From a7533cbd8bd60a8d4e4752f9ca4ba573d52aec65 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sat, 10 Sep 2022 13:39:07 +0200 Subject: [PATCH] [core] Move storage to SQLite database (#30) fixes #29 --- .gitignore | 2 + README.md | 42 +- action_script.go | 2 +- actions.go | 2 +- auth.go | 16 +- botEditor.go | 2 +- botUserState.go | 2 +- config.go | 2 +- configEditor.go | 2 +- configEditor_general.go | 10 +- functions.go | 8 - functions_counter.go | 27 -- go.mod | 11 +- go.sum | 49 +++ helpers.go | 5 +- .../actors/counter/actor.go | 60 ++- internal/actors/counter/database.go | 55 +++ internal/actors/counter/schema/001.sql | 4 + internal/actors/modchannel/actor.go | 2 +- internal/actors/nuke/actor.go | 2 +- internal/actors/punish/actor.go | 122 +----- internal/actors/punish/database.go | 142 ++++++ internal/actors/punish/schema/001.sql | 6 + internal/actors/quotedb/actor.go | 146 +------ internal/actors/quotedb/database.go | 173 ++++++++ internal/actors/quotedb/http.go | 36 +- internal/actors/quotedb/schema/001.sql | 7 + .../actors/variables/actor.go | 56 ++- internal/actors/variables/database.go | 56 +++ internal/actors/variables/schema/001.sql | 4 + internal/apimodules/overlays/database.go | 74 ++++ internal/apimodules/overlays/overlays.go | 90 +--- internal/apimodules/overlays/schema/001.sql | 9 + internal/service/access/access.go | 190 ++++++++ internal/service/timer/schema/001.sql | 4 + internal/service/timer/timer.go | 103 +++++ internal/v2migrator/core.go | 90 ++++ {crypt => internal/v2migrator/crypt}/crypt.go | 0 internal/v2migrator/modOverlays.go | 46 ++ internal/v2migrator/modPunish.go | 39 ++ internal/v2migrator/modQuoteDB.go | 35 ++ internal/v2migrator/store.go | 119 +++++ irc.go | 4 +- main.go | 121 ++++-- pkg/database/connector.go | 67 +++ pkg/database/connector_test.go | 98 +++++ pkg/database/coreKV.go | 90 ++++ pkg/database/crypt.go | 17 + pkg/database/database.go | 50 +++ pkg/database/migration.go | 94 ++++ pkg/database/migration_embedfs.go | 35 ++ pkg/database/migration_test.go | 42 ++ pkg/database/schema/001.sql | 6 + pkg/database/schema/core.sql | 6 + pkg/database/testdata/migration1/001.sql | 4 + pkg/database/testdata/migration2/001.sql | 4 + pkg/database/testdata/migration2/002.sql | 1 + {twitch => pkg/twitch}/badges.go | 0 {twitch => pkg/twitch}/eventsub.go | 0 {twitch => pkg/twitch}/http.go | 0 {twitch => pkg/twitch}/scopes.go | 0 {twitch => pkg/twitch}/twitch.go | 0 {twitch => pkg/twitch}/twitchAPICache.go | 0 plugins/interface.go | 7 +- plugins/rule.go | 34 +- plugins/rule_test.go | 2 +- plugins/timerstore.go | 24 +- plugins_core.go | 18 +- scopes.go | 2 +- store.go | 405 ------------------ timers.go | 50 --- twitchWatcher.go | 15 +- 72 files changed, 2127 insertions(+), 921 deletions(-) delete mode 100644 functions_counter.go rename action_counter.go => internal/actors/counter/actor.go (73%) create mode 100644 internal/actors/counter/database.go create mode 100644 internal/actors/counter/schema/001.sql create mode 100644 internal/actors/punish/database.go create mode 100644 internal/actors/punish/schema/001.sql create mode 100644 internal/actors/quotedb/database.go create mode 100644 internal/actors/quotedb/schema/001.sql rename action_setvar.go => internal/actors/variables/actor.go (70%) create mode 100644 internal/actors/variables/database.go create mode 100644 internal/actors/variables/schema/001.sql create mode 100644 internal/apimodules/overlays/database.go create mode 100644 internal/apimodules/overlays/schema/001.sql create mode 100644 internal/service/access/access.go create mode 100644 internal/service/timer/schema/001.sql create mode 100644 internal/service/timer/timer.go create mode 100644 internal/v2migrator/core.go rename {crypt => internal/v2migrator/crypt}/crypt.go (100%) create mode 100644 internal/v2migrator/modOverlays.go create mode 100644 internal/v2migrator/modPunish.go create mode 100644 internal/v2migrator/modQuoteDB.go create mode 100644 internal/v2migrator/store.go create mode 100644 pkg/database/connector.go create mode 100644 pkg/database/connector_test.go create mode 100644 pkg/database/coreKV.go create mode 100644 pkg/database/crypt.go create mode 100644 pkg/database/database.go create mode 100644 pkg/database/migration.go create mode 100644 pkg/database/migration_embedfs.go create mode 100644 pkg/database/migration_test.go create mode 100644 pkg/database/schema/001.sql create mode 100644 pkg/database/schema/core.sql create mode 100644 pkg/database/testdata/migration1/001.sql create mode 100644 pkg/database/testdata/migration2/001.sql create mode 100644 pkg/database/testdata/migration2/002.sql rename {twitch => pkg/twitch}/badges.go (100%) rename {twitch => pkg/twitch}/eventsub.go (100%) rename {twitch => pkg/twitch}/http.go (100%) rename {twitch => pkg/twitch}/scopes.go (100%) rename {twitch => pkg/twitch}/twitch.go (100%) rename {twitch => pkg/twitch}/twitchAPICache.go (100%) delete mode 100644 store.go delete mode 100644 timers.go diff --git a/.gitignore b/.gitignore index eb86de0..0ed644a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,7 @@ editor/app.js editor/bundle.* .env node_modules +storage.db +storage.db-journal storage.json.gz twitch-bot diff --git a/README.md b/README.md index 87d1f1d..31b0738 100644 --- a/README.md +++ b/README.md @@ -23,16 +23,56 @@ Usage of twitch-bot: --log-level string Log level (debug, info, warn, error, fatal) (default "info") --plugin-dir string Where to find and load plugins (default "/usr/lib/twitch-bot") --rate-limit duration How often to send a message (default: 20/30s=1500ms, if your bot is mod everywhere: 100/30s=300ms, different for known/verified bots) (default 1.5s) + --storage-database string Database file to store data in (default "./storage.db") --storage-encryption-pass string Passphrase to encrypt secrets inside storage (defaults to twitch-client:twitch-client-secret) - --storage-file string Where to store the data (default "./storage.json.gz") --twitch-client string Client ID to act as --twitch-client-secret string Secret for the Client ID --twitch-token string OAuth token valid for client (fallback if no token was set in interface) -v, --validate-config Loads the config, logs any errors and quits with status 0 on success --version Prints current version and exits + # twitch-bot help Supported sub-commands are: actor-docs Generate markdown documentation for available actors api-token Generate an api-token to be entered into the config + migrate-v2 Migrate old (*.json.gz) storage file into new database help Prints this help message ``` + +## Upgrade from `v2.x` to `v3.x` + +With the release of `v3.0.0` the bot changed a lot introducing a new storage format. As that storage backend is not compatible with the `v2.x` storage you need to migrate it manually before starting a `v3.x` bot version the first time. + +**Before starting the migration make sure to fully stop the bot!** + +This section assumes you were starting your bot the following way: + +```console +# twitch-bot --storage-file storage.json.gz --twitch-client --twitch-client-secret +``` + +To execute the migration we need to provide the same `storage-encryption-pass` or `twitch-client` / `twitch-client-secret` combination if no `storage-encryption-pass` was used. + +```console +# twitch-bot --storage-database storage.db --twitch-client --twitch-client-secret migrate-v2 storage.json.gz +WARN[0000] No storage encryption passphrase was set, falling back to client-id:client-secret +WARN[0000] Module registered unhandled query-param type module=status type=integer +WARN[0000] Overlays dir not specified, no dir or non existent dir= +INFO[0000] Starting migration... module=variables +INFO[0000] Starting migration... module=mod_punish +INFO[0000] Starting migration... module=mod_overlays +INFO[0000] Starting migration... module=mod_quotedb +INFO[0000] Starting migration... module=core +INFO[0000] Starting migration... module=counter +INFO[0000] Starting migration... module=permissions +INFO[0000] Starting migration... module=timers +INFO[0000] v2 storage file was migrated +``` + +If you see the `v2 storage file was migrated` message the contents of your old storage file were migrated to the new `storage-database`. The old file is not modified in this step. + +Afterwards your need to adjust the start parameters of the bot: + +```console +# twitch-bot --storage-database storage.db --twitch-client --twitch-client-secret +``` diff --git a/action_script.go b/action_script.go index c1ab93f..f8af49e 100644 --- a/action_script.go +++ b/action_script.go @@ -10,8 +10,8 @@ import ( "github.com/go-irc/irc" "github.com/pkg/errors" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) func init() { diff --git a/actions.go b/actions.go index 2a9d9e7..83d9d8a 100644 --- a/actions.go +++ b/actions.go @@ -85,7 +85,7 @@ func handleMessage(c *irc.Client, m *irc.Message, event *string, eventData *plug // Lock command if !preventCooldown { - r.SetCooldown(timerStore, m, eventData) + r.SetCooldown(timerService, m, eventData) } } } diff --git a/auth.go b/auth.go index 4ddb5fc..4db5312 100644 --- a/auth.go +++ b/auth.go @@ -11,8 +11,8 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) var instanceState = uuid.Must(uuid.NewV4()).String() @@ -85,18 +85,14 @@ func handleAuthUpdateBotToken(w http.ResponseWriter, r *http.Request) { return } - if err = store.UpdateBotToken(rData.AccessToken, rData.RefreshToken); err != nil { + if err = accessService.SetBotTwitchCredentials(rData.AccessToken, rData.RefreshToken); err != nil { http.Error(w, errors.Wrap(err, "storing access token").Error(), http.StatusInternalServerError) return } twitchClient.UpdateToken(rData.AccessToken, rData.RefreshToken) - if err = store.SetExtendedPermissions(botUser, storageExtendedPermission{ - AccessToken: rData.AccessToken, - RefreshToken: rData.RefreshToken, - Scopes: rData.Scope, - }, true); err != nil { + if err = accessService.SetExtendedTwitchCredentials(botUser, rData.AccessToken, rData.RefreshToken, rData.Scope); err != nil { http.Error(w, errors.Wrap(err, "storing access scopes").Error(), http.StatusInternalServerError) return } @@ -145,11 +141,7 @@ func handleAuthUpdateChannelGrant(w http.ResponseWriter, r *http.Request) { return } - if err = store.SetExtendedPermissions(grantUser, storageExtendedPermission{ - AccessToken: rData.AccessToken, - RefreshToken: rData.RefreshToken, - Scopes: rData.Scope, - }, false); err != nil { + if err = accessService.SetExtendedTwitchCredentials(grantUser, rData.AccessToken, rData.RefreshToken, rData.Scope); err != nil { http.Error(w, errors.Wrap(err, "storing access token").Error(), http.StatusInternalServerError) return } diff --git a/botEditor.go b/botEditor.go index bfa5798..f13311c 100644 --- a/botEditor.go +++ b/botEditor.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" "github.com/Luzifer/go_helpers/v2/str" - "github.com/Luzifer/twitch-bot/twitch" + "github.com/Luzifer/twitch-bot/pkg/twitch" ) func getAuthorizationFromRequest(r *http.Request) (string, *twitch.Client, error) { diff --git a/botUserState.go b/botUserState.go index c443d2c..24938d4 100644 --- a/botUserState.go +++ b/botUserState.go @@ -6,7 +6,7 @@ import ( "github.com/go-irc/irc" - "github.com/Luzifer/twitch-bot/twitch" + "github.com/Luzifer/twitch-bot/pkg/twitch" ) type ( diff --git a/config.go b/config.go index 547dbde..7dcecd3 100644 --- a/config.go +++ b/config.go @@ -257,7 +257,7 @@ func (c configFile) GetMatchingRules(m *irc.Message, event *string, eventData *p var out []*plugins.Rule for _, r := range c.Rules { - if r.Matches(m, event, timerStore, formatMessage, twitchClient, eventData) { + if r.Matches(m, event, timerService, formatMessage, twitchClient, eventData) { out = append(out, r) } } diff --git a/configEditor.go b/configEditor.go index 92355b7..caea21a 100644 --- a/configEditor.go +++ b/configEditor.go @@ -11,8 +11,8 @@ import ( "github.com/gorilla/websocket" "github.com/pkg/errors" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) const websocketPingInterval = 30 * time.Second diff --git a/configEditor_general.go b/configEditor_general.go index 2d06e4e..96f2c5f 100644 --- a/configEditor_general.go +++ b/configEditor_general.go @@ -183,10 +183,16 @@ func configEditorHandleGeneralDeleteAuthToken(w http.ResponseWriter, r *http.Req } func configEditorHandleGeneralGet(w http.ResponseWriter, r *http.Request) { - elevated := make(map[string]bool) + var ( + elevated = make(map[string]bool) + err error + ) for _, ch := range config.Channels { - elevated[ch] = store.UserHasGrantedScopes(ch, channelDefaultScopes...) && store.UserHasExtendedAuth(ch) + if elevated[ch], err = accessService.HasPermissionsForChannel(ch, channelDefaultScopes...); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } } var uName *string diff --git a/functions.go b/functions.go index 42a39c9..09458ec 100644 --- a/functions.go +++ b/functions.go @@ -99,12 +99,4 @@ func init() { tplFuncs.Register("toLower", plugins.GenericTemplateFunctionGetter(strings.ToLower)) tplFuncs.Register("toUpper", plugins.GenericTemplateFunctionGetter(strings.ToUpper)) - - tplFuncs.Register("variable", plugins.GenericTemplateFunctionGetter(func(name string, defVal ...string) string { - value := store.GetVariable(name) - if value == "" && len(defVal) > 0 { - return defVal[0] - } - return value - })) } diff --git a/functions_counter.go b/functions_counter.go deleted file mode 100644 index 8fc8513..0000000 --- a/functions_counter.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "strings" - - "github.com/go-irc/irc" - "github.com/pkg/errors" - - "github.com/Luzifer/twitch-bot/plugins" -) - -func init() { - tplFuncs.Register("channelCounter", func(m *irc.Message, r *plugins.Rule, fields *plugins.FieldCollection) interface{} { - return func(name string) (string, error) { - channel, err := fields.String("channel") - if err != nil { - return "", errors.Wrap(err, "channel not available") - } - - return strings.Join([]string{channel, name}, ":"), nil - } - }) - - tplFuncs.Register("counterValue", plugins.GenericTemplateFunctionGetter(func(name string, _ ...string) int64 { - return store.GetCounterValue(name) - })) -} diff --git a/go.mod b/go.mod index ee73cb7..9262c71 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,13 @@ require ( github.com/Luzifer/go_helpers/v2 v2.12.2 github.com/Luzifer/korvike/functions v0.6.1 github.com/Luzifer/rconfig/v2 v2.3.0 + github.com/glebarez/go-sqlite v1.18.1 github.com/go-irc/irc v2.1.0+incompatible github.com/gofrs/uuid v4.2.0+incompatible github.com/gofrs/uuid/v3 v3.1.2 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 + github.com/jmoiron/sqlx v1.3.5 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 @@ -26,6 +28,7 @@ require ( github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -37,20 +40,26 @@ require ( github.com/hashicorp/vault/sdk v0.2.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sergi/go-diff v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/src-d/gcfg v1.4.0 // indirect github.com/xanzy/ssh-agent v0.2.1 // indirect golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect + golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect gopkg.in/validator.v2 v2.0.0-20210331031555-b37d688a7fb0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + modernc.org/libc v1.16.19 // indirect + modernc.org/mathutil v1.4.1 // indirect + modernc.org/memory v1.1.1 // indirect + modernc.org/sqlite v1.18.1 // indirect ) diff --git a/go.sum b/go.sum index d62cb12..b956fba 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,7 @@ github.com/docker/docker v1.4.2-0.20200319182547-c7ad2b866182/go.mod h1:eEKB0N0r github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -72,6 +73,8 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/glebarez/go-sqlite v1.18.1 h1:w0xtxKWktqYsUsXg//SQK+l1IcpKb3rGOQHmMptvL2U= +github.com/glebarez/go-sqlite v1.18.1/go.mod h1:ydXIGq2M4OzF4YyNhH129SPp7jWoVvgkEgb6pldmS0s= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-irc/irc v2.1.0+incompatible h1:pg7pMVq5OYQbqTxceByD/EN8VIsba7DtKn49rsCnG8Y= @@ -84,6 +87,7 @@ github.com/go-ldap/ldap/v3 v3.1.10/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9p github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -118,8 +122,11 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= @@ -191,9 +198,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -209,6 +219,7 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leekchan/gtf v0.0.0-20190214083521-5fba33c5b00b/go.mod h1:thNruaSwydMhkQ8dXzapABF9Sc1Tz08ZBcDdgott9RA= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -216,6 +227,10 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -286,6 +301,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -400,6 +417,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4= +golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -425,6 +446,7 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -494,3 +516,30 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19 h1:S8flPn5ZeXx6iw/8yNa986hwTQDrY8RXU7tObZuAozo= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1 h1:ko32eKt3jf7eqIkCgPAeHMBXw3riNSLhl2f3loEF7o8= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= diff --git a/helpers.go b/helpers.go index 3f6c00b..92fa71b 100644 --- a/helpers.go +++ b/helpers.go @@ -1,6 +1,3 @@ package main -var ( - ptrBoolFalse = func(v bool) *bool { return &v }(false) - ptrStringEmpty = func(v string) *string { return &v }("") -) +var ptrBoolFalse = func(v bool) *bool { return &v }(false) diff --git a/action_counter.go b/internal/actors/counter/actor.go similarity index 73% rename from action_counter.go rename to internal/actors/counter/actor.go index 0ed6709..af67231 100644 --- a/action_counter.go +++ b/internal/actors/counter/actor.go @@ -1,21 +1,38 @@ -package main +package counter import ( "fmt" "net/http" "strconv" + "strings" "github.com/go-irc/irc" "github.com/gorilla/mux" "github.com/pkg/errors" + "github.com/Luzifer/twitch-bot/pkg/database" "github.com/Luzifer/twitch-bot/plugins" ) -func init() { - registerAction("counter", func() plugins.Actor { return &ActorCounter{} }) +var ( + db database.Connector + formatMessage plugins.MsgFormatter - registerActorDocumentation(plugins.ActionDocumentation{ + ptrStringEmpty = func(s string) *string { return &s }("") +) + +//nolint:funlen // This function is a few lines too long but only contains definitions +func Register(args plugins.RegistrationArguments) error { + db = args.GetDatabaseConnector() + if err := db.Migrate("counter", database.NewEmbedFSMigrator(schema, "schema")); err != nil { + return errors.Wrap(err, "applying schema migration") + } + + formatMessage = args.FormatMessage + + args.RegisterActor("counter", func() plugins.Actor { return &ActorCounter{} }) + + args.RegisterActorDocumentation(plugins.ActionDocumentation{ Description: "Update counter values", Name: "Modify Counter", Type: "counter", @@ -51,7 +68,7 @@ func init() { }, }) - registerRoute(plugins.HTTPRouteRegistrationArgs{ + args.RegisterAPIRoute(plugins.HTTPRouteRegistrationArgs{ Description: "Returns the (formatted) value as a plain string", HandlerFunc: routeActorCounterGetValue, Method: http.MethodGet, @@ -75,7 +92,7 @@ func init() { }, }) - registerRoute(plugins.HTTPRouteRegistrationArgs{ + args.RegisterAPIRoute(plugins.HTTPRouteRegistrationArgs{ Description: "Updates the value of the counter", HandlerFunc: routeActorCounterSetValue, Method: http.MethodPatch, @@ -104,6 +121,23 @@ func init() { }, }, }) + + args.RegisterTemplateFunction("channelCounter", func(m *irc.Message, r *plugins.Rule, fields *plugins.FieldCollection) interface{} { + return func(name string) (string, error) { + channel, err := fields.String("channel") + if err != nil { + return "", errors.Wrap(err, "channel not available") + } + + return strings.Join([]string{channel, name}, ":"), nil + } + }) + + args.RegisterTemplateFunction("counterValue", plugins.GenericTemplateFunctionGetter(func(name string, _ ...string) (int64, error) { + return getCounterValue(name) + })) + + return nil } type ActorCounter struct{} @@ -126,7 +160,7 @@ func (a ActorCounter) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule, ev } return false, errors.Wrap( - store.UpdateCounter(counterName, counterValue, true), + updateCounter(counterName, counterValue, true), "set counter", ) } @@ -145,7 +179,7 @@ func (a ActorCounter) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule, ev } return false, errors.Wrap( - store.UpdateCounter(counterName, counterStep, false), + updateCounter(counterName, counterStep, false), "update counter", ) } @@ -167,8 +201,14 @@ func routeActorCounterGetValue(w http.ResponseWriter, r *http.Request) { template = "%d" } + cv, err := getCounterValue(mux.Vars(r)["name"]) + if err != nil { + http.Error(w, errors.Wrap(err, "getting value").Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text-plain") - fmt.Fprintf(w, template, store.GetCounterValue(mux.Vars(r)["name"])) + fmt.Fprintf(w, template, cv) } func routeActorCounterSetValue(w http.ResponseWriter, r *http.Request) { @@ -183,7 +223,7 @@ func routeActorCounterSetValue(w http.ResponseWriter, r *http.Request) { return } - if err = store.UpdateCounter(mux.Vars(r)["name"], value, absolute); err != nil { + if err = updateCounter(mux.Vars(r)["name"], value, absolute); err != nil { http.Error(w, errors.Wrap(err, "updating value").Error(), http.StatusInternalServerError) return } diff --git a/internal/actors/counter/database.go b/internal/actors/counter/database.go new file mode 100644 index 0000000..715c50e --- /dev/null +++ b/internal/actors/counter/database.go @@ -0,0 +1,55 @@ +package counter + +import ( + "database/sql" + "embed" + + "github.com/pkg/errors" +) + +//go:embed schema/** +var schema embed.FS + +func getCounterValue(counter string) (int64, error) { + row := db.DB().QueryRow( + `SELECT value + FROM counters + WHERE name = $1`, + counter, + ) + + var cv int64 + err := row.Scan(&cv) + switch { + case err == nil: + return cv, nil + + case errors.Is(err, sql.ErrNoRows): + return 0, nil + + default: + return 0, errors.Wrap(err, "querying counter") + } +} + +func updateCounter(counter string, value int64, absolute bool) error { + if !absolute { + cv, err := getCounterValue(counter) + if err != nil { + return errors.Wrap(err, "getting previous value") + } + + value += cv + } + + _, err := db.DB().Exec( + `INSERT INTO counters + (name, value) + VALUES ($1, $2) + ON CONFLICT DO UPDATE + SET value = excluded.value;`, + counter, value, + ) + + return errors.Wrap(err, "storing counter value") +} diff --git a/internal/actors/counter/schema/001.sql b/internal/actors/counter/schema/001.sql new file mode 100644 index 0000000..1eb2f30 --- /dev/null +++ b/internal/actors/counter/schema/001.sql @@ -0,0 +1,4 @@ +CREATE TABLE counters ( + name STRING NOT NULL PRIMARY KEY, + value INTEGER +); diff --git a/internal/actors/modchannel/actor.go b/internal/actors/modchannel/actor.go index 584d28e..b784cf2 100644 --- a/internal/actors/modchannel/actor.go +++ b/internal/actors/modchannel/actor.go @@ -7,8 +7,8 @@ import ( "github.com/go-irc/irc" "github.com/pkg/errors" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) const actorName = "modchannel" diff --git a/internal/actors/nuke/actor.go b/internal/actors/nuke/actor.go index 9c5c2be..3417d8f 100644 --- a/internal/actors/nuke/actor.go +++ b/internal/actors/nuke/actor.go @@ -12,8 +12,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/Luzifer/go_helpers/v2/str" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) const ( diff --git a/internal/actors/punish/actor.go b/internal/actors/punish/actor.go index 53628d1..ff86895 100644 --- a/internal/actors/punish/actor.go +++ b/internal/actors/punish/actor.go @@ -1,16 +1,15 @@ package punish import ( - "encoding/json" "math" "strconv" "strings" - "sync" "time" "github.com/go-irc/irc" "github.com/pkg/errors" + "github.com/Luzifer/twitch-bot/pkg/database" "github.com/Luzifer/twitch-bot/plugins" ) @@ -23,16 +22,19 @@ const ( ) var ( + db database.Connector formatMessage plugins.MsgFormatter ptrDefaultCooldown = func(v time.Duration) *time.Duration { return &v }(oneWeek) ptrStringEmpty = func(v string) *string { return &v }("") - store plugins.StorageManager - storedObject = newStorage() ) func Register(args plugins.RegistrationArguments) error { + db = args.GetDatabaseConnector() + if err := db.Migrate("punish", database.NewEmbedFSMigrator(schema, "schema")); err != nil { + return errors.Wrap(err, "applying schema migration") + } + formatMessage = args.FormatMessage - store = args.GetStorageManager() args.RegisterActor(actorNamePunish, func() plugins.Actor { return &actorPunish{} }) args.RegisterActor(actorNameResetPunish, func() plugins.Actor { return &actorResetPunish{} }) @@ -118,10 +120,7 @@ func Register(args plugins.RegistrationArguments) error { }, }) - return errors.Wrap( - store.GetModuleStore(moduleUUID, storedObject), - "loading module storage", - ) + return nil } type ( @@ -133,12 +132,6 @@ type ( Executed time.Time `json:"executed"` Cooldown time.Duration `json:"cooldown"` } - - storage struct { - ActiveLevels map[string]*levelConfig `json:"active_levels"` - - lock sync.Mutex - } ) // Punish @@ -160,7 +153,10 @@ func (a actorPunish) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule, eve return false, errors.Wrap(err, "preparing user") } - lvl := storedObject.GetPunishment(plugins.DeriveChannel(m, eventData), user, uuid) + lvl, err := getPunishment(plugins.DeriveChannel(m, eventData), user, uuid) + if err != nil { + return false, errors.Wrap(err, "getting stored punishment") + } nLvl := int(math.Min(float64(len(levels)-1), float64(lvl.LastLevel+1))) var cmd []string @@ -207,7 +203,7 @@ func (a actorPunish) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule, eve lvl.LastLevel = nLvl return false, errors.Wrap( - store.SetModuleStore(moduleUUID, storedObject), + setPunishment(plugins.DeriveChannel(m, eventData), user, uuid, lvl), "storing punishment level", ) } @@ -239,10 +235,8 @@ func (a actorResetPunish) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule return false, errors.Wrap(err, "preparing user") } - storedObject.ResetLevel(plugins.DeriveChannel(m, eventData), user, uuid) - return false, errors.Wrap( - store.SetModuleStore(moduleUUID, storedObject), + deletePunishment(plugins.DeriveChannel(m, eventData), user, uuid), "resetting punishment level", ) } @@ -257,91 +251,3 @@ func (a actorResetPunish) Validate(attrs *plugins.FieldCollection) (err error) { return nil } - -// Storage - -func newStorage() *storage { - return &storage{ - ActiveLevels: make(map[string]*levelConfig), - } -} - -func (s *storage) GetPunishment(channel, user, uuid string) *levelConfig { - s.lock.Lock() - defer s.lock.Unlock() - - // Ensure old state is cleared - s.calculateCooldowns() - - var ( - id = s.getCacheKey(channel, user, uuid) - lvl = s.ActiveLevels[id] - ) - - if lvl == nil { - // Initialize a non-triggered state - lvl = &levelConfig{LastLevel: -1} - s.ActiveLevels[id] = lvl - } - - return lvl -} - -func (s *storage) ResetLevel(channel, user, uuid string) { - s.lock.Lock() - defer s.lock.Unlock() - - delete(s.ActiveLevels, s.getCacheKey(channel, user, uuid)) -} - -func (s *storage) getCacheKey(channel, user, uuid string) string { - return strings.Join([]string{channel, user, uuid}, "::") -} - -func (s *storage) calculateCooldowns() { - // This MUST NOT be locked, the lock MUST be set by calling method - - var clear []string - - for id, lvl := range s.ActiveLevels { - for { - cooldownTime := lvl.Executed.Add(lvl.Cooldown) - if cooldownTime.After(time.Now()) { - break - } - - lvl.Executed = cooldownTime - lvl.LastLevel-- - } - - // Level 0 is the first punishment level, so only remove if it drops below 0 - if lvl.LastLevel < 0 { - clear = append(clear, id) - } - } - - for _, id := range clear { - delete(s.ActiveLevels, id) - } -} - -// Implement marshaller interfaces -func (s *storage) MarshalStoredObject() ([]byte, error) { - s.lock.Lock() - defer s.lock.Unlock() - - s.calculateCooldowns() - return json.Marshal(s) -} - -func (s *storage) UnmarshalStoredObject(data []byte) error { - if data == nil { - // No data set yet, don't try to unmarshal - return nil - } - - s.lock.Lock() - defer s.lock.Unlock() - - return json.Unmarshal(data, s) -} diff --git a/internal/actors/punish/database.go b/internal/actors/punish/database.go new file mode 100644 index 0000000..9128a7c --- /dev/null +++ b/internal/actors/punish/database.go @@ -0,0 +1,142 @@ +package punish + +import ( + "database/sql" + "embed" + "strings" + "time" + + "github.com/pkg/errors" +) + +//go:embed schema/** +var schema embed.FS + +func calculateCurrentPunishments() error { + rows, err := db.DB().Query( + `SELECT key, last_level, executed, cooldown + FROM punish_levels;`, + ) + if err != nil { + return errors.Wrap(err, "querying punish_levels") + } + + for rows.Next() { + if err = rows.Err(); err != nil { + return errors.Wrap(err, "advancing rows") + } + + var ( + key string + lastLevel, executed, cooldown int64 + + actUpdate bool + ) + if err = rows.Scan(&key, &lastLevel, &executed, &cooldown); err != nil { + return errors.Wrap(err, "advancing rows") + } + + lvl := &levelConfig{ + LastLevel: int(lastLevel), + Cooldown: time.Duration(cooldown), + Executed: time.Unix(executed, 0), + } + + for { + cooldownTime := lvl.Executed.Add(lvl.Cooldown) + if cooldownTime.After(time.Now()) { + break + } + + lvl.Executed = cooldownTime + lvl.LastLevel-- + actUpdate = true + } + + // Level 0 is the first punishment level, so only remove if it drops below 0 + if lvl.LastLevel < 0 { + if err = deletePunishmentForKey(key); err != nil { + return errors.Wrap(err, "cleaning up expired punishment") + } + continue + } + + if actUpdate { + if err = setPunishmentForKey(key, lvl); err != nil { + return errors.Wrap(err, "updating punishment") + } + } + } + + return errors.Wrap(rows.Err(), "finishing rows processing") +} + +func deletePunishment(channel, user, uuid string) error { + return deletePunishmentForKey(getDBKey(channel, user, uuid)) +} + +func deletePunishmentForKey(key string) error { + _, err := db.DB().Exec( + `DELETE FROM punish_levels + WHERE key = $1;`, + key, + ) + + return errors.Wrap(err, "deleting punishment info") +} + +func getPunishment(channel, user, uuid string) (*levelConfig, error) { + if err := calculateCurrentPunishments(); err != nil { + return nil, errors.Wrap(err, "updating punishment states") + } + + row := db.DB().QueryRow( + `SELECT last_level, executed, cooldown + FROM punish_levels + WHERE key = $1;`, + getDBKey(channel, user, uuid), + ) + + lc := &levelConfig{LastLevel: -1} + + var lastLevel, executed, cooldown int64 + err := row.Scan(&lastLevel, &executed, &cooldown) + switch { + case err == nil: + lc.LastLevel = int(lastLevel) + lc.Cooldown = time.Duration(cooldown) + lc.Executed = time.Unix(executed, 0) + + return lc, nil + + case errors.Is(err, sql.ErrNoRows): + return lc, nil + + default: + return nil, errors.Wrap(err, "getting punishment from database") + } +} + +func setPunishment(channel, user, uuid string, lc *levelConfig) error { + return setPunishmentForKey(getDBKey(channel, user, uuid), lc) +} + +func setPunishmentForKey(key string, lc *levelConfig) error { + _, err := db.DB().Exec( + `INSERT INTO punish_levels + (key, last_level, executed, cooldown) + VALUES ($1, $2, $3, $4) + ON CONFLICT DO UPDATE + SET last_level = excluded.last_level, + executed = excluded.executed, + cooldown = excluded.cooldown;`, + key, + lc.LastLevel, lc.Executed.UTC().Unix(), int64(lc.Cooldown), + ) + + return errors.Wrap(err, "updating punishment info") +} + +func getDBKey(channel, user, uuid string) string { + return strings.Join([]string{channel, user, uuid}, "::") +} diff --git a/internal/actors/punish/schema/001.sql b/internal/actors/punish/schema/001.sql new file mode 100644 index 0000000..3f9935b --- /dev/null +++ b/internal/actors/punish/schema/001.sql @@ -0,0 +1,6 @@ +CREATE TABLE punish_levels ( + key STRING NOT NULL PRIMARY KEY, + last_level INTEGER, + executed INTEGER, -- time.Time + cooldown INTEGER -- time.Duration +); diff --git a/internal/actors/quotedb/actor.go b/internal/actors/quotedb/actor.go index 62ffae8..2fff440 100644 --- a/internal/actors/quotedb/actor.go +++ b/internal/actors/quotedb/actor.go @@ -1,14 +1,12 @@ package quotedb import ( - "encoding/json" - "math/rand" "strconv" - "sync" "github.com/go-irc/irc" "github.com/pkg/errors" + "github.com/Luzifer/twitch-bot/pkg/database" "github.com/Luzifer/twitch-bot/plugins" ) @@ -18,9 +16,8 @@ const ( ) var ( + db database.Connector formatMessage plugins.MsgFormatter - store plugins.StorageManager - storedObject = newStorage() ptrStringEmpty = func(v string) *string { return &v }("") ptrStringOutFormat = func(v string) *string { return &v }("Quote #{{ .index }}: {{ .quote }}") @@ -28,8 +25,12 @@ var ( ) func Register(args plugins.RegistrationArguments) error { + db = args.GetDatabaseConnector() + if err := db.Migrate(actorName, database.NewEmbedFSMigrator(schema, "schema")); err != nil { + return errors.Wrap(err, "applying schema migration") + } + formatMessage = args.FormatMessage - store = args.GetStorageManager() args.RegisterActor(actorName, func() plugins.Actor { return &actor{} }) @@ -81,25 +82,16 @@ func Register(args plugins.RegistrationArguments) error { registerAPI(args.RegisterAPIRoute) args.RegisterTemplateFunction("lastQuoteIndex", func(m *irc.Message, r *plugins.Rule, fields *plugins.FieldCollection) interface{} { - return func() int { - return storedObject.GetMaxQuoteIdx(plugins.DeriveChannel(m, nil)) + return func() (int, error) { + return getMaxQuoteIdx(plugins.DeriveChannel(m, nil)) } }) - return errors.Wrap( - store.GetModuleStore(moduleUUID, storedObject), - "loading module storage", - ) + return nil } type ( actor struct{} - - storage struct { - ChannelQuotes map[string][]string `json:"channel_quotes"` - - lock sync.RWMutex - } ) func (a actor) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule, eventData *plugins.FieldCollection, attrs *plugins.FieldCollection) (preventCooldown bool, err error) { @@ -129,21 +121,22 @@ func (a actor) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule, eventData return false, errors.Wrap(err, "formatting quote") } - storedObject.AddQuote(plugins.DeriveChannel(m, eventData), quote) return false, errors.Wrap( - store.SetModuleStore(moduleUUID, storedObject), - "storing quote database", + addQuote(plugins.DeriveChannel(m, eventData), quote), + "adding quote", ) case "del": - storedObject.DelQuote(plugins.DeriveChannel(m, eventData), index) return false, errors.Wrap( - store.SetModuleStore(moduleUUID, storedObject), + delQuote(plugins.DeriveChannel(m, eventData), index), "storing quote database", ) case "get": - idx, quote := storedObject.GetQuote(plugins.DeriveChannel(m, eventData), index) + idx, quote, err := getQuote(plugins.DeriveChannel(m, eventData), index) + if err != nil { + return false, errors.Wrap(err, "getting quote") + } if idx == 0 { // No quote was found for the given idx @@ -201,108 +194,3 @@ func (a actor) Validate(attrs *plugins.FieldCollection) (err error) { return nil } - -// Storage - -func newStorage() *storage { - return &storage{ - ChannelQuotes: make(map[string][]string), - } -} - -func (s *storage) AddQuote(channel, quote string) { - s.lock.Lock() - defer s.lock.Unlock() - - s.ChannelQuotes[channel] = append(s.ChannelQuotes[channel], quote) -} - -func (s *storage) DelQuote(channel string, quote int) { - s.lock.Lock() - defer s.lock.Unlock() - - var quotes []string - for i, q := range s.ChannelQuotes[channel] { - if i == quote { - continue - } - quotes = append(quotes, q) - } - - s.ChannelQuotes[channel] = quotes -} - -func (s *storage) GetChannelQuotes(channel string) []string { - s.lock.RLock() - defer s.lock.RUnlock() - - var out []string - out = append(out, s.ChannelQuotes[channel]...) - return out -} - -func (s *storage) GetMaxQuoteIdx(channel string) int { - s.lock.RLock() - defer s.lock.RUnlock() - - return len(s.ChannelQuotes[channel]) -} - -func (s *storage) GetQuote(channel string, quote int) (int, string) { - s.lock.RLock() - defer s.lock.RUnlock() - - if quote == 0 { - quote = rand.Intn(len(s.ChannelQuotes[channel])) + 1 // #nosec G404 // no need for cryptographic safety - } - - if quote > len(s.ChannelQuotes[channel]) { - return 0, "" - } - - return quote, s.ChannelQuotes[channel][quote-1] -} - -func (s *storage) SetQuotes(channel string, quotes []string) { - s.lock.Lock() - defer s.lock.Unlock() - - s.ChannelQuotes[channel] = quotes -} - -func (s *storage) UpdateQuote(channel string, idx int, quote string) { - s.lock.Lock() - defer s.lock.Unlock() - - var quotes []string - for i := range s.ChannelQuotes[channel] { - if i == idx { - quotes = append(quotes, quote) - continue - } - - quotes = append(quotes, s.ChannelQuotes[channel][i]) - } - - s.ChannelQuotes[channel] = quotes -} - -// Implement marshaller interfaces -func (s *storage) MarshalStoredObject() ([]byte, error) { - s.lock.RLock() - defer s.lock.RUnlock() - - return json.Marshal(s) -} - -func (s *storage) UnmarshalStoredObject(data []byte) error { - if data == nil { - // No data set yet, don't try to unmarshal - return nil - } - - s.lock.Lock() - defer s.lock.Unlock() - - return json.Unmarshal(data, s) -} diff --git a/internal/actors/quotedb/database.go b/internal/actors/quotedb/database.go new file mode 100644 index 0000000..c76d8c8 --- /dev/null +++ b/internal/actors/quotedb/database.go @@ -0,0 +1,173 @@ +package quotedb + +import ( + "database/sql" + "embed" + "math/rand" + "time" + + "github.com/pkg/errors" +) + +//go:embed schema/** +var schema embed.FS + +func addQuote(channel, quote string) error { + _, err := db.DB().Exec( + `INSERT INTO quotedb + (channel, created_at, quote) + VALUES ($1, $2, $3);`, + channel, time.Now().UnixNano(), quote, + ) + + return errors.Wrap(err, "adding quote to database") +} + +func delQuote(channel string, quote int) error { + _, createdAt, _, err := getQuoteRaw(channel, quote) + if err != nil { + return errors.Wrap(err, "fetching specified quote") + } + + _, err = db.DB().Exec( + `DELETE FROM quotedb + WHERE channel = $1 AND created_at = $2;`, + channel, createdAt, + ) + + return errors.Wrap(err, "deleting quote") +} + +func getChannelQuotes(channel string) ([]string, error) { + rows, err := db.DB().Query( + `SELECT quote + FROM quotedb + WHERE channel = $1 + ORDER BY created_at ASC`, + channel, + ) + if err != nil { + return nil, errors.Wrap(err, "querying quotes") + } + + var quotes []string + for rows.Next() { + if err = rows.Err(); err != nil { + return nil, errors.Wrap(err, "advancing row read") + } + + var quote string + if err = rows.Scan("e); err != nil { + return nil, errors.Wrap(err, "scanning row") + } + + quotes = append(quotes, quote) + } + + return quotes, errors.Wrap(rows.Err(), "advancing row read") +} + +func getMaxQuoteIdx(channel string) (int, error) { + row := db.DB().QueryRow( + `SELECT COUNT(1) as quoteCount + FROM quotedb + WHERE channel = $1;`, + channel, + ) + + var count int + err := row.Scan(&count) + + return count, errors.Wrap(err, "getting quote count") +} + +func getQuote(channel string, quote int) (int, string, error) { + quoteIdx, _, quoteText, err := getQuoteRaw(channel, quote) + return quoteIdx, quoteText, err +} + +func getQuoteRaw(channel string, quote int) (int, int64, string, error) { + if quote == 0 { + max, err := getMaxQuoteIdx(channel) + if err != nil { + return 0, 0, "", errors.Wrap(err, "getting max quote idx") + } + quote = rand.Intn(max) + 1 // #nosec G404 // no need for cryptographic safety + } + + row := db.DB().QueryRow( + `SELECT created_at, quote + FROM quotedb + WHERE channel = $1 + ORDER BY created_at ASC + LIMIT 1 OFFSET $2`, + channel, quote-1, + ) + + var ( + createdAt int64 + quoteText string + ) + + err := row.Scan(&createdAt, "eText) + switch { + case err == nil: + return quote, createdAt, quoteText, nil + + case errors.Is(err, sql.ErrNoRows): + return 0, 0, "", nil + + default: + return 0, 0, "", errors.Wrap(err, "getting quote from DB") + } +} + +func setQuotes(channel string, quotes []string) error { + tx, err := db.DB().Begin() + if err != nil { + return errors.Wrap(err, "creating transaction") + } + + if _, err = tx.Exec( + `DELETE FROM quotedb + WHERE channel = $1;`, + channel, + ); err != nil { + defer tx.Rollback() + return errors.Wrap(err, "deleting quotes for channel") + } + + t := time.Now() + for _, quote := range quotes { + if _, err = tx.Exec( + `INSERT INTO quotedb + (channel, created_at, quote) + VALUES ($1, $2, $3);`, + channel, t.UnixNano(), quote, + ); err != nil { + defer tx.Rollback() + return errors.Wrap(err, "adding quote for channel") + } + + t = t.Add(time.Nanosecond) // Increase by one ns to adhere to unique index + } + + return errors.Wrap(tx.Commit(), "committing change") +} + +func updateQuote(channel string, idx int, quote string) error { + _, createdAt, _, err := getQuoteRaw(channel, idx) + if err != nil { + return errors.Wrap(err, "fetching specified quote") + } + + _, err = db.DB().Exec( + `UPDATE quotedb + SET quote = $3 + WHERE channel = $1 + AND created_at = $2;`, + channel, createdAt, quote, + ) + + return errors.Wrap(err, "updating quote") +} diff --git a/internal/actors/quotedb/http.go b/internal/actors/quotedb/http.go index d0176af..e7140c8 100644 --- a/internal/actors/quotedb/http.go +++ b/internal/actors/quotedb/http.go @@ -133,12 +133,10 @@ func handleAddQuotes(w http.ResponseWriter, r *http.Request) { } for _, q := range quotes { - storedObject.AddQuote(channel, q) - } - - if err := store.SetModuleStore(moduleUUID, storedObject); err != nil { - http.Error(w, errors.Wrap(err, "storing quote database").Error(), http.StatusInternalServerError) - return + if err := addQuote(channel, q); err != nil { + http.Error(w, errors.Wrap(err, "adding quote").Error(), http.StatusInternalServerError) + return + } } w.WriteHeader(http.StatusCreated) @@ -156,10 +154,8 @@ func handleDeleteQuote(w http.ResponseWriter, r *http.Request) { return } - storedObject.DelQuote(channel, idx) - - if err := store.SetModuleStore(moduleUUID, storedObject); err != nil { - http.Error(w, errors.Wrap(err, "storing quote database").Error(), http.StatusInternalServerError) + if err = delQuote(channel, idx); err != nil { + http.Error(w, errors.Wrap(err, "deleting quote").Error(), http.StatusInternalServerError) return } @@ -175,9 +171,13 @@ func handleListQuotes(w http.ResponseWriter, r *http.Request) { channel := "#" + strings.TrimLeft(mux.Vars(r)["channel"], "#") - quotes := storedObject.GetChannelQuotes(channel) + quotes, err := getChannelQuotes(channel) + if err != nil { + http.Error(w, errors.Wrap(err, "getting quotes").Error(), http.StatusInternalServerError) + return + } - if err := json.NewEncoder(w).Encode(quotes); err != nil { + if err = json.NewEncoder(w).Encode(quotes); err != nil { http.Error(w, errors.Wrap(err, "enocding quote list").Error(), http.StatusInternalServerError) return } @@ -192,10 +192,8 @@ func handleReplaceQuotes(w http.ResponseWriter, r *http.Request) { return } - storedObject.SetQuotes(channel, quotes) - - if err := store.SetModuleStore(moduleUUID, storedObject); err != nil { - http.Error(w, errors.Wrap(err, "storing quote database").Error(), http.StatusInternalServerError) + if err := setQuotes(channel, quotes); err != nil { + http.Error(w, errors.Wrap(err, "replacing quotes").Error(), http.StatusInternalServerError) return } @@ -230,10 +228,8 @@ func handleUpdateQuote(w http.ResponseWriter, r *http.Request) { return } - storedObject.UpdateQuote(channel, idx, quotes[0]) - - if err := store.SetModuleStore(moduleUUID, storedObject); err != nil { - http.Error(w, errors.Wrap(err, "storing quote database").Error(), http.StatusInternalServerError) + if err = updateQuote(channel, idx, quotes[0]); err != nil { + http.Error(w, errors.Wrap(err, "updating quote").Error(), http.StatusInternalServerError) return } diff --git a/internal/actors/quotedb/schema/001.sql b/internal/actors/quotedb/schema/001.sql new file mode 100644 index 0000000..99232c9 --- /dev/null +++ b/internal/actors/quotedb/schema/001.sql @@ -0,0 +1,7 @@ +CREATE TABLE quotedb ( + channel STRING NOT NULL, + created_at INTEGER, + quote STRING NOT NULL, + + UNIQUE(channel, created_at) +); diff --git a/action_setvar.go b/internal/actors/variables/actor.go similarity index 70% rename from action_setvar.go rename to internal/actors/variables/actor.go index cc1d7e7..063a25a 100644 --- a/action_setvar.go +++ b/internal/actors/variables/actor.go @@ -1,4 +1,4 @@ -package main +package variables import ( "fmt" @@ -8,13 +8,29 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" + "github.com/Luzifer/twitch-bot/pkg/database" "github.com/Luzifer/twitch-bot/plugins" ) -func init() { - registerAction("setvariable", func() plugins.Actor { return &ActorSetVariable{} }) +var ( + db database.Connector + formatMessage plugins.MsgFormatter - registerActorDocumentation(plugins.ActionDocumentation{ + ptrBoolFalse = func(b bool) *bool { return &b }(false) + ptrStringEmpty = func(s string) *string { return &s }("") +) + +func Register(args plugins.RegistrationArguments) error { + db = args.GetDatabaseConnector() + if err := db.Migrate("setvariable", database.NewEmbedFSMigrator(schema, "schema")); err != nil { + return errors.Wrap(err, "applying schema migration") + } + + formatMessage = args.FormatMessage + + args.RegisterActor("setvariable", func() plugins.Actor { return &ActorSetVariable{} }) + + args.RegisterActorDocumentation(plugins.ActionDocumentation{ Description: "Modify variable contents", Name: "Modify Variable", Type: "setvariable", @@ -50,7 +66,7 @@ func init() { }, }) - registerRoute(plugins.HTTPRouteRegistrationArgs{ + args.RegisterAPIRoute(plugins.HTTPRouteRegistrationArgs{ Description: "Returns the value as a plain string", HandlerFunc: routeActorSetVarGetValue, Method: http.MethodGet, @@ -66,7 +82,7 @@ func init() { }, }) - registerRoute(plugins.HTTPRouteRegistrationArgs{ + args.RegisterAPIRoute(plugins.HTTPRouteRegistrationArgs{ Description: "Updates the value of the variable", HandlerFunc: routeActorSetVarSetValue, Method: http.MethodPatch, @@ -89,6 +105,20 @@ func init() { }, }, }) + + args.RegisterTemplateFunction("variable", plugins.GenericTemplateFunctionGetter(func(name string, defVal ...string) (string, error) { + value, err := getVariable(name) + if err != nil { + return "", errors.Wrap(err, "getting variable") + } + + if value == "" && len(defVal) > 0 { + return defVal[0], nil + } + return value, nil + })) + + return nil } type ActorSetVariable struct{} @@ -101,7 +131,7 @@ func (a ActorSetVariable) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule if attrs.MustBool("clear", ptrBoolFalse) { return false, errors.Wrap( - store.RemoveVariable(varName), + removeVariable(varName), "removing variable", ) } @@ -112,7 +142,7 @@ func (a ActorSetVariable) Execute(c *irc.Client, m *irc.Message, r *plugins.Rule } return false, errors.Wrap( - store.SetVariable(varName, value), + setVariable(varName, value), "setting variable", ) } @@ -129,12 +159,18 @@ func (a ActorSetVariable) Validate(attrs *plugins.FieldCollection) (err error) { } func routeActorSetVarGetValue(w http.ResponseWriter, r *http.Request) { + vc, err := getVariable(mux.Vars(r)["name"]) + if err != nil { + http.Error(w, errors.Wrap(err, "getting value").Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text-plain") - fmt.Fprint(w, store.GetVariable(mux.Vars(r)["name"])) + fmt.Fprint(w, vc) } func routeActorSetVarSetValue(w http.ResponseWriter, r *http.Request) { - if err := store.SetVariable(mux.Vars(r)["name"], r.FormValue("value")); err != nil { + if err := setVariable(mux.Vars(r)["name"], r.FormValue("value")); err != nil { http.Error(w, errors.Wrap(err, "updating value").Error(), http.StatusInternalServerError) return } diff --git a/internal/actors/variables/database.go b/internal/actors/variables/database.go new file mode 100644 index 0000000..3ff2f8f --- /dev/null +++ b/internal/actors/variables/database.go @@ -0,0 +1,56 @@ +package variables + +import ( + "database/sql" + "embed" + + "github.com/pkg/errors" +) + +//go:embed schema/** +var schema embed.FS + +func getVariable(key string) (string, error) { + row := db.DB().QueryRow( + `SELECT value + FROM variables + WHERE name = $1`, + key, + ) + + var vc string + err := row.Scan(&vc) + switch { + case err == nil: + return vc, nil + + case errors.Is(err, sql.ErrNoRows): + return "", nil // Compatibility to old behavior + + default: + return "", errors.Wrap(err, "getting value from database") + } +} + +func setVariable(key, value string) error { + _, err := db.DB().Exec( + `INSERT INTO variables + (name, value) + VALUES ($1, $2) + ON CONFLICT DO UPDATE + SET value = excluded.value;`, + key, value, + ) + + return errors.Wrap(err, "updating value in database") +} + +func removeVariable(key string) error { + _, err := db.DB().Exec( + `DELETE FROM variables + WHERE name = $1;`, + key, + ) + + return errors.Wrap(err, "deleting value in database") +} diff --git a/internal/actors/variables/schema/001.sql b/internal/actors/variables/schema/001.sql new file mode 100644 index 0000000..02121ee --- /dev/null +++ b/internal/actors/variables/schema/001.sql @@ -0,0 +1,4 @@ +CREATE TABLE variables ( + name STRING NOT NULL PRIMARY KEY, + value STRING +); diff --git a/internal/apimodules/overlays/database.go b/internal/apimodules/overlays/database.go new file mode 100644 index 0000000..9e3eaa0 --- /dev/null +++ b/internal/apimodules/overlays/database.go @@ -0,0 +1,74 @@ +package overlays + +import ( + "bytes" + "embed" + "encoding/json" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/Luzifer/twitch-bot/plugins" +) + +//go:embed schema/** +var schema embed.FS + +func addEvent(channel string, evt socketMessage) error { + buf := new(bytes.Buffer) + if err := json.NewEncoder(buf).Encode(evt.Fields); err != nil { + return errors.Wrap(err, "encoding fields") + } + + _, err := db.DB().Exec( + `INSERT INTO overlays_events + (channel, created_at, event_type, fields) + VALUES ($1, $2, $3, $4);`, + channel, evt.Time.UnixNano(), evt.Type, strings.TrimSpace(buf.String()), + ) + + return errors.Wrap(err, "storing event to database") +} + +func getChannelEvents(channel string) ([]socketMessage, error) { + rows, err := db.DB().Query( + `SELECT created_at, event_type, fields + FROM overlays_events + WHERE channel = $1 + ORDER BY created_at;`, + channel, + ) + if err != nil { + return nil, errors.Wrap(err, "querying channel events") + } + + var out []socketMessage + for rows.Next() { + if err = rows.Err(); err != nil { + return nil, errors.Wrap(err, "advancing row read") + } + + var ( + createdAt int64 + eventType, rawFields string + ) + if err = rows.Scan(&createdAt, &eventType, &rawFields); err != nil { + return nil, errors.Wrap(err, "scanning row") + } + + fields := new(plugins.FieldCollection) + if err = json.NewDecoder(strings.NewReader(rawFields)).Decode(fields); err != nil { + return nil, errors.Wrap(err, "decoding fields") + } + + out = append(out, socketMessage{ + IsLive: false, + Time: time.Unix(0, createdAt), + Type: eventType, + Fields: fields, + }) + } + + return out, errors.Wrap(rows.Err(), "advancing row read") +} diff --git a/internal/apimodules/overlays/overlays.go b/internal/apimodules/overlays/overlays.go index fb47965..be7ce81 100644 --- a/internal/apimodules/overlays/overlays.go +++ b/internal/apimodules/overlays/overlays.go @@ -17,6 +17,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/Luzifer/go_helpers/v2/str" + "github.com/Luzifer/twitch-bot/pkg/database" "github.com/Luzifer/twitch-bot/plugins" ) @@ -25,18 +26,10 @@ const ( bufferSizeByte = 1024 socketKeepAlive = 5 * time.Second - moduleUUID = "f9ca2b3a-baf6-45ea-a347-c626168665e8" - msgTypeRequestAuth = "_auth" ) type ( - storage struct { - ChannelEvents map[string][]socketMessage `json:"channel_events"` - - lock sync.RWMutex - } - socketMessage struct { IsLive bool `json:"is_live"` Time time.Time `json:"time"` @@ -49,15 +42,15 @@ var ( //go:embed default/** embeddedOverlays embed.FS + db database.Connector + fsStack httpFSStack ptrStringEmpty = func(v string) *string { return &v }("") - store plugins.StorageManager storeExemption = []string{ "join", "part", // Those make no sense for replay } - storedObject = newStorage() subscribers = map[string]func(event string, eventData *plugins.FieldCollection){} subscribersLock sync.RWMutex @@ -71,7 +64,11 @@ var ( ) func Register(args plugins.RegistrationArguments) error { - store = args.GetStorageManager() + db = args.GetDatabaseConnector() + if err := db.Migrate("overlays", database.NewEmbedFSMigrator(schema, "schema")); err != nil { + return errors.Wrap(err, "applying schema migration") + } + validateToken = args.ValidateToken args.RegisterAPIRoute(plugins.HTTPRouteRegistrationArgs{ @@ -131,16 +128,14 @@ func Register(args plugins.RegistrationArguments) error { return nil } - storedObject.AddEvent(plugins.DeriveChannel(nil, eventData), socketMessage{ - IsLive: false, - Time: time.Now(), - Type: event, - Fields: eventData, - }) - return errors.Wrap( - store.SetModuleStore(moduleUUID, storedObject), - "storing events database", + addEvent(plugins.DeriveChannel(nil, eventData), socketMessage{ + IsLive: false, + Time: time.Now(), + Type: event, + Fields: eventData, + }), + "storing event", ) }) @@ -155,10 +150,7 @@ func Register(args plugins.RegistrationArguments) error { fsStack = append(httpFSStack{http.Dir(overlaysDir)}, fsStack...) } - return errors.Wrap( - store.GetModuleStore(moduleUUID, storedObject), - "loading module storage", - ) + return nil } func handleEventsReplay(w http.ResponseWriter, r *http.Request) { @@ -172,7 +164,13 @@ func handleEventsReplay(w http.ResponseWriter, r *http.Request) { since = s } - for _, msg := range storedObject.GetChannelEvents("#" + strings.TrimLeft(channel, "#")) { + events, err := getChannelEvents("#" + strings.TrimLeft(channel, "#")) + if err != nil { + http.Error(w, errors.Wrap(err, "getting channel events").Error(), http.StatusInternalServerError) + return + } + + for _, msg := range events { if msg.Time.Before(since) { continue } @@ -351,45 +349,3 @@ func unsubscribeSocket(id string) { delete(subscribers, id) } - -// Storage - -func newStorage() *storage { - return &storage{ - ChannelEvents: make(map[string][]socketMessage), - } -} - -func (s *storage) AddEvent(channel string, evt socketMessage) { - s.lock.Lock() - defer s.lock.Unlock() - - s.ChannelEvents[channel] = append(s.ChannelEvents[channel], evt) -} - -func (s *storage) GetChannelEvents(channel string) []socketMessage { - s.lock.RLock() - defer s.lock.RUnlock() - - return s.ChannelEvents[channel] -} - -// Implement marshaller interfaces -func (s *storage) MarshalStoredObject() ([]byte, error) { - s.lock.RLock() - defer s.lock.RUnlock() - - return json.Marshal(s) -} - -func (s *storage) UnmarshalStoredObject(data []byte) error { - if data == nil { - // No data set yet, don't try to unmarshal - return nil - } - - s.lock.Lock() - defer s.lock.Unlock() - - return json.Unmarshal(data, s) -} diff --git a/internal/apimodules/overlays/schema/001.sql b/internal/apimodules/overlays/schema/001.sql new file mode 100644 index 0000000..0e546d7 --- /dev/null +++ b/internal/apimodules/overlays/schema/001.sql @@ -0,0 +1,9 @@ +CREATE TABLE overlays_events ( + channel STRING NOT NULL, + created_at INTEGER, + event_type STRING, + fields STRING +); + +CREATE INDEX overlays_events_sort_idx + ON overlays_events (channel, created_at DESC); diff --git a/internal/service/access/access.go b/internal/service/access/access.go new file mode 100644 index 0000000..3672602 --- /dev/null +++ b/internal/service/access/access.go @@ -0,0 +1,190 @@ +package access + +import ( + "database/sql" + "strings" + + "github.com/pkg/errors" + + "github.com/Luzifer/go_helpers/v2/str" + "github.com/Luzifer/twitch-bot/pkg/database" + "github.com/Luzifer/twitch-bot/pkg/twitch" +) + +const ( + coreMetaKeyBotToken = "bot_access_token" + coreMetaKeyBotRefreshToken = "bot_refresh_token" //#nosec:G101 // That's a key, not a credential +) + +type ( + ClientConfig struct { + TwitchClient string + TwitchClientSecret string + + FallbackToken string + + TokenUpdateHook func() + } + + Service struct{ db database.Connector } +) + +func New(db database.Connector) *Service { + return &Service{db} +} + +func (s Service) GetBotTwitchClient(cfg ClientConfig) (*twitch.Client, error) { + var botAccessToken, botRefreshToken string + + err := s.db.ReadEncryptedCoreMeta(coreMetaKeyBotToken, &botAccessToken) + switch { + case errors.Is(err, nil): + // This is fine + + case errors.Is(err, database.ErrCoreMetaNotFound): + botAccessToken = cfg.FallbackToken + + default: + return nil, errors.Wrap(err, "getting bot access token from database") + } + + if err = s.db.ReadEncryptedCoreMeta(coreMetaKeyBotToken, &botAccessToken); err != nil && !errors.Is(err, database.ErrCoreMetaNotFound) { + return nil, errors.Wrap(err, "getting bot refresh token from database") + } + + twitchClient := twitch.New(cfg.TwitchClient, cfg.TwitchClientSecret, botAccessToken, botRefreshToken) + twitchClient.SetTokenUpdateHook(s.SetBotTwitchCredentials) + + return twitchClient, nil +} + +func (s Service) GetTwitchClientForChannel(channel string, cfg ClientConfig) (*twitch.Client, error) { + var err error + row := s.db.DB().QueryRow( + `SELECT access_token, refresh_token, scopes + FROM extended_permissions + WHERE channel = $1`, + channel, + ) + + var accessToken, refreshToken, scopeStr string + if err = row.Scan(&accessToken, &refreshToken, &scopeStr); err != nil { + return nil, errors.Wrap(err, "getting twitch credentials from database") + } + + if accessToken, err = s.db.DecryptField(accessToken); err != nil { + return nil, errors.Wrap(err, "decrypting access token") + } + + if refreshToken, err = s.db.DecryptField(refreshToken); err != nil { + return nil, errors.Wrap(err, "decrypting refresh token") + } + + scopes := strings.Split(scopeStr, " ") + + tc := twitch.New(cfg.TwitchClient, cfg.TwitchClientSecret, accessToken, refreshToken) + tc.SetTokenUpdateHook(func(at, rt string) error { + return errors.Wrap(s.SetExtendedTwitchCredentials(channel, at, rt, scopes), "updating extended permissions token") + }) + + return tc, nil +} + +func (s Service) HasAnyPermissionForChannel(channel string, scopes ...string) (bool, error) { + row := s.db.DB().QueryRow( + `SELECT scopes + FROM extended_permissions + WHERE channel = $1`, + channel, + ) + + var scopeStr string + if err := row.Scan(&scopeStr); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return false, errors.Wrap(err, "getting scopes from database") + } + + storedScopes := strings.Split(scopeStr, " ") + + for _, scope := range scopes { + if str.StringInSlice(scope, storedScopes) { + return true, nil + } + } + + return false, nil +} + +func (s Service) HasPermissionsForChannel(channel string, scopes ...string) (bool, error) { + row := s.db.DB().QueryRow( + `SELECT scopes + FROM extended_permissions + WHERE channel = $1`, + channel, + ) + + var scopeStr string + if err := row.Scan(&scopeStr); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return false, errors.Wrap(err, "getting scopes from database") + } + + storedScopes := strings.Split(scopeStr, " ") + + for _, scope := range scopes { + if !str.StringInSlice(scope, storedScopes) { + return false, nil + } + } + + return true, nil +} + +func (s Service) RemoveExendedTwitchCredentials(channel string) error { + _, err := s.db.DB().Exec( + `DELETE FROM extended_permissions + WHERE channel = $1`, + channel, + ) + + return errors.Wrap(err, "deleting data from table") +} + +func (s Service) SetBotTwitchCredentials(accessToken, refreshToken string) (err error) { + if err = s.db.StoreEncryptedCoreMeta(coreMetaKeyBotToken, accessToken); err != nil { + return errors.Wrap(err, "storing bot access token") + } + + if err = s.db.StoreEncryptedCoreMeta(coreMetaKeyBotRefreshToken, refreshToken); err != nil { + return errors.Wrap(err, "storing bot refresh token") + } + + return nil +} + +func (s Service) SetExtendedTwitchCredentials(channel, accessToken, refreshToken string, scope []string) (err error) { + if accessToken, err = s.db.EncryptField(accessToken); err != nil { + return errors.Wrap(err, "encrypting access token") + } + + if refreshToken, err = s.db.EncryptField(refreshToken); err != nil { + return errors.Wrap(err, "encrypting refresh token") + } + + _, err = s.db.DB().Exec( + `INSERT INTO extended_permissions + (channel, access_token, refresh_token, scopes) + VALUES ($1, $2, $3, $4) + ON CONFLICT DO UPDATE SET + access_token=excluded.access_token, + refresh_token=excluded.refresh_token, + scopes=excluded.scopes;`, + channel, accessToken, refreshToken, strings.Join(scope, " "), + ) + + return errors.Wrap(err, "inserting data into table") +} diff --git a/internal/service/timer/schema/001.sql b/internal/service/timer/schema/001.sql new file mode 100644 index 0000000..b4a72ac --- /dev/null +++ b/internal/service/timer/schema/001.sql @@ -0,0 +1,4 @@ +CREATE TABLE timers ( + id STRING NOT NULL PRIMARY KEY, + expires_at INTEGER +); diff --git a/internal/service/timer/timer.go b/internal/service/timer/timer.go new file mode 100644 index 0000000..122a6d1 --- /dev/null +++ b/internal/service/timer/timer.go @@ -0,0 +1,103 @@ +package timer + +import ( + "crypto/sha256" + "embed" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/Luzifer/twitch-bot/pkg/database" + "github.com/Luzifer/twitch-bot/plugins" +) + +type ( + Service struct { + db database.Connector + permitTimeout time.Duration + } +) + +var ( + _ plugins.TimerStore = (*Service)(nil) + + //go:embed schema/** + schema embed.FS +) + +func New(db database.Connector) (*Service, error) { + s := &Service{ + db: db, + } + + return s, errors.Wrap(s.db.Migrate("timersvc", database.NewEmbedFSMigrator(schema, "schema")), "applying migrations") +} + +func (s *Service) UpdatePermitTimeout(d time.Duration) { + s.permitTimeout = d +} + +// Cooldown timer + +func (s Service) AddCooldown(tt plugins.TimerType, limiter, ruleID string, expiry time.Time) error { + return s.setTimer(s.getCooldownTimerKey(tt, limiter, ruleID), expiry) +} + +func (s Service) InCooldown(tt plugins.TimerType, limiter, ruleID string) (bool, error) { + return s.hasTimer(s.getCooldownTimerKey(tt, limiter, ruleID)) +} + +func (Service) getCooldownTimerKey(tt plugins.TimerType, limiter, ruleID string) string { + h := sha256.New() + fmt.Fprintf(h, "%d:%s:%s", tt, limiter, ruleID) + return fmt.Sprintf("sha256:%x", h.Sum(nil)) +} + +// Permit timer + +func (s Service) AddPermit(channel, username string) error { + return s.setTimer(s.getPermitTimerKey(channel, username), time.Now().Add(s.permitTimeout)) +} + +func (s Service) HasPermit(channel, username string) (bool, error) { + return s.hasTimer(s.getPermitTimerKey(channel, username)) +} + +func (Service) getPermitTimerKey(channel, username string) string { + h := sha256.New() + fmt.Fprintf(h, "%d:%s:%s", plugins.TimerTypePermit, channel, strings.ToLower(strings.TrimLeft(username, "@"))) + return fmt.Sprintf("sha256:%x", h.Sum(nil)) +} + +// Generic timer + +func (s Service) hasTimer(id string) (bool, error) { + row := s.db.DB().QueryRow( + `SELECT COUNT(1) as active_counters + FROM timers + WHERE id = $1 AND expires_at >= $2`, + id, time.Now().UTC().Unix(), + ) + + var nCounters int64 + if err := row.Scan(&nCounters); err != nil { + return false, errors.Wrap(err, "getting active counters from database") + } + + return nCounters > 0, nil +} + +func (s Service) setTimer(id string, expiry time.Time) error { + _, err := s.db.DB().Exec( + `INSERT INTO timers + (id, expires_at) + VALUES ($1, $2) + ON CONFLICT DO UPDATE + SET expires_at = excluded.expires_at;`, + id, expiry.UTC().Unix(), + ) + + return errors.Wrap(err, "storing counter in database") +} diff --git a/internal/v2migrator/core.go b/internal/v2migrator/core.go new file mode 100644 index 0000000..636754f --- /dev/null +++ b/internal/v2migrator/core.go @@ -0,0 +1,90 @@ +package v2migrator + +import ( + "github.com/pkg/errors" + + "github.com/Luzifer/twitch-bot/internal/service/access" + "github.com/Luzifer/twitch-bot/pkg/database" +) + +func (s storageFile) migrateCoreKV(db database.Connector) (err error) { + as := access.New(db) + + if err = as.SetBotTwitchCredentials(s.BotAccessToken, s.BotRefreshToken); err != nil { + return errors.Wrap(err, "setting bot credentials") + } + + if err = db.StoreEncryptedCoreMeta("event_sub_secret", s.EventSubSecret); err != nil { + return errors.Wrap(err, "storing bot eventsub token") + } + + return nil +} + +func (s storageFile) migrateCounters(db database.Connector) (err error) { + for counter, value := range s.Counters { + if _, err = db.DB().Exec( + `INSERT INTO counters + (name, value) + VALUES ($1, $2) + ON CONFLICT DO UPDATE + SET value = excluded.value;`, + counter, value, + ); err != nil { + return errors.Wrap(err, "storing counter value") + } + } + + return nil +} + +func (s storageFile) migratePermissions(db database.Connector) (err error) { + as := access.New(db) + + for channel, perms := range s.ExtendedPermissions { + if err = as.SetExtendedTwitchCredentials( + channel, + perms.AccessToken, + perms.RefreshToken, + perms.Scopes, + ); err != nil { + return errors.Wrapf(err, "storing channel %q credentials", channel) + } + } + + return nil +} + +func (s storageFile) migrateTimers(db database.Connector) (err error) { + for id, expiry := range s.Timers { + if _, err := db.DB().Exec( + `INSERT INTO timers + (id, expires_at) + VALUES ($1, $2) + ON CONFLICT DO UPDATE + SET expires_at = excluded.expires_at;`, + id, expiry.Time.Unix(), + ); err != nil { + return errors.Wrap(err, "storing counter in database") + } + } + + return nil +} + +func (s storageFile) migrateVariables(db database.Connector) (err error) { + for key, value := range s.Variables { + if _, err = db.DB().Exec( + `INSERT INTO variables + (name, value) + VALUES ($1, $2) + ON CONFLICT DO UPDATE + SET value = excluded.value;`, + key, value, + ); err != nil { + return errors.Wrap(err, "updating value in database") + } + } + + return nil +} diff --git a/crypt/crypt.go b/internal/v2migrator/crypt/crypt.go similarity index 100% rename from crypt/crypt.go rename to internal/v2migrator/crypt/crypt.go diff --git a/internal/v2migrator/modOverlays.go b/internal/v2migrator/modOverlays.go new file mode 100644 index 0000000..6b4db8f --- /dev/null +++ b/internal/v2migrator/modOverlays.go @@ -0,0 +1,46 @@ +package v2migrator + +import ( + "bytes" + "encoding/json" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/Luzifer/twitch-bot/pkg/database" + "github.com/Luzifer/twitch-bot/plugins" +) + +type ( + storageModOverlays struct { + ChannelEvents map[string][]struct { + IsLive bool `json:"is_live"` + Time time.Time `json:"time"` + Type string `json:"type"` + Fields *plugins.FieldCollection `json:"fields"` + } `json:"channel_events"` + } +) + +func (s storageModOverlays) migrate(db database.Connector) (err error) { + for channel, evts := range s.ChannelEvents { + for _, evt := range evts { + buf := new(bytes.Buffer) + if err = json.NewEncoder(buf).Encode(evt.Fields); err != nil { + return errors.Wrap(err, "encoding fields") + } + + if _, err = db.DB().Exec( + `INSERT INTO overlays_events + (channel, created_at, event_type, fields) + VALUES ($1, $2, $3, $4);`, + channel, evt.Time.UnixNano(), evt.Type, strings.TrimSpace(buf.String()), + ); err != nil { + return errors.Wrap(err, "storing event to database") + } + } + } + + return nil +} diff --git a/internal/v2migrator/modPunish.go b/internal/v2migrator/modPunish.go new file mode 100644 index 0000000..8b0041d --- /dev/null +++ b/internal/v2migrator/modPunish.go @@ -0,0 +1,39 @@ +package v2migrator + +import ( + "time" + + "github.com/pkg/errors" + + "github.com/Luzifer/twitch-bot/pkg/database" +) + +type ( + storageModPunish struct { + ActiveLevels map[string]*struct { + LastLevel int `json:"last_level"` + Executed time.Time `json:"executed"` + Cooldown time.Duration `json:"cooldown"` + } `json:"active_levels"` + } +) + +func (s storageModPunish) migrate(db database.Connector) (err error) { + for key, lc := range s.ActiveLevels { + if _, err = db.DB().Exec( + `INSERT INTO punish_levels + (key, last_level, executed, cooldown) + VALUES ($1, $2, $3, $4) + ON CONFLICT DO UPDATE + SET last_level = excluded.last_level, + executed = excluded.executed, + cooldown = excluded.cooldown;`, + key, + lc.LastLevel, lc.Executed.UTC().Unix(), int64(lc.Cooldown), + ); err != nil { + return errors.Wrap(err, "updating punishment info") + } + } + + return nil +} diff --git a/internal/v2migrator/modQuoteDB.go b/internal/v2migrator/modQuoteDB.go new file mode 100644 index 0000000..57977a3 --- /dev/null +++ b/internal/v2migrator/modQuoteDB.go @@ -0,0 +1,35 @@ +package v2migrator + +import ( + "time" + + "github.com/pkg/errors" + + "github.com/Luzifer/twitch-bot/pkg/database" +) + +type ( + storageModQuoteDB struct { + ChannelQuotes map[string][]string `json:"channel_quotes"` + } +) + +func (s storageModQuoteDB) migrate(db database.Connector) (err error) { + for channel, quotes := range s.ChannelQuotes { + t := time.Now() + for _, quote := range quotes { + if _, err = db.DB().Exec( + `INSERT INTO quotedb + (channel, created_at, quote) + VALUES ($1, $2, $3);`, + channel, t.UnixNano(), quote, + ); err != nil { + return errors.Wrap(err, "adding quote for channel") + } + + t = t.Add(time.Nanosecond) // Increase by one ns to adhere to unique index + } + } + + return nil +} diff --git a/internal/v2migrator/store.go b/internal/v2migrator/store.go new file mode 100644 index 0000000..7b8e89e --- /dev/null +++ b/internal/v2migrator/store.go @@ -0,0 +1,119 @@ +package v2migrator + +import ( + "compress/gzip" + "encoding/json" + "os" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/Luzifer/twitch-bot/internal/v2migrator/crypt" + "github.com/Luzifer/twitch-bot/pkg/database" + "github.com/Luzifer/twitch-bot/plugins" +) + +type ( + Migrator interface { + Load(filename, encryptionPass string) error + Migrate(db database.Connector) error + } + + storageExtendedPermission struct { + AccessToken string `encrypt:"true" json:"access_token,omitempty"` + RefreshToken string `encrypt:"true" json:"refresh_token,omitempty"` + Scopes []string `json:"scopes,omitempty"` + } + + storageFile struct { + Counters map[string]int64 `json:"counters"` + Timers map[string]plugins.TimerEntry `json:"timers"` + Variables map[string]string `json:"variables"` + + ModuleStorage struct { + ModPunish storageModPunish `json:"44ab4646-ce50-4e16-9353-c1f0eb68962b"` + ModOverlays storageModOverlays `json:"f9ca2b3a-baf6-45ea-a347-c626168665e8"` + ModQuoteDB storageModQuoteDB `json:"917c83ee-ed40-41e4-a558-1c2e59fdf1f5"` + } `json:"module_storage"` + + ExtendedPermissions map[string]*storageExtendedPermission `json:"extended_permissions"` + + EventSubSecret string `encrypt:"true" json:"event_sub_secret,omitempty"` + + BotAccessToken string `encrypt:"true" json:"bot_access_token,omitempty"` + BotRefreshToken string `encrypt:"true" json:"bot_refresh_token,omitempty"` + } +) + +func NewStorageFile() Migrator { + return &storageFile{ + Counters: map[string]int64{}, + Timers: map[string]plugins.TimerEntry{}, + Variables: map[string]string{}, + + ExtendedPermissions: map[string]*storageExtendedPermission{}, + } +} + +func (s *storageFile) Load(filename, encryptionPass string) error { + f, err := os.Open(filename) + if err != nil { + if os.IsNotExist(err) { + // Store init state + return nil + } + return errors.Wrap(err, "open storage file") + } + defer f.Close() + + zf, err := gzip.NewReader(f) + if err != nil { + return errors.Wrap(err, "create gzip reader") + } + defer zf.Close() + + if err = json.NewDecoder(zf).Decode(s); err != nil { + return errors.Wrap(err, "decode storage object") + } + + if err = crypt.DecryptFields(s, encryptionPass); err != nil { + return errors.Wrap(err, "decrypting storage object") + } + + return nil +} + +func (s storageFile) Migrate(db database.Connector) error { + var bat string + err := db.ReadCoreMeta("bot_access_token", &bat) + switch { + case err == nil: + return errors.New("Access token is set, database already initialized") + + case errors.Is(err, database.ErrCoreMetaNotFound): + // This is the expected state + + default: + return errors.Wrap(err, "checking for bot access token") + } + + for name, fn := range map[string]func(database.Connector) error{ + // Core + "core": s.migrateCoreKV, + "counter": s.migrateCounters, + "permissions": s.migratePermissions, + "timers": s.migrateTimers, + "variables": s.migrateVariables, + // Modules + "mod_punish": s.ModuleStorage.ModPunish.migrate, + "mod_overlays": s.ModuleStorage.ModOverlays.migrate, + "mod_quotedb": s.ModuleStorage.ModQuoteDB.migrate, + } { + logrus.WithField("module", name).Info("Starting migration...") + if err = fn(db); err != nil { + return errors.Wrapf(err, "executing %q migration", name) + } + } + + return nil +} diff --git a/irc.go b/irc.go index aaa84df..92a830a 100644 --- a/irc.go +++ b/irc.go @@ -13,8 +13,8 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) var ( @@ -297,7 +297,7 @@ func (i ircHandler) handlePermit(m *irc.Message) { }) log.WithFields(fields.Data()).Debug("Added permit") - timerStore.AddPermit(m.Params[0], username) + timerService.AddPermit(m.Params[0], username) go handleMessage(i.c, m, eventTypePermit, fields) } diff --git a/main.go b/main.go index e417920..a522c69 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,8 @@ package main import ( "bytes" "context" + "crypto/rand" + "encoding/hex" "fmt" "io" "math" @@ -23,7 +25,11 @@ import ( "github.com/Luzifer/go_helpers/v2/str" "github.com/Luzifer/rconfig/v2" - "github.com/Luzifer/twitch-bot/twitch" + "github.com/Luzifer/twitch-bot/internal/service/access" + "github.com/Luzifer/twitch-bot/internal/service/timer" + "github.com/Luzifer/twitch-bot/internal/v2migrator" + "github.com/Luzifer/twitch-bot/pkg/database" + "github.com/Luzifer/twitch-bot/pkg/twitch" ) const ( @@ -34,6 +40,9 @@ const ( maxIRCRetryBackoff = time.Minute httpReadHeaderTimeout = 5 * time.Second + + coreMetaKeyEventSubSecret = "event_sub_secret" + eventSubSecretLength = 32 ) var ( @@ -44,7 +53,7 @@ var ( IRCRateLimit time.Duration `flag:"rate-limit" default:"1500ms" description:"How often to send a message (default: 20/30s=1500ms, if your bot is mod everywhere: 100/30s=300ms, different for known/verified bots)"` LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` PluginDir string `flag:"plugin-dir" default:"/usr/lib/twitch-bot" description:"Where to find and load plugins"` - StorageFile string `flag:"storage-file" default:"./storage.json.gz" description:"Where to store the data"` + StorageDatabase string `flag:"storage-database" default:"./storage.db" description:"Database file to store data in"` StorageEncryptionPass string `flag:"storage-encryption-pass" default:"" description:"Passphrase to encrypt secrets inside storage (defaults to twitch-client:twitch-client-secret)"` TwitchClient string `flag:"twitch-client" default:"" description:"Client ID to act as"` TwitchClientSecret string `flag:"twitch-client-secret" default:"" description:"Secret for the Client ID"` @@ -64,7 +73,10 @@ var ( runID = uuid.Must(uuid.NewV4()).String() externalHTTPAvailable bool - store = newStorageFile(false) + db database.Connector + accessService *access.Service + timerService *timer.Service + twitchClient *twitch.Client twitchEventSubClient *twitch.EventSubClient @@ -72,14 +84,6 @@ var ( ) func init() { - for _, a := range os.Args { - if strings.HasPrefix(a, "-test.") { - // Skip initialize for test run - store = newStorageFile(true) // Use in-mem-store for tests - return - } - } - rconfig.AutoEnv(true) if err := rconfig.ParseAndValidate(&cfg); err != nil { log.Fatalf("Unable to parse commandline options: %s", err) @@ -105,6 +109,35 @@ func init() { } } +func getEventSubSecret() (secret, handle string, err error) { + var eventSubSecret string + + err = db.ReadEncryptedCoreMeta(coreMetaKeyEventSubSecret, &eventSubSecret) + switch { + case errors.Is(err, nil): + return eventSubSecret, eventSubSecret[:5], nil + + case errors.Is(err, database.ErrCoreMetaNotFound): + // We need to generate a new secret below + + default: + return "", "", errors.Wrap(err, "reading secret from database") + } + + key := make([]byte, eventSubSecretLength) + n, err := rand.Read(key) + if err != nil { + return "", "", errors.Wrap(err, "generating random secret") + } + if n != eventSubSecretLength { + return "", "", errors.Errorf("read only %d of %d byte", n, eventSubSecretLength) + } + + eventSubSecret = hex.EncodeToString(key) + + return eventSubSecret, eventSubSecret[:5], errors.Wrap(db.StoreEncryptedCoreMeta(coreMetaKeyEventSubSecret, eventSubSecret), "storing secret to database") +} + func handleSubCommand(args []string) { switch args[0] { @@ -144,8 +177,25 @@ func handleSubCommand(args []string) { fmt.Println("Supported sub-commands are:") fmt.Println(" actor-docs Generate markdown documentation for available actors") fmt.Println(" api-token Generate an api-token to be entered into the config") + fmt.Println(" migrate-v2 Migrate old (*.json.gz) storage file into new database") fmt.Println(" help Prints this help message") + case "migrate-v2": + if len(args) < 2 { //nolint:gomnd // Just a count of parameters + log.Fatalf("Usage: twitch-bot migrate-v2 ") + } + + v2s := v2migrator.NewStorageFile() + if err := v2s.Load(args[1], cfg.StorageEncryptionPass); err != nil { + log.WithError(err).Fatal("loading v2 storage file") + } + + if err := v2s.Migrate(db); err != nil { + log.WithError(err).Fatal("migrating v2 storage file") + } + + log.Info("v2 storage file was migrated") + default: handleSubCommand([]string{"help"}) log.Fatalf("Unknown sub-command %q", args[0]) @@ -157,26 +207,39 @@ func handleSubCommand(args []string) { func main() { var err error - if err = store.Load(); err != nil { - log.WithError(err).Fatal("Unable to load storage file") + databaseConnectionString := strings.Join([]string{ + cfg.StorageDatabase, + strings.Join([]string{ + "_pragma=locking_mode(EXCLUSIVE)", + "_pragma=synchronous(FULL)", + }, "&"), + }, "?") + + if db, err = database.New("sqlite", databaseConnectionString, cfg.StorageEncryptionPass); err != nil { + log.WithError(err).Fatal("Unable to open storage database") + } + + accessService = access.New(db) + if timerService, err = timer.New(db); err != nil { + log.WithError(err).Fatal("Unable to apply timer migration") } cronService = cron.New() - twitchClient = twitch.New(cfg.TwitchClient, cfg.TwitchClientSecret, store.GetBotToken(cfg.TwitchToken), store.BotRefreshToken) - twitchClient.SetTokenUpdateHook(func(at, rt string) error { - if err := store.UpdateBotToken(at, rt); err != nil { - return errors.Wrap(err, "updating store") - } - - // Misuse the config reload hook to let the frontend reload its state - configReloadHooksLock.RLock() - defer configReloadHooksLock.RUnlock() - for _, fn := range configReloadHooks { - fn() - } - - return nil - }) + if twitchClient, err = accessService.GetBotTwitchClient(access.ClientConfig{ + TwitchClient: cfg.TwitchClient, + TwitchClientSecret: cfg.TwitchClientSecret, + FallbackToken: cfg.TwitchToken, + TokenUpdateHook: func() { + // Misuse the config reload hook to let the frontend reload its state + configReloadHooksLock.RLock() + defer configReloadHooksLock.RUnlock() + for _, fn := range configReloadHooks { + fn() + } + }, + }); err != nil { + log.WithError(err).Fatal("Unable to initialize Twitch client") + } twitchWatch := newTwitchWatcher() // Query may run that often as the twitchClient has an internal @@ -265,7 +328,7 @@ func main() { checkExternalHTTP() if externalHTTPAvailable && cfg.TwitchClient != "" && cfg.TwitchClientSecret != "" { - secret, handle, err := store.GetOrGenerateEventSubSecret() + secret, handle, err := getEventSubSecret() if err != nil { log.WithError(err).Fatal("Unable to get or create eventsub secret") } diff --git a/pkg/database/connector.go b/pkg/database/connector.go new file mode 100644 index 0000000..74da92a --- /dev/null +++ b/pkg/database/connector.go @@ -0,0 +1,67 @@ +package database + +import ( + "embed" + "regexp" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) + +type ( + connector struct { + db *sqlx.DB + encryptionSecret string + } +) + +var ( + // ErrCoreMetaNotFound is the error thrown when reading a non-existent + // core_kv key + ErrCoreMetaNotFound = errors.New("core meta entry not found") + + //go:embed schema/** + schema embed.FS + + migrationFilename = regexp.MustCompile(`^([0-9]+)\.sql$`) +) + +// New creates a new Connector with the given driver and database +func New(driverName, dataSourceName, encryptionSecret string) (Connector, error) { + db, err := sqlx.Connect(driverName, dataSourceName) + if err != nil { + return nil, errors.Wrap(err, "connecting database") + } + + db.SetConnMaxIdleTime(0) + db.SetConnMaxLifetime(0) + db.SetMaxIdleConns(1) + db.SetMaxOpenConns(1) + + conn := &connector{ + db: db, + encryptionSecret: encryptionSecret, + } + return conn, errors.Wrap(conn.applyCoreSchema(), "applying core schema") +} + +func (c connector) Close() error { + return errors.Wrap(c.db.Close(), "closing database") +} + +func (c connector) DB() *sqlx.DB { + return c.db +} + +func (c connector) applyCoreSchema() error { + coreSQL, err := schema.ReadFile("schema/core.sql") + if err != nil { + return errors.Wrap(err, "reading core.sql content") + } + + if _, err = c.db.Exec(string(coreSQL)); err != nil { + return errors.Wrap(err, "applying core schema") + } + + return errors.Wrap(c.Migrate("core", NewEmbedFSMigrator(schema, "schema")), "applying core migration") +} diff --git a/pkg/database/connector_test.go b/pkg/database/connector_test.go new file mode 100644 index 0000000..eef6217 --- /dev/null +++ b/pkg/database/connector_test.go @@ -0,0 +1,98 @@ +package database + +import ( + "testing" + + "github.com/pkg/errors" +) + +const testEncryptionPass = "password123" + +func TestNewConnector(t *testing.T) { + dbc, err := New("sqlite", ":memory:", testEncryptionPass) + if err != nil { + t.Fatalf("creating database connector: %s", err) + } + defer dbc.Close() + + row := dbc.DB().QueryRow("SELECT count(1) AS tables FROM sqlite_master WHERE type='table' AND name='core_kv';") + + var count int + if err = row.Scan(&count); err != nil { + t.Fatalf("reading table count result") + } + + if count != 1 { + t.Errorf("expected to find one result, got %d in count of core_kv table", count) + } +} + +func TestCoreMetaRoundtrip(t *testing.T) { + dbc, err := New("sqlite", ":memory:", testEncryptionPass) + if err != nil { + t.Fatalf("creating database connector: %s", err) + } + defer dbc.Close() + + var ( + arbitrary struct{ A string } + testKey = "arbitrary" + ) + + if err = dbc.ReadCoreMeta(testKey, &arbitrary); !errors.Is(err, ErrCoreMetaNotFound) { + t.Error("expected core_kv not to contain key after init") + } + + checkWriteRead := func(testString string) { + arbitrary.A = testString + if err = dbc.StoreCoreMeta(testKey, arbitrary); err != nil { + t.Errorf("storing core_kv: %s", err) + } + + arbitrary.A = "" // Clear to test unmarshal + if err = dbc.ReadCoreMeta(testKey, &arbitrary); err != nil { + t.Errorf("reading core_kv: %s", err) + } + + if arbitrary.A != testString { + t.Errorf("expected meta entry to have %q, got %q", testString, arbitrary.A) + } + } + + checkWriteRead("just a string") // Turn one: Init from not existing + checkWriteRead("another random string") // Turn two: Overwrite +} + +func TestCoreMetaEncryption(t *testing.T) { + dbc, err := New("sqlite", ":memory:", testEncryptionPass) + if err != nil { + t.Fatalf("creating database connector: %s", err) + } + defer dbc.Close() + + var ( + arbitrary struct{ A string } + testKey = "arbitrary" + testString = "foobar" + ) + + arbitrary.A = testString + + if err = dbc.StoreEncryptedCoreMeta(testKey, arbitrary); err != nil { + t.Fatalf("storing encrypted core meta: %s", err) + } + + if err = dbc.ReadCoreMeta(testKey, &arbitrary); err == nil { + t.Error("reading encrypted meta without decryption succeeded") + } + + arbitrary.A = "" + + if err = dbc.ReadEncryptedCoreMeta(testKey, &arbitrary); err != nil { + t.Errorf("reading encrypted meta: %s", err) + } + + if arbitrary.A != testString { + t.Errorf("unexpected value: %q != %q", arbitrary.A, testString) + } +} diff --git a/pkg/database/coreKV.go b/pkg/database/coreKV.go new file mode 100644 index 0000000..5c46147 --- /dev/null +++ b/pkg/database/coreKV.go @@ -0,0 +1,90 @@ +package database + +import ( + "bytes" + "database/sql" + "encoding/json" + "strings" + + "github.com/pkg/errors" +) + +// ReadCoreMeta reads an entry of the core_kv table specified by +// the given `key` and unmarshals it into the `value`. The value must +// be a valid variable to `json.NewDecoder(...).Decode(value)` +// (pointer to struct, string, int, ...). In case the key does not +// exist a check to 'errors.Is(err, sql.ErrNoRows)' will succeed +func (c connector) ReadCoreMeta(key string, value any) error { + return c.readCoreMeta(key, value, nil) +} + +// StoreCoreMeta stores an entry to the core_kv table soecified by +// the given `key`. The value given must be a valid variable to +// `json.NewEncoder(...).Encode(value)`. +func (c connector) StoreCoreMeta(key string, value any) error { + return c.storeCoreMeta(key, value, nil) +} + +// ReadEncryptedCoreMeta works like ReadCoreMeta but decrypts the +// stored value before unmarshalling it +func (c connector) ReadEncryptedCoreMeta(key string, value any) error { + return c.readCoreMeta(key, value, c.DecryptField) +} + +// StoreEncryptedCoreMeta works like StoreCoreMeta but encrypts the +// marshalled value before storing it +func (c connector) StoreEncryptedCoreMeta(key string, value any) error { + return c.storeCoreMeta(key, value, c.EncryptField) +} + +func (c connector) readCoreMeta(key string, value any, processor func(string) (string, error)) (err error) { + var data struct{ Key, Value string } + data.Key = key + + if err = c.db.Get(&data, "SELECT * FROM core_kv WHERE key = $1", data.Key); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrCoreMetaNotFound + } + return errors.Wrap(err, "querying core meta table") + } + + if data.Value == "" { + return errors.New("empty value returned") + } + + if processor != nil { + if data.Value, err = processor(data.Value); err != nil { + return errors.Wrap(err, "processing stored value") + } + } + + if err := json.NewDecoder(strings.NewReader(data.Value)).Decode(value); err != nil { + return errors.Wrap(err, "JSON decoding value") + } + + return nil +} + +func (c connector) storeCoreMeta(key string, value any, processor func(string) (string, error)) (err error) { + buf := new(bytes.Buffer) + if err := json.NewEncoder(buf).Encode(value); err != nil { + return errors.Wrap(err, "JSON encoding value") + } + + encValue := strings.TrimSpace(buf.String()) + if processor != nil { + if encValue, err = processor(encValue); err != nil { + return errors.Wrap(err, "processing value to store") + } + } + + _, err = c.db.NamedExec( + "INSERT INTO core_kv (key, value) VALUES (:key, :value) ON CONFLICT DO UPDATE SET value=excluded.value;", + map[string]any{ + "key": key, + "value": encValue, + }, + ) + + return errors.Wrap(err, "upserting core meta value") +} diff --git a/pkg/database/crypt.go b/pkg/database/crypt.go new file mode 100644 index 0000000..4f50e6a --- /dev/null +++ b/pkg/database/crypt.go @@ -0,0 +1,17 @@ +package database + +import ( + "github.com/pkg/errors" + + "github.com/Luzifer/go-openssl/v4" +) + +func (c connector) DecryptField(dec string) (string, error) { + dv, err := openssl.New().DecryptBytes(c.encryptionSecret, []byte(dec), openssl.PBKDF2SHA512) + return string(dv), errors.Wrap(err, "decrypting value") +} + +func (c connector) EncryptField(enc string) (string, error) { + ev, err := openssl.New().EncryptBytes(c.encryptionSecret, []byte(enc), openssl.PBKDF2SHA512) + return string(ev), errors.Wrap(err, "encrypting value") +} diff --git a/pkg/database/database.go b/pkg/database/database.go new file mode 100644 index 0000000..6459ea7 --- /dev/null +++ b/pkg/database/database.go @@ -0,0 +1,50 @@ +// Package database represents a connector to the sqlite storage +// backend to store persistent data from core and plugins +package database + +import ( + "io/fs" + + "github.com/jmoiron/sqlx" + + // Included support for pure-go sqlite + _ "github.com/glebarez/go-sqlite" +) + +type ( + // Connector represents a database connection having some extra + // convenience methods + Connector interface { + Close() error + DB() *sqlx.DB + Migrate(module string, migrations MigrationStorage) error + ReadCoreMeta(key string, value any) error + StoreCoreMeta(key string, value any) error + ReadEncryptedCoreMeta(key string, value any) error + StoreEncryptedCoreMeta(key string, value any) error + DecryptField(string) (string, error) + EncryptField(string) (string, error) + } + + // MigrationStorage represents a file storage containing migration + // files to migrate a namespace to its desired state. The files + // MUST be named in the schema `[0-9]+\.sql`. + // + // The storage is scanned recursively and all files are then + // string-sorted by their base-name (`/migrations/001.sql => 001.sql`). + // The last executed number is stored in numeric format, the next + // migration which basename evaluates to higher numeric will be + // executed. + // + // Numbers MUST be consecutive and MUST NOT leave out a number. A + // missing number will result in the migration processing not to + // catch up any migration afterwards. + // + // The first migration MUST be number 1 + // + // Previously executed migrations MUST NOT be modified! + MigrationStorage interface { + ReadDir(name string) ([]fs.DirEntry, error) + ReadFile(name string) ([]byte, error) + } +) diff --git a/pkg/database/migration.go b/pkg/database/migration.go new file mode 100644 index 0000000..512ee47 --- /dev/null +++ b/pkg/database/migration.go @@ -0,0 +1,94 @@ +package database + +import ( + "path" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +func (c connector) Migrate(module string, migrations MigrationStorage) error { + m, err := collectMigrations(migrations, "/") + if err != nil { + return errors.Wrap(err, "collecting migrations") + } + + migrationKey := strings.Join([]string{"migration_state", module}, "-") + + var lastMigration int + if err = c.ReadCoreMeta(migrationKey, &lastMigration); err != nil && !errors.Is(err, ErrCoreMetaNotFound) { + return errors.Wrap(err, "getting last migration") + } + + nextMigration := lastMigration + for { + nextMigration++ + filename := m[nextMigration] + if filename == "" { + break + } + + if err = c.applyMigration(migrations, filename); err != nil { + return errors.Wrapf(err, "applying migration %d", nextMigration) + } + + if err = c.StoreCoreMeta(migrationKey, nextMigration); err != nil { + return errors.Wrap(err, "updating migration number") + } + } + + return nil +} + +func (c connector) applyMigration(migrations MigrationStorage, filename string) error { + rawMigration, err := migrations.ReadFile(filename) + if err != nil { + return errors.Wrap(err, "reading migration file") + } + + _, err = c.db.Exec(string(rawMigration)) + return errors.Wrap(err, "executing migration statement(s)") +} + +func collectMigrations(migrations MigrationStorage, dir string) (map[int]string, error) { + out := map[int]string{} + + entries, err := migrations.ReadDir(dir) + if err != nil { + return nil, errors.Wrapf(err, "reading dir %q", dir) + } + + for _, e := range entries { + if e.IsDir() { + sout, err := collectMigrations(migrations, path.Join(dir, e.Name())) + if err != nil { + return nil, errors.Wrapf(err, "scanning subdir %q", e.Name()) + } + + for n, p := range sout { + if out[n] != "" { + return nil, errors.Errorf("migration %d found more than once", n) + } + + out[n] = p + } + + continue + } + + if !migrationFilename.MatchString(e.Name()) { + continue + } + + matches := migrationFilename.FindStringSubmatch(e.Name()) + n, err := strconv.Atoi(matches[1]) + if err != nil { + return nil, errors.Wrap(err, "parsing migration number") + } + + out[n] = path.Join(dir, e.Name()) + } + + return out, nil +} diff --git a/pkg/database/migration_embedfs.go b/pkg/database/migration_embedfs.go new file mode 100644 index 0000000..eff19ab --- /dev/null +++ b/pkg/database/migration_embedfs.go @@ -0,0 +1,35 @@ +package database + +import ( + "embed" + "io/fs" + "path" + "strings" +) + +type ( + // EmbedFSMigrator is a wrapper around embed.FS enabling ReadDir("/") + // which normally would cause an error as path "/" is not available + // within an embed.FS + EmbedFSMigrator struct { + BasePath string + embed.FS + } +) + +// NewEmbedFSMigrator creates a new EmbedFSMigrator +func NewEmbedFSMigrator(fs embed.FS, basePath string) MigrationStorage { + return EmbedFSMigrator{BasePath: basePath, FS: fs} +} + +// ReadDir Wraps embed.FS.ReadDir with adjustment of the path prefix +func (e EmbedFSMigrator) ReadDir(name string) ([]fs.DirEntry, error) { + name = path.Join(e.BasePath, strings.TrimPrefix(name, "/")) + return e.FS.ReadDir(name) +} + +// ReadFile Wraps embed.FS.ReadFile with adjustment of the path prefix +func (e EmbedFSMigrator) ReadFile(name string) ([]byte, error) { + name = path.Join(e.BasePath, strings.TrimPrefix(name, "/")) + return e.FS.ReadFile(name) +} diff --git a/pkg/database/migration_test.go b/pkg/database/migration_test.go new file mode 100644 index 0000000..e1584a7 --- /dev/null +++ b/pkg/database/migration_test.go @@ -0,0 +1,42 @@ +package database + +import ( + "embed" + "testing" +) + +var ( + //go:embed testdata/migration1/** + testMigration1 embed.FS + //go:embed testdata/migration2/** + testMigration2 embed.FS +) + +func TestMigration(t *testing.T) { + dbc, err := New("sqlite", ":memory:", testEncryptionPass) + if err != nil { + t.Fatalf("creating database connector: %s", err) + } + defer dbc.Close() + + var ( + tm1 = NewEmbedFSMigrator(testMigration1, "testdata") + tm2 = NewEmbedFSMigrator(testMigration2, "testdata") + ) + + if err = dbc.Migrate("test", tm1); err != nil { + t.Errorf("migration 1 take 1: %s", err) + } + + if err = dbc.Migrate("test", tm1); err != nil { + t.Errorf("migration 1 take 2: %s", err) + } + + if err = dbc.Migrate("test", tm2); err != nil { + t.Errorf("migration 2 take 1: %s", err) + } + + if err = dbc.Migrate("test", tm2); err != nil { + t.Errorf("migration 2 take 2: %s", err) + } +} diff --git a/pkg/database/schema/001.sql b/pkg/database/schema/001.sql new file mode 100644 index 0000000..55d5213 --- /dev/null +++ b/pkg/database/schema/001.sql @@ -0,0 +1,6 @@ +CREATE TABLE extended_permissions ( + channel STRING NOT NULL PRIMARY KEY, + access_token STRING, + refresh_token STRING, + scopes STRING +); diff --git a/pkg/database/schema/core.sql b/pkg/database/schema/core.sql new file mode 100644 index 0000000..88f1b60 --- /dev/null +++ b/pkg/database/schema/core.sql @@ -0,0 +1,6 @@ +-- Core database structure, to be applied before any migration + +CREATE TABLE IF NOT EXISTS core_kv ( + key STRING NOT NULL PRIMARY KEY, + value STRING +); diff --git a/pkg/database/testdata/migration1/001.sql b/pkg/database/testdata/migration1/001.sql new file mode 100644 index 0000000..ec3f960 --- /dev/null +++ b/pkg/database/testdata/migration1/001.sql @@ -0,0 +1,4 @@ +CREATE TABLE testdata ( + key STRING NOT NULL PRIMARY KEY, + value STRING +); diff --git a/pkg/database/testdata/migration2/001.sql b/pkg/database/testdata/migration2/001.sql new file mode 100644 index 0000000..ec3f960 --- /dev/null +++ b/pkg/database/testdata/migration2/001.sql @@ -0,0 +1,4 @@ +CREATE TABLE testdata ( + key STRING NOT NULL PRIMARY KEY, + value STRING +); diff --git a/pkg/database/testdata/migration2/002.sql b/pkg/database/testdata/migration2/002.sql new file mode 100644 index 0000000..c5ba077 --- /dev/null +++ b/pkg/database/testdata/migration2/002.sql @@ -0,0 +1 @@ +ALTER TABLE testdata ADD COLUMN another_value STRING; diff --git a/twitch/badges.go b/pkg/twitch/badges.go similarity index 100% rename from twitch/badges.go rename to pkg/twitch/badges.go diff --git a/twitch/eventsub.go b/pkg/twitch/eventsub.go similarity index 100% rename from twitch/eventsub.go rename to pkg/twitch/eventsub.go diff --git a/twitch/http.go b/pkg/twitch/http.go similarity index 100% rename from twitch/http.go rename to pkg/twitch/http.go diff --git a/twitch/scopes.go b/pkg/twitch/scopes.go similarity index 100% rename from twitch/scopes.go rename to pkg/twitch/scopes.go diff --git a/twitch/twitch.go b/pkg/twitch/twitch.go similarity index 100% rename from twitch/twitch.go rename to pkg/twitch/twitch.go diff --git a/twitch/twitchAPICache.go b/pkg/twitch/twitchAPICache.go similarity index 100% rename from twitch/twitchAPICache.go rename to pkg/twitch/twitchAPICache.go diff --git a/plugins/interface.go b/plugins/interface.go index 2f9d91a..15d8c8e 100644 --- a/plugins/interface.go +++ b/plugins/interface.go @@ -5,7 +5,8 @@ import ( "github.com/robfig/cron/v3" log "github.com/sirupsen/logrus" - "github.com/Luzifer/twitch-bot/twitch" + "github.com/Luzifer/twitch-bot/pkg/database" + "github.com/Luzifer/twitch-bot/pkg/twitch" ) type ( @@ -51,10 +52,10 @@ type ( CreateEvent EventHandlerFunc // FormatMessage is a method to convert templates into strings using internally known variables / configs FormatMessage MsgFormatter + // GetDatabaseConnector returns an active database.Connector to access the backend storage database + GetDatabaseConnector func() database.Connector // GetLogger returns a sirupsen log.Entry pre-configured with the module name GetLogger LoggerCreationFunc - // GetStorageManager returns an interface to access the modules storage - GetStorageManager func() StorageManager // GetTwitchClient retrieves a fully configured Twitch client with initialized cache GetTwitchClient func() *twitch.Client // GetTwitchClientForChannel retrieves a fully configured Twitch client with initialized cache for extended permission channels diff --git a/plugins/rule.go b/plugins/rule.go index e3c3264..ca87fab 100644 --- a/plugins/rule.go +++ b/plugins/rule.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/Luzifer/go_helpers/v2/str" - "github.com/Luzifer/twitch-bot/twitch" + "github.com/Luzifer/twitch-bot/pkg/twitch" ) type ( @@ -164,7 +164,13 @@ func (r *Rule) allowExecuteChannelCooldown(logger *log.Entry, m *irc.Message, ev return true } - if !r.timerStore.InCooldown(TimerTypeCooldown, DeriveChannel(m, evtData), r.MatcherID()) { + inCooldown, err := r.timerStore.InCooldown(TimerTypeCooldown, DeriveChannel(m, evtData), r.MatcherID()) + if err != nil { + logger.WithError(err).Error("checking channel cooldown") + return false + } + + if !inCooldown { return true } @@ -225,7 +231,13 @@ func (r *Rule) allowExecuteDisableOnOffline(logger *log.Entry, m *irc.Message, e } func (r *Rule) allowExecuteDisableOnPermit(logger *log.Entry, m *irc.Message, event *string, badges twitch.BadgeCollection, evtData *FieldCollection) bool { - if r.DisableOnPermit != nil && *r.DisableOnPermit && DeriveChannel(m, evtData) != "" && r.timerStore.HasPermit(DeriveChannel(m, evtData), DeriveUser(m, evtData)) { + hasPermit, err := r.timerStore.HasPermit(DeriveChannel(m, evtData), DeriveUser(m, evtData)) + if err != nil { + logger.WithError(err).Error("checking permit") + return false + } + + if r.DisableOnPermit != nil && *r.DisableOnPermit && DeriveChannel(m, evtData) != "" && hasPermit { logger.Trace("Non-Match: Permit") return false } @@ -328,7 +340,13 @@ func (r *Rule) allowExecuteRuleCooldown(logger *log.Entry, m *irc.Message, event return true } - if !r.timerStore.InCooldown(TimerTypeCooldown, "", r.MatcherID()) { + inCooldown, err := r.timerStore.InCooldown(TimerTypeCooldown, "", r.MatcherID()) + if err != nil { + logger.WithError(err).Error("checking rule cooldown") + return false + } + + if !inCooldown { return true } @@ -347,7 +365,13 @@ func (r *Rule) allowExecuteUserCooldown(logger *log.Entry, m *irc.Message, event return true } - if DeriveUser(m, evtData) == "" || !r.timerStore.InCooldown(TimerTypeCooldown, DeriveUser(m, evtData), r.MatcherID()) { + inCooldown, err := r.timerStore.InCooldown(TimerTypeCooldown, DeriveUser(m, evtData), r.MatcherID()) + if err != nil { + logger.WithError(err).Error("checking user cooldown") + return false + } + + if DeriveUser(m, evtData) == "" || !inCooldown { return true } diff --git a/plugins/rule_test.go b/plugins/rule_test.go index 19e7bef..2c69869 100644 --- a/plugins/rule_test.go +++ b/plugins/rule_test.go @@ -8,7 +8,7 @@ import ( "github.com/go-irc/irc" "github.com/sirupsen/logrus" - "github.com/Luzifer/twitch-bot/twitch" + "github.com/Luzifer/twitch-bot/pkg/twitch" ) var ( diff --git a/plugins/timerstore.go b/plugins/timerstore.go index 921ac21..71ce94d 100644 --- a/plugins/timerstore.go +++ b/plugins/timerstore.go @@ -21,10 +21,10 @@ type ( } TimerStore interface { - AddCooldown(tt TimerType, limiter, ruleID string, expiry time.Time) - InCooldown(tt TimerType, limiter, ruleID string) bool - AddPermit(channel, username string) - HasPermit(channel, username string) bool + AddCooldown(tt TimerType, limiter, ruleID string, expiry time.Time) error + InCooldown(tt TimerType, limiter, ruleID string) (bool, error) + AddPermit(channel, username string) error + HasPermit(channel, username string) (bool, error) } testTimerStore struct { @@ -32,16 +32,19 @@ type ( } ) +var _ TimerStore = (*testTimerStore)(nil) + func newTestTimerStore() *testTimerStore { return &testTimerStore{timers: map[string]time.Time{}} } // Cooldown timer -func (t *testTimerStore) AddCooldown(tt TimerType, limiter, ruleID string, expiry time.Time) { +func (t *testTimerStore) AddCooldown(tt TimerType, limiter, ruleID string, expiry time.Time) error { t.timers[t.getCooldownTimerKey(tt, limiter, ruleID)] = expiry + return nil } -func (t *testTimerStore) InCooldown(tt TimerType, limiter, ruleID string) bool { - return t.timers[t.getCooldownTimerKey(tt, limiter, ruleID)].After(time.Now()) +func (t *testTimerStore) InCooldown(tt TimerType, limiter, ruleID string) (bool, error) { + return t.timers[t.getCooldownTimerKey(tt, limiter, ruleID)].After(time.Now()), nil } func (testTimerStore) getCooldownTimerKey(tt TimerType, limiter, ruleID string) string { @@ -52,12 +55,13 @@ func (testTimerStore) getCooldownTimerKey(tt TimerType, limiter, ruleID string) // Permit timer -func (t *testTimerStore) AddPermit(channel, username string) { +func (t *testTimerStore) AddPermit(channel, username string) error { t.timers[t.getPermitTimerKey(channel, username)] = time.Now().Add(time.Minute) + return nil } -func (t *testTimerStore) HasPermit(channel, username string) bool { - return t.timers[t.getPermitTimerKey(channel, username)].After(time.Now()) +func (t *testTimerStore) HasPermit(channel, username string) (bool, error) { + return t.timers[t.getPermitTimerKey(channel, username)].After(time.Now()), nil } func (testTimerStore) getPermitTimerKey(channel, username string) string { diff --git a/plugins_core.go b/plugins_core.go index 7b9157e..f9bee2b 100644 --- a/plugins_core.go +++ b/plugins_core.go @@ -11,6 +11,7 @@ import ( "github.com/Luzifer/go_helpers/v2/backoff" "github.com/Luzifer/go_helpers/v2/str" "github.com/Luzifer/twitch-bot/internal/actors/ban" + "github.com/Luzifer/twitch-bot/internal/actors/counter" "github.com/Luzifer/twitch-bot/internal/actors/delay" deleteactor "github.com/Luzifer/twitch-bot/internal/actors/delete" "github.com/Luzifer/twitch-bot/internal/actors/filesay" @@ -21,15 +22,18 @@ import ( "github.com/Luzifer/twitch-bot/internal/actors/raw" "github.com/Luzifer/twitch-bot/internal/actors/respond" "github.com/Luzifer/twitch-bot/internal/actors/timeout" + "github.com/Luzifer/twitch-bot/internal/actors/variables" "github.com/Luzifer/twitch-bot/internal/actors/whisper" "github.com/Luzifer/twitch-bot/internal/apimodules/customevent" "github.com/Luzifer/twitch-bot/internal/apimodules/msgformat" "github.com/Luzifer/twitch-bot/internal/apimodules/overlays" + "github.com/Luzifer/twitch-bot/internal/service/access" "github.com/Luzifer/twitch-bot/internal/template/numeric" "github.com/Luzifer/twitch-bot/internal/template/random" "github.com/Luzifer/twitch-bot/internal/template/slice" + "github.com/Luzifer/twitch-bot/pkg/database" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) const ircHandleWaitRetries = 10 @@ -38,6 +42,7 @@ var ( corePluginRegistrations = []plugins.RegisterFunc{ // Actors ban.Register, + counter.Register, delay.Register, deleteactor.Register, filesay.Register, @@ -48,6 +53,7 @@ var ( raw.Register, respond.Register, timeout.Register, + variables.Register, whisper.Register, // Template functions @@ -113,10 +119,9 @@ func getRegistrationArguments() plugins.RegistrationArguments { return nil }, FormatMessage: formatMessage, + GetDatabaseConnector: func() database.Connector { return db }, GetLogger: func(moduleName string) *log.Entry { return log.WithField("module", moduleName) }, - GetStorageManager: func() plugins.StorageManager { return store }, GetTwitchClient: func() *twitch.Client { return twitchClient }, - GetTwitchClientForChannel: store.GetTwitchClientForChannel, RegisterActor: registerAction, RegisterActorDocumentation: registerActorDocumentation, RegisterAPIRoute: registerRoute, @@ -126,6 +131,13 @@ func getRegistrationArguments() plugins.RegistrationArguments { RegisterTemplateFunction: tplFuncs.Register, SendMessage: sendMessage, ValidateToken: validateAuthToken, + + GetTwitchClientForChannel: func(channel string) (*twitch.Client, error) { + return accessService.GetTwitchClientForChannel(channel, access.ClientConfig{ + TwitchClient: cfg.TwitchClient, + TwitchClientSecret: cfg.TwitchClientSecret, + }) + }, } } diff --git a/scopes.go b/scopes.go index 9515d06..6d12430 100644 --- a/scopes.go +++ b/scopes.go @@ -1,6 +1,6 @@ package main -import "github.com/Luzifer/twitch-bot/twitch" +import "github.com/Luzifer/twitch-bot/pkg/twitch" var ( channelDefaultScopes = []string{ diff --git a/store.go b/store.go deleted file mode 100644 index a3f0609..0000000 --- a/store.go +++ /dev/null @@ -1,405 +0,0 @@ -package main - -import ( - "compress/gzip" - "crypto/rand" - "encoding/hex" - "encoding/json" - "os" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/Luzifer/go_helpers/v2/str" - "github.com/Luzifer/twitch-bot/crypt" - "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" -) - -const eventSubSecretLength = 32 - -var errExtendedPermissionsMissing = errors.New("no extended permissions greanted") - -type ( - storageExtendedPermission struct { - AccessToken string `encrypt:"true" json:"access_token,omitempty"` - RefreshToken string `encrypt:"true" json:"refresh_token,omitempty"` - Scopes []string `json:"scopes,omitempty"` - } - - storageFile struct { - Counters map[string]int64 `json:"counters"` - Timers map[string]plugins.TimerEntry `json:"timers"` - Variables map[string]string `json:"variables"` - - ModuleStorage map[string]json.RawMessage `json:"module_storage"` - - GrantedScopes map[string][]string `json:"granted_scopes,omitempty"` // Deprecated, Read-Only - ExtendedPermissions map[string]*storageExtendedPermission `json:"extended_permissions"` - - EventSubSecret string `encrypt:"true" json:"event_sub_secret,omitempty"` - - BotAccessToken string `encrypt:"true" json:"bot_access_token,omitempty"` - BotRefreshToken string `encrypt:"true" json:"bot_refresh_token,omitempty"` - - inMem bool - lock *sync.RWMutex - } -) - -func newStorageFile(inMemStore bool) *storageFile { - return &storageFile{ - Counters: map[string]int64{}, - Timers: map[string]plugins.TimerEntry{}, - Variables: map[string]string{}, - - ModuleStorage: map[string]json.RawMessage{}, - - GrantedScopes: map[string][]string{}, - ExtendedPermissions: map[string]*storageExtendedPermission{}, - - inMem: inMemStore, - lock: new(sync.RWMutex), - } -} - -func (s *storageFile) DeleteExtendedPermissions(user string) error { - s.lock.Lock() - defer s.lock.Unlock() - - delete(s.ExtendedPermissions, user) - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) DeleteModuleStore(moduleUUID string) error { - s.lock.Lock() - defer s.lock.Unlock() - - delete(s.ModuleStorage, moduleUUID) - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) GetBotToken(fallback string) string { - s.lock.RLock() - defer s.lock.RUnlock() - - if v := s.BotAccessToken; v != "" { - return v - } - return fallback -} - -func (s *storageFile) GetCounterValue(counter string) int64 { - s.lock.RLock() - defer s.lock.RUnlock() - - return s.Counters[counter] -} - -func (s *storageFile) GetOrGenerateEventSubSecret() (string, string, error) { - s.lock.Lock() - defer s.lock.Unlock() - - if s.EventSubSecret != "" { - return s.EventSubSecret, s.EventSubSecret[:5], nil - } - - key := make([]byte, eventSubSecretLength) - n, err := rand.Read(key) - if err != nil { - return "", "", errors.Wrap(err, "generating random secret") - } - if n != eventSubSecretLength { - return "", "", errors.Errorf("read only %d of %d byte", n, eventSubSecretLength) - } - - s.EventSubSecret = hex.EncodeToString(key) - - return s.EventSubSecret, s.EventSubSecret[:5], errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) GetModuleStore(moduleUUID string, storedObject plugins.StorageUnmarshaller) error { - s.lock.RLock() - defer s.lock.RUnlock() - - return errors.Wrap( - storedObject.UnmarshalStoredObject(s.ModuleStorage[moduleUUID]), - "unmarshalling stored object", - ) -} - -func (s *storageFile) GetTwitchClientForChannel(channel string) (*twitch.Client, error) { - s.lock.RLock() - defer s.lock.RUnlock() - - perms := s.ExtendedPermissions[channel] - if perms == nil { - return nil, errExtendedPermissionsMissing - } - - tc := twitch.New(cfg.TwitchClient, cfg.TwitchClientSecret, perms.AccessToken, perms.RefreshToken) - tc.SetTokenUpdateHook(func(at, rt string) error { - return errors.Wrap(s.SetExtendedPermissions(channel, storageExtendedPermission{ - AccessToken: at, - RefreshToken: rt, - }, true), "updating extended permissions token") - }) - - return tc, nil -} - -func (s *storageFile) GetVariable(key string) string { - s.lock.RLock() - defer s.lock.RUnlock() - - return s.Variables[key] -} - -func (s *storageFile) HasTimer(id string) bool { - s.lock.RLock() - defer s.lock.RUnlock() - - return s.Timers[id].Time.After(time.Now()) -} - -func (s *storageFile) Load() error { - s.lock.Lock() - defer s.lock.Unlock() - - if s.inMem { - // In-Memory store is active, do not load from disk - // for testing purposes only! - return nil - } - - f, err := os.Open(cfg.StorageFile) - if err != nil { - if os.IsNotExist(err) { - // Store init state - return nil - } - return errors.Wrap(err, "open storage file") - } - defer f.Close() - - zf, err := gzip.NewReader(f) - if err != nil { - return errors.Wrap(err, "create gzip reader") - } - defer zf.Close() - - if err = json.NewDecoder(zf).Decode(s); err != nil { - return errors.Wrap(err, "decode storage object") - } - - if err = crypt.DecryptFields(s, cfg.StorageEncryptionPass); err != nil { - return errors.Wrap(err, "decrypting storage object") - } - - return errors.Wrap(s.migrate(), "migrating storage") -} - -func (s *storageFile) Save() error { - // NOTE(kahlers): DO NOT LOCK THIS, all calling functions are - // modifying functions and must have locks in place - - if s.inMem { - // In-Memory store is active, do not store to disk - // for testing purposes only! - return nil - } - - // Cleanup timers - var timerIDs []string - for id := range s.Timers { - timerIDs = append(timerIDs, id) - } - - for _, i := range timerIDs { - if s.Timers[i].Time.Before(time.Now()) { - delete(s.Timers, i) - } - } - - // Encrypt fields in memory before writing - if err := crypt.EncryptFields(s, cfg.StorageEncryptionPass); err != nil { - return errors.Wrap(err, "encrypting storage object") - } - - // Write store to disk - f, err := os.Create(cfg.StorageFile) - if err != nil { - return errors.Wrap(err, "create storage file") - } - defer f.Close() - - zf := gzip.NewWriter(f) - defer zf.Close() - - if err = json.NewEncoder(zf).Encode(s); err != nil { - return errors.Wrap(err, "encode storage object") - } - - // Decrypt the values to make them accessible again - if err = crypt.DecryptFields(s, cfg.StorageEncryptionPass); err != nil { - return errors.Wrap(err, "decrypting storage object") - } - - return nil -} - -func (s *storageFile) SetExtendedPermissions(user string, data storageExtendedPermission, merge bool) error { - s.lock.Lock() - defer s.lock.Unlock() - - prev := s.ExtendedPermissions[user] - if merge && prev != nil { - for _, sc := range prev.Scopes { - if !str.StringInSlice(sc, data.Scopes) { - data.Scopes = append(data.Scopes, sc) - } - } - - if data.AccessToken == "" && prev.AccessToken != "" { - data.AccessToken = prev.AccessToken - } - - if data.RefreshToken == "" && prev.RefreshToken != "" { - data.RefreshToken = prev.RefreshToken - } - } - - s.ExtendedPermissions[user] = &data - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) SetModuleStore(moduleUUID string, storedObject plugins.StorageMarshaller) error { - s.lock.Lock() - defer s.lock.Unlock() - - data, err := storedObject.MarshalStoredObject() - if err != nil { - return errors.Wrap(err, "marshalling stored object") - } - - s.ModuleStorage[moduleUUID] = data - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) SetTimer(kind plugins.TimerType, id string, expiry time.Time) error { - s.lock.Lock() - defer s.lock.Unlock() - - s.Timers[id] = plugins.TimerEntry{Kind: kind, Time: expiry} - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) SetVariable(key, value string) error { - s.lock.Lock() - defer s.lock.Unlock() - - s.Variables[key] = value - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) RemoveVariable(key string) error { - s.lock.Lock() - defer s.lock.Unlock() - - delete(s.Variables, key) - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) UpdateBotToken(accessToken, refreshToken string) error { - s.lock.Lock() - defer s.lock.Unlock() - - s.BotAccessToken = accessToken - s.BotRefreshToken = refreshToken - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) UpdateCounter(counter string, value int64, absolute bool) error { - s.lock.Lock() - defer s.lock.Unlock() - - if !absolute { - value = s.Counters[counter] + value - } - - if value == 0 { - delete(s.Counters, counter) - } else { - s.Counters[counter] = value - } - - return errors.Wrap(s.Save(), "saving store") -} - -func (s *storageFile) UserHasExtendedAuth(user string) bool { - s.lock.RLock() - defer s.lock.RUnlock() - - ep := s.ExtendedPermissions[user] - return ep != nil && ep.AccessToken != "" && ep.RefreshToken != "" -} - -func (s *storageFile) UserHasGrantedAnyScope(user string, scopes ...string) bool { - s.lock.RLock() - defer s.lock.RUnlock() - - if s.ExtendedPermissions[user] == nil { - return false - } - - grantedScopes := s.ExtendedPermissions[user].Scopes - for _, scope := range scopes { - if str.StringInSlice(scope, grantedScopes) { - return true - } - } - - return false -} - -func (s *storageFile) UserHasGrantedScopes(user string, scopes ...string) bool { - s.lock.RLock() - defer s.lock.RUnlock() - - if s.ExtendedPermissions[user] == nil { - return false - } - - grantedScopes := s.ExtendedPermissions[user].Scopes - for _, scope := range scopes { - if !str.StringInSlice(scope, grantedScopes) { - return false - } - } - - return true -} - -func (s *storageFile) migrate() error { - // Do NOT lock, use during locked call - - // Migration: Transform GrantedScopes and delete - for ch, scopes := range s.GrantedScopes { - if s.ExtendedPermissions[ch] != nil { - continue - } - s.ExtendedPermissions[ch] = &storageExtendedPermission{Scopes: scopes} - } - s.GrantedScopes = nil - - return nil -} diff --git a/timers.go b/timers.go deleted file mode 100644 index 5e917c8..0000000 --- a/timers.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "crypto/sha256" - "fmt" - "strings" - "time" - - "github.com/Luzifer/twitch-bot/plugins" -) - -var timerStore plugins.TimerStore = newTimer() - -type timer struct{} - -func newTimer() *timer { - return &timer{} -} - -// Cooldown timer - -func (t *timer) AddCooldown(tt plugins.TimerType, limiter, ruleID string, expiry time.Time) { - store.SetTimer(plugins.TimerTypeCooldown, t.getCooldownTimerKey(tt, limiter, ruleID), expiry) -} - -func (t *timer) InCooldown(tt plugins.TimerType, limiter, ruleID string) bool { - return store.HasTimer(t.getCooldownTimerKey(tt, limiter, ruleID)) -} - -func (timer) getCooldownTimerKey(tt plugins.TimerType, limiter, ruleID string) string { - h := sha256.New() - fmt.Fprintf(h, "%d:%s:%s", tt, limiter, ruleID) - return fmt.Sprintf("sha256:%x", h.Sum(nil)) -} - -// Permit timer - -func (t *timer) AddPermit(channel, username string) { - store.SetTimer(plugins.TimerTypePermit, t.getPermitTimerKey(channel, username), time.Now().Add(config.PermitTimeout)) -} - -func (t *timer) HasPermit(channel, username string) bool { - return store.HasTimer(t.getPermitTimerKey(channel, username)) -} - -func (timer) getPermitTimerKey(channel, username string) string { - h := sha256.New() - fmt.Fprintf(h, "%d:%s:%s", plugins.TimerTypePermit, channel, strings.ToLower(strings.TrimLeft(username, "@"))) - return fmt.Sprintf("sha256:%x", h.Sum(nil)) -} diff --git a/twitchWatcher.go b/twitchWatcher.go index 05f6e0a..890768d 100644 --- a/twitchWatcher.go +++ b/twitchWatcher.go @@ -7,8 +7,8 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/Luzifer/twitch-bot/pkg/twitch" "github.com/Luzifer/twitch-bot/plugins" - "github.com/Luzifer/twitch-bot/twitch" ) type ( @@ -171,7 +171,7 @@ func (t *twitchWatcher) handleEventUserAuthRevoke(m json.RawMessage) error { } return errors.Wrap( - store.DeleteExtendedPermissions(payload.UserLogin), + accessService.RemoveExendedTwitchCredentials(payload.UserLogin), "deleting granted scopes", ) } @@ -284,12 +284,17 @@ func (t *twitchWatcher) registerEventSubCallbacks(channel string) (func(), error }) if len(tr.RequiredScopes) > 0 { - fn := store.UserHasGrantedScopes + fn := accessService.HasPermissionsForChannel if tr.AnyScope { - fn = store.UserHasGrantedAnyScope + fn = accessService.HasAnyPermissionForChannel } - if !fn(channel, tr.RequiredScopes...) { + hasScopes, err := fn(channel, tr.RequiredScopes...) + if err != nil { + return nil, errors.Wrap(err, "checking granted scopes") + } + + if !hasScopes { logger.Debug("Missing scopes for eventsub topic") continue }