From 29df9e59b575d2bcbe18b2637d1ff12a1069fc8e Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sat, 26 Aug 2023 00:20:10 +0200 Subject: [PATCH] [core] Remove v2 migration Signed-off-by: Knut Ahlers --- README.md | 53 +-------- cli_migrateV2.go | 33 ------ docs/content/configuration/templating.md | 11 +- internal/actors/punish/actor.go | 1 - internal/actors/quotedb/actor.go | 3 +- internal/service/access/access.go | 14 --- internal/v2migrator/core.go | 84 -------------- internal/v2migrator/crypt/crypt.go | 139 ----------------------- internal/v2migrator/modOverlays.go | 26 ----- internal/v2migrator/modQuoteDB.go | 24 ---- internal/v2migrator/store.go | 117 ------------------- tplDocs.tpl | 9 -- 12 files changed, 3 insertions(+), 511 deletions(-) delete mode 100644 cli_migrateV2.go delete mode 100644 internal/v2migrator/core.go delete mode 100644 internal/v2migrator/crypt/crypt.go delete mode 100644 internal/v2migrator/modOverlays.go delete mode 100644 internal/v2migrator/modQuoteDB.go delete mode 100644 internal/v2migrator/store.go diff --git a/README.md b/README.md index b19c7ce..2a07cf3 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,8 @@ Usage of twitch-bot: Supported sub-commands are: actor-docs Generate markdown documentation for available actors api-token [...scope] Generate an api-token to be entered into the config - migrate-v2 Migrate old (*.json.gz) storage file into new database reset-secrets Remove encrypted data to reset encryption passphrase + tpl-docs Generate markdown documentation for available template functions validate-config Try to load configuration file and report errors if any ``` @@ -93,54 +93,3 @@ Just pass the filename you want to use. --storage-conn-string 'storage.db' \ ... ``` - - -## Upgrade from `v2.x` to `v3.x` - -With the release of `v3.0.0` the bot changed a lot introducing a new storage format. As that storage backend is not compatible with the `v2.x` storage you need to migrate it manually before starting a `v3.x` bot version the first time. - -**Before starting the migration make sure to fully stop the bot!** - -This section assumes you were starting your `v2.x` bot the following way: - -```console -# twitch-bot \ - --storage-file storage.json.gz - --twitch-client \ - --twitch-client-secret -``` - -To execute the migration we need to provide the same `storage-encryption-pass` or `twitch-client` / `twitch-client-secret` combination if no `storage-encryption-pass` was used. - -```console -# twitch-bot \ - --storage-conn-type \ - --storage-conn-string \ - --twitch-client \ - --twitch-client-secret \ - migrate-v2 storage.json.gz -WARN[0000] No storage encryption passphrase was set, falling back to client-id:client-secret -WARN[0000] Module registered unhandled query-param type module=status type=integer -WARN[0000] Overlays dir not specified, no dir or non existent dir= -INFO[0000] Starting migration... module=variables -INFO[0000] Starting migration... module=mod_punish -INFO[0000] Starting migration... module=mod_overlays -INFO[0000] Starting migration... module=mod_quotedb -INFO[0000] Starting migration... module=core -INFO[0000] Starting migration... module=counter -INFO[0000] Starting migration... module=permissions -INFO[0000] Starting migration... module=timers -INFO[0000] v2 storage file was migrated -``` - -If you see the `v2 storage file was migrated` message the contents of your old storage file were migrated to the new database. The old file is not modified in this step. - -Afterwards your need to adjust the start parameters of the bot: - -```console -# twitch-bot \ - --storage-conn-type \ - --storage-conn-string \ - --twitch-client \ - --twitch-client-secret \ -``` diff --git a/cli_migrateV2.go b/cli_migrateV2.go deleted file mode 100644 index c5b3e37..0000000 --- a/cli_migrateV2.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - - "github.com/Luzifer/twitch-bot/v3/internal/v2migrator" -) - -func init() { - cli.Add(cliRegistryEntry{ - Name: "migrate-v2", - Description: "Migrate old (*.json.gz) storage file into new database", - Params: []string{""}, - Run: func(args []string) error { - if len(args) < 2 { //nolint:gomnd // Just a count of parameters - return errors.New("Usage: twitch-bot migrate-v2 ") - } - - v2s := v2migrator.NewStorageFile() - if err := v2s.Load(args[1], cfg.StorageEncryptionPass); err != nil { - return errors.Wrap(err, "loading v2 storage file") - } - - if err := v2s.Migrate(db); err != nil { - return errors.Wrap(err, "migrating v2 storage file") - } - - log.Info("v2 storage file was migrated") - return nil - }, - }) -} diff --git a/docs/content/configuration/templating.md b/docs/content/configuration/templating.md index 3a8331a..442808f 100644 --- a/docs/content/configuration/templating.md +++ b/docs/content/configuration/templating.md @@ -376,7 +376,7 @@ Example: ``` # Your int this hour: {{ printf "%.0f" (mulf (seededRandom (list "int" .username (now | date "2006-01-02 15") | join ":")) 100) }}% -< Your int this hour: 84% +< Your int this hour: 9% ``` ### `streamUptime` @@ -458,12 +458,3 @@ Example: # {{ variable "foo" "fallback" }} - {{ variable "unsetvar" "fallback" }} * test - fallback ``` - -## Upgrade from `v2.x` to `v3.x` - -When adding [sprig](https://masterminds.github.io/sprig/) function collection some functions collided and needed replacement. You need to adapt your templates accordingly: - -- Math functions (`add`, `div`, `mod`, `mul`, `multiply`, `sub`) were replaced with their sprig-equivalent and are now working with integers instead of floats. If you need them to continue to work with floats you need to use their [float-variants](https://masterminds.github.io/sprig/mathf.html). -- `now` does no longer format the current date as a string but return the current date. You need to replace this: `now "2006-01-02"` becomes `now | date "2006-01-02"`. -- `concat` is now used to concat arrays. To join strings you will need to modify your code: `concat ":" "string1" "string2"` becomes `lists "string1" "string2" | join ":"`. -- `toLower` / `toUpper` need to be replaced with their sprig equivalent `lower` and `upper`. diff --git a/internal/actors/punish/actor.go b/internal/actors/punish/actor.go index 51377f5..0ddf720 100644 --- a/internal/actors/punish/actor.go +++ b/internal/actors/punish/actor.go @@ -16,7 +16,6 @@ import ( const ( actorNamePunish = "punish" actorNameResetPunish = "reset-punish" - moduleUUID = "44ab4646-ce50-4e16-9353-c1f0eb68962b" oneWeek = 168 * time.Hour ) diff --git a/internal/actors/quotedb/actor.go b/internal/actors/quotedb/actor.go index 70bc15a..bd98ed9 100644 --- a/internal/actors/quotedb/actor.go +++ b/internal/actors/quotedb/actor.go @@ -11,8 +11,7 @@ import ( ) const ( - actorName = "quotedb" - moduleUUID = "917c83ee-ed40-41e4-a558-1c2e59fdf1f5" + actorName = "quotedb" ) var ( diff --git a/internal/service/access/access.go b/internal/service/access/access.go index b6d92db..56bca78 100644 --- a/internal/service/access/access.go +++ b/internal/service/access/access.go @@ -228,20 +228,6 @@ func (s Service) RemoveExendedTwitchCredentials(channel string) error { ) } -// Deprecated: Use SetBotUsername and SetExtendedTwitchCredentials -// instead. This function is only required for the v2 migration tool. -func (s Service) SetBotTwitchCredentials(accessToken, refreshToken string) (err error) { - if err = s.db.StoreEncryptedCoreMeta(coreMetaKeyBotToken, accessToken); err != nil { - return errors.Wrap(err, "storing bot access token") - } - - if err = s.db.StoreEncryptedCoreMeta(coreMetaKeyBotRefreshToken, refreshToken); err != nil { - return errors.Wrap(err, "storing bot refresh token") - } - - return nil -} - func (s Service) SetBotUsername(channel string) (err error) { return errors.Wrap( s.db.StoreCoreMeta(coreMetaKeyBotUsername, strings.TrimLeft(channel, "#")), diff --git a/internal/v2migrator/core.go b/internal/v2migrator/core.go deleted file mode 100644 index 66378ea..0000000 --- a/internal/v2migrator/core.go +++ /dev/null @@ -1,84 +0,0 @@ -package v2migrator - -import ( - "github.com/pkg/errors" - - "github.com/Luzifer/twitch-bot/v3/internal/actors/counter" - "github.com/Luzifer/twitch-bot/v3/internal/actors/variables" - "github.com/Luzifer/twitch-bot/v3/internal/service/access" - "github.com/Luzifer/twitch-bot/v3/internal/service/timer" - "github.com/Luzifer/twitch-bot/v3/pkg/database" -) - -func (s storageFile) migrateCoreKV(db database.Connector) (err error) { - as, err := access.New(db) - if err != nil { - return errors.Wrap(err, "creating access service") - } - - //nolint:staticcheck // Use of deprecated function is fine for this purpose - if err = as.SetBotTwitchCredentials(s.BotAccessToken, s.BotRefreshToken); err != nil { - return errors.Wrap(err, "setting bot credentials") - } - - if err = db.StoreEncryptedCoreMeta("event_sub_secret", s.EventSubSecret); err != nil { - return errors.Wrap(err, "storing bot eventsub token") - } - - return nil -} - -func (s storageFile) migrateCounters(db database.Connector) (err error) { - for counterName, value := range s.Counters { - if err = counter.UpdateCounter(db, counterName, value, true); err != nil { - return errors.Wrap(err, "storing counter value") - } - } - - return nil -} - -func (s storageFile) migratePermissions(db database.Connector) (err error) { - as, err := access.New(db) - if err != nil { - return errors.Wrap(err, "creating access service") - } - - for channel, perms := range s.ExtendedPermissions { - if err = as.SetExtendedTwitchCredentials( - channel, - perms.AccessToken, - perms.RefreshToken, - perms.Scopes, - ); err != nil { - return errors.Wrapf(err, "storing channel %q credentials", channel) - } - } - - return nil -} - -func (s storageFile) migrateTimers(db database.Connector) (err error) { - ts, err := timer.New(db, nil) - if err != nil { - return errors.Wrap(err, "creating timer service") - } - - for id, expiry := range s.Timers { - if err := ts.SetTimer(id, expiry.Time); err != nil { - return errors.Wrap(err, "storing counter in database") - } - } - - return nil -} - -func (s storageFile) migrateVariables(db database.Connector) (err error) { - for key, value := range s.Variables { - if err := variables.SetVariable(db, key, value); err != nil { - return errors.Wrap(err, "updating value in database") - } - } - - return nil -} diff --git a/internal/v2migrator/crypt/crypt.go b/internal/v2migrator/crypt/crypt.go deleted file mode 100644 index f102afa..0000000 --- a/internal/v2migrator/crypt/crypt.go +++ /dev/null @@ -1,139 +0,0 @@ -package crypt - -import ( - "reflect" - "strings" - "time" - - "github.com/pkg/errors" - - "github.com/Luzifer/go-openssl/v4" -) - -const encryptedValuePrefix = "enc:" - -type encryptAction uint8 - -const ( - handleTagsDecrypt encryptAction = iota - handleTagsEncrypt -) - -var osslClient = openssl.New() - -// DecryptFields iterates through the given struct and decrypts all -// fields marked with a struct tag of `encrypt:"true"`. The fields -// are directly manipulated and the value is replaced. -// -// The input object needs to be a pointer to a struct! -func DecryptFields(obj interface{}, passphrase string) error { - return handleEncryptedTags(obj, passphrase, handleTagsDecrypt) -} - -// EncryptFields iterates through the given struct and encrypts all -// fields marked with a struct tag of `encrypt:"true"`. The fields -// are directly manipulated and the value is replaced. -// -// The input object needs to be a pointer to a struct! -func EncryptFields(obj interface{}, passphrase string) error { - return handleEncryptedTags(obj, passphrase, handleTagsEncrypt) -} - -//nolint:gocognit,gocyclo // Reflect loop, cannot reduce complexity -func handleEncryptedTags(obj interface{}, passphrase string, action encryptAction) error { - // Check we got a pointer and can manipulate the struct - if kind := reflect.TypeOf(obj).Kind(); kind != reflect.Ptr { - return errors.Errorf("expected pointer to struct, got %s", kind) - } - - // Check we got a struct in the pointer - if kind := reflect.ValueOf(obj).Elem().Kind(); kind != reflect.Struct { - return errors.Errorf("expected pointer to struct, got pointer to %s", kind) - } - - // Iterate over fields to find encrypted fields to manipulate - st := reflect.ValueOf(obj).Elem() - for i := 0; i < st.NumField(); i++ { - v := st.Field(i) - t := st.Type().Field(i) - - if t.PkgPath != "" && !t.Anonymous { - // Caught us an non-exported field, ignore that one - continue - } - - hasEncryption := t.Tag.Get("encrypt") == "true" - - switch t.Type.Kind() { - // Type: Map - see whether value is struct - case reflect.Map: - if t.Type.Elem().Kind() == reflect.Ptr && t.Type.Elem().Elem().Kind() == reflect.Struct { - for _, k := range v.MapKeys() { - if err := handleEncryptedTags(v.MapIndex(k).Interface(), passphrase, action); err != nil { - return err - } - } - } - - // Type: Pointer - Recurse if not nil and struct inside - case reflect.Ptr: - if !v.IsNil() && v.Elem().Kind() == reflect.Struct && t.Type != reflect.TypeOf(&time.Time{}) { - if err := handleEncryptedTags(v.Interface(), passphrase, action); err != nil { - return err - } - } - - // Type: String - Replace value if required - case reflect.String: - if hasEncryption { - newValue, err := manipulateValue(v.String(), passphrase, action) - if err != nil { - return errors.Wrap(err, "manipulating value") - } - v.SetString(newValue) - } - - // Type: Struct - Welcome to recursion - case reflect.Struct: - if t.Type != reflect.TypeOf(time.Time{}) { - if err := handleEncryptedTags(v.Addr().Interface(), passphrase, action); err != nil { - return err - } - } - - // We don't support anything else. Yet. - default: - if hasEncryption { - return errors.Errorf("unsupported field type for encyption: %s", t.Type.Kind()) - } - } - } - - return nil -} - -func manipulateValue(val, passphrase string, action encryptAction) (string, error) { - switch action { - case handleTagsDecrypt: - if !strings.HasPrefix(val, encryptedValuePrefix) { - // This is not an encrypted string: Return the value itself for - // working with legacy values in storage - return val, nil - } - - d, err := osslClient.DecryptBytes(passphrase, []byte(strings.TrimPrefix(val, encryptedValuePrefix)), openssl.PBKDF2SHA256) - return string(d), errors.Wrap(err, "decrypting value") - - case handleTagsEncrypt: - if strings.HasPrefix(val, encryptedValuePrefix) { - // This is an encrypted string: shouldn't happen but whatever - return val, nil - } - - e, err := osslClient.EncryptBytes(passphrase, []byte(val), openssl.PBKDF2SHA256) - return encryptedValuePrefix + string(e), errors.Wrap(err, "encrypting value") - - default: - return "", errors.New("invalid action") - } -} diff --git a/internal/v2migrator/modOverlays.go b/internal/v2migrator/modOverlays.go deleted file mode 100644 index 016f13d..0000000 --- a/internal/v2migrator/modOverlays.go +++ /dev/null @@ -1,26 +0,0 @@ -package v2migrator - -import ( - "github.com/pkg/errors" - - "github.com/Luzifer/twitch-bot/v3/internal/apimodules/overlays" - "github.com/Luzifer/twitch-bot/v3/pkg/database" -) - -type ( - storageModOverlays struct { - ChannelEvents map[string][]overlays.SocketMessage `json:"channel_events"` - } -) - -func (s storageModOverlays) migrate(db database.Connector) (err error) { - for channel, evts := range s.ChannelEvents { - for _, evt := range evts { - if err := overlays.AddChannelEvent(db, channel, evt); err != nil { - return errors.Wrap(err, "storing event to database") - } - } - } - - return nil -} diff --git a/internal/v2migrator/modQuoteDB.go b/internal/v2migrator/modQuoteDB.go deleted file mode 100644 index 7377a60..0000000 --- a/internal/v2migrator/modQuoteDB.go +++ /dev/null @@ -1,24 +0,0 @@ -package v2migrator - -import ( - "github.com/pkg/errors" - - "github.com/Luzifer/twitch-bot/v3/internal/actors/quotedb" - "github.com/Luzifer/twitch-bot/v3/pkg/database" -) - -type ( - storageModQuoteDB struct { - ChannelQuotes map[string][]string `json:"channel_quotes"` - } -) - -func (s storageModQuoteDB) migrate(db database.Connector) (err error) { - for channel, quotes := range s.ChannelQuotes { - if err := quotedb.SetQuotes(db, channel, quotes); err != nil { - return errors.Wrap(err, "setting quotes for channel") - } - } - - return nil -} diff --git a/internal/v2migrator/store.go b/internal/v2migrator/store.go deleted file mode 100644 index a4ece6c..0000000 --- a/internal/v2migrator/store.go +++ /dev/null @@ -1,117 +0,0 @@ -package v2migrator - -import ( - "compress/gzip" - "encoding/json" - "os" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - "github.com/Luzifer/twitch-bot/v3/internal/v2migrator/crypt" - "github.com/Luzifer/twitch-bot/v3/pkg/database" - "github.com/Luzifer/twitch-bot/v3/plugins" -) - -type ( - Migrator interface { - Load(filename, encryptionPass string) error - Migrate(db database.Connector) error - } - - storageExtendedPermission struct { - AccessToken string `encrypt:"true" json:"access_token,omitempty"` - RefreshToken string `encrypt:"true" json:"refresh_token,omitempty"` - Scopes []string `json:"scopes,omitempty"` - } - - storageFile struct { - Counters map[string]int64 `json:"counters"` - Timers map[string]plugins.TimerEntry `json:"timers"` - Variables map[string]string `json:"variables"` - - ModuleStorage struct { - ModOverlays storageModOverlays `json:"f9ca2b3a-baf6-45ea-a347-c626168665e8"` - ModQuoteDB storageModQuoteDB `json:"917c83ee-ed40-41e4-a558-1c2e59fdf1f5"` - } `json:"module_storage"` - - ExtendedPermissions map[string]*storageExtendedPermission `json:"extended_permissions"` - - EventSubSecret string `encrypt:"true" json:"event_sub_secret,omitempty"` - - BotAccessToken string `encrypt:"true" json:"bot_access_token,omitempty"` - BotRefreshToken string `encrypt:"true" json:"bot_refresh_token,omitempty"` - } -) - -func NewStorageFile() Migrator { - return &storageFile{ - Counters: map[string]int64{}, - Timers: map[string]plugins.TimerEntry{}, - Variables: map[string]string{}, - - ExtendedPermissions: map[string]*storageExtendedPermission{}, - } -} - -func (s *storageFile) Load(filename, encryptionPass string) error { - f, err := os.Open(filename) - if err != nil { - if os.IsNotExist(err) { - // Store init state - return nil - } - return errors.Wrap(err, "open storage file") - } - defer f.Close() - - zf, err := gzip.NewReader(f) - if err != nil { - return errors.Wrap(err, "create gzip reader") - } - defer zf.Close() - - if err = json.NewDecoder(zf).Decode(s); err != nil { - return errors.Wrap(err, "decode storage object") - } - - if err = crypt.DecryptFields(s, encryptionPass); err != nil { - return errors.Wrap(err, "decrypting storage object") - } - - return nil -} - -func (s storageFile) Migrate(db database.Connector) error { - var bat string - err := db.ReadCoreMeta("bot_access_token", &bat) - switch { - case err == nil: - return errors.New("Access token is set, database already initialized") - - case errors.Is(err, database.ErrCoreMetaNotFound): - // This is the expected state - - default: - return errors.Wrap(err, "checking for bot access token") - } - - for name, fn := range map[string]func(database.Connector) error{ - // Core - "core": s.migrateCoreKV, - "counter": s.migrateCounters, - "permissions": s.migratePermissions, - "timers": s.migrateTimers, - "variables": s.migrateVariables, - // Modules - "mod_overlays": s.ModuleStorage.ModOverlays.migrate, - "mod_quotedb": s.ModuleStorage.ModQuoteDB.migrate, - } { - logrus.WithField("module", name).Info("Starting migration...") - if err = fn(db); err != nil { - return errors.Wrapf(err, "executing %q migration", name) - } - } - - return nil -} diff --git a/tplDocs.tpl b/tplDocs.tpl index d0b592c..0de32ce 100644 --- a/tplDocs.tpl +++ b/tplDocs.tpl @@ -67,13 +67,4 @@ Example: {{ end -}} {{- end -}} -## Upgrade from `v2.x` to `v3.x` - -When adding [sprig](https://masterminds.github.io/sprig/) function collection some functions collided and needed replacement. You need to adapt your templates accordingly: - -- Math functions (`add`, `div`, `mod`, `mul`, `multiply`, `sub`) were replaced with their sprig-equivalent and are now working with integers instead of floats. If you need them to continue to work with floats you need to use their [float-variants](https://masterminds.github.io/sprig/mathf.html). -- `now` does no longer format the current date as a string but return the current date. You need to replace this: `now "2006-01-02"` becomes `now | date "2006-01-02"`. -- `concat` is now used to concat arrays. To join strings you will need to modify your code: `concat ":" "string1" "string2"` becomes `lists "string1" "string2" | join ":"`. -- `toLower` / `toUpper` need to be replaced with their sprig equivalent `lower` and `upper`. - {{ if false }}{{ end }}