diff --git a/README.md b/README.md index 2a07cf3..9d6d67f 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,12 @@ Usage of twitch-bot: # twitch-bot help Supported sub-commands are: - actor-docs Generate markdown documentation for available actors - api-token [...scope] Generate an api-token to be entered into the config - reset-secrets Remove encrypted data to reset encryption passphrase - tpl-docs Generate markdown documentation for available template functions - validate-config Try to load configuration file and report errors if any + actor-docs Generate markdown documentation for available actors + api-token [...scope] Generate an api-token to be entered into the config + copy-database Copies database contents to a new storage DSN i.e. for migrating to a new DBMS + reset-secrets Remove encrypted data to reset encryption passphrase + tpl-docs Generate markdown documentation for available template functions + validate-config Try to load configuration file and report errors if any ``` ### Database Connection Strings diff --git a/cli_migrateDatabase.go b/cli_migrateDatabase.go new file mode 100644 index 0000000..acca430 --- /dev/null +++ b/cli_migrateDatabase.go @@ -0,0 +1,67 @@ +package main + +import ( + "sync" + + "github.com/Luzifer/twitch-bot/v3/pkg/database" + "github.com/Luzifer/twitch-bot/v3/plugins" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +var ( + dbCopyFuncs = map[string]plugins.DatabaseCopyFunc{} + dbCopyFuncsLock sync.Mutex +) + +func init() { + cli.Add(cliRegistryEntry{ + Name: "copy-database", + Description: "Copies database contents to a new storage DSN i.e. for migrating to a new DBMS", + Params: []string{"", ""}, + Run: func(args []string) error { + if len(args) < 3 { //nolint:gomnd // Just a count of parameters + return errors.New("Usage: twitch-bot copy-database ") + } + + // Core functions cannot register themselves, we take that for them + registerDatabaseCopyFunc("core-values", db.CopyDatabase) + registerDatabaseCopyFunc("permissions", accessService.CopyDatabase) + registerDatabaseCopyFunc("timers", timerService.CopyDatabase) + + targetDB, err := database.New(args[1], args[2], cfg.StorageEncryptionPass) + if err != nil { + return errors.Wrap(err, "connecting to target db") + } + defer func() { + if err := targetDB.Close(); err != nil { + logrus.WithError(err).Error("closing connection to target db") + } + }() + + return errors.Wrap( + targetDB.DB().Transaction(func(tx *gorm.DB) (err error) { + for name, dbcf := range dbCopyFuncs { + logrus.WithField("name", name).Info("running migration") + if err = dbcf(db.DB(), tx); err != nil { + return errors.Wrapf(err, "running DatabaseCopyFunc %q", name) + } + } + + logrus.Info("database has been copied successfully") + + return nil + }), + "copying database to target", + ) + }, + }) +} + +func registerDatabaseCopyFunc(name string, fn plugins.DatabaseCopyFunc) { + dbCopyFuncsLock.Lock() + defer dbCopyFuncsLock.Unlock() + + dbCopyFuncs[name] = fn +} diff --git a/internal/actors/counter/actor.go b/internal/actors/counter/actor.go index c3682e3..077a5a9 100644 --- a/internal/actors/counter/actor.go +++ b/internal/actors/counter/actor.go @@ -9,6 +9,7 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "gopkg.in/irc.v4" + "gorm.io/gorm" "github.com/Luzifer/twitch-bot/v3/pkg/database" "github.com/Luzifer/twitch-bot/v3/plugins" @@ -28,6 +29,10 @@ func Register(args plugins.RegistrationArguments) error { return errors.Wrap(err, "applying schema migration") } + args.RegisterCopyDatabaseFunc("counter", func(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &Counter{}) + }) + formatMessage = args.FormatMessage args.RegisterActor("counter", func() plugins.Actor { return &ActorCounter{} }) diff --git a/internal/actors/punish/actor.go b/internal/actors/punish/actor.go index c52f90c..56557c1 100644 --- a/internal/actors/punish/actor.go +++ b/internal/actors/punish/actor.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "gopkg.in/irc.v4" + "gorm.io/gorm" "github.com/Luzifer/twitch-bot/v3/pkg/database" "github.com/Luzifer/twitch-bot/v3/pkg/twitch" @@ -34,6 +35,10 @@ func Register(args plugins.RegistrationArguments) error { return errors.Wrap(err, "applying schema migration") } + args.RegisterCopyDatabaseFunc("punish", func(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &punishLevel{}) + }) + botTwitchClient = args.GetTwitchClient() formatMessage = args.FormatMessage diff --git a/internal/actors/quotedb/actor.go b/internal/actors/quotedb/actor.go index 9215528..752c719 100644 --- a/internal/actors/quotedb/actor.go +++ b/internal/actors/quotedb/actor.go @@ -5,6 +5,7 @@ import ( "github.com/pkg/errors" "gopkg.in/irc.v4" + "gorm.io/gorm" "github.com/Luzifer/twitch-bot/v3/pkg/database" "github.com/Luzifer/twitch-bot/v3/plugins" @@ -30,6 +31,10 @@ func Register(args plugins.RegistrationArguments) error { return errors.Wrap(err, "applying schema migration") } + args.RegisterCopyDatabaseFunc("quote", func(src, target *gorm.DB) error { + return database.CopyObjects(src, target, "e{}) + }) + formatMessage = args.FormatMessage send = args.SendMessage diff --git a/internal/actors/variables/actor.go b/internal/actors/variables/actor.go index 46cb939..9349084 100644 --- a/internal/actors/variables/actor.go +++ b/internal/actors/variables/actor.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "gopkg.in/irc.v4" + "gorm.io/gorm" "github.com/Luzifer/twitch-bot/v3/pkg/database" "github.com/Luzifer/twitch-bot/v3/plugins" @@ -27,6 +28,10 @@ func Register(args plugins.RegistrationArguments) error { return errors.Wrap(err, "applying schema migration") } + args.RegisterCopyDatabaseFunc("variable", func(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &variable{}) + }) + formatMessage = args.FormatMessage args.RegisterActor("setvariable", func() plugins.Actor { return &ActorSetVariable{} }) diff --git a/internal/apimodules/customevent/customevent.go b/internal/apimodules/customevent/customevent.go index 92be660..7382f7c 100644 --- a/internal/apimodules/customevent/customevent.go +++ b/internal/apimodules/customevent/customevent.go @@ -10,6 +10,7 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" + "gorm.io/gorm" "github.com/Luzifer/twitch-bot/v3/pkg/database" "github.com/Luzifer/twitch-bot/v3/plugins" @@ -32,6 +33,10 @@ func Register(args plugins.RegistrationArguments) error { return errors.Wrap(err, "applying schema migration") } + args.RegisterCopyDatabaseFunc("custom_event", func(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &storedCustomEvent{}) + }) + mc = &memoryCache{dbc: db} eventCreatorFunc = args.CreateEvent diff --git a/internal/apimodules/overlays/database.go b/internal/apimodules/overlays/database.go index d893e52..701616c 100644 --- a/internal/apimodules/overlays/database.go +++ b/internal/apimodules/overlays/database.go @@ -14,6 +14,7 @@ import ( type ( overlaysEvent struct { + ID uint64 `gorm:"primaryKey"` Channel string `gorm:"not null;index:overlays_events_sort_idx"` CreatedAt time.Time `gorm:"index:overlays_events_sort_idx"` EventType string @@ -28,7 +29,7 @@ func AddChannelEvent(db database.Connector, channel string, evt SocketMessage) e } return errors.Wrap( - db.DB().Create(overlaysEvent{ + db.DB().Create(&overlaysEvent{ Channel: channel, CreatedAt: evt.Time.UTC(), EventType: evt.Type, diff --git a/internal/apimodules/overlays/overlays.go b/internal/apimodules/overlays/overlays.go index 745ebe9..b60db50 100644 --- a/internal/apimodules/overlays/overlays.go +++ b/internal/apimodules/overlays/overlays.go @@ -15,6 +15,7 @@ import ( "github.com/gorilla/websocket" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "gorm.io/gorm" "github.com/Luzifer/go_helpers/v2/str" "github.com/Luzifer/twitch-bot/v3/pkg/database" @@ -69,6 +70,10 @@ func Register(args plugins.RegistrationArguments) error { return errors.Wrap(err, "applying schema migration") } + args.RegisterCopyDatabaseFunc("overlay_events", func(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &overlaysEvent{}) + }) + validateToken = args.ValidateToken args.RegisterAPIRoute(plugins.HTTPRouteRegistrationArgs{ diff --git a/internal/apimodules/raffle/raffle.go b/internal/apimodules/raffle/raffle.go index 9903ab0..04c0b00 100644 --- a/internal/apimodules/raffle/raffle.go +++ b/internal/apimodules/raffle/raffle.go @@ -5,6 +5,7 @@ package raffle import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "gorm.io/gorm" "github.com/Luzifer/twitch-bot/v3/pkg/database" "github.com/Luzifer/twitch-bot/v3/pkg/twitch" @@ -28,6 +29,10 @@ func Register(args plugins.RegistrationArguments) (err error) { return errors.Wrap(err, "applying schema migration") } + args.RegisterCopyDatabaseFunc("raffle", func(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &raffle{}, &raffleEntry{}) + }) + dbc = newDBClient(db) if err = dbc.RefreshActiveRaffles(); err != nil { return errors.Wrap(err, "refreshing active raffle cache") diff --git a/internal/service/access/access.go b/internal/service/access/access.go index 5c80a40..5d8807f 100644 --- a/internal/service/access/access.go +++ b/internal/service/access/access.go @@ -47,6 +47,10 @@ func New(db database.Connector) (*Service, error) { ) } +func (s *Service) CopyDatabase(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &extendedPermission{}) +} + func (s Service) GetBotUsername() (string, error) { var botUsername string err := s.db.ReadCoreMeta(coreMetaKeyBotUsername, &botUsername) diff --git a/internal/service/timer/timer.go b/internal/service/timer/timer.go index c9cd01c..5191117 100644 --- a/internal/service/timer/timer.go +++ b/internal/service/timer/timer.go @@ -44,6 +44,10 @@ func New(db database.Connector, cronService *cron.Cron) (*Service, error) { return s, errors.Wrap(s.db.DB().AutoMigrate(&timer{}), "applying migrations") } +func (s *Service) CopyDatabase(src, target *gorm.DB) error { + return database.CopyObjects(src, target, &timer{}) +} + func (s *Service) UpdatePermitTimeout(d time.Duration) { s.permitTimeout = d } diff --git a/pkg/database/connector.go b/pkg/database/connector.go index ece29a3..d711c8e 100644 --- a/pkg/database/connector.go +++ b/pkg/database/connector.go @@ -42,7 +42,7 @@ func New(driverName, connString, encryptionSecret string) (Connector, error) { switch driverName { case "mysql": - mysqlDriver.SetLogger(newLogrusLogWriterWithLevel(logrus.ErrorLevel, driverName)) + mysqlDriver.SetLogger(NewLogrusLogWriterWithLevel(logrus.StandardLogger(), logrus.ErrorLevel, driverName)) innerDB = mysql.Open(connString) dbTuner = tuneMySQLDatabase @@ -63,7 +63,13 @@ func New(driverName, connString, encryptionSecret string) (Connector, error) { db, err := gorm.Open(innerDB, &gorm.Config{ DisableForeignKeyConstraintWhenMigrating: true, - Logger: logger.New(newLogrusLogWriterWithLevel(logrus.TraceLevel, driverName), logger.Config{}), + Logger: logger.New(NewLogrusLogWriterWithLevel(logrus.StandardLogger(), logrus.TraceLevel, driverName), logger.Config{ + SlowThreshold: time.Second, + Colorful: false, + IgnoreRecordNotFoundError: false, + ParameterizedQueries: false, + LogLevel: logger.Info, + }), }) if err != nil { return nil, errors.Wrap(err, "connecting database") @@ -87,6 +93,10 @@ func (c connector) Close() error { return nil } +func (c connector) CopyDatabase(src, target *gorm.DB) error { + return CopyObjects(src, target, &coreKV{}) +} + func (c connector) DB() *gorm.DB { return c.db } diff --git a/pkg/database/copyhelper.go b/pkg/database/copyhelper.go new file mode 100644 index 0000000..5810f12 --- /dev/null +++ b/pkg/database/copyhelper.go @@ -0,0 +1,42 @@ +package database + +import ( + "reflect" + + "github.com/pkg/errors" + "gorm.io/gorm" +) + +const copyBatchSize = 100 + +// CopyObjects is a helper to copy elements of a given type from the +// src to the target GORM database interface +func CopyObjects(src, target *gorm.DB, objects ...any) (err error) { + for _, obj := range objects { + copySlice := reflect.New(reflect.SliceOf(reflect.TypeOf(obj))).Elem().Addr().Interface() + + if err = target.AutoMigrate(obj); err != nil { + return errors.Wrap(err, "applying migration to target") + } + + if err = target.Where("1 = 1").Delete(obj).Error; err != nil { + return errors.Wrap(err, "cleaning target table") + } + + if err = src.FindInBatches(copySlice, copyBatchSize, func(tx *gorm.DB, _ int) error { + if err = target.Save(copySlice).Error; err != nil { + if errors.Is(err, gorm.ErrEmptySlice) { + // That's fine and no reason to exit here + return nil + } + return errors.Wrap(err, "inserting collected elements") + } + + return nil + }).Error; err != nil { + return errors.Wrap(err, "batch-copying data") + } + } + + return nil +} diff --git a/pkg/database/database.go b/pkg/database/database.go index 7694605..544facf 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -11,6 +11,7 @@ type ( // convenience methods Connector interface { Close() error + CopyDatabase(src, target *gorm.DB) error DB() *gorm.DB DeleteCoreMeta(key string) error ReadCoreMeta(key string, value any) error diff --git a/pkg/database/logger.go b/pkg/database/logger.go index d042056..76e50c3 100644 --- a/pkg/database/logger.go +++ b/pkg/database/logger.go @@ -8,18 +8,18 @@ import ( ) type ( - logWriter struct{ io.Writer } + LogWriter struct{ io.Writer } ) -func newLogrusLogWriterWithLevel(level logrus.Level, dbDriver string) logWriter { - writer := logrus.WithField("database", dbDriver).WriterLevel(level) - return logWriter{writer} +func NewLogrusLogWriterWithLevel(logger *logrus.Logger, level logrus.Level, dbDriver string) LogWriter { + writer := logger.WithField("database", dbDriver).WriterLevel(level) + return LogWriter{writer} } -func (l logWriter) Print(a ...any) { +func (l LogWriter) Print(a ...any) { fmt.Fprint(l.Writer, a...) } -func (l logWriter) Printf(format string, a ...any) { +func (l LogWriter) Printf(format string, a ...any) { fmt.Fprintf(l.Writer, format, a...) } diff --git a/plugins/interface.go b/plugins/interface.go index c915fdd..bef1102 100644 --- a/plugins/interface.go +++ b/plugins/interface.go @@ -5,6 +5,7 @@ import ( "github.com/robfig/cron/v3" log "github.com/sirupsen/logrus" "gopkg.in/irc.v4" + "gorm.io/gorm" "github.com/Luzifer/twitch-bot/v3/pkg/database" "github.com/Luzifer/twitch-bot/v3/pkg/twitch" @@ -38,6 +39,8 @@ type ( CronRegistrationFunc func(spec string, cmd func()) (cron.EntryID, error) + DatabaseCopyFunc func(src, target *gorm.DB) error + EventHandlerFunc func(evt string, eventData *FieldCollection) error EventHandlerRegisterFunc func(EventHandlerFunc) error @@ -83,6 +86,10 @@ type ( RegisterActorDocumentation ActorDocumentationRegistrationFunc // RegisterAPIRoute registers a new HTTP handler function including documentation RegisterAPIRoute HTTPRouteRegistrationFunc + // RegisterCopyDatabaseFunc registers a DatabaseCopyFunc for the + // database migration tool. Modules not registering such a func + // will not be copied over when migrating to another database. + RegisterCopyDatabaseFunc func(name string, fn DatabaseCopyFunc) // RegisterCron is a method to register cron functions in the global cron instance RegisterCron CronRegistrationFunc // RegisterEventHandler is a method to register a handler function receiving ALL events diff --git a/plugins_core.go b/plugins_core.go index cfac10d..58aefca 100644 --- a/plugins_core.go +++ b/plugins_core.go @@ -163,6 +163,7 @@ func getRegistrationArguments() plugins.RegistrationArguments { RegisterActorDocumentation: registerActorDocumentation, RegisterAPIRoute: registerRoute, RegisterCron: cronService.AddFunc, + RegisterCopyDatabaseFunc: registerDatabaseCopyFunc, RegisterEventHandler: registerEventHandlers, RegisterMessageModFunc: registerChatcommand, RegisterRawMessageHandler: registerRawMessageHandler,