package database import ( "errors" "fmt" "os" "path/filepath" "reflect" "sort" "strings" "time" "github.com/glebarez/sqlite" "go.uber.org/zap" "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/gorm/logger" "gorm.io/gorm/schema" "github.com/ShukeBta/MediaStationGo/internal/config" "github.com/ShukeBta/MediaStationGo/internal/model" ) // MigrateSQLiteToCurrentIfNeeded copies an existing SQLite database into // PostgreSQL. Redis is not migrated because it is a rebuildable cache, not a // source of truth. const sqliteMigrationCompleteSettingKey = "database.sqlite_migration_complete" func MigrateSQLiteToCurrentIfNeeded(cfg *config.Config, target *gorm.DB, log *zap.Logger) error { if cfg == nil || target == nil || target.Dialector == nil || target.Dialector.Name() != "postgres" { return nil } sqlitePath, err := sqliteMigrationSourcePath(cfg, log) if err != nil { return err } if sqlitePath == "" { return nil } if complete, err := sqliteMigrationMarkedComplete(target); err != nil { return err } else if complete { if log != nil { log.Info("skip sqlite to postgres migration: already completed") } return nil } src, err := openSQLiteMigrationSource(cfg, sqlitePath) if err != nil { return fmt.Errorf("open sqlite migration source: %w", err) } sqlDB, err := src.DB() if err == nil { defer sqlDB.Close() } started := time.Now() if err := resetBootstrapTargetBeforeSQLiteMigrationIfSafe(src, target, log); err != nil { return err } copied, err := copyModelTables(src, target, 500) if err != nil { return err } if err := markSQLiteMigrationComplete(target); err != nil { return err } if log != nil { log.Info("sqlite data migrated to postgres", zap.String("source", sqlitePath), zap.Int64("rows", copied), zap.Duration("duration", time.Since(started))) } return nil } func openSQLiteMigrationSource(cfg *config.Config, sqlitePath string) (*gorm.DB, error) { srcCfg := *cfg srcCfg.Database.Type = "sqlite" srcCfg.Database.DBPath = sqlitePath return gorm.Open(sqlite.Open(buildSQLiteDSN(&srcCfg)), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) } func sqliteMigrationSourcePath(cfg *config.Config, log *zap.Logger) (string, error) { configured := strings.TrimSpace(cfg.Database.DBPath) if configured != "" { exists, err := regularFileExists(configured) if err != nil { return "", fmt.Errorf("stat sqlite migration source: %w", err) } if exists { return configured, nil } } fallback := filepath.Join(strings.TrimSpace(cfg.App.DataDir), "mediastation.db") if fallback == "" || sameCleanPath(configured, fallback) { return "", nil } exists, err := regularFileExists(fallback) if err != nil { return "", fmt.Errorf("stat default sqlite migration source: %w", err) } if !exists { return "", nil } if log != nil && configured != "" { log.Warn("configured sqlite migration source not found; using data-dir default", zap.String("configured", configured), zap.String("fallback", fallback)) } return fallback, nil } func regularFileExists(path string) (bool, error) { if strings.TrimSpace(path) == "" { return false, nil } info, err := os.Stat(path) if err != nil { if errors.Is(err, os.ErrNotExist) { return false, nil } return false, err } return !info.IsDir(), nil } func sameCleanPath(a, b string) bool { if a == "" || b == "" { return false } return filepath.Clean(a) == filepath.Clean(b) } func resetBootstrapTargetBeforeSQLiteMigrationIfSafe(src, target *gorm.DB, log *zap.Logger) error { hasRows, err := sqliteSourceHasMigratableRows(src) if err != nil { return err } if !hasRows { return nil } bootstrapOnly, err := targetLooksLikeBootstrapOnly(target) if err != nil || !bootstrapOnly { return err } for i := len(model.AllModels()) - 1; i >= 0; i-- { m := model.AllModels()[i] if !target.Migrator().HasTable(m) { continue } if err := target.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(m).Error; err != nil { return fmt.Errorf("clear bootstrap target table %T: %w", m, err) } } if log != nil { log.Warn("cleared bootstrap postgres rows before sqlite migration") } return nil } func sqliteSourceHasMigratableRows(src *gorm.DB) (bool, error) { for _, table := range []string{"users", "libraries", "media", "settings"} { exists, err := sqliteTableExists(src, table) if err != nil { return false, err } if !exists { continue } var count int64 if err := src.Raw("SELECT COUNT(1) FROM " + quoteIdent(table)).Scan(&count).Error; err != nil { return false, fmt.Errorf("count sqlite table %s: %w", table, err) } if count > 0 { return true, nil } } return false, nil } func targetLooksLikeBootstrapOnly(target *gorm.DB) (bool, error) { for _, m := range []any{ &model.Library{}, &model.Series{}, &model.Media{}, &model.PlaybackHistory{}, &model.Favorite{}, &model.Playlist{}, &model.PlaylistItem{}, &model.DownloadTask{}, &model.Subscription{}, } { if !target.Migrator().HasTable(m) { continue } var count int64 if err := target.Unscoped().Model(m).Count(&count).Error; err != nil { return false, err } if count > 0 { return false, nil } } var userCount int64 if !target.Migrator().HasTable(&model.User{}) { return true, nil } if err := target.Model(&model.User{}).Count(&userCount).Error; err != nil { return false, err } if userCount == 0 { return true, nil } if userCount != 1 { return false, nil } var user model.User if err := target.Unscoped().Where("username = ?", "admin").First(&user).Error; err != nil { return false, nil } return user.Role == "admin", nil } func sqliteMigrationMarkedComplete(db *gorm.DB) (bool, error) { var value string err := db.Raw("SELECT value FROM "+quoteIdent("settings")+" WHERE "+quoteIdent("key")+" = ?", sqliteMigrationCompleteSettingKey).Scan(&value).Error if err != nil { return false, fmt.Errorf("check sqlite migration marker: %w", err) } return strings.EqualFold(strings.TrimSpace(value), "true"), nil } func markSQLiteMigrationComplete(db *gorm.DB) error { now := time.Now() if err := db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "key"}}, DoUpdates: clause.AssignmentColumns([]string{"value", "updated_at"}), }).Create(&model.Setting{ Key: sqliteMigrationCompleteSettingKey, Value: "true", UpdatedAt: now, }).Error; err != nil { return fmt.Errorf("mark sqlite migration complete: %w", err) } return nil } func copyModelTables(src, target *gorm.DB, batchSize int) (int64, error) { if batchSize <= 0 { batchSize = 500 } var copied int64 for _, m := range model.AllModels() { table, err := modelTableName(src, m) if err != nil { return copied, err } primaryColumns, err := modelPrimaryColumns(src, m) if err != nil { return copied, fmt.Errorf("inspect model %T primary keys: %w", m, err) } exists, err := sqliteTableExists(src, table) if err != nil { return copied, err } if !exists { continue } var sourceCount int64 if err := src.Raw("SELECT COUNT(1) FROM " + quoteIdent(table)).Scan(&sourceCount).Error; err != nil { return copied, fmt.Errorf("count sqlite table %s: %w", table, err) } if sourceCount == 0 { continue } var targetCount int64 if err := target.Raw("SELECT COUNT(1) FROM " + quoteIdent(table)).Scan(&targetCount).Error; err != nil { return copied, fmt.Errorf("count target table %s: %w", table, err) } modelType := reflect.TypeOf(m) if modelType.Kind() != reflect.Ptr { return copied, fmt.Errorf("model %T is not a pointer", m) } sliceType := reflect.SliceOf(modelType.Elem()) slicePtr := reflect.New(sliceType) if err := src.Unscoped().Find(slicePtr.Interface()).Error; err != nil { return copied, fmt.Errorf("read sqlite table %s: %w", table, err) } filtered := slicePtr.Elem() if targetCount > 0 { primaryKeySet, err := targetPrimaryKeySet(target, table, primaryColumns) if err != nil { return copied, err } filtered = filterRowsMissingInTarget(target, table, primaryColumns, filtered, primaryKeySet) } if filtered.Len() == 0 { continue } filteredPtr := reflect.New(filtered.Type()) filteredPtr.Elem().Set(filtered) if err := target.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(filteredPtr.Interface(), batchSize).Error; err != nil { return copied, fmt.Errorf("copy sqlite table %s: %w", table, err) } copied += int64(filtered.Len()) } return copied, nil } func modelPrimaryColumns(db *gorm.DB, m any) ([]string, error) { stmt := &gorm.Statement{DB: db} if err := stmt.Parse(m); err != nil { return nil, err } var cols []string for _, field := range stmt.Schema.PrimaryFields { cols = append(cols, field.DBName) } if len(cols) == 0 { return nil, fmt.Errorf("no primary key columns") } return cols, nil } func targetPrimaryKeySet(target *gorm.DB, table string, primaryColumns []string) (map[string]struct{}, error) { if len(primaryColumns) != 1 { return nil, nil } var values []string if err := target.Raw("SELECT " + quoteIdent(primaryColumns[0]) + " FROM " + quoteIdent(table)).Scan(&values).Error; err != nil { return nil, fmt.Errorf("read target primary keys for table %s: %w", table, err) } set := make(map[string]struct{}, len(values)) for _, value := range values { set[value] = struct{}{} } return set, nil } func filterRowsMissingInTarget(target *gorm.DB, table string, primaryColumns []string, rows reflect.Value, primaryKeySet map[string]struct{}) reflect.Value { if rows.Kind() != reflect.Slice || rows.Len() == 0 || len(primaryColumns) == 0 { return rows } out := reflect.MakeSlice(rows.Type(), 0, rows.Len()) for i := 0; i < rows.Len(); i++ { row := rows.Index(i) keys, ok := rowPrimaryKeys(row, primaryColumns) if !ok { out = reflect.Append(out, row) continue } if primaryKeySet != nil { if _, exists := primaryKeySet[fmt.Sprint(keys[primaryColumns[0]])]; !exists { out = reflect.Append(out, row) } continue } if !targetHasPrimaryKey(target, table, keys) { out = reflect.Append(out, row) } } return out } func rowPrimaryKeys(row reflect.Value, primaryColumns []string) (map[string]any, bool) { if row.Kind() == reflect.Pointer { if row.IsNil() { return nil, false } row = row.Elem() } if row.Kind() != reflect.Struct { return nil, false } keys := make(map[string]any, len(primaryColumns)) for _, column := range primaryColumns { value, ok := fieldByDBName(row, column) if !ok || value.IsZero() { return nil, false } keys[column] = value.Interface() } return keys, true } func fieldByDBName(row reflect.Value, column string) (reflect.Value, bool) { rowType := row.Type() for i := 0; i < row.NumField(); i++ { fieldType := rowType.Field(i) field := row.Field(i) if fieldType.Anonymous { if value, ok := fieldByDBName(field, column); ok { return value, true } } if columnNameForStructField(fieldType) == column { if field.Kind() == reflect.Pointer && field.IsNil() { return reflect.Value{}, false } return field, field.CanInterface() } } return reflect.Value{}, false } func columnNameForStructField(field reflect.StructField) string { if field.PkgPath != "" && !field.Anonymous { return "" } tag := field.Tag.Get("gorm") settings := schema.ParseTagSetting(tag, ";") if column := settings["COLUMN"]; column != "" { return column } return schema.NamingStrategy{}.ColumnName("", field.Name) } func targetHasPrimaryKey(target *gorm.DB, table string, keys map[string]any) bool { where := make([]string, 0, len(keys)) args := make([]any, 0, len(keys)) for _, column := range sortedMapKeys(keys) { where = append(where, quoteIdent(column)+" = ?") args = append(args, keys[column]) } var count int64 err := target.Raw("SELECT COUNT(1) FROM "+quoteIdent(table)+" WHERE "+strings.Join(where, " AND "), args...).Scan(&count).Error return err == nil && count > 0 } func sortedMapKeys(m map[string]any) []string { keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) } sort.Strings(keys) return keys } func sqliteTableExists(db *gorm.DB, table string) (bool, error) { var count int64 if err := db.Raw(`SELECT COUNT(1) FROM sqlite_master WHERE type = 'table' AND name = ?`, table).Scan(&count).Error; err != nil { return false, fmt.Errorf("inspect sqlite table %s: %w", table, err) } return count > 0, nil } func modelTableName(db *gorm.DB, m any) (string, error) { stmt := &gorm.Statement{DB: db} if err := stmt.Parse(m); err != nil { return "", err } return stmt.Schema.Table, nil } func quoteIdent(value string) string { return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` }