From fea1b1dda118e3c2316e3c7c278857e6eb40e82b Mon Sep 17 00:00:00 2001 From: ShukeBta <272197458+ShukeBta@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:13:30 +0800 Subject: [PATCH] Fix PostgreSQL migration resume and home crash guard --- README.md | 3 +- README_EN.md | 3 +- internal/database/database.go | 233 +++++++++++++++++++++++++---- internal/database/database_test.go | 97 ++++++++++++ internal/model/model.go | 2 +- web/src/App.tsx | 47 +++++- web/src/pages/HomePage.tsx | 9 +- web/src/utils/groupSeries.ts | 6 +- 8 files changed, 356 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 807c7fd..dd590db 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,8 @@ environment: - 新部署:直接 `docker compose up -d`,会使用 PostgreSQL,不会创建新的 SQLite 主库。 - 旧版本升级:如果存在 `./data/mediastation.db`,首次启动新版 compose 时会自动导入到 PostgreSQL。 -- 导入只在 PostgreSQL 目标库为空时运行;PG 里已有数据时会跳过,避免覆盖现有数据。 +- 导入按主键补齐缺失数据,已有行会跳过;如果中途失败,修复后再次启动会继续补剩余表。 +- 成功导入后会在 PostgreSQL 的 `settings` 表写入完成标记,之后即使旧 SQLite 文件还在也不会重复导入。 - Redis 是热缓存,OpenSearch 是搜索索引;它们都不是源数据库,丢失后可以重建。 旧 SQLite 升级到 PostgreSQL 的建议步骤: diff --git a/README_EN.md b/README_EN.md index 90f09c1..d06f154 100644 --- a/README_EN.md +++ b/README_EN.md @@ -144,7 +144,8 @@ environment: - Fresh installs: `docker compose up -d` uses PostgreSQL and does not create a new SQLite primary database. - Upgrades: if `./data/mediastation.db` exists, the first start with the new compose file imports it into PostgreSQL. -- Migration only runs when the PostgreSQL target tables are empty. If PostgreSQL already has data, the SQLite import is skipped. +- Migration fills missing rows by primary key and skips rows that already exist. If it fails partway through, a later start continues the remaining tables. +- After a successful import, PostgreSQL gets a completion marker in the `settings` table, so the old SQLite file is not imported again. - Redis is a hot cache and OpenSearch is a search index; neither is a source database. Recommended SQLite to PostgreSQL upgrade flow: diff --git a/internal/database/database.go b/internal/database/database.go index 7c5e258..3b42c7d 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "reflect" + "sort" "strings" "time" @@ -18,6 +19,7 @@ import ( "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/gorm/logger" + "gorm.io/gorm/schema" "github.com/ShukeBta/MediaStationGo/internal/config" "github.com/ShukeBta/MediaStationGo/internal/model" @@ -104,11 +106,11 @@ func databaseDialector(cfg *config.Config, dialect string) (gorm.Dialector, erro } } -// MigrateSQLiteToCurrentIfNeeded copies an existing SQLite database into a new -// PostgreSQL database once. It is intentionally conservative: it only runs when -// the current DB is PostgreSQL, the configured SQLite file exists, and the -// target business tables are empty. Redis is not migrated because it is a -// rebuildable cache, not a source of truth. +// MigrateSQLiteToCurrentIfNeeded copies an existing SQLite database into +// PostgreSQL. Redis is not migrated because it is a rebuildable cache, not a +// source of truth. +const sqliteMigrationCompleteSettingKey = "database.sqlite_migration_complete" + func MigrateSQLiteToCurrentIfNeeded(cfg *config.Config, target *gorm.DB, log *zap.Logger) error { if cfg == nil || target == nil || target.Dialector == nil || target.Dialector.Name() != "postgres" { return nil @@ -123,11 +125,11 @@ func MigrateSQLiteToCurrentIfNeeded(cfg *config.Config, target *gorm.DB, log *za } return fmt.Errorf("stat sqlite migration source: %w", err) } - if empty, err := targetBusinessTablesEmpty(target); err != nil { + if complete, err := sqliteMigrationMarkedComplete(target); err != nil { return err - } else if !empty { + } else if complete { if log != nil { - log.Info("skip sqlite to postgres migration: target database already has data") + log.Info("skip sqlite to postgres migration: already completed") } return nil } @@ -150,7 +152,10 @@ func MigrateSQLiteToCurrentIfNeeded(cfg *config.Config, target *gorm.DB, log *za if err != nil { return err } - if copied > 0 && log != nil { + if err := markSQLiteMigrationComplete(target); err != nil { + return err + } + if log != nil { log.Info("sqlite data migrated to postgres", zap.String("source", sqlitePath), zap.Int64("rows", copied), @@ -159,21 +164,28 @@ func MigrateSQLiteToCurrentIfNeeded(cfg *config.Config, target *gorm.DB, log *za return nil } -func targetBusinessTablesEmpty(db *gorm.DB) (bool, error) { - for _, m := range model.AllModels() { - table, err := modelTableName(db, m) - if err != nil { - return false, err - } - var count int64 - if err := db.Raw("SELECT COUNT(1) FROM " + quoteIdent(table)).Scan(&count).Error; err != nil { - return false, fmt.Errorf("count target table %s: %w", table, err) - } - if count > 0 { - return false, nil - } +func sqliteMigrationMarkedComplete(db *gorm.DB) (bool, error) { + var value string + err := db.Raw("SELECT value FROM "+quoteIdent("settings")+" WHERE "+quoteIdent("key")+" = ?", sqliteMigrationCompleteSettingKey).Scan(&value).Error + if err != nil { + return false, fmt.Errorf("check sqlite migration marker: %w", err) } - return true, nil + return strings.EqualFold(strings.TrimSpace(value), "true"), nil +} + +func markSQLiteMigrationComplete(db *gorm.DB) error { + now := time.Now() + if err := db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "key"}}, + DoUpdates: clause.AssignmentColumns([]string{"value", "updated_at"}), + }).Create(&model.Setting{ + Key: sqliteMigrationCompleteSettingKey, + Value: "true", + UpdatedAt: now, + }).Error; err != nil { + return fmt.Errorf("mark sqlite migration complete: %w", err) + } + return nil } func copyModelTables(src, target *gorm.DB, batchSize int) (int64, error) { @@ -186,6 +198,10 @@ func copyModelTables(src, target *gorm.DB, batchSize int) (int64, error) { if err != nil { return copied, err } + primaryColumns, err := modelPrimaryColumns(src, m) + if err != nil { + return copied, fmt.Errorf("inspect model %T primary keys: %w", m, err) + } exists, err := sqliteTableExists(src, table) if err != nil { return copied, err @@ -204,10 +220,6 @@ func copyModelTables(src, target *gorm.DB, batchSize int) (int64, error) { if err := target.Raw("SELECT COUNT(1) FROM " + quoteIdent(table)).Scan(&targetCount).Error; err != nil { return copied, fmt.Errorf("count target table %s: %w", table, err) } - if targetCount > 0 { - continue - } - modelType := reflect.TypeOf(m) if modelType.Kind() != reflect.Ptr { return copied, fmt.Errorf("model %T is not a pointer", m) @@ -217,17 +229,156 @@ func copyModelTables(src, target *gorm.DB, batchSize int) (int64, error) { if err := src.Unscoped().Find(slicePtr.Interface()).Error; err != nil { return copied, fmt.Errorf("read sqlite table %s: %w", table, err) } - if slicePtr.Elem().Len() == 0 { + filtered := slicePtr.Elem() + if targetCount > 0 { + primaryKeySet, err := targetPrimaryKeySet(target, table, primaryColumns) + if err != nil { + return copied, err + } + filtered = filterRowsMissingInTarget(target, table, primaryColumns, filtered, primaryKeySet) + } + if filtered.Len() == 0 { continue } - if err := target.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(slicePtr.Interface(), batchSize).Error; err != nil { + filteredPtr := reflect.New(filtered.Type()) + filteredPtr.Elem().Set(filtered) + if err := target.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(filteredPtr.Interface(), batchSize).Error; err != nil { return copied, fmt.Errorf("copy sqlite table %s: %w", table, err) } - copied += int64(slicePtr.Elem().Len()) + copied += int64(filtered.Len()) } return copied, nil } +func modelPrimaryColumns(db *gorm.DB, m any) ([]string, error) { + stmt := &gorm.Statement{DB: db} + if err := stmt.Parse(m); err != nil { + return nil, err + } + var cols []string + for _, field := range stmt.Schema.PrimaryFields { + cols = append(cols, field.DBName) + } + if len(cols) == 0 { + return nil, fmt.Errorf("no primary key columns") + } + return cols, nil +} + +func targetPrimaryKeySet(target *gorm.DB, table string, primaryColumns []string) (map[string]struct{}, error) { + if len(primaryColumns) != 1 { + return nil, nil + } + var values []string + if err := target.Raw("SELECT " + quoteIdent(primaryColumns[0]) + " FROM " + quoteIdent(table)).Scan(&values).Error; err != nil { + return nil, fmt.Errorf("read target primary keys for table %s: %w", table, err) + } + set := make(map[string]struct{}, len(values)) + for _, value := range values { + set[value] = struct{}{} + } + return set, nil +} + +func filterRowsMissingInTarget(target *gorm.DB, table string, primaryColumns []string, rows reflect.Value, primaryKeySet map[string]struct{}) reflect.Value { + if rows.Kind() != reflect.Slice || rows.Len() == 0 || len(primaryColumns) == 0 { + return rows + } + out := reflect.MakeSlice(rows.Type(), 0, rows.Len()) + for i := 0; i < rows.Len(); i++ { + row := rows.Index(i) + keys, ok := rowPrimaryKeys(row, primaryColumns) + if !ok { + out = reflect.Append(out, row) + continue + } + if primaryKeySet != nil { + if _, exists := primaryKeySet[fmt.Sprint(keys[primaryColumns[0]])]; !exists { + out = reflect.Append(out, row) + } + continue + } + if !targetHasPrimaryKey(target, table, keys) { + out = reflect.Append(out, row) + } + } + return out +} + +func rowPrimaryKeys(row reflect.Value, primaryColumns []string) (map[string]any, bool) { + if row.Kind() == reflect.Pointer { + if row.IsNil() { + return nil, false + } + row = row.Elem() + } + if row.Kind() != reflect.Struct { + return nil, false + } + keys := make(map[string]any, len(primaryColumns)) + for _, column := range primaryColumns { + value, ok := fieldByDBName(row, column) + if !ok || value.IsZero() { + return nil, false + } + keys[column] = value.Interface() + } + return keys, true +} + +func fieldByDBName(row reflect.Value, column string) (reflect.Value, bool) { + rowType := row.Type() + for i := 0; i < row.NumField(); i++ { + fieldType := rowType.Field(i) + field := row.Field(i) + if fieldType.Anonymous { + if value, ok := fieldByDBName(field, column); ok { + return value, true + } + } + if columnNameForStructField(fieldType) == column { + if field.Kind() == reflect.Pointer && field.IsNil() { + return reflect.Value{}, false + } + return field, field.CanInterface() + } + } + return reflect.Value{}, false +} + +func columnNameForStructField(field reflect.StructField) string { + if field.PkgPath != "" && !field.Anonymous { + return "" + } + tag := field.Tag.Get("gorm") + settings := schema.ParseTagSetting(tag, ";") + if column := settings["COLUMN"]; column != "" { + return column + } + return schema.NamingStrategy{}.ColumnName("", field.Name) +} + +func targetHasPrimaryKey(target *gorm.DB, table string, keys map[string]any) bool { + where := make([]string, 0, len(keys)) + args := make([]any, 0, len(keys)) + for _, column := range sortedMapKeys(keys) { + where = append(where, quoteIdent(column)+" = ?") + args = append(args, keys[column]) + } + var count int64 + err := target.Raw("SELECT COUNT(1) FROM "+quoteIdent(table)+" WHERE "+strings.Join(where, " AND "), args...).Scan(&count).Error + return err == nil && count > 0 +} + +func sortedMapKeys(m map[string]any) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + func sqliteTableExists(db *gorm.DB, table string) (bool, error) { var count int64 if err := db.Raw(`SELECT COUNT(1) FROM sqlite_master WHERE type = 'table' AND name = ?`, table).Scan(&count).Error; err != nil { @@ -341,6 +492,9 @@ func AutoMigrate(db *gorm.DB) error { if err := db.AutoMigrate(model.AllModels()...); err != nil { return err } + if err := ensurePostgresColumnCompatibility(db); err != nil { + return err + } if err := enforceTelegramBindingOneToOne(db); err != nil { return err } @@ -353,6 +507,21 @@ func AutoMigrate(db *gorm.DB) error { return nil } +func ensurePostgresColumnCompatibility(db *gorm.DB) error { + if !isPostgres(db) { + return nil + } + statements := []string{ + `ALTER TABLE media ALTER COLUMN container TYPE varchar(128)`, + } + for _, stmt := range statements { + if err := db.Exec(stmt).Error; err != nil { + return err + } + } + return nil +} + func ensurePerformanceIndexes(db *gorm.DB) error { statements := []string{ `CREATE INDEX IF NOT EXISTS idx_media_library_created_active ON media(library_id, created_at DESC) WHERE deleted_at IS NULL`, @@ -386,6 +555,10 @@ func isSQLite(db *gorm.DB) bool { return db != nil && db.Dialector != nil && db.Dialector.Name() == "sqlite" } +func isPostgres(db *gorm.DB) bool { + return db != nil && db.Dialector != nil && db.Dialector.Name() == "postgres" +} + // mediaSearchIndexSchemaVersion 标识 FTS 索引的物理布局版本。 // v2:FTS 行的 rowid 与 media.rowid 对齐,并由触发器实时维护。 const mediaSearchIndexSchemaVersion = 2 diff --git a/internal/database/database_test.go b/internal/database/database_test.go index cf2af81..bde9636 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -115,3 +115,100 @@ func TestCopyModelTablesMigratesExistingSQLiteRows(t *testing.T) { t.Fatalf("user not preserved: %#v", got) } } + +func TestCopyModelTablesResumesPartialSQLiteMigration(t *testing.T) { + src, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + dst, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + for _, db := range []*gorm.DB{src, dst} { + if err := db.AutoMigrate(&model.User{}, &model.Media{}, &model.Setting{}); err != nil { + t.Fatal(err) + } + } + user := model.User{Username: "admin", PasswordHash: "hash", Role: "admin", IsActive: true} + if err := src.Create(&user).Error; err != nil { + t.Fatal(err) + } + if err := dst.Create(&user).Error; err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: "library-1", + Title: "Resume Migration", + Path: "/media/resume.mp4", + Container: "mov,mp4,m4a,3gp,3g2,mj2", + ScrapeStatus: "matched", + OriginalName: "Resume Migration", + PosterURL: "/media/poster.jpg", + BackdropURL: "/media/backdrop.jpg", + VideoCodec: "hevc", + AudioCodec: "eac3", + DurationSec: 120, + SizeBytes: 1024, + Width: 3840, + Height: 2160, + SeasonNum: 1, + EpisodeNum: 1, + } + if err := src.Create(&media).Error; err != nil { + t.Fatal(err) + } + if err := src.Create(&model.Setting{Key: "organize.auto", Value: "false"}).Error; err != nil { + t.Fatal(err) + } + + copied, err := copyModelTables(src, dst, 2) + if err != nil { + t.Fatal(err) + } + if copied != 2 { + t.Fatalf("copied rows = %d, want 2", copied) + } + var got model.Media + if err := dst.First(&got, "path = ?", media.Path).Error; err != nil { + t.Fatal(err) + } + if got.Container != media.Container { + t.Fatalf("container = %q, want %q", got.Container, media.Container) + } + + copied, err = copyModelTables(src, dst, 2) + if err != nil { + t.Fatal(err) + } + if copied != 0 { + t.Fatalf("second copy rows = %d, want 0", copied) + } +} + +func TestSQLiteMigrationCompleteMarker(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Setting{}); err != nil { + t.Fatal(err) + } + complete, err := sqliteMigrationMarkedComplete(db) + if err != nil { + t.Fatal(err) + } + if complete { + t.Fatal("fresh database should not be marked migrated") + } + if err := markSQLiteMigrationComplete(db); err != nil { + t.Fatal(err) + } + complete, err = sqliteMigrationMarkedComplete(db) + if err != nil { + t.Fatal(err) + } + if !complete { + t.Fatal("database should be marked migrated") + } +} diff --git a/internal/model/model.go b/internal/model/model.go index f002cf4..45f9b86 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -80,7 +80,7 @@ type Media struct { Height int `json:"height"` VideoCodec string `gorm:"size:32" json:"video_codec,omitempty"` AudioCodec string `gorm:"size:32" json:"audio_codec,omitempty"` - Container string `gorm:"size:16" json:"container,omitempty"` + Container string `gorm:"size:128" json:"container,omitempty"` PosterURL string `gorm:"size:1024" json:"poster_url,omitempty"` BackdropURL string `gorm:"size:1024" json:"backdrop_url,omitempty"` Overview string `gorm:"type:text" json:"overview,omitempty"` diff --git a/web/src/App.tsx b/web/src/App.tsx index 0f1e24a..5311490 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,4 +1,4 @@ -import { Suspense, lazy } from 'react' +import { Component, Suspense, lazy, type ErrorInfo, type ReactNode } from 'react' import { Navigate, Route, Routes } from 'react-router-dom' import { Layout } from './components/Layout' @@ -105,10 +105,46 @@ const LicensePage = lazy(() => const Loading = () =>
加载中…
+class AppErrorBoundary extends Component<{ children: ReactNode }, { hasError: boolean }> { + state = { hasError: false } + + static getDerivedStateFromError() { + return { hasError: true } + } + + componentDidCatch(error: Error, info: ErrorInfo) { + console.error('MediaStation UI crashed', error, info) + } + + render() { + if (this.state.hasError) { + return ( +页面加载失败
++ 当前页面遇到异常,刷新后会重新加载资源和登录状态。 +
+ +