diff --git a/openflare_server/model/main.go b/openflare_server/model/main.go index 028738c9..3854e941 100644 --- a/openflare_server/model/main.go +++ b/openflare_server/model/main.go @@ -123,9 +123,21 @@ func openDatabase() (*gorm.DB, string, error) { } func autoMigrateAll(db *gorm.DB) error { + return autoMigrateAllExcept(db, nil) +} + +func autoMigrateAllExcept(db *gorm.DB, excludedTables map[string]bool) error { models := registeredModels() for i, item := range models { name := fmt.Sprintf("%T", item) + tableName, err := tableNameForModel(item) + if err != nil { + return fmt.Errorf("resolve table name for %s failed: %w", name, err) + } + if excludedTables[tableName] { + slog.Info("autoMigrateAll: skipped model", "index", fmt.Sprintf("%d/%d", i+1, len(models)), "model", name, "table", tableName) + continue + } slog.Info("autoMigrateAll: migrating model", "index", fmt.Sprintf("%d/%d", i+1, len(models)), "model", name) if err := db.AutoMigrate(item); err != nil { return fmt.Errorf("AutoMigrate %s failed: %w", name, err) @@ -135,6 +147,16 @@ func autoMigrateAll(db *gorm.DB) error { return nil } +func tableNameForModel(item any) (string, error) { + namer := schema.NamingStrategy{} + cache := &sync.Map{} + parsed, err := schema.Parse(item, cache, namer) + if err != nil { + return "", err + } + return parsed.Table, nil +} + func isDatabaseEmpty(db *gorm.DB) (bool, error) { models, err := buildDBModels() if err != nil { diff --git a/openflare_server/model/main_test.go b/openflare_server/model/main_test.go index bad9ec0e..cf337731 100644 --- a/openflare_server/model/main_test.go +++ b/openflare_server/model/main_test.go @@ -522,6 +522,76 @@ func TestEnsureDatabaseSchemaUpToDateAddsNodeIPManualOverride(t *testing.T) { } } +func TestEnsureDatabaseSchemaUpToDateV21BackfillsNodeColumnsWhenNewColumnsAlreadyExist(t *testing.T) { + db := openBareTestSQLiteDB(t, "node-v21-existing-target-columns.db") + if err := registerSharding(db, "sqlite"); err != nil { + t.Fatalf("register sharding: %v", err) + } + if err := applyCurrentSchema(db, "sqlite"); err != nil { + t.Fatalf("apply current schema: %v", err) + } + if err := ensureDefaultWAFRuleGroup(db); err != nil { + t.Fatalf("ensure default waf rule group: %v", err) + } + for _, stmt := range []string{ + `ALTER TABLE nodes ADD COLUMN agent_token text`, + `ALTER TABLE nodes ADD COLUMN agent_version text`, + `ALTER TABLE nodes ADD COLUMN nginx_version text`, + `ALTER TABLE nodes ADD COLUMN relay_version text`, + `ALTER TABLE nodes ADD COLUMN relay_frp_version text`, + `ALTER TABLE nodes ADD COLUMN relay_frps_connections integer`, + `ALTER TABLE nodes ADD COLUMN relay_frps_proxy_count integer`, + } { + if err := db.Exec(stmt).Error; err != nil { + t.Fatalf("prepare legacy node column with %q: %v", stmt, err) + } + } + now := time.Now() + if err := db.Exec(` + INSERT INTO nodes ( + node_id, name, ip, access_token, version, ext_version, + agent_token, agent_version, nginx_version, + status, last_seen_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, "node-v21", "Node v21", "127.0.0.1", "", "", "", "legacy-token", "v2.0.0", "openresty/1.25.3", "offline", now, now, now).Error; err != nil { + t.Fatalf("seed node with legacy columns: %v", err) + } + if err := saveDatabaseSchemaVersion(db, 20); err != nil { + t.Fatalf("save schema version: %v", err) + } + + if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil { + t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err) + } + + var node Node + if err := db.Where("node_id = ?", "node-v21").First(&node).Error; err != nil { + t.Fatalf("query migrated node: %v", err) + } + if node.AccessToken != "legacy-token" { + t.Fatalf("unexpected access_token: got %q", node.AccessToken) + } + if node.Version != "v2.0.0" { + t.Fatalf("unexpected version: got %q", node.Version) + } + if node.ExtVersion != "openresty/1.25.3" { + t.Fatalf("unexpected ext_version: got %q", node.ExtVersion) + } + if !db.Migrator().HasColumn(&Node{}, "agent_token") { + t.Fatal("expected migration to keep legacy nodes.agent_token column") + } + version, exists, err := loadDatabaseSchemaVersion(db) + if err != nil { + t.Fatalf("loadDatabaseSchemaVersion: %v", err) + } + if !exists { + t.Fatal("expected schema version record to exist") + } + if version != currentDatabaseSchemaVersion { + t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion) + } +} + func TestAllRegisteredMigrationsHaveValidationDefined(t *testing.T) { ctx := databaseSchemaMigrationContext{} for _, migration := range databaseSchemaMigrations() { diff --git a/openflare_server/model/migrate/migration.go b/openflare_server/model/migrate/migration.go index c333f685..e6a628d2 100644 --- a/openflare_server/model/migrate/migration.go +++ b/openflare_server/model/migrate/migration.go @@ -10,6 +10,7 @@ const BaseDatabaseSchemaVersion = 7 type Context interface { ApplyCurrentSchema(db *gorm.DB, backend string) error + ApplyCurrentSchemaExcept(db *gorm.DB, backend string, excludedTables ...string) error BackfillOriginsFromProxyRoutes(db *gorm.DB) error BackfillProxyRouteSiteFields(db *gorm.DB) error EnsureProxyRouteSiteNameUniqueIndex(db *gorm.DB) error diff --git a/openflare_server/model/migrate/v21.go b/openflare_server/model/migrate/v21.go index 6a7c6328..b7c43c4c 100644 --- a/openflare_server/model/migrate/v21.go +++ b/openflare_server/model/migrate/v21.go @@ -31,6 +31,10 @@ func migrateV21(ctx Context, db *gorm.DB, backend string) error { slog.Info("starting v21 database migration (Node Optimization & Observation Split)") migrator := db.Migrator() + dropLegacyNodeColumn := func(col string) error { + slog.Info("v21: keeping legacy nodes column to avoid lock-heavy schema rewrite", "backend", backend, "column", col) + return nil + } // Rename agent_token → access_token (target may already exist if AutoMigrate ran earlier) if migrator.HasColumn(&nodeV21{}, "agent_token") { @@ -39,7 +43,7 @@ func migrateV21(ctx Context, db *gorm.DB, backend string) error { if err := db.Exec(`UPDATE nodes SET access_token = agent_token WHERE access_token IS NULL OR access_token = ''`).Error; err != nil { return fmt.Errorf("failed to backfill access_token from agent_token: %w", err) } - if err := migrator.DropColumn(&nodeV21{}, "agent_token"); err != nil { + if err := dropLegacyNodeColumn("agent_token"); err != nil { slog.Warn("failed to drop agent_token after backfill", "error", err) } } else { @@ -57,7 +61,7 @@ func migrateV21(ctx Context, db *gorm.DB, backend string) error { if err := db.Exec(`UPDATE nodes SET version = agent_version WHERE version = ''`).Error; err != nil { return fmt.Errorf("failed to backfill version from agent_version: %w", err) } - if err := migrator.DropColumn(&nodeV21{}, "agent_version"); err != nil { + if err := dropLegacyNodeColumn("agent_version"); err != nil { slog.Warn("failed to drop agent_version after backfill", "error", err) } } else { @@ -74,7 +78,7 @@ func migrateV21(ctx Context, db *gorm.DB, backend string) error { if err := db.Exec(`UPDATE nodes SET ext_version = nginx_version WHERE ext_version IS NULL OR ext_version = ''`).Error; err != nil { return fmt.Errorf("failed to backfill ext_version from nginx_version: %w", err) } - if err := migrator.DropColumn(&nodeV21{}, "nginx_version"); err != nil { + if err := dropLegacyNodeColumn("nginx_version"); err != nil { slog.Warn("failed to drop nginx_version after backfill", "error", err) } } else { @@ -94,13 +98,13 @@ func migrateV21(ctx Context, db *gorm.DB, backend string) error { for _, col := range columnsToDrop { if migrator.HasColumn(&nodeV21{}, col) { - if err := migrator.DropColumn(&nodeV21{}, col); err != nil { + if err := dropLegacyNodeColumn(col); err != nil { slog.Warn("failed to drop column in v21 migration", "column", col, "error", err) } } } - if err := ctx.ApplyCurrentSchema(db, backend); err != nil { + if err := ctx.ApplyCurrentSchemaExcept(db, backend, "nodes"); err != nil { return err } diff --git a/openflare_server/model/migrations.go b/openflare_server/model/migrations.go index 4f9b6949..1ddf5b72 100644 --- a/openflare_server/model/migrations.go +++ b/openflare_server/model/migrations.go @@ -28,6 +28,10 @@ func (databaseSchemaMigrationContext) ApplyCurrentSchema(db *gorm.DB, backend st return applyCurrentSchema(db, backend) } +func (databaseSchemaMigrationContext) ApplyCurrentSchemaExcept(db *gorm.DB, backend string, excludedTables ...string) error { + return applyCurrentSchemaExcept(db, backend, excludedTables...) +} + func (databaseSchemaMigrationContext) BackfillOriginsFromProxyRoutes(db *gorm.DB) error { return backfillOriginsFromProxyRoutes(db) } @@ -184,6 +188,16 @@ func migrateObservabilityLegacyColumns(db *gorm.DB) error { } func applyCurrentSchema(db *gorm.DB, backend string) error { + return applyCurrentSchemaExcept(db, backend) +} + +func applyCurrentSchemaExcept(db *gorm.DB, backend string, excludedTables ...string) error { + excluded := make(map[string]bool, len(excludedTables)) + for _, table := range excludedTables { + if table != "" { + excluded[table] = true + } + } slog.Info("applyCurrentSchema: step 1/5 - auto migrate schema metadata") if err := autoMigrateSchemaMetadata(db); err != nil { return err @@ -193,7 +207,7 @@ func applyCurrentSchema(db *gorm.DB, backend string) error { return err } slog.Info("applyCurrentSchema: step 3/5 - auto migrate all models") - if err := autoMigrateAll(db); err != nil { + if err := autoMigrateAllExcept(db, excluded); err != nil { return err } slog.Info("applyCurrentSchema: step 4/5 - migrate text columns")