From ae72f2da9adfccab40ce42ef771d47f6b4eb8809 Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 19 Mar 2026 16:45:22 +0800 Subject: [PATCH] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E7=89=88=E6=9C=AC=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E4=B8=8E=E8=BF=81=E7=A7=BB=E9=80=BB=E8=BE=91=EF=BC=8C=E7=A1=AE?= =?UTF-8?q?=E4=BF=9D=E6=95=B0=E6=8D=AE=E5=BA=93=E7=BB=93=E6=9E=84=E4=B8=8E?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E4=B8=80=E8=87=B4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- docs/development-guidelines.md | 12 + .../model/database_schema_version.go | 19 ++ openflare_server/model/main.go | 227 ++++++++++++++++-- openflare_server/model/main_test.go | 109 +++++++-- 5 files changed, 339 insertions(+), 31 deletions(-) create mode 100644 openflare_server/model/database_schema_version.go diff --git a/.gitignore b/.gitignore index 715ad0e0..ed495b7c 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,5 @@ go.work.sum *.log .DS_Store -.codex-cache \ No newline at end of file +.codex-cache +/.gomodcache/ diff --git a/docs/development-guidelines.md b/docs/development-guidelines.md index 96dac923..1f6fba09 100644 --- a/docs/development-guidelines.md +++ b/docs/development-guidelines.md @@ -126,6 +126,18 @@ * 快照与聚合结果采用追加式模型，不覆盖历史 * 原始访问明细必须有受控保留策略 +### 3.1 数据库版本与迁移 + +* 任何涉及表结构、索引、列类型、分表规则或内部持久化元数据的修改，都必须同步提升数据库版本号 +* 数据库版本号定义在 `openflare_server/model`，不得只依赖 `AutoMigrate` 隐式升级存量数据库 +* 每次提升数据库版本号时，必须补充从上一版本升级到新版本的显式迁移方法 +* 迁移方法必须包含升级后的校验逻辑；只有校验通过，才能写入新的数据库版本记录 +* 新包启动后必须先检查数据库当前版本，再按顺序逐步升级到目标版本；禁止跳过中间升级步骤直接写目标版本 +* 空库初始化可以直接建立当前版本结构，但初始化完成后仍必须执行同版本校验，并落库当前数据库版本 +* 数据库版本元数据属于内部控制信息，必须保存在独立内部表中，不能混入业务配置表 +* 如果迁移失败或校验失败，启动流程必须中止，且不得提升数据库版本记录 +* 涉及数据库版本变更的提交，必须补充对应的迁移测试或等效回归测试 + ## 4. API 与鉴权规范 ### 4.1 API diff --git a/openflare_server/model/database_schema_version.go b/openflare_server/model/database_schema_version.go new file mode 100644 index 00000000..c8738253 --- /dev/null +++ b/openflare_server/model/database_schema_version.go @@ -0,0 +1,19 @@ +package model + +import "time" + +const ( + legacyDatabaseSchemaVersion = 1 + currentDatabaseSchemaVersion = 2 + databaseSchemaVersionRowID = 1 +) + +type DatabaseSchemaVersion struct { + ID uint `json:"id" gorm:"primaryKey"` + Version int `json:"version" gorm:"not null"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (DatabaseSchemaVersion) TableName() string { + return "database_schema_versions" +} diff --git a/openflare_server/model/main.go b/openflare_server/model/main.go index 464d26e2..c1307415 100644 --- a/openflare_server/model/main.go +++ b/openflare_server/model/main.go @@ -2,6 +2,7 @@ package model import ( "encoding/json" + "errors" "fmt" "github.com/glebarez/sqlite" "gorm.io/driver/postgres" @@ -23,6 +24,13 @@ type dbModel struct { hasIDPK bool } +type databaseSchemaMigration struct { + fromVersion int + toVersion int + migrate func(db *gorm.DB, backend string) error + validate func(db *gorm.DB, backend string) error +} + func registeredModels() []any { return []any{ &File{}, @@ -42,6 +50,12 @@ func registeredModels() []any { } } +func schemaMetadataModels() []any { + return []any{ + &DatabaseSchemaVersion{}, + } +} + func buildDBModels() ([]dbModel, error) { models := registeredModels() result := make([]dbModel, 0, len(models)) @@ -123,6 +137,15 @@ func autoMigrateAll(db *gorm.DB) error { return nil } +func autoMigrateSchemaMetadata(db *gorm.DB) error { + for _, item := range schemaMetadataModels() { + if err := db.AutoMigrate(item); err != nil { + return err + } + } + return nil +} + func migrateTextColumns(db *gorm.DB, backend string) error { if backend != "postgres" { return nil @@ -194,10 +217,196 @@ func migrateObservabilityLegacyColumns(db *gorm.DB) error { return nil } +func applyCurrentSchema(db *gorm.DB, backend string) error { + if err := autoMigrateSchemaMetadata(db); err != nil { + return err + } + if err := migrateProxyRouteEnableHTTPSColumn(db); err != nil { + return err + } + if err := autoMigrateAll(db); err != nil { + return err + } + if err := migrateTextColumns(db, backend); err != nil { + return err + } + if err := migrateObservabilityLegacyColumns(db); err != nil { + return err + } + return nil +} + +func loadDatabaseSchemaVersion(db *gorm.DB) (int, bool, error) { + if db == nil { + return 0, false, nil + } + if !db.Migrator().HasTable(&DatabaseSchemaVersion{}) { + return 0, false, nil + } + var state DatabaseSchemaVersion + err := db.Where("id = ?", databaseSchemaVersionRowID).First(&state).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return 0, false, nil + } + if err != nil { + return 0, false, err + } + return state.Version, true, nil +} + +func saveDatabaseSchemaVersion(db *gorm.DB, version int) error { + return db.Save(&DatabaseSchemaVersion{ + ID: databaseSchemaVersionRowID, + Version: version, + }).Error +} + +func validateDatabaseSchemaV2(db *gorm.DB, backend string) error { + if db == nil { + return fmt.Errorf("database handle is nil") + } + if !db.Migrator().HasTable(&DatabaseSchemaVersion{}) { + return fmt.Errorf("table %s is missing", (&DatabaseSchemaVersion{}).TableName()) + } + models, err := buildDBModels() + if err != nil { + return err + } + for _, item := range models { + if isShardedObservabilityTable(item.tableName) { + for _, table := range observabilityShardTables(item.tableName) { + if !db.Migrator().HasTable(table) { + return fmt.Errorf("sharded table %s is missing", table) + } + } + continue + } + if !db.Migrator().HasTable(item.value) { + return fmt.Errorf("table %s is missing", item.tableName) + } + } + if !db.Migrator().HasColumn(&NodeHealthEvent{}, "metadata_json") { + return fmt.Errorf("column node_health_events.metadata_json is missing") + } + _ = backend + return nil +} + +func databaseSchemaMigrations() []databaseSchemaMigration { + return []databaseSchemaMigration{ + { + fromVersion: 1, + toVersion: 2, + migrate: applyCurrentSchema, + validate: validateDatabaseSchemaV2, + }, + } +} + +func databaseSchemaMigrationMap() map[int]databaseSchemaMigration { + migrations := make(map[int]databaseSchemaMigration, len(databaseSchemaMigrations())) + for _, item := range databaseSchemaMigrations() { + migrations[item.fromVersion] = item + } + return migrations +} + +func runDatabaseSchemaMigration(db *gorm.DB, backend string, migration databaseSchemaMigration) error { + return db.Transaction(func(tx *gorm.DB) error { + if err := migration.migrate(tx, backend); err != nil { + return fmt.Errorf("migrate database schema from v%d to v%d failed: %w", migration.fromVersion, migration.toVersion, err) + } + if err := migration.validate(tx, backend); err != nil { + return fmt.Errorf("validate database schema v%d failed: %w", migration.toVersion, err) + } + if err := saveDatabaseSchemaVersion(tx, migration.toVersion); err != nil { + return fmt.Errorf("persist database schema version v%d failed: %w", migration.toVersion, err) + } + return nil + }) +} + +func upgradeDatabaseSchema(db *gorm.DB, backend string, version int) error { + if version > currentDatabaseSchemaVersion { + return fmt.Errorf("database schema version %d is newer than application version %d", version, currentDatabaseSchemaVersion) + } + if version == currentDatabaseSchemaVersion { + return nil + } + migrationMap := databaseSchemaMigrationMap() + for version < currentDatabaseSchemaVersion { + migration, ok := migrationMap[version] + if !ok { + return fmt.Errorf("database schema migration from v%d is not defined", version) + } + if err := runDatabaseSchemaMigration(db, backend, migration); err != nil { + return err + } + version = migration.toVersion + } + return nil +} + +func initializeFreshDatabaseSchema(db *gorm.DB, backend string) error { + if err := applyCurrentSchema(db, backend); err != nil { + return err + } + if err := migrateSQLiteDataIfNeeded(db, backend); err != nil { + return err + } + if err := validateDatabaseSchemaV2(db, backend); err != nil { + return err + } + return saveDatabaseSchemaVersion(db, currentDatabaseSchemaVersion) +} + +func ensureDatabaseSchemaUpToDate(db *gorm.DB, backend string) error { + version, exists, err := loadDatabaseSchemaVersion(db) + if err != nil { + return err + } + if exists { + return upgradeDatabaseSchema(db, backend, version) + } + empty, err := isDatabaseEmpty(db) + if err != nil { + return err + } + if empty { + return initializeFreshDatabaseSchema(db, backend) + } + if err := autoMigrateSchemaMetadata(db); err != nil { + return err + } + return upgradeDatabaseSchema(db, backend, legacyDatabaseSchemaVersion) +} + func isDatabaseEmpty(db *gorm.DB) (bool, error) { - for _, item := range registeredModels() { + models, err := buildDBModels() + if err != nil { + return false, err + } + for _, item := range models { + if isShardedObservabilityTable(item.tableName) { + for _, table := range observabilityShardTables(item.tableName) { + if !db.Migrator().HasTable(table) { + continue + } + var count int64 + if err := db.Table(table).Limit(1).Count(&count).Error; err != nil { + return false, err + } + if count > 0 { + return false, nil + } + } + continue + } + if !db.Migrator().HasTable(item.value) { + continue + } var count int64 - if err := db.Model(item).Limit(1).Count(&count).Error; err != nil { + if err := db.Model(item.value).Limit(1).Count(&count).Error; err != nil { return false, err } if count > 0 { @@ -346,19 +555,7 @@ func InitDB() (err error) { if err = registerSharding(db, backend); err != nil { return err } - if err = migrateProxyRouteEnableHTTPSColumn(db); err != nil { - return err - } - if err = autoMigrateAll(db); err != nil { - return err - } - if err = migrateTextColumns(db, backend); err != nil { - return err - } - if err = migrateObservabilityLegacyColumns(db); err != nil { - return err - } - if err = migrateSQLiteDataIfNeeded(db, backend); err != nil { + if err = ensureDatabaseSchemaUpToDate(db, backend); err != nil { return err } return createRootAccountIfNeed() diff --git a/openflare_server/model/main_test.go b/openflare_server/model/main_test.go index 5c6664c3..dd5d6041 100644 --- a/openflare_server/model/main_test.go +++ b/openflare_server/model/main_test.go @@ -10,16 +10,13 @@ import ( "gorm.io/gorm" ) -func openTestSQLiteDB(t *testing.T, name string) *gorm.DB { +func openBareTestSQLiteDB(t *testing.T, name string) *gorm.DB { t.Helper() db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), name)), &gorm.Config{}) if err != nil { t.Fatalf("open sqlite db: %v", err) } - if err := autoMigrateAll(db); err != nil { - t.Fatalf("auto migrate db: %v", err) - } sqlDB, err := db.DB() if err != nil { t.Fatalf("get sql db: %v", err) @@ -30,6 +27,16 @@ func openTestSQLiteDB(t *testing.T, name string) *gorm.DB { return db } +func openTestSQLiteDB(t *testing.T, name string) *gorm.DB { + t.Helper() + + db := openBareTestSQLiteDB(t, name) + if err := autoMigrateAll(db); err != nil { + t.Fatalf("auto migrate db: %v", err) + } + return db +} + func findDBModelByTableName(t *testing.T, tableName string) dbModel { t.Helper() @@ -125,17 +132,7 @@ func TestMigrateTableDataCopiesRows(t *testing.T) { } func TestRegisterShardingAutoMigratesShardTables(t *testing.T) { - db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "sharded.db")), &gorm.Config{}) - if err != nil { - t.Fatalf("open sqlite db: %v", err) - } - sqlDB, err := db.DB() - if err != nil { - t.Fatalf("get sql db: %v", err) - } - t.Cleanup(func() { - _ = sqlDB.Close() - }) + db := openBareTestSQLiteDB(t, "sharded.db") if err := registerSharding(db, "sqlite"); err != nil { t.Fatalf("register sharding: %v", err) } @@ -202,3 +199,85 @@ func TestMigrateObservabilityLegacyColumnsBackfillsHealthEventMetadata(t *testin t.Fatal("expected metadata_json to be backfilled") } } + +func TestEnsureDatabaseSchemaUpToDateInitializesFreshDatabase(t *testing.T) { + db := openBareTestSQLiteDB(t, "fresh-schema.db") + if err := registerSharding(db, "sqlite"); err != nil { + t.Fatalf("register sharding: %v", err) + } + + if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil { + t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err) + } + + version, exists, err := loadDatabaseSchemaVersion(db) + if err != nil { + t.Fatalf("loadDatabaseSchemaVersion: %v", err) + } + if !exists { + t.Fatal("expected database schema version to be recorded") + } + if version != currentDatabaseSchemaVersion { + t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion) + } +} + +func TestEnsureDatabaseSchemaUpToDateUpgradesLegacyDatabase(t *testing.T) { + db := openBareTestSQLiteDB(t, "legacy-schema.db") + if err := registerSharding(db, "sqlite"); err != nil { + t.Fatalf("register sharding: %v", err) + } + if err := autoMigrateAll(db); err != nil { + t.Fatalf("auto migrate db: %v", err) + } + if err := db.Create(&User{ + Username: "legacy", + Password: "secret", + DisplayName: "Legacy User", + Role: 1, + Status: 1, + }).Error; err != nil { + t.Fatalf("seed legacy user: %v", err) + } + + if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil { + t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err) + } + + version, exists, err := loadDatabaseSchemaVersion(db) + if err != nil { + t.Fatalf("loadDatabaseSchemaVersion: %v", err) + } + if !exists { + t.Fatal("expected legacy database to gain a schema version record") + } + if version != currentDatabaseSchemaVersion { + t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion) + } +} + +func TestRunDatabaseSchemaMigrationDoesNotAdvanceVersionWhenValidationFails(t *testing.T) { + db := openBareTestSQLiteDB(t, "failed-validation.db") + + err := runDatabaseSchemaMigration(db, "sqlite", databaseSchemaMigration{ + fromVersion: legacyDatabaseSchemaVersion, + toVersion: currentDatabaseSchemaVersion, + migrate: func(tx *gorm.DB, backend string) error { + return autoMigrateSchemaMetadata(tx) + }, + validate: func(tx *gorm.DB, backend string) error { + return gorm.ErrInvalidDB + }, + }) + if err == nil { + t.Fatal("expected migration validation to fail") + } + + _, exists, loadErr := loadDatabaseSchemaVersion(db) + if loadErr != nil { + t.Fatalf("loadDatabaseSchemaVersion: %v", loadErr) + } + if exists { + t.Fatal("expected schema version to remain unset after failed validation") + } +}