[优化] 重构数据库迁移逻辑,整合遗留和Goose迁移处理

This commit is contained in:
ryan
2026-06-02 22:27:04 +08:00
parent dfd040a9de
commit 4ca6f2957b
9 changed files with 508 additions and 71 deletions
@@ -6,11 +6,14 @@ import (
)
const (
legacyDatabaseSchemaVersion = migrate.BaseDatabaseSchemaVersion
databaseSchemaVersionRowID = 1
legacyDatabaseSchemaVersion = migrate.BaseDatabaseSchemaVersion
legacyMigrationTerminalVersion = 17
databaseSchemaVersionRowID = 1
)
var currentDatabaseSchemaVersion = migrate.CurrentVersion()
// currentDatabaseSchemaVersion tracks the current physical schema validated by the
// legacy validator set. Goose owns only post-v17 migrations, and none exist yet.
var currentDatabaseSchemaVersion = legacyMigrationTerminalVersion
type DatabaseSchemaVersion struct {
ID uint `json:"id" gorm:"primaryKey"`
+194
View File
@@ -0,0 +1,194 @@
package model
import (
"database/sql"
"errors"
"fmt"
"log/slog"
"gorm.io/gorm"
)
type schemaMigrationState int
const (
schemaMigrationStateFresh schemaMigrationState = iota
schemaMigrationStateLegacyOnly
schemaMigrationStateGooseOnly
schemaMigrationStateLegacyBootstrap
schemaMigrationStateMixed
)
func detectLegacySchemaState(db *gorm.DB) (schemaMigrationState, error) {
hasLegacyTable := db.Migrator().HasTable(&DatabaseSchemaVersion{})
hasGooseTable := db.Migrator().HasTable("goose_db_version")
switch {
case hasLegacyTable && hasGooseTable:
return schemaMigrationStateMixed, nil
case hasLegacyTable:
return schemaMigrationStateLegacyOnly, nil
case hasGooseTable:
return schemaMigrationStateGooseOnly, nil
}
empty, err := isDatabaseEmpty(db)
if err != nil {
return 0, err
}
if empty {
return schemaMigrationStateFresh, nil
}
return schemaMigrationStateLegacyBootstrap, nil
}
func loadGooseDatabaseVersion(db *gorm.DB) (int, bool, error) {
if db == nil || !db.Migrator().HasTable("goose_db_version") {
return 0, false, nil
}
var version int64
err := db.Table("goose_db_version").
Where("is_applied = ?", true).
Order("version_id DESC").
Select("version_id").
Limit(1).
Row().
Scan(&version)
if errors.Is(err, sql.ErrNoRows) {
return 0, false, nil
}
if err != nil {
return 0, false, err
}
return int(version), true, nil
}
func bootstrapLegacySchemaVersion(db *gorm.DB) error {
if err := autoMigrateLegacySchemaMetadata(db); err != nil {
return err
}
version, exists, err := loadLegacyDatabaseSchemaVersion(db)
if err != nil {
return err
}
if exists {
if version > legacyMigrationTerminalVersion {
return fmt.Errorf("legacy schema version %d is newer than supported terminal version %d", version, legacyMigrationTerminalVersion)
}
return nil
}
return saveLegacyDatabaseSchemaVersion(db, legacyDatabaseSchemaVersion)
}
func upgradeLegacyToLegacyTerminal(db *gorm.DB, backend string) error {
if err := bootstrapLegacySchemaVersion(db); err != nil {
return err
}
version, exists, err := loadLegacyDatabaseSchemaVersion(db)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("legacy schema version record is missing after bootstrap")
}
return upgradeLegacyDatabaseSchema(db, backend, version)
}
func validateGooseBridgeState(db *gorm.DB) error {
version, exists, err := loadGooseDatabaseVersion(db)
if err != nil {
return err
}
if !exists {
return nil
}
if version < legacyMigrationTerminalVersion {
return fmt.Errorf("goose schema version %d is below legacy bridge baseline %d", version, legacyMigrationTerminalVersion)
}
if int64(version) > currentGooseTargetVersion() {
return fmt.Errorf("goose schema version %d is newer than application target version %d", version, currentGooseTargetVersion())
}
return nil
}
func finalizeLegacyToGooseBridge(db *gorm.DB) error {
gooseVersion, exists, err := loadGooseDatabaseVersion(db)
if err != nil {
return err
}
if !exists || gooseVersion < legacyMigrationTerminalVersion {
return nil
}
if !db.Migrator().HasTable(&DatabaseSchemaVersion{}) {
return nil
}
if err := db.Migrator().DropTable(&DatabaseSchemaVersion{}); err != nil {
return fmt.Errorf("drop legacy schema versions table failed: %w", err)
}
slog.Info("completed legacy-to-goose migration bridge", "goose_version", gooseVersion)
return nil
}
func repairCurrentSchemaState(db *gorm.DB, backend string) error {
if err := dropLegacyNodeColumns(db, backend); err != nil {
return err
}
if err := ensureDefaultGitHubAuthSource(db); err != nil {
return err
}
if err := ensureDefaultWAFRuleGroup(db); err != nil {
return err
}
return nil
}
func ensureDatabaseSchemaUpToDate(db *gorm.DB, backend string) error {
state, err := detectLegacySchemaState(db)
if err != nil {
return err
}
switch state {
case schemaMigrationStateFresh:
if err := initializeFreshDatabaseSchema(db, backend); err != nil {
return err
}
case schemaMigrationStateLegacyOnly:
if err := upgradeLegacyToLegacyTerminal(db, backend); err != nil {
return err
}
case schemaMigrationStateGooseOnly:
if err := validateGooseBridgeState(db); err != nil {
return err
}
case schemaMigrationStateLegacyBootstrap:
if err := upgradeLegacyToLegacyTerminal(db, backend); err != nil {
return err
}
case schemaMigrationStateMixed:
legacyVersion, exists, err := loadLegacyDatabaseSchemaVersion(db)
if err != nil {
return err
}
if exists && legacyVersion != legacyMigrationTerminalVersion {
return fmt.Errorf("incomplete mixed migration state: legacy schema version %d does not match bridge terminal version %d", legacyVersion, legacyMigrationTerminalVersion)
}
if err := validateGooseBridgeState(db); err != nil {
return err
}
default:
return fmt.Errorf("unknown schema migration state: %d", state)
}
if err := runGooseMigrations(db, backend); err != nil {
return err
}
if err := finalizeLegacyToGooseBridge(db); err != nil {
return err
}
if err := repairCurrentSchemaState(db, backend); err != nil {
return err
}
return validateCurrentDatabaseSchema(db, backend)
}
@@ -0,0 +1,62 @@
package model
import (
"context"
"database/sql"
"fmt"
"github.com/pressly/goose/v3"
"gorm.io/gorm"
)
func newGooseBaselineMigration() *goose.Migration {
migration := goose.NewGoMigration(int64(legacyMigrationTerminalVersion), nil, nil)
migration.Source = fmt.Sprintf("%05d_legacy_terminal_baseline.go", legacyMigrationTerminalVersion)
return migration
}
func newGooseGORMMigration(
version int64,
source string,
backend string,
up func(ctx databaseSchemaMigrationContext, db *gorm.DB, backend string) error,
) *goose.Migration {
migration := goose.NewGoMigration(version, &goose.GoFunc{
RunDB: func(_ context.Context, sqlDB *sql.DB) error {
gormDB, err := getGORMDBFromSQLDB(sqlDB, backend)
if err != nil {
return err
}
ctx := databaseSchemaMigrationContext{}
if backend == "postgres" {
return gormDB.Transaction(func(tx *gorm.DB) error {
return up(ctx, tx, backend)
})
}
return up(ctx, gormDB, backend)
},
}, nil)
migration.Source = source
return migration
}
func registeredGooseMigrations(backend string) []*goose.Migration {
_ = backend
return nil
}
func buildGooseMigrations(backend string) []*goose.Migration {
migrations := []*goose.Migration{newGooseBaselineMigration()}
migrations = append(migrations, registeredGooseMigrations(backend)...)
return migrations
}
func currentGooseTargetVersion() int64 {
var maxVersion int64 = legacyMigrationTerminalVersion
for _, migration := range buildGooseMigrations("sqlite") {
if migration.Version > maxVersion {
maxVersion = migration.Version
}
}
return maxVersion
}
+76
View File
@@ -0,0 +1,76 @@
package model
import (
"context"
"database/sql"
"fmt"
"github.com/glebarez/sqlite"
"github.com/pressly/goose/v3"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/schema"
)
func gooseDialectForBackend(backend string) (goose.Dialect, error) {
switch backend {
case "postgres":
return goose.DialectPostgres, nil
case "sqlite":
return goose.DialectSQLite3, nil
default:
return "", fmt.Errorf("unsupported database backend: %s", backend)
}
}
func getGORMDBFromSQLDB(db *sql.DB, backend string) (*gorm.DB, error) {
var dialector gorm.Dialector
switch backend {
case "postgres":
dialector = postgres.New(postgres.Config{Conn: db})
case "sqlite":
dialector = &sqlite.Dialector{Conn: db}
default:
return nil, fmt.Errorf("unsupported database backend: %s", backend)
}
gormDB, err := gorm.Open(dialector, &gorm.Config{
NamingStrategy: schema.NamingStrategy{},
})
if err != nil {
return nil, err
}
if err := registerSharding(gormDB, backend); err != nil {
return nil, err
}
return gormDB, nil
}
func buildGooseProvider(db *gorm.DB, backend string) (*goose.Provider, error) {
sqlDB, err := db.DB()
if err != nil {
return nil, err
}
dialect, err := gooseDialectForBackend(backend)
if err != nil {
return nil, err
}
return goose.NewProvider(
dialect,
sqlDB,
nil,
goose.WithDisableGlobalRegistry(true),
goose.WithGoMigrations(buildGooseMigrations(backend)...),
)
}
func runGooseMigrations(db *gorm.DB, backend string) error {
provider, err := buildGooseProvider(db, backend)
if err != nil {
return fmt.Errorf("build goose provider: %w", err)
}
if _, err := provider.Up(context.Background()); err != nil {
return fmt.Errorf("goose up failed: %w", err)
}
return nil
}
+12 -1
View File
@@ -54,12 +54,23 @@ func registeredModels() []any {
}
}
func schemaMetadataModels() []any {
func currentSchemaMetadataModels() []any {
return nil
}
func legacySchemaMetadataModels() []any {
return []any{
&DatabaseSchemaVersion{},
}
}
func schemaMetadataModels() []any {
models := make([]any, 0, len(currentSchemaMetadataModels())+len(legacySchemaMetadataModels()))
models = append(models, currentSchemaMetadataModels()...)
models = append(models, legacySchemaMetadataModels()...)
return models
}
func buildDBModels() ([]dbModel, error) {
models := registeredModels()
result := make([]dbModel, 0, len(models))
+55 -5
View File
@@ -195,8 +195,8 @@ func TestUpgradeDatabaseSchemaV15ToV16AppliesCompressedReleaseSchema(t *testing.
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
if err := autoMigrateLegacySchemaMetadata(db); err != nil {
t.Fatalf("auto migrate legacy schema metadata: %v", err)
}
if err := applyCurrentSchema(db, "sqlite"); err != nil {
t.Fatalf("apply current schema: %v", err)
@@ -306,6 +306,12 @@ func TestEnsureDatabaseSchemaUpToDateInitializesFreshDatabase(t *testing.T) {
if version != currentDatabaseSchemaVersion {
t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion)
}
if db.Migrator().HasTable(&DatabaseSchemaVersion{}) {
t.Fatal("expected fresh database to avoid legacy database_schema_versions table")
}
if !db.Migrator().HasTable("goose_db_version") {
t.Fatal("expected fresh database to initialize goose_db_version")
}
}
func TestEnsureDatabaseSchemaUpToDateUpgradesLegacyDatabase(t *testing.T) {
@@ -340,6 +346,12 @@ func TestEnsureDatabaseSchemaUpToDateUpgradesLegacyDatabase(t *testing.T) {
if version != currentDatabaseSchemaVersion {
t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion)
}
if db.Migrator().HasTable(&DatabaseSchemaVersion{}) {
t.Fatal("expected legacy database_schema_versions table to be removed after bridging to goose")
}
if !db.Migrator().HasTable("goose_db_version") {
t.Fatal("expected legacy upgrade to initialize goose_db_version")
}
}
func TestMigrateOriginsSchemaBackfillsOrigins(t *testing.T) {
@@ -402,8 +414,8 @@ func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteDomainCertificateFields(t *te
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
if err := autoMigrateLegacySchemaMetadata(db); err != nil {
t.Fatalf("auto migrate legacy schema metadata: %v", err)
}
for _, item := range registeredModels() {
@@ -478,7 +490,7 @@ func TestRunDatabaseSchemaMigrationDoesNotAdvanceVersionWhenValidationFails(t *t
fromVersion: legacyDatabaseSchemaVersion,
toVersion: 11,
migrate: func(tx *gorm.DB, backend string) error {
return autoMigrateSchemaMetadata(tx)
return autoMigrateLegacySchemaMetadata(tx)
},
validate: func(tx *gorm.DB, backend string) error {
return gorm.ErrInvalidDB
@@ -655,6 +667,44 @@ func TestEnsureDatabaseSchemaUpToDateV16DropsLegacyNodeColumnsWhenAlreadyCurrent
t.Fatalf("expected current-schema cleanup to drop legacy nodes.%s column", column)
}
}
if db.Migrator().HasTable(&DatabaseSchemaVersion{}) {
t.Fatal("expected current-schema legacy version table to be removed after goose bridge")
}
if !db.Migrator().HasTable("goose_db_version") {
t.Fatal("expected current-schema goose_db_version table to exist")
}
}
func TestEnsureDatabaseSchemaUpToDateKeepsGooseOnlyDatabaseOnReentry(t *testing.T) {
db := openBareTestSQLiteDB(t, "goose-only-reentry.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("first ensureDatabaseSchemaUpToDate: %v", err)
}
if db.Migrator().HasTable(&DatabaseSchemaVersion{}) {
t.Fatal("expected first initialization to avoid legacy table")
}
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("second ensureDatabaseSchemaUpToDate: %v", err)
}
if db.Migrator().HasTable(&DatabaseSchemaVersion{}) {
t.Fatal("expected goose-only database to remain free of legacy version table")
}
version, exists, err := loadGooseDatabaseVersion(db)
if err != nil {
t.Fatalf("loadGooseDatabaseVersion: %v", err)
}
if !exists {
t.Fatal("expected goose-only database to keep goose version record")
}
if version != currentDatabaseSchemaVersion {
t.Fatalf("unexpected goose version: got %d want %d", version, currentDatabaseSchemaVersion)
}
}
func TestAllRegisteredMigrationsHaveValidationDefined(t *testing.T) {
+44 -40
View File
@@ -93,8 +93,17 @@ func (databaseSchemaMigrationContext) ValidateDatabaseSchemaVersion(db *gorm.DB,
}
}
func autoMigrateSchemaMetadata(db *gorm.DB) error {
for _, item := range schemaMetadataModels() {
func autoMigrateCurrentSchemaMetadata(db *gorm.DB) error {
for _, item := range currentSchemaMetadataModels() {
if err := db.AutoMigrate(item); err != nil {
return err
}
}
return nil
}
func autoMigrateLegacySchemaMetadata(db *gorm.DB) error {
for _, item := range legacySchemaMetadataModels() {
if err := db.AutoMigrate(item); err != nil {
return err
}
@@ -245,7 +254,7 @@ func applyCurrentSchemaExcept(db *gorm.DB, backend string, excludedTables ...str
}
}
slog.Info("applyCurrentSchema: step 1/5 - auto migrate schema metadata")
if err := autoMigrateSchemaMetadata(db); err != nil {
if err := autoMigrateCurrentSchemaMetadata(db); err != nil {
return err
}
slog.Info("applyCurrentSchema: step 2/5 - migrate proxy route https column")
@@ -268,7 +277,7 @@ func applyCurrentSchemaExcept(db *gorm.DB, backend string, excludedTables ...str
return nil
}
func loadDatabaseSchemaVersion(db *gorm.DB) (int, bool, error) {
func loadLegacyDatabaseSchemaVersion(db *gorm.DB) (int, bool, error) {
if db == nil {
return 0, false, nil
}
@@ -286,19 +295,37 @@ func loadDatabaseSchemaVersion(db *gorm.DB) (int, bool, error) {
return state.Version, true, nil
}
func saveDatabaseSchemaVersion(db *gorm.DB, version int) error {
func saveLegacyDatabaseSchemaVersion(db *gorm.DB, version int) error {
if err := autoMigrateLegacySchemaMetadata(db); err != nil {
return err
}
return db.Save(&DatabaseSchemaVersion{
ID: databaseSchemaVersionRowID,
Version: version,
}).Error
}
func loadDatabaseSchemaVersion(db *gorm.DB) (int, bool, error) {
version, exists, err := loadGooseDatabaseVersion(db)
if err != nil {
return 0, false, err
}
if exists {
return version, true, nil
}
return loadLegacyDatabaseSchemaVersion(db)
}
func saveDatabaseSchemaVersion(db *gorm.DB, version int) error {
return saveLegacyDatabaseSchemaVersion(db, version)
}
func validateDatabaseSchemaV2(db *gorm.DB, backend string) error {
if db == nil {
return fmt.Errorf("database handle is nil")
}
if !db.Migrator().HasTable(&DatabaseSchemaVersion{}) {
return fmt.Errorf("table %s is missing", (&DatabaseSchemaVersion{}).TableName())
if !db.Migrator().HasTable(&DatabaseSchemaVersion{}) && !db.Migrator().HasTable("goose_db_version") {
return fmt.Errorf("neither %s nor goose_db_version exists", (&DatabaseSchemaVersion{}).TableName())
}
models, err := buildDBModels()
if err != nil {
@@ -1385,7 +1412,7 @@ func runDatabaseSchemaMigration(db *gorm.DB, backend string, migration databaseS
if err := migration.validate(db, backend); err != nil {
return fmt.Errorf("validate database schema v%d failed: %w", migration.toVersion, err)
}
if err := saveDatabaseSchemaVersion(db, migration.toVersion); err != nil {
if err := saveLegacyDatabaseSchemaVersion(db, migration.toVersion); err != nil {
return fmt.Errorf("persist database schema version v%d failed: %w", migration.toVersion, err)
}
return nil
@@ -1398,26 +1425,26 @@ func runDatabaseSchemaMigration(db *gorm.DB, backend string, migration databaseS
if err := migration.validate(tx, backend); err != nil {
return fmt.Errorf("validate database schema v%d failed: %w", migration.toVersion, err)
}
if err := saveDatabaseSchemaVersion(tx, migration.toVersion); err != nil {
if err := saveLegacyDatabaseSchemaVersion(tx, migration.toVersion); err != nil {
return fmt.Errorf("persist database schema version v%d failed: %w", migration.toVersion, err)
}
return nil
})
}
func upgradeDatabaseSchema(db *gorm.DB, backend string, version int) error {
if version > currentDatabaseSchemaVersion {
return fmt.Errorf("database schema version %d is newer than application version %d", version, currentDatabaseSchemaVersion)
func upgradeLegacyDatabaseSchema(db *gorm.DB, backend string, version int) error {
if version > legacyMigrationTerminalVersion {
return fmt.Errorf("database schema version %d is newer than legacy migration terminal version %d", version, legacyMigrationTerminalVersion)
}
if version < legacyDatabaseSchemaVersion {
slog.Warn("database schema version is below supported baseline; treating it as historical initial schema", "version", version, "baseline", legacyDatabaseSchemaVersion)
version = legacyDatabaseSchemaVersion
}
if version == currentDatabaseSchemaVersion {
if version == legacyMigrationTerminalVersion {
return nil
}
migrationMap := databaseSchemaMigrationMap()
for version < currentDatabaseSchemaVersion {
for version < legacyMigrationTerminalVersion {
migration, ok := migrationMap[version]
if !ok {
return fmt.Errorf("database schema migration from v%d is not defined", version)
@@ -1458,32 +1485,9 @@ func initializeFreshDatabaseSchema(db *gorm.DB, backend string) error {
if err := ensureDefaultWAFRuleGroup(db); err != nil {
return err
}
if err := validateCurrentDatabaseSchema(db, backend); err != nil {
return err
}
return saveDatabaseSchemaVersion(db, currentDatabaseSchemaVersion)
return nil
}
func ensureDatabaseSchemaUpToDate(db *gorm.DB, backend string) error {
version, exists, err := loadDatabaseSchemaVersion(db)
if err != nil {
return err
}
if exists {
if err := upgradeDatabaseSchema(db, backend, version); err != nil {
return err
}
return dropLegacyNodeColumns(db, backend)
}
empty, err := isDatabaseEmpty(db)
if err != nil {
return err
}
if empty {
return initializeFreshDatabaseSchema(db, backend)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
return err
}
return upgradeDatabaseSchema(db, backend, legacyDatabaseSchemaVersion)
func upgradeDatabaseSchema(db *gorm.DB, backend string, version int) error {
return upgradeLegacyDatabaseSchema(db, backend, version)
}