From 32e1a071579e0130dcd8ed192f275e16f2d0efd9 Mon Sep 17 00:00:00 2001 From: ryan Date: Sat, 6 Jun 2026 11:50:57 +0800 Subject: [PATCH] =?UTF-8?q?[=E4=BC=98=E5=8C=96]=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E5=8E=86=E5=8F=B2=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E6=9E=B6=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog/index.md | 4 +- openflare-server/model/goose/bridge.go | 84 +++++- openflare-server/model/migrations.go | 368 +++++-------------------- 3 files changed, 160 insertions(+), 296 deletions(-) diff --git a/docs/changelog/index.md b/docs/changelog/index.md index d9d5cf84..1b81fa29 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -32,9 +32,11 @@ sidebar: false - 重构 `install-agent.sh` 安装脚本与 `uninstall-agent.sh` 卸载脚本以兼容交互式导引、非交互式命令行参数及 Docker 部署/卸载参数（`--docker`/`--method docker`） - 重构 Go 包依赖结构为统一模块（Monorepo），模块命名为 `github.com/rain-kl/openflare` - 移除各子目录下独立的 `go.mod`/`go.sum` 文件，统一由根目录 `go.mod` 进行全局依赖管理与依赖版本锁定 -- 替换全仓库 Go 源文件中的内部引用路径，由本地相对路径迁移为标准 GitHub 绝对导入路径 +- 替换全仓库 Go源文件中的内部引用路径，由本地相对路径迁移为标准 GitHub 绝对导入路径 - 适配 Docker 镜像构建，所有组件镜像的 Dockerfile 调整为基于根目录的上下文编译 - 更新 GitHub release 自动化发布流水线，适配全新 monorepo 包结构与符号信息注入路径 +- 简化并重构数据库历史迁移校验逻辑，将版本 2 至 6 的中间校验函数合并到基线校验函数 `validateDatabaseSchemaV7` 中，消除冗余代码 +- 重构数据库历史迁移校验架构，引入基于 GORM 反射解析（`schema.Parse`）的通用自动表结构校验，彻底废弃老版本中大量手动编写的 `HasTable`/`HasColumn` 结构字段存在性检测代码 --- diff --git a/openflare-server/model/goose/bridge.go b/openflare-server/model/goose/bridge.go index 1866a795..de999ad1 100644 --- a/openflare-server/model/goose/bridge.go +++ b/openflare-server/model/goose/bridge.go @@ -4,7 +4,9 @@ import ( "database/sql" "errors" "fmt" + "io" "log/slog" + "os" "gorm.io/gorm" ) @@ -169,7 +171,22 @@ func ValidateRegisteredSchema(db *gorm.DB) error { return nil } -func EnsureDatabaseSchemaUpToDate(db *gorm.DB, backend string, ctx BridgeContext) error { +func EnsureDatabaseSchemaUpToDate(db *gorm.DB, backend string, ctx BridgeContext) (returnedErr error) { + if backend == "sqlite" { + backupPath, restore, err := backupSQLiteDatabase(db) + if err != nil { + slog.Warn("failed to backup sqlite database before migration", "error", err) + } else if backupPath != "" { + defer func() { + if returnedErr != nil { + restore() + } else { + os.Remove(backupPath) + } + }() + } + } + var startDesc string legacyVer, hasLegacy, _ := loadLegacyDatabaseSchemaVersion(db) gooseVer, hasGoose, _ := LoadDatabaseVersion(db) @@ -242,3 +259,68 @@ func EnsureDatabaseSchemaUpToDate(db *gorm.DB, backend string, ctx BridgeContext } return nil } + +func backupSQLiteDatabase(db *gorm.DB) (string, func(), error) { + var dbList []struct { + Seq int + Name string + File string + } + if err := db.Raw("PRAGMA database_list").Scan(&dbList).Error; err != nil { + return "", nil, err + } + var dbPath string + for _, item := range dbList { + if item.Name == "main" && item.File != "" { + dbPath = item.File + break + } + } + if dbPath == "" { + return "", nil, nil + } + + backupPath := dbPath + ".bak" + + src, err := os.Open(dbPath) + if err != nil { + return "", nil, err + } + defer src.Close() + + dst, err := os.Create(backupPath) + if err != nil { + return "", nil, err + } + defer dst.Close() + + if _, err := io.Copy(dst, src); err != nil { + return "", nil, err + } + dst.Sync() + + restoreFunc := func() { + src, err := os.Open(backupPath) + if err != nil { + slog.Error("failed to open sqlite backup for restore", "error", err) + return + } + defer src.Close() + + dst, err := os.OpenFile(dbPath, os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + slog.Error("failed to open sqlite db for restore", "error", err) + return + } + defer dst.Close() + + if _, err := io.Copy(dst, src); err != nil { + slog.Error("failed to restore sqlite backup", "error", err) + } else { + dst.Sync() + slog.Warn("restored sqlite database from backup due to migration failure") + } + } + + return backupPath, restoreFunc, nil +} diff --git a/openflare-server/model/migrations.go b/openflare-server/model/migrations.go index 4c28ace8..7aaf7bf9 100644 --- a/openflare-server/model/migrations.go +++ b/openflare-server/model/migrations.go @@ -10,10 +10,12 @@ import ( "net" "net/url" "strings" + "sync" schemamigrate "github.com/rain-kl/openflare/openflare-server/model/migrate" "gorm.io/gorm" + "gorm.io/gorm/schema" ) type databaseSchemaMigration struct { @@ -65,30 +67,64 @@ func (databaseSchemaMigrationContext) DropLegacyNodeColumns(db *gorm.DB, backend return dropLegacyNodeColumns(db, backend) } +func validateAllModelsSchema(db *gorm.DB) error { + models := registeredModels() + namer := schema.NamingStrategy{} + cache := &sync.Map{} + migrator := db.Migrator() + + for _, model := range models { + parsed, err := schema.Parse(model, cache, namer) + if err != nil { + return fmt.Errorf("parse model schema failed: %w", err) + } + + if isShardedObservabilityTable(parsed.Table) { + for _, table := range observabilityShardTables(parsed.Table) { + if !migrator.HasTable(table) { + return fmt.Errorf("sharded table %s is missing", table) + } + for _, field := range parsed.Fields { + if field.DBName != "" && !field.IgnoreMigration { + if !migrator.HasColumn(table, field.DBName) { + return fmt.Errorf("sharded column %s.%s is missing", table, field.DBName) + } + } + } + } + continue + } + + if !migrator.HasTable(model) { + return fmt.Errorf("table %s is missing", parsed.Table) + } + + for _, field := range parsed.Fields { + if field.DBName != "" && !field.IgnoreMigration { + if !migrator.HasColumn(model, field.DBName) { + return fmt.Errorf("column %s.%s is missing", parsed.Table, field.DBName) + } + } + } + } + return nil +} + func (databaseSchemaMigrationContext) ValidateDatabaseSchemaVersion(db *gorm.DB, backend string, version int) error { + if db == nil { + return fmt.Errorf("database handle is nil") + } switch version { case 7: return validateDatabaseSchemaV7(db, backend) case 8: return validateDatabaseSchemaV8(db, backend) - case 9: - return validateDatabaseSchemaV9(db, backend) - case 10: - return validateDatabaseSchemaV10(db, backend) - case 11: - return validateDatabaseSchemaV11(db, backend) - case 12: - return validateDatabaseSchemaV12(db, backend) + case 9, 10, 11, 12, 14, 15, 17: + return nil case 13: return validateDatabaseSchemaV13(db, backend) - case 14: - return validateDatabaseSchemaV14(db, backend) - case 15: - return validateDatabaseSchemaV15(db, backend) case 16: return validateDatabaseSchemaV16(db, backend) - case 17: - return validateDatabaseSchemaV17(db, backend) default: return fmt.Errorf("database schema validation for v%d is not defined", version) } @@ -321,65 +357,6 @@ func saveDatabaseSchemaVersion(db *gorm.DB, version int) error { return saveLegacyDatabaseSchemaVersion(db, version) } -func validateDatabaseSchemaV2(db *gorm.DB, backend string) error { - if db == nil { - return fmt.Errorf("database handle is nil") - } - if !db.Migrator().HasTable(&DatabaseSchemaVersion{}) && !db.Migrator().HasTable("goose_db_version") { - return fmt.Errorf("neither %s nor goose_db_version exists", (&DatabaseSchemaVersion{}).TableName()) - } - models, err := buildDBModels() - if err != nil { - return err - } - for _, item := range models { - if isShardedObservabilityTable(item.tableName) { - for _, table := range observabilityShardTables(item.tableName) { - if !db.Migrator().HasTable(table) { - return fmt.Errorf("sharded table %s is missing", table) - } - } - continue - } - if !db.Migrator().HasTable(item.value) { - return fmt.Errorf("table %s is missing", item.tableName) - } - } - if !db.Migrator().HasColumn(&NodeHealthEvent{}, "metadata_json") { - return fmt.Errorf("column node_health_events.metadata_json is missing") - } - _ = backend - return nil -} - -func validateDatabaseSchemaV3(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV2(db, backend); err != nil { - return err - } - for _, baseTable := range shardedObservabilityBaseTables() { - for _, table := range observabilityShardTables(baseTable) { - legacyTable := legacyObservabilityShardTableName(table) - if db.Migrator().HasTable(legacyTable) { - return fmt.Errorf("legacy sharded table %s still exists", legacyTable) - } - } - } - return nil -} - -func validateDatabaseSchemaV4(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV3(db, backend); err != nil { - return err - } - if !db.Migrator().HasTable(&Origin{}) { - return fmt.Errorf("table origins is missing") - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "origin_id") { - return fmt.Errorf("column proxy_routes.origin_id is missing") - } - return nil -} - func normalizeProxyRouteDomainForMigration(raw string) string { return strings.ToLower(strings.TrimSpace(raw)) } @@ -790,17 +767,18 @@ func backfillProxyRouteDomainCertificateFields(db *gorm.DB) error { return nil } -func validateDatabaseSchemaV5(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV4(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "site_name") { - return fmt.Errorf("column proxy_routes.site_name is missing") - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "domains") { - return fmt.Errorf("column proxy_routes.domains is missing") +func validateDatabaseSchemaV7(db *gorm.DB, backend string) error { + // Validate legacy sharded tables do not exist + for _, baseTable := range shardedObservabilityBaseTables() { + for _, table := range observabilityShardTables(baseTable) { + legacyTable := legacyObservabilityShardTableName(table) + if db.Migrator().HasTable(legacyTable) { + return fmt.Errorf("legacy sharded table %s still exists", legacyTable) + } + } } + // Fetch all proxy routes for data validation (site names, domains, certificates) var routes []ProxyRoute if err := db.Order("id asc").Find(&routes).Error; err != nil { return fmt.Errorf("list proxy routes for validation failed: %w", err) @@ -808,7 +786,9 @@ func validateDatabaseSchemaV5(db *gorm.DB, backend string) error { siteNames := make(map[string]uint, len(routes)) domainOwners := make(map[string]uint, len(routes)) + for _, route := range routes { + // Domains and Site Name Validation domains, err := decodeProxyRouteDomainsForMigration(route.Domains, route.Domain) if err != nil { return fmt.Errorf("proxy route %d domains are invalid: %w", route.ID, err) @@ -840,39 +820,8 @@ func validateDatabaseSchemaV5(db *gorm.DB, backend string) error { } domainOwners[domain] = route.ID } - } - return nil -} -func validateDatabaseSchemaV6(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV5(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "limit_conn_per_server") { - return fmt.Errorf("column proxy_routes.limit_conn_per_server is missing") - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "limit_conn_per_ip") { - return fmt.Errorf("column proxy_routes.limit_conn_per_ip is missing") - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "limit_rate") { - return fmt.Errorf("column proxy_routes.limit_rate is missing") - } - return nil -} - -func validateDatabaseSchemaV7(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV6(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "cert_ids") { - return fmt.Errorf("column proxy_routes.cert_ids is missing") - } - - var routes []ProxyRoute - if err := db.Order("id asc").Find(&routes).Error; err != nil { - return fmt.Errorf("list proxy routes for certificate validation failed: %w", err) - } - for _, route := range routes { + // Certificate Mapping Validation certIDs, err := decodeProxyRouteCertIDsForMigration(route.CertIDs, route.CertID) if err != nil { return fmt.Errorf("proxy route %d cert_ids are invalid: %w", route.ID, err) @@ -893,17 +842,12 @@ func validateDatabaseSchemaV7(db *gorm.DB, backend string) error { return fmt.Errorf("proxy route %d primary cert_id mirror is invalid", route.ID) } } + + _ = backend return nil } func validateDatabaseSchemaV8(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV7(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "domain_cert_ids") { - return fmt.Errorf("column proxy_routes.domain_cert_ids is missing") - } - var routes []ProxyRoute if err := db.Order("id asc").Find(&routes).Error; err != nil { return fmt.Errorf("list proxy routes for domain certificate validation failed: %w", err) @@ -941,6 +885,7 @@ func validateDatabaseSchemaV8(db *gorm.DB, backend string) error { return fmt.Errorf("proxy route %d primary cert_id mirror is invalid", route.ID) } } + _ = backend return nil } @@ -1128,70 +1073,6 @@ func ensureDefaultGitHubAuthSource(db *gorm.DB) error { return nil } -func validateDatabaseSchemaV9(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV8(db, backend); err != nil { - return err - } - hasAppliedDropPoW := false - if db.Migrator().HasTable("goose_db_version") { - var count int64 - _ = db.Table("goose_db_version"). - Where("version_id = ? AND is_applied = ?", 202606030003, true). - Count(&count).Error - if count > 0 { - hasAppliedDropPoW = true - } - } - if !hasAppliedDropPoW { - if !db.Migrator().HasColumn(&ProxyRoute{}, "pow_enabled") { - return fmt.Errorf("column proxy_routes.pow_enabled is missing") - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "pow_config") { - return fmt.Errorf("column proxy_routes.pow_config is missing") - } - } - return nil -} - -func validateDatabaseSchemaV10(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV9(db, backend); err != nil { - return err - } - if !db.Migrator().HasTable(&AuthSource{}) { - return fmt.Errorf("table auth_sources is missing") - } - if !db.Migrator().HasTable(&ExternalAccount{}) { - return fmt.Errorf("table external_accounts is missing") - } - return nil -} - -func validateDatabaseSchemaV11(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV10(db, backend); err != nil { - return err - } - if !db.Migrator().HasTable(&AcmeAccount{}) { - return fmt.Errorf("table acme_accounts is missing") - } - if !db.Migrator().HasTable(&DnsAccount{}) { - return fmt.Errorf("table dns_accounts is missing") - } - if !db.Migrator().HasColumn(&TLSCertificate{}, "provider") { - return fmt.Errorf("column tls_certificates.provider is missing") - } - return nil -} - -func validateDatabaseSchemaV12(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV11(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&ProxyRoute{}, "basic_auth_enabled") { - return fmt.Errorf("column proxy_routes.basic_auth_enabled is missing") - } - return nil -} - func ensureDefaultWAFRuleGroup(db *gorm.DB) error { if db == nil { return fmt.Errorf("database handle is nil") @@ -1230,15 +1111,6 @@ func ensureDefaultWAFRuleGroup(db *gorm.DB) error { } func validateDatabaseSchemaV13(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV12(db, backend); err != nil { - return err - } - if !db.Migrator().HasTable(&WAFRuleGroup{}) { - return fmt.Errorf("table waf_rule_groups is missing") - } - if !db.Migrator().HasTable(&WAFRuleGroupBinding{}) { - return fmt.Errorf("table waf_rule_group_bindings is missing") - } var count int64 if err := db.Model(&WAFRuleGroup{}).Where("is_global = ?", true).Count(&count).Error; err != nil { return fmt.Errorf("count global waf rule groups failed: %w", err) @@ -1246,75 +1118,16 @@ func validateDatabaseSchemaV13(db *gorm.DB, backend string) error { if count != 1 { return fmt.Errorf("expected exactly one global waf rule group, got %d", count) } - return nil -} - -func validateDatabaseSchemaV14(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV13(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&WAFRuleGroup{}, "pow_enabled") { - return fmt.Errorf("column waf_rule_groups.pow_enabled is missing") - } - if !db.Migrator().HasColumn(&WAFRuleGroup{}, "pow_config") { - return fmt.Errorf("column waf_rule_groups.pow_config is missing") - } - return nil -} - -func validateDatabaseSchemaV15(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV14(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&Node{}, "ip_manual_override") { - return fmt.Errorf("column nodes.ip_manual_override is missing") - } + _ = backend return nil } func validateDatabaseSchemaV16(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV15(db, backend); err != nil { - return err - } - if !db.Migrator().HasColumn(&Node{}, "access_token") { - return fmt.Errorf("column nodes.access_token is missing") - } - if !db.Migrator().HasColumn(&Node{}, "version") { - return fmt.Errorf("column nodes.version is missing") - } - if !db.Migrator().HasColumn(&Node{}, "ext_version") { - return fmt.Errorf("column nodes.ext_version is missing") - } - if !db.Migrator().HasColumn(&Node{}, "node_type") { - return fmt.Errorf("column nodes.node_type is missing") - } - for _, column := range []string{ - "relay_bind_port", - "relay_vhost_http_port", - "relay_auth_token", - "relay_agent_access_addr", - "relay_client_access_addr", - "relay_client_proxy_url", - "relay_status", - } { - if !db.Migrator().HasColumn(&Node{}, column) { - return fmt.Errorf("column nodes.%s is missing", column) - } - } - for _, column := range []string{ - "upstream_type", - "tunnel_node_id", - "tunnel_target_addr", - "tunnel_target_protocol", - } { - if !db.Migrator().HasColumn(&ProxyRoute{}, column) { - return fmt.Errorf("column proxy_routes.%s is missing", column) - } - } - if db.Migrator().HasTable("tunnels") { + migrator := db.Migrator() + if migrator.HasTable("tunnels") { return fmt.Errorf("table tunnels should not exist in v16") } - if db.Migrator().HasColumn(&ProxyRoute{}, "tunnel_id") { + if migrator.HasColumn(&ProxyRoute{}, "tunnel_id") { return fmt.Errorf("column proxy_routes.tunnel_id should not exist in v16") } for _, column := range []string{ @@ -1334,43 +1147,7 @@ func validateDatabaseSchemaV16(db *gorm.DB, backend string) error { return fmt.Errorf("column nodes.%s should not exist in v16", column) } } - if !db.Migrator().HasTable(&WAFIPGroup{}) { - return fmt.Errorf("table waf_ip_groups is missing") - } - if !db.Migrator().HasColumn(&WAFRuleGroup{}, "ip_whitelist_groups") { - return fmt.Errorf("column waf_rule_groups.ip_whitelist_groups is missing") - } - if !db.Migrator().HasColumn(&WAFRuleGroup{}, "ip_blacklist_groups") { - return fmt.Errorf("column waf_rule_groups.ip_blacklist_groups is missing") - } - if !db.Migrator().HasColumn(&WAFIPGroup{}, "ext_ips") { - return fmt.Errorf("column waf_ip_groups.ext_ips is missing") - } - return nil -} - -func validateDatabaseSchemaV17(db *gorm.DB, backend string) error { - if err := validateDatabaseSchemaV16(db, backend); err != nil { - return err - } - if db == nil { - return fmt.Errorf("database handle is nil") - } - - migrator := db.Migrator() - if !migrator.HasColumn(&Node{}, "relay_web_server_enabled") { - return fmt.Errorf("column nodes.relay_web_server_enabled is missing") - } - - // Validate columns on a sharded partition table - for _, shard := range []string{"node_observation_frps_00"} { - for _, column := range []string{"frps_client_count", "frps_proxies"} { - if !migrator.HasColumn(shard, column) { - return fmt.Errorf("column %s.%s is missing", shard, column) - } - } - } - + _ = backend return nil } @@ -1406,6 +1183,9 @@ func validateExternalDatabaseSchema(ctx databaseSchemaMigrationContext, db *gorm } func validateCurrentDatabaseSchema(db *gorm.DB, backend string) error { + if err := validateAllModelsSchema(db); err != nil { + return err + } return validateExternalDatabaseSchema(databaseSchemaMigrationContext{}, db, backend, currentDatabaseSchemaVersion) }