[优化] 重构数据库迁移逻辑,添加版本管理和验证功能

This commit is contained in:
ryan
2026-05-31 14:32:43 +08:00
parent 4ae6c2718f
commit f365b3d331
16 changed files with 421 additions and 864 deletions
@@ -1,13 +1,17 @@
package model
import "time"
import (
"openflare/model/migrate"
"time"
)
const (
legacyDatabaseSchemaVersion = 1
currentDatabaseSchemaVersion = 15
databaseSchemaVersionRowID = 1
legacyDatabaseSchemaVersion = migrate.BaseDatabaseSchemaVersion
databaseSchemaVersionRowID = 1
)
var currentDatabaseSchemaVersion = migrate.CurrentVersion()
type DatabaseSchemaVersion struct {
ID uint `json:"id" gorm:"primaryKey"`
Version int `json:"version" gorm:"not null"`
-443
View File
@@ -10,85 +10,6 @@ import (
"gorm.io/gorm"
)
type legacyProxyRouteV4 struct {
ID uint `gorm:"primaryKey"`
Domain string `gorm:"uniqueIndex;size:255;not null"`
OriginID *uint `gorm:"index"`
OriginURL string `gorm:"size:2048;not null"`
OriginHost string `gorm:"size:255"`
Upstreams string `gorm:"type:text;not null;default:'[]'"`
Enabled bool `gorm:"not null;default:true"`
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
CertID *uint
RedirectHTTP bool `gorm:"not null;default:false"`
CacheEnabled bool `gorm:"not null;default:false"`
CachePolicy string `gorm:"size:32;not null;default:''"`
CacheRules string `gorm:"type:text;not null;default:'[]'"`
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
Remark string `gorm:"size:255"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (legacyProxyRouteV4) TableName() string {
return "proxy_routes"
}
type legacyProxyRouteV5 struct {
ID uint `gorm:"primaryKey"`
SiteName string `gorm:"size:255;not null;default:''"`
Domain string `gorm:"uniqueIndex;size:255;not null"`
Domains string `gorm:"type:text;not null;default:'[]'"`
OriginID *uint `gorm:"index"`
OriginURL string `gorm:"size:2048;not null"`
OriginHost string `gorm:"size:255"`
Upstreams string `gorm:"type:text;not null;default:'[]'"`
Enabled bool `gorm:"not null;default:true"`
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
CertID *uint
RedirectHTTP bool `gorm:"not null;default:false"`
CacheEnabled bool `gorm:"not null;default:false"`
CachePolicy string `gorm:"size:32;not null;default:''"`
CacheRules string `gorm:"type:text;not null;default:'[]'"`
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
Remark string `gorm:"size:255"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (legacyProxyRouteV5) TableName() string {
return "proxy_routes"
}
type legacyProxyRouteV6 struct {
ID uint `gorm:"primaryKey"`
SiteName string `gorm:"size:255;not null;default:''"`
Domain string `gorm:"uniqueIndex;size:255;not null"`
Domains string `gorm:"type:text;not null;default:'[]'"`
OriginID *uint `gorm:"index"`
OriginURL string `gorm:"size:2048;not null"`
OriginHost string `gorm:"size:255"`
Upstreams string `gorm:"type:text;not null;default:'[]'"`
Enabled bool `gorm:"not null;default:true"`
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
CertID *uint
RedirectHTTP bool `gorm:"not null;default:false"`
LimitConnPerServer int `gorm:"not null;default:0"`
LimitConnPerIP int `gorm:"not null;default:0"`
LimitRate string `gorm:"size:32;not null;default:''"`
CacheEnabled bool `gorm:"not null;default:false"`
CachePolicy string `gorm:"size:32;not null;default:''"`
CacheRules string `gorm:"type:text;not null;default:'[]'"`
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
Remark string `gorm:"size:255"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (legacyProxyRouteV6) TableName() string {
return "proxy_routes"
}
type legacyProxyRouteV7 struct {
ID uint `gorm:"primaryKey"`
SiteName string `gorm:"size:255;not null;default:''"`
@@ -365,159 +286,6 @@ func TestEnsureDatabaseSchemaUpToDateUpgradesLegacyDatabase(t *testing.T) {
}
}
func TestEnsureDatabaseSchemaUpToDateMigratesObservabilityShardsToID(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-observability-shards.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateAll(db); err != nil {
t.Fatalf("auto migrate db: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
now := time.Now().UTC()
if err := db.Table("node_metric_snapshots_00").Create(&NodeMetricSnapshot{
ID: 1,
NodeID: "node-a",
CapturedAt: now.Add(-2 * time.Minute),
CPUUsagePercent: 22,
MemoryUsedBytes: 2,
MemoryTotalBytes: 8,
}).Error; err != nil {
t.Fatalf("seed metric snapshot shard 00: %v", err)
}
if err := db.Table("node_metric_snapshots_01").Create(&NodeMetricSnapshot{
ID: 1,
NodeID: "node-b",
CapturedAt: now.Add(-time.Minute),
CPUUsagePercent: 44,
MemoryUsedBytes: 4,
MemoryTotalBytes: 8,
}).Error; err != nil {
t.Fatalf("seed metric snapshot shard 01: %v", err)
}
if err := db.Table("node_request_reports_00").Create(&NodeRequestReport{
ID: 1,
NodeID: "node-a",
WindowStartedAt: now.Add(-3 * time.Minute),
WindowEndedAt: now.Add(-2 * time.Minute),
RequestCount: 12,
ErrorCount: 1,
UniqueVisitorCount: 6,
}).Error; err != nil {
t.Fatalf("seed request report shard 00: %v", err)
}
if err := db.Table("node_request_reports_01").Create(&NodeRequestReport{
ID: 1,
NodeID: "node-b",
WindowStartedAt: now.Add(-2 * time.Minute),
WindowEndedAt: now.Add(-time.Minute),
RequestCount: 21,
ErrorCount: 2,
UniqueVisitorCount: 9,
}).Error; err != nil {
t.Fatalf("seed request report shard 01: %v", err)
}
if err := db.Table("node_access_logs_00").Create(&NodeAccessLog{
ID: 1,
NodeID: "node-a",
LoggedAt: now.Add(-90 * time.Second),
RemoteAddr: "203.0.113.10",
Host: "a.example.com",
Path: "/alpha",
StatusCode: 200,
}).Error; err != nil {
t.Fatalf("seed access log shard 00: %v", err)
}
if err := db.Table("node_access_logs_01").Create(&NodeAccessLog{
ID: 1,
NodeID: "node-b",
LoggedAt: now.Add(-60 * time.Second),
RemoteAddr: "203.0.113.11",
Host: "b.example.com",
Path: "/beta",
StatusCode: 502,
}).Error; err != nil {
t.Fatalf("seed access log shard 01: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 2); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
version, exists, err := loadDatabaseSchemaVersion(db)
if err != nil {
t.Fatalf("loadDatabaseSchemaVersion: %v", err)
}
if !exists {
t.Fatal("expected migrated database to keep schema version record")
}
if version != currentDatabaseSchemaVersion {
t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion)
}
for _, baseTable := range shardedObservabilityBaseTables() {
for _, table := range observabilityShardTables(baseTable) {
legacyTable := legacyObservabilityShardTableName(table)
if db.Migrator().HasTable(legacyTable) {
t.Fatalf("expected legacy shard table %s to be removed", legacyTable)
}
}
}
snapshots, err := ListMetricSnapshotsSince(time.Time{})
if err != nil {
t.Fatalf("ListMetricSnapshotsSince failed: %v", err)
}
if len(snapshots) != 2 {
t.Fatalf("expected 2 migrated metric snapshots, got %+v", snapshots)
}
reports, err := ListRequestReportsSince(time.Time{})
if err != nil {
t.Fatalf("ListRequestReportsSince failed: %v", err)
}
if len(reports) != 2 {
t.Fatalf("expected 2 migrated request reports, got %+v", reports)
}
logs, err := ListNodeAccessLogs(NodeAccessLogQuery{Page: 0, PageSize: 10})
if err != nil {
t.Fatalf("ListNodeAccessLogs failed: %v", err)
}
if len(logs) != 2 {
t.Fatalf("expected 2 migrated access logs, got %+v", logs)
}
seenSnapshotIDs := make(map[uint]struct{}, len(snapshots))
for _, item := range snapshots {
if item == nil || item.ID == 0 {
t.Fatalf("expected migrated metric snapshot to have a new non-zero id: %+v", item)
}
if _, exists := seenSnapshotIDs[item.ID]; exists {
t.Fatalf("expected migrated metric snapshot ids to be unique, got duplicate %d", item.ID)
}
seenSnapshotIDs[item.ID] = struct{}{}
targetTable := observabilityShardTableForID("node_metric_snapshots", item.ID)
var count int64
if err := db.Table(targetTable).Where("id = ?", item.ID).Count(&count).Error; err != nil {
t.Fatalf("count migrated metric snapshot in target shard: %v", err)
}
if count != 1 {
t.Fatalf("expected migrated metric snapshot id %d to be stored in %s", item.ID, targetTable)
}
}
}
func TestMigrateOriginsSchemaBackfillsOrigins(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-origins.db")
if err := registerSharding(db, "sqlite"); err != nil {
@@ -573,217 +341,6 @@ func TestMigrateOriginsSchemaBackfillsOrigins(t *testing.T) {
}
}
func TestEnsureDatabaseSchemaUpToDateBackfillsProxyRouteSiteFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-sites.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
for _, item := range registeredModels() {
if _, ok := item.(*ProxyRoute); ok {
continue
}
if err := db.AutoMigrate(item); err != nil {
t.Fatalf("auto migrate supporting table: %v", err)
}
}
if err := db.AutoMigrate(&legacyProxyRouteV4{}); err != nil {
t.Fatalf("auto migrate legacy proxy_routes: %v", err)
}
now := time.Now().UTC()
if err := db.Create(&legacyProxyRouteV4{
Domain: "app.example.com",
OriginURL: "https://origin-a.internal:8443",
Upstreams: `["https://origin-a.internal:8443","https://origin-b.internal:8443"]`,
Enabled: true,
EnableHTTPS: false,
RedirectHTTP: false,
CacheEnabled: false,
CachePolicy: "",
CacheRules: `[]`,
CustomHeaders: `[]`,
CreatedAt: now,
UpdatedAt: now,
}).Error; err != nil {
t.Fatalf("seed legacy proxy route: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 4); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
var route ProxyRoute
if err := db.First(&route).Error; err != nil {
t.Fatalf("query migrated proxy route: %v", err)
}
if route.SiteName != "app.example.com" {
t.Fatalf("unexpected site_name after migration: %s", route.SiteName)
}
if route.Domain != "app.example.com" {
t.Fatalf("unexpected domain mirror after migration: %s", route.Domain)
}
var domains []string
if err := json.Unmarshal([]byte(route.Domains), &domains); err != nil {
t.Fatalf("decode migrated domains: %v", err)
}
if len(domains) != 1 || domains[0] != "app.example.com" {
t.Fatalf("unexpected migrated domains: %#v", domains)
}
}
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteRateLimitFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-rate-limits.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
for _, item := range registeredModels() {
if _, ok := item.(*ProxyRoute); ok {
continue
}
if err := db.AutoMigrate(item); err != nil {
t.Fatalf("auto migrate supporting table: %v", err)
}
}
if err := db.AutoMigrate(&legacyProxyRouteV5{}); err != nil {
t.Fatalf("auto migrate legacy proxy_routes v5: %v", err)
}
now := time.Now().UTC()
if err := db.Create(&legacyProxyRouteV5{
SiteName: "main-site",
Domain: "app.example.com",
Domains: `["app.example.com","www.example.com"]`,
OriginURL: "https://origin-a.internal:8443",
Upstreams: `["https://origin-a.internal:8443"]`,
Enabled: true,
EnableHTTPS: false,
RedirectHTTP: false,
CacheEnabled: false,
CachePolicy: "",
CacheRules: `[]`,
CustomHeaders: `[]`,
CreatedAt: now,
UpdatedAt: now,
}).Error; err != nil {
t.Fatalf("seed legacy proxy route v5: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 5); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
var route ProxyRoute
if err := db.First(&route).Error; err != nil {
t.Fatalf("query migrated proxy route: %v", err)
}
if route.LimitConnPerServer != 0 || route.LimitConnPerIP != 0 || route.LimitRate != "" {
t.Fatalf("expected new rate limit fields to default to disabled values, got %+v", route)
}
}
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteCertificateListFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-cert-ids.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
for _, item := range registeredModels() {
if _, ok := item.(*ProxyRoute); ok {
continue
}
if err := db.AutoMigrate(item); err != nil {
t.Fatalf("auto migrate supporting table: %v", err)
}
}
if err := db.AutoMigrate(&legacyProxyRouteV6{}); err != nil {
t.Fatalf("auto migrate legacy proxy_routes v6: %v", err)
}
now := time.Now().UTC()
certID := uint(9)
if err := db.Create(&legacyProxyRouteV6{
SiteName: "secure-site",
Domain: "secure.example.com",
Domains: `["secure.example.com","www.secure.example.com"]`,
OriginURL: "https://origin-secure.internal:8443",
Upstreams: `["https://origin-secure.internal:8443"]`,
Enabled: true,
EnableHTTPS: true,
CertID: &certID,
RedirectHTTP: true,
LimitConnPerServer: 120,
LimitConnPerIP: 12,
LimitRate: "512k",
CacheEnabled: false,
CachePolicy: "",
CacheRules: `[]`,
CustomHeaders: `[]`,
CreatedAt: now,
UpdatedAt: now,
}).Error; err != nil {
t.Fatalf("seed legacy proxy route v6: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 6); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
var route ProxyRoute
if err := db.First(&route).Error; err != nil {
t.Fatalf("query migrated proxy route: %v", err)
}
if route.CertID == nil || *route.CertID != certID {
t.Fatalf("expected cert_id mirror to be preserved, got %+v", route.CertID)
}
var certIDs []uint
if err := json.Unmarshal([]byte(route.CertIDs), &certIDs); err != nil {
t.Fatalf("decode migrated cert_ids: %v", err)
}
if len(certIDs) != 1 || certIDs[0] != certID {
t.Fatalf("unexpected migrated cert_ids: %#v", certIDs)
}
}
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteDomainCertificateFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-domain-cert-ids.db")
if err := registerSharding(db, "sqlite"); err != nil {
@@ -0,0 +1,4 @@
package migrate
// Versions 1 through 7 are treated as the historical baseline. There are no
// supported deployments below v8, so new upgrades start from this base version.
@@ -0,0 +1,52 @@
package migrate
import (
"sort"
"gorm.io/gorm"
)
const BaseDatabaseSchemaVersion = 7
type Context interface {
ApplyCurrentSchema(db *gorm.DB, backend string) error
BackfillOriginsFromProxyRoutes(db *gorm.DB) error
BackfillProxyRouteSiteFields(db *gorm.DB) error
EnsureProxyRouteSiteNameUniqueIndex(db *gorm.DB) error
BackfillProxyRouteCertificateFields(db *gorm.DB) error
BackfillProxyRouteDomainCertificateFields(db *gorm.DB) error
EnsureDefaultGitHubAuthSource(db *gorm.DB) error
EnsureDefaultWAFRuleGroup(db *gorm.DB) error
ValidateDatabaseSchemaVersion(db *gorm.DB, backend string, version int) error
}
type Migration struct {
FromVersion int
ToVersion int
Migrate func(ctx Context, db *gorm.DB, backend string) error
Validate func(ctx Context, db *gorm.DB, backend string) error
}
var registeredMigrations []Migration
func Register(migration Migration) {
registeredMigrations = append(registeredMigrations, migration)
}
func Migrations() []Migration {
migrations := append([]Migration{}, registeredMigrations...)
sort.Slice(migrations, func(i int, j int) bool {
return migrations[i].FromVersion < migrations[j].FromVersion
})
return migrations
}
func CurrentVersion() int {
version := BaseDatabaseSchemaVersion
for _, migration := range registeredMigrations {
if migration.ToVersion > version {
version = migration.ToVersion
}
}
return version
}
@@ -0,0 +1,23 @@
package migrate
import "testing"
func TestMigrationsAreContinuousFromBaseVersion(t *testing.T) {
migrations := Migrations()
if len(migrations) == 0 {
t.Fatal("expected at least one registered migration")
}
expectedFrom := BaseDatabaseSchemaVersion
for _, migration := range migrations {
if migration.FromVersion != expectedFrom {
t.Fatalf("expected migration from v%d, got v%d -> v%d", expectedFrom, migration.FromVersion, migration.ToVersion)
}
if migration.ToVersion != migration.FromVersion+1 {
t.Fatalf("expected one-step migration, got v%d -> v%d", migration.FromVersion, migration.ToVersion)
}
expectedFrom = migration.ToVersion
}
if CurrentVersion() != expectedFrom {
t.Fatalf("unexpected current version: got %d want %d", CurrentVersion(), expectedFrom)
}
}
+27
View File
@@ -0,0 +1,27 @@
package migrate
import "gorm.io/gorm"
func init() {
Register(V10())
}
func V10() Migration {
return Migration{
FromVersion: 9,
ToVersion: 10,
Migrate: migrateV10,
Validate: validateV10,
}
}
func migrateV10(ctx Context, db *gorm.DB, backend string) error {
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
return err
}
return ctx.EnsureDefaultGitHubAuthSource(db)
}
func validateV10(ctx Context, db *gorm.DB, backend string) error {
return ctx.ValidateDatabaseSchemaVersion(db, backend, 10)
}
+24
View File
@@ -0,0 +1,24 @@
package migrate
import "gorm.io/gorm"
func init() {
Register(V11())
}
func V11() Migration {
return Migration{
FromVersion: 10,
ToVersion: 11,
Migrate: migrateV11,
Validate: validateV11,
}
}
func migrateV11(ctx Context, db *gorm.DB, backend string) error {
return ctx.ApplyCurrentSchema(db, backend)
}
func validateV11(ctx Context, db *gorm.DB, backend string) error {
return ctx.ValidateDatabaseSchemaVersion(db, backend, 11)
}
+24
View File
@@ -0,0 +1,24 @@
package migrate
import "gorm.io/gorm"
func init() {
Register(V12())
}
func V12() Migration {
return Migration{
FromVersion: 11,
ToVersion: 12,
Migrate: migrateV12,
Validate: validateV12,
}
}
func migrateV12(ctx Context, db *gorm.DB, backend string) error {
return ctx.ApplyCurrentSchema(db, backend)
}
func validateV12(ctx Context, db *gorm.DB, backend string) error {
return ctx.ValidateDatabaseSchemaVersion(db, backend, 12)
}
+27
View File
@@ -0,0 +1,27 @@
package migrate
import "gorm.io/gorm"
func init() {
Register(V13())
}
func V13() Migration {
return Migration{
FromVersion: 12,
ToVersion: 13,
Migrate: migrateV13,
Validate: validateV13,
}
}
func migrateV13(ctx Context, db *gorm.DB, backend string) error {
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
return err
}
return ctx.EnsureDefaultWAFRuleGroup(db)
}
func validateV13(ctx Context, db *gorm.DB, backend string) error {
return ctx.ValidateDatabaseSchemaVersion(db, backend, 13)
}
+27
View File
@@ -0,0 +1,27 @@
package migrate
import "gorm.io/gorm"
func init() {
Register(V14())
}
func V14() Migration {
return Migration{
FromVersion: 13,
ToVersion: 14,
Migrate: migrateV14,
Validate: validateV14,
}
}
func migrateV14(ctx Context, db *gorm.DB, backend string) error {
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
return err
}
return ctx.EnsureDefaultWAFRuleGroup(db)
}
func validateV14(ctx Context, db *gorm.DB, backend string) error {
return ctx.ValidateDatabaseSchemaVersion(db, backend, 14)
}
+42
View File
@@ -0,0 +1,42 @@
package migrate
import (
"fmt"
"gorm.io/gorm"
)
type nodeV15 struct {
IPManualOverride bool `gorm:"column:ip_manual_override;not null;default:false"`
}
func init() {
Register(V15())
}
func V15() Migration {
return Migration{
FromVersion: 14,
ToVersion: 15,
Migrate: migrateV15,
Validate: validateV15,
}
}
func (nodeV15) TableName() string {
return "nodes"
}
func migrateV15(ctx Context, db *gorm.DB, backend string) error {
return ctx.ApplyCurrentSchema(db, backend)
}
func validateV15(ctx Context, db *gorm.DB, backend string) error {
if err := ctx.ValidateDatabaseSchemaVersion(db, backend, 14); err != nil {
return err
}
if db == nil || !db.Migrator().HasColumn(&nodeV15{}, "ip_manual_override") {
return fmt.Errorf("column nodes.ip_manual_override is missing")
}
return nil
}
+39
View File
@@ -0,0 +1,39 @@
package migrate
import "gorm.io/gorm"
func init() {
Register(V8())
}
func V8() Migration {
return Migration{
FromVersion: 7,
ToVersion: 8,
Migrate: migrateV8,
Validate: validateV8,
}
}
func migrateV8(ctx Context, db *gorm.DB, backend string) error {
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
return err
}
if err := ctx.BackfillOriginsFromProxyRoutes(db); err != nil {
return err
}
if err := ctx.BackfillProxyRouteSiteFields(db); err != nil {
return err
}
if err := ctx.EnsureProxyRouteSiteNameUniqueIndex(db); err != nil {
return err
}
if err := ctx.BackfillProxyRouteCertificateFields(db); err != nil {
return err
}
return ctx.BackfillProxyRouteDomainCertificateFields(db)
}
func validateV8(ctx Context, db *gorm.DB, backend string) error {
return ctx.ValidateDatabaseSchemaVersion(db, backend, 8)
}
+27
View File
@@ -0,0 +1,27 @@
package migrate
import "gorm.io/gorm"
func init() {
Register(V9())
}
func V9() Migration {
return Migration{
FromVersion: 8,
ToVersion: 9,
Migrate: migrateV9,
Validate: validateV9,
}
}
func migrateV9(ctx Context, db *gorm.DB, backend string) error {
if err := migrateV8(ctx, db, backend); err != nil {
return err
}
return nil
}
func validateV9(ctx Context, db *gorm.DB, backend string) error {
return ctx.ValidateDatabaseSchemaVersion(db, backend, 9)
}
+93 -417
View File
@@ -6,8 +6,10 @@ import (
"encoding/pem"
"errors"
"fmt"
"log/slog"
"net"
"net/url"
schemamigrate "openflare/model/migrate"
"strings"
"gorm.io/gorm"
@@ -20,6 +22,63 @@ type databaseSchemaMigration struct {
validate func(db *gorm.DB, backend string) error
}
type databaseSchemaMigrationContext struct{}
func (databaseSchemaMigrationContext) ApplyCurrentSchema(db *gorm.DB, backend string) error {
return applyCurrentSchema(db, backend)
}
func (databaseSchemaMigrationContext) BackfillOriginsFromProxyRoutes(db *gorm.DB) error {
return backfillOriginsFromProxyRoutes(db)
}
func (databaseSchemaMigrationContext) BackfillProxyRouteSiteFields(db *gorm.DB) error {
return backfillProxyRouteSiteFields(db)
}
func (databaseSchemaMigrationContext) EnsureProxyRouteSiteNameUniqueIndex(db *gorm.DB) error {
return ensureProxyRouteSiteNameUniqueIndex(db)
}
func (databaseSchemaMigrationContext) BackfillProxyRouteCertificateFields(db *gorm.DB) error {
return backfillProxyRouteCertificateFields(db)
}
func (databaseSchemaMigrationContext) BackfillProxyRouteDomainCertificateFields(db *gorm.DB) error {
return backfillProxyRouteDomainCertificateFields(db)
}
func (databaseSchemaMigrationContext) EnsureDefaultGitHubAuthSource(db *gorm.DB) error {
return ensureDefaultGitHubAuthSource(db)
}
func (databaseSchemaMigrationContext) EnsureDefaultWAFRuleGroup(db *gorm.DB) error {
return ensureDefaultWAFRuleGroup(db)
}
func (databaseSchemaMigrationContext) ValidateDatabaseSchemaVersion(db *gorm.DB, backend string, version int) error {
switch version {
case 7:
return validateDatabaseSchemaV7(db, backend)
case 8:
return validateDatabaseSchemaV8(db, backend)
case 9:
return validateDatabaseSchemaV9(db, backend)
case 10:
return validateDatabaseSchemaV10(db, backend)
case 11:
return validateDatabaseSchemaV11(db, backend)
case 12:
return validateDatabaseSchemaV12(db, backend)
case 13:
return validateDatabaseSchemaV13(db, backend)
case 14:
return validateDatabaseSchemaV14(db, backend)
default:
return fmt.Errorf("database schema validation for v%d is not defined", version)
}
}
func autoMigrateSchemaMetadata(db *gorm.DB) error {
for _, item := range schemaMetadataModels() {
if err := db.AutoMigrate(item); err != nil {
@@ -789,225 +848,6 @@ func uintSlicesEqualForMigration(left []uint, right []uint) bool {
return true
}
func renameLegacyObservabilityShardTables(db *gorm.DB) error {
for _, baseTable := range shardedObservabilityBaseTables() {
for _, table := range observabilityShardTables(baseTable) {
legacyTable := legacyObservabilityShardTableName(table)
if db.Migrator().HasTable(legacyTable) {
return fmt.Errorf("legacy sharded table %s already exists", legacyTable)
}
if !db.Migrator().HasTable(table) {
continue
}
if err := db.Migrator().RenameTable(table, legacyTable); err != nil {
return fmt.Errorf("rename sharded table %s to %s failed: %w", table, legacyTable, err)
}
if err := dropLegacyObservabilitySecondaryIndexes(db, legacyTable); err != nil {
return err
}
}
}
return nil
}
func dropLegacyObservabilitySecondaryIndexes(db *gorm.DB, table string) error {
db = sessionIgnoringSharding(db)
if db == nil {
return fmt.Errorf("database handle is nil")
}
backend := baseDialector(db).Name()
indexes := make([]string, 0)
switch backend {
case "sqlite":
if err := db.Raw(
`SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ? AND name LIKE 'idx_%'`,
table,
).Scan(&indexes).Error; err != nil {
return fmt.Errorf("list indexes for %s failed: %w", table, err)
}
case "postgres":
if err := db.Raw(
`SELECT indexname FROM pg_indexes WHERE schemaname = current_schema() AND tablename = ? AND indexname LIKE 'idx_%'`,
table,
).Scan(&indexes).Error; err != nil {
return fmt.Errorf("list indexes for %s failed: %w", table, err)
}
default:
return fmt.Errorf("unsupported database backend %s", backend)
}
for _, indexName := range indexes {
if err := db.Exec(fmt.Sprintf(`DROP INDEX IF EXISTS "%s"`, indexName)).Error; err != nil {
return fmt.Errorf("drop legacy index %s failed: %w", indexName, err)
}
}
return nil
}
func autoMigrateObservabilityShardTables(db *gorm.DB) error {
db = sessionIgnoringSharding(db)
if db == nil {
return fmt.Errorf("database handle is nil")
}
dialector := baseDialector(db)
if dialector == nil {
return fmt.Errorf("database dialector is nil")
}
type shardedTable struct {
model any
base string
}
tables := []shardedTable{
{model: &NodeMetricSnapshot{}, base: "node_metric_snapshots"},
{model: &NodeRequestReport{}, base: "node_request_reports"},
{model: &NodeAccessLog{}, base: "node_access_logs"},
}
for _, item := range tables {
for _, table := range observabilityShardTables(item.base) {
tx := db.Table(table)
if err := dialector.Migrator(tx).AutoMigrate(item.model); err != nil {
return fmt.Errorf("auto migrate sharded table %s failed: %w", table, err)
}
}
}
return nil
}
func dropLegacyObservabilityShardTables(db *gorm.DB) error {
db = sessionIgnoringSharding(db)
if db == nil {
return fmt.Errorf("database handle is nil")
}
for _, baseTable := range shardedObservabilityBaseTables() {
for _, table := range observabilityShardTables(baseTable) {
legacyTable := legacyObservabilityShardTableName(table)
if !db.Migrator().HasTable(legacyTable) {
continue
}
if err := db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS "%s"`, legacyTable)).Error; err != nil {
return fmt.Errorf("drop legacy sharded table %s failed: %w", legacyTable, err)
}
}
}
return nil
}
func migrateLegacyNodeMetricSnapshots(db *gorm.DB) error {
for _, table := range observabilityShardTables("node_metric_snapshots") {
legacyTable := legacyObservabilityShardTableName(table)
if !db.Migrator().HasTable(legacyTable) {
continue
}
var lastSeenID uint
for {
var rows []NodeMetricSnapshot
query := db.Table(legacyTable).Order("id ASC").Limit(500)
if lastSeenID > 0 {
query = query.Where("id > ?", lastSeenID)
}
if err := query.Find(&rows).Error; err != nil {
return fmt.Errorf("query legacy sharded table %s failed: %w", legacyTable, err)
}
if len(rows) == 0 {
break
}
lastSeenID = rows[len(rows)-1].ID
grouped := make(map[string][]NodeMetricSnapshot, observabilityShardCount)
for index := range rows {
rows[index].ID = 0
if err := assignObservabilityID(&rows[index].ID); err != nil {
return err
}
targetTable := observabilityShardTableForID("node_metric_snapshots", rows[index].ID)
grouped[targetTable] = append(grouped[targetTable], rows[index])
}
for targetTable, batch := range grouped {
if err := db.Table(targetTable).Create(&batch).Error; err != nil {
return fmt.Errorf("write migrated rows into %s failed: %w", targetTable, err)
}
}
}
}
return nil
}
func migrateLegacyNodeRequestReports(db *gorm.DB) error {
for _, table := range observabilityShardTables("node_request_reports") {
legacyTable := legacyObservabilityShardTableName(table)
if !db.Migrator().HasTable(legacyTable) {
continue
}
var lastSeenID uint
for {
var rows []NodeRequestReport
query := db.Table(legacyTable).Order("id ASC").Limit(500)
if lastSeenID > 0 {
query = query.Where("id > ?", lastSeenID)
}
if err := query.Find(&rows).Error; err != nil {
return fmt.Errorf("query legacy sharded table %s failed: %w", legacyTable, err)
}
if len(rows) == 0 {
break
}
lastSeenID = rows[len(rows)-1].ID
grouped := make(map[string][]NodeRequestReport, observabilityShardCount)
for index := range rows {
rows[index].ID = 0
if err := assignObservabilityID(&rows[index].ID); err != nil {
return err
}
targetTable := observabilityShardTableForID("node_request_reports", rows[index].ID)
grouped[targetTable] = append(grouped[targetTable], rows[index])
}
for targetTable, batch := range grouped {
if err := db.Table(targetTable).Create(&batch).Error; err != nil {
return fmt.Errorf("write migrated rows into %s failed: %w", targetTable, err)
}
}
}
}
return nil
}
func migrateLegacyNodeAccessLogs(db *gorm.DB) error {
for _, table := range observabilityShardTables("node_access_logs") {
legacyTable := legacyObservabilityShardTableName(table)
if !db.Migrator().HasTable(legacyTable) {
continue
}
var lastSeenID uint
for {
var rows []NodeAccessLog
query := db.Table(legacyTable).Order("id ASC").Limit(500)
if lastSeenID > 0 {
query = query.Where("id > ?", lastSeenID)
}
if err := query.Find(&rows).Error; err != nil {
return fmt.Errorf("query legacy sharded table %s failed: %w", legacyTable, err)
}
if len(rows) == 0 {
break
}
lastSeenID = rows[len(rows)-1].ID
grouped := make(map[string][]NodeAccessLog, observabilityShardCount)
for index := range rows {
rows[index].ID = 0
if err := assignObservabilityID(&rows[index].ID); err != nil {
return err
}
targetTable := observabilityShardTableForID("node_access_logs", rows[index].ID)
grouped[targetTable] = append(grouped[targetTable], rows[index])
}
for targetTable, batch := range grouped {
if err := db.Table(targetTable).Create(&batch).Error; err != nil {
return fmt.Errorf("write migrated rows into %s failed: %w", targetTable, err)
}
}
}
}
return nil
}
func normalizeOriginAddressForMigration(raw string) string {
return strings.ToLower(strings.TrimSpace(raw))
}
@@ -1086,136 +926,6 @@ func backfillOriginsFromProxyRoutes(db *gorm.DB) error {
return nil
}
// migrateV2 upgrades the legacy schema to the first versioned schema by
// creating schema metadata, applying the current tables, and backfilling
// compatibility columns.
func migrateV2(db *gorm.DB, backend string) error {
return applyCurrentSchema(db, backend)
}
// migrateV3 upgrades observability shard tables from legacy ID layout to the
// current ID-sharded layout and migrates existing shard data into the new tables.
func migrateV3(db *gorm.DB, backend string) error {
if db == nil {
return fmt.Errorf("database handle is nil")
}
_ = backend
if err := renameLegacyObservabilityShardTables(db); err != nil {
return err
}
if err := autoMigrateObservabilityShardTables(db); err != nil {
return err
}
if err := migrateLegacyNodeMetricSnapshots(db); err != nil {
return err
}
if err := migrateLegacyNodeRequestReports(db); err != nil {
return err
}
if err := migrateLegacyNodeAccessLogs(db); err != nil {
return err
}
return dropLegacyObservabilityShardTables(db)
}
// migrateV4 introduces the origins schema and backfills proxy route origin
// references from existing origin_url values.
func migrateV4(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
return backfillOriginsFromProxyRoutes(db)
}
// migrateV5 upgrades proxy_routes to website-level identity fields by
// backfilling site_name and domains while keeping domain as the primary-domain
// compatibility mirror.
func migrateV5(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
if err := backfillOriginsFromProxyRoutes(db); err != nil {
return err
}
if err := backfillProxyRouteSiteFields(db); err != nil {
return err
}
return ensureProxyRouteSiteNameUniqueIndex(db)
}
// migrateV6 adds structured website-level rate limit fields to proxy_routes.
func migrateV6(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
if err := backfillOriginsFromProxyRoutes(db); err != nil {
return err
}
if err := backfillProxyRouteSiteFields(db); err != nil {
return err
}
return ensureProxyRouteSiteNameUniqueIndex(db)
}
// migrateV7 adds structured website-level certificate lists to proxy_routes
// while keeping cert_id as the primary certificate compatibility mirror.
func migrateV7(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
if err := backfillOriginsFromProxyRoutes(db); err != nil {
return err
}
if err := backfillProxyRouteSiteFields(db); err != nil {
return err
}
if err := ensureProxyRouteSiteNameUniqueIndex(db); err != nil {
return err
}
return backfillProxyRouteCertificateFields(db)
}
// migrateV8 adds per-domain certificate assignments to proxy_routes while
// keeping cert_ids as the website-level compatibility mirror.
func migrateV8(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
if err := backfillOriginsFromProxyRoutes(db); err != nil {
return err
}
if err := backfillProxyRouteSiteFields(db); err != nil {
return err
}
if err := ensureProxyRouteSiteNameUniqueIndex(db); err != nil {
return err
}
if err := backfillProxyRouteCertificateFields(db); err != nil {
return err
}
return backfillProxyRouteDomainCertificateFields(db)
}
// migrateV9 adds PoW (Proof-of-Work) anti-bot protection fields to proxy_routes.
func migrateV9(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
if err := backfillOriginsFromProxyRoutes(db); err != nil {
return err
}
if err := backfillProxyRouteSiteFields(db); err != nil {
return err
}
if err := ensureProxyRouteSiteNameUniqueIndex(db); err != nil {
return err
}
if err := backfillProxyRouteCertificateFields(db); err != nil {
return err
}
return backfillProxyRouteDomainCertificateFields(db)
}
func ensureDefaultGitHubAuthSource(db *gorm.DB) error {
if db == nil || !db.Migrator().HasTable(&AuthSource{}) || !db.Migrator().HasTable(&ExternalAccount{}) {
return nil
@@ -1310,14 +1020,6 @@ func ensureDefaultGitHubAuthSource(db *gorm.DB) error {
return nil
}
// migrateV10 adds configurable auth sources and external account bindings.
func migrateV10(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
return ensureDefaultGitHubAuthSource(db)
}
func validateDatabaseSchemaV9(db *gorm.DB, backend string) error {
if err := validateDatabaseSchemaV8(db, backend); err != nil {
return err
@@ -1344,15 +1046,6 @@ func validateDatabaseSchemaV10(db *gorm.DB, backend string) error {
return nil
}
// migrateV11 adds acme and dns accounts and extends tls_certificates.
func migrateV11(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
// Default values will be applied by gorm for new columns automatically during AutoMigrate.
return nil
}
func validateDatabaseSchemaV11(db *gorm.DB, backend string) error {
if err := validateDatabaseSchemaV10(db, backend); err != nil {
return err
@@ -1369,15 +1062,6 @@ func validateDatabaseSchemaV11(db *gorm.DB, backend string) error {
return nil
}
// migrateV12 adds basic authentication fields to proxy_routes.
func migrateV12(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
// Default values will be applied by gorm for new columns automatically during AutoMigrate.
return nil
}
func validateDatabaseSchemaV12(db *gorm.DB, backend string) error {
if err := validateDatabaseSchemaV11(db, backend); err != nil {
return err
@@ -1423,14 +1107,6 @@ func ensureDefaultWAFRuleGroup(db *gorm.DB) error {
return nil
}
// migrateV13 adds WAF rule groups and website bindings.
func migrateV13(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
return ensureDefaultWAFRuleGroup(db)
}
func validateDatabaseSchemaV13(db *gorm.DB, backend string) error {
if err := validateDatabaseSchemaV12(db, backend); err != nil {
return err
@@ -1451,14 +1127,6 @@ func validateDatabaseSchemaV13(db *gorm.DB, backend string) error {
return nil
}
// migrateV14 adds PoW policy fields to WAF rule groups.
func migrateV14(db *gorm.DB, backend string) error {
if err := applyCurrentSchema(db, backend); err != nil {
return err
}
return ensureDefaultWAFRuleGroup(db)
}
func validateDatabaseSchemaV14(db *gorm.DB, backend string) error {
if err := validateDatabaseSchemaV13(db, backend); err != nil {
return err
@@ -1472,38 +1140,42 @@ func validateDatabaseSchemaV14(db *gorm.DB, backend string) error {
return nil
}
// migrateV15 adds the node IP manual override flag.
func migrateV15(db *gorm.DB, backend string) error {
return applyCurrentSchema(db, backend)
func databaseSchemaMigrations() []databaseSchemaMigration {
ctx := databaseSchemaMigrationContext{}
migrations := []databaseSchemaMigration{}
for _, item := range schemamigrate.Migrations() {
external := item
migrations = append(migrations, databaseSchemaMigration{
fromVersion: external.FromVersion,
toVersion: external.ToVersion,
migrate: func(db *gorm.DB, backend string) error {
return external.Migrate(ctx, db, backend)
},
validate: func(db *gorm.DB, backend string) error {
return validateExternalDatabaseSchema(ctx, db, backend, external.ToVersion)
},
})
}
return migrations
}
func validateDatabaseSchemaV15(db *gorm.DB, backend string) error {
if err := validateDatabaseSchemaV14(db, backend); err != nil {
return err
func validateExternalDatabaseSchema(ctx databaseSchemaMigrationContext, db *gorm.DB, backend string, targetVersion int) error {
if targetVersion <= schemamigrate.BaseDatabaseSchemaVersion {
return ctx.ValidateDatabaseSchemaVersion(db, backend, targetVersion)
}
if !db.Migrator().HasColumn(&Node{}, "ip_manual_override") {
return fmt.Errorf("column nodes.ip_manual_override is missing")
for _, migration := range schemamigrate.Migrations() {
if migration.ToVersion > targetVersion {
continue
}
if err := migration.Validate(ctx, db, backend); err != nil {
return err
}
}
return nil
}
func databaseSchemaMigrations() []databaseSchemaMigration {
return []databaseSchemaMigration{
{fromVersion: 1, toVersion: 2, migrate: migrateV2, validate: validateDatabaseSchemaV2},
{fromVersion: 2, toVersion: 3, migrate: migrateV3, validate: validateDatabaseSchemaV3},
{fromVersion: 3, toVersion: 4, migrate: migrateV4, validate: validateDatabaseSchemaV4},
{fromVersion: 4, toVersion: 5, migrate: migrateV5, validate: validateDatabaseSchemaV5},
{fromVersion: 5, toVersion: 6, migrate: migrateV6, validate: validateDatabaseSchemaV6},
{fromVersion: 6, toVersion: 7, migrate: migrateV7, validate: validateDatabaseSchemaV7},
{fromVersion: 7, toVersion: 8, migrate: migrateV8, validate: validateDatabaseSchemaV8},
{fromVersion: 8, toVersion: 9, migrate: migrateV9, validate: validateDatabaseSchemaV9},
{fromVersion: 9, toVersion: 10, migrate: migrateV10, validate: validateDatabaseSchemaV10},
{fromVersion: 10, toVersion: 11, migrate: migrateV11, validate: validateDatabaseSchemaV11},
{fromVersion: 11, toVersion: 12, migrate: migrateV12, validate: validateDatabaseSchemaV12},
{fromVersion: 12, toVersion: 13, migrate: migrateV13, validate: validateDatabaseSchemaV13},
{fromVersion: 13, toVersion: 14, migrate: migrateV14, validate: validateDatabaseSchemaV14},
{fromVersion: 14, toVersion: 15, migrate: migrateV15, validate: validateDatabaseSchemaV15},
}
func validateCurrentDatabaseSchema(db *gorm.DB, backend string) error {
return validateExternalDatabaseSchema(databaseSchemaMigrationContext{}, db, backend, currentDatabaseSchemaVersion)
}
func databaseSchemaMigrationMap() map[int]databaseSchemaMigration {
@@ -1546,6 +1218,10 @@ func upgradeDatabaseSchema(db *gorm.DB, backend string, version int) error {
if version > currentDatabaseSchemaVersion {
return fmt.Errorf("database schema version %d is newer than application version %d", version, currentDatabaseSchemaVersion)
}
if version < legacyDatabaseSchemaVersion {
slog.Warn("database schema version is below supported baseline; treating it as historical initial schema", "version", version, "baseline", legacyDatabaseSchemaVersion)
version = legacyDatabaseSchemaVersion
}
if version == currentDatabaseSchemaVersion {
return nil
}
@@ -1591,7 +1267,7 @@ func initializeFreshDatabaseSchema(db *gorm.DB, backend string) error {
if err := ensureDefaultWAFRuleGroup(db); err != nil {
return err
}
if err := validateDatabaseSchemaV15(db, backend); err != nil {
if err := validateCurrentDatabaseSchema(db, backend); err != nil {
return err
}
return saveDatabaseSchemaVersion(db, currentDatabaseSchemaVersion)