mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-06 15:46:37 +08:00
[优化] 重构数据库迁移逻辑,添加版本管理和验证功能
This commit is contained in:
@@ -1,13 +1,17 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"openflare/model/migrate"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
legacyDatabaseSchemaVersion = 1
|
||||
currentDatabaseSchemaVersion = 15
|
||||
databaseSchemaVersionRowID = 1
|
||||
legacyDatabaseSchemaVersion = migrate.BaseDatabaseSchemaVersion
|
||||
databaseSchemaVersionRowID = 1
|
||||
)
|
||||
|
||||
var currentDatabaseSchemaVersion = migrate.CurrentVersion()
|
||||
|
||||
type DatabaseSchemaVersion struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
Version int `json:"version" gorm:"not null"`
|
||||
|
||||
@@ -10,85 +10,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type legacyProxyRouteV4 struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
Domain string `gorm:"uniqueIndex;size:255;not null"`
|
||||
OriginID *uint `gorm:"index"`
|
||||
OriginURL string `gorm:"size:2048;not null"`
|
||||
OriginHost string `gorm:"size:255"`
|
||||
Upstreams string `gorm:"type:text;not null;default:'[]'"`
|
||||
Enabled bool `gorm:"not null;default:true"`
|
||||
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
|
||||
CertID *uint
|
||||
RedirectHTTP bool `gorm:"not null;default:false"`
|
||||
CacheEnabled bool `gorm:"not null;default:false"`
|
||||
CachePolicy string `gorm:"size:32;not null;default:''"`
|
||||
CacheRules string `gorm:"type:text;not null;default:'[]'"`
|
||||
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
|
||||
Remark string `gorm:"size:255"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (legacyProxyRouteV4) TableName() string {
|
||||
return "proxy_routes"
|
||||
}
|
||||
|
||||
type legacyProxyRouteV5 struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
SiteName string `gorm:"size:255;not null;default:''"`
|
||||
Domain string `gorm:"uniqueIndex;size:255;not null"`
|
||||
Domains string `gorm:"type:text;not null;default:'[]'"`
|
||||
OriginID *uint `gorm:"index"`
|
||||
OriginURL string `gorm:"size:2048;not null"`
|
||||
OriginHost string `gorm:"size:255"`
|
||||
Upstreams string `gorm:"type:text;not null;default:'[]'"`
|
||||
Enabled bool `gorm:"not null;default:true"`
|
||||
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
|
||||
CertID *uint
|
||||
RedirectHTTP bool `gorm:"not null;default:false"`
|
||||
CacheEnabled bool `gorm:"not null;default:false"`
|
||||
CachePolicy string `gorm:"size:32;not null;default:''"`
|
||||
CacheRules string `gorm:"type:text;not null;default:'[]'"`
|
||||
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
|
||||
Remark string `gorm:"size:255"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (legacyProxyRouteV5) TableName() string {
|
||||
return "proxy_routes"
|
||||
}
|
||||
|
||||
type legacyProxyRouteV6 struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
SiteName string `gorm:"size:255;not null;default:''"`
|
||||
Domain string `gorm:"uniqueIndex;size:255;not null"`
|
||||
Domains string `gorm:"type:text;not null;default:'[]'"`
|
||||
OriginID *uint `gorm:"index"`
|
||||
OriginURL string `gorm:"size:2048;not null"`
|
||||
OriginHost string `gorm:"size:255"`
|
||||
Upstreams string `gorm:"type:text;not null;default:'[]'"`
|
||||
Enabled bool `gorm:"not null;default:true"`
|
||||
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
|
||||
CertID *uint
|
||||
RedirectHTTP bool `gorm:"not null;default:false"`
|
||||
LimitConnPerServer int `gorm:"not null;default:0"`
|
||||
LimitConnPerIP int `gorm:"not null;default:0"`
|
||||
LimitRate string `gorm:"size:32;not null;default:''"`
|
||||
CacheEnabled bool `gorm:"not null;default:false"`
|
||||
CachePolicy string `gorm:"size:32;not null;default:''"`
|
||||
CacheRules string `gorm:"type:text;not null;default:'[]'"`
|
||||
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
|
||||
Remark string `gorm:"size:255"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (legacyProxyRouteV6) TableName() string {
|
||||
return "proxy_routes"
|
||||
}
|
||||
|
||||
type legacyProxyRouteV7 struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
SiteName string `gorm:"size:255;not null;default:''"`
|
||||
@@ -365,159 +286,6 @@ func TestEnsureDatabaseSchemaUpToDateUpgradesLegacyDatabase(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDatabaseSchemaUpToDateMigratesObservabilityShardsToID(t *testing.T) {
|
||||
db := openBareTestSQLiteDB(t, "legacy-observability-shards.db")
|
||||
if err := registerSharding(db, "sqlite"); err != nil {
|
||||
t.Fatalf("register sharding: %v", err)
|
||||
}
|
||||
if err := autoMigrateAll(db); err != nil {
|
||||
t.Fatalf("auto migrate db: %v", err)
|
||||
}
|
||||
if err := autoMigrateSchemaMetadata(db); err != nil {
|
||||
t.Fatalf("auto migrate schema metadata: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
if err := db.Table("node_metric_snapshots_00").Create(&NodeMetricSnapshot{
|
||||
ID: 1,
|
||||
NodeID: "node-a",
|
||||
CapturedAt: now.Add(-2 * time.Minute),
|
||||
CPUUsagePercent: 22,
|
||||
MemoryUsedBytes: 2,
|
||||
MemoryTotalBytes: 8,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed metric snapshot shard 00: %v", err)
|
||||
}
|
||||
if err := db.Table("node_metric_snapshots_01").Create(&NodeMetricSnapshot{
|
||||
ID: 1,
|
||||
NodeID: "node-b",
|
||||
CapturedAt: now.Add(-time.Minute),
|
||||
CPUUsagePercent: 44,
|
||||
MemoryUsedBytes: 4,
|
||||
MemoryTotalBytes: 8,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed metric snapshot shard 01: %v", err)
|
||||
}
|
||||
if err := db.Table("node_request_reports_00").Create(&NodeRequestReport{
|
||||
ID: 1,
|
||||
NodeID: "node-a",
|
||||
WindowStartedAt: now.Add(-3 * time.Minute),
|
||||
WindowEndedAt: now.Add(-2 * time.Minute),
|
||||
RequestCount: 12,
|
||||
ErrorCount: 1,
|
||||
UniqueVisitorCount: 6,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed request report shard 00: %v", err)
|
||||
}
|
||||
if err := db.Table("node_request_reports_01").Create(&NodeRequestReport{
|
||||
ID: 1,
|
||||
NodeID: "node-b",
|
||||
WindowStartedAt: now.Add(-2 * time.Minute),
|
||||
WindowEndedAt: now.Add(-time.Minute),
|
||||
RequestCount: 21,
|
||||
ErrorCount: 2,
|
||||
UniqueVisitorCount: 9,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed request report shard 01: %v", err)
|
||||
}
|
||||
if err := db.Table("node_access_logs_00").Create(&NodeAccessLog{
|
||||
ID: 1,
|
||||
NodeID: "node-a",
|
||||
LoggedAt: now.Add(-90 * time.Second),
|
||||
RemoteAddr: "203.0.113.10",
|
||||
Host: "a.example.com",
|
||||
Path: "/alpha",
|
||||
StatusCode: 200,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed access log shard 00: %v", err)
|
||||
}
|
||||
if err := db.Table("node_access_logs_01").Create(&NodeAccessLog{
|
||||
ID: 1,
|
||||
NodeID: "node-b",
|
||||
LoggedAt: now.Add(-60 * time.Second),
|
||||
RemoteAddr: "203.0.113.11",
|
||||
Host: "b.example.com",
|
||||
Path: "/beta",
|
||||
StatusCode: 502,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed access log shard 01: %v", err)
|
||||
}
|
||||
if err := saveDatabaseSchemaVersion(db, 2); err != nil {
|
||||
t.Fatalf("save schema version: %v", err)
|
||||
}
|
||||
|
||||
previousDB := DB
|
||||
DB = db
|
||||
t.Cleanup(func() {
|
||||
DB = previousDB
|
||||
})
|
||||
|
||||
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
|
||||
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
|
||||
}
|
||||
|
||||
version, exists, err := loadDatabaseSchemaVersion(db)
|
||||
if err != nil {
|
||||
t.Fatalf("loadDatabaseSchemaVersion: %v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatal("expected migrated database to keep schema version record")
|
||||
}
|
||||
if version != currentDatabaseSchemaVersion {
|
||||
t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion)
|
||||
}
|
||||
|
||||
for _, baseTable := range shardedObservabilityBaseTables() {
|
||||
for _, table := range observabilityShardTables(baseTable) {
|
||||
legacyTable := legacyObservabilityShardTableName(table)
|
||||
if db.Migrator().HasTable(legacyTable) {
|
||||
t.Fatalf("expected legacy shard table %s to be removed", legacyTable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
snapshots, err := ListMetricSnapshotsSince(time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListMetricSnapshotsSince failed: %v", err)
|
||||
}
|
||||
if len(snapshots) != 2 {
|
||||
t.Fatalf("expected 2 migrated metric snapshots, got %+v", snapshots)
|
||||
}
|
||||
reports, err := ListRequestReportsSince(time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListRequestReportsSince failed: %v", err)
|
||||
}
|
||||
if len(reports) != 2 {
|
||||
t.Fatalf("expected 2 migrated request reports, got %+v", reports)
|
||||
}
|
||||
logs, err := ListNodeAccessLogs(NodeAccessLogQuery{Page: 0, PageSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("ListNodeAccessLogs failed: %v", err)
|
||||
}
|
||||
if len(logs) != 2 {
|
||||
t.Fatalf("expected 2 migrated access logs, got %+v", logs)
|
||||
}
|
||||
|
||||
seenSnapshotIDs := make(map[uint]struct{}, len(snapshots))
|
||||
for _, item := range snapshots {
|
||||
if item == nil || item.ID == 0 {
|
||||
t.Fatalf("expected migrated metric snapshot to have a new non-zero id: %+v", item)
|
||||
}
|
||||
if _, exists := seenSnapshotIDs[item.ID]; exists {
|
||||
t.Fatalf("expected migrated metric snapshot ids to be unique, got duplicate %d", item.ID)
|
||||
}
|
||||
seenSnapshotIDs[item.ID] = struct{}{}
|
||||
targetTable := observabilityShardTableForID("node_metric_snapshots", item.ID)
|
||||
var count int64
|
||||
if err := db.Table(targetTable).Where("id = ?", item.ID).Count(&count).Error; err != nil {
|
||||
t.Fatalf("count migrated metric snapshot in target shard: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("expected migrated metric snapshot id %d to be stored in %s", item.ID, targetTable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateOriginsSchemaBackfillsOrigins(t *testing.T) {
|
||||
db := openBareTestSQLiteDB(t, "legacy-origins.db")
|
||||
if err := registerSharding(db, "sqlite"); err != nil {
|
||||
@@ -573,217 +341,6 @@ func TestMigrateOriginsSchemaBackfillsOrigins(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDatabaseSchemaUpToDateBackfillsProxyRouteSiteFields(t *testing.T) {
|
||||
db := openBareTestSQLiteDB(t, "legacy-proxy-route-sites.db")
|
||||
if err := registerSharding(db, "sqlite"); err != nil {
|
||||
t.Fatalf("register sharding: %v", err)
|
||||
}
|
||||
if err := autoMigrateSchemaMetadata(db); err != nil {
|
||||
t.Fatalf("auto migrate schema metadata: %v", err)
|
||||
}
|
||||
|
||||
for _, item := range registeredModels() {
|
||||
if _, ok := item.(*ProxyRoute); ok {
|
||||
continue
|
||||
}
|
||||
if err := db.AutoMigrate(item); err != nil {
|
||||
t.Fatalf("auto migrate supporting table: %v", err)
|
||||
}
|
||||
}
|
||||
if err := db.AutoMigrate(&legacyProxyRouteV4{}); err != nil {
|
||||
t.Fatalf("auto migrate legacy proxy_routes: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
if err := db.Create(&legacyProxyRouteV4{
|
||||
Domain: "app.example.com",
|
||||
OriginURL: "https://origin-a.internal:8443",
|
||||
Upstreams: `["https://origin-a.internal:8443","https://origin-b.internal:8443"]`,
|
||||
Enabled: true,
|
||||
EnableHTTPS: false,
|
||||
RedirectHTTP: false,
|
||||
CacheEnabled: false,
|
||||
CachePolicy: "",
|
||||
CacheRules: `[]`,
|
||||
CustomHeaders: `[]`,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed legacy proxy route: %v", err)
|
||||
}
|
||||
if err := saveDatabaseSchemaVersion(db, 4); err != nil {
|
||||
t.Fatalf("save schema version: %v", err)
|
||||
}
|
||||
|
||||
previousDB := DB
|
||||
DB = db
|
||||
t.Cleanup(func() {
|
||||
DB = previousDB
|
||||
})
|
||||
|
||||
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
|
||||
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
|
||||
}
|
||||
|
||||
var route ProxyRoute
|
||||
if err := db.First(&route).Error; err != nil {
|
||||
t.Fatalf("query migrated proxy route: %v", err)
|
||||
}
|
||||
if route.SiteName != "app.example.com" {
|
||||
t.Fatalf("unexpected site_name after migration: %s", route.SiteName)
|
||||
}
|
||||
if route.Domain != "app.example.com" {
|
||||
t.Fatalf("unexpected domain mirror after migration: %s", route.Domain)
|
||||
}
|
||||
|
||||
var domains []string
|
||||
if err := json.Unmarshal([]byte(route.Domains), &domains); err != nil {
|
||||
t.Fatalf("decode migrated domains: %v", err)
|
||||
}
|
||||
if len(domains) != 1 || domains[0] != "app.example.com" {
|
||||
t.Fatalf("unexpected migrated domains: %#v", domains)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteRateLimitFields(t *testing.T) {
|
||||
db := openBareTestSQLiteDB(t, "legacy-proxy-route-rate-limits.db")
|
||||
if err := registerSharding(db, "sqlite"); err != nil {
|
||||
t.Fatalf("register sharding: %v", err)
|
||||
}
|
||||
if err := autoMigrateSchemaMetadata(db); err != nil {
|
||||
t.Fatalf("auto migrate schema metadata: %v", err)
|
||||
}
|
||||
|
||||
for _, item := range registeredModels() {
|
||||
if _, ok := item.(*ProxyRoute); ok {
|
||||
continue
|
||||
}
|
||||
if err := db.AutoMigrate(item); err != nil {
|
||||
t.Fatalf("auto migrate supporting table: %v", err)
|
||||
}
|
||||
}
|
||||
if err := db.AutoMigrate(&legacyProxyRouteV5{}); err != nil {
|
||||
t.Fatalf("auto migrate legacy proxy_routes v5: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
if err := db.Create(&legacyProxyRouteV5{
|
||||
SiteName: "main-site",
|
||||
Domain: "app.example.com",
|
||||
Domains: `["app.example.com","www.example.com"]`,
|
||||
OriginURL: "https://origin-a.internal:8443",
|
||||
Upstreams: `["https://origin-a.internal:8443"]`,
|
||||
Enabled: true,
|
||||
EnableHTTPS: false,
|
||||
RedirectHTTP: false,
|
||||
CacheEnabled: false,
|
||||
CachePolicy: "",
|
||||
CacheRules: `[]`,
|
||||
CustomHeaders: `[]`,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed legacy proxy route v5: %v", err)
|
||||
}
|
||||
if err := saveDatabaseSchemaVersion(db, 5); err != nil {
|
||||
t.Fatalf("save schema version: %v", err)
|
||||
}
|
||||
|
||||
previousDB := DB
|
||||
DB = db
|
||||
t.Cleanup(func() {
|
||||
DB = previousDB
|
||||
})
|
||||
|
||||
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
|
||||
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
|
||||
}
|
||||
|
||||
var route ProxyRoute
|
||||
if err := db.First(&route).Error; err != nil {
|
||||
t.Fatalf("query migrated proxy route: %v", err)
|
||||
}
|
||||
if route.LimitConnPerServer != 0 || route.LimitConnPerIP != 0 || route.LimitRate != "" {
|
||||
t.Fatalf("expected new rate limit fields to default to disabled values, got %+v", route)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteCertificateListFields(t *testing.T) {
|
||||
db := openBareTestSQLiteDB(t, "legacy-proxy-route-cert-ids.db")
|
||||
if err := registerSharding(db, "sqlite"); err != nil {
|
||||
t.Fatalf("register sharding: %v", err)
|
||||
}
|
||||
if err := autoMigrateSchemaMetadata(db); err != nil {
|
||||
t.Fatalf("auto migrate schema metadata: %v", err)
|
||||
}
|
||||
|
||||
for _, item := range registeredModels() {
|
||||
if _, ok := item.(*ProxyRoute); ok {
|
||||
continue
|
||||
}
|
||||
if err := db.AutoMigrate(item); err != nil {
|
||||
t.Fatalf("auto migrate supporting table: %v", err)
|
||||
}
|
||||
}
|
||||
if err := db.AutoMigrate(&legacyProxyRouteV6{}); err != nil {
|
||||
t.Fatalf("auto migrate legacy proxy_routes v6: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
certID := uint(9)
|
||||
if err := db.Create(&legacyProxyRouteV6{
|
||||
SiteName: "secure-site",
|
||||
Domain: "secure.example.com",
|
||||
Domains: `["secure.example.com","www.secure.example.com"]`,
|
||||
OriginURL: "https://origin-secure.internal:8443",
|
||||
Upstreams: `["https://origin-secure.internal:8443"]`,
|
||||
Enabled: true,
|
||||
EnableHTTPS: true,
|
||||
CertID: &certID,
|
||||
RedirectHTTP: true,
|
||||
LimitConnPerServer: 120,
|
||||
LimitConnPerIP: 12,
|
||||
LimitRate: "512k",
|
||||
CacheEnabled: false,
|
||||
CachePolicy: "",
|
||||
CacheRules: `[]`,
|
||||
CustomHeaders: `[]`,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed legacy proxy route v6: %v", err)
|
||||
}
|
||||
if err := saveDatabaseSchemaVersion(db, 6); err != nil {
|
||||
t.Fatalf("save schema version: %v", err)
|
||||
}
|
||||
|
||||
previousDB := DB
|
||||
DB = db
|
||||
t.Cleanup(func() {
|
||||
DB = previousDB
|
||||
})
|
||||
|
||||
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
|
||||
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
|
||||
}
|
||||
|
||||
var route ProxyRoute
|
||||
if err := db.First(&route).Error; err != nil {
|
||||
t.Fatalf("query migrated proxy route: %v", err)
|
||||
}
|
||||
if route.CertID == nil || *route.CertID != certID {
|
||||
t.Fatalf("expected cert_id mirror to be preserved, got %+v", route.CertID)
|
||||
}
|
||||
|
||||
var certIDs []uint
|
||||
if err := json.Unmarshal([]byte(route.CertIDs), &certIDs); err != nil {
|
||||
t.Fatalf("decode migrated cert_ids: %v", err)
|
||||
}
|
||||
if len(certIDs) != 1 || certIDs[0] != certID {
|
||||
t.Fatalf("unexpected migrated cert_ids: %#v", certIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteDomainCertificateFields(t *testing.T) {
|
||||
db := openBareTestSQLiteDB(t, "legacy-proxy-route-domain-cert-ids.db")
|
||||
if err := registerSharding(db, "sqlite"); err != nil {
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
package migrate
|
||||
|
||||
// Versions 1 through 7 are treated as the historical baseline. There are no
|
||||
// supported deployments below v8, so new upgrades start from this base version.
|
||||
@@ -0,0 +1,52 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const BaseDatabaseSchemaVersion = 7
|
||||
|
||||
type Context interface {
|
||||
ApplyCurrentSchema(db *gorm.DB, backend string) error
|
||||
BackfillOriginsFromProxyRoutes(db *gorm.DB) error
|
||||
BackfillProxyRouteSiteFields(db *gorm.DB) error
|
||||
EnsureProxyRouteSiteNameUniqueIndex(db *gorm.DB) error
|
||||
BackfillProxyRouteCertificateFields(db *gorm.DB) error
|
||||
BackfillProxyRouteDomainCertificateFields(db *gorm.DB) error
|
||||
EnsureDefaultGitHubAuthSource(db *gorm.DB) error
|
||||
EnsureDefaultWAFRuleGroup(db *gorm.DB) error
|
||||
ValidateDatabaseSchemaVersion(db *gorm.DB, backend string, version int) error
|
||||
}
|
||||
|
||||
type Migration struct {
|
||||
FromVersion int
|
||||
ToVersion int
|
||||
Migrate func(ctx Context, db *gorm.DB, backend string) error
|
||||
Validate func(ctx Context, db *gorm.DB, backend string) error
|
||||
}
|
||||
|
||||
var registeredMigrations []Migration
|
||||
|
||||
func Register(migration Migration) {
|
||||
registeredMigrations = append(registeredMigrations, migration)
|
||||
}
|
||||
|
||||
func Migrations() []Migration {
|
||||
migrations := append([]Migration{}, registeredMigrations...)
|
||||
sort.Slice(migrations, func(i int, j int) bool {
|
||||
return migrations[i].FromVersion < migrations[j].FromVersion
|
||||
})
|
||||
return migrations
|
||||
}
|
||||
|
||||
func CurrentVersion() int {
|
||||
version := BaseDatabaseSchemaVersion
|
||||
for _, migration := range registeredMigrations {
|
||||
if migration.ToVersion > version {
|
||||
version = migration.ToVersion
|
||||
}
|
||||
}
|
||||
return version
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package migrate
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMigrationsAreContinuousFromBaseVersion(t *testing.T) {
|
||||
migrations := Migrations()
|
||||
if len(migrations) == 0 {
|
||||
t.Fatal("expected at least one registered migration")
|
||||
}
|
||||
expectedFrom := BaseDatabaseSchemaVersion
|
||||
for _, migration := range migrations {
|
||||
if migration.FromVersion != expectedFrom {
|
||||
t.Fatalf("expected migration from v%d, got v%d -> v%d", expectedFrom, migration.FromVersion, migration.ToVersion)
|
||||
}
|
||||
if migration.ToVersion != migration.FromVersion+1 {
|
||||
t.Fatalf("expected one-step migration, got v%d -> v%d", migration.FromVersion, migration.ToVersion)
|
||||
}
|
||||
expectedFrom = migration.ToVersion
|
||||
}
|
||||
if CurrentVersion() != expectedFrom {
|
||||
t.Fatalf("unexpected current version: got %d want %d", CurrentVersion(), expectedFrom)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package migrate
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
func init() {
|
||||
Register(V10())
|
||||
}
|
||||
|
||||
func V10() Migration {
|
||||
return Migration{
|
||||
FromVersion: 9,
|
||||
ToVersion: 10,
|
||||
Migrate: migrateV10,
|
||||
Validate: validateV10,
|
||||
}
|
||||
}
|
||||
|
||||
func migrateV10(ctx Context, db *gorm.DB, backend string) error {
|
||||
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.EnsureDefaultGitHubAuthSource(db)
|
||||
}
|
||||
|
||||
func validateV10(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, 10)
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package migrate
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
func init() {
|
||||
Register(V11())
|
||||
}
|
||||
|
||||
func V11() Migration {
|
||||
return Migration{
|
||||
FromVersion: 10,
|
||||
ToVersion: 11,
|
||||
Migrate: migrateV11,
|
||||
Validate: validateV11,
|
||||
}
|
||||
}
|
||||
|
||||
func migrateV11(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ApplyCurrentSchema(db, backend)
|
||||
}
|
||||
|
||||
func validateV11(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, 11)
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package migrate
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
func init() {
|
||||
Register(V12())
|
||||
}
|
||||
|
||||
func V12() Migration {
|
||||
return Migration{
|
||||
FromVersion: 11,
|
||||
ToVersion: 12,
|
||||
Migrate: migrateV12,
|
||||
Validate: validateV12,
|
||||
}
|
||||
}
|
||||
|
||||
func migrateV12(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ApplyCurrentSchema(db, backend)
|
||||
}
|
||||
|
||||
func validateV12(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, 12)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package migrate
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
func init() {
|
||||
Register(V13())
|
||||
}
|
||||
|
||||
func V13() Migration {
|
||||
return Migration{
|
||||
FromVersion: 12,
|
||||
ToVersion: 13,
|
||||
Migrate: migrateV13,
|
||||
Validate: validateV13,
|
||||
}
|
||||
}
|
||||
|
||||
func migrateV13(ctx Context, db *gorm.DB, backend string) error {
|
||||
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.EnsureDefaultWAFRuleGroup(db)
|
||||
}
|
||||
|
||||
func validateV13(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, 13)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package migrate
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
func init() {
|
||||
Register(V14())
|
||||
}
|
||||
|
||||
func V14() Migration {
|
||||
return Migration{
|
||||
FromVersion: 13,
|
||||
ToVersion: 14,
|
||||
Migrate: migrateV14,
|
||||
Validate: validateV14,
|
||||
}
|
||||
}
|
||||
|
||||
func migrateV14(ctx Context, db *gorm.DB, backend string) error {
|
||||
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.EnsureDefaultWAFRuleGroup(db)
|
||||
}
|
||||
|
||||
func validateV14(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, 14)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type nodeV15 struct {
|
||||
IPManualOverride bool `gorm:"column:ip_manual_override;not null;default:false"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
Register(V15())
|
||||
}
|
||||
|
||||
func V15() Migration {
|
||||
return Migration{
|
||||
FromVersion: 14,
|
||||
ToVersion: 15,
|
||||
Migrate: migrateV15,
|
||||
Validate: validateV15,
|
||||
}
|
||||
}
|
||||
|
||||
func (nodeV15) TableName() string {
|
||||
return "nodes"
|
||||
}
|
||||
|
||||
func migrateV15(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ApplyCurrentSchema(db, backend)
|
||||
}
|
||||
|
||||
func validateV15(ctx Context, db *gorm.DB, backend string) error {
|
||||
if err := ctx.ValidateDatabaseSchemaVersion(db, backend, 14); err != nil {
|
||||
return err
|
||||
}
|
||||
if db == nil || !db.Migrator().HasColumn(&nodeV15{}, "ip_manual_override") {
|
||||
return fmt.Errorf("column nodes.ip_manual_override is missing")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package migrate
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
func init() {
|
||||
Register(V8())
|
||||
}
|
||||
|
||||
func V8() Migration {
|
||||
return Migration{
|
||||
FromVersion: 7,
|
||||
ToVersion: 8,
|
||||
Migrate: migrateV8,
|
||||
Validate: validateV8,
|
||||
}
|
||||
}
|
||||
|
||||
func migrateV8(ctx Context, db *gorm.DB, backend string) error {
|
||||
if err := ctx.ApplyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.BackfillOriginsFromProxyRoutes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.BackfillProxyRouteSiteFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.EnsureProxyRouteSiteNameUniqueIndex(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.BackfillProxyRouteCertificateFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.BackfillProxyRouteDomainCertificateFields(db)
|
||||
}
|
||||
|
||||
func validateV8(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, 8)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package migrate
|
||||
|
||||
import "gorm.io/gorm"
|
||||
|
||||
func init() {
|
||||
Register(V9())
|
||||
}
|
||||
|
||||
func V9() Migration {
|
||||
return Migration{
|
||||
FromVersion: 8,
|
||||
ToVersion: 9,
|
||||
Migrate: migrateV9,
|
||||
Validate: validateV9,
|
||||
}
|
||||
}
|
||||
|
||||
func migrateV9(ctx Context, db *gorm.DB, backend string) error {
|
||||
if err := migrateV8(ctx, db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateV9(ctx Context, db *gorm.DB, backend string) error {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, 9)
|
||||
}
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/url"
|
||||
schemamigrate "openflare/model/migrate"
|
||||
"strings"
|
||||
|
||||
"gorm.io/gorm"
|
||||
@@ -20,6 +22,63 @@ type databaseSchemaMigration struct {
|
||||
validate func(db *gorm.DB, backend string) error
|
||||
}
|
||||
|
||||
type databaseSchemaMigrationContext struct{}
|
||||
|
||||
func (databaseSchemaMigrationContext) ApplyCurrentSchema(db *gorm.DB, backend string) error {
|
||||
return applyCurrentSchema(db, backend)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) BackfillOriginsFromProxyRoutes(db *gorm.DB) error {
|
||||
return backfillOriginsFromProxyRoutes(db)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) BackfillProxyRouteSiteFields(db *gorm.DB) error {
|
||||
return backfillProxyRouteSiteFields(db)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) EnsureProxyRouteSiteNameUniqueIndex(db *gorm.DB) error {
|
||||
return ensureProxyRouteSiteNameUniqueIndex(db)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) BackfillProxyRouteCertificateFields(db *gorm.DB) error {
|
||||
return backfillProxyRouteCertificateFields(db)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) BackfillProxyRouteDomainCertificateFields(db *gorm.DB) error {
|
||||
return backfillProxyRouteDomainCertificateFields(db)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) EnsureDefaultGitHubAuthSource(db *gorm.DB) error {
|
||||
return ensureDefaultGitHubAuthSource(db)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) EnsureDefaultWAFRuleGroup(db *gorm.DB) error {
|
||||
return ensureDefaultWAFRuleGroup(db)
|
||||
}
|
||||
|
||||
func (databaseSchemaMigrationContext) ValidateDatabaseSchemaVersion(db *gorm.DB, backend string, version int) error {
|
||||
switch version {
|
||||
case 7:
|
||||
return validateDatabaseSchemaV7(db, backend)
|
||||
case 8:
|
||||
return validateDatabaseSchemaV8(db, backend)
|
||||
case 9:
|
||||
return validateDatabaseSchemaV9(db, backend)
|
||||
case 10:
|
||||
return validateDatabaseSchemaV10(db, backend)
|
||||
case 11:
|
||||
return validateDatabaseSchemaV11(db, backend)
|
||||
case 12:
|
||||
return validateDatabaseSchemaV12(db, backend)
|
||||
case 13:
|
||||
return validateDatabaseSchemaV13(db, backend)
|
||||
case 14:
|
||||
return validateDatabaseSchemaV14(db, backend)
|
||||
default:
|
||||
return fmt.Errorf("database schema validation for v%d is not defined", version)
|
||||
}
|
||||
}
|
||||
|
||||
func autoMigrateSchemaMetadata(db *gorm.DB) error {
|
||||
for _, item := range schemaMetadataModels() {
|
||||
if err := db.AutoMigrate(item); err != nil {
|
||||
@@ -789,225 +848,6 @@ func uintSlicesEqualForMigration(left []uint, right []uint) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func renameLegacyObservabilityShardTables(db *gorm.DB) error {
|
||||
for _, baseTable := range shardedObservabilityBaseTables() {
|
||||
for _, table := range observabilityShardTables(baseTable) {
|
||||
legacyTable := legacyObservabilityShardTableName(table)
|
||||
if db.Migrator().HasTable(legacyTable) {
|
||||
return fmt.Errorf("legacy sharded table %s already exists", legacyTable)
|
||||
}
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := db.Migrator().RenameTable(table, legacyTable); err != nil {
|
||||
return fmt.Errorf("rename sharded table %s to %s failed: %w", table, legacyTable, err)
|
||||
}
|
||||
if err := dropLegacyObservabilitySecondaryIndexes(db, legacyTable); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dropLegacyObservabilitySecondaryIndexes(db *gorm.DB, table string) error {
|
||||
db = sessionIgnoringSharding(db)
|
||||
if db == nil {
|
||||
return fmt.Errorf("database handle is nil")
|
||||
}
|
||||
backend := baseDialector(db).Name()
|
||||
indexes := make([]string, 0)
|
||||
switch backend {
|
||||
case "sqlite":
|
||||
if err := db.Raw(
|
||||
`SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ? AND name LIKE 'idx_%'`,
|
||||
table,
|
||||
).Scan(&indexes).Error; err != nil {
|
||||
return fmt.Errorf("list indexes for %s failed: %w", table, err)
|
||||
}
|
||||
case "postgres":
|
||||
if err := db.Raw(
|
||||
`SELECT indexname FROM pg_indexes WHERE schemaname = current_schema() AND tablename = ? AND indexname LIKE 'idx_%'`,
|
||||
table,
|
||||
).Scan(&indexes).Error; err != nil {
|
||||
return fmt.Errorf("list indexes for %s failed: %w", table, err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported database backend %s", backend)
|
||||
}
|
||||
for _, indexName := range indexes {
|
||||
if err := db.Exec(fmt.Sprintf(`DROP INDEX IF EXISTS "%s"`, indexName)).Error; err != nil {
|
||||
return fmt.Errorf("drop legacy index %s failed: %w", indexName, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func autoMigrateObservabilityShardTables(db *gorm.DB) error {
|
||||
db = sessionIgnoringSharding(db)
|
||||
if db == nil {
|
||||
return fmt.Errorf("database handle is nil")
|
||||
}
|
||||
dialector := baseDialector(db)
|
||||
if dialector == nil {
|
||||
return fmt.Errorf("database dialector is nil")
|
||||
}
|
||||
type shardedTable struct {
|
||||
model any
|
||||
base string
|
||||
}
|
||||
tables := []shardedTable{
|
||||
{model: &NodeMetricSnapshot{}, base: "node_metric_snapshots"},
|
||||
{model: &NodeRequestReport{}, base: "node_request_reports"},
|
||||
{model: &NodeAccessLog{}, base: "node_access_logs"},
|
||||
}
|
||||
for _, item := range tables {
|
||||
for _, table := range observabilityShardTables(item.base) {
|
||||
tx := db.Table(table)
|
||||
if err := dialector.Migrator(tx).AutoMigrate(item.model); err != nil {
|
||||
return fmt.Errorf("auto migrate sharded table %s failed: %w", table, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dropLegacyObservabilityShardTables(db *gorm.DB) error {
|
||||
db = sessionIgnoringSharding(db)
|
||||
if db == nil {
|
||||
return fmt.Errorf("database handle is nil")
|
||||
}
|
||||
for _, baseTable := range shardedObservabilityBaseTables() {
|
||||
for _, table := range observabilityShardTables(baseTable) {
|
||||
legacyTable := legacyObservabilityShardTableName(table)
|
||||
if !db.Migrator().HasTable(legacyTable) {
|
||||
continue
|
||||
}
|
||||
if err := db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS "%s"`, legacyTable)).Error; err != nil {
|
||||
return fmt.Errorf("drop legacy sharded table %s failed: %w", legacyTable, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateLegacyNodeMetricSnapshots(db *gorm.DB) error {
|
||||
for _, table := range observabilityShardTables("node_metric_snapshots") {
|
||||
legacyTable := legacyObservabilityShardTableName(table)
|
||||
if !db.Migrator().HasTable(legacyTable) {
|
||||
continue
|
||||
}
|
||||
var lastSeenID uint
|
||||
for {
|
||||
var rows []NodeMetricSnapshot
|
||||
query := db.Table(legacyTable).Order("id ASC").Limit(500)
|
||||
if lastSeenID > 0 {
|
||||
query = query.Where("id > ?", lastSeenID)
|
||||
}
|
||||
if err := query.Find(&rows).Error; err != nil {
|
||||
return fmt.Errorf("query legacy sharded table %s failed: %w", legacyTable, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
lastSeenID = rows[len(rows)-1].ID
|
||||
grouped := make(map[string][]NodeMetricSnapshot, observabilityShardCount)
|
||||
for index := range rows {
|
||||
rows[index].ID = 0
|
||||
if err := assignObservabilityID(&rows[index].ID); err != nil {
|
||||
return err
|
||||
}
|
||||
targetTable := observabilityShardTableForID("node_metric_snapshots", rows[index].ID)
|
||||
grouped[targetTable] = append(grouped[targetTable], rows[index])
|
||||
}
|
||||
for targetTable, batch := range grouped {
|
||||
if err := db.Table(targetTable).Create(&batch).Error; err != nil {
|
||||
return fmt.Errorf("write migrated rows into %s failed: %w", targetTable, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateLegacyNodeRequestReports(db *gorm.DB) error {
|
||||
for _, table := range observabilityShardTables("node_request_reports") {
|
||||
legacyTable := legacyObservabilityShardTableName(table)
|
||||
if !db.Migrator().HasTable(legacyTable) {
|
||||
continue
|
||||
}
|
||||
var lastSeenID uint
|
||||
for {
|
||||
var rows []NodeRequestReport
|
||||
query := db.Table(legacyTable).Order("id ASC").Limit(500)
|
||||
if lastSeenID > 0 {
|
||||
query = query.Where("id > ?", lastSeenID)
|
||||
}
|
||||
if err := query.Find(&rows).Error; err != nil {
|
||||
return fmt.Errorf("query legacy sharded table %s failed: %w", legacyTable, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
lastSeenID = rows[len(rows)-1].ID
|
||||
grouped := make(map[string][]NodeRequestReport, observabilityShardCount)
|
||||
for index := range rows {
|
||||
rows[index].ID = 0
|
||||
if err := assignObservabilityID(&rows[index].ID); err != nil {
|
||||
return err
|
||||
}
|
||||
targetTable := observabilityShardTableForID("node_request_reports", rows[index].ID)
|
||||
grouped[targetTable] = append(grouped[targetTable], rows[index])
|
||||
}
|
||||
for targetTable, batch := range grouped {
|
||||
if err := db.Table(targetTable).Create(&batch).Error; err != nil {
|
||||
return fmt.Errorf("write migrated rows into %s failed: %w", targetTable, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateLegacyNodeAccessLogs(db *gorm.DB) error {
|
||||
for _, table := range observabilityShardTables("node_access_logs") {
|
||||
legacyTable := legacyObservabilityShardTableName(table)
|
||||
if !db.Migrator().HasTable(legacyTable) {
|
||||
continue
|
||||
}
|
||||
var lastSeenID uint
|
||||
for {
|
||||
var rows []NodeAccessLog
|
||||
query := db.Table(legacyTable).Order("id ASC").Limit(500)
|
||||
if lastSeenID > 0 {
|
||||
query = query.Where("id > ?", lastSeenID)
|
||||
}
|
||||
if err := query.Find(&rows).Error; err != nil {
|
||||
return fmt.Errorf("query legacy sharded table %s failed: %w", legacyTable, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
lastSeenID = rows[len(rows)-1].ID
|
||||
grouped := make(map[string][]NodeAccessLog, observabilityShardCount)
|
||||
for index := range rows {
|
||||
rows[index].ID = 0
|
||||
if err := assignObservabilityID(&rows[index].ID); err != nil {
|
||||
return err
|
||||
}
|
||||
targetTable := observabilityShardTableForID("node_access_logs", rows[index].ID)
|
||||
grouped[targetTable] = append(grouped[targetTable], rows[index])
|
||||
}
|
||||
for targetTable, batch := range grouped {
|
||||
if err := db.Table(targetTable).Create(&batch).Error; err != nil {
|
||||
return fmt.Errorf("write migrated rows into %s failed: %w", targetTable, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeOriginAddressForMigration(raw string) string {
|
||||
return strings.ToLower(strings.TrimSpace(raw))
|
||||
}
|
||||
@@ -1086,136 +926,6 @@ func backfillOriginsFromProxyRoutes(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateV2 upgrades the legacy schema to the first versioned schema by
|
||||
// creating schema metadata, applying the current tables, and backfilling
|
||||
// compatibility columns.
|
||||
func migrateV2(db *gorm.DB, backend string) error {
|
||||
return applyCurrentSchema(db, backend)
|
||||
}
|
||||
|
||||
// migrateV3 upgrades observability shard tables from legacy ID layout to the
|
||||
// current ID-sharded layout and migrates existing shard data into the new tables.
|
||||
func migrateV3(db *gorm.DB, backend string) error {
|
||||
if db == nil {
|
||||
return fmt.Errorf("database handle is nil")
|
||||
}
|
||||
_ = backend
|
||||
if err := renameLegacyObservabilityShardTables(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := autoMigrateObservabilityShardTables(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateLegacyNodeMetricSnapshots(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateLegacyNodeRequestReports(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateLegacyNodeAccessLogs(db); err != nil {
|
||||
return err
|
||||
}
|
||||
return dropLegacyObservabilityShardTables(db)
|
||||
}
|
||||
|
||||
// migrateV4 introduces the origins schema and backfills proxy route origin
|
||||
// references from existing origin_url values.
|
||||
func migrateV4(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return backfillOriginsFromProxyRoutes(db)
|
||||
}
|
||||
|
||||
// migrateV5 upgrades proxy_routes to website-level identity fields by
|
||||
// backfilling site_name and domains while keeping domain as the primary-domain
|
||||
// compatibility mirror.
|
||||
func migrateV5(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillOriginsFromProxyRoutes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillProxyRouteSiteFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureProxyRouteSiteNameUniqueIndex(db)
|
||||
}
|
||||
|
||||
// migrateV6 adds structured website-level rate limit fields to proxy_routes.
|
||||
func migrateV6(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillOriginsFromProxyRoutes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillProxyRouteSiteFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureProxyRouteSiteNameUniqueIndex(db)
|
||||
}
|
||||
|
||||
// migrateV7 adds structured website-level certificate lists to proxy_routes
|
||||
// while keeping cert_id as the primary certificate compatibility mirror.
|
||||
func migrateV7(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillOriginsFromProxyRoutes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillProxyRouteSiteFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureProxyRouteSiteNameUniqueIndex(db); err != nil {
|
||||
return err
|
||||
}
|
||||
return backfillProxyRouteCertificateFields(db)
|
||||
}
|
||||
|
||||
// migrateV8 adds per-domain certificate assignments to proxy_routes while
|
||||
// keeping cert_ids as the website-level compatibility mirror.
|
||||
func migrateV8(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillOriginsFromProxyRoutes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillProxyRouteSiteFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureProxyRouteSiteNameUniqueIndex(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillProxyRouteCertificateFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
return backfillProxyRouteDomainCertificateFields(db)
|
||||
}
|
||||
|
||||
// migrateV9 adds PoW (Proof-of-Work) anti-bot protection fields to proxy_routes.
|
||||
func migrateV9(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillOriginsFromProxyRoutes(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillProxyRouteSiteFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureProxyRouteSiteNameUniqueIndex(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backfillProxyRouteCertificateFields(db); err != nil {
|
||||
return err
|
||||
}
|
||||
return backfillProxyRouteDomainCertificateFields(db)
|
||||
}
|
||||
|
||||
func ensureDefaultGitHubAuthSource(db *gorm.DB) error {
|
||||
if db == nil || !db.Migrator().HasTable(&AuthSource{}) || !db.Migrator().HasTable(&ExternalAccount{}) {
|
||||
return nil
|
||||
@@ -1310,14 +1020,6 @@ func ensureDefaultGitHubAuthSource(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateV10 adds configurable auth sources and external account bindings.
|
||||
func migrateV10(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureDefaultGitHubAuthSource(db)
|
||||
}
|
||||
|
||||
func validateDatabaseSchemaV9(db *gorm.DB, backend string) error {
|
||||
if err := validateDatabaseSchemaV8(db, backend); err != nil {
|
||||
return err
|
||||
@@ -1344,15 +1046,6 @@ func validateDatabaseSchemaV10(db *gorm.DB, backend string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateV11 adds acme and dns accounts and extends tls_certificates.
|
||||
func migrateV11(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
// Default values will be applied by gorm for new columns automatically during AutoMigrate.
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDatabaseSchemaV11(db *gorm.DB, backend string) error {
|
||||
if err := validateDatabaseSchemaV10(db, backend); err != nil {
|
||||
return err
|
||||
@@ -1369,15 +1062,6 @@ func validateDatabaseSchemaV11(db *gorm.DB, backend string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateV12 adds basic authentication fields to proxy_routes.
|
||||
func migrateV12(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
// Default values will be applied by gorm for new columns automatically during AutoMigrate.
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDatabaseSchemaV12(db *gorm.DB, backend string) error {
|
||||
if err := validateDatabaseSchemaV11(db, backend); err != nil {
|
||||
return err
|
||||
@@ -1423,14 +1107,6 @@ func ensureDefaultWAFRuleGroup(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateV13 adds WAF rule groups and website bindings.
|
||||
func migrateV13(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureDefaultWAFRuleGroup(db)
|
||||
}
|
||||
|
||||
func validateDatabaseSchemaV13(db *gorm.DB, backend string) error {
|
||||
if err := validateDatabaseSchemaV12(db, backend); err != nil {
|
||||
return err
|
||||
@@ -1451,14 +1127,6 @@ func validateDatabaseSchemaV13(db *gorm.DB, backend string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateV14 adds PoW policy fields to WAF rule groups.
|
||||
func migrateV14(db *gorm.DB, backend string) error {
|
||||
if err := applyCurrentSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return ensureDefaultWAFRuleGroup(db)
|
||||
}
|
||||
|
||||
func validateDatabaseSchemaV14(db *gorm.DB, backend string) error {
|
||||
if err := validateDatabaseSchemaV13(db, backend); err != nil {
|
||||
return err
|
||||
@@ -1472,38 +1140,42 @@ func validateDatabaseSchemaV14(db *gorm.DB, backend string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateV15 adds the node IP manual override flag.
|
||||
func migrateV15(db *gorm.DB, backend string) error {
|
||||
return applyCurrentSchema(db, backend)
|
||||
func databaseSchemaMigrations() []databaseSchemaMigration {
|
||||
ctx := databaseSchemaMigrationContext{}
|
||||
migrations := []databaseSchemaMigration{}
|
||||
for _, item := range schemamigrate.Migrations() {
|
||||
external := item
|
||||
migrations = append(migrations, databaseSchemaMigration{
|
||||
fromVersion: external.FromVersion,
|
||||
toVersion: external.ToVersion,
|
||||
migrate: func(db *gorm.DB, backend string) error {
|
||||
return external.Migrate(ctx, db, backend)
|
||||
},
|
||||
validate: func(db *gorm.DB, backend string) error {
|
||||
return validateExternalDatabaseSchema(ctx, db, backend, external.ToVersion)
|
||||
},
|
||||
})
|
||||
}
|
||||
return migrations
|
||||
}
|
||||
|
||||
func validateDatabaseSchemaV15(db *gorm.DB, backend string) error {
|
||||
if err := validateDatabaseSchemaV14(db, backend); err != nil {
|
||||
return err
|
||||
func validateExternalDatabaseSchema(ctx databaseSchemaMigrationContext, db *gorm.DB, backend string, targetVersion int) error {
|
||||
if targetVersion <= schemamigrate.BaseDatabaseSchemaVersion {
|
||||
return ctx.ValidateDatabaseSchemaVersion(db, backend, targetVersion)
|
||||
}
|
||||
if !db.Migrator().HasColumn(&Node{}, "ip_manual_override") {
|
||||
return fmt.Errorf("column nodes.ip_manual_override is missing")
|
||||
for _, migration := range schemamigrate.Migrations() {
|
||||
if migration.ToVersion > targetVersion {
|
||||
continue
|
||||
}
|
||||
if err := migration.Validate(ctx, db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func databaseSchemaMigrations() []databaseSchemaMigration {
|
||||
return []databaseSchemaMigration{
|
||||
{fromVersion: 1, toVersion: 2, migrate: migrateV2, validate: validateDatabaseSchemaV2},
|
||||
{fromVersion: 2, toVersion: 3, migrate: migrateV3, validate: validateDatabaseSchemaV3},
|
||||
{fromVersion: 3, toVersion: 4, migrate: migrateV4, validate: validateDatabaseSchemaV4},
|
||||
{fromVersion: 4, toVersion: 5, migrate: migrateV5, validate: validateDatabaseSchemaV5},
|
||||
{fromVersion: 5, toVersion: 6, migrate: migrateV6, validate: validateDatabaseSchemaV6},
|
||||
{fromVersion: 6, toVersion: 7, migrate: migrateV7, validate: validateDatabaseSchemaV7},
|
||||
{fromVersion: 7, toVersion: 8, migrate: migrateV8, validate: validateDatabaseSchemaV8},
|
||||
{fromVersion: 8, toVersion: 9, migrate: migrateV9, validate: validateDatabaseSchemaV9},
|
||||
{fromVersion: 9, toVersion: 10, migrate: migrateV10, validate: validateDatabaseSchemaV10},
|
||||
{fromVersion: 10, toVersion: 11, migrate: migrateV11, validate: validateDatabaseSchemaV11},
|
||||
{fromVersion: 11, toVersion: 12, migrate: migrateV12, validate: validateDatabaseSchemaV12},
|
||||
{fromVersion: 12, toVersion: 13, migrate: migrateV13, validate: validateDatabaseSchemaV13},
|
||||
{fromVersion: 13, toVersion: 14, migrate: migrateV14, validate: validateDatabaseSchemaV14},
|
||||
{fromVersion: 14, toVersion: 15, migrate: migrateV15, validate: validateDatabaseSchemaV15},
|
||||
}
|
||||
func validateCurrentDatabaseSchema(db *gorm.DB, backend string) error {
|
||||
return validateExternalDatabaseSchema(databaseSchemaMigrationContext{}, db, backend, currentDatabaseSchemaVersion)
|
||||
}
|
||||
|
||||
func databaseSchemaMigrationMap() map[int]databaseSchemaMigration {
|
||||
@@ -1546,6 +1218,10 @@ func upgradeDatabaseSchema(db *gorm.DB, backend string, version int) error {
|
||||
if version > currentDatabaseSchemaVersion {
|
||||
return fmt.Errorf("database schema version %d is newer than application version %d", version, currentDatabaseSchemaVersion)
|
||||
}
|
||||
if version < legacyDatabaseSchemaVersion {
|
||||
slog.Warn("database schema version is below supported baseline; treating it as historical initial schema", "version", version, "baseline", legacyDatabaseSchemaVersion)
|
||||
version = legacyDatabaseSchemaVersion
|
||||
}
|
||||
if version == currentDatabaseSchemaVersion {
|
||||
return nil
|
||||
}
|
||||
@@ -1591,7 +1267,7 @@ func initializeFreshDatabaseSchema(db *gorm.DB, backend string) error {
|
||||
if err := ensureDefaultWAFRuleGroup(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDatabaseSchemaV15(db, backend); err != nil {
|
||||
if err := validateCurrentDatabaseSchema(db, backend); err != nil {
|
||||
return err
|
||||
}
|
||||
return saveDatabaseSchemaVersion(db, currentDatabaseSchemaVersion)
|
||||
|
||||
Reference in New Issue
Block a user