tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-10 17:26:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

[优化] 重构数据库迁移逻辑，添加版本管理和验证功能

Browse Source
This commit is contained in:
ryan
2026-05-31 14:32:43 +08:00
parent 4ae6c2718f
commit f365b3d331
16 changed files with 421 additions and 864 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openflare_server/model/main_test.go
-443
View File
@@ -10,85 +10,6 @@ import (
"gorm.io/gorm"
)
type legacyProxyRouteV4 struct {
ID uint `gorm:"primaryKey"`
Domain string `gorm:"uniqueIndex;size:255;not null"`
OriginID *uint `gorm:"index"`
OriginURL string `gorm:"size:2048;not null"`
OriginHost string `gorm:"size:255"`
Upstreams string `gorm:"type:text;not null;default:'[]'"`
Enabled bool `gorm:"not null;default:true"`
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
CertID *uint
RedirectHTTP bool `gorm:"not null;default:false"`
CacheEnabled bool `gorm:"not null;default:false"`
CachePolicy string `gorm:"size:32;not null;default:''"`
CacheRules string `gorm:"type:text;not null;default:'[]'"`
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
Remark string `gorm:"size:255"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (legacyProxyRouteV4) TableName() string {
return "proxy_routes"
}
type legacyProxyRouteV5 struct {
ID uint `gorm:"primaryKey"`
SiteName string `gorm:"size:255;not null;default:''"`
Domain string `gorm:"uniqueIndex;size:255;not null"`
Domains string `gorm:"type:text;not null;default:'[]'"`
OriginID *uint `gorm:"index"`
OriginURL string `gorm:"size:2048;not null"`
OriginHost string `gorm:"size:255"`
Upstreams string `gorm:"type:text;not null;default:'[]'"`
Enabled bool `gorm:"not null;default:true"`
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
CertID *uint
RedirectHTTP bool `gorm:"not null;default:false"`
CacheEnabled bool `gorm:"not null;default:false"`
CachePolicy string `gorm:"size:32;not null;default:''"`
CacheRules string `gorm:"type:text;not null;default:'[]'"`
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
Remark string `gorm:"size:255"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (legacyProxyRouteV5) TableName() string {
return "proxy_routes"
}
type legacyProxyRouteV6 struct {
ID uint `gorm:"primaryKey"`
SiteName string `gorm:"size:255;not null;default:''"`
Domain string `gorm:"uniqueIndex;size:255;not null"`
Domains string `gorm:"type:text;not null;default:'[]'"`
OriginID *uint `gorm:"index"`
OriginURL string `gorm:"size:2048;not null"`
OriginHost string `gorm:"size:255"`
Upstreams string `gorm:"type:text;not null;default:'[]'"`
Enabled bool `gorm:"not null;default:true"`
EnableHTTPS bool `gorm:"column:enable_https;not null;default:false"`
CertID *uint
RedirectHTTP bool `gorm:"not null;default:false"`
LimitConnPerServer int `gorm:"not null;default:0"`
LimitConnPerIP int `gorm:"not null;default:0"`
LimitRate string `gorm:"size:32;not null;default:''"`
CacheEnabled bool `gorm:"not null;default:false"`
CachePolicy string `gorm:"size:32;not null;default:''"`
CacheRules string `gorm:"type:text;not null;default:'[]'"`
CustomHeaders string `gorm:"type:text;not null;default:'[]'"`
Remark string `gorm:"size:255"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (legacyProxyRouteV6) TableName() string {
return "proxy_routes"
}
type legacyProxyRouteV7 struct {
ID uint `gorm:"primaryKey"`
SiteName string `gorm:"size:255;not null;default:''"`
@@ -365,159 +286,6 @@ func TestEnsureDatabaseSchemaUpToDateUpgradesLegacyDatabase(t *testing.T) {
}
}
func TestEnsureDatabaseSchemaUpToDateMigratesObservabilityShardsToID(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-observability-shards.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateAll(db); err != nil {
t.Fatalf("auto migrate db: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
now := time.Now().UTC()
if err := db.Table("node_metric_snapshots_00").Create(&NodeMetricSnapshot{
ID: 1,
NodeID: "node-a",
CapturedAt: now.Add(-2 * time.Minute),
CPUUsagePercent: 22,
MemoryUsedBytes: 2,
MemoryTotalBytes: 8,
}).Error; err != nil {
t.Fatalf("seed metric snapshot shard 00: %v", err)
}
if err := db.Table("node_metric_snapshots_01").Create(&NodeMetricSnapshot{
ID: 1,
NodeID: "node-b",
CapturedAt: now.Add(-time.Minute),
CPUUsagePercent: 44,
MemoryUsedBytes: 4,
MemoryTotalBytes: 8,
}).Error; err != nil {
t.Fatalf("seed metric snapshot shard 01: %v", err)
}
if err := db.Table("node_request_reports_00").Create(&NodeRequestReport{
ID: 1,
NodeID: "node-a",
WindowStartedAt: now.Add(-3 * time.Minute),
WindowEndedAt: now.Add(-2 * time.Minute),
RequestCount: 12,
ErrorCount: 1,
UniqueVisitorCount: 6,
}).Error; err != nil {
t.Fatalf("seed request report shard 00: %v", err)
}
if err := db.Table("node_request_reports_01").Create(&NodeRequestReport{
ID: 1,
NodeID: "node-b",
WindowStartedAt: now.Add(-2 * time.Minute),
WindowEndedAt: now.Add(-time.Minute),
RequestCount: 21,
ErrorCount: 2,
UniqueVisitorCount: 9,
}).Error; err != nil {
t.Fatalf("seed request report shard 01: %v", err)
}
if err := db.Table("node_access_logs_00").Create(&NodeAccessLog{
ID: 1,
NodeID: "node-a",
LoggedAt: now.Add(-90 * time.Second),
RemoteAddr: "203.0.113.10",
Host: "a.example.com",
Path: "/alpha",
StatusCode: 200,
}).Error; err != nil {
t.Fatalf("seed access log shard 00: %v", err)
}
if err := db.Table("node_access_logs_01").Create(&NodeAccessLog{
ID: 1,
NodeID: "node-b",
LoggedAt: now.Add(-60 * time.Second),
RemoteAddr: "203.0.113.11",
Host: "b.example.com",
Path: "/beta",
StatusCode: 502,
}).Error; err != nil {
t.Fatalf("seed access log shard 01: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 2); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
version, exists, err := loadDatabaseSchemaVersion(db)
if err != nil {
t.Fatalf("loadDatabaseSchemaVersion: %v", err)
}
if !exists {
t.Fatal("expected migrated database to keep schema version record")
}
if version != currentDatabaseSchemaVersion {
t.Fatalf("unexpected schema version: got %d want %d", version, currentDatabaseSchemaVersion)
}
for _, baseTable := range shardedObservabilityBaseTables() {
for _, table := range observabilityShardTables(baseTable) {
legacyTable := legacyObservabilityShardTableName(table)
if db.Migrator().HasTable(legacyTable) {
t.Fatalf("expected legacy shard table %s to be removed", legacyTable)
}
}
}
snapshots, err := ListMetricSnapshotsSince(time.Time{})
if err != nil {
t.Fatalf("ListMetricSnapshotsSince failed: %v", err)
}
if len(snapshots) != 2 {
t.Fatalf("expected 2 migrated metric snapshots, got %+v", snapshots)
}
reports, err := ListRequestReportsSince(time.Time{})
if err != nil {
t.Fatalf("ListRequestReportsSince failed: %v", err)
}
if len(reports) != 2 {
t.Fatalf("expected 2 migrated request reports, got %+v", reports)
}
logs, err := ListNodeAccessLogs(NodeAccessLogQuery{Page: 0, PageSize: 10})
if err != nil {
t.Fatalf("ListNodeAccessLogs failed: %v", err)
}
if len(logs) != 2 {
t.Fatalf("expected 2 migrated access logs, got %+v", logs)
}
seenSnapshotIDs := make(map[uint]struct{}, len(snapshots))
for _, item := range snapshots {
if item == nil || item.ID == 0 {
t.Fatalf("expected migrated metric snapshot to have a new non-zero id: %+v", item)
}
if _, exists := seenSnapshotIDs[item.ID]; exists {
t.Fatalf("expected migrated metric snapshot ids to be unique, got duplicate %d", item.ID)
}
seenSnapshotIDs[item.ID] = struct{}{}
targetTable := observabilityShardTableForID("node_metric_snapshots", item.ID)
var count int64
if err := db.Table(targetTable).Where("id = ?", item.ID).Count(&count).Error; err != nil {
t.Fatalf("count migrated metric snapshot in target shard: %v", err)
}
if count != 1 {
t.Fatalf("expected migrated metric snapshot id %d to be stored in %s", item.ID, targetTable)
}
}
}
func TestMigrateOriginsSchemaBackfillsOrigins(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-origins.db")
if err := registerSharding(db, "sqlite"); err != nil {
@@ -573,217 +341,6 @@ func TestMigrateOriginsSchemaBackfillsOrigins(t *testing.T) {
}
}
func TestEnsureDatabaseSchemaUpToDateBackfillsProxyRouteSiteFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-sites.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
for _, item := range registeredModels() {
if _, ok := item.(*ProxyRoute); ok {
continue
}
if err := db.AutoMigrate(item); err != nil {
t.Fatalf("auto migrate supporting table: %v", err)
}
}
if err := db.AutoMigrate(&legacyProxyRouteV4{}); err != nil {
t.Fatalf("auto migrate legacy proxy_routes: %v", err)
}
now := time.Now().UTC()
if err := db.Create(&legacyProxyRouteV4{
Domain: "app.example.com",
OriginURL: "https://origin-a.internal:8443",
Upstreams: `["https://origin-a.internal:8443","https://origin-b.internal:8443"]`,
Enabled: true,
EnableHTTPS: false,
RedirectHTTP: false,
CacheEnabled: false,
CachePolicy: "",
CacheRules: `[]`,
CustomHeaders: `[]`,
CreatedAt: now,
UpdatedAt: now,
}).Error; err != nil {
t.Fatalf("seed legacy proxy route: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 4); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
var route ProxyRoute
if err := db.First(&route).Error; err != nil {
t.Fatalf("query migrated proxy route: %v", err)
}
if route.SiteName != "app.example.com" {
t.Fatalf("unexpected site_name after migration: %s", route.SiteName)
}
if route.Domain != "app.example.com" {
t.Fatalf("unexpected domain mirror after migration: %s", route.Domain)
}
var domains []string
if err := json.Unmarshal([]byte(route.Domains), &domains); err != nil {
t.Fatalf("decode migrated domains: %v", err)
}
if len(domains) != 1 || domains[0] != "app.example.com" {
t.Fatalf("unexpected migrated domains: %#v", domains)
}
}
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteRateLimitFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-rate-limits.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
for _, item := range registeredModels() {
if _, ok := item.(*ProxyRoute); ok {
continue
}
if err := db.AutoMigrate(item); err != nil {
t.Fatalf("auto migrate supporting table: %v", err)
}
}
if err := db.AutoMigrate(&legacyProxyRouteV5{}); err != nil {
t.Fatalf("auto migrate legacy proxy_routes v5: %v", err)
}
now := time.Now().UTC()
if err := db.Create(&legacyProxyRouteV5{
SiteName: "main-site",
Domain: "app.example.com",
Domains: `["app.example.com","www.example.com"]`,
OriginURL: "https://origin-a.internal:8443",
Upstreams: `["https://origin-a.internal:8443"]`,
Enabled: true,
EnableHTTPS: false,
RedirectHTTP: false,
CacheEnabled: false,
CachePolicy: "",
CacheRules: `[]`,
CustomHeaders: `[]`,
CreatedAt: now,
UpdatedAt: now,
}).Error; err != nil {
t.Fatalf("seed legacy proxy route v5: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 5); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
var route ProxyRoute
if err := db.First(&route).Error; err != nil {
t.Fatalf("query migrated proxy route: %v", err)
}
if route.LimitConnPerServer != 0 || route.LimitConnPerIP != 0 || route.LimitRate != "" {
t.Fatalf("expected new rate limit fields to default to disabled values, got %+v", route)
}
}
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteCertificateListFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-cert-ids.db")
if err := registerSharding(db, "sqlite"); err != nil {
t.Fatalf("register sharding: %v", err)
}
if err := autoMigrateSchemaMetadata(db); err != nil {
t.Fatalf("auto migrate schema metadata: %v", err)
}
for _, item := range registeredModels() {
if _, ok := item.(*ProxyRoute); ok {
continue
}
if err := db.AutoMigrate(item); err != nil {
t.Fatalf("auto migrate supporting table: %v", err)
}
}
if err := db.AutoMigrate(&legacyProxyRouteV6{}); err != nil {
t.Fatalf("auto migrate legacy proxy_routes v6: %v", err)
}
now := time.Now().UTC()
certID := uint(9)
if err := db.Create(&legacyProxyRouteV6{
SiteName: "secure-site",
Domain: "secure.example.com",
Domains: `["secure.example.com","www.secure.example.com"]`,
OriginURL: "https://origin-secure.internal:8443",
Upstreams: `["https://origin-secure.internal:8443"]`,
Enabled: true,
EnableHTTPS: true,
CertID: &certID,
RedirectHTTP: true,
LimitConnPerServer: 120,
LimitConnPerIP: 12,
LimitRate: "512k",
CacheEnabled: false,
CachePolicy: "",
CacheRules: `[]`,
CustomHeaders: `[]`,
CreatedAt: now,
UpdatedAt: now,
}).Error; err != nil {
t.Fatalf("seed legacy proxy route v6: %v", err)
}
if err := saveDatabaseSchemaVersion(db, 6); err != nil {
t.Fatalf("save schema version: %v", err)
}
previousDB := DB
DB = db
t.Cleanup(func() {
DB = previousDB
})
if err := ensureDatabaseSchemaUpToDate(db, "sqlite"); err != nil {
t.Fatalf("ensureDatabaseSchemaUpToDate: %v", err)
}
var route ProxyRoute
if err := db.First(&route).Error; err != nil {
t.Fatalf("query migrated proxy route: %v", err)
}
if route.CertID == nil || *route.CertID != certID {
t.Fatalf("expected cert_id mirror to be preserved, got %+v", route.CertID)
}
var certIDs []uint
if err := json.Unmarshal([]byte(route.CertIDs), &certIDs); err != nil {
t.Fatalf("decode migrated cert_ids: %v", err)
}
if len(certIDs) != 1 || certIDs[0] != certID {
t.Fatalf("unexpected migrated cert_ids: %#v", certIDs)
}
}
func TestEnsureDatabaseSchemaUpToDateAddsProxyRouteDomainCertificateFields(t *testing.T) {
db := openBareTestSQLiteDB(t, "legacy-proxy-route-domain-cert-ids.db")
if err := registerSharding(db, "sqlite"); err != nil {