merge: 合并 PR #22 Pages 部署源 V2 到 feat/pages-source-sync-v2

基于最新 main 合并 deqiying/feat/pages-source-sync-v2,
解决 docs/changelog/index.md 与限流相关条目的冲突。
This commit is contained in:
ryan
2026-07-19 19:36:48 +08:00
127 changed files with 24144 additions and 1933 deletions
@@ -0,0 +1,82 @@
-- +goose Up
ALTER TABLE of_pages_projects
ADD COLUMN IF NOT EXISTS content_config_version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE of_pages_deployments
ADD COLUMN IF NOT EXISTS source_type VARCHAR(32) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS source_identity CHAR(64),
ADD COLUMN IF NOT EXISTS source_revision CHAR(64),
ADD COLUMN IF NOT EXISTS source_label VARCHAR(255) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS source_meta TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS trigger_type VARCHAR(32) NOT NULL DEFAULT '';
UPDATE of_pages_deployments
SET source_type = 'manual_upload',
trigger_type = 'manual_upload';
CREATE TABLE IF NOT EXISTS of_pages_project_sources (
id BIGSERIAL PRIMARY KEY,
project_id BIGINT NOT NULL,
source_type VARCHAR(32) NOT NULL DEFAULT '',
remote_url TEXT NOT NULL DEFAULT '',
remote_network_policy VARCHAR(32) NOT NULL DEFAULT '',
github_repository VARCHAR(255) NOT NULL DEFAULT '',
release_selector VARCHAR(16) NOT NULL DEFAULT '',
release_tag VARCHAR(255) NOT NULL DEFAULT '',
asset_name VARCHAR(255) NOT NULL DEFAULT '',
auto_update_enabled BOOLEAN NOT NULL DEFAULT FALSE,
check_interval_minutes INTEGER NOT NULL DEFAULT 0,
config_version INTEGER NOT NULL DEFAULT 0,
source_identity CHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_project_sources_project_id
ON of_pages_project_sources (project_id);
CREATE TABLE IF NOT EXISTS of_pages_project_source_runtime (
source_id BIGINT PRIMARY KEY,
etag VARCHAR(512) NOT NULL DEFAULT '',
last_seen_revision CHAR(64) NOT NULL DEFAULT '',
last_seen_detail TEXT NOT NULL DEFAULT '',
last_applied_revision CHAR(64) NOT NULL DEFAULT '',
last_applied_detail TEXT NOT NULL DEFAULT '',
sync_status VARCHAR(32) NOT NULL DEFAULT '',
last_error TEXT NOT NULL DEFAULT '',
last_checked_at TIMESTAMPTZ,
last_synced_at TIMESTAMPTZ,
next_check_at TIMESTAMPTZ,
lease_expires_at TIMESTAMPTZ,
lease_token VARCHAR(64) NOT NULL DEFAULT '',
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_pages_project_source_runtime_next_check_at
ON of_pages_project_source_runtime (next_check_at);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_project_number
ON of_pages_deployments (project_id, deployment_number);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_source_revision
ON of_pages_deployments (project_id, source_identity, source_revision)
WHERE source_identity IS NOT NULL AND source_revision IS NOT NULL;
-- +goose Down
DROP INDEX IF EXISTS idx_of_pages_deployments_source_revision;
DROP INDEX IF EXISTS idx_of_pages_deployments_project_number;
DROP INDEX IF EXISTS idx_of_pages_project_source_runtime_next_check_at;
DROP TABLE IF EXISTS of_pages_project_source_runtime;
DROP INDEX IF EXISTS idx_of_pages_project_sources_project_id;
DROP TABLE IF EXISTS of_pages_project_sources;
ALTER TABLE of_pages_deployments
DROP COLUMN IF EXISTS trigger_type,
DROP COLUMN IF EXISTS source_meta,
DROP COLUMN IF EXISTS source_label,
DROP COLUMN IF EXISTS source_revision,
DROP COLUMN IF EXISTS source_identity,
DROP COLUMN IF EXISTS source_type;
ALTER TABLE of_pages_projects
DROP COLUMN IF EXISTS content_config_version;
@@ -0,0 +1,38 @@
-- +goose Up
-- Earlier built-in schedules used explicit IDs, so advance the identity only
-- when it trails either existing rows or an already-higher sequence value.
SELECT setval(
pg_get_serial_sequence('w_schedules', 'id'),
GREATEST(
1,
COALESCE((SELECT MAX(id) FROM w_schedules), 0),
COALESCE((
SELECT sequences.last_value
FROM pg_sequences AS sequences
WHERE format('%I.%I', sequences.schemaname, sequences.sequencename)::regclass =
pg_get_serial_sequence('w_schedules', 'id')::regclass
), 0)
),
TRUE
);
INSERT INTO w_schedules (name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT
'OpenFlare Pages 部署源扫描',
'of_pages_source_scan',
'*/5 * * * *',
'{}',
TRUE,
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
WHERE NOT EXISTS (
SELECT 1 FROM w_schedules WHERE task_type = 'of_pages_source_scan'
);
-- +goose Down
DELETE FROM w_schedules
WHERE task_type = 'of_pages_source_scan'
AND name = 'OpenFlare Pages 部署源扫描'
AND cron = '*/5 * * * *'
AND payload = '{}'
AND is_active = TRUE;
@@ -0,0 +1,151 @@
-- +goose Up
ALTER TABLE of_pages_projects
ADD COLUMN content_config_version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE of_pages_deployments
ADD COLUMN source_type TEXT NOT NULL DEFAULT '';
ALTER TABLE of_pages_deployments
ADD COLUMN source_identity TEXT;
ALTER TABLE of_pages_deployments
ADD COLUMN source_revision TEXT;
ALTER TABLE of_pages_deployments
ADD COLUMN source_label TEXT NOT NULL DEFAULT '';
ALTER TABLE of_pages_deployments
ADD COLUMN source_meta TEXT NOT NULL DEFAULT '';
ALTER TABLE of_pages_deployments
ADD COLUMN trigger_type TEXT NOT NULL DEFAULT '';
UPDATE of_pages_deployments
SET source_type = 'manual_upload',
trigger_type = 'manual_upload';
CREATE TABLE IF NOT EXISTS of_pages_project_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
project_id INTEGER NOT NULL,
source_type TEXT NOT NULL DEFAULT '',
remote_url TEXT NOT NULL DEFAULT '',
remote_network_policy TEXT NOT NULL DEFAULT '',
github_repository TEXT NOT NULL DEFAULT '',
release_selector TEXT NOT NULL DEFAULT '',
release_tag TEXT NOT NULL DEFAULT '',
asset_name TEXT NOT NULL DEFAULT '',
auto_update_enabled INTEGER NOT NULL DEFAULT 0,
check_interval_minutes INTEGER NOT NULL DEFAULT 0,
config_version INTEGER NOT NULL DEFAULT 0,
source_identity TEXT NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_project_sources_project_id
ON of_pages_project_sources (project_id);
CREATE TABLE IF NOT EXISTS of_pages_project_source_runtime (
source_id INTEGER PRIMARY KEY,
etag TEXT NOT NULL DEFAULT '',
last_seen_revision TEXT NOT NULL DEFAULT '',
last_seen_detail TEXT NOT NULL DEFAULT '',
last_applied_revision TEXT NOT NULL DEFAULT '',
last_applied_detail TEXT NOT NULL DEFAULT '',
sync_status TEXT NOT NULL DEFAULT '',
last_error TEXT NOT NULL DEFAULT '',
last_checked_at DATETIME,
last_synced_at DATETIME,
next_check_at DATETIME,
lease_expires_at DATETIME,
lease_token TEXT NOT NULL DEFAULT '',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_pages_project_source_runtime_next_check_at
ON of_pages_project_source_runtime (next_check_at);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_project_number
ON of_pages_deployments (project_id, deployment_number);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_source_revision
ON of_pages_deployments (project_id, source_identity, source_revision)
WHERE source_identity IS NOT NULL AND source_revision IS NOT NULL;
-- +goose Down
DROP INDEX IF EXISTS idx_of_pages_deployments_source_revision;
DROP INDEX IF EXISTS idx_of_pages_deployments_project_number;
DROP INDEX IF EXISTS idx_of_pages_project_source_runtime_next_check_at;
DROP TABLE IF EXISTS of_pages_project_source_runtime;
DROP INDEX IF EXISTS idx_of_pages_project_sources_project_id;
DROP TABLE IF EXISTS of_pages_project_sources;
-- SQLite 的 Down 必须重建受影响表,完整移除新增列并保留原有数据与索引。
CREATE TABLE of_pages_deployments_before_source (
id INTEGER PRIMARY KEY AUTOINCREMENT,
project_id INTEGER NOT NULL,
deployment_number INTEGER NOT NULL,
checksum TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'uploaded',
upload_id INTEGER NOT NULL DEFAULT 0,
artifact_path TEXT NOT NULL,
file_count INTEGER NOT NULL DEFAULT 0,
total_size INTEGER NOT NULL DEFAULT 0,
created_by TEXT NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
activated_at DATETIME
);
INSERT INTO of_pages_deployments_before_source (
id, project_id, deployment_number, checksum, status, upload_id, artifact_path,
file_count, total_size, created_by, created_at, activated_at
)
SELECT
id, project_id, deployment_number, checksum, status, upload_id, artifact_path,
file_count, total_size, created_by, created_at, activated_at
FROM of_pages_deployments;
DROP TABLE of_pages_deployments;
ALTER TABLE of_pages_deployments_before_source RENAME TO of_pages_deployments;
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_project_id
ON of_pages_deployments (project_id);
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_checksum
ON of_pages_deployments (checksum);
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_status
ON of_pages_deployments (status);
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_upload_id
ON of_pages_deployments (upload_id);
CREATE TABLE of_pages_projects_before_source (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
slug TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
enabled INTEGER NOT NULL DEFAULT 1,
spa_fallback_enabled INTEGER NOT NULL DEFAULT 0,
spa_fallback_path TEXT NOT NULL DEFAULT '/index.html',
api_proxy_enabled INTEGER NOT NULL DEFAULT 0,
api_proxy_path TEXT NOT NULL DEFAULT '',
api_proxy_pass TEXT NOT NULL DEFAULT '',
api_proxy_rewrite TEXT NOT NULL DEFAULT '',
active_deployment_id INTEGER,
root_dir TEXT NOT NULL DEFAULT '',
entry_file TEXT NOT NULL DEFAULT 'index.html',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO of_pages_projects_before_source (
id, name, slug, description, enabled, spa_fallback_enabled, spa_fallback_path,
api_proxy_enabled, api_proxy_path, api_proxy_pass, api_proxy_rewrite,
active_deployment_id, root_dir, entry_file, created_at, updated_at
)
SELECT
id, name, slug, description, enabled, spa_fallback_enabled, spa_fallback_path,
api_proxy_enabled, api_proxy_path, api_proxy_pass, api_proxy_rewrite,
active_deployment_id, root_dir, entry_file, created_at, updated_at
FROM of_pages_projects;
DROP TABLE of_pages_projects;
ALTER TABLE of_pages_projects_before_source RENAME TO of_pages_projects;
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_projects_slug
ON of_pages_projects (slug);
CREATE INDEX IF NOT EXISTS idx_of_pages_projects_active_deployment_id
ON of_pages_projects (active_deployment_id);
@@ -0,0 +1,21 @@
-- +goose Up
INSERT INTO w_schedules (name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT
'OpenFlare Pages 部署源扫描',
'of_pages_source_scan',
'*/5 * * * *',
'{}',
1,
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
WHERE NOT EXISTS (
SELECT 1 FROM w_schedules WHERE task_type = 'of_pages_source_scan'
);
-- +goose Down
DELETE FROM w_schedules
WHERE task_type = 'of_pages_source_scan'
AND name = 'OpenFlare Pages 部署源扫描'
AND cron = '*/5 * * * *'
AND payload = '{}'
AND is_active = 1;
+3 -3
View File
@@ -18,9 +18,9 @@ import (
"gorm.io/gorm"
)
// expectedMigratedSystemConfigCount 包含初始 32 项系统配置,以及 202606220004
// 从 of_options 迁移过来的 48 项业务配置(OpenFlare/UptimeKuma/OpenResty)。
const expectedMigratedSystemConfigCount = 80
// expectedMigratedSystemConfigCount 包含初始 32 项系统配置、202606220004
// 从 of_options 迁移过来的 48 项业务配置,以及 Pages 的 2 项业务配置。
const expectedMigratedSystemConfigCount = 82
func TestMigrateInitializesSQLiteDatabase(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
@@ -0,0 +1,322 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"database/sql"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/glebarez/sqlite"
"github.com/pressly/goose/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
const (
pagesSourcePreviousMigration = int64(202607180001)
pagesSourceMigration = int64(202607190001)
pagesMigrationProjectID = uint(900001)
pagesMigrationDeploymentID = uint(900001)
)
func TestPagesSourceMigrationSQLiteUpDownUp(t *testing.T) {
dbPath := t.TempDir() + "/pages-source-migration.db"
gormDB, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
sqlDB, err := gormDB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
runPagesSourceMigrationUpDownUp(t, gormDB, sqlDB, dialectSqlite, "goose/sqlite")
var indexSQL string
require.NoError(t, gormDB.Raw(
"SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?",
"idx_of_pages_deployments_source_revision",
).Scan(&indexSQL).Error)
assert.Contains(t, strings.ToUpper(indexSQL), "WHERE SOURCE_IDENTITY IS NOT NULL AND SOURCE_REVISION IS NOT NULL")
}
func TestPagesSourceMigrationPostgresUpDownUp(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("OPENFLARE_TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("OPENFLARE_TEST_POSTGRES_DSN is not set")
}
gormDB, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
sqlDB, err := gormDB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("pages_source_migration_%d", time.Now().UnixNano())
require.Regexp(t, `^[a-z0-9_]+$`, schema)
require.NoError(t, gormDB.Exec(`CREATE SCHEMA "`+schema+`"`).Error)
require.NoError(t, gormDB.Exec(`SET search_path TO "`+schema+`"`).Error)
t.Cleanup(func() {
assert.NoError(t, gormDB.Exec("SET search_path TO public").Error)
assert.NoError(t, gormDB.Exec(`DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`).Error)
assert.NoError(t, sqlDB.Close())
})
runPagesSourceMigrationUpDownUp(t, gormDB, sqlDB, dialectPostgres, "goose/postgres")
}
func runPagesSourceMigrationUpDownUp(
t *testing.T,
gormDB *gorm.DB,
sqlDB *sql.DB,
dialect string,
dir string,
) {
t.Helper()
goose.SetBaseFS(migrationFS)
require.NoError(t, goose.SetDialect(dialect))
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourcePreviousMigration))
seedPrePagesSourceMigrationData(t, gormDB)
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceMigration))
assertPagesSourceMigrationUp(t, gormDB)
require.NoError(t, goose.DownTo(sqlDB, dir, pagesSourcePreviousMigration))
assertPagesSourceMigrationDown(t, gormDB)
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceMigration))
assertPagesSourceMigrationUpAgain(t, gormDB)
}
func seedPrePagesSourceMigrationData(t *testing.T, gormDB *gorm.DB) {
t.Helper()
require.NoError(t, gormDB.Exec(`
INSERT INTO of_pages_projects (
id, name, slug, description, enabled, active_deployment_id, root_dir, entry_file
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
pagesMigrationProjectID,
"Migration Site",
"migration-site",
"keep-project-data",
true,
pagesMigrationDeploymentID,
"public",
"home.html",
).Error)
require.NoError(t, gormDB.Exec(`
INSERT INTO of_pages_deployments (
id, project_id, deployment_number, checksum, status, upload_id, artifact_path,
file_count, total_size, created_by
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
pagesMigrationDeploymentID,
pagesMigrationProjectID,
1,
strings.Repeat("a", 64),
model.PagesDeploymentStatusActive,
uint64(700001),
"legacy/package.zip",
2,
int64(128),
"user:1",
).Error)
}
func assertPagesSourceMigrationUp(t *testing.T, gormDB *gorm.DB) {
t.Helper()
migrator := gormDB.Migrator()
assert.True(t, migrator.HasTable(&model.PagesProjectSource{}))
assert.True(t, migrator.HasTable(&model.PagesProjectSourceRuntime{}))
assert.True(t, migrator.HasColumn(&model.PagesProject{}, "ContentConfigVersion"))
assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceType"))
assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceIdentity"))
assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceRevision"))
assert.True(t, migrator.HasIndex(&model.PagesProjectSource{}, "idx_of_pages_project_sources_project_id"))
assert.True(t, migrator.HasIndex(&model.PagesProjectSourceRuntime{}, "idx_of_pages_project_source_runtime_next_check_at"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_number"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_source_revision"))
var project model.PagesProject
require.NoError(t, gormDB.First(&project, pagesMigrationProjectID).Error)
assert.Equal(t, 0, project.ContentConfigVersion)
assert.Equal(t, "keep-project-data", project.Description)
assert.Equal(t, "public", project.RootDir)
assert.Equal(t, "home.html", project.EntryFile)
var deployment model.PagesDeployment
require.NoError(t, gormDB.First(&deployment, pagesMigrationDeploymentID).Error)
assert.Equal(t, "manual_upload", deployment.SourceType)
assert.Equal(t, "manual_upload", deployment.TriggerType)
assert.Nil(t, deployment.SourceIdentity)
assert.Nil(t, deployment.SourceRevision)
assert.Equal(t, uint64(700001), deployment.UploadID)
sourceID := createMigrationSourceRuntime(t, gormDB)
assertPagesSourceConstraints(t, gormDB, sourceID)
}
func createMigrationSourceRuntime(t *testing.T, gormDB *gorm.DB) uint {
t.Helper()
source := model.PagesProjectSource{
ProjectID: pagesMigrationProjectID,
SourceType: "remote_url",
RemoteURL: "https://example.com/site.zip?token=secret",
RemoteNetworkPolicy: "public",
CheckIntervalMinutes: 0,
ConfigVersion: 1,
SourceIdentity: strings.Repeat("b", 64),
}
require.NoError(t, gormDB.Create(&source).Error)
require.NotZero(t, source.ID)
require.NoError(t, gormDB.Create(&model.PagesProjectSourceRuntime{
SourceID: source.ID,
SyncStatus: "idle",
}).Error)
return source.ID
}
func assertPagesSourceConstraints(t *testing.T, gormDB *gorm.DB, sourceID uint) {
t.Helper()
duplicateSource := model.PagesProjectSource{
ProjectID: pagesMigrationProjectID,
SourceType: "remote_url",
ConfigVersion: 1,
SourceIdentity: strings.Repeat("c", 64),
}
assert.Error(t, gormDB.Create(&duplicateSource).Error)
for number := 2; number <= 3; number++ {
require.NoError(t, createMigrationDeployment(
gormDB,
number,
strings.Repeat(string(rune('a'+number)), 64),
nil,
nil,
))
}
identity := strings.Repeat("d", 64)
revision := strings.Repeat("e", 64)
require.NoError(t, createMigrationDeployment(
gormDB,
4,
strings.Repeat("f", 64),
&identity,
&revision,
))
assert.Error(t, createMigrationDeployment(
gormDB,
5,
strings.Repeat("0", 64),
&identity,
&revision,
))
assert.Error(t, createMigrationDeployment(
gormDB,
1,
strings.Repeat("1", 64),
nil,
nil,
))
var runtime model.PagesProjectSourceRuntime
require.NoError(t, gormDB.First(&runtime, sourceID).Error)
assert.Equal(t, "idle", runtime.SyncStatus)
}
func createMigrationDeployment(
gormDB *gorm.DB,
deploymentNumber int,
checksum string,
identity *string,
revision *string,
) error {
return gormDB.Create(&model.PagesDeployment{
ProjectID: pagesMigrationProjectID,
DeploymentNumber: deploymentNumber,
Checksum: checksum,
Status: model.PagesDeploymentStatusUploaded,
UploadID: uint64(710000 + deploymentNumber),
ArtifactPath: fmt.Sprintf("legacy/%d.zip", deploymentNumber),
SourceType: "manual_upload",
SourceIdentity: identity,
SourceRevision: revision,
TriggerType: "manual_upload",
}).Error
}
func assertPagesSourceMigrationDown(t *testing.T, gormDB *gorm.DB) {
t.Helper()
migrator := gormDB.Migrator()
assert.False(t, migrator.HasTable(&model.PagesProjectSource{}))
assert.False(t, migrator.HasTable(&model.PagesProjectSourceRuntime{}))
assert.False(t, migrator.HasColumn(&model.PagesProject{}, "ContentConfigVersion"))
assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceType"))
assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceIdentity"))
assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceRevision"))
assert.False(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_number"))
assert.False(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_source_revision"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_id"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_upload_id"))
var project struct {
Description string
RootDir string
EntryFile string
ActiveDeploymentID *uint
}
require.NoError(t, gormDB.Table("of_pages_projects").Where("id = ?", pagesMigrationProjectID).Take(&project).Error)
assert.Equal(t, "keep-project-data", project.Description)
assert.Equal(t, "public", project.RootDir)
assert.Equal(t, "home.html", project.EntryFile)
require.NotNil(t, project.ActiveDeploymentID)
assert.Equal(t, pagesMigrationDeploymentID, *project.ActiveDeploymentID)
var deployment struct {
UploadID uint64
ArtifactPath string
FileCount int
TotalSize int64
}
require.NoError(t, gormDB.Table("of_pages_deployments").Where("id = ?", pagesMigrationDeploymentID).Take(&deployment).Error)
assert.Equal(t, uint64(700001), deployment.UploadID)
assert.Equal(t, "legacy/package.zip", deployment.ArtifactPath)
assert.Equal(t, 2, deployment.FileCount)
assert.Equal(t, int64(128), deployment.TotalSize)
var count int64
require.NoError(t, gormDB.Table("of_pages_deployments").Where("project_id = ?", pagesMigrationProjectID).Count(&count).Error)
assert.Equal(t, int64(4), count)
}
func assertPagesSourceMigrationUpAgain(t *testing.T, gormDB *gorm.DB) {
t.Helper()
assert.True(t, gormDB.Migrator().HasTable(&model.PagesProjectSource{}))
assert.True(t, gormDB.Migrator().HasTable(&model.PagesProjectSourceRuntime{}))
assert.True(t, gormDB.Migrator().HasColumn(&model.PagesProject{}, "ContentConfigVersion"))
assert.True(t, gormDB.Migrator().HasColumn(&model.PagesDeployment{}, "SourceRevision"))
var count int64
require.NoError(t, gormDB.Table("of_pages_deployments").
Where("project_id = ? AND source_type = ? AND trigger_type = ?", pagesMigrationProjectID, "manual_upload", "manual_upload").
Count(&count).Error)
assert.Equal(t, int64(4), count)
require.NoError(t, gormDB.Table("of_pages_project_sources").Count(&count).Error)
assert.Zero(t, count, "source config is intentionally removed by Down and is not reconstructable")
}
@@ -0,0 +1,160 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"database/sql"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/glebarez/sqlite"
"github.com/pressly/goose/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
const (
pagesSourceScanPreviousMigration = int64(202607190001)
pagesSourceScanMigration = int64(202607190002)
pagesSourceScanTaskType = "of_pages_source_scan"
)
func TestPagesSourceScanScheduleMigrationSQLite(t *testing.T) {
dbPath := t.TempDir() + "/pages-source-scan-migration.db"
gormDB, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
sqlDB, err := gormDB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
runPagesSourceScanScheduleMigration(t, gormDB, sqlDB, dialectSqlite, "goose/sqlite")
}
func TestPagesSourceScanScheduleMigrationPostgres(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("OPENFLARE_TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("OPENFLARE_TEST_POSTGRES_DSN is not set")
}
gormDB, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
sqlDB, err := gormDB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("pages_source_scan_migration_%d", time.Now().UnixNano())
require.Regexp(t, `^[a-z0-9_]+$`, schema)
require.NoError(t, gormDB.Exec(`CREATE SCHEMA "`+schema+`"`).Error)
require.NoError(t, gormDB.Exec(`SET search_path TO "`+schema+`"`).Error)
t.Cleanup(func() {
assert.NoError(t, gormDB.Exec("SET search_path TO public").Error)
assert.NoError(t, gormDB.Exec(`DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`).Error)
assert.NoError(t, sqlDB.Close())
})
runPagesSourceScanScheduleMigration(t, gormDB, sqlDB, dialectPostgres, "goose/postgres")
}
func runPagesSourceScanScheduleMigration(
t *testing.T,
gormDB *gorm.DB,
sqlDB *sql.DB,
dialect string,
dir string,
) {
t.Helper()
goose.SetBaseFS(migrationFS)
require.NoError(t, goose.SetDialect(dialect))
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceScanPreviousMigration))
var previousMaxID uint64
require.NoError(t, gormDB.Table("w_schedules").Select("COALESCE(MAX(id), 0)").Scan(&previousMaxID).Error)
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceScanMigration))
seeded := assertPagesSourceScanSchedule(t, gormDB)
assert.NotZero(t, seeded.ID)
if dialect == dialectPostgres {
assert.Greater(t, seeded.ID, previousMaxID)
}
require.NoError(t, goose.DownTo(sqlDB, dir, pagesSourceScanPreviousMigration))
assertPagesSourceScanScheduleMissing(t, gormDB)
custom := model.Schedule{
ID: 900001,
Name: "用户保留的 Pages 扫描任务",
TaskType: pagesSourceScanTaskType,
Cron: "0 * * * *",
Payload: `{"custom":true}`,
IsActive: false,
}
require.NoError(t, gormDB.Create(&custom).Error)
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceScanMigration))
var schedules []model.Schedule
require.NoError(t, gormDB.Where("task_type = ?", pagesSourceScanTaskType).Find(&schedules).Error)
require.Len(t, schedules, 1)
assert.Equal(t, custom.ID, schedules[0].ID)
assert.Equal(t, custom.Name, schedules[0].Name)
require.NoError(t, goose.DownTo(sqlDB, dir, pagesSourceScanPreviousMigration))
var retained model.Schedule
require.NoError(t, gormDB.First(&retained, custom.ID).Error)
assert.Equal(t, custom.TaskType, retained.TaskType)
}
func TestPagesSourceScanScheduleMigrationsUseDatabaseGeneratedIDs(t *testing.T) {
for _, name := range []string{
"goose/postgres/202607190002_seed_pages_source_scan.sql",
"goose/sqlite/202607190002_seed_pages_source_scan.sql",
} {
t.Run(name, func(t *testing.T) {
content, err := migrationFS.ReadFile(name)
require.NoError(t, err)
normalized := strings.ToLower(string(content))
assert.NotContains(t, normalized, "insert into w_schedules (id,")
assert.NotContains(t, normalized, "coalesce(max(id)")
})
}
postgresContent, err := migrationFS.ReadFile("goose/postgres/202607190002_seed_pages_source_scan.sql")
require.NoError(t, err)
compactPostgres := strings.Join(strings.Fields(strings.ToLower(string(postgresContent))), " ")
assert.Contains(
t,
compactPostgres,
"select setval( pg_get_serial_sequence('w_schedules', 'id'), greatest( 1,",
"sequence synchronization must retain a valid lower bound for an empty table",
)
}
func assertPagesSourceScanSchedule(t *testing.T, gormDB *gorm.DB) model.Schedule {
t.Helper()
var schedules []model.Schedule
require.NoError(t, gormDB.Where("task_type = ?", pagesSourceScanTaskType).Find(&schedules).Error)
require.Len(t, schedules, 1)
schedule := schedules[0]
assert.Equal(t, "OpenFlare Pages 部署源扫描", schedule.Name)
assert.Equal(t, "*/5 * * * *", schedule.Cron)
assert.Equal(t, "{}", schedule.Payload)
assert.True(t, schedule.IsActive)
return schedule
}
func assertPagesSourceScanScheduleMissing(t *testing.T, gormDB *gorm.DB) {
t.Helper()
var count int64
require.NoError(t, gormDB.Model(&model.Schedule{}).
Where("task_type = ?", pagesSourceScanTaskType).
Count(&count).Error)
assert.Zero(t, count)
}
+21 -16
View File
@@ -57,13 +57,10 @@ func initSQLite() {
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", sqlitePath),
attribute.String("db.system", "SQLite"),
),
),
newGORMTracingPlugin([]attribute.KeyValue{
attribute.String("db.instance", sqlitePath),
attribute.String("db.system", "SQLite"),
}),
); err != nil {
log.Fatalf("[SQLite] init trace failed: %v\n", err)
}
@@ -98,15 +95,12 @@ func initPostgres() {
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", dbConfig.Database),
attribute.String("db.ip", dbConfig.Host),
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
attribute.String("db.system", "PostgreSQL"),
),
),
newGORMTracingPlugin([]attribute.KeyValue{
attribute.String("db.instance", dbConfig.Database),
attribute.String("db.ip", dbConfig.Host),
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
attribute.String("db.system", "PostgreSQL"),
}),
); err != nil {
log.Fatalf("[PostgreSQL] init trace failed: %v\n", err)
}
@@ -160,6 +154,17 @@ func initPostgres() {
}
// newGORMTracingPlugin 构造数据库链路追踪插件。查询参数只保留占位符,避免凭据等绑定值进入 Span。
func newGORMTracingPlugin(attrs []attribute.KeyValue, extraOptions ...tracing.Option) gorm.Plugin {
options := []tracing.Option{
tracing.WithoutMetrics(),
tracing.WithoutQueryVariables(),
tracing.WithAttributes(attrs...),
}
options = append(options, extraOptions...)
return tracing.NewPlugin(options...)
}
// buildDSN 构建 PostgreSQL DSN
func buildDSN(host string, port int, username, password string) string {
cfg := config.Config.Database
+5
View File
@@ -49,6 +49,11 @@ func (l *gormZapLogger) Error(ctx context.Context, fmt string, args ...interface
}
}
// ParamsFilter 让 GORM 的 Trace 回调只接收参数化 SQL,避免绑定值被 Dialector.Explain 展开到日志。
func (l *gormZapLogger) ParamsFilter(_ context.Context, sql string, _ ...interface{}) (string, []interface{}) {
return sql, nil
}
func (l *gormZapLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
elapsed := time.Since(begin)
switch {
+76
View File
@@ -4,11 +4,40 @@
package db
import (
"context"
"strings"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
)
type paramsFilterCaptureLogger struct {
filter *gormZapLogger
traces []string
}
func (l *paramsFilterCaptureLogger) LogMode(gormLogger.LogLevel) gormLogger.Interface {
return l
}
func (l *paramsFilterCaptureLogger) Info(context.Context, string, ...interface{}) {}
func (l *paramsFilterCaptureLogger) Warn(context.Context, string, ...interface{}) {}
func (l *paramsFilterCaptureLogger) Error(context.Context, string, ...interface{}) {}
func (l *paramsFilterCaptureLogger) ParamsFilter(ctx context.Context, sql string, params ...interface{}) (string, []interface{}) {
return l.filter.ParamsFilter(ctx, sql, params...)
}
func (l *paramsFilterCaptureLogger) Trace(_ context.Context, _ time.Time, fc func() (string, int64), _ error) {
sql, _ := fc()
l.traces = append(l.traces, sql)
}
func TestParseLogLevel(t *testing.T) {
t.Parallel()
@@ -37,3 +66,50 @@ func TestParseLogLevel(t *testing.T) {
})
}
}
func TestGormZapLoggerParamsFilterDropsBoundValues(t *testing.T) {
t.Parallel()
const (
query = "UPDATE openflare_pages_sources SET remote_url = ? WHERE id = ?"
secret = "https://example.test/release.zip?token=super-secret"
)
filteredSQL, filteredParams := (&gormZapLogger{}).ParamsFilter(t.Context(), query, secret, int64(42))
if filteredSQL != query {
t.Fatalf("ParamsFilter() sql = %q, want %q", filteredSQL, query)
}
if filteredParams != nil {
t.Fatalf("ParamsFilter() params = %#v, want nil", filteredParams)
}
}
func TestGormZapLoggerKeepsParameterizedSQLInTrace(t *testing.T) {
t.Parallel()
const secret = "https://example.test/release.zip?token=trace-secret"
capture := &paramsFilterCaptureLogger{filter: &gormZapLogger{}}
testDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: capture})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := testDB.Exec("CREATE TABLE source_secrets (remote_url TEXT NOT NULL)").Error; err != nil {
t.Fatalf("create table: %v", err)
}
capture.traces = nil
if err := testDB.Exec("INSERT INTO source_secrets (remote_url) VALUES (?)", secret).Error; err != nil {
t.Fatalf("insert source secret: %v", err)
}
if len(capture.traces) != 1 {
t.Fatalf("trace count = %d, want 1", len(capture.traces))
}
traceSQL := capture.traces[0]
if strings.Contains(traceSQL, secret) || strings.Contains(traceSQL, "trace-secret") {
t.Fatalf("trace SQL leaked bound value: %q", traceSQL)
}
if !strings.Contains(traceSQL, "VALUES (?)") {
t.Fatalf("trace SQL = %q, want parameter placeholder", traceSQL)
}
}
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"strings"
"testing"
"github.com/glebarez/sqlite"
"go.opentelemetry.io/otel/attribute"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
semconv "go.opentelemetry.io/otel/semconv/v1.30.0"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
"gorm.io/plugin/opentelemetry/tracing"
)
func TestGORMTracingPluginDoesNotRecordQueryVariables(t *testing.T) {
t.Parallel()
const secret = "https://example.test/release.zip?token=otel-secret"
spanRecorder := tracetest.NewSpanRecorder()
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder))
t.Cleanup(func() {
if err := tracerProvider.Shutdown(context.Background()); err != nil {
t.Errorf("shutdown tracer provider: %v", err)
}
})
testDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
Logger: gormLogger.Default.LogMode(gormLogger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := testDB.Use(newGORMTracingPlugin(
[]attribute.KeyValue{attribute.String("db.instance", "trace-test")},
tracing.WithTracerProvider(tracerProvider),
)); err != nil {
t.Fatalf("register tracing plugin: %v", err)
}
if err := testDB.Exec("CREATE TABLE source_secrets (remote_url TEXT NOT NULL)").Error; err != nil {
t.Fatalf("create table: %v", err)
}
if err := testDB.Exec("INSERT INTO source_secrets (remote_url) VALUES (?)", secret).Error; err != nil {
t.Fatalf("insert source secret: %v", err)
}
var queryText string
for _, span := range spanRecorder.Ended() {
for _, attr := range span.Attributes() {
if attr.Key == semconv.DBQueryTextKey && strings.Contains(attr.Value.AsString(), "INSERT INTO source_secrets") {
queryText = attr.Value.AsString()
}
}
}
if queryText == "" {
t.Fatal("database query text attribute not found")
}
if strings.Contains(queryText, secret) || strings.Contains(queryText, "otel-secret") {
t.Fatalf("db.query.text leaked bound value: %q", queryText)
}
if !strings.Contains(queryText, "VALUES (?)") {
t.Fatalf("db.query.text = %q, want parameter placeholder", queryText)
}
}