tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-09 09:06:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(pages): 支持 Remote 部署源同步

Browse Source 
新增部署源配置与运行态模型、安全下载、租约续期、原子激活和失败补偿。

接入内部任务与脱敏前端交互，并阻止数据库 Trace 和日志展开敏感查询参数。
This commit is contained in:
deqiying
2026-07-19 17:36:51 +08:00
parent 4e8ec23264
commit 38b0516937
58 changed files with 9858 additions and 1075 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apps/admin/task/routers.go
+15 -3
View File
@@ -66,7 +66,7 @@ func DispatchTask(c *gin.Context) {
return
}
meta := task.GetTaskMeta(req.TaskType)
meta := getAdminTaskMeta(req.TaskType)
if meta == nil {
response.AbortBadRequest(c, InvalidTaskType)
return
@@ -256,7 +256,7 @@ func CreateSchedule(c *gin.Context) {
}
// 校验关联的异步任务类型
meta := task.GetTaskMeta(req.TaskType)
meta := getAdminTaskMeta(req.TaskType)
if meta == nil {
response.AbortBadRequest(c, InvalidTaskType)
return
@@ -338,6 +338,10 @@ func UpdateSchedule(c *gin.Context) {
response.AbortNotFound(c, ScheduleNotFound)
return
}
if existingMeta := task.GetTaskMeta(schedule.TaskType); existingMeta != nil && existingMeta.InternalOnly {
response.AbortBadRequest(c, InvalidTaskType)
return
}
// 校验 Cron 表达式
if _, err := cron.ParseStandard(req.Cron); err != nil {
@@ -346,7 +350,7 @@ func UpdateSchedule(c *gin.Context) {
}
// 校验关联的异步任务类型
meta := task.GetTaskMeta(req.TaskType)
meta := getAdminTaskMeta(req.TaskType)
if meta == nil {
response.AbortBadRequest(c, InvalidTaskType)
return
@@ -382,6 +386,14 @@ func UpdateSchedule(c *gin.Context) {
c.JSON(http.StatusOK, response.OK(schedule))
}
func getAdminTaskMeta(taskType string) *task.TaskMeta {
meta := task.GetTaskMeta(taskType)
if meta == nil || meta.InternalOnly {
return nil
}
return meta
}
// DeleteSchedule 删除定时任务
// @Summary 删除定时任务
// @Description 删除指定的定时任务配置，并触发调度器热加载，需要管理员权限
internal/apps/admin/task/routers_test.go
+133
View File
@@ -29,6 +29,20 @@ import (
"github.com/Rain-kl/Wavelet/internal/common/response"
)
const (
testInternalOnlyTaskType = "test_internal_only_admin"
testInternalOnlyAsynqTask = "test:internal_only_admin"
)
func registerInternalOnlyTaskMeta() {
task.RegisterTaskMeta(task.TaskMeta{
Type: testInternalOnlyTaskType,
AsynqTask: testInternalOnlyAsynqTask,
Name: "内部测试任务",
InternalOnly: true,
})
}
func setupTaskTestEnvironment(t *testing.T) func() {
_, mr, cleanup := testhelper.SetupTestEnvironment(t)
bootstrap.RegisterTasks()
@@ -61,12 +75,15 @@ func setupTestRouter(authUser *model.User) *gin.Engine {
adminGroup.GET("/tasks/executions", ListTaskExecutions)
adminGroup.GET("/tasks/executions/:id", GetTaskExecution)
adminGroup.POST("/tasks/executions/:id/retry", RetryTask)
adminGroup.POST("/tasks/schedules", CreateSchedule)
adminGroup.PUT("/tasks/schedules/:id", UpdateSchedule)
return r
}
func TestListTaskTypes(t *testing.T) {
cleanup := setupTaskTestEnvironment(t)
defer cleanup()
registerInternalOnlyTaskMeta()
adminUser := &model.User{ID: 1001, Username: "admin", IsAdmin: true}
router := setupTestRouter(adminUser)
@@ -93,6 +110,9 @@ func TestListTaskTypes(t *testing.T) {
foundCleanup := false
foundWarmImageCache := false
for _, m := range taskMetas {
if m.Type == testInternalOnlyTaskType {
t.Errorf("internal-only task type %s must not be listed", testInternalOnlyTaskType)
}
if m.Type == uploadtask.TaskTypeSystemCleanup {
foundCleanup = true
}
@@ -108,6 +128,119 @@ func TestListTaskTypes(t *testing.T) {
}
}
func TestInternalOnlyTaskAdminBoundaries(t *testing.T) {
cleanup := setupTaskTestEnvironment(t)
defer cleanup()
registerInternalOnlyTaskMeta()
adminUser := &model.User{ID: 1001, Username: "admin", IsAdmin: true}
router := setupTestRouter(adminUser)
ctx := context.Background()
t.Run("dispatch rejects internal-only task", func(t *testing.T) {
body, err := json.Marshal(DispatchTaskRequest{TaskType: testInternalOnlyTaskType})
require.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/tasks/dispatch", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
var resp response.Any
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
assert.Equal(t, InvalidTaskType, resp.ErrorMsg)
})
t.Run("create schedule rejects internal-only task", func(t *testing.T) {
isActive := true
body, err := json.Marshal(CreateScheduleRequest{
Name: "内部任务排程",
TaskType: testInternalOnlyTaskType,
Cron: "0 * * * *",
IsActive: &isActive,
})
require.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/tasks/schedules", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
var resp response.Any
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
assert.Equal(t, InvalidTaskType, resp.ErrorMsg)
})
t.Run("update cannot convert existing internal schedule to public task", func(t *testing.T) {
schedule := &model.Schedule{
Name: "系统内部排程",
TaskType: testInternalOnlyTaskType,
Cron: "0 * * * *",
IsActive: true,
}
require.NoError(t, model.CreateSchedule(ctx, schedule))
isActive := false
body, err := json.Marshal(UpdateScheduleRequest{
Name: "尝试修改内部排程",
TaskType: uploadtask.TaskTypeSystemCleanup,
Cron: "5 * * * *",
IsActive: &isActive,
})
require.NoError(t, err)
req := httptest.NewRequest(
http.MethodPut,
fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID),
bytes.NewReader(body),
)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
unchanged, err := model.GetScheduleByID(ctx, schedule.ID)
require.NoError(t, err)
assert.Equal(t, "系统内部排程", unchanged.Name)
assert.Equal(t, testInternalOnlyTaskType, unchanged.TaskType)
assert.True(t, unchanged.IsActive)
})
t.Run("update public schedule rejects internal-only target task", func(t *testing.T) {
schedule := &model.Schedule{
Name: "公开排程",
TaskType: uploadtask.TaskTypeSystemCleanup,
Cron: "0 * * * *",
IsActive: true,
}
require.NoError(t, model.CreateSchedule(ctx, schedule))
isActive := true
body, err := json.Marshal(UpdateScheduleRequest{
Name: "尝试切入内部任务",
TaskType: testInternalOnlyTaskType,
Cron: "10 * * * *",
IsActive: &isActive,
})
require.NoError(t, err)
req := httptest.NewRequest(
http.MethodPut,
fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID),
bytes.NewReader(body),
)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
unchanged, err := model.GetScheduleByID(ctx, schedule.ID)
require.NoError(t, err)
assert.Equal(t, "公开排程", unchanged.Name)
assert.Equal(t, uploadtask.TaskTypeSystemCleanup, unchanged.TaskType)
})
}
func TestDispatchTask(t *testing.T) {
cleanup := setupTaskTestEnvironment(t)
defer cleanup()
internal/apps/openflare/pages/download_url.go
+24 -242
View File
@@ -5,211 +5,38 @@ package pages
import (
"context"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"errors"
"fmt"
"io"
"mime"
"net"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/Rain-kl/Wavelet/pkg/pagesarchive"
)
const (
pagesURLDownloadTimeout = 10 * time.Minute
pagesURLMaxRedirects = 5
pagesMagicSniffBytes = 512
pagesURLDialTimeout = 30 * time.Second
pagesURLTLSHandshake = 15 * time.Second
pagesBrowserUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
pagesBrowserAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
pagesBrowserAcceptLang = "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7"
pagesBrowserSecCHUA = `"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"`
pagesBrowserSecCHUAMobile = "?0"
pagesBrowserSecCHUAPlat = `"Windows"`
)
// downloadPagesPackageFromURL fetches a remote archive with browser-like headers
// and writes it to a temp file. Allows private/LAN hosts and insecure TLS certs
// (self-signed / internal CA) so operators can pull from internal artifact stores.
func downloadPagesPackageFromURL(ctx context.Context, rawURL string, maxPackageBytes int64) (tempPath string, checksum string, size int64, format pagesarchive.Format, fileName string, err error) {
parsed, err := parseAndValidatePagesDownloadURL(rawURL)
if err != nil {
// downloadPagesPackageFromURL is the deprecated one-shot URL adapter. It uses
// the same bounded downloader as persisted sources, with the legacy trusted
// network policy that permits operator-managed internal artifact services.
func downloadPagesPackageFromURL(
ctx context.Context,
rawURL string,
maxPackageBytes int64,
) (tempPath string, checksum string, size int64, format pagesarchive.Format, fileName string, err error) {
if _, err := parseAndValidatePagesDownloadURL(rawURL); err != nil {
return "", "", 0, "", "", err
}
resp, err := doBrowserDownload(ctx, parsed)
candidate, err := FetchRemoteSource(ctx, RemoteSourceRequest{
URL: strings.TrimSpace(rawURL),
NetworkPolicy: RemoteNetworkPolicyTrustedInternal,
MaxPackageBytes: maxPackageBytes,
})
if err != nil {
return "", "", 0, "", "", err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", "", 0, "", "", fmt.Errorf("%s: HTTP %d", errPagesPackageURLDownloadFailed, resp.StatusCode)
}
if resp.ContentLength > 0 && resp.ContentLength > maxPackageBytes {
return "", "", 0, "", "", errors.New(errPagesPackageURLTooLarge)
}
fileName = fileNameFromDownload(resp, parsed)
format, _ = pagesarchive.DetectFormatFromName(fileName)
tempPath, checksum, size, err = writeLimitedPackageTemp(resp.Body, format, maxPackageBytes)
if err != nil {
return "", "", 0, "", "", err
}
format, fileName, err = ensurePackageFormat(tempPath, format, fileName)
if err != nil {
_ = os.Remove(tempPath)
return "", "", 0, "", "", err
}
return tempPath, checksum, size, format, fileName, nil
}
func newPagesURLDownloadClient() *http.Client {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: pagesURLDialTimeout,
KeepAlive: pagesURLDialTimeout,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 32,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: pagesURLTLSHandshake,
ExpectContinueTimeout: time.Second,
// Allow self-signed / internal certificates for artifact hosts.
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // intentional for internal/dev artifact URLs
}
client := &http.Client{
Timeout: pagesURLDownloadTimeout,
Transport: transport,
}
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= pagesURLMaxRedirects {
return errors.New(errPagesPackageURLDownloadFailed)
if strings.Contains(err.Error(), errPagesSourceRemoteURLInvalid) {
return "", "", 0, "", "", errors.New(errPagesPackageURLInvalid)
}
if err := validatePagesDownloadURLValue(req.URL); err != nil {
return err
}
applyBrowserDownloadHeaders(req, via[0].URL.String())
return nil
}
return client
}
func doBrowserDownload(ctx context.Context, parsed *url.URL) (*http.Response, error) {
client := newPagesURLDownloadClient()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return nil, errors.New(errPagesPackageURLInvalid)
}
applyBrowserDownloadHeaders(req, "")
resp, err := client.Do(req) //nolint:gosec // scheme validated; private hosts and insecure TLS intentionally allowed
if err != nil {
return nil, fmt.Errorf("%s: %w", errPagesPackageURLDownloadFailed, err)
}
return resp, nil
}
func writeLimitedPackageTemp(body io.Reader, format pagesarchive.Format, maxPackageBytes int64) (tempPath, checksum string, size int64, err error) {
temp, err := os.CreateTemp("", "openflare-pages-url-*."+safeTempSuffixOrBin(format))
if err != nil {
return "", "", 0, err
}
tempPath = temp.Name()
defer func() {
_ = temp.Close()
if err != nil {
_ = os.Remove(tempPath)
}
}()
hash := sha256.New()
written, copyErr := io.Copy(io.MultiWriter(temp, hash), io.LimitReader(body, maxPackageBytes+1))
if copyErr != nil {
err = fmt.Errorf("%s: %w", errPagesPackageURLDownloadFailed, copyErr)
return "", "", 0, err
}
if written > maxPackageBytes {
err = errors.New(errPagesPackageURLTooLarge)
return "", "", 0, err
}
if written == 0 {
err = errors.New(errPagesPackageEmpty)
return "", "", 0, err
}
return tempPath, hex.EncodeToString(hash.Sum(nil)), written, nil
}
func ensurePackageFormat(tempPath string, format pagesarchive.Format, fileName string) (pagesarchive.Format, string, error) {
if format != "" {
return format, fileName, nil
}
detected, ok := sniffPackageFormat(tempPath)
if !ok {
return "", fileName, errors.New(errPagesPackageUnsupported)
}
if !strings.Contains(strings.ToLower(fileName), ".") {
fileName = fileName + "." + pagesarchive.Extension(detected)
}
return detected, fileName, nil
}
func sniffPackageFormat(tempPath string) (pagesarchive.Format, bool) {
file, err := os.Open(tempPath) //nolint:gosec // temp path created by us
if err != nil {
return "", false
}
defer func() { _ = file.Close() }()
head := make([]byte, pagesMagicSniffBytes)
n, _ := io.ReadFull(file, head)
if n <= 0 {
return "", false
}
return pagesarchive.DetectFormatFromBytes(head[:n])
}
func safeTempSuffixOrBin(format pagesarchive.Format) string {
if format == "" {
return "bin"
}
return safeTempSuffix(format)
}
func applyBrowserDownloadHeaders(req *http.Request, referer string) {
if req == nil {
return
}
req.Header.Set("User-Agent", pagesBrowserUserAgent)
req.Header.Set("Accept", pagesBrowserAccept)
req.Header.Set("Accept-Language", pagesBrowserAcceptLang)
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Pragma", "no-cache")
req.Header.Set("Upgrade-Insecure-Requests", "1")
req.Header.Set("Sec-Fetch-Dest", "document")
req.Header.Set("Sec-Fetch-Mode", "navigate")
req.Header.Set("Sec-Fetch-Site", "none")
req.Header.Set("Sec-Fetch-User", "?1")
req.Header.Set("Sec-Ch-Ua", pagesBrowserSecCHUA)
req.Header.Set("Sec-Ch-Ua-Mobile", pagesBrowserSecCHUAMobile)
req.Header.Set("Sec-Ch-Ua-Platform", pagesBrowserSecCHUAPlat)
if referer != "" {
req.Header.Set("Referer", referer)
req.Header.Set("Sec-Fetch-Site", "cross-site")
return
}
if req.URL != nil {
req.Header.Set("Referer", req.URL.Scheme+"://"+req.URL.Host+"/")
return "", "", 0, "", "", err
}
// Ownership transfers to the existing one-shot caller, which removes the
// temporary file after the candidate deployment has been created.
return candidate.TempPath, candidate.Checksum, candidate.PackageSize, candidate.Format, candidate.SafeLabel, nil
}
func parseAndValidatePagesDownloadURL(raw string) (*url.URL, error) {
@@ -218,57 +45,12 @@ func parseAndValidatePagesDownloadURL(raw string) (*url.URL, error) {
return nil, errors.New(errPagesPackageURLRequired)
}
parsed, err := url.Parse(value)
if err != nil {
if err != nil || parsed.User != nil || parsed.Fragment != "" || parsed.Opaque != "" {
return nil, errors.New(errPagesPackageURLInvalid)
}
if err := validatePagesDownloadURLValue(parsed); err != nil {
return nil, err
scheme := strings.ToLower(strings.TrimSpace(parsed.Scheme))
if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(parsed.Hostname()) == "" {
return nil, errors.New(errPagesPackageURLInvalid)
}
return parsed, nil
}
func validatePagesDownloadURLValue(parsed *url.URL) error {
if parsed == nil {
return errors.New(errPagesPackageURLInvalid)
}
scheme := strings.ToLower(strings.TrimSpace(parsed.Scheme))
if scheme != "http" && scheme != "https" {
return errors.New(errPagesPackageURLInvalid)
}
if strings.TrimSpace(parsed.Hostname()) == "" {
return errors.New(errPagesPackageURLInvalid)
}
return nil
}
func fileNameFromDownload(resp *http.Response, parsed *url.URL) string {
if name := fileNameFromContentDisposition(resp); name != "" {
return name
}
if parsed != nil {
base := path.Base(parsed.Path)
if base != "" && base != "." && base != "/" {
return base
}
}
return "package.bin"
}
func fileNameFromContentDisposition(resp *http.Response) string {
if resp == nil {
return ""
}
cd := resp.Header.Get("Content-Disposition")
if cd == "" {
return ""
}
_, params, err := mime.ParseMediaType(cd)
if err != nil {
return ""
}
name := strings.TrimSpace(params["filename"])
if name == "" {
return ""
}
return path.Base(filepath.ToSlash(name))
}
internal/apps/openflare/pages/download_url_test.go
+5 -12
View File
@@ -10,7 +10,6 @@ import (
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -48,10 +47,10 @@ func TestDownloadPagesPackageFromURLAllowsPrivateHost(t *testing.T) {
require.NoError(t, zw.Close())
zipBytes := body.Bytes()
var sawBrowserUA bool
var sawProviderUA bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
sawBrowserUA = true
if r.Header.Get("User-Agent") == remoteSourceUserAgent {
sawProviderUA = true
}
w.Header().Set("Content-Disposition", `attachment; filename="remote-site.zip"`)
w.Header().Set("Content-Type", "application/zip")
@@ -59,12 +58,6 @@ func TestDownloadPagesPackageFromURLAllowsPrivateHost(t *testing.T) {
}))
t.Cleanup(server.Close)
req, err := http.NewRequest(http.MethodGet, server.URL+"/pkg.zip", nil)
require.NoError(t, err)
applyBrowserDownloadHeaders(req, "")
assert.Contains(t, req.Header.Get("User-Agent"), "Mozilla")
assert.Contains(t, req.Header.Get("Sec-Fetch-Mode"), "navigate")
tempPath, checksum, size, format, fileName, err := downloadPagesPackageFromURL(
context.Background(),
server.URL+"/pkg.zip",
@@ -72,11 +65,11 @@ func TestDownloadPagesPackageFromURLAllowsPrivateHost(t *testing.T) {
)
require.NoError(t, err)
t.Cleanup(func() { _ = os.Remove(tempPath) })
assert.True(t, sawBrowserUA)
assert.True(t, sawProviderUA)
assert.NotEmpty(t, checksum)
assert.Positive(t, size)
assert.Equal(t, "zip", string(format))
assert.Equal(t, "remote-site.zip", fileName)
assert.Equal(t, "pkg.zip", fileName)
}
func TestUploadDeploymentFromURLPrivateHost(t *testing.T) {
internal/apps/openflare/pages/errs.go
+16
View File
@@ -36,4 +36,20 @@ const (
errPagesInvalidSnapshotFormat = "配置快照格式无效"
errPagesActorMissing = "无法识别当前用户"
errPagesEntryFileMissing = "当前激活部署中不存在指定入口文件"
errPagesSourceNotFound = "pages 部署源不存在"
errPagesSourceTypeRequired = "请选择 pages 部署源类型"
errPagesSourceTypeUnsupported = "当前阶段仅支持远程地址部署源"
errPagesSourceRemoteFields = "远程地址来源不能包含 GitHub 或自动更新配置"
errPagesSourceRemoteURLRequired = "请提供远程部署包地址"
errPagesSourceRemoteURLMode = "remote_url_set 与 remote_url 参数不匹配"
errPagesSourceRemoteURLInvalid = "远程部署包地址无效，仅支持不含用户信息和片段的 http/https 地址"
errPagesSourceNetworkPolicy = "远程地址网络策略仅支持 public 或 trusted_internal"
errPagesSourceCheckUnsupported = "远程地址来源不支持检查更新，请使用立即同步"
errPagesSourceActionBusy = "pages 部署源任务正在执行"
errPagesSourceActionInvalid = "pages 部署源任务参数无效"
errPagesSourceActionStale = "pages 部署源配置已变化，本次任务已跳过"
errPagesSourceLeaseLost = "pages 部署源任务执行权已失效"
errPagesSourceSyncFailed = "pages 部署源同步失败"
errPagesSourceTaskDispatchFailed = "pages 部署源任务入队失败"
errPagesSourceInternal = "pages 部署源操作失败，请稍后重试"
)
internal/apps/openflare/pages/helpers.go
+21 -4
View File
@@ -38,6 +38,7 @@ const (
pagesIngestMarkerKey = "pages_ingest_marker"
pagesIngestMarkerV2 = "pages_deployment_v2"
pagesProjectIDMetadataKey = "pages_project_id"
pagesSourceIDMetadataKey = "pages_source_id"
pagesMaxPathLength = 512
bytesPerMiB = 1024 * 1024
pagesExtractedSizeMultiplier = 4
@@ -263,10 +264,29 @@ func ingestPagesDeploymentPackage(
projectID uint,
fileName string,
format pagesarchive.Format,
) (upload.IngestResult, error) {
return ingestPagesDeploymentPackageWithSource(ctx, localPath, checksum, projectID, 0, fileName, format)
}
func ingestPagesDeploymentPackageWithSource(
ctx context.Context,
localPath string,
checksum string,
projectID uint,
sourceID uint,
fileName string,
format pagesarchive.Format,
) (upload.IngestResult, error) {
systemUser := repository.GetSystemUser(ctx)
accessMode := 0
extension := pagesarchive.NormalizeNameExtension(fileName, format)
extra := map[string]any{
pagesIngestMarkerKey: pagesIngestMarkerV2,
pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10),
}
if sourceID != 0 {
extra[pagesSourceIDMetadataKey] = strconv.FormatUint(uint64(sourceID), 10)
}
return upload.IngestFromLocalPath(ctx, localPath, upload.IngestRequest{
UserID: systemUser.ID,
FileName: fileName,
@@ -278,10 +298,7 @@ func ingestPagesDeploymentPackage(
SkipExtensionCheck: true,
Policy: upload.PolicyDedupNewRecord,
Metadata: model.UploadMetadata{
Extra: map[string]any{
pagesIngestMarkerKey: pagesIngestMarkerV2,
pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10),
},
Extra: extra,
},
})
}
internal/apps/openflare/pages/logics.go
+278 -69
View File
@@ -60,6 +60,9 @@ type DeploymentView struct {
FileCount int `json:"file_count"`
TotalSize int64 `json:"total_size"`
CreatedBy string `json:"created_by"`
SourceType string `json:"source_type"`
SourceLabel string `json:"source_label"`
TriggerType string `json:"trigger_type"`
CreatedAt time.Time `json:"created_at"`
ActivatedAt *time.Time `json:"activated_at"`
}
@@ -151,13 +154,14 @@ func UpdateProject(ctx context.Context, id uint, input Input) (*View, error) {
if err != nil {
return err
}
if (existing.RootDir != project.RootDir || existing.EntryFile != project.EntryFile) &&
contentConfigChanged := existing.RootDir != project.RootDir || existing.EntryFile != project.EntryFile
if contentConfigChanged &&
existing.ActiveDeploymentID != nil && *existing.ActiveDeploymentID != 0 {
if err := ensureDeploymentEntry(tx, *existing.ActiveDeploymentID, project.RootDir, project.EntryFile); err != nil {
return err
}
}
return tx.Model(&existing).Updates(map[string]any{
updates := map[string]any{
"name": project.Name,
"slug": project.Slug,
"description": project.Description,
@@ -170,7 +174,23 @@ func UpdateProject(ctx context.Context, id uint, input Input) (*View, error) {
"api_proxy_rewrite": project.APIProxyRewrite,
"root_dir": project.RootDir,
"entry_file": project.EntryFile,
}).Error
}
if contentConfigChanged {
updates["content_config_version"] = existing.ContentConfigVersion + 1
var source model.PagesProjectSource
sourceErr := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("project_id = ?", existing.ID).
First(&source).Error
if sourceErr != nil && !errors.Is(sourceErr, gorm.ErrRecordNotFound) {
return sourceErr
}
if sourceErr == nil {
if err := fenceAndNormalizeRuntime(tx, source.ID); err != nil {
return err
}
}
}
return tx.Model(&existing).Updates(updates).Error
})
if err != nil {
if isUniqueConstraintError(err) {
@@ -221,6 +241,28 @@ func DeleteProject(ctx context.Context, id uint) error {
return errors.New(errPagesDeleteReferenced)
}
}
var source model.PagesProjectSource
sourceErr := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("project_id = ?", project.ID).
First(&source).Error
if sourceErr != nil && !errors.Is(sourceErr, gorm.ErrRecordNotFound) {
return sourceErr
}
if sourceErr == nil {
var runtime model.PagesProjectSourceRuntime
runtimeErr := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", source.ID).
First(&runtime).Error
if runtimeErr != nil && !errors.Is(runtimeErr, gorm.ErrRecordNotFound) {
return runtimeErr
}
if err := tx.Where("source_id = ?", source.ID).Delete(&model.PagesProjectSourceRuntime{}).Error; err != nil {
return err
}
if err := tx.Delete(&source).Error; err != nil {
return err
}
}
if err := tx.Where("project_id = ?", project.ID).Find(&deployments).Error; err != nil {
return err
}
@@ -233,7 +275,7 @@ func DeleteProject(ctx context.Context, id uint) error {
if err := tx.Where("project_id = ?", project.ID).Delete(&model.PagesDeployment{}).Error; err != nil {
return err
}
if err := tx.Delete(project).Error; err != nil {
if err := tx.Delete(&lockedProject).Error; err != nil {
return err
}
return nil
@@ -301,7 +343,18 @@ func UploadDeployment(ctx context.Context, projectID uint, fileHeader *multipart
return nil, err
}
defer func() { _ = os.Remove(tempPath) }()
return createDeploymentFromTempPackage(ctx, project, tempPath, checksum, format, fileHeader.Filename, createdBy, limits)
return createDeploymentFromTempPackage(
ctx,
project,
tempPath,
checksum,
format,
fileHeader.Filename,
createdBy,
"manual_upload",
"manual_upload",
limits,
)
}
// UploadFromURLInput is the request body for downloading a deployment package from a remote URL.
@@ -321,7 +374,18 @@ func UploadDeploymentFromURL(ctx context.Context, projectID uint, rawURL string,
return nil, err
}
defer func() { _ = os.Remove(tempPath) }()
return createDeploymentFromTempPackage(ctx, project, tempPath, checksum, format, fileName, createdBy, limits)
return createDeploymentFromTempPackage(
ctx,
project,
tempPath,
checksum,
format,
fileName,
createdBy,
"manual_url",
"manual_url",
limits,
)
}
func createDeploymentFromTempPackage(
@@ -332,6 +396,8 @@ func createDeploymentFromTempPackage(
format pagesarchive.Format,
fileName string,
createdBy string,
sourceType string,
triggerType string,
limits pagesLimits,
) (*DeploymentView, error) {
if project == nil {
@@ -377,6 +443,15 @@ func createDeploymentFromTempPackage(
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&lockedProject, project.ID).Error; err != nil {
return err
}
var uploadRecord model.Upload
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id = ?", ingestResult.Upload.ID).
First(&uploadRecord).Error; err != nil {
return errors.New(errPagesPackageUploadMissing)
}
if uploadRecord.Status != model.UploadStatusUsed || uploadRecord.Type != upload.ReservedPagesDeploymentType {
return errors.New(errPagesPackageUploadMissing)
}
var maxNumber int
if err := tx.Model(&model.PagesDeployment{}).
Where("project_id = ?", project.ID).
@@ -393,6 +468,9 @@ func createDeploymentFromTempPackage(
FileCount: manifest.FileCount,
TotalSize: manifest.TotalSize,
CreatedBy: strings.TrimSpace(createdBy),
SourceType: sourceType,
SourceLabel: safeRemoteSourceLabel(fileName),
TriggerType: triggerType,
}
if err := tx.Create(deployment).Error; err != nil {
return err
@@ -582,72 +660,27 @@ func selectDeploymentsToPrune(
// ActivateDeployment 激活 Pages 部署。
func ActivateDeployment(ctx context.Context, projectID uint, deploymentID uint) (*View, error) {
project, err := model.GetPagesProjectByID(ctx, projectID)
return ActivateDeploymentAs(ctx, projectID, deploymentID, "system:pages-manual-activation")
}
// ActivateDeploymentAs activates a historical deployment and fences any
// configured source when the active deployment actually changes.
func ActivateDeploymentAs(ctx context.Context, projectID uint, deploymentID uint, actor string) (*View, error) {
if err := ensureActivationDeploymentUpload(ctx, projectID, deploymentID); err != nil {
return nil, err
}
audit, err := activateDeploymentTransaction(ctx, projectID, deploymentID, time.Now())
if err != nil {
return nil, err
}
deployment, err := model.GetPagesDeploymentByID(ctx, deploymentID)
if err != nil {
return nil, err
}
if deployment.ProjectID != project.ID {
return nil, errors.New(errPagesDeploymentMismatch)
}
if deployment.UploadID == 0 {
if err = ensureDeploymentUploadRecord(ctx, deployment); err != nil {
return nil, err
}
}
now := time.Now()
if err = db.DB(ctx).Transaction(func(tx *gorm.DB) error {
var project model.PagesProject
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil {
return err
}
var deployment model.PagesDeployment
if err := tx.First(&deployment, deploymentID).Error; err != nil {
return err
}
if deployment.ProjectID != project.ID {
return errors.New(errPagesDeploymentMismatch)
}
rootDir, err := validateAndNormalizePagesRootDir(project.RootDir)
if err != nil {
return err
}
entryFile, err := validateAndNormalizePagesEntryFile(project.EntryFile)
if err != nil {
return err
}
if err := ensureDeploymentEntry(tx, deployment.ID, rootDir, entryFile); err != nil {
return err
}
var uploadRecord model.Upload
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id = ?", deployment.UploadID).
First(&uploadRecord).Error; err != nil {
return errors.New(errPagesPackageUploadMissing)
}
if uploadRecord.Status != model.UploadStatusUsed || uploadRecord.Type != upload.ReservedPagesDeploymentType {
return errors.New(errPagesPackageUploadMissing)
}
if err := tx.Model(&model.PagesDeployment{}).
Where("project_id = ?", project.ID).
Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil {
return err
}
if err := tx.Model(&deployment).Updates(map[string]any{
"status": model.PagesDeploymentStatusActive,
"activated_at": &now,
}).Error; err != nil {
return err
}
return tx.Model(&project).Updates(map[string]any{
"active_deployment_id": deployment.ID,
}).Error
}); err != nil {
return nil, err
if audit.Noop {
return GetProject(ctx, projectID)
}
logger.InfoF(ctx,
"[Pages] manual activation: actor=%s project_id=%d old_deployment_id=%d new_deployment_id=%d source_type=%s source_identity=%s auto_disabled=%t",
strings.TrimSpace(actor), projectID, audit.OldDeploymentID, deploymentID,
audit.SourceType, audit.SourceIdentity, audit.AutoDisabled,
)
limits := resolvePagesLimits(ctx)
if pruneErr := pruneProjectDeploymentHistory(ctx, projectID, limits.HistoryCount, 0); pruneErr != nil {
logger.ErrorF(ctx,
@@ -658,6 +691,179 @@ func ActivateDeployment(ctx context.Context, projectID uint, deploymentID uint)
return GetProject(ctx, projectID)
}
type deploymentActivationAudit struct {
OldDeploymentID uint
SourceType string
SourceIdentity string
AutoDisabled bool
Noop bool
}
type deploymentActivationSource struct {
Source *model.PagesProjectSource
Runtime *model.PagesProjectSourceRuntime
}
func ensureActivationDeploymentUpload(ctx context.Context, projectID uint, deploymentID uint) error {
deployment, err := model.GetPagesDeploymentByID(ctx, deploymentID)
if err != nil {
return err
}
if deployment.ProjectID != projectID {
return errors.New(errPagesDeploymentMismatch)
}
if deployment.UploadID != 0 {
return nil
}
return ensureDeploymentUploadRecord(ctx, deployment)
}
func activateDeploymentTransaction(
ctx context.Context,
projectID uint,
deploymentID uint,
now time.Time,
) (deploymentActivationAudit, error) {
audit := deploymentActivationAudit{}
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
var project model.PagesProject
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil {
return err
}
if project.ActiveDeploymentID != nil {
audit.OldDeploymentID = *project.ActiveDeploymentID
}
if audit.OldDeploymentID == deploymentID {
audit.Noop = true
return nil
}
sourceState, err := lockDeploymentActivationSource(tx, project.ID)
if err != nil {
return err
}
deployment, err := loadDeploymentActivationTarget(tx, &project, deploymentID)
if err != nil {
return err
}
if err := fenceDeploymentActivationSource(tx, sourceState, deployment, &audit); err != nil {
return err
}
return switchActiveDeploymentTx(tx, &project, deployment, now)
})
return audit, err
}
func lockDeploymentActivationSource(tx *gorm.DB, projectID uint) (*deploymentActivationSource, error) {
var source model.PagesProjectSource
err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("project_id = ?", projectID).
First(&source).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
var runtime model.PagesProjectSourceRuntime
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", source.ID).
First(&runtime).Error; err != nil {
return nil, err
}
return &deploymentActivationSource{Source: &source, Runtime: &runtime}, nil
}
func loadDeploymentActivationTarget(
tx *gorm.DB,
project *model.PagesProject,
deploymentID uint,
) (*model.PagesDeployment, error) {
var deployment model.PagesDeployment
if err := tx.First(&deployment, deploymentID).Error; err != nil {
return nil, err
}
if deployment.ProjectID != project.ID {
return nil, errors.New(errPagesDeploymentMismatch)
}
rootDir, err := validateAndNormalizePagesRootDir(project.RootDir)
if err != nil {
return nil, err
}
entryFile, err := validateAndNormalizePagesEntryFile(project.EntryFile)
if err != nil {
return nil, err
}
if err := ensureDeploymentEntry(tx, deployment.ID, rootDir, entryFile); err != nil {
return nil, err
}
var uploadRecord model.Upload
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id = ?", deployment.UploadID).
First(&uploadRecord).Error; err != nil {
return nil, errors.New(errPagesPackageUploadMissing)
}
if uploadRecord.Status != model.UploadStatusUsed || uploadRecord.Type != upload.ReservedPagesDeploymentType {
return nil, errors.New(errPagesPackageUploadMissing)
}
return &deployment, nil
}
func fenceDeploymentActivationSource(
tx *gorm.DB,
state *deploymentActivationSource,
deployment *model.PagesDeployment,
audit *deploymentActivationAudit,
) error {
if state == nil {
return nil
}
audit.SourceType = state.Source.SourceType
audit.SourceIdentity = state.Source.SourceIdentity
audit.AutoDisabled = state.Source.AutoUpdateEnabled
if err := tx.Model(state.Source).Updates(map[string]any{
"config_version": state.Source.ConfigVersion + 1,
"auto_update_enabled": false,
}).Error; err != nil {
return err
}
if deployment.SourceIdentity != nil && *deployment.SourceIdentity == state.Source.SourceIdentity &&
deployment.SourceRevision != nil {
state.Runtime.LastAppliedRevision = *deployment.SourceRevision
state.Runtime.LastAppliedDetail = deployment.SourceMeta
} else {
state.Runtime.LastAppliedRevision = ""
state.Runtime.LastAppliedDetail = ""
}
return tx.Model(state.Runtime).Updates(map[string]any{
"last_applied_revision": state.Runtime.LastAppliedRevision,
"last_applied_detail": state.Runtime.LastAppliedDetail,
sourceRuntimeColumnSyncStatus: normalizedSourceRuntimeStatus(state.Runtime),
sourceRuntimeColumnLeaseToken: "",
sourceRuntimeColumnLeaseExpiresAt: nil,
}).Error
}
func switchActiveDeploymentTx(
tx *gorm.DB,
project *model.PagesProject,
deployment *model.PagesDeployment,
now time.Time,
) error {
if err := tx.Model(&model.PagesDeployment{}).
Where("project_id = ?", project.ID).
Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil {
return err
}
if err := tx.Model(deployment).Updates(map[string]any{
"status": model.PagesDeploymentStatusActive,
"activated_at": &now,
}).Error; err != nil {
return err
}
return tx.Model(project).Update("active_deployment_id", deployment.ID).Error
}
// GetDeploymentPackageHash returns the upload SHA-256 hash of the deployment package.
// Prefer GetProjectLatestPackageHash for Agent latest-pointer pulls.
func GetDeploymentPackageHash(ctx context.Context, deploymentID uint) (string, error) {
@@ -1104,7 +1310,7 @@ func buildProject(existing *model.PagesProject, input Input) (*model.PagesProjec
return nil, errors.New(errPagesAPIProxyPassRequired)
}
parsedURL, err := url.Parse(apiProxyPass)
if err != nil || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") || parsedURL.Host == "" {
if err != nil || (parsedURL.Scheme != remoteSourceSchemeHTTP && parsedURL.Scheme != remoteSourceSchemeHTTPS) || parsedURL.Host == "" {
return nil, errors.New(errPagesAPIProxyPassInvalid)
}
}
@@ -1177,6 +1383,9 @@ func buildDeploymentView(deployment *model.PagesDeployment) DeploymentView {
FileCount: deployment.FileCount,
TotalSize: deployment.TotalSize,
CreatedBy: deployment.CreatedBy,
SourceType: deployment.SourceType,
SourceLabel: deployment.SourceLabel,
TriggerType: deployment.TriggerType,
CreatedAt: deployment.CreatedAt,
ActivatedAt: deployment.ActivatedAt,
}
internal/apps/openflare/pages/logics_test.go
+2
View File
@@ -43,6 +43,8 @@ func setupPagesTestDB(t *testing.T) func() {
&model.PagesProject{},
&model.PagesDeployment{},
&model.PagesDeploymentFile{},
&model.PagesProjectSource{},
&model.PagesProjectSourceRuntime{},
&model.ConfigVersion{},
&model.SystemConfig{},
))
internal/apps/openflare/pages/routers.go
+217 -2
View File
@@ -4,7 +4,10 @@
package pages
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
@@ -12,7 +15,9 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/openflare/apiutil"
"github.com/Rain-kl/Wavelet/internal/common/response"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
func handleLogicError(c *gin.Context, err error) bool {
@@ -22,6 +27,53 @@ func handleLogicError(c *gin.Context, err error) bool {
return apiutil.AbortNotFoundIfMissing(c, err, errPagesProjectNotFound)
}
func handleSourceLogicError(c *gin.Context, err error) bool {
if err == nil {
return false
}
if errors.Is(err, gorm.ErrRecordNotFound) || err.Error() == errPagesSourceNotFound {
response.AbortNotFound(c, errPagesSourceNotFound)
return true
}
switch err.Error() {
case errPagesSourceActionBusy:
response.AbortConflict(c, errPagesSourceActionBusy)
case errPagesSourceTypeRequired,
errPagesSourceTypeUnsupported,
errPagesSourceRemoteFields,
errPagesSourceRemoteURLRequired,
errPagesSourceRemoteURLMode,
errPagesSourceRemoteURLInvalid,
errPagesSourceNetworkPolicy,
errPagesSourceCheckUnsupported,
errPagesSourceActionInvalid:
response.AbortBadRequest(c, err.Error())
case errPagesSourceTaskDispatchFailed:
response.AbortInternal(c, errPagesSourceInternal)
default:
logger.ErrorF(c.Request.Context(), "[PagesSource] API operation failed: error=%v", err)
response.AbortInternal(c, errPagesSourceInternal)
}
return true
}
func decodeStrictJSON(c *gin.Context, target any, allowEmpty bool) bool {
decoder := json.NewDecoder(c.Request.Body)
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
if allowEmpty && errors.Is(err, io.EOF) {
return true
}
response.AbortBadRequest(c, errPagesSourceActionInvalid)
return false
}
if err := ensureJSONEOF(decoder); err != nil {
response.AbortBadRequest(c, errPagesSourceActionInvalid)
return false
}
return true
}
func deploymentIDParam(c *gin.Context) (uint, bool) {
raw := c.Param("deployment_id")
if raw == "" {
@@ -174,6 +226,164 @@ func DeleteProjectHandler(c *gin.Context) {
c.JSON(http.StatusOK, response.OKNil())
}
// GetSourceHandler 获取 Pages 项目的部署源。
// @Summary 获取 Pages 部署源
// @Description 返回脱敏后的项目部署源配置与运行状态，需要管理员权限
// @Tags openflare-pages
// @Produce json
// @Security SessionCookie
// @Param id path int true "项目 ID"
// @Success 200 {object} response.Any{data=pages.SourceView} "部署源"
// @Failure 400 {object} response.Any "参数错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "项目或部署源不存在"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/d/pages/{id}/source [get]
func GetSourceHandler(c *gin.Context) {
projectID, ok := apiutil.IDParam(c)
if !ok {
return
}
source, err := GetSource(c.Request.Context(), projectID)
if handleSourceLogicError(c, err) {
return
}
c.JSON(http.StatusOK, response.OK(source))
}
// UpdateSourceHandler 创建或更新 Pages 项目部署源。
// @Summary 更新 Pages 部署源
// @Description Phase 1 支持 Remote URL 来源；完整地址仅写入，不会在响应中返回
// @Tags openflare-pages
// @Accept json
// @Produce json
// @Security SessionCookie
// @Param id path int true "项目 ID"
// @Param request body pages.SourceUpdateInput true "部署源配置"
// @Success 200 {object} response.Any{data=pages.SourceUpdateResult} "更新结果"
// @Failure 400 {object} response.Any "配置无效"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "项目不存在"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/d/pages/{id}/source/update [post]
func UpdateSourceHandler(c *gin.Context) {
projectID, ok := apiutil.IDParam(c)
if !ok {
return
}
var input SourceUpdateInput
if !decodeStrictJSON(c, &input, false) {
return
}
result, err := UpdateSource(c.Request.Context(), projectID, input)
if handleSourceLogicError(c, err) {
return
}
c.JSON(http.StatusOK, response.OK(result))
}
// DeleteSourceHandler 将 Pages 项目切换回手动部署模式。
// @Summary 删除 Pages 部署源
// @Description 幂等删除持久部署源；已有部署历史与当前生产部署保持不变
// @Tags openflare-pages
// @Produce json
// @Security SessionCookie
// @Param id path int true "项目 ID"
// @Success 200 {object} response.Any{data=pages.SourceView} "手动来源视图"
// @Failure 400 {object} response.Any "参数错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "项目不存在"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/d/pages/{id}/source/delete [post]
func DeleteSourceHandler(c *gin.Context) {
projectID, ok := apiutil.IDParam(c)
if !ok {
return
}
source, err := DeleteSource(c.Request.Context(), projectID)
if handleSourceLogicError(c, err) {
return
}
c.JSON(http.StatusOK, response.OK(source))
}
// CheckSourceHandler 请求检查 Pages 部署源。
// @Summary 检查 Pages 部署源
// @Description Remote URL 来源不支持检查更新；该端点为后续 GitHub Release 来源保留统一动作契约
// @Tags openflare-pages
// @Produce json
// @Security SessionCookie
// @Param id path int true "项目 ID"
// @Success 200 {object} response.Any{data=pages.SourceActionReceipt} "任务回执"
// @Failure 400 {object} response.Any "当前来源不支持检查"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "部署源不存在"
// @Failure 409 {object} response.Any "来源任务正在执行"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/d/pages/{id}/source/check [post]
func CheckSourceHandler(c *gin.Context) {
projectID, ok := apiutil.IDParam(c)
if !ok {
return
}
actor, ok := currentPagesActor(c)
if !ok {
return
}
receipt, err := DispatchSourceAction(c.Request.Context(), projectID, sourceActionCheck, actor, "")
if handleSourceLogicError(c, err) {
return
}
c.JSON(http.StatusOK, response.OK(receipt))
}
// SourceSyncInput is the optional source sync action payload.
type SourceSyncInput struct {
ConfirmedRevision string `json:"confirmed_revision"`
}
// SyncSourceHandler 请求同步并发布 Pages 部署源。
// @Summary 同步并发布 Pages 部署源
// @Description 异步下载、校验并原子激活来源部署包；空请求体与空 JSON 对象均有效
// @Tags openflare-pages
// @Accept json
// @Produce json
// @Security SessionCookie
// @Param id path int true "项目 ID"
// @Param request body pages.SourceSyncInput false "同步参数"
// @Success 200 {object} response.Any{data=pages.SourceActionReceipt} "任务回执"
// @Failure 400 {object} response.Any "参数或来源类型无效"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "部署源不存在"
// @Failure 409 {object} response.Any "来源任务正在执行"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/d/pages/{id}/source/sync [post]
func SyncSourceHandler(c *gin.Context) {
projectID, ok := apiutil.IDParam(c)
if !ok {
return
}
var input SourceSyncInput
if !decodeStrictJSON(c, &input, true) {
return
}
actor, ok := currentPagesActor(c)
if !ok {
return
}
receipt, err := DispatchSourceAction(
c.Request.Context(),
projectID,
sourceActionSync,
actor,
input.ConfirmedRevision,
)
if handleSourceLogicError(c, err) {
return
}
c.JSON(http.StatusOK, response.OK(receipt))
}
// ListDeploymentsHandler 列出项目的全部部署。
// @Summary 列出 Pages 部署
// @Description 返回指定项目的全部部署记录，需要管理员权限
@@ -239,7 +449,8 @@ func UploadDeploymentHandler(c *gin.Context) {
// UploadDeploymentFromURLHandler 从 URL 下载并创建 Pages 部署。
// @Summary 从 URL 导入 Pages 部署包
// @Description 从用户提供的 HTTP(S) 链接下载部署包并创建部署记录；服务端使用浏览器伪装请求头拉取，允许内网地址与不安全 TLS 证书，需要管理员权限
// @Description 已弃用的一次性 URL 导入；使用 trusted_internal 策略兼容内网与自签名证书，不创建持久部署源
// @Deprecated
// @Tags openflare-pages
// @Accept json
// @Produce json
@@ -298,7 +509,11 @@ func ActivateDeploymentHandler(c *gin.Context) {
if !ok {
return
}
project, err := ActivateDeployment(c.Request.Context(), projectID, deploymentID)
actor, ok := currentPagesActor(c)
if !ok {
return
}
project, err := ActivateDeploymentAs(c.Request.Context(), projectID, deploymentID, actor)
if handleLogicError(c, err) {
return
}
internal/apps/openflare/pages/routers_source_test.go
+224
View File
@@ -0,0 +1,224 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/task"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/alicebob/miniredis/v2"
"github.com/gin-gonic/gin"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
type sourceHandlerEnvelope struct {
ErrorMsg string `json:"error_msg"`
Data json.RawMessage `json:"data"`
}
func newPagesSourceTestRouter(userID uint64) *gin.Engine {
router := testhelper.NewTestGinEngine(func(ctx *gin.Context) {
oauth.SetToContext(ctx, oauth.UserObjKey, &model.User{ID: userID})
ctx.Next()
})
router.GET("/api/v1/d/pages/:id/source", GetSourceHandler)
router.POST("/api/v1/d/pages/:id/source/update", UpdateSourceHandler)
router.POST("/api/v1/d/pages/:id/source/delete", DeleteSourceHandler)
router.POST("/api/v1/d/pages/:id/source/check", CheckSourceHandler)
router.POST("/api/v1/d/pages/:id/source/sync", SyncSourceHandler)
return router
}
func performPagesSourceRequest(
t *testing.T,
router http.Handler,
method string,
path string,
body []byte,
) (int, sourceHandlerEnvelope) {
t.Helper()
request := httptest.NewRequest(method, path, bytes.NewReader(body))
if body != nil {
request.Header.Set("Content-Type", "application/json")
}
recorder := httptest.NewRecorder()
router.ServeHTTP(recorder, request)
var envelope sourceHandlerEnvelope
if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil {
t.Fatalf("json.Unmarshal(%s %s response %q) error = %v, want nil", method, path, recorder.Body.String(), err)
}
return recorder.Code, envelope
}
func setupPagesSourceDispatchTest(t *testing.T) {
t.Helper()
miniRedis, err := miniredis.Run()
if err != nil {
t.Fatalf("miniredis.Run() error = %v, want nil", err)
}
redisClient := redis.NewClient(&redis.Options{Addr: miniRedis.Addr()})
asynqClient := asynq.NewClient(asynq.RedisClientOpt{Addr: miniRedis.Addr()})
previousRedis := db.Redis
previousAsynqClient := task.AsynqClient
db.Redis = redisClient
task.AsynqClient = asynqClient
task.RegisterTaskMeta(PagesSourceActionMeta)
t.Cleanup(func() {
_ = asynqClient.Close()
_ = redisClient.Close()
miniRedis.Close()
task.AsynqClient = previousAsynqClient
db.Redis = previousRedis
})
}
func TestPagesSourceHandlersReturnStableActionErrors(t *testing.T) {
ctx := setupPagesSourceTest(t)
router := newPagesSourceTestRouter(42)
manualProject := mustCreatePagesSourceProject(t, ctx, "handler-no-source")
code, envelope := performPagesSourceRequest(
t,
router,
http.MethodPost,
fmt.Sprintf("/api/v1/d/pages/%d/source/sync", manualProject.ID),
nil,
)
if got, want := code, http.StatusNotFound; got != want {
t.Errorf("POST source/sync without source status = %d, want %d", got, want)
}
if got, want := envelope.ErrorMsg, errPagesSourceNotFound; got != want {
t.Errorf("POST source/sync without source error = %q, want %q", got, want)
}
remoteProject := mustCreatePagesSourceProject(t, ctx, "handler-check")
_, _ = mustConfigureRemoteSource(
t,
ctx,
remoteProject.ID,
"https://example.com/site.zip?token=handler-secret",
RemoteNetworkPolicyPublic,
)
code, envelope = performPagesSourceRequest(
t,
router,
http.MethodPost,
fmt.Sprintf("/api/v1/d/pages/%d/source/check", remoteProject.ID),
nil,
)
if got, want := code, http.StatusBadRequest; got != want {
t.Errorf("POST remote source/check status = %d, want %d", got, want)
}
if got, want := envelope.ErrorMsg, errPagesSourceCheckUnsupported; got != want {
t.Errorf("POST remote source/check error = %q, want %q", got, want)
}
if strings.Contains(string(envelope.Data), "handler-secret") || strings.Contains(envelope.ErrorMsg, "handler-secret") {
t.Errorf("POST remote source/check response = %+v, want no URL secret", envelope)
}
busyProject := mustCreatePagesSourceProject(t, ctx, "handler-busy")
busySource, _ := mustConfigureRemoteSource(
t,
ctx,
busyProject.ID,
"https://example.com/site.zip",
RemoteNetworkPolicyPublic,
)
future := time.Now().Add(time.Minute)
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", busySource.ID).
Updates(map[string]any{
"sync_status": pagesSourceStatusSyncing,
"lease_token": "busy-owner",
"lease_expires_at": &future,
}).Error; err != nil {
t.Fatalf("seed busy source runtime error = %v, want nil", err)
}
code, envelope = performPagesSourceRequest(
t,
router,
http.MethodPost,
fmt.Sprintf("/api/v1/d/pages/%d/source/sync", busyProject.ID),
[]byte(`{}`),
)
if got, want := code, http.StatusConflict; got != want {
t.Errorf("POST busy source/sync status = %d, want %d", got, want)
}
if got, want := envelope.ErrorMsg, errPagesSourceActionBusy; got != want {
t.Errorf("POST busy source/sync error = %q, want %q", got, want)
}
}
func TestSyncSourceHandlerAcceptsEmptyBodyAndEmptyObject(t *testing.T) {
ctx := setupPagesSourceTest(t)
setupPagesSourceDispatchTest(t)
router := newPagesSourceTestRouter(77)
project := mustCreatePagesSourceProject(t, ctx, "handler-empty-sync")
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip?token=dispatch-secret",
RemoteNetworkPolicyPublic,
)
path := fmt.Sprintf("/api/v1/d/pages/%d/source/sync", project.ID)
for _, test := range []struct {
name string
body []byte
}{
{name: "empty body", body: nil},
{name: "empty object", body: []byte(`{}`)},
} {
t.Run(test.name, func(t *testing.T) {
code, envelope := performPagesSourceRequest(t, router, http.MethodPost, path, test.body)
if got, want := code, http.StatusOK; got != want {
t.Fatalf("POST source/sync (%s) status = %d, want %d; error=%q", test.name, got, want, envelope.ErrorMsg)
}
if envelope.ErrorMsg != "" {
t.Errorf("POST source/sync (%s) error = %q, want empty", test.name, envelope.ErrorMsg)
}
var receipt SourceActionReceipt
if err := json.Unmarshal(envelope.Data, &receipt); err != nil {
t.Fatalf("json.Unmarshal(source/sync %s receipt) error = %v, want nil", test.name, err)
}
if receipt.TaskID == "" || receipt.ExecutionID == "" || receipt.Action != sourceActionSync {
t.Errorf("POST source/sync (%s) receipt = %+v, want task/execution IDs and action %q", test.name, receipt, sourceActionSync)
}
})
}
var executions []model.TaskExecution
if err := db.DB(ctx).Where("task_type = ?", PagesSourceActionTask).Order("id asc").Find(&executions).Error; err != nil {
t.Fatalf("list Pages source task executions error = %v, want nil", err)
}
if got, want := len(executions), 2; got != want {
t.Fatalf("Pages source task execution count = %d, want %d", got, want)
}
for _, execution := range executions {
if strings.Contains(execution.Payload, "dispatch-secret") || strings.Contains(execution.Payload, "http") {
t.Errorf("task execution %q payload = %s, want no Remote URL secret", execution.TaskID, execution.Payload)
}
var payload SourceActionPayload
if err := json.Unmarshal([]byte(execution.Payload), &payload); err != nil {
t.Errorf("json.Unmarshal(task execution %q payload) error = %v, want nil", execution.TaskID, err)
continue
}
if payload.SourceID != source.ID || payload.ConfigVersion != source.ConfigVersion || payload.Actor != "user:77" {
t.Errorf("task execution %q payload = %+v, want source=%d config=%d actor=user:77", execution.TaskID, payload, source.ID, source.ConfigVersion)
}
}
}
internal/apps/openflare/pages/source.go
+495
View File
@@ -0,0 +1,495 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"net"
"net/url"
"path"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const (
// PagesSourceTypeManual represents projects without a persisted source row.
PagesSourceTypeManual = "manual"
// PagesSourceTypeRemoteURL represents a persisted artifact URL.
PagesSourceTypeRemoteURL = "remote_url"
// PagesSourceTypeGitHubRelease is reserved for Phase 2.
PagesSourceTypeGitHubRelease = "github_release"
pagesSourceStatusIdle = "idle"
pagesSourceStatusChecking = "checking"
pagesSourceStatusUpdateAvailable = "update_available"
pagesSourceStatusSyncing = "syncing"
pagesSourceStatusFailed = "failed"
pagesSourceStatusAttention = "attention"
defaultRemoteAssetLabel = "pages-package"
)
// SourceUpdateInput is the discriminated source configuration payload.
// GitHub fields are accepted by the decoder so mode-incompatible values can be
// rejected deterministically; GitHub itself is enabled in Phase 2.
type SourceUpdateInput struct {
SourceType string `json:"source_type"`
RemoteURLSet bool `json:"remote_url_set"`
RemoteURL string `json:"remote_url"`
RemoteNetworkPolicy string `json:"remote_network_policy"`
RepositoryURL string `json:"repository_url"`
ReleaseSelector string `json:"release_selector"`
ReleaseTag string `json:"release_tag"`
AssetName string `json:"asset_name"`
AutoUpdateEnabled bool `json:"auto_update_enabled"`
CheckIntervalMinutes int `json:"check_interval_minutes"`
}
// SourceRevisionView is a credential-free source cursor.
type SourceRevisionView struct {
Revision string `json:"revision"`
Label string `json:"label"`
AssetName string `json:"asset_name,omitempty"`
}
// SourceView is the safe discriminated source view returned to the console.
type SourceView struct {
SourceType string `json:"source_type"`
HasRemoteURL bool `json:"has_remote_url,omitempty"`
DisplayURL string `json:"display_url,omitempty"`
RemoteNetworkPolicy string `json:"remote_network_policy,omitempty"`
GitHubRepository string `json:"github_repository,omitempty"`
ReleaseSelector string `json:"release_selector,omitempty"`
ReleaseTag string `json:"release_tag,omitempty"`
AssetName string `json:"asset_name,omitempty"`
AutoUpdateEnabled bool `json:"auto_update_enabled,omitempty"`
CheckIntervalMinutes int `json:"check_interval_minutes,omitempty"`
SyncStatus string `json:"sync_status,omitempty"`
UpdateAvailable bool `json:"update_available,omitempty"`
LastSeen *SourceRevisionView `json:"last_seen,omitempty"`
LastApplied *SourceRevisionView `json:"last_applied,omitempty"`
LastCheckedAt *time.Time `json:"last_checked_at,omitempty"`
LastSyncedAt *time.Time `json:"last_synced_at,omitempty"`
NextCheckAt *time.Time `json:"next_check_at,omitempty"`
LastError string `json:"last_error,omitempty"`
}
// SourceActionReceipt identifies the internal task execution created by an action API.
type SourceActionReceipt struct {
TaskID string `json:"task_id"`
ExecutionID string `json:"execution_id"`
Action string `json:"action"`
}
// SourceUpdateResult is returned after persisting source configuration.
type SourceUpdateResult struct {
Source *SourceView `json:"source"`
CheckTask *SourceActionReceipt `json:"check_task"`
Warning string `json:"warning"`
}
type sourceDetail struct {
Provider string `json:"provider"`
Label string `json:"label"`
AssetName string `json:"asset_name,omitempty"`
ReleaseID string `json:"release_id,omitempty"`
}
type remoteSourceConfig struct {
URL string
Policy string
Identity string
}
// GetSource returns the current persisted source or a manual discriminator.
func GetSource(ctx context.Context, projectID uint) (*SourceView, error) {
if _, err := model.GetPagesProjectByID(ctx, projectID); err != nil {
return nil, err
}
source, runtime, err := loadSourceByProject(ctx, projectID)
if errors.Is(err, gorm.ErrRecordNotFound) {
return &SourceView{SourceType: PagesSourceTypeManual}, nil
}
if err != nil {
return nil, err
}
return buildSourceView(source, runtime)
}
// UpdateSource creates or updates a Remote URL source and its 1:1 runtime row.
func UpdateSource(ctx context.Context, projectID uint, input SourceUpdateInput) (*SourceUpdateResult, error) {
if err := validateRemoteSourceInput(input); err != nil {
return nil, err
}
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return updateRemoteSourceTx(tx, projectID, input)
})
if err != nil {
return nil, err
}
view, err := GetSource(ctx, projectID)
if err != nil {
return nil, err
}
return &SourceUpdateResult{Source: view, Warning: ""}, nil
}
func updateRemoteSourceTx(tx *gorm.DB, projectID uint, input SourceUpdateInput) error {
var project model.PagesProject
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil {
return err
}
existing, hasExisting, err := loadProjectSourceForUpdate(tx, projectID)
if err != nil {
return err
}
config, err := buildRemoteSourceConfig(existing, hasExisting, input)
if err != nil {
return err
}
if !hasExisting {
return createRemoteSourceTx(tx, projectID, config)
}
return updateExistingRemoteSourceTx(tx, existing, config)
}
func loadProjectSourceForUpdate(tx *gorm.DB, projectID uint) (*model.PagesProjectSource, bool, error) {
var source model.PagesProjectSource
err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("project_id = ?", projectID).
First(&source).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return &source, false, nil
}
if err != nil {
return nil, false, err
}
return &source, true, nil
}
func buildRemoteSourceConfig(
existing *model.PagesProjectSource,
hasExisting bool,
input SourceUpdateInput,
) (remoteSourceConfig, error) {
remoteURL, err := resolveUpdatedRemoteURL(existing, hasExisting, input)
if err != nil {
return remoteSourceConfig{}, err
}
parsedURL, err := parseRemoteSourceURL(remoteURL)
if err != nil {
return remoteSourceConfig{}, err
}
policy := strings.TrimSpace(input.RemoteNetworkPolicy)
if policy == "" {
policy = RemoteNetworkPolicyPublic
}
return remoteSourceConfig{
URL: remoteURL,
Policy: policy,
Identity: remoteSourceIdentity(parsedURL),
}, nil
}
func createRemoteSourceTx(tx *gorm.DB, projectID uint, config remoteSourceConfig) error {
source := &model.PagesProjectSource{
ProjectID: projectID,
SourceType: PagesSourceTypeRemoteURL,
RemoteURL: config.URL,
RemoteNetworkPolicy: config.Policy,
AutoUpdateEnabled: false,
CheckIntervalMinutes: 0,
ConfigVersion: 1,
SourceIdentity: config.Identity,
}
if err := tx.Create(source).Error; err != nil {
return err
}
return tx.Create(&model.PagesProjectSourceRuntime{
SourceID: source.ID,
SyncStatus: pagesSourceStatusIdle,
}).Error
}
func updateExistingRemoteSourceTx(
tx *gorm.DB,
existing *model.PagesProjectSource,
config remoteSourceConfig,
) error {
if !remoteSourceConfigChanged(existing, config) {
return nil
}
var runtime model.PagesProjectSourceRuntime
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", existing.ID).
First(&runtime).Error; err != nil {
return err
}
identityChanged := existing.SourceIdentity != config.Identity
if err := tx.Model(existing).Updates(map[string]any{
"source_type": PagesSourceTypeRemoteURL,
"remote_url": config.URL,
"remote_network_policy": config.Policy,
"github_repository": "",
"release_selector": "",
"release_tag": "",
"asset_name": "",
"auto_update_enabled": false,
"check_interval_minutes": 0,
"config_version": existing.ConfigVersion + 1,
"source_identity": config.Identity,
}).Error; err != nil {
return err
}
return resetRuntimeAfterSourceUpdate(tx, &runtime, identityChanged)
}
func remoteSourceConfigChanged(existing *model.PagesProjectSource, config remoteSourceConfig) bool {
return existing.SourceType != PagesSourceTypeRemoteURL ||
existing.RemoteURL != config.URL ||
existing.RemoteNetworkPolicy != config.Policy ||
existing.GitHubRepository != "" ||
existing.ReleaseSelector != "" ||
existing.ReleaseTag != "" ||
existing.AssetName != "" ||
existing.AutoUpdateEnabled ||
existing.CheckIntervalMinutes != 0
}
// DeleteSource idempotently switches a project back to manual mode.
func DeleteSource(ctx context.Context, projectID uint) (*SourceView, error) {
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
var project model.PagesProject
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil {
return err
}
var source model.PagesProjectSource
err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("project_id = ?", projectID).
First(&source).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
if err != nil {
return err
}
var runtime model.PagesProjectSourceRuntime
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", source.ID).
First(&runtime).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
if err := tx.Where("source_id = ?", source.ID).Delete(&model.PagesProjectSourceRuntime{}).Error; err != nil {
return err
}
return tx.Delete(&source).Error
})
if err != nil {
return nil, err
}
return &SourceView{SourceType: PagesSourceTypeManual}, nil
}
func validateRemoteSourceInput(input SourceUpdateInput) error {
sourceType := strings.TrimSpace(input.SourceType)
if sourceType == "" {
return errors.New(errPagesSourceTypeRequired)
}
if sourceType != PagesSourceTypeRemoteURL {
return errors.New(errPagesSourceTypeUnsupported)
}
if strings.TrimSpace(input.RepositoryURL) != "" || strings.TrimSpace(input.ReleaseSelector) != "" ||
strings.TrimSpace(input.ReleaseTag) != "" || strings.TrimSpace(input.AssetName) != "" ||
input.AutoUpdateEnabled || input.CheckIntervalMinutes != 0 {
return errors.New(errPagesSourceRemoteFields)
}
policy := strings.TrimSpace(input.RemoteNetworkPolicy)
if policy != "" && policy != RemoteNetworkPolicyPublic && policy != RemoteNetworkPolicyTrustedInternal {
return errors.New(errPagesSourceNetworkPolicy)
}
if !input.RemoteURLSet && strings.TrimSpace(input.RemoteURL) != "" {
return errors.New(errPagesSourceRemoteURLMode)
}
if input.RemoteURLSet && strings.TrimSpace(input.RemoteURL) == "" {
return errors.New(errPagesSourceRemoteURLRequired)
}
return nil
}
func resolveUpdatedRemoteURL(existing *model.PagesProjectSource, hasExisting bool, input SourceUpdateInput) (string, error) {
if input.RemoteURLSet {
return strings.TrimSpace(input.RemoteURL), nil
}
if !hasExisting || existing.SourceType != PagesSourceTypeRemoteURL || strings.TrimSpace(existing.RemoteURL) == "" {
return "", errors.New(errPagesSourceRemoteURLRequired)
}
return existing.RemoteURL, nil
}
func parseRemoteSourceURL(raw string) (*url.URL, error) {
parsed, err := url.Parse(strings.TrimSpace(raw))
if err != nil || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
return nil, errors.New(errPagesSourceRemoteURLInvalid)
}
parsed.Scheme = strings.ToLower(parsed.Scheme)
if parsed.Scheme != remoteSourceSchemeHTTP && parsed.Scheme != remoteSourceSchemeHTTPS {
return nil, errors.New(errPagesSourceRemoteURLInvalid)
}
if strings.TrimSpace(parsed.Hostname()) == "" {
return nil, errors.New(errPagesSourceRemoteURLInvalid)
}
return parsed, nil
}
func remoteSourceIdentity(parsed *url.URL) string {
hostname := strings.ToLower(parsed.Hostname())
port := parsed.Port()
if (parsed.Scheme == "https" && port == "443") || (parsed.Scheme == "http" && port == "80") {
port = ""
}
host := hostname
if port != "" {
host = net.JoinHostPort(hostname, port)
} else if strings.Contains(hostname, ":") {
host = "[" + hostname + "]"
}
canonicalPath := parsed.EscapedPath()
if canonicalPath == "" {
canonicalPath = "/"
}
canonicalPath = path.Clean("/" + strings.TrimPrefix(canonicalPath, "/"))
canonical := parsed.Scheme + "://" + host + canonicalPath
sum := sha256.Sum256([]byte("remote_url|" + canonical))
return hex.EncodeToString(sum[:])
}
func displayRemoteSourceURL(raw string) string {
parsed, err := parseRemoteSourceURL(raw)
if err != nil {
return ""
}
hadQuery := parsed.RawQuery != ""
parsed.RawQuery = ""
display := parsed.String()
if hadQuery {
display += "?***"
}
return display
}
func loadSourceByProject(ctx context.Context, projectID uint) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime, error) {
var source model.PagesProjectSource
if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil {
return nil, nil, err
}
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
return nil, nil, err
}
return &source, &runtime, nil
}
func buildSourceView(source *model.PagesProjectSource, runtime *model.PagesProjectSourceRuntime) (*SourceView, error) {
if source == nil || runtime == nil {
return nil, errors.New(errPagesSourceNotFound)
}
view := &SourceView{
SourceType: source.SourceType,
SyncStatus: runtime.SyncStatus,
LastSyncedAt: runtime.LastSyncedAt,
LastError: runtime.LastError,
}
if runtime.LastAppliedRevision != "" {
view.LastApplied = revisionView(runtime.LastAppliedRevision, runtime.LastAppliedDetail)
}
switch source.SourceType {
case PagesSourceTypeRemoteURL:
view.HasRemoteURL = strings.TrimSpace(source.RemoteURL) != ""
view.DisplayURL = displayRemoteSourceURL(source.RemoteURL)
view.RemoteNetworkPolicy = source.RemoteNetworkPolicy
case PagesSourceTypeGitHubRelease:
view.LastCheckedAt = runtime.LastCheckedAt
view.NextCheckAt = runtime.NextCheckAt
view.UpdateAvailable = runtime.LastSeenRevision != "" && runtime.LastSeenRevision != runtime.LastAppliedRevision
if runtime.LastSeenRevision != "" {
view.LastSeen = revisionView(runtime.LastSeenRevision, runtime.LastSeenDetail)
}
view.GitHubRepository = source.GitHubRepository
view.ReleaseSelector = source.ReleaseSelector
view.ReleaseTag = source.ReleaseTag
view.AssetName = source.AssetName
view.AutoUpdateEnabled = source.AutoUpdateEnabled
view.CheckIntervalMinutes = source.CheckIntervalMinutes
default:
return nil, errors.New(errPagesSourceTypeUnsupported)
}
return view, nil
}
func revisionView(revision string, detailJSON string) *SourceRevisionView {
detail := sourceDetail{}
_ = unmarshalSourceDetail(detailJSON, &detail)
label := strings.TrimSpace(detail.Label)
if label == "" {
label = defaultRemoteAssetLabel
}
return &SourceRevisionView{
Revision: revision,
Label: label,
AssetName: detail.AssetName,
}
}
func unmarshalSourceDetail(raw string, detail *sourceDetail) error {
if detail == nil || strings.TrimSpace(raw) == "" {
return nil
}
return json.Unmarshal([]byte(raw), detail)
}
func resetRuntimeAfterSourceUpdate(tx *gorm.DB, runtime *model.PagesProjectSourceRuntime, identityChanged bool) error {
updates := map[string]any{
sourceRuntimeColumnLeaseToken: "",
sourceRuntimeColumnLeaseExpiresAt: nil,
sourceRuntimeColumnLastError: "",
}
if identityChanged {
updates["etag"] = ""
updates["last_seen_revision"] = ""
updates["last_seen_detail"] = ""
updates["last_applied_revision"] = ""
updates["last_applied_detail"] = ""
updates["last_checked_at"] = nil
updates["last_synced_at"] = nil
updates["next_check_at"] = nil
updates[sourceRuntimeColumnSyncStatus] = pagesSourceStatusIdle
} else {
updates[sourceRuntimeColumnSyncStatus] = normalizedSourceRuntimeStatus(runtime)
}
return tx.Model(runtime).Updates(updates).Error
}
func normalizedSourceRuntimeStatus(runtime *model.PagesProjectSourceRuntime) string {
if runtime == nil {
return pagesSourceStatusIdle
}
if sourceHasSameReleaseReplacement(runtime) {
return pagesSourceStatusAttention
}
if runtime.LastSeenRevision != "" && runtime.LastSeenRevision != runtime.LastAppliedRevision {
return pagesSourceStatusUpdateAvailable
}
return pagesSourceStatusIdle
}
internal/apps/openflare/pages/source_provider.go
+549
View File
@@ -0,0 +1,549 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"context"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"errors"
"fmt"
"io"
"math"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/Rain-kl/Wavelet/pkg/httppool"
"github.com/Rain-kl/Wavelet/pkg/pagesarchive"
)
const (
// RemoteNetworkPolicyPublic only permits publicly routable targets and
// performs DNS validation again for every connection.
RemoteNetworkPolicyPublic = "public"
// RemoteNetworkPolicyTrustedInternal permits private targets and self-signed
// TLS certificates. It is an explicit administrator trust boundary.
RemoteNetworkPolicyTrustedInternal = "trusted_internal"
remoteSourceDownloadTimeout = 10 * time.Minute
remoteSourceResponseHeaderTimeout = 30 * time.Second
remoteSourceDialTimeout = 30 * time.Second
remoteSourceDialKeepAlive = 30 * time.Second
remoteSourceMaxRedirects = 5
remoteSourceMagicSniffBytes = 512
remoteSourceMaxSafeLabelBytes = 255
remoteSourceFallbackLabel = "package"
remoteSourceUserAgent = "OpenFlare Pages Source/2"
remoteSourceSchemeHTTP = "http"
remoteSourceSchemeHTTPS = "https"
)
type remoteProviderError string
func (providerError remoteProviderError) Error() string {
return string(providerError)
}
const (
errRemoteProviderInvalidPolicy remoteProviderError = "远程来源网络策略无效"
errRemoteProviderInvalidLimit remoteProviderError = "远程来源部署包大小限制无效"
errRemoteProviderBlockedAddress remoteProviderError = "远程来源 public 策略禁止访问非公网地址"
errRemoteProviderResolveFailed remoteProviderError = "远程来源地址解析失败"
errRemoteProviderRedirectLimit remoteProviderError = "远程来源重定向次数超过限制"
errRemoteProviderDownloadFailed remoteProviderError = errPagesPackageURLDownloadFailed
errRemoteProviderTooLarge remoteProviderError = errPagesPackageURLTooLarge
errRemoteProviderEmpty remoteProviderError = errPagesPackageEmpty
errRemoteProviderUnsupported remoteProviderError = errPagesPackageUnsupported
errRemoteProviderCleanupFailed remoteProviderError = "清理远程来源临时文件失败"
)
var remoteSourceNonPublicPrefixes = []netip.Prefix{
// IPv4 special-use, private, link-local, documentation, multicast and
// reserved ranges. A conservative deny list is intentional for SSRF safety.
netip.MustParsePrefix("0.0.0.0/8"),
netip.MustParsePrefix("10.0.0.0/8"),
netip.MustParsePrefix("100.64.0.0/10"),
netip.MustParsePrefix("127.0.0.0/8"),
netip.MustParsePrefix("169.254.0.0/16"),
netip.MustParsePrefix("172.16.0.0/12"),
netip.MustParsePrefix("192.0.0.0/24"),
netip.MustParsePrefix("192.0.2.0/24"),
netip.MustParsePrefix("192.88.99.0/24"),
netip.MustParsePrefix("192.168.0.0/16"),
netip.MustParsePrefix("198.18.0.0/15"),
netip.MustParsePrefix("198.51.100.0/24"),
netip.MustParsePrefix("203.0.113.0/24"),
netip.MustParsePrefix("224.0.0.0/4"),
netip.MustParsePrefix("240.0.0.0/4"),
// IPv6 protocol-assignment, documentation and transition ranges that are
// not acceptable as direct public artifact origins.
netip.MustParsePrefix("2001::/23"),
netip.MustParsePrefix("2001:db8::/32"),
netip.MustParsePrefix("2002::/16"),
netip.MustParsePrefix("3fff::/20"),
}
var remoteSourcePublicIPv6Prefix = netip.MustParsePrefix("2000::/3")
// RemoteSourceRequest describes one immutable Remote URL package fetch.
type RemoteSourceRequest struct {
URL string
NetworkPolicy string
MaxPackageBytes int64
}
// SourceCandidate is a constrained, immutable archive downloaded to a
// provider-owned temporary file. The caller owns the file after a successful
// fetch and must call Cleanup when processing finishes.
type SourceCandidate struct {
TempPath string
Checksum string
PackageSize int64
Format pagesarchive.Format
SafeLabel string
}
// Cleanup removes the candidate temporary file. It is safe to call repeatedly.
func (candidate *SourceCandidate) Cleanup() error {
if candidate == nil || candidate.TempPath == "" {
return nil
}
tempPath := candidate.TempPath
err := os.Remove(tempPath)
if err == nil || errors.Is(err, os.ErrNotExist) {
candidate.TempPath = ""
return nil
}
return errRemoteProviderCleanupFailed
}
type remoteSourceResolver interface {
LookupNetIP(context.Context, string, string) ([]netip.Addr, error)
}
type remoteSourceDependencies struct {
resolver remoteSourceResolver
dialContext func(context.Context, string, string) (net.Conn, error)
createTemp func(string, string) (*os.File, error)
}
// FetchRemoteSource downloads a Remote URL package without writing deployment
// state. Errors are reduced to safe domain messages and never contain the raw
// URL, query, response headers or response body.
func FetchRemoteSource(ctx context.Context, request RemoteSourceRequest) (*SourceCandidate, error) {
dialer := &net.Dialer{
Timeout: remoteSourceDialTimeout,
KeepAlive: remoteSourceDialKeepAlive,
}
dependencies := remoteSourceDependencies{
resolver: net.DefaultResolver,
dialContext: dialer.DialContext,
createTemp: os.CreateTemp,
}
return fetchRemoteSource(ctx, request, dependencies)
}
func fetchRemoteSource(ctx context.Context, request RemoteSourceRequest, dependencies remoteSourceDependencies) (*SourceCandidate, error) {
if request.MaxPackageBytes <= 0 {
return nil, errRemoteProviderInvalidLimit
}
if dependencies.dialContext == nil || dependencies.createTemp == nil {
return nil, errRemoteProviderDownloadFailed
}
policy, err := normalizeRemoteNetworkPolicy(request.NetworkPolicy)
if err != nil {
return nil, err
}
parsed, err := parseRemoteSourceURL(request.URL)
if err != nil {
return nil, err
}
if err := validateRemoteSourceTarget(ctx, parsed, policy, dependencies.resolver); err != nil {
return nil, sanitizeRemoteProviderError(ctx, err)
}
safeLabel, namedFormat := remoteSourceLabel(parsed)
client := newRemoteSourceClient(policy, dependencies)
defer client.CloseIdleConnections()
response, err := requestRemoteSource(ctx, client, parsed)
if err != nil {
return nil, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
return nil, fmt.Errorf("%w: HTTP %d", errRemoteProviderDownloadFailed, response.StatusCode)
}
if response.ContentLength > request.MaxPackageBytes {
return nil, errRemoteProviderTooLarge
}
tempPath, checksum, packageSize, err := streamRemoteSourcePackage(
response.Body,
request.MaxPackageBytes,
dependencies.createTemp,
)
if err != nil {
return nil, sanitizeRemoteProviderError(ctx, err)
}
format, safeLabel, err := detectRemoteSourceFormat(tempPath, safeLabel, namedFormat)
if err != nil {
if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
return nil, errRemoteProviderCleanupFailed
}
return nil, err
}
return &SourceCandidate{
TempPath: tempPath,
Checksum: checksum,
PackageSize: packageSize,
Format: format,
SafeLabel: safeLabel,
}, nil
}
func normalizeRemoteNetworkPolicy(policy string) (string, error) {
switch strings.TrimSpace(policy) {
case "", RemoteNetworkPolicyPublic:
return RemoteNetworkPolicyPublic, nil
case RemoteNetworkPolicyTrustedInternal:
return RemoteNetworkPolicyTrustedInternal, nil
default:
return "", errRemoteProviderInvalidPolicy
}
}
func newRemoteSourceClient(policy string, dependencies remoteSourceDependencies) *http.Client {
tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
dialContext := dependencies.dialContext
if policy == RemoteNetworkPolicyPublic {
dialContext = newPublicRemoteSourceDialer(dependencies.resolver, dependencies.dialContext)
} else {
// trusted_internal is an explicit administrator-selected boundary for
// private artifact services using an internal CA or self-signed cert.
tlsConfig.InsecureSkipVerify = true //nolint:gosec // required trusted_internal semantics
}
client := &http.Client{
Timeout: remoteSourceDownloadTimeout,
Transport: httppool.NewTransport(httppool.TransportOptions{
Proxy: nil,
DialContext: dialContext,
TLSClientConfig: tlsConfig,
ResponseHeaderTimeout: remoteSourceResponseHeaderTimeout,
TraceFilter: remoteSourceTraceFilter,
}),
}
client.CheckRedirect = func(next *http.Request, previous []*http.Request) error {
if len(previous) > remoteSourceMaxRedirects {
return errRemoteProviderRedirectLimit
}
stripRemoteSourceRedirectHeaders(next)
if err := validateRemoteSourceTarget(next.Context(), next.URL, policy, dependencies.resolver); err != nil {
return err
}
applyRemoteSourceHeaders(next)
return nil
}
return client
}
func requestRemoteSource(ctx context.Context, client *http.Client, parsed *url.URL) (*http.Response, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return nil, errors.New(errPagesSourceRemoteURLInvalid)
}
applyRemoteSourceHeaders(request)
response, err := client.Do(request) //nolint:gosec // scheme and every dial target are validated above
if err == nil {
return response, nil
}
if response != nil && response.Body != nil {
_ = response.Body.Close()
}
return nil, sanitizeRemoteProviderError(ctx, err)
}
func applyRemoteSourceHeaders(request *http.Request) {
request.Header.Set("User-Agent", remoteSourceUserAgent)
request.Header.Set("Accept", "application/octet-stream,application/zip,application/x-tar,application/gzip,*/*;q=0.1")
// Preserve the artifact bytes exactly as stored. Automatic HTTP gzip
// decompression would change the checksum, size and archive format.
request.Header.Set("Accept-Encoding", "identity")
}
func stripRemoteSourceRedirectHeaders(request *http.Request) {
request.Header.Del("Authorization")
request.Header.Del("Cookie")
request.Header.Del("Proxy-Authorization")
request.Header.Del("Referer")
}
func remoteSourceTraceFilter(request *http.Request) bool {
// otelhttp records url.full. Signed query strings must never enter traces.
return request.URL == nil || request.URL.RawQuery == ""
}
func validateRemoteSourceTarget(ctx context.Context, target *url.URL, policy string, resolver remoteSourceResolver) error {
if target == nil || target.User != nil || target.Fragment != "" || target.Opaque != "" {
return errors.New(errPagesSourceRemoteURLInvalid)
}
scheme := strings.ToLower(strings.TrimSpace(target.Scheme))
if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(target.Hostname()) == "" {
return errors.New(errPagesSourceRemoteURLInvalid)
}
if policy != RemoteNetworkPolicyPublic {
return nil
}
_, err := resolvePublicRemoteSourceIPs(ctx, resolver, target.Hostname())
return err
}
func newPublicRemoteSourceDialer(
resolver remoteSourceResolver,
directDial func(context.Context, string, string) (net.Conn, error),
) func(context.Context, string, string) (net.Conn, error) {
return func(ctx context.Context, network string, address string) (net.Conn, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return nil, errRemoteProviderDownloadFailed
}
addresses, err := resolvePublicRemoteSourceIPs(ctx, resolver, host)
if err != nil {
return nil, err
}
for _, address := range addresses {
if !remoteSourceIPMatchesNetwork(address, network) {
continue
}
connection, dialErr := directDial(ctx, network, net.JoinHostPort(address.String(), port))
if dialErr == nil {
return connection, nil
}
}
return nil, errRemoteProviderDownloadFailed
}
}
func resolvePublicRemoteSourceIPs(ctx context.Context, resolver remoteSourceResolver, host string) ([]netip.Addr, error) {
if strings.Contains(host, "%") {
return nil, errRemoteProviderBlockedAddress
}
if literal, parseErr := netip.ParseAddr(host); parseErr == nil {
if !isPublicRemoteSourceIP(literal) {
return nil, errRemoteProviderBlockedAddress
}
return []netip.Addr{literal}, nil
}
if resolver == nil {
return nil, errRemoteProviderResolveFailed
}
addresses, err := resolver.LookupNetIP(ctx, "ip", host)
if err != nil || len(addresses) == 0 {
return nil, errRemoteProviderResolveFailed
}
for _, address := range addresses {
if !isPublicRemoteSourceIP(address) {
return nil, errRemoteProviderBlockedAddress
}
}
return addresses, nil
}
func isPublicRemoteSourceIP(address netip.Addr) bool {
if !address.IsValid() || address.Zone() != "" {
return false
}
address = address.Unmap()
if !address.IsGlobalUnicast() {
return false
}
if address.Is6() && !remoteSourcePublicIPv6Prefix.Contains(address) {
return false
}
for _, prefix := range remoteSourceNonPublicPrefixes {
if prefix.Contains(address) {
return false
}
}
return true
}
func remoteSourceIPMatchesNetwork(address netip.Addr, network string) bool {
switch network {
case "tcp4":
return address.Unmap().Is4()
case "tcp6":
return address.Unmap().Is6()
default:
return true
}
}
func streamRemoteSourcePackage(
body io.Reader,
maxPackageBytes int64,
createTemp func(string, string) (*os.File, error),
) (tempPath string, checksum string, packageSize int64, err error) {
if createTemp == nil {
return "", "", 0, errRemoteProviderDownloadFailed
}
tempFile, err := createTemp("", "openflare-pages-source-*")
if err != nil {
return "", "", 0, errRemoteProviderDownloadFailed
}
createdTempPath := tempFile.Name()
tempPath = createdTempPath
defer func() {
closeErr := tempFile.Close()
if err == nil && closeErr != nil {
err = errRemoteProviderDownloadFailed
}
if err != nil {
if removeErr := os.Remove(createdTempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
err = errRemoteProviderCleanupFailed
}
}
}()
hasher := sha256.New()
readLimit := maxPackageBytes
if readLimit < math.MaxInt64 {
readLimit++
}
packageSize, err = io.Copy(io.MultiWriter(tempFile, hasher), io.LimitReader(body, readLimit))
if err != nil {
return "", "", 0, errRemoteProviderDownloadFailed
}
if packageSize > maxPackageBytes {
return "", "", 0, errRemoteProviderTooLarge
}
if packageSize == 0 {
return "", "", 0, errRemoteProviderEmpty
}
checksum = hex.EncodeToString(hasher.Sum(nil))
return tempPath, checksum, packageSize, nil
}
func detectRemoteSourceFormat(
tempPath string,
safeLabel string,
namedFormat pagesarchive.Format,
) (pagesarchive.Format, string, error) {
if namedFormat != "" {
return namedFormat, safeLabel, nil
}
tempFile, err := os.Open(tempPath) //nolint:gosec // path is a provider-created temporary file
if err != nil {
return "", safeLabel, errRemoteProviderDownloadFailed
}
defer func() { _ = tempFile.Close() }()
head := make([]byte, remoteSourceMagicSniffBytes)
readBytes, readErr := io.ReadFull(tempFile, head)
if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) {
return "", safeLabel, errRemoteProviderDownloadFailed
}
format, ok := pagesarchive.DetectFormatFromBytes(head[:readBytes])
if !ok {
return "", safeLabel, errRemoteProviderUnsupported
}
return format, appendRemoteSourceLabelExtension(safeLabel, format), nil
}
func remoteSourceLabel(parsed *url.URL) (string, pagesarchive.Format) {
baseName := path.Base(parsed.Path)
if baseName == "" || baseName == "." || baseName == "/" {
baseName = remoteSourceFallbackLabel
}
safeLabel := sanitizeRemoteSourceLabel(baseName)
format, _ := pagesarchive.DetectFormatFromName(safeLabel)
return limitRemoteSourceLabel(safeLabel, format), format
}
func sanitizeRemoteSourceLabel(label string) string {
var builder strings.Builder
lastReplacement := false
for _, character := range label {
if isRemoteSourceLabelCharacter(character) {
builder.WriteRune(character)
lastReplacement = false
continue
}
if !lastReplacement {
builder.WriteByte('-')
lastReplacement = true
}
}
safeLabel := strings.TrimSpace(builder.String())
if safeLabel == "" || strings.Trim(safeLabel, "._-") == "" {
return remoteSourceFallbackLabel
}
return safeLabel
}
func isRemoteSourceLabelCharacter(character rune) bool {
return character >= 'a' && character <= 'z' ||
character >= 'A' && character <= 'Z' ||
character >= '0' && character <= '9' ||
character == '.' || character == '-' || character == '_'
}
func limitRemoteSourceLabel(label string, format pagesarchive.Format) string {
if len(label) <= remoteSourceMaxSafeLabelBytes {
return label
}
if format == "" {
return strings.TrimRight(label[:remoteSourceMaxSafeLabelBytes], ".-_")
}
extension := "." + pagesarchive.Extension(format)
prefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
prefix := strings.TrimRight(label[:prefixLength], ".-_")
if prefix == "" {
prefix = remoteSourceFallbackLabel
}
return prefix + extension
}
func appendRemoteSourceLabelExtension(label string, format pagesarchive.Format) string {
extension := "." + pagesarchive.Extension(format)
maxPrefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
if len(label) > maxPrefixLength {
label = strings.TrimRight(label[:maxPrefixLength], ".-_")
}
if label == "" {
label = remoteSourceFallbackLabel
}
return label + extension
}
func sanitizeRemoteProviderError(ctx context.Context, err error) error {
if ctxErr := ctx.Err(); ctxErr != nil {
return fmt.Errorf("%w: %w", errRemoteProviderDownloadFailed, ctxErr)
}
for _, safeError := range []error{
errRemoteProviderInvalidPolicy,
errRemoteProviderInvalidLimit,
errRemoteProviderBlockedAddress,
errRemoteProviderResolveFailed,
errRemoteProviderRedirectLimit,
errRemoteProviderTooLarge,
errRemoteProviderEmpty,
errRemoteProviderUnsupported,
errRemoteProviderCleanupFailed,
} {
if errors.Is(err, safeError) {
return safeError
}
}
return errRemoteProviderDownloadFailed
}
internal/apps/openflare/pages/source_provider_test.go
+459
View File
@@ -0,0 +1,459 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"log"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"strconv"
"strings"
"sync/atomic"
"testing"
)
type remoteSourceResolverFunc func(context.Context, string, string) ([]netip.Addr, error)
func (function remoteSourceResolverFunc) LookupNetIP(
ctx context.Context,
network string,
host string,
) ([]netip.Addr, error) {
return function(ctx, network, host)
}
func TestFetchRemoteSourceTrustedInternalSelfSignedAndSafeLabel(t *testing.T) {
packageBytes := makeRemoteSourceZIP(t)
server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Query().Get("token") != "source-secret" {
t.Error("signed query did not reach the artifact server")
}
if request.Header.Get("Accept-Encoding") != "identity" {
t.Error("artifact request must disable automatic HTTP decompression")
}
writer.Header().Set("Content-Disposition", `attachment; filename="redirected.tar.gz"`)
_, _ = writer.Write(packageBytes)
}))
server.Config.ErrorLog = log.New(io.Discard, "", 0)
t.Cleanup(server.Close)
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: server.URL + "/original/site.zip?token=source-secret",
NetworkPolicy: RemoteNetworkPolicyTrustedInternal,
MaxPackageBytes: int64(len(packageBytes) + 1),
})
if err != nil {
t.Fatalf("FetchRemoteSource() error = %v", err)
}
if candidate.Format != "zip" {
t.Fatalf("Format = %q, want zip", candidate.Format)
}
if candidate.SafeLabel != "site.zip" {
t.Fatalf("SafeLabel = %q, want original path basename", candidate.SafeLabel)
}
if candidate.PackageSize != int64(len(packageBytes)) {
t.Fatalf("PackageSize = %d, want %d", candidate.PackageSize, len(packageBytes))
}
wantChecksum := sha256.Sum256(packageBytes)
if candidate.Checksum != hex.EncodeToString(wantChecksum[:]) {
t.Fatalf("Checksum = %q, want SHA-256", candidate.Checksum)
}
downloaded, err := os.ReadFile(candidate.TempPath) //nolint:gosec // provider-owned test temp file
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if !bytes.Equal(downloaded, packageBytes) {
t.Fatal("downloaded package differs from response body")
}
tempPath := candidate.TempPath
if err := candidate.Cleanup(); err != nil {
t.Fatalf("Cleanup() error = %v", err)
}
if err := candidate.Cleanup(); err != nil {
t.Fatalf("second Cleanup() error = %v", err)
}
if _, err := os.Stat(tempPath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("temporary file still exists: %v", err)
}
}
func TestFetchRemoteSourceKeepsOriginalLabelAcrossRedirect(t *testing.T) {
packageBytes := makeRemoteSourceZIP(t)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path == "/original/site.zip" {
writer.Header().Set("Location", "/delivery/final.tar.gz?token=redirect-secret")
writer.WriteHeader(http.StatusFound)
return
}
if request.Header.Get("Referer") != "" {
t.Error("redirect must not forward a signed source URL as Referer")
}
writer.Header().Set("Content-Disposition", `attachment; filename="response.7z"`)
_, _ = writer.Write(packageBytes)
}))
t.Cleanup(server.Close)
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: server.URL + "/original/site.zip?token=initial-secret",
NetworkPolicy: RemoteNetworkPolicyTrustedInternal,
MaxPackageBytes: int64(len(packageBytes) + 1),
})
if err != nil {
t.Fatalf("FetchRemoteSource() error = %v", err)
}
defer func() { _ = candidate.Cleanup() }()
if candidate.SafeLabel != "site.zip" || candidate.Format != "zip" {
t.Fatalf("candidate = label %q format %q, want original site.zip", candidate.SafeLabel, candidate.Format)
}
}
func TestFetchRemoteSourcePublicRejectsNonPublicAddresses(t *testing.T) {
tests := []string{
"http://127.0.0.1/site.zip?token=loopback-secret",
"http://[::1]/site.zip?token=ipv6-secret",
"http://100.64.0.1/site.zip?token=cgnat-secret",
"http://192.0.2.1/site.zip?token=documentation-secret",
}
for _, rawURL := range tests {
t.Run(rawURL, func(t *testing.T) {
_, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: rawURL,
NetworkPolicy: RemoteNetworkPolicyPublic,
MaxPackageBytes: 1024,
})
if !errors.Is(err, errRemoteProviderBlockedAddress) {
t.Fatalf("FetchRemoteSource() error = %v, want blocked address", err)
}
assertRemoteSourceErrorRedacted(t, err, rawURL, "secret", "token=")
})
}
}
func TestFetchRemoteSourcePublicDialsValidatedIP(t *testing.T) {
packageBytes := makeRemoteSourceZIP(t)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
_, _ = writer.Write(packageBytes)
}))
t.Cleanup(server.Close)
var dialedAddress string
dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver())
dependencies.dialContext = func(ctx context.Context, network string, address string) (net.Conn, error) {
dialedAddress = address
dialer := &net.Dialer{}
return dialer.DialContext(ctx, network, server.Listener.Addr().String())
}
candidate, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: "http://artifact.example/site.zip",
NetworkPolicy: RemoteNetworkPolicyPublic,
MaxPackageBytes: int64(len(packageBytes) + 1),
}, dependencies)
if err != nil {
t.Fatalf("fetchRemoteSource() error = %v", err)
}
defer func() { _ = candidate.Cleanup() }()
if dialedAddress != "93.184.216.34:80" {
t.Fatalf("direct dial address = %q, want validated IP", dialedAddress)
}
}
func TestFetchRemoteSourcePublicRejectsPrivateRedirect(t *testing.T) {
var requestCount atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
requestCount.Add(1)
writer.Header().Set("Location", "http://127.0.0.1/private.zip?token=redirect-secret")
writer.WriteHeader(http.StatusFound)
}))
t.Cleanup(server.Close)
dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver())
rawURL := "http://artifact.example/start.zip?token=initial-secret"
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: rawURL,
NetworkPolicy: RemoteNetworkPolicyPublic,
MaxPackageBytes: 1024,
}, dependencies)
if !errors.Is(err, errRemoteProviderBlockedAddress) {
t.Fatalf("fetchRemoteSource() error = %v, want blocked redirect", err)
}
if requestCount.Load() != 1 {
t.Fatalf("request count = %d, private redirect must not be requested", requestCount.Load())
}
assertRemoteSourceErrorRedacted(t, err, rawURL, "initial-secret", "redirect-secret", "token=")
}
func TestFetchRemoteSourcePublicRejectsDNSRebinding(t *testing.T) {
var lookupCount atomic.Int32
resolver := remoteSourceResolverFunc(func(context.Context, string, string) ([]netip.Addr, error) {
if lookupCount.Add(1) == 1 {
return []netip.Addr{netip.MustParseAddr("93.184.216.34")}, nil
}
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
})
var dialCount atomic.Int32
dependencies := remoteSourceDependencies{
resolver: resolver,
dialContext: func(context.Context, string, string) (net.Conn, error) {
dialCount.Add(1)
return nil, errors.New("unexpected dial")
},
createTemp: os.CreateTemp,
}
rawURL := "http://rebind.example/site.zip?signature=dns-secret"
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: rawURL,
NetworkPolicy: RemoteNetworkPolicyPublic,
MaxPackageBytes: 1024,
}, dependencies)
if !errors.Is(err, errRemoteProviderBlockedAddress) {
t.Fatalf("fetchRemoteSource() error = %v, want DNS rebinding rejection", err)
}
if lookupCount.Load() != 2 {
t.Fatalf("DNS lookup count = %d, want preflight plus dial validation", lookupCount.Load())
}
if dialCount.Load() != 0 {
t.Fatalf("direct dial count = %d, rebound address must not be dialed", dialCount.Load())
}
assertRemoteSourceErrorRedacted(t, err, rawURL, "dns-secret", "signature=")
}
func TestFetchRemoteSourcePublicRejectsSelfSignedTLS(t *testing.T) {
packageBytes := makeRemoteSourceZIP(t)
server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
_, _ = writer.Write(packageBytes)
}))
server.Config.ErrorLog = log.New(io.Discard, "", 0)
t.Cleanup(server.Close)
dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver())
rawURL := "https://artifact.example/site.zip?signature=tls-secret"
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: rawURL,
NetworkPolicy: RemoteNetworkPolicyPublic,
MaxPackageBytes: int64(len(packageBytes) + 1),
}, dependencies)
if !errors.Is(err, errRemoteProviderDownloadFailed) {
t.Fatalf("fetchRemoteSource() error = %v, want strict TLS failure", err)
}
assertRemoteSourceErrorRedacted(t, err, rawURL, "tls-secret", "signature=")
}
func TestFetchRemoteSourceRejectsChunkedBodyOverLimitAndCleansTemp(t *testing.T) {
const maxPackageBytes = int64(64)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
_, _ = writer.Write(bytes.Repeat([]byte{'x'}, int(maxPackageBytes)))
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
_, _ = writer.Write([]byte("overflow"))
}))
t.Cleanup(server.Close)
tempDir := t.TempDir()
dependencies := defaultRemoteSourceDependenciesForTest()
dependencies.createTemp = func(_ string, pattern string) (*os.File, error) {
return os.CreateTemp(tempDir, pattern)
}
rawURL := server.URL + "/site.zip?token=chunk-secret"
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: rawURL,
NetworkPolicy: RemoteNetworkPolicyTrustedInternal,
MaxPackageBytes: maxPackageBytes,
}, dependencies)
if !errors.Is(err, errRemoteProviderTooLarge) {
t.Fatalf("fetchRemoteSource() error = %v, want actual stream limit", err)
}
assertRemoteSourceTempDirEmpty(t, tempDir)
assertRemoteSourceErrorRedacted(t, err, rawURL, "chunk-secret", "token=")
}
func TestFetchRemoteSourceRejectsContentLengthBeforeCreatingTemp(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
writer.Header().Set("Content-Length", "4096")
writer.WriteHeader(http.StatusOK)
}))
t.Cleanup(server.Close)
var createCount atomic.Int32
dependencies := defaultRemoteSourceDependenciesForTest()
dependencies.createTemp = func(directory string, pattern string) (*os.File, error) {
createCount.Add(1)
return os.CreateTemp(directory, pattern)
}
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: server.URL + "/site.zip",
NetworkPolicy: RemoteNetworkPolicyTrustedInternal,
MaxPackageBytes: 1024,
}, dependencies)
if !errors.Is(err, errRemoteProviderTooLarge) {
t.Fatalf("fetchRemoteSource() error = %v, want Content-Length rejection", err)
}
if createCount.Load() != 0 {
t.Fatalf("CreateTemp called %d times before Content-Length rejection", createCount.Load())
}
}
func TestFetchRemoteSourceSniffsAtLeast512BytesForTar(t *testing.T) {
packageBytes := make([]byte, remoteSourceMagicSniffBytes)
copy(packageBytes[257:], []byte("ustar"))
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
_, _ = writer.Write(packageBytes)
}))
t.Cleanup(server.Close)
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: server.URL + "/download",
NetworkPolicy: RemoteNetworkPolicyTrustedInternal,
MaxPackageBytes: int64(len(packageBytes) + 1),
})
if err != nil {
t.Fatalf("FetchRemoteSource() error = %v", err)
}
defer func() { _ = candidate.Cleanup() }()
if candidate.Format != "tar" {
t.Fatalf("Format = %q, want tar detected at byte 257", candidate.Format)
}
if candidate.SafeLabel != "download.tar" {
t.Fatalf("SafeLabel = %q, want download.tar", candidate.SafeLabel)
}
}
func TestFetchRemoteSourceRedactsURLHeadersAndBodyFromErrors(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
writer.Header().Set("X-Artifact-Secret", "header-secret")
_, _ = writer.Write([]byte("response-body-secret"))
}))
t.Cleanup(server.Close)
tempDir := t.TempDir()
dependencies := defaultRemoteSourceDependenciesForTest()
dependencies.createTemp = func(_ string, pattern string) (*os.File, error) {
return os.CreateTemp(tempDir, pattern)
}
rawURL := server.URL + "/download?token=query-secret"
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: rawURL,
NetworkPolicy: RemoteNetworkPolicyTrustedInternal,
MaxPackageBytes: 1024,
}, dependencies)
if !errors.Is(err, errRemoteProviderUnsupported) {
t.Fatalf("fetchRemoteSource() error = %v, want unsupported archive", err)
}
assertRemoteSourceTempDirEmpty(t, tempDir)
assertRemoteSourceErrorRedacted(
t,
err,
rawURL,
"query-secret",
"header-secret",
"response-body-secret",
"token=",
)
}
func TestFetchRemoteSourceAllowsFiveRedirectsOnly(t *testing.T) {
packageBytes := makeRemoteSourceZIP(t)
var requestCount atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
requestCount.Add(1)
redirectNumber, _ := strconv.Atoi(strings.TrimPrefix(request.URL.Path, "/"))
if redirectNumber < remoteSourceMaxRedirects+1 {
writer.Header().Set("Location", "/"+strconv.Itoa(redirectNumber+1))
writer.WriteHeader(http.StatusFound)
return
}
_, _ = writer.Write(packageBytes)
}))
t.Cleanup(server.Close)
dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver())
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
URL: "http://artifact.example/0",
NetworkPolicy: RemoteNetworkPolicyPublic,
MaxPackageBytes: int64(len(packageBytes) + 1),
}, dependencies)
if !errors.Is(err, errRemoteProviderRedirectLimit) {
t.Fatalf("fetchRemoteSource() error = %v, want redirect limit", err)
}
if requestCount.Load() != remoteSourceMaxRedirects+1 {
t.Fatalf("request count = %d, want initial plus five redirects", requestCount.Load())
}
}
func staticPublicRemoteSourceResolver() remoteSourceResolver {
return remoteSourceResolverFunc(func(context.Context, string, string) ([]netip.Addr, error) {
return []netip.Addr{netip.MustParseAddr("93.184.216.34")}, nil
})
}
func defaultRemoteSourceDependenciesForTest() remoteSourceDependencies {
dialer := &net.Dialer{}
return remoteSourceDependencies{
resolver: net.DefaultResolver,
dialContext: dialer.DialContext,
createTemp: os.CreateTemp,
}
}
func mappedRemoteSourceDependencies(targetAddress string, resolver remoteSourceResolver) remoteSourceDependencies {
dialer := &net.Dialer{}
return remoteSourceDependencies{
resolver: resolver,
dialContext: func(ctx context.Context, network string, _ string) (net.Conn, error) {
return dialer.DialContext(ctx, network, targetAddress)
},
createTemp: os.CreateTemp,
}
}
func makeRemoteSourceZIP(t *testing.T) []byte {
t.Helper()
var buffer bytes.Buffer
archive := zip.NewWriter(&buffer)
file, err := archive.Create("index.html")
if err != nil {
t.Fatalf("zip.Create() error = %v", err)
}
if _, err := file.Write([]byte("<h1>OpenFlare</h1>")); err != nil {
t.Fatalf("zip entry Write() error = %v", err)
}
if err := archive.Close(); err != nil {
t.Fatalf("zip.Close() error = %v", err)
}
return buffer.Bytes()
}
func assertRemoteSourceTempDirEmpty(t *testing.T, directory string) {
t.Helper()
entries, err := os.ReadDir(directory)
if err != nil {
t.Fatalf("ReadDir() error = %v", err)
}
if len(entries) != 0 {
t.Fatalf("temporary directory contains %d leaked files", len(entries))
}
}
func assertRemoteSourceErrorRedacted(t *testing.T, err error, sensitiveValues ...string) {
t.Helper()
if err == nil {
t.Fatal("expected an error")
}
message := err.Error()
for _, sensitiveValue := range sensitiveValues {
if sensitiveValue != "" && strings.Contains(message, sensitiveValue) {
t.Fatalf("error %q contains sensitive value %q", message, sensitiveValue)
}
}
}
internal/apps/openflare/pages/source_runtime.go
+309
View File
@@ -0,0 +1,309 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const (
pagesSourceCheckLeaseDuration = 2 * time.Minute
pagesSourceSyncLeaseDuration = 15 * time.Minute
sourceLeaseTokenBytes = 32
sourceRuntimeErrorMaxBytes = 512
sourceRevisionHexLength = 64
sourceRuntimeColumnSyncStatus = "sync_status"
sourceRuntimeColumnLastError = "last_error"
sourceRuntimeColumnLeaseToken = "lease_token"
sourceRuntimeColumnLeaseExpiresAt = "lease_expires_at"
)
type sourceLeaseOutcome string
const (
sourceLeaseAcquired sourceLeaseOutcome = "acquired"
sourceLeaseBusy sourceLeaseOutcome = "busy"
sourceLeaseStale sourceLeaseOutcome = "stale"
)
// sourceExecutionSnapshot captures every mutable value that can affect archive
// validation or the atomic activation decision. The queued payload deliberately
// does not carry project content configuration.
type sourceExecutionSnapshot struct {
ProjectID uint
SourceID uint
SourceConfigVersion int
ContentConfigVersion int
SourceType string
SourceIdentity string
RemoteURL string
RemoteNetworkPolicy string
RootDir string
EntryFile string
LeaseToken string
LeaseExpiresAt time.Time
}
func acquireSourceLease(
ctx context.Context,
sourceID uint,
expectedConfigVersion int,
action string,
) (*sourceExecutionSnapshot, sourceLeaseOutcome, error) {
leaseDuration, status, err := sourceLeaseParameters(action)
if err != nil {
return nil, sourceLeaseStale, err
}
token, err := newSourceLeaseToken()
if err != nil {
return nil, sourceLeaseStale, err
}
now := time.Now()
expiresAt := now.Add(leaseDuration)
result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", sourceID).
Where("lease_expires_at IS NULL OR lease_expires_at <= ?", now).
Where(
"EXISTS (SELECT 1 FROM of_pages_project_sources source WHERE source.id = ? AND source.config_version = ?)",
sourceID,
expectedConfigVersion,
).
Updates(map[string]any{
sourceRuntimeColumnLeaseToken: token,
sourceRuntimeColumnLeaseExpiresAt: expiresAt,
sourceRuntimeColumnSyncStatus: status,
sourceRuntimeColumnLastError: "",
})
if result.Error != nil {
return nil, sourceLeaseStale, result.Error
}
if result.RowsAffected == 0 {
outcome, inspectErr := inspectSourceLeaseMiss(ctx, sourceID, expectedConfigVersion, now)
return nil, outcome, inspectErr
}
snapshot, err := loadSourceExecutionSnapshot(ctx, sourceID, token)
if err != nil {
if errors.Is(err, errSourceLeaseSnapshotStale) {
return nil, sourceLeaseStale, nil
}
return nil, sourceLeaseStale, err
}
return snapshot, sourceLeaseAcquired, nil
}
var errSourceLeaseSnapshotStale = errors.New("source lease snapshot stale")
func loadSourceExecutionSnapshot(
ctx context.Context,
sourceID uint,
token string,
) (*sourceExecutionSnapshot, error) {
var snapshot sourceExecutionSnapshot
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
var source model.PagesProjectSource
if err := tx.Where("id = ?", sourceID).First(&source).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errSourceLeaseSnapshotStale
}
return err
}
var project model.PagesProject
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
First(&project, source.ProjectID).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errSourceLeaseSnapshotStale
}
return err
}
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id = ?", sourceID).
First(&source).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errSourceLeaseSnapshotStale
}
return err
}
var runtime model.PagesProjectSourceRuntime
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", source.ID).
First(&runtime).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errSourceLeaseSnapshotStale
}
return err
}
// 必须在 runtime 行锁拿到后重新取时间，避免锁等待跨过
// lease expiry 时仍使用事务开始前的旧时间继续执行。
now := time.Now()
if runtime.LeaseToken != token || runtime.LeaseExpiresAt == nil || !runtime.LeaseExpiresAt.After(now) {
return errSourceLeaseSnapshotStale
}
snapshot = sourceExecutionSnapshot{
ProjectID: project.ID,
SourceID: source.ID,
SourceConfigVersion: source.ConfigVersion,
ContentConfigVersion: project.ContentConfigVersion,
SourceType: source.SourceType,
SourceIdentity: source.SourceIdentity,
RemoteURL: source.RemoteURL,
RemoteNetworkPolicy: source.RemoteNetworkPolicy,
RootDir: project.RootDir,
EntryFile: project.EntryFile,
LeaseToken: token,
LeaseExpiresAt: *runtime.LeaseExpiresAt,
}
return nil
})
if err != nil {
return nil, err
}
return &snapshot, nil
}
func inspectSourceLeaseMiss(
ctx context.Context,
sourceID uint,
expectedConfigVersion int,
now time.Time,
) (sourceLeaseOutcome, error) {
var source model.PagesProjectSource
if err := db.DB(ctx).Where("id = ?", sourceID).First(&source).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return sourceLeaseStale, nil
}
return sourceLeaseStale, err
}
if source.ConfigVersion != expectedConfigVersion {
return sourceLeaseStale, nil
}
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return sourceLeaseStale, nil
}
return sourceLeaseStale, err
}
if runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(now) {
return sourceLeaseBusy, nil
}
return sourceLeaseStale, nil
}
func sourceLeaseParameters(action string) (time.Duration, string, error) {
switch action {
case sourceActionCheck:
return pagesSourceCheckLeaseDuration, pagesSourceStatusChecking, nil
case sourceActionSync:
return pagesSourceSyncLeaseDuration, pagesSourceStatusSyncing, nil
default:
return 0, "", errors.New(errPagesSourceActionInvalid)
}
}
func newSourceLeaseToken() (string, error) {
value := make([]byte, sourceLeaseTokenBytes)
if _, err := rand.Read(value); err != nil {
return "", err
}
return hex.EncodeToString(value), nil
}
func renewSourceLease(ctx context.Context, snapshot *sourceExecutionSnapshot, duration time.Duration) (bool, error) {
if snapshot == nil || snapshot.SourceID == 0 || snapshot.LeaseToken == "" || duration <= 0 {
return false, errors.New(errPagesSourceLeaseLost)
}
now := time.Now()
expiresAt := now.Add(duration)
result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?", snapshot.SourceID, snapshot.LeaseToken, now).
Updates(map[string]any{sourceRuntimeColumnLeaseExpiresAt: expiresAt})
if result.Error != nil {
return false, result.Error
}
if result.RowsAffected == 0 {
return false, nil
}
snapshot.LeaseExpiresAt = expiresAt
return true, nil
}
func failSourceLease(ctx context.Context, snapshot *sourceExecutionSnapshot, message string) error {
if snapshot == nil || snapshot.SourceID == 0 || snapshot.LeaseToken == "" {
return nil
}
message = safeSourceRuntimeError(message)
now := time.Now()
return db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?", snapshot.SourceID, snapshot.LeaseToken, now).
Updates(map[string]any{
sourceRuntimeColumnSyncStatus: pagesSourceStatusFailed,
sourceRuntimeColumnLastError: message,
sourceRuntimeColumnLeaseToken: "",
sourceRuntimeColumnLeaseExpiresAt: nil,
}).Error
}
func safeSourceRuntimeError(message string) string {
message = strings.TrimSpace(message)
if message == "" {
return errPagesSourceSyncFailed
}
if len(message) > sourceRuntimeErrorMaxBytes {
message = message[:sourceRuntimeErrorMaxBytes]
}
return message
}
func sourceLeaseIsBusy(ctx context.Context, sourceID uint) (bool, error) {
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil {
return false, err
}
return runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(time.Now()), nil
}
// fenceAndNormalizeRuntime invalidates in-flight work while preserving safe
// seen/applied cursors. The caller must already hold the source row lock.
func fenceAndNormalizeRuntime(tx *gorm.DB, sourceID uint) error {
var runtime model.PagesProjectSourceRuntime
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", sourceID).
First(&runtime).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return err
}
return tx.Model(&runtime).Updates(map[string]any{
sourceRuntimeColumnLeaseToken: "",
sourceRuntimeColumnLeaseExpiresAt: nil,
sourceRuntimeColumnSyncStatus: normalizedSourceRuntimeStatus(&runtime),
}).Error
}
func sourceHasSameReleaseReplacement(runtime *model.PagesProjectSourceRuntime) bool {
if runtime == nil || runtime.LastSeenRevision == "" || runtime.LastSeenRevision == runtime.LastAppliedRevision {
return false
}
seen := sourceDetail{}
applied := sourceDetail{}
if unmarshalSourceDetail(runtime.LastSeenDetail, &seen) != nil ||
unmarshalSourceDetail(runtime.LastAppliedDetail, &applied) != nil {
return false
}
return seen.ReleaseID != "" && seen.ReleaseID == applied.ReleaseID
}
internal/apps/openflare/pages/source_runtime_test.go
+350
View File
@@ -0,0 +1,350 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"errors"
"strings"
"sync"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
)
func TestSourceLeaseHeartbeatRenewsAndCancelsOnOwnershipLoss(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "lease-heartbeat")
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip",
RemoteNetworkPolicyPublic,
)
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
if err != nil || outcome != sourceLeaseAcquired || snapshot == nil {
t.Fatalf("acquireSourceLease(heartbeat) = (%+v, %q, %v), want acquired", snapshot, outcome, err)
}
workCtx, heartbeat, err := startSourceLeaseHeartbeat(ctx, snapshot, 500*time.Millisecond, 20*time.Millisecond)
if err != nil {
t.Fatalf("startSourceLeaseHeartbeat() error = %v, want nil", err)
}
t.Cleanup(func() { _ = heartbeat.stop() })
var initial model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&initial).Error; err != nil {
t.Fatalf("load initial heartbeat runtime error = %v, want nil", err)
}
if initial.LeaseExpiresAt == nil {
t.Fatal("initial heartbeat expiry = nil, want non-nil")
}
deadline := time.Now().Add(2 * time.Second)
for {
var renewedRuntime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&renewedRuntime).Error; err != nil {
t.Fatalf("load renewed heartbeat runtime error = %v, want nil", err)
}
if renewedRuntime.LeaseExpiresAt != nil && renewedRuntime.LeaseExpiresAt.After(*initial.LeaseExpiresAt) {
break
}
if time.Now().After(deadline) {
t.Fatal("heartbeat did not extend lease before deadline")
}
time.Sleep(10 * time.Millisecond)
}
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", source.ID).
Update("lease_token", "replacement-owner").Error; err != nil {
t.Fatalf("replace heartbeat lease owner error = %v, want nil", err)
}
select {
case <-workCtx.Done():
case <-time.After(2 * time.Second):
t.Fatal("heartbeat work context was not canceled after ownership loss")
}
if err := heartbeat.stop(); !errors.Is(err, errSourceLeaseHeartbeatLost) {
t.Fatalf("heartbeat.stop() error = %v, want %v", err, errSourceLeaseHeartbeatLost)
}
if err := heartbeat.stop(); !errors.Is(err, errSourceLeaseHeartbeatLost) {
t.Fatalf("heartbeat.stop() second error = %v, want stable %v", err, errSourceLeaseHeartbeatLost)
}
}
func TestAcquireSourceLeaseConcurrentOnlyOneOwner(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "lease-concurrent")
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip",
RemoteNetworkPolicyPublic,
)
type leaseResult struct {
snapshot *sourceExecutionSnapshot
outcome sourceLeaseOutcome
err error
}
results := make(chan leaseResult, 2)
var workers sync.WaitGroup
workers.Add(2)
for range 2 {
go func() {
defer workers.Done()
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
results <- leaseResult{snapshot: snapshot, outcome: outcome, err: err}
}()
}
workers.Wait()
close(results)
acquired := 0
busy := 0
for result := range results {
if result.err != nil {
t.Errorf("acquireSourceLease(concurrent) error = %v, want nil", result.err)
continue
}
switch result.outcome {
case sourceLeaseAcquired:
acquired++
if result.snapshot == nil || result.snapshot.LeaseToken == "" {
t.Errorf("acquireSourceLease(concurrent acquired) snapshot = %+v, want token-bearing snapshot", result.snapshot)
}
case sourceLeaseBusy:
busy++
if result.snapshot != nil {
t.Errorf("acquireSourceLease(concurrent busy) snapshot = %+v, want nil", result.snapshot)
}
default:
t.Errorf("acquireSourceLease(concurrent) outcome = %q, want acquired or busy", result.outcome)
}
}
if acquired != 1 || busy != 1 {
t.Errorf("concurrent lease outcomes = acquired:%d busy:%d, want 1 and 1", acquired, busy)
}
}
func TestAcquireSourceLeaseMutualExclusionExpiryAndTerminalOwnership(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "lease-cas")
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip",
RemoteNetworkPolicyPublic,
)
first, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
if err != nil {
t.Fatalf("acquireSourceLease(first) error = %v, want nil", err)
}
if got, want := outcome, sourceLeaseAcquired; got != want {
t.Fatalf("acquireSourceLease(first) outcome = %q, want %q", got, want)
}
if first == nil || first.LeaseToken == "" {
t.Fatalf("acquireSourceLease(first) snapshot = %+v, want token-bearing snapshot", first)
}
second, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
if err != nil {
t.Fatalf("acquireSourceLease(duplicate) error = %v, want nil", err)
}
if got, want := outcome, sourceLeaseBusy; got != want {
t.Errorf("acquireSourceLease(duplicate) outcome = %q, want %q", got, want)
}
if second != nil {
t.Errorf("acquireSourceLease(duplicate) snapshot = %+v, want nil", second)
}
past := time.Now().Add(-time.Second)
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", source.ID).
Update("lease_expires_at", &past).Error; err != nil {
t.Fatalf("expire first lease error = %v, want nil", err)
}
takeover, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
if err != nil {
t.Fatalf("acquireSourceLease(takeover) error = %v, want nil", err)
}
if got, want := outcome, sourceLeaseAcquired; got != want {
t.Fatalf("acquireSourceLease(takeover) outcome = %q, want %q", got, want)
}
if takeover == nil {
t.Fatal("acquireSourceLease(takeover) snapshot = nil, want non-nil")
}
if takeover.LeaseToken == "" || takeover.LeaseToken == first.LeaseToken {
t.Fatalf("takeover LeaseToken = %q, want non-empty token distinct from %q", takeover.LeaseToken, first.LeaseToken)
}
renewed, err := renewSourceLease(ctx, first, pagesSourceSyncLeaseDuration)
if err != nil {
t.Fatalf("renewSourceLease(expired owner) error = %v, want nil", err)
}
if renewed {
t.Error("renewSourceLease(expired owner) = true, want false")
}
if err := failSourceLease(ctx, first, "stale worker must not win"); err != nil {
t.Fatalf("failSourceLease(expired owner) error = %v, want nil", err)
}
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
t.Fatalf("load runtime after takeover error = %v, want nil", err)
}
if got, want := runtime.LeaseToken, takeover.LeaseToken; got != want {
t.Errorf("runtime LeaseToken after stale terminal write = %q, want %q", got, want)
}
if got, want := runtime.SyncStatus, pagesSourceStatusSyncing; got != want {
t.Errorf("runtime SyncStatus after stale terminal write = %q, want %q", got, want)
}
renewed, err = renewSourceLease(ctx, takeover, pagesSourceSyncLeaseDuration)
if err != nil {
t.Fatalf("renewSourceLease(current owner) error = %v, want nil", err)
}
if !renewed {
t.Error("renewSourceLease(current owner) = false, want true")
}
if err := failSourceLease(ctx, takeover, errPagesSourceSyncFailed); err != nil {
t.Fatalf("failSourceLease(current owner) error = %v, want nil", err)
}
var failedRuntime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&failedRuntime).Error; err != nil {
t.Fatalf("load failed runtime error = %v, want nil", err)
}
if got, want := failedRuntime.SyncStatus, pagesSourceStatusFailed; got != want {
t.Errorf("failed runtime SyncStatus = %q, want %q", got, want)
}
if failedRuntime.LeaseToken != "" || failedRuntime.LeaseExpiresAt != nil {
t.Errorf("failed runtime lease = (%q, %v), want cleared", failedRuntime.LeaseToken, failedRuntime.LeaseExpiresAt)
}
}
func TestSourceConfigAndProjectContentChangesFenceLease(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "lease-fence")
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip?token=first",
RemoteNetworkPolicyPublic,
)
configSnapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
if err != nil || outcome != sourceLeaseAcquired {
t.Fatalf("acquireSourceLease(config fence) = (%+v, %q, %v), want acquired", configSnapshot, outcome, err)
}
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: "https://example.com/site.zip?token=second",
RemoteNetworkPolicy: RemoteNetworkPolicyPublic,
}); err != nil {
t.Fatalf("UpdateSource(config fence) error = %v, want nil", err)
}
renewed, err := renewSourceLease(ctx, configSnapshot, pagesSourceSyncLeaseDuration)
if err != nil {
t.Fatalf("renewSourceLease(after source update) error = %v, want nil", err)
}
if renewed {
t.Error("renewSourceLease(after source update) = true, want false")
}
var updatedSource model.PagesProjectSource
if err := db.DB(ctx).Where("id = ?", source.ID).First(&updatedSource).Error; err != nil {
t.Fatalf("load updated source error = %v, want nil", err)
}
if got, want := updatedSource.ConfigVersion, source.ConfigVersion+1; got != want {
t.Errorf("updated source ConfigVersion = %d, want %d", got, want)
}
if snapshot, staleOutcome, staleErr := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync); staleErr != nil || staleOutcome != sourceLeaseStale || snapshot != nil {
t.Errorf("acquireSourceLease(old config) = (%+v, %q, %v), want (nil, %q, nil)", snapshot, staleOutcome, staleErr, sourceLeaseStale)
}
contentSnapshot, outcome, err := acquireSourceLease(ctx, source.ID, updatedSource.ConfigVersion, sourceActionSync)
if err != nil || outcome != sourceLeaseAcquired {
t.Fatalf("acquireSourceLease(content fence) = (%+v, %q, %v), want acquired", contentSnapshot, outcome, err)
}
if _, err := UpdateProject(ctx, project.ID, Input{
Name: project.Name,
Slug: project.Slug,
Enabled: true,
RootDir: "dist",
EntryFile: "index.html",
}); err != nil {
t.Fatalf("UpdateProject(content fence) error = %v, want nil", err)
}
renewed, err = renewSourceLease(ctx, contentSnapshot, pagesSourceSyncLeaseDuration)
if err != nil {
t.Fatalf("renewSourceLease(after content update) error = %v, want nil", err)
}
if renewed {
t.Error("renewSourceLease(after content update) = true, want false")
}
storedProject, err := model.GetPagesProjectByID(ctx, project.ID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
}
if got, want := storedProject.ContentConfigVersion, project.ContentConfigVersion+1; got != want {
t.Errorf("ContentConfigVersion after RootDir update = %d, want %d", got, want)
}
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
t.Fatalf("load fenced runtime error = %v, want nil", err)
}
if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil {
t.Errorf("content-fenced runtime lease = (%q, %v), want cleared", runtime.LeaseToken, runtime.LeaseExpiresAt)
}
}
func TestSourceRuntimeUsesOnlySixDocumentedStates(t *testing.T) {
states := []string{
pagesSourceStatusIdle,
pagesSourceStatusChecking,
pagesSourceStatusUpdateAvailable,
pagesSourceStatusSyncing,
pagesSourceStatusFailed,
pagesSourceStatusAttention,
}
seen := make(map[string]struct{}, len(states))
for _, state := range states {
if strings.TrimSpace(state) == "" {
t.Errorf("documented source state = %q, want non-empty", state)
}
if _, exists := seen[state]; exists {
t.Errorf("documented source state %q is duplicated", state)
}
seen[state] = struct{}{}
}
if got, want := len(seen), 6; got != want {
t.Errorf("unique source states = %d, want %d", got, want)
}
updateRuntime := &model.PagesProjectSourceRuntime{
LastSeenRevision: strings.Repeat("a", 64),
LastAppliedRevision: strings.Repeat("b", 64),
LastSeenDetail: `{"release_id":"new"}`,
LastAppliedDetail: `{"release_id":"old"}`,
}
if got, want := normalizedSourceRuntimeStatus(updateRuntime), pagesSourceStatusUpdateAvailable; got != want {
t.Errorf("normalizedSourceRuntimeStatus(update) = %q, want %q", got, want)
}
attentionRuntime := &model.PagesProjectSourceRuntime{
LastSeenRevision: strings.Repeat("a", 64),
LastAppliedRevision: strings.Repeat("b", 64),
LastSeenDetail: `{"release_id":"same"}`,
LastAppliedDetail: `{"release_id":"same"}`,
}
if got, want := normalizedSourceRuntimeStatus(attentionRuntime), pagesSourceStatusAttention; got != want {
t.Errorf("normalizedSourceRuntimeStatus(attention) = %q, want %q", got, want)
}
if got, want := normalizedSourceRuntimeStatus(&model.PagesProjectSourceRuntime{}), pagesSourceStatusIdle; got != want {
t.Errorf("normalizedSourceRuntimeStatus(idle) = %q, want %q", got, want)
}
}
internal/apps/openflare/pages/source_sync.go
+715
View File
@@ -0,0 +1,715 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/task"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const (
pagesSourceTriggerManualSync = "manual_sync"
pagesSourceCreatedBySystem = "system:pages-source-sync"
pagesSourceHeartbeatInterval = pagesSourceSyncLeaseDuration / 3
pagesSourceCleanupTimeout = 15 * time.Second
)
var (
errSourceFinalFence = errors.New("pages source final fence rejected")
errSourceLeaseHeartbeatLost = errors.New("pages source lease heartbeat lost")
sourceCommitNow = time.Now
)
type sourceSyncOutcome struct {
Deployment *DeploymentView
Reused bool
Stale bool
}
type preparedRemoteSource struct {
Candidate *SourceCandidate
Manifest *deploymentManifest
Detail sourceDetail
DetailJSON string
}
type sourceIngestState struct {
Result upload.IngestResult
HasIngest bool
Referenced bool
}
type sourceCommitState struct {
Project *model.PagesProject
Source *model.PagesProjectSource
Runtime *model.PagesProjectSourceRuntime
Now time.Time
}
type sourceLeaseHeartbeat struct {
cancel context.CancelFunc
done <-chan error
stopOnce sync.Once
stopErr error
}
func startSourceLeaseHeartbeat(
ctx context.Context,
snapshot *sourceExecutionSnapshot,
leaseDuration time.Duration,
interval time.Duration,
) (context.Context, *sourceLeaseHeartbeat, error) {
if snapshot == nil || leaseDuration <= 0 || interval <= 0 || interval >= leaseDuration {
return nil, nil, errors.New(errPagesSourceLeaseLost)
}
renewed, err := renewSourceLease(ctx, snapshot, leaseDuration)
if err != nil {
return nil, nil, err
}
if !renewed {
return nil, nil, errSourceLeaseHeartbeatLost
}
workCtx, cancel := context.WithCancel(ctx)
done := make(chan error, 1)
heartbeat := &sourceLeaseHeartbeat{cancel: cancel, done: done}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-workCtx.Done():
done <- nil
return
case <-ticker.C:
renewed, renewErr := renewSourceLease(workCtx, snapshot, leaseDuration)
if renewErr != nil {
if workCtx.Err() != nil {
done <- nil
return
}
done <- renewErr
cancel()
return
}
if !renewed {
done <- errSourceLeaseHeartbeatLost
cancel()
return
}
}
}
}()
return workCtx, heartbeat, nil
}
func (heartbeat *sourceLeaseHeartbeat) stop() error {
if heartbeat == nil {
return nil
}
heartbeat.stopOnce.Do(func() {
heartbeat.cancel()
heartbeat.stopErr = <-heartbeat.done
})
return heartbeat.stopErr
}
func sourceHeartbeatOutcome(err error) (*sourceSyncOutcome, error) {
if errors.Is(err, errSourceLeaseHeartbeatLost) {
return &sourceSyncOutcome{Stale: true}, nil
}
return nil, err
}
func recordSourceLeaseFailure(ctx context.Context, snapshot *sourceExecutionSnapshot) {
cleanupCtx, cancel := sourceCleanupContext(ctx)
defer cancel()
if err := failSourceLease(cleanupCtx, snapshot, errPagesSourceSyncFailed); err != nil {
var sourceID uint
if snapshot != nil {
sourceID = snapshot.SourceID
}
logger.WarnF(cleanupCtx, "[PagesSource] record failed runtime state failed: source_id=%d error=%v", sourceID, err)
}
}
func sourceCleanupContext(ctx context.Context) (context.Context, context.CancelFunc) {
return context.WithTimeout(context.WithoutCancel(ctx), pagesSourceCleanupTimeout)
}
func syncRemoteSource(
ctx context.Context,
snapshot *sourceExecutionSnapshot,
actor string,
) (outcome *sourceSyncOutcome, resultErr error) {
if snapshot == nil || snapshot.SourceType != PagesSourceTypeRemoteURL {
return nil, errors.New(errPagesSourceTypeUnsupported)
}
actor = strings.TrimSpace(actor)
if actor == "" {
return nil, errors.New(errPagesSourceActionInvalid)
}
defer func() {
if resultErr != nil {
recordSourceLeaseFailure(ctx, snapshot)
}
}()
workCtx, heartbeat, err := startSourceLeaseHeartbeat(
ctx,
snapshot,
pagesSourceSyncLeaseDuration,
pagesSourceHeartbeatInterval,
)
if err != nil {
return sourceHeartbeatOutcome(err)
}
defer func() {
_ = heartbeat.stop()
}()
limits := resolvePagesLimits(workCtx)
prepared, err := prepareRemoteSource(workCtx, snapshot, limits)
if err != nil {
if heartbeatErr := heartbeat.stop(); heartbeatErr != nil {
return sourceHeartbeatOutcome(heartbeatErr)
}
return nil, err
}
defer func() {
if cleanupErr := prepared.Candidate.Cleanup(); cleanupErr != nil {
logger.WarnF(ctx, "[PagesSource] cleanup temporary package failed: source_id=%d error=%v", snapshot.SourceID, cleanupErr)
}
}()
ingestState, err := resolveSourceIngest(workCtx, snapshot, prepared)
if err != nil {
if heartbeatErr := heartbeat.stop(); heartbeatErr != nil {
return sourceHeartbeatOutcome(heartbeatErr)
}
return nil, err
}
defer func() {
compensateSourceIngest(ctx, snapshot, ingestState)
}()
if heartbeatErr := heartbeat.stop(); heartbeatErr != nil {
return sourceHeartbeatOutcome(heartbeatErr)
}
renewed, err := renewSourceLease(ctx, snapshot, pagesSourceSyncLeaseDuration)
if err != nil {
return nil, err
}
if !renewed {
return &sourceSyncOutcome{Stale: true}, nil
}
task.AppendLog(ctx, "[activate] 正在原子切换生产部署")
deployment, reused, referenced, err := commitSourceDeployment(
ctx,
snapshot,
prepared.Candidate.Checksum,
prepared.Detail,
prepared.DetailJSON,
actor,
prepared.Manifest,
ingestState.Result,
ingestState.HasIngest,
)
ingestState.Referenced = referenced
if errors.Is(err, errSourceFinalFence) {
return &sourceSyncOutcome{Stale: true}, nil
}
if err != nil {
return nil, err
}
ingestState.Referenced = ingestState.HasIngest && deployment.UploadID == ingestState.Result.Upload.ID
if pruneErr := pruneProjectDeploymentHistory(ctx, snapshot.ProjectID, limits.HistoryCount, 0); pruneErr != nil {
logger.ErrorF(ctx,
"[PagesSource] strict prune failed: project_id=%d source_id=%d keep=%d error=%v",
snapshot.ProjectID,
snapshot.SourceID,
limits.HistoryCount,
pruneErr,
)
}
view := buildDeploymentView(deployment)
return &sourceSyncOutcome{Deployment: &view, Reused: reused}, nil
}
func prepareRemoteSource(
ctx context.Context,
snapshot *sourceExecutionSnapshot,
limits pagesLimits,
) (*preparedRemoteSource, error) {
task.AppendLog(ctx, "[download] 正在获取远程部署包")
candidate, err := FetchRemoteSource(ctx, RemoteSourceRequest{
URL: snapshot.RemoteURL,
NetworkPolicy: snapshot.RemoteNetworkPolicy,
MaxPackageBytes: limits.PackageBytes,
})
if err != nil {
return nil, err
}
if candidate == nil || candidate.TempPath == "" || candidate.Checksum == "" || candidate.Format == "" {
return nil, errors.New(errPagesSourceSyncFailed)
}
rootDir, err := validateAndNormalizePagesRootDir(snapshot.RootDir)
if err != nil {
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
return nil, err
}
entryFile, err := validateAndNormalizePagesEntryFile(snapshot.EntryFile)
if err != nil {
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
return nil, err
}
task.AppendLog(ctx, "[verify] 正在校验归档结构与入口文件")
manifest, err := inspectPagesPackage(candidate.TempPath, candidate.Format, rootDir, entryFile, limits)
if err != nil {
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
return nil, err
}
detail := sourceDetail{Provider: PagesSourceTypeRemoteURL, Label: safeRemoteSourceLabel(candidate.SafeLabel)}
detailJSON, err := json.Marshal(detail)
if err != nil {
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
return nil, err
}
return &preparedRemoteSource{
Candidate: candidate,
Manifest: manifest,
Detail: detail,
DetailJSON: string(detailJSON),
}, nil
}
func cleanupFailedRemoteCandidate(
ctx context.Context,
snapshot *sourceExecutionSnapshot,
candidate *SourceCandidate,
) {
if err := candidate.Cleanup(); err != nil {
logger.WarnF(ctx, "[PagesSource] cleanup failed preparation package: source_id=%d error=%v", snapshot.SourceID, err)
}
}
func resolveSourceIngest(
ctx context.Context,
snapshot *sourceExecutionSnapshot,
prepared *preparedRemoteSource,
) (*sourceIngestState, error) {
_, err := findSourceDeployment(
ctx,
snapshot.ProjectID,
snapshot.SourceIdentity,
prepared.Candidate.Checksum,
)
if err == nil {
return &sourceIngestState{}, nil
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
task.AppendLog(ctx, "[ingest] 正在保存受管部署包")
result, err := ingestPagesDeploymentPackageWithSource(
ctx,
prepared.Candidate.TempPath,
prepared.Candidate.Checksum,
snapshot.ProjectID,
snapshot.SourceID,
prepared.Detail.Label,
prepared.Candidate.Format,
)
if err != nil {
return nil, err
}
return &sourceIngestState{Result: result, HasIngest: true}, nil
}
func compensateSourceIngest(
ctx context.Context,
snapshot *sourceExecutionSnapshot,
state *sourceIngestState,
) {
if state == nil || !state.HasIngest || !state.Result.Created || state.Referenced {
return
}
cleanupCtx, cancel := sourceCleanupContext(ctx)
defer cancel()
task.AppendLog(cleanupCtx, "[cleanup] 正在补偿未引用的部署包记录")
if err := removePagesUploadIfUnreferenced(cleanupCtx, snapshot.ProjectID, state.Result.Upload.ID); err != nil {
logger.ErrorF(cleanupCtx,
"[PagesSource] compensate upload failed: project_id=%d source_id=%d upload_id=%d error=%v",
snapshot.ProjectID, snapshot.SourceID, state.Result.Upload.ID, err,
)
}
}
func findSourceDeployment(
ctx context.Context,
projectID uint,
sourceIdentity string,
revision string,
) (*model.PagesDeployment, error) {
var deployment model.PagesDeployment
err := db.DB(ctx).
Where("project_id = ? AND source_identity = ? AND source_revision = ?", projectID, sourceIdentity, revision).
First(&deployment).Error
if err != nil {
return nil, err
}
return &deployment, nil
}
func commitSourceDeployment(
ctx context.Context,
snapshot *sourceExecutionSnapshot,
revision string,
detail sourceDetail,
detailJSON string,
actor string,
manifest *deploymentManifest,
ingestResult upload.IngestResult,
hasIngest bool,
) (*model.PagesDeployment, bool, bool, error) {
if snapshot == nil || manifest == nil {
return nil, false, false, errors.New(errPagesSourceSyncFailed)
}
var committed model.PagesDeployment
reused := false
ingestReferenced := false
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
state, err := lockSourceCommitState(tx, snapshot)
if err != nil {
return err
}
target, targetReused, err := resolveSourceDeploymentTx(
tx, state, revision, detail, detailJSON, actor, manifest, ingestResult, hasIngest,
)
if err != nil {
return err
}
if err := lockSourceDeploymentUploadsTx(tx, target, ingestResult, hasIngest); err != nil {
return err
}
if err := ensureDeploymentEntry(tx, target.ID, state.Project.RootDir, state.Project.EntryFile); err != nil {
return err
}
if err := refreshSourceCommitLease(state, snapshot); err != nil {
return err
}
if err := activateSourceDeploymentTx(tx, state, target, revision, detailJSON); err != nil {
return err
}
committed = *target
reused = targetReused
ingestReferenced = hasIngest && target.UploadID == ingestResult.Upload.ID
return nil
})
if err != nil {
return nil, false, false, err
}
return &committed, reused, ingestReferenced, nil
}
func lockSourceCommitState(tx *gorm.DB, snapshot *sourceExecutionSnapshot) (*sourceCommitState, error) {
state := &sourceCommitState{}
var project model.PagesProject
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
First(&project, snapshot.ProjectID).Error; err != nil {
return nil, sourceFenceRecordError(err)
}
if project.ContentConfigVersion != snapshot.ContentConfigVersion {
return nil, errSourceFinalFence
}
var source model.PagesProjectSource
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id = ? AND project_id = ?", snapshot.SourceID, snapshot.ProjectID).
First(&source).Error; err != nil {
return nil, sourceFenceRecordError(err)
}
if source.ConfigVersion != snapshot.SourceConfigVersion ||
source.SourceIdentity != snapshot.SourceIdentity ||
source.SourceType != snapshot.SourceType {
return nil, errSourceFinalFence
}
var runtime model.PagesProjectSourceRuntime
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", source.ID).
First(&runtime).Error; err != nil {
return nil, sourceFenceRecordError(err)
}
state.Project = &project
state.Source = &source
state.Runtime = &runtime
if err := refreshSourceCommitLease(state, snapshot); err != nil {
return nil, err
}
return state, nil
}
func refreshSourceCommitLease(state *sourceCommitState, snapshot *sourceExecutionSnapshot) error {
if state == nil || state.Runtime == nil || snapshot == nil {
return errSourceFinalFence
}
now := sourceCommitNow()
if state.Runtime.LeaseToken != snapshot.LeaseToken || state.Runtime.LeaseExpiresAt == nil ||
!state.Runtime.LeaseExpiresAt.After(now) {
return errSourceFinalFence
}
state.Now = now
return nil
}
func sourceFenceRecordError(err error) error {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errSourceFinalFence
}
return err
}
func resolveSourceDeploymentTx(
tx *gorm.DB,
state *sourceCommitState,
revision string,
detail sourceDetail,
detailJSON string,
actor string,
manifest *deploymentManifest,
ingestResult upload.IngestResult,
hasIngest bool,
) (*model.PagesDeployment, bool, error) {
var target model.PagesDeployment
err := tx.Where(
"project_id = ? AND source_identity = ? AND source_revision = ?",
state.Project.ID,
state.Source.SourceIdentity,
revision,
).First(&target).Error
if err == nil {
return &target, true, nil
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, false, err
}
if !hasIngest {
return nil, false, errSourceFinalFence
}
return createSourceDeploymentTx(
tx, state, revision, detail, detailJSON, actor, manifest, ingestResult,
)
}
func createSourceDeploymentTx(
tx *gorm.DB,
state *sourceCommitState,
revision string,
detail sourceDetail,
detailJSON string,
actor string,
manifest *deploymentManifest,
ingestResult upload.IngestResult,
) (*model.PagesDeployment, bool, error) {
var maxNumber int
if err := tx.Model(&model.PagesDeployment{}).
Where("project_id = ?", state.Project.ID).
Select("COALESCE(MAX(deployment_number), 0)").
Scan(&maxNumber).Error; err != nil {
return nil, false, err
}
identity := state.Source.SourceIdentity
revisionValue := revision
target := &model.PagesDeployment{
ProjectID: state.Project.ID,
DeploymentNumber: maxNumber + 1,
Checksum: revision,
Status: model.PagesDeploymentStatusUploaded,
UploadID: ingestResult.Upload.ID,
FileCount: manifest.FileCount,
TotalSize: manifest.TotalSize,
CreatedBy: actor,
SourceType: state.Source.SourceType,
SourceIdentity: &identity,
SourceRevision: &revisionValue,
SourceLabel: detail.Label,
SourceMeta: detailJSON,
TriggerType: pagesSourceTriggerManualSync,
}
result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(target)
if result.Error != nil {
return nil, false, result.Error
}
if result.RowsAffected == 0 {
return reloadSourceDeploymentTx(tx, state.Project.ID, identity, revision)
}
if err := createSourceDeploymentFilesTx(tx, target.ID, manifest.Files); err != nil {
return nil, false, err
}
return target, false, nil
}
func reloadSourceDeploymentTx(
tx *gorm.DB,
projectID uint,
identity string,
revision string,
) (*model.PagesDeployment, bool, error) {
var target model.PagesDeployment
err := tx.Where(
"project_id = ? AND source_identity = ? AND source_revision = ?",
projectID,
identity,
revision,
).First(&target).Error
return &target, true, err
}
func createSourceDeploymentFilesTx(tx *gorm.DB, deploymentID uint, files []model.PagesDeploymentFile) error {
if len(files) == 0 {
return nil
}
for index := range files {
files[index].DeploymentID = deploymentID
}
return tx.Create(&files).Error
}
func lockSourceDeploymentUploadsTx(
tx *gorm.DB,
target *model.PagesDeployment,
ingestResult upload.IngestResult,
hasIngest bool,
) error {
uploadIDs := []uint64{target.UploadID}
if hasIngest && ingestResult.Upload.ID != 0 && ingestResult.Upload.ID != target.UploadID {
uploadIDs = append(uploadIDs, ingestResult.Upload.ID)
}
sort.Slice(uploadIDs, func(i, j int) bool { return uploadIDs[i] < uploadIDs[j] })
var records []model.Upload
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id IN ?", uploadIDs).
Order("id asc").
Find(&records).Error; err != nil {
return err
}
for index := range records {
if records[index].ID != target.UploadID {
continue
}
if records[index].Status == model.UploadStatusUsed && records[index].Type == upload.ReservedPagesDeploymentType {
return nil
}
break
}
return errSourceFinalFence
}
func activateSourceDeploymentTx(
tx *gorm.DB,
state *sourceCommitState,
target *model.PagesDeployment,
revision string,
detailJSON string,
) error {
if err := tx.Model(&model.PagesDeployment{}).
Where("project_id = ?", state.Project.ID).
Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil {
return err
}
if err := tx.Model(target).Updates(map[string]any{
"status": model.PagesDeploymentStatusActive,
"activated_at": &state.Now,
}).Error; err != nil {
return err
}
if err := tx.Model(state.Project).Update("active_deployment_id", target.ID).Error; err != nil {
return err
}
finishedAt := sourceCommitNow()
result := tx.Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?",
state.Runtime.SourceID,
state.Runtime.LeaseToken,
finishedAt,
).
Updates(map[string]any{
"last_seen_revision": revision,
"last_seen_detail": detailJSON,
"last_applied_revision": revision,
"last_applied_detail": detailJSON,
sourceRuntimeColumnSyncStatus: pagesSourceStatusIdle,
sourceRuntimeColumnLastError: "",
"last_checked_at": &finishedAt,
"last_synced_at": &finishedAt,
"next_check_at": nil,
sourceRuntimeColumnLeaseToken: "",
sourceRuntimeColumnLeaseExpiresAt: nil,
})
if result.Error != nil {
return result.Error
}
if result.RowsAffected != 1 {
return errSourceFinalFence
}
return nil
}
func safeRemoteSourceLabel(raw string) string {
value := strings.ReplaceAll(strings.ToValidUTF8(raw, ""), "\\", "/")
value = path.Base(strings.TrimSpace(value))
if value == "." || value == "/" {
value = ""
}
var builder strings.Builder
for _, character := range value {
if character >= 0x20 && character != 0x7f {
builder.WriteRune(character)
}
}
value = strings.TrimSpace(builder.String())
if value == "" {
value = defaultRemoteAssetLabel
}
if len(value) > remoteSourceMaxSafeLabelBytes {
value = value[:remoteSourceMaxSafeLabelBytes]
for !utf8.ValidString(value) {
_, size := utf8.DecodeLastRuneInString(value)
value = value[:len(value)-size]
}
}
return value
}
func sourceSyncResultDetail(outcome *sourceSyncOutcome) string {
if outcome == nil || outcome.Deployment == nil {
return ""
}
payload := map[string]any{
"deployment_id": outcome.Deployment.ID,
"reused": outcome.Reused,
}
encoded, err := json.Marshal(payload)
if err != nil {
return fmt.Sprintf(`{"deployment_id":%d}`, outcome.Deployment.ID)
}
return string(encoded)
}
internal/apps/openflare/pages/source_sync_test.go
+604
View File
@@ -0,0 +1,604 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/pagesarchive"
"gorm.io/gorm"
)
func setupPagesSourceSyncTest(t *testing.T) context.Context {
t.Helper()
ctx := setupPagesSourceTest(t)
_, disableStorage := setupPagesStorageMock(t)
t.Cleanup(disableStorage)
return ctx
}
func mustAcquireRemoteSyncLease(
t *testing.T,
ctx context.Context,
source *model.PagesProjectSource,
) *sourceExecutionSnapshot {
t.Helper()
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
if err != nil {
t.Fatalf("acquireSourceLease(source=%d) error = %v, want nil", source.ID, err)
}
if got, want := outcome, sourceLeaseAcquired; got != want {
t.Fatalf("acquireSourceLease(source=%d) outcome = %q, want %q", source.ID, got, want)
}
if snapshot == nil {
t.Fatalf("acquireSourceLease(source=%d) snapshot = nil, want non-nil", source.ID)
}
return snapshot
}
func mustCreateActiveManualDeployment(
t *testing.T,
ctx context.Context,
projectID uint,
content string,
) *model.PagesDeployment {
t.Helper()
view, err := UploadDeployment(
ctx,
projectID,
testPagesMultipartFile(t, "manual.zip", testPagesZip(t, map[string]string{"index.html": content})),
"user:1",
)
if err != nil {
t.Fatalf("UploadDeployment(project=%d) error = %v, want nil", projectID, err)
}
if _, err := ActivateDeploymentAs(ctx, projectID, view.ID, "user:1"); err != nil {
t.Fatalf("ActivateDeploymentAs(project=%d, deployment=%d) error = %v, want nil", projectID, view.ID, err)
}
deployment, err := model.GetPagesDeploymentByID(ctx, view.ID)
if err != nil {
t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", view.ID, err)
}
return deployment
}
func newPagesArchiveServer(t *testing.T, status int, body []byte, beforeWrite func() error) *httptest.Server {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
if beforeWrite != nil {
if err := beforeWrite(); err != nil {
writer.WriteHeader(http.StatusInternalServerError)
return
}
}
writer.Header().Set("Content-Type", "application/zip")
writer.Header().Set("Content-Disposition", `attachment; filename="site.zip"`)
writer.WriteHeader(status)
_, _ = writer.Write(body)
}))
t.Cleanup(server.Close)
return server
}
func TestSyncRemoteSourceAtomicallyActivatesAndReusesChecksum(t *testing.T) {
ctx := setupPagesSourceSyncTest(t)
project := mustCreatePagesSourceProject(t, ctx, "sync-success")
packageBytes := testPagesZip(t, map[string]string{
"index.html": "remote-v1",
"assets/app.js": "console.log('ok')",
})
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil)
secret := "sync-query-secret"
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
server.URL+"/site.zip?token="+secret,
RemoteNetworkPolicyTrustedInternal,
)
firstSnapshot := mustAcquireRemoteSyncLease(t, ctx, source)
first, err := syncRemoteSource(ctx, firstSnapshot, "user:42")
if err != nil {
t.Fatalf("syncRemoteSource(first) error = %v, want nil", err)
}
if first == nil || first.Stale || first.Reused || first.Deployment == nil {
t.Fatalf("syncRemoteSource(first) = %+v, want new active deployment", first)
}
storedProject, err := model.GetPagesProjectByID(ctx, project.ID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
}
if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != first.Deployment.ID {
t.Fatalf("project ActiveDeploymentID = %v, want %d", storedProject.ActiveDeploymentID, first.Deployment.ID)
}
deployment, err := model.GetPagesDeploymentByID(ctx, first.Deployment.ID)
if err != nil {
t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", first.Deployment.ID, err)
}
expectedHash := sha256.Sum256(packageBytes)
if got, want := deployment.Checksum, hex.EncodeToString(expectedHash[:]); got != want {
t.Errorf("deployment Checksum = %q, want %q", got, want)
}
if got, want := deployment.Status, model.PagesDeploymentStatusActive; got != want {
t.Errorf("deployment Status = %q, want %q", got, want)
}
if got, want := deployment.SourceType, PagesSourceTypeRemoteURL; got != want {
t.Errorf("deployment SourceType = %q, want %q", got, want)
}
if deployment.SourceIdentity == nil || *deployment.SourceIdentity != source.SourceIdentity {
t.Errorf("deployment SourceIdentity = %v, want %q", deployment.SourceIdentity, source.SourceIdentity)
}
if deployment.SourceRevision == nil || *deployment.SourceRevision != deployment.Checksum {
t.Errorf("deployment SourceRevision = %v, want %q", deployment.SourceRevision, deployment.Checksum)
}
if got, want := deployment.CreatedBy, "user:42"; got != want {
t.Errorf("deployment CreatedBy = %q, want %q", got, want)
}
if got, want := deployment.TriggerType, pagesSourceTriggerManualSync; got != want {
t.Errorf("deployment TriggerType = %q, want %q", got, want)
}
if strings.Contains(deployment.SourceMeta, secret) || strings.Contains(deployment.SourceLabel, secret) {
t.Errorf("deployment provenance = label:%q meta:%q, want no query secret", deployment.SourceLabel, deployment.SourceMeta)
}
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
t.Fatalf("load source runtime error = %v, want nil", err)
}
if got, want := runtime.SyncStatus, pagesSourceStatusIdle; got != want {
t.Errorf("runtime SyncStatus = %q, want %q", got, want)
}
if got, want := runtime.LastAppliedRevision, deployment.Checksum; got != want {
t.Errorf("runtime LastAppliedRevision = %q, want %q", got, want)
}
if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil {
t.Errorf("runtime lease = (%q, %v), want cleared", runtime.LeaseToken, runtime.LeaseExpiresAt)
}
var uploadRecord model.Upload
if err := db.DB(ctx).First(&uploadRecord, deployment.UploadID).Error; err != nil {
t.Fatalf("load deployment upload %d error = %v, want nil", deployment.UploadID, err)
}
if got, want := uploadRecord.Status, model.UploadStatusUsed; got != want {
t.Errorf("deployment upload Status = %q, want %q", got, want)
}
if got, want := uploadRecord.Type, upload.ReservedPagesDeploymentType; got != want {
t.Errorf("deployment upload Type = %q, want %q", got, want)
}
if got, want := fmt.Sprint(uploadRecord.Metadata.Extra[pagesSourceIDMetadataKey]), fmt.Sprint(source.ID); got != want {
t.Errorf("deployment upload pages_source_id = %q, want %q", got, want)
}
secondSnapshot := mustAcquireRemoteSyncLease(t, ctx, source)
second, err := syncRemoteSource(ctx, secondSnapshot, "user:42")
if err != nil {
t.Fatalf("syncRemoteSource(second) error = %v, want nil", err)
}
if second == nil || second.Stale || !second.Reused || second.Deployment == nil {
t.Fatalf("syncRemoteSource(second) = %+v, want reused active deployment", second)
}
if got, want := second.Deployment.ID, first.Deployment.ID; got != want {
t.Errorf("reused deployment ID = %d, want %d", got, want)
}
var deploymentCount, uploadCount int64
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("project_id = ?", project.ID).Count(&deploymentCount).Error; err != nil {
t.Fatalf("count source deployments error = %v, want nil", err)
}
if err := db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error; err != nil {
t.Fatalf("count source uploads error = %v, want nil", err)
}
if got, want := deploymentCount, int64(1); got != want {
t.Errorf("deployment count after identical sync = %d, want %d", got, want)
}
if got, want := uploadCount, int64(1); got != want {
t.Errorf("upload count after identical sync = %d, want %d", got, want)
}
}
func TestSyncRemoteSourceDownloadFailureKeepsOldActive(t *testing.T) {
ctx := setupPagesSourceSyncTest(t)
project := mustCreatePagesSourceProject(t, ctx, "sync-download-fail")
oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active")
server := newPagesArchiveServer(t, http.StatusBadGateway, []byte("upstream failed"), nil)
secret := "download-failure-secret"
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
server.URL+"/site.zip?token="+secret,
RemoteNetworkPolicyTrustedInternal,
)
_, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:2")
if err == nil {
t.Fatal("syncRemoteSource(download failure) error = nil, want non-nil")
}
if strings.Contains(err.Error(), secret) {
t.Errorf("syncRemoteSource(download failure) error = %q, want no query secret", err)
}
assertPagesSyncFailureState(t, ctx, project.ID, source.ID, oldActive.ID, 1)
}
func TestSyncRemoteSourceArchiveFailureKeepsOldActive(t *testing.T) {
ctx := setupPagesSourceSyncTest(t)
project := mustCreatePagesSourceProject(t, ctx, "sync-archive-fail")
oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active")
server := newPagesArchiveServer(t, http.StatusOK, []byte("not-a-valid-zip"), nil)
secret := "archive-failure-secret"
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
server.URL+"/site.zip?token="+secret,
RemoteNetworkPolicyTrustedInternal,
)
_, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:3")
if err == nil {
t.Fatal("syncRemoteSource(archive failure) error = nil, want non-nil")
}
if strings.Contains(err.Error(), secret) {
t.Errorf("syncRemoteSource(archive failure) error = %q, want no query secret", err)
}
assertPagesSyncFailureState(t, ctx, project.ID, source.ID, oldActive.ID, 1)
}
func TestSyncRemoteSourceFinalFenceCompensatesIngest(t *testing.T) {
ctx := setupPagesSourceSyncTest(t)
project := mustCreatePagesSourceProject(t, ctx, "sync-final-fence")
oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active")
packageBytes := testPagesZip(t, map[string]string{"index.html": "never-activate"})
mutationResult := make(chan error, 1)
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, func() error {
err := db.DB(context.Background()).Model(&model.PagesProject{}).
Where("id = ?", project.ID).
Update("content_config_version", gorm.Expr("content_config_version + 1")).Error
mutationResult <- err
return err
})
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
server.URL+"/site.zip?token=final-fence-secret",
RemoteNetworkPolicyTrustedInternal,
)
outcome, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:4")
if err != nil {
t.Fatalf("syncRemoteSource(final fence) error = %v, want nil stale outcome", err)
}
select {
case mutationErr := <-mutationResult:
if mutationErr != nil {
t.Fatalf("content version mutation error = %v, want nil", mutationErr)
}
case <-time.After(2 * time.Second):
t.Fatal("content version mutation was not observed")
}
if outcome == nil || !outcome.Stale || outcome.Deployment != nil {
t.Fatalf("syncRemoteSource(final fence) = %+v, want stale outcome without deployment", outcome)
}
storedProject, err := model.GetPagesProjectByID(ctx, project.ID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
}
if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != oldActive.ID {
t.Errorf("ActiveDeploymentID after final fence = %v, want %d", storedProject.ActiveDeploymentID, oldActive.ID)
}
var deployments []model.PagesDeployment
if err := db.DB(ctx).Where("project_id = ?", project.ID).Find(&deployments).Error; err != nil {
t.Fatalf("list deployments after final fence error = %v, want nil", err)
}
if got, want := len(deployments), 1; got != want {
t.Errorf("deployment count after final fence = %d, want %d", got, want)
}
var uploads []model.Upload
if err := db.DB(ctx).Order("id asc").Find(&uploads).Error; err != nil {
t.Fatalf("list uploads after final fence error = %v, want nil", err)
}
var compensated *model.Upload
for index := range uploads {
if fmt.Sprint(uploads[index].Metadata.Extra[pagesSourceIDMetadataKey]) == fmt.Sprint(source.ID) {
compensated = &uploads[index]
break
}
}
if compensated == nil {
t.Fatalf("source upload after final fence = nil, want compensated upload record")
}
if got, want := compensated.Status, model.UploadStatusDeleted; got != want {
t.Errorf("compensated upload Status = %q, want %q", got, want)
}
var danglingCount int64
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
Where("upload_id = ?", compensated.ID).
Count(&danglingCount).Error; err != nil {
t.Fatalf("count compensated upload references error = %v, want nil", err)
}
if got, want := danglingCount, int64(0); got != want {
t.Errorf("deployments referencing compensated upload = %d, want %d", got, want)
}
}
func TestCommitSourceDeploymentRechecksLeaseAfterUploadLocks(t *testing.T) {
ctx := setupPagesSourceSyncTest(t)
project := mustCreatePagesSourceProject(t, ctx, "sync-expiry-recheck")
packageBytes := testPagesZip(t, map[string]string{"index.html": "expiry-recheck"})
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil)
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
server.URL+"/site.zip",
RemoteNetworkPolicyTrustedInternal,
)
first, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:5")
if err != nil || first == nil || first.Deployment == nil {
t.Fatalf("syncRemoteSource(seed) = (%+v, %v), want deployment", first, err)
}
deployment, err := model.GetPagesDeploymentByID(ctx, first.Deployment.ID)
if err != nil {
t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", first.Deployment.ID, err)
}
if err := db.DB(ctx).Model(&model.PagesProject{}).
Where("id = ?", project.ID).
Update("active_deployment_id", nil).Error; err != nil {
t.Fatalf("clear active deployment error = %v, want nil", err)
}
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
Where("id = ?", deployment.ID).
Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil {
t.Fatalf("reset deployment status error = %v, want nil", err)
}
snapshot := mustAcquireRemoteSyncLease(t, ctx, source)
expiresAt := time.Now().Add(time.Hour)
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ? AND lease_token = ?", source.ID, snapshot.LeaseToken).
Update("lease_expires_at", &expiresAt).Error; err != nil {
t.Fatalf("set deterministic lease expiry error = %v, want nil", err)
}
originalNow := sourceCommitNow
nowCalls := 0
sourceCommitNow = func() time.Time {
nowCalls++
if nowCalls == 1 {
return expiresAt.Add(-time.Second)
}
return expiresAt.Add(time.Second)
}
t.Cleanup(func() { sourceCommitNow = originalNow })
_, _, _, err = commitSourceDeployment(
ctx,
snapshot,
deployment.Checksum,
sourceDetail{Provider: PagesSourceTypeRemoteURL, Label: deployment.SourceLabel},
deployment.SourceMeta,
"user:5",
&deploymentManifest{},
upload.IngestResult{},
false,
)
if !errors.Is(err, errSourceFinalFence) {
t.Fatalf("commitSourceDeployment(expired after upload lock) error = %v, want %v", err, errSourceFinalFence)
}
if nowCalls != 2 {
t.Fatalf("sourceCommitNow calls = %d, want runtime-lock and post-upload checks", nowCalls)
}
storedProject, err := model.GetPagesProjectByID(ctx, project.ID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
}
if storedProject.ActiveDeploymentID != nil {
t.Errorf("ActiveDeploymentID after expiry recheck = %v, want nil", storedProject.ActiveDeploymentID)
}
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
if err != nil {
t.Fatalf("GetPagesDeploymentByID(%d) after expiry error = %v, want nil", deployment.ID, err)
}
if got, want := storedDeployment.Status, model.PagesDeploymentStatusUploaded; got != want {
t.Errorf("deployment status after expiry recheck = %q, want %q", got, want)
}
}
func TestCompensateSourceIngestSurvivesCanceledParentContext(t *testing.T) {
ctx := setupPagesSourceSyncTest(t)
project := mustCreatePagesSourceProject(t, ctx, "sync-canceled-compensation")
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip",
RemoteNetworkPolicyPublic,
)
packageBytes := testPagesZip(t, map[string]string{"index.html": "cancel-compensation"})
packagePath := filepath.Join(t.TempDir(), "site.zip")
if err := os.WriteFile(packagePath, packageBytes, 0o600); err != nil {
t.Fatalf("write test package error = %v, want nil", err)
}
digest := sha256.Sum256(packageBytes)
result, err := ingestPagesDeploymentPackageWithSource(
ctx,
packagePath,
hex.EncodeToString(digest[:]),
project.ID,
source.ID,
"site.zip",
pagesarchive.FormatZip,
)
if err != nil {
t.Fatalf("ingestPagesDeploymentPackageWithSource() error = %v, want nil", err)
}
if !result.Created {
t.Fatal("ingest result Created = false, want a compensatable record")
}
canceledCtx, cancel := context.WithCancel(ctx)
cancel()
compensateSourceIngest(canceledCtx, &sourceExecutionSnapshot{
ProjectID: project.ID,
SourceID: source.ID,
}, &sourceIngestState{
Result: result,
HasIngest: true,
})
var uploadRecord model.Upload
if err := db.DB(ctx).Where("id = ?", result.Upload.ID).First(&uploadRecord).Error; err != nil {
t.Fatalf("load compensated upload error = %v, want nil", err)
}
if got, want := uploadRecord.Status, model.UploadStatusDeleted; got != want {
t.Errorf("compensated upload status = %q, want %q", got, want)
}
}
func assertPagesSyncFailureState(
t *testing.T,
ctx context.Context,
projectID uint,
sourceID uint,
oldActiveID uint,
wantDeploymentCount int64,
) {
t.Helper()
project, err := model.GetPagesProjectByID(ctx, projectID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", projectID, err)
}
if project.ActiveDeploymentID == nil || *project.ActiveDeploymentID != oldActiveID {
t.Errorf("project %d ActiveDeploymentID = %v, want %d", projectID, project.ActiveDeploymentID, oldActiveID)
}
var deploymentCount int64
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
Where("project_id = ?", projectID).
Count(&deploymentCount).Error; err != nil {
t.Fatalf("count project %d deployments error = %v, want nil", projectID, err)
}
if got, want := deploymentCount, wantDeploymentCount; got != want {
t.Errorf("project %d deployment count = %d, want %d", projectID, got, want)
}
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil {
t.Fatalf("load source %d runtime error = %v, want nil", sourceID, err)
}
if got, want := runtime.SyncStatus, pagesSourceStatusFailed; got != want {
t.Errorf("source %d runtime SyncStatus = %q, want %q", sourceID, got, want)
}
if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil {
t.Errorf("source %d runtime lease = (%q, %v), want cleared", sourceID, runtime.LeaseToken, runtime.LeaseExpiresAt)
}
}
func TestCommitSourceDeploymentRejectsDeletedTargetUpload(t *testing.T) {
ctx := setupPagesSourceSyncTest(t)
project := mustCreatePagesSourceProject(t, ctx, "sync-deleted-upload")
packageBytes := testPagesZip(t, map[string]string{"index.html": "content"})
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil)
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
server.URL+"/site.zip",
RemoteNetworkPolicyTrustedInternal,
)
snapshot := mustAcquireRemoteSyncLease(t, ctx, source)
// A pre-existing source deployment whose upload was removed must never be
// reactivated into a dangling active pointer.
identity := source.SourceIdentity
revision := strings.Repeat("d", 64)
uploadRecord := &model.Upload{
ID: 987654321,
UserID: 999,
FileName: "deleted.zip",
FilePath: "deleted.zip",
FileSize: 1,
MimeType: "application/zip",
Extension: "zip",
Hash: revision,
Type: upload.ReservedPagesDeploymentType,
Status: model.UploadStatusDeleted,
AccessMode: 0,
}
if err := db.DB(ctx).Create(uploadRecord).Error; err != nil {
t.Fatalf("create deleted upload error = %v, want nil", err)
}
deployment := &model.PagesDeployment{
ProjectID: project.ID,
DeploymentNumber: 1,
Checksum: revision,
Status: model.PagesDeploymentStatusUploaded,
UploadID: uploadRecord.ID,
FileCount: 1,
TotalSize: 1,
CreatedBy: "user:1",
SourceType: PagesSourceTypeRemoteURL,
SourceIdentity: &identity,
SourceRevision: &revision,
SourceLabel: "deleted.zip",
SourceMeta: `{"provider":"remote_url","label":"deleted.zip"}`,
TriggerType: pagesSourceTriggerManualSync,
}
if err := db.DB(ctx).Create(deployment).Error; err != nil {
t.Fatalf("create source deployment error = %v, want nil", err)
}
if err := db.DB(ctx).Create(&model.PagesDeploymentFile{
DeploymentID: deployment.ID,
Path: "index.html",
Size: 1,
Checksum: revision,
}).Error; err != nil {
t.Fatalf("create source deployment file error = %v, want nil", err)
}
manifest := &deploymentManifest{
FileCount: 1,
TotalSize: 1,
EntryFile: "index.html",
}
_, _, _, err := commitSourceDeployment(
ctx,
snapshot,
revision,
sourceDetail{Provider: PagesSourceTypeRemoteURL, Label: "deleted.zip"},
`{"provider":"remote_url","label":"deleted.zip"}`,
"user:1",
manifest,
upload.IngestResult{},
false,
)
if !errors.Is(err, errSourceFinalFence) {
t.Errorf("commitSourceDeployment(deleted upload) error = %v, want %v", err, errSourceFinalFence)
}
storedProject, err := model.GetPagesProjectByID(ctx, project.ID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
}
if storedProject.ActiveDeploymentID != nil {
t.Errorf("ActiveDeploymentID after deleted upload rejection = %v, want nil", storedProject.ActiveDeploymentID)
}
var activeCount int64
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
Where("project_id = ? AND status = ?", project.ID, model.PagesDeploymentStatusActive).
Count(&activeCount).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
t.Fatalf("count active deployments error = %v, want nil", err)
}
if got, want := activeCount, int64(0); got != want {
t.Errorf("active deployment count after deleted upload rejection = %d, want %d", got, want)
}
}
internal/apps/openflare/pages/source_tasks.go
+275
View File
@@ -0,0 +1,275 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"io"
"strconv"
"strings"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/task"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
)
const (
// PagesSourceActionTask is the private Asynq task type for source actions.
PagesSourceActionTask = "openflare:pages_source_action"
// TaskTypePagesSourceAction is the internal task meta type.
TaskTypePagesSourceAction = "of_pages_source_action"
sourceActionCheck = "check"
sourceActionSync = "sync"
)
var errUnexpectedJSONTrailingValue = errors.New("unexpected trailing JSON value")
// PagesSourceActionMeta is intentionally hidden from generic Admin task APIs.
var PagesSourceActionMeta = task.TaskMeta{
Type: TaskTypePagesSourceAction,
AsynqTask: PagesSourceActionTask,
Name: "OpenFlare Pages 部署源操作",
Description: "检查或同步 Pages 项目部署源",
SupportsTime: false,
MaxRetry: 2,
Queue: task.QueueDefault,
Retryable: false,
InternalOnly: true,
}
// SourceActionPayload is the credential-free internal queue contract.
type SourceActionPayload struct {
SourceID uint `json:"source_id"`
ConfigVersion int `json:"config_version"`
Action string `json:"action"`
Actor string `json:"actor"`
TargetRevision string `json:"target_revision"`
ConfirmedRevision string `json:"confirmed_revision"`
}
// SourceActionHandler executes a validated source action.
type SourceActionHandler struct{}
// ValidatePayload rejects unknown keys and normalizes the internal contract.
func (h *SourceActionHandler) ValidatePayload(payload []byte) ([]byte, error) {
var input SourceActionPayload
decoder := json.NewDecoder(bytes.NewReader(payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&input); err != nil {
return nil, errors.New(errPagesSourceActionInvalid)
}
if err := ensureJSONEOF(decoder); err != nil {
return nil, errors.New(errPagesSourceActionInvalid)
}
input.Action = strings.TrimSpace(input.Action)
input.Actor = strings.TrimSpace(input.Actor)
input.TargetRevision = strings.TrimSpace(input.TargetRevision)
input.ConfirmedRevision = strings.TrimSpace(input.ConfirmedRevision)
if input.SourceID == 0 || input.ConfigVersion <= 0 ||
(input.Action != sourceActionCheck && input.Action != sourceActionSync) ||
!validPagesSourceActor(input.Actor) ||
!validOptionalSourceRevision(input.TargetRevision) ||
!validOptionalSourceRevision(input.ConfirmedRevision) {
return nil, errors.New(errPagesSourceActionInvalid)
}
return json.Marshal(input)
}
// Execute validates again inside the worker and performs the source action.
func (h *SourceActionHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
normalized, err := h.ValidatePayload(payload)
if err != nil {
return nil, task.PermanentError(errPagesSourceActionInvalid)
}
var input SourceActionPayload
if err := json.Unmarshal(normalized, &input); err != nil {
return nil, task.PermanentError(errPagesSourceActionInvalid)
}
var source model.PagesProjectSource
if err := db.DB(ctx).Where("id = ?", input.SourceID).First(&source).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
task.AppendLog(ctx, "[resolve] 来源已不存在，本次任务跳过")
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
}
logger.ErrorF(ctx, "[PagesSource] load source failed: source_id=%d error=%v", input.SourceID, err)
return nil, errors.New(errPagesSourceSyncFailed)
}
if source.ConfigVersion != input.ConfigVersion {
task.AppendLog(ctx, "[resolve] 来源配置已变化，本次任务跳过")
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
}
if input.Action == sourceActionCheck && source.SourceType == PagesSourceTypeRemoteURL {
return nil, task.PermanentError(errPagesSourceCheckUnsupported)
}
if source.SourceType != PagesSourceTypeRemoteURL {
return nil, task.PermanentError(errPagesSourceTypeUnsupported)
}
if input.TargetRevision != "" || input.ConfirmedRevision != "" {
return nil, task.PermanentError(errPagesSourceActionInvalid)
}
task.AppendLog(ctx, "[resolve] 正在获取来源执行权")
snapshot, outcome, err := acquireSourceLease(ctx, input.SourceID, input.ConfigVersion, input.Action)
if err != nil {
logger.ErrorF(ctx, "[PagesSource] acquire lease failed: source_id=%d error=%v", input.SourceID, err)
return nil, errors.New(errPagesSourceSyncFailed)
}
switch outcome {
case sourceLeaseBusy:
task.AppendLog(ctx, "[resolve] 已有来源任务正在执行，本次任务跳过")
return &task.TaskResult{Message: errPagesSourceActionBusy}, nil
case sourceLeaseStale:
task.AppendLog(ctx, "[resolve] 来源配置或执行权已变化，本次任务跳过")
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
}
result, err := syncRemoteSource(ctx, snapshot, input.Actor)
if err != nil {
logger.ErrorF(ctx, "[PagesSource] sync failed: project_id=%d source_id=%d error=%v", snapshot.ProjectID, snapshot.SourceID, err)
if isPermanentSourceSyncError(err) {
return nil, task.PermanentError(errPagesSourceSyncFailed)
}
return nil, errors.New(errPagesSourceSyncFailed)
}
if result == nil || result.Stale {
task.AppendLog(ctx, "[activate] 来源配置或执行权已变化，本次任务未切换部署")
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
}
message := "Pages 部署源同步并发布成功"
if result.Reused {
message = "Pages 部署源内容未变化，已重新激活现有部署"
}
return &task.TaskResult{Message: message, Detail: sourceSyncResultDetail(result)}, nil
}
func ensureJSONEOF(decoder *json.Decoder) error {
var trailing any
err := decoder.Decode(&trailing)
if errors.Is(err, io.EOF) {
return nil
}
if err == nil {
return errUnexpectedJSONTrailingValue
}
return err
}
func validPagesSourceActor(actor string) bool {
if actor == pagesSourceCreatedBySystem {
return true
}
if !strings.HasPrefix(actor, "user:") {
return false
}
id, err := strconv.ParseUint(strings.TrimPrefix(actor, "user:"), 10, 64)
return err == nil && id > 0
}
func validOptionalSourceRevision(value string) bool {
if value == "" {
return true
}
if len(value) != sourceRevisionHexLength {
return false
}
decoded, err := hex.DecodeString(value)
return err == nil && len(decoded) == 32
}
func isPermanentSourceSyncError(err error) bool {
if err == nil {
return false
}
message := err.Error()
return strings.Contains(message, errPagesPackageUnsupported) ||
strings.Contains(message, errPagesPackageInvalid) ||
strings.Contains(message, errPagesPackageEmpty) ||
strings.Contains(message, errPagesPackageExtractedTooLarge) ||
strings.Contains(message, errPagesPackageFileTooLarge) ||
strings.Contains(message, errPagesEntryFileMissing) ||
strings.Contains(message, errPagesSourceRemoteURLInvalid) ||
strings.Contains(message, errPagesSourceNetworkPolicy)
}
// DispatchSourceAction performs API preflight and enqueues a credential-free action.
func DispatchSourceAction(
ctx context.Context,
projectID uint,
action string,
actor string,
confirmedRevision string,
) (*SourceActionReceipt, error) {
action = strings.TrimSpace(action)
confirmedRevision = strings.TrimSpace(confirmedRevision)
if action != sourceActionCheck && action != sourceActionSync {
return nil, errors.New(errPagesSourceActionInvalid)
}
if !validPagesSourceActor(actor) {
return nil, errors.New(errPagesSourceActionInvalid)
}
var source model.PagesProjectSource
if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, errors.New(errPagesSourceNotFound)
}
return nil, err
}
if source.SourceType != PagesSourceTypeRemoteURL {
return nil, errors.New(errPagesSourceTypeUnsupported)
}
if action == sourceActionCheck {
return nil, errors.New(errPagesSourceCheckUnsupported)
}
if confirmedRevision != "" {
return nil, errors.New(errPagesSourceActionInvalid)
}
busy, err := sourceLeaseIsBusy(ctx, source.ID)
if err != nil {
return nil, err
}
if busy {
return nil, errors.New(errPagesSourceActionBusy)
}
handler := &SourceActionHandler{}
rawPayload, err := json.Marshal(SourceActionPayload{
SourceID: source.ID,
ConfigVersion: source.ConfigVersion,
Action: action,
Actor: actor,
TargetRevision: "",
ConfirmedRevision: confirmedRevision,
})
if err != nil {
return nil, errors.New(errPagesSourceActionInvalid)
}
payload, err := handler.ValidatePayload(rawPayload)
if err != nil {
return nil, err
}
taskID, err := task.DispatchTask(ctx, TaskTypePagesSourceAction, payload, "manual")
if err != nil {
logger.ErrorF(ctx, "[PagesSource] dispatch action failed: project_id=%d source_id=%d action=%s error=%v", projectID, source.ID, action, err)
return nil, errors.New(errPagesSourceTaskDispatchFailed)
}
execution, err := model.GetTaskExecutionByTaskID(ctx, taskID)
if err != nil {
logger.ErrorF(ctx, "[PagesSource] load dispatched execution failed: source_id=%d task_id=%s error=%v", source.ID, taskID, err)
return nil, errors.New(errPagesSourceTaskDispatchFailed)
}
return &SourceActionReceipt{
TaskID: taskID,
ExecutionID: strconv.FormatUint(execution.ID, 10),
Action: action,
}, nil
}
internal/apps/openflare/pages/source_tasks_test.go
+166
View File
@@ -0,0 +1,166 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"encoding/json"
"errors"
"strings"
"testing"
"github.com/hibiken/asynq"
)
func TestSourceActionPayloadValidationIsStrictAndCredentialFree(t *testing.T) {
handler := &SourceActionHandler{}
valid := SourceActionPayload{
SourceID: 7,
ConfigVersion: 3,
Action: sourceActionSync,
Actor: "user:42",
}
raw, err := json.Marshal(valid)
if err != nil {
t.Fatalf("json.Marshal(valid payload) error = %v, want nil", err)
}
normalized, err := handler.ValidatePayload(raw)
if err != nil {
t.Fatalf("ValidatePayload(valid) error = %v, want nil", err)
}
var got SourceActionPayload
if err := json.Unmarshal(normalized, &got); err != nil {
t.Fatalf("json.Unmarshal(normalized payload) error = %v, want nil", err)
}
if got != valid {
t.Errorf("ValidatePayload(valid) = %+v, want %+v", got, valid)
}
for _, forbidden := range []string{"remote_url", "content_config_version", "expected_revision", "lease_token", "etag"} {
if strings.Contains(string(normalized), forbidden) {
t.Errorf("normalized payload = %s, want no forbidden field %q", normalized, forbidden)
}
}
invalidPayloads := []struct {
name string
raw string
}{
{
name: "unknown remote URL field",
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42","remote_url":"https://example.com/site.zip?token=secret"}`,
},
{
name: "unknown content version field",
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42","content_config_version":9}`,
},
{
name: "empty actor",
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":""}`,
},
{
name: "untrusted system actor",
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"system"}`,
},
{
name: "zero user actor",
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:0"}`,
},
{
name: "invalid action",
raw: `{"source_id":7,"config_version":3,"action":"activate","actor":"user:42"}`,
},
{
name: "multiple JSON values",
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42"} {}`,
},
}
for _, test := range invalidPayloads {
t.Run(test.name, func(t *testing.T) {
normalized, err := handler.ValidatePayload([]byte(test.raw))
if err == nil {
t.Errorf("ValidatePayload(%s) = %s, nil; want non-nil error", test.raw, normalized)
}
if err != nil && strings.Contains(err.Error(), "secret") {
t.Errorf("ValidatePayload(%s) error = %q, want credential-free error", test.name, err)
}
})
}
}
func TestSourceActionPayloadAcceptsOnlyRealActors(t *testing.T) {
tests := []struct {
actor string
want bool
}{
{actor: "user:1", want: true},
{actor: "user:18446744073709551615", want: true},
{actor: pagesSourceCreatedBySystem, want: true},
{actor: "", want: false},
{actor: "user:0", want: false},
{actor: "user:-1", want: false},
{actor: "user:not-a-number", want: false},
{actor: "system", want: false},
}
for _, test := range tests {
if got := validPagesSourceActor(test.actor); got != test.want {
t.Errorf("validPagesSourceActor(%q) = %t, want %t", test.actor, got, test.want)
}
}
}
func TestRemoteCheckActionIsPermanentWithoutExposingURL(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "task-remote-check")
secret := "task-query-secret"
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip?token="+secret,
RemoteNetworkPolicyPublic,
)
raw, err := json.Marshal(SourceActionPayload{
SourceID: source.ID,
ConfigVersion: source.ConfigVersion,
Action: sourceActionCheck,
Actor: "user:9",
})
if err != nil {
t.Fatalf("json.Marshal(check payload) error = %v, want nil", err)
}
result, err := (&SourceActionHandler{}).Execute(ctx, raw)
if result != nil {
t.Errorf("SourceActionHandler.Execute(remote check) result = %+v, want nil", result)
}
if err == nil {
t.Fatal("SourceActionHandler.Execute(remote check) error = nil, want permanent error")
}
if !errors.Is(err, asynq.SkipRetry) {
t.Errorf("SourceActionHandler.Execute(remote check) error = %v, want errors.Is(asynq.SkipRetry)", err)
}
if got, want := err.Error(), errPagesSourceCheckUnsupported; got != want {
t.Errorf("SourceActionHandler.Execute(remote check) error = %q, want %q", got, want)
}
if strings.Contains(err.Error(), secret) || strings.Contains(string(raw), secret) {
t.Errorf("remote check result error/payload = %q / %s, want no URL secret", err, raw)
}
}
func TestPagesSourceActionMetaIsInternalOnly(t *testing.T) {
if !PagesSourceActionMeta.InternalOnly {
t.Error("PagesSourceActionMeta.InternalOnly = false, want true")
}
if PagesSourceActionMeta.Type != TaskTypePagesSourceAction {
t.Errorf("PagesSourceActionMeta.Type = %q, want %q", PagesSourceActionMeta.Type, TaskTypePagesSourceAction)
}
if PagesSourceActionMeta.AsynqTask != PagesSourceActionTask {
t.Errorf("PagesSourceActionMeta.AsynqTask = %q, want %q", PagesSourceActionMeta.AsynqTask, PagesSourceActionTask)
}
if PagesSourceActionMeta.Retryable {
t.Error("PagesSourceActionMeta.Retryable = true, want false for manual retry API")
}
if PagesSourceActionMeta.MaxRetry <= 0 {
t.Errorf("PagesSourceActionMeta.MaxRetry = %d, want bounded transient retries", PagesSourceActionMeta.MaxRetry)
}
}
internal/apps/openflare/pages/source_test.go
+386
View File
@@ -0,0 +1,386 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pages
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
)
func setupPagesSourceTest(t *testing.T) context.Context {
t.Helper()
cleanup := setupPagesTestDB(t)
t.Cleanup(cleanup)
sqlDB, err := db.DB(t.Context()).DB()
if err != nil {
t.Fatalf("db.DB().DB() error = %v, want nil", err)
}
// SQLite :memory: is scoped to one connection. Keeping one connection also
// makes lease tests exercise the production CAS without creating empty
// per-connection databases.
sqlDB.SetMaxOpenConns(1)
return t.Context()
}
func mustCreatePagesSourceProject(t *testing.T, ctx context.Context, slug string) *model.PagesProject {
t.Helper()
view, err := CreateProject(ctx, Input{
Name: "Source " + slug,
Slug: slug,
Enabled: true,
EntryFile: "index.html",
})
if err != nil {
t.Fatalf("CreateProject(%q) error = %v, want nil", slug, err)
}
project, err := model.GetPagesProjectByID(ctx, view.ID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", view.ID, err)
}
return project
}
func mustConfigureRemoteSource(
t *testing.T,
ctx context.Context,
projectID uint,
remoteURL string,
policy string,
) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime) {
t.Helper()
_, err := UpdateSource(ctx, projectID, SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: remoteURL,
RemoteNetworkPolicy: policy,
})
if err != nil {
t.Fatalf("UpdateSource(%d, %q) error = %v, want nil", projectID, remoteURL, err)
}
var source model.PagesProjectSource
if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil {
t.Fatalf("load source for project %d error = %v, want nil", projectID, err)
}
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
t.Fatalf("load runtime for source %d error = %v, want nil", source.ID, err)
}
return &source, &runtime
}
func TestValidateRemoteSourceInputRejectsModeIncompatibleFields(t *testing.T) {
tests := []struct {
name string
input SourceUpdateInput
}{
{
name: "missing source type",
input: SourceUpdateInput{
RemoteURLSet: true,
RemoteURL: "https://example.com/site.zip",
},
},
{
name: "github type reserved for phase two",
input: SourceUpdateInput{
SourceType: PagesSourceTypeGitHubRelease,
RepositoryURL: "https://github.com/example/site",
},
},
{
name: "remote rejects repository field",
input: SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: "https://example.com/site.zip",
RepositoryURL: "https://github.com/example/site",
},
},
{
name: "remote rejects automatic updates",
input: SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: "https://example.com/site.zip",
AutoUpdateEnabled: true,
},
},
{
name: "url value requires replacement flag",
input: SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURL: "https://example.com/site.zip",
},
},
{
name: "invalid network policy",
input: SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: "https://example.com/site.zip",
RemoteNetworkPolicy: "private",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if err := validateRemoteSourceInput(test.input); err == nil {
t.Errorf("validateRemoteSourceInput(%+v) error = nil, want non-nil", test.input)
}
})
}
}
func TestUpdateSourceNewRemoteRequiresExplicitURL(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "remote-requires-url")
_, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
})
if err == nil {
t.Fatal("UpdateSource(new remote without URL) error = nil, want non-nil")
}
if got, want := err.Error(), errPagesSourceRemoteURLRequired; got != want {
t.Errorf("UpdateSource(new remote without URL) error = %q, want %q", got, want)
}
}
func TestRemoteSourceCRUDPreservesSecretAndResetsRuntimeByIdentity(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "remote-crud")
firstURL := "https://Artifacts.Example.com:443/dist/site.zip?token=first-secret&expires=1"
source, runtime := mustConfigureRemoteSource(t, ctx, project.ID, firstURL, RemoteNetworkPolicyPublic)
if got, want := source.ConfigVersion, 1; got != want {
t.Errorf("new source ConfigVersion = %d, want %d", got, want)
}
if got, want := runtime.SyncStatus, pagesSourceStatusIdle; got != want {
t.Errorf("new runtime SyncStatus = %q, want %q", got, want)
}
view, err := GetSource(ctx, project.ID)
if err != nil {
t.Fatalf("GetSource(%d) error = %v, want nil", project.ID, err)
}
if got, want := view.DisplayURL, "https://Artifacts.Example.com:443/dist/site.zip?***"; got != want {
t.Errorf("GetSource(%d).DisplayURL = %q, want %q", project.ID, got, want)
}
encodedView, err := json.Marshal(view)
if err != nil {
t.Fatalf("json.Marshal(GetSource(%d)) error = %v, want nil", project.ID, err)
}
if strings.Contains(string(encodedView), "first-secret") || strings.Contains(string(encodedView), "expires=1") {
t.Errorf("GetSource(%d) JSON = %s, want credential-free view", project.ID, encodedView)
}
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: firstURL,
RemoteNetworkPolicy: RemoteNetworkPolicyPublic,
}); err != nil {
t.Fatalf("UpdateSource(%d, no-op) error = %v, want nil", project.ID, err)
}
var unchangedSource model.PagesProjectSource
if err := db.DB(ctx).Where("id = ?", source.ID).First(&unchangedSource).Error; err != nil {
t.Fatalf("load no-op source error = %v, want nil", err)
}
if got, want := unchangedSource.ConfigVersion, source.ConfigVersion; got != want {
t.Errorf("no-op source ConfigVersion = %d, want unchanged %d", got, want)
}
seenRevision := strings.Repeat("a", 64)
appliedRevision := strings.Repeat("b", 64)
future := time.Now().Add(time.Hour)
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", source.ID).
Updates(map[string]any{
"last_seen_revision": seenRevision,
"last_seen_detail": `{"provider":"remote_url","label":"new.zip"}`,
"last_applied_revision": appliedRevision,
"last_applied_detail": `{"provider":"remote_url","label":"old.zip"}`,
"sync_status": pagesSourceStatusSyncing,
"lease_token": "in-flight",
"lease_expires_at": &future,
}).Error; err != nil {
t.Fatalf("seed source runtime error = %v, want nil", err)
}
// Omit the secret URL while changing policy. The stored URL and cursor must
// survive, while the in-flight lease is fenced.
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: false,
RemoteNetworkPolicy: RemoteNetworkPolicyTrustedInternal,
}); err != nil {
t.Fatalf("UpdateSource(%d, preserve URL) error = %v, want nil", project.ID, err)
}
preservedSource, preservedRuntime, err := loadSourceByProject(ctx, project.ID)
if err != nil {
t.Fatalf("loadSourceByProject(%d) error = %v, want nil", project.ID, err)
}
if got, want := preservedSource.RemoteURL, firstURL; got != want {
t.Errorf("preserved RemoteURL = %q, want %q", got, want)
}
if got, want := preservedSource.ConfigVersion, 2; got != want {
t.Errorf("preserved source ConfigVersion = %d, want %d", got, want)
}
if got, want := preservedSource.SourceIdentity, source.SourceIdentity; got != want {
t.Errorf("preserved source identity = %q, want %q", got, want)
}
if got, want := preservedRuntime.LastSeenRevision, seenRevision; got != want {
t.Errorf("preserved LastSeenRevision = %q, want %q", got, want)
}
if got, want := preservedRuntime.SyncStatus, pagesSourceStatusUpdateAvailable; got != want {
t.Errorf("preserved runtime SyncStatus = %q, want %q", got, want)
}
if preservedRuntime.LeaseToken != "" || preservedRuntime.LeaseExpiresAt != nil {
t.Errorf("preserved runtime lease = (%q, %v), want cleared", preservedRuntime.LeaseToken, preservedRuntime.LeaseExpiresAt)
}
// Replacing only the query secret keeps the canonical identity and cursors.
queryReplacementURL := "https://artifacts.example.com/dist/site.zip?token=second-secret"
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: queryReplacementURL,
RemoteNetworkPolicy: RemoteNetworkPolicyTrustedInternal,
}); err != nil {
t.Fatalf("UpdateSource(%d, query replacement) error = %v, want nil", project.ID, err)
}
querySource, queryRuntime, err := loadSourceByProject(ctx, project.ID)
if err != nil {
t.Fatalf("loadSourceByProject(%d) after query replacement error = %v, want nil", project.ID, err)
}
if got, want := querySource.SourceIdentity, source.SourceIdentity; got != want {
t.Errorf("query replacement identity = %q, want %q", got, want)
}
if got, want := queryRuntime.LastSeenRevision, seenRevision; got != want {
t.Errorf("query replacement LastSeenRevision = %q, want %q", got, want)
}
// Replacing the path changes identity and clears all remote cursors.
pathReplacementURL := "https://artifacts.example.com/dist/other.zip?token=third-secret"
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
SourceType: PagesSourceTypeRemoteURL,
RemoteURLSet: true,
RemoteURL: pathReplacementURL,
RemoteNetworkPolicy: RemoteNetworkPolicyTrustedInternal,
}); err != nil {
t.Fatalf("UpdateSource(%d, path replacement) error = %v, want nil", project.ID, err)
}
pathSource, pathRuntime, err := loadSourceByProject(ctx, project.ID)
if err != nil {
t.Fatalf("loadSourceByProject(%d) after path replacement error = %v, want nil", project.ID, err)
}
if pathSource.SourceIdentity == source.SourceIdentity {
t.Errorf("path replacement identity = %q, want a new identity", pathSource.SourceIdentity)
}
if pathRuntime.LastSeenRevision != "" || pathRuntime.LastAppliedRevision != "" {
t.Errorf("path replacement cursors = (%q, %q), want empty", pathRuntime.LastSeenRevision, pathRuntime.LastAppliedRevision)
}
if got, want := pathRuntime.SyncStatus, pagesSourceStatusIdle; got != want {
t.Errorf("path replacement SyncStatus = %q, want %q", got, want)
}
pathView, err := GetSource(ctx, project.ID)
if err != nil {
t.Fatalf("GetSource(%d) after path replacement error = %v, want nil", project.ID, err)
}
pathJSON, err := json.Marshal(pathView)
if err != nil {
t.Fatalf("json.Marshal(path view) error = %v, want nil", err)
}
for _, secret := range []string{"first-secret", "second-secret", "third-secret"} {
if strings.Contains(string(pathJSON), secret) {
t.Errorf("path view JSON = %s, want no secret %q", pathJSON, secret)
}
}
}
func TestRemoteSourceIdentityIgnoresQueryAndNormalizesDefaultPort(t *testing.T) {
first, err := parseRemoteSourceURL("HTTPS://Artifacts.Example.com:443/dist/../dist/site.zip?token=one")
if err != nil {
t.Fatalf("parseRemoteSourceURL(first) error = %v, want nil", err)
}
second, err := parseRemoteSourceURL("https://artifacts.example.com/dist/site.zip?token=two")
if err != nil {
t.Fatalf("parseRemoteSourceURL(second) error = %v, want nil", err)
}
if got, want := remoteSourceIdentity(first), remoteSourceIdentity(second); got != want {
t.Errorf("remoteSourceIdentity(first) = %q, want %q", got, want)
}
}
func TestDeleteSourceIsIdempotentAndKeepsDeploymentState(t *testing.T) {
ctx := setupPagesSourceTest(t)
project := mustCreatePagesSourceProject(t, ctx, "source-delete")
source, _ := mustConfigureRemoteSource(
t,
ctx,
project.ID,
"https://example.com/site.zip?token=delete-secret",
RemoteNetworkPolicyPublic,
)
deployment := &model.PagesDeployment{
ProjectID: project.ID,
DeploymentNumber: 1,
Checksum: strings.Repeat("c", 64),
Status: model.PagesDeploymentStatusActive,
CreatedBy: "user:1",
SourceType: "manual_upload",
TriggerType: "manual_upload",
}
if err := db.DB(ctx).Create(deployment).Error; err != nil {
t.Fatalf("create deployment error = %v, want nil", err)
}
if err := db.DB(ctx).Model(&model.PagesProject{}).
Where("id = ?", project.ID).
Update("active_deployment_id", deployment.ID).Error; err != nil {
t.Fatalf("set active deployment error = %v, want nil", err)
}
for attempt := 1; attempt <= 2; attempt++ {
view, err := DeleteSource(ctx, project.ID)
if err != nil {
t.Fatalf("DeleteSource(%d), attempt %d error = %v, want nil", project.ID, attempt, err)
}
if got, want := view.SourceType, PagesSourceTypeManual; got != want {
t.Errorf("DeleteSource(%d), attempt %d SourceType = %q, want %q", project.ID, attempt, got, want)
}
}
var sourceCount, runtimeCount, deploymentCount int64
if err := db.DB(ctx).Model(&model.PagesProjectSource{}).Where("id = ?", source.ID).Count(&sourceCount).Error; err != nil {
t.Fatalf("count source error = %v, want nil", err)
}
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).Where("source_id = ?", source.ID).Count(&runtimeCount).Error; err != nil {
t.Fatalf("count runtime error = %v, want nil", err)
}
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("id = ?", deployment.ID).Count(&deploymentCount).Error; err != nil {
t.Fatalf("count deployment error = %v, want nil", err)
}
if sourceCount != 0 || runtimeCount != 0 || deploymentCount != 1 {
t.Errorf("DeleteSource counts = source:%d runtime:%d deployment:%d, want 0, 0, 1", sourceCount, runtimeCount, deploymentCount)
}
storedProject, err := model.GetPagesProjectByID(ctx, project.ID)
if err != nil {
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
}
if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != deployment.ID {
t.Errorf("active deployment = %v, want %d", storedProject.ActiveDeploymentID, deployment.ID)
}
manual, err := GetSource(ctx, project.ID)
if err != nil {
t.Fatalf("GetSource(%d) after delete error = %v, want nil", project.ID, err)
}
if got, want := fmt.Sprint(manual.SourceType), PagesSourceTypeManual; got != want {
t.Errorf("GetSource(%d).SourceType = %q, want %q", project.ID, got, want)
}
}
internal/db/migrator/goose/postgres/202607190001_add_pages_source_runtime.sql
+82
View File
@@ -0,0 +1,82 @@
-- +goose Up
ALTER TABLE of_pages_projects
ADD COLUMN IF NOT EXISTS content_config_version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE of_pages_deployments
ADD COLUMN IF NOT EXISTS source_type VARCHAR(32) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS source_identity CHAR(64),
ADD COLUMN IF NOT EXISTS source_revision CHAR(64),
ADD COLUMN IF NOT EXISTS source_label VARCHAR(255) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS source_meta TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS trigger_type VARCHAR(32) NOT NULL DEFAULT '';
UPDATE of_pages_deployments
SET source_type = 'manual_upload',
trigger_type = 'manual_upload';
CREATE TABLE IF NOT EXISTS of_pages_project_sources (
id BIGSERIAL PRIMARY KEY,
project_id BIGINT NOT NULL,
source_type VARCHAR(32) NOT NULL DEFAULT '',
remote_url TEXT NOT NULL DEFAULT '',
remote_network_policy VARCHAR(32) NOT NULL DEFAULT '',
github_repository VARCHAR(255) NOT NULL DEFAULT '',
release_selector VARCHAR(16) NOT NULL DEFAULT '',
release_tag VARCHAR(255) NOT NULL DEFAULT '',
asset_name VARCHAR(255) NOT NULL DEFAULT '',
auto_update_enabled BOOLEAN NOT NULL DEFAULT FALSE,
check_interval_minutes INTEGER NOT NULL DEFAULT 0,
config_version INTEGER NOT NULL DEFAULT 0,
source_identity CHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_project_sources_project_id
ON of_pages_project_sources (project_id);
CREATE TABLE IF NOT EXISTS of_pages_project_source_runtime (
source_id BIGINT PRIMARY KEY,
etag VARCHAR(512) NOT NULL DEFAULT '',
last_seen_revision CHAR(64) NOT NULL DEFAULT '',
last_seen_detail TEXT NOT NULL DEFAULT '',
last_applied_revision CHAR(64) NOT NULL DEFAULT '',
last_applied_detail TEXT NOT NULL DEFAULT '',
sync_status VARCHAR(32) NOT NULL DEFAULT '',
last_error TEXT NOT NULL DEFAULT '',
last_checked_at TIMESTAMPTZ,
last_synced_at TIMESTAMPTZ,
next_check_at TIMESTAMPTZ,
lease_expires_at TIMESTAMPTZ,
lease_token VARCHAR(64) NOT NULL DEFAULT '',
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_pages_project_source_runtime_next_check_at
ON of_pages_project_source_runtime (next_check_at);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_project_number
ON of_pages_deployments (project_id, deployment_number);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_source_revision
ON of_pages_deployments (project_id, source_identity, source_revision)
WHERE source_identity IS NOT NULL AND source_revision IS NOT NULL;
-- +goose Down
DROP INDEX IF EXISTS idx_of_pages_deployments_source_revision;
DROP INDEX IF EXISTS idx_of_pages_deployments_project_number;
DROP INDEX IF EXISTS idx_of_pages_project_source_runtime_next_check_at;
DROP TABLE IF EXISTS of_pages_project_source_runtime;
DROP INDEX IF EXISTS idx_of_pages_project_sources_project_id;
DROP TABLE IF EXISTS of_pages_project_sources;
ALTER TABLE of_pages_deployments
DROP COLUMN IF EXISTS trigger_type,
DROP COLUMN IF EXISTS source_meta,
DROP COLUMN IF EXISTS source_label,
DROP COLUMN IF EXISTS source_revision,
DROP COLUMN IF EXISTS source_identity,
DROP COLUMN IF EXISTS source_type;
ALTER TABLE of_pages_projects
DROP COLUMN IF EXISTS content_config_version;
internal/db/migrator/goose/sqlite/202607190001_add_pages_source_runtime.sql
+151
View File
@@ -0,0 +1,151 @@
-- +goose Up
ALTER TABLE of_pages_projects
ADD COLUMN content_config_version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE of_pages_deployments
ADD COLUMN source_type TEXT NOT NULL DEFAULT '';
ALTER TABLE of_pages_deployments
ADD COLUMN source_identity TEXT;
ALTER TABLE of_pages_deployments
ADD COLUMN source_revision TEXT;
ALTER TABLE of_pages_deployments
ADD COLUMN source_label TEXT NOT NULL DEFAULT '';
ALTER TABLE of_pages_deployments
ADD COLUMN source_meta TEXT NOT NULL DEFAULT '';
ALTER TABLE of_pages_deployments
ADD COLUMN trigger_type TEXT NOT NULL DEFAULT '';
UPDATE of_pages_deployments
SET source_type = 'manual_upload',
trigger_type = 'manual_upload';
CREATE TABLE IF NOT EXISTS of_pages_project_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
project_id INTEGER NOT NULL,
source_type TEXT NOT NULL DEFAULT '',
remote_url TEXT NOT NULL DEFAULT '',
remote_network_policy TEXT NOT NULL DEFAULT '',
github_repository TEXT NOT NULL DEFAULT '',
release_selector TEXT NOT NULL DEFAULT '',
release_tag TEXT NOT NULL DEFAULT '',
asset_name TEXT NOT NULL DEFAULT '',
auto_update_enabled INTEGER NOT NULL DEFAULT 0,
check_interval_minutes INTEGER NOT NULL DEFAULT 0,
config_version INTEGER NOT NULL DEFAULT 0,
source_identity TEXT NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_project_sources_project_id
ON of_pages_project_sources (project_id);
CREATE TABLE IF NOT EXISTS of_pages_project_source_runtime (
source_id INTEGER PRIMARY KEY,
etag TEXT NOT NULL DEFAULT '',
last_seen_revision TEXT NOT NULL DEFAULT '',
last_seen_detail TEXT NOT NULL DEFAULT '',
last_applied_revision TEXT NOT NULL DEFAULT '',
last_applied_detail TEXT NOT NULL DEFAULT '',
sync_status TEXT NOT NULL DEFAULT '',
last_error TEXT NOT NULL DEFAULT '',
last_checked_at DATETIME,
last_synced_at DATETIME,
next_check_at DATETIME,
lease_expires_at DATETIME,
lease_token TEXT NOT NULL DEFAULT '',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_pages_project_source_runtime_next_check_at
ON of_pages_project_source_runtime (next_check_at);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_project_number
ON of_pages_deployments (project_id, deployment_number);
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_source_revision
ON of_pages_deployments (project_id, source_identity, source_revision)
WHERE source_identity IS NOT NULL AND source_revision IS NOT NULL;
-- +goose Down
DROP INDEX IF EXISTS idx_of_pages_deployments_source_revision;
DROP INDEX IF EXISTS idx_of_pages_deployments_project_number;
DROP INDEX IF EXISTS idx_of_pages_project_source_runtime_next_check_at;
DROP TABLE IF EXISTS of_pages_project_source_runtime;
DROP INDEX IF EXISTS idx_of_pages_project_sources_project_id;
DROP TABLE IF EXISTS of_pages_project_sources;
-- SQLite 的 Down 必须重建受影响表，完整移除新增列并保留原有数据与索引。
CREATE TABLE of_pages_deployments_before_source (
id INTEGER PRIMARY KEY AUTOINCREMENT,
project_id INTEGER NOT NULL,
deployment_number INTEGER NOT NULL,
checksum TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'uploaded',
upload_id INTEGER NOT NULL DEFAULT 0,
artifact_path TEXT NOT NULL,
file_count INTEGER NOT NULL DEFAULT 0,
total_size INTEGER NOT NULL DEFAULT 0,
created_by TEXT NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
activated_at DATETIME
);
INSERT INTO of_pages_deployments_before_source (
id, project_id, deployment_number, checksum, status, upload_id, artifact_path,
file_count, total_size, created_by, created_at, activated_at
)
SELECT
id, project_id, deployment_number, checksum, status, upload_id, artifact_path,
file_count, total_size, created_by, created_at, activated_at
FROM of_pages_deployments;
DROP TABLE of_pages_deployments;
ALTER TABLE of_pages_deployments_before_source RENAME TO of_pages_deployments;
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_project_id
ON of_pages_deployments (project_id);
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_checksum
ON of_pages_deployments (checksum);
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_status
ON of_pages_deployments (status);
CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_upload_id
ON of_pages_deployments (upload_id);
CREATE TABLE of_pages_projects_before_source (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
slug TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
enabled INTEGER NOT NULL DEFAULT 1,
spa_fallback_enabled INTEGER NOT NULL DEFAULT 0,
spa_fallback_path TEXT NOT NULL DEFAULT '/index.html',
api_proxy_enabled INTEGER NOT NULL DEFAULT 0,
api_proxy_path TEXT NOT NULL DEFAULT '',
api_proxy_pass TEXT NOT NULL DEFAULT '',
api_proxy_rewrite TEXT NOT NULL DEFAULT '',
active_deployment_id INTEGER,
root_dir TEXT NOT NULL DEFAULT '',
entry_file TEXT NOT NULL DEFAULT 'index.html',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO of_pages_projects_before_source (
id, name, slug, description, enabled, spa_fallback_enabled, spa_fallback_path,
api_proxy_enabled, api_proxy_path, api_proxy_pass, api_proxy_rewrite,
active_deployment_id, root_dir, entry_file, created_at, updated_at
)
SELECT
id, name, slug, description, enabled, spa_fallback_enabled, spa_fallback_path,
api_proxy_enabled, api_proxy_path, api_proxy_pass, api_proxy_rewrite,
active_deployment_id, root_dir, entry_file, created_at, updated_at
FROM of_pages_projects;
DROP TABLE of_pages_projects;
ALTER TABLE of_pages_projects_before_source RENAME TO of_pages_projects;
CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_projects_slug
ON of_pages_projects (slug);
CREATE INDEX IF NOT EXISTS idx_of_pages_projects_active_deployment_id
ON of_pages_projects (active_deployment_id);
internal/db/migrator/migrator_test.go
+3 -3
View File
@@ -18,9 +18,9 @@ import (
"gorm.io/gorm"
)
// expectedMigratedSystemConfigCount 包含初始 32 项系统配置，以及 202606220004
// 从 of_options 迁移过来的 48 项业务配置（OpenFlare/UptimeKuma/OpenResty）。
const expectedMigratedSystemConfigCount = 80
// expectedMigratedSystemConfigCount 包含初始 32 项系统配置、202606220004
// 从 of_options 迁移过来的 48 项业务配置，以及 Pages 的 2 项业务配置。
const expectedMigratedSystemConfigCount = 82
func TestMigrateInitializesSQLiteDatabase(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
internal/db/migrator/pages_source_migration_test.go
+322
View File
@@ -0,0 +1,322 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"database/sql"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/glebarez/sqlite"
"github.com/pressly/goose/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/postgres"
"gorm.io/gorm"
)
const (
pagesSourcePreviousMigration = int64(202607180001)
pagesSourceMigration = int64(202607190001)
pagesMigrationProjectID = uint(900001)
pagesMigrationDeploymentID = uint(900001)
)
func TestPagesSourceMigrationSQLiteUpDownUp(t *testing.T) {
dbPath := t.TempDir() + "/pages-source-migration.db"
gormDB, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
sqlDB, err := gormDB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
runPagesSourceMigrationUpDownUp(t, gormDB, sqlDB, dialectSqlite, "goose/sqlite")
var indexSQL string
require.NoError(t, gormDB.Raw(
"SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?",
"idx_of_pages_deployments_source_revision",
).Scan(&indexSQL).Error)
assert.Contains(t, strings.ToUpper(indexSQL), "WHERE SOURCE_IDENTITY IS NOT NULL AND SOURCE_REVISION IS NOT NULL")
}
func TestPagesSourceMigrationPostgresUpDownUp(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("OPENFLARE_TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("OPENFLARE_TEST_POSTGRES_DSN is not set")
}
gormDB, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
sqlDB, err := gormDB.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("pages_source_migration_%d", time.Now().UnixNano())
require.Regexp(t, `^[a-z0-9_]+$`, schema)
require.NoError(t, gormDB.Exec(`CREATE SCHEMA "`+schema+`"`).Error)
require.NoError(t, gormDB.Exec(`SET search_path TO "`+schema+`"`).Error)
t.Cleanup(func() {
assert.NoError(t, gormDB.Exec("SET search_path TO public").Error)
assert.NoError(t, gormDB.Exec(`DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`).Error)
assert.NoError(t, sqlDB.Close())
})
runPagesSourceMigrationUpDownUp(t, gormDB, sqlDB, dialectPostgres, "goose/postgres")
}
func runPagesSourceMigrationUpDownUp(
t *testing.T,
gormDB *gorm.DB,
sqlDB *sql.DB,
dialect string,
dir string,
) {
t.Helper()
goose.SetBaseFS(migrationFS)
require.NoError(t, goose.SetDialect(dialect))
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourcePreviousMigration))
seedPrePagesSourceMigrationData(t, gormDB)
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceMigration))
assertPagesSourceMigrationUp(t, gormDB)
require.NoError(t, goose.DownTo(sqlDB, dir, pagesSourcePreviousMigration))
assertPagesSourceMigrationDown(t, gormDB)
require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceMigration))
assertPagesSourceMigrationUpAgain(t, gormDB)
}
func seedPrePagesSourceMigrationData(t *testing.T, gormDB *gorm.DB) {
t.Helper()
require.NoError(t, gormDB.Exec(`
INSERT INTO of_pages_projects (
id, name, slug, description, enabled, active_deployment_id, root_dir, entry_file
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
pagesMigrationProjectID,
"Migration Site",
"migration-site",
"keep-project-data",
true,
pagesMigrationDeploymentID,
"public",
"home.html",
).Error)
require.NoError(t, gormDB.Exec(`
INSERT INTO of_pages_deployments (
id, project_id, deployment_number, checksum, status, upload_id, artifact_path,
file_count, total_size, created_by
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
pagesMigrationDeploymentID,
pagesMigrationProjectID,
1,
strings.Repeat("a", 64),
model.PagesDeploymentStatusActive,
uint64(700001),
"legacy/package.zip",
2,
int64(128),
"user:1",
).Error)
}
func assertPagesSourceMigrationUp(t *testing.T, gormDB *gorm.DB) {
t.Helper()
migrator := gormDB.Migrator()
assert.True(t, migrator.HasTable(&model.PagesProjectSource{}))
assert.True(t, migrator.HasTable(&model.PagesProjectSourceRuntime{}))
assert.True(t, migrator.HasColumn(&model.PagesProject{}, "ContentConfigVersion"))
assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceType"))
assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceIdentity"))
assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceRevision"))
assert.True(t, migrator.HasIndex(&model.PagesProjectSource{}, "idx_of_pages_project_sources_project_id"))
assert.True(t, migrator.HasIndex(&model.PagesProjectSourceRuntime{}, "idx_of_pages_project_source_runtime_next_check_at"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_number"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_source_revision"))
var project model.PagesProject
require.NoError(t, gormDB.First(&project, pagesMigrationProjectID).Error)
assert.Equal(t, 0, project.ContentConfigVersion)
assert.Equal(t, "keep-project-data", project.Description)
assert.Equal(t, "public", project.RootDir)
assert.Equal(t, "home.html", project.EntryFile)
var deployment model.PagesDeployment
require.NoError(t, gormDB.First(&deployment, pagesMigrationDeploymentID).Error)
assert.Equal(t, "manual_upload", deployment.SourceType)
assert.Equal(t, "manual_upload", deployment.TriggerType)
assert.Nil(t, deployment.SourceIdentity)
assert.Nil(t, deployment.SourceRevision)
assert.Equal(t, uint64(700001), deployment.UploadID)
sourceID := createMigrationSourceRuntime(t, gormDB)
assertPagesSourceConstraints(t, gormDB, sourceID)
}
func createMigrationSourceRuntime(t *testing.T, gormDB *gorm.DB) uint {
t.Helper()
source := model.PagesProjectSource{
ProjectID: pagesMigrationProjectID,
SourceType: "remote_url",
RemoteURL: "https://example.com/site.zip?token=secret",
RemoteNetworkPolicy: "public",
CheckIntervalMinutes: 0,
ConfigVersion: 1,
SourceIdentity: strings.Repeat("b", 64),
}
require.NoError(t, gormDB.Create(&source).Error)
require.NotZero(t, source.ID)
require.NoError(t, gormDB.Create(&model.PagesProjectSourceRuntime{
SourceID: source.ID,
SyncStatus: "idle",
}).Error)
return source.ID
}
func assertPagesSourceConstraints(t *testing.T, gormDB *gorm.DB, sourceID uint) {
t.Helper()
duplicateSource := model.PagesProjectSource{
ProjectID: pagesMigrationProjectID,
SourceType: "remote_url",
ConfigVersion: 1,
SourceIdentity: strings.Repeat("c", 64),
}
assert.Error(t, gormDB.Create(&duplicateSource).Error)
for number := 2; number <= 3; number++ {
require.NoError(t, createMigrationDeployment(
gormDB,
number,
strings.Repeat(string(rune('a'+number)), 64),
nil,
nil,
))
}
identity := strings.Repeat("d", 64)
revision := strings.Repeat("e", 64)
require.NoError(t, createMigrationDeployment(
gormDB,
4,
strings.Repeat("f", 64),
&identity,
&revision,
))
assert.Error(t, createMigrationDeployment(
gormDB,
5,
strings.Repeat("0", 64),
&identity,
&revision,
))
assert.Error(t, createMigrationDeployment(
gormDB,
1,
strings.Repeat("1", 64),
nil,
nil,
))
var runtime model.PagesProjectSourceRuntime
require.NoError(t, gormDB.First(&runtime, sourceID).Error)
assert.Equal(t, "idle", runtime.SyncStatus)
}
func createMigrationDeployment(
gormDB *gorm.DB,
deploymentNumber int,
checksum string,
identity *string,
revision *string,
) error {
return gormDB.Create(&model.PagesDeployment{
ProjectID: pagesMigrationProjectID,
DeploymentNumber: deploymentNumber,
Checksum: checksum,
Status: model.PagesDeploymentStatusUploaded,
UploadID: uint64(710000 + deploymentNumber),
ArtifactPath: fmt.Sprintf("legacy/%d.zip", deploymentNumber),
SourceType: "manual_upload",
SourceIdentity: identity,
SourceRevision: revision,
TriggerType: "manual_upload",
}).Error
}
func assertPagesSourceMigrationDown(t *testing.T, gormDB *gorm.DB) {
t.Helper()
migrator := gormDB.Migrator()
assert.False(t, migrator.HasTable(&model.PagesProjectSource{}))
assert.False(t, migrator.HasTable(&model.PagesProjectSourceRuntime{}))
assert.False(t, migrator.HasColumn(&model.PagesProject{}, "ContentConfigVersion"))
assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceType"))
assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceIdentity"))
assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceRevision"))
assert.False(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_number"))
assert.False(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_source_revision"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_id"))
assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_upload_id"))
var project struct {
Description string
RootDir string
EntryFile string
ActiveDeploymentID *uint
}
require.NoError(t, gormDB.Table("of_pages_projects").Where("id = ?", pagesMigrationProjectID).Take(&project).Error)
assert.Equal(t, "keep-project-data", project.Description)
assert.Equal(t, "public", project.RootDir)
assert.Equal(t, "home.html", project.EntryFile)
require.NotNil(t, project.ActiveDeploymentID)
assert.Equal(t, pagesMigrationDeploymentID, *project.ActiveDeploymentID)
var deployment struct {
UploadID uint64
ArtifactPath string
FileCount int
TotalSize int64
}
require.NoError(t, gormDB.Table("of_pages_deployments").Where("id = ?", pagesMigrationDeploymentID).Take(&deployment).Error)
assert.Equal(t, uint64(700001), deployment.UploadID)
assert.Equal(t, "legacy/package.zip", deployment.ArtifactPath)
assert.Equal(t, 2, deployment.FileCount)
assert.Equal(t, int64(128), deployment.TotalSize)
var count int64
require.NoError(t, gormDB.Table("of_pages_deployments").Where("project_id = ?", pagesMigrationProjectID).Count(&count).Error)
assert.Equal(t, int64(4), count)
}
func assertPagesSourceMigrationUpAgain(t *testing.T, gormDB *gorm.DB) {
t.Helper()
assert.True(t, gormDB.Migrator().HasTable(&model.PagesProjectSource{}))
assert.True(t, gormDB.Migrator().HasTable(&model.PagesProjectSourceRuntime{}))
assert.True(t, gormDB.Migrator().HasColumn(&model.PagesProject{}, "ContentConfigVersion"))
assert.True(t, gormDB.Migrator().HasColumn(&model.PagesDeployment{}, "SourceRevision"))
var count int64
require.NoError(t, gormDB.Table("of_pages_deployments").
Where("project_id = ? AND source_type = ? AND trigger_type = ?", pagesMigrationProjectID, "manual_upload", "manual_upload").
Count(&count).Error)
assert.Equal(t, int64(4), count)
require.NoError(t, gormDB.Table("of_pages_project_sources").Count(&count).Error)
assert.Zero(t, count, "source config is intentionally removed by Down and is not reconstructable")
}
internal/db/postgres.go
+21 -16
View File
@@ -57,13 +57,10 @@ func initSQLite() {
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", sqlitePath),
attribute.String("db.system", "SQLite"),
),
),
newGORMTracingPlugin([]attribute.KeyValue{
attribute.String("db.instance", sqlitePath),
attribute.String("db.system", "SQLite"),
}),
); err != nil {
log.Fatalf("[SQLite] init trace failed: %v\n", err)
}
@@ -98,15 +95,12 @@ func initPostgres() {
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", dbConfig.Database),
attribute.String("db.ip", dbConfig.Host),
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
attribute.String("db.system", "PostgreSQL"),
),
),
newGORMTracingPlugin([]attribute.KeyValue{
attribute.String("db.instance", dbConfig.Database),
attribute.String("db.ip", dbConfig.Host),
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
attribute.String("db.system", "PostgreSQL"),
}),
); err != nil {
log.Fatalf("[PostgreSQL] init trace failed: %v\n", err)
}
@@ -160,6 +154,17 @@ func initPostgres() {
}
// newGORMTracingPlugin 构造数据库链路追踪插件。查询参数只保留占位符，避免凭据等绑定值进入 Span。
func newGORMTracingPlugin(attrs []attribute.KeyValue, extraOptions ...tracing.Option) gorm.Plugin {
options := []tracing.Option{
tracing.WithoutMetrics(),
tracing.WithoutQueryVariables(),
tracing.WithAttributes(attrs...),
}
options = append(options, extraOptions...)
return tracing.NewPlugin(options...)
}
// buildDSN 构建 PostgreSQL DSN
func buildDSN(host string, port int, username, password string) string {
cfg := config.Config.Database
internal/db/postgres_logger.go
+5
View File
@@ -49,6 +49,11 @@ func (l *gormZapLogger) Error(ctx context.Context, fmt string, args ...interface
}
}
// ParamsFilter 让 GORM 的 Trace 回调只接收参数化 SQL，避免绑定值被 Dialector.Explain 展开到日志。
func (l *gormZapLogger) ParamsFilter(_ context.Context, sql string, _ ...interface{}) (string, []interface{}) {
return sql, nil
}
func (l *gormZapLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
elapsed := time.Since(begin)
switch {
internal/db/postgres_logger_test.go
+76
View File
@@ -4,11 +4,40 @@
package db
import (
"context"
"strings"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
)
type paramsFilterCaptureLogger struct {
filter *gormZapLogger
traces []string
}
func (l *paramsFilterCaptureLogger) LogMode(gormLogger.LogLevel) gormLogger.Interface {
return l
}
func (l *paramsFilterCaptureLogger) Info(context.Context, string, ...interface{}) {}
func (l *paramsFilterCaptureLogger) Warn(context.Context, string, ...interface{}) {}
func (l *paramsFilterCaptureLogger) Error(context.Context, string, ...interface{}) {}
func (l *paramsFilterCaptureLogger) ParamsFilter(ctx context.Context, sql string, params ...interface{}) (string, []interface{}) {
return l.filter.ParamsFilter(ctx, sql, params...)
}
func (l *paramsFilterCaptureLogger) Trace(_ context.Context, _ time.Time, fc func() (string, int64), _ error) {
sql, _ := fc()
l.traces = append(l.traces, sql)
}
func TestParseLogLevel(t *testing.T) {
t.Parallel()
@@ -37,3 +66,50 @@ func TestParseLogLevel(t *testing.T) {
})
}
}
func TestGormZapLoggerParamsFilterDropsBoundValues(t *testing.T) {
t.Parallel()
const (
query = "UPDATE openflare_pages_sources SET remote_url = ? WHERE id = ?"
secret = "https://example.test/release.zip?token=super-secret"
)
filteredSQL, filteredParams := (&gormZapLogger{}).ParamsFilter(t.Context(), query, secret, int64(42))
if filteredSQL != query {
t.Fatalf("ParamsFilter() sql = %q, want %q", filteredSQL, query)
}
if filteredParams != nil {
t.Fatalf("ParamsFilter() params = %#v, want nil", filteredParams)
}
}
func TestGormZapLoggerKeepsParameterizedSQLInTrace(t *testing.T) {
t.Parallel()
const secret = "https://example.test/release.zip?token=trace-secret"
capture := &paramsFilterCaptureLogger{filter: &gormZapLogger{}}
testDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: capture})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := testDB.Exec("CREATE TABLE source_secrets (remote_url TEXT NOT NULL)").Error; err != nil {
t.Fatalf("create table: %v", err)
}
capture.traces = nil
if err := testDB.Exec("INSERT INTO source_secrets (remote_url) VALUES (?)", secret).Error; err != nil {
t.Fatalf("insert source secret: %v", err)
}
if len(capture.traces) != 1 {
t.Fatalf("trace count = %d, want 1", len(capture.traces))
}
traceSQL := capture.traces[0]
if strings.Contains(traceSQL, secret) || strings.Contains(traceSQL, "trace-secret") {
t.Fatalf("trace SQL leaked bound value: %q", traceSQL)
}
if !strings.Contains(traceSQL, "VALUES (?)") {
t.Fatalf("trace SQL = %q, want parameter placeholder", traceSQL)
}
}
internal/db/postgres_tracing_test.go
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"strings"
"testing"
"github.com/glebarez/sqlite"
"go.opentelemetry.io/otel/attribute"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
semconv "go.opentelemetry.io/otel/semconv/v1.30.0"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
"gorm.io/plugin/opentelemetry/tracing"
)
func TestGORMTracingPluginDoesNotRecordQueryVariables(t *testing.T) {
t.Parallel()
const secret = "https://example.test/release.zip?token=otel-secret"
spanRecorder := tracetest.NewSpanRecorder()
tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder))
t.Cleanup(func() {
if err := tracerProvider.Shutdown(context.Background()); err != nil {
t.Errorf("shutdown tracer provider: %v", err)
}
})
testDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
Logger: gormLogger.Default.LogMode(gormLogger.Silent),
})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := testDB.Use(newGORMTracingPlugin(
[]attribute.KeyValue{attribute.String("db.instance", "trace-test")},
tracing.WithTracerProvider(tracerProvider),
)); err != nil {
t.Fatalf("register tracing plugin: %v", err)
}
if err := testDB.Exec("CREATE TABLE source_secrets (remote_url TEXT NOT NULL)").Error; err != nil {
t.Fatalf("create table: %v", err)
}
if err := testDB.Exec("INSERT INTO source_secrets (remote_url) VALUES (?)", secret).Error; err != nil {
t.Fatalf("insert source secret: %v", err)
}
var queryText string
for _, span := range spanRecorder.Ended() {
for _, attr := range span.Attributes() {
if attr.Key == semconv.DBQueryTextKey && strings.Contains(attr.Value.AsString(), "INSERT INTO source_secrets") {
queryText = attr.Value.AsString()
}
}
}
if queryText == "" {
t.Fatal("database query text attribute not found")
}
if strings.Contains(queryText, secret) || strings.Contains(queryText, "otel-secret") {
t.Fatalf("db.query.text leaked bound value: %q", queryText)
}
if !strings.Contains(queryText, "VALUES (?)") {
t.Fatalf("db.query.text = %q, want parameter placeholder", queryText)
}
}
internal/model/openflare_pages.go
+25 -18
View File
@@ -18,22 +18,23 @@ const (
// PagesProject OpenFlare Pages 静态托管项目。
type PagesProject struct {
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
Name string `json:"name" gorm:"size:255;not null"`
Slug string `json:"slug" gorm:"uniqueIndex;size:128;not null"`
Description string `json:"description" gorm:"type:text;not null;default:''"`
Enabled bool `json:"enabled" gorm:"not null;default:true"`
SPAFallbackEnabled bool `json:"spa_fallback_enabled" gorm:"not null;default:false"`
SPAFallbackPath string `json:"spa_fallback_path" gorm:"size:512;not null;default:'/index.html'"`
APIProxyEnabled bool `json:"api_proxy_enabled" gorm:"not null;default:false"`
APIProxyPath string `json:"api_proxy_path" gorm:"size:255;not null;default:''"`
APIProxyPass string `json:"api_proxy_pass" gorm:"size:2048;not null;default:''"`
APIProxyRewrite string `json:"api_proxy_rewrite" gorm:"size:255;not null;default:''"`
ActiveDeploymentID *uint `json:"active_deployment_id" gorm:"index"`
RootDir string `json:"root_dir" gorm:"size:512;not null;default:''"`
EntryFile string `json:"entry_file" gorm:"size:512;not null;default:'index.html'"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
Name string `json:"name" gorm:"size:255;not null"`
Slug string `json:"slug" gorm:"uniqueIndex;size:128;not null"`
Description string `json:"description" gorm:"type:text;not null;default:''"`
Enabled bool `json:"enabled" gorm:"not null;default:true"`
SPAFallbackEnabled bool `json:"spa_fallback_enabled" gorm:"not null;default:false"`
SPAFallbackPath string `json:"spa_fallback_path" gorm:"size:512;not null;default:'/index.html'"`
APIProxyEnabled bool `json:"api_proxy_enabled" gorm:"not null;default:false"`
APIProxyPath string `json:"api_proxy_path" gorm:"size:255;not null;default:''"`
APIProxyPass string `json:"api_proxy_pass" gorm:"size:2048;not null;default:''"`
APIProxyRewrite string `json:"api_proxy_rewrite" gorm:"size:255;not null;default:''"`
ActiveDeploymentID *uint `json:"active_deployment_id" gorm:"index"`
RootDir string `json:"root_dir" gorm:"size:512;not null;default:''"`
EntryFile string `json:"entry_file" gorm:"size:512;not null;default:'index.html'"`
ContentConfigVersion int `json:"-" gorm:"not null;default:0"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
}
// TableName 表名。
@@ -44,8 +45,8 @@ func (PagesProject) TableName() string {
// PagesDeployment OpenFlare Pages 不可变部署记录。
type PagesDeployment struct {
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
ProjectID uint `json:"project_id" gorm:"not null;index"`
DeploymentNumber int `json:"deployment_number" gorm:"not null"`
ProjectID uint `json:"project_id" gorm:"not null;index;uniqueIndex:idx_of_pages_deployments_project_number,priority:1;uniqueIndex:idx_of_pages_deployments_source_revision,priority:1,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"`
DeploymentNumber int `json:"deployment_number" gorm:"not null;uniqueIndex:idx_of_pages_deployments_project_number,priority:2"`
Checksum string `json:"checksum" gorm:"size:64;not null;index"`
Status string `json:"status" gorm:"size:32;not null;default:'uploaded';index"`
UploadID uint64 `json:"upload_id,string" gorm:"not null;default:0;index"`
@@ -53,6 +54,12 @@ type PagesDeployment struct {
FileCount int `json:"file_count" gorm:"not null;default:0"`
TotalSize int64 `json:"total_size" gorm:"not null;default:0"`
CreatedBy string `json:"created_by" gorm:"size:64;not null;default:''"`
SourceType string `json:"source_type" gorm:"size:32;not null;default:''"`
SourceIdentity *string `json:"-" gorm:"type:char(64);uniqueIndex:idx_of_pages_deployments_source_revision,priority:2,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"`
SourceRevision *string `json:"-" gorm:"type:char(64);uniqueIndex:idx_of_pages_deployments_source_revision,priority:3,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"`
SourceLabel string `json:"source_label" gorm:"size:255;not null;default:''"`
SourceMeta string `json:"-" gorm:"type:text;not null;default:''"`
TriggerType string `json:"trigger_type" gorm:"size:32;not null;default:''"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
ActivatedAt *time.Time `json:"activated_at"`
}
internal/model/openflare_pages_source.go
+59
View File
@@ -0,0 +1,59 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package model
import "time"
// PagesProjectSource 保存 Pages 项目的持久部署源配置。
//
// RemoteURL 可能包含签名参数，禁止直接序列化 model；对外接口必须映射到
// pages 包内的脱敏 source view。
type PagesProjectSource struct {
ID uint `json:"-" gorm:"primaryKey;autoIncrement"`
ProjectID uint `json:"-" gorm:"not null;uniqueIndex:idx_of_pages_project_sources_project_id"`
SourceType string `json:"-" gorm:"size:32;not null;default:''"`
RemoteURL string `json:"-" gorm:"type:text;not null;default:''"`
RemoteNetworkPolicy string `json:"-" gorm:"size:32;not null;default:''"`
GitHubRepository string `json:"-" gorm:"column:github_repository;size:255;not null;default:''"`
ReleaseSelector string `json:"-" gorm:"size:16;not null;default:''"`
ReleaseTag string `json:"-" gorm:"size:255;not null;default:''"`
AssetName string `json:"-" gorm:"size:255;not null;default:''"`
AutoUpdateEnabled bool `json:"-" gorm:"not null;default:false"`
CheckIntervalMinutes int `json:"-" gorm:"not null;default:0"`
ConfigVersion int `json:"-" gorm:"not null;default:0"`
SourceIdentity string `json:"-" gorm:"type:char(64);not null;default:''"`
CreatedAt time.Time `json:"-" gorm:"autoCreateTime"`
UpdatedAt time.Time `json:"-" gorm:"autoUpdateTime"`
}
// TableName 返回 Pages 项目部署源配置表名。
func (PagesProjectSource) TableName() string {
return "of_pages_project_sources"
}
// PagesProjectSourceRuntime 保存 Pages 项目部署源的可变运行态。
//
// Runtime 不冗余 project_id；调用方通过 SourceID 关联配置，并在最终提交时
// 同时校验 source config version 与 project content config version。
type PagesProjectSourceRuntime struct {
SourceID uint `json:"-" gorm:"primaryKey;autoIncrement:false"`
ETag string `json:"-" gorm:"column:etag;size:512;not null;default:''"`
LastSeenRevision string `json:"-" gorm:"type:char(64);not null;default:''"`
LastSeenDetail string `json:"-" gorm:"type:text;not null;default:''"`
LastAppliedRevision string `json:"-" gorm:"type:char(64);not null;default:''"`
LastAppliedDetail string `json:"-" gorm:"type:text;not null;default:''"`
SyncStatus string `json:"-" gorm:"size:32;not null;default:''"`
LastError string `json:"-" gorm:"type:text;not null;default:''"`
LastCheckedAt *time.Time `json:"-"`
LastSyncedAt *time.Time `json:"-"`
NextCheckAt *time.Time `json:"-" gorm:"index:idx_of_pages_project_source_runtime_next_check_at"`
LeaseExpiresAt *time.Time `json:"-"`
LeaseToken string `json:"-" gorm:"size:64;not null;default:''"`
UpdatedAt time.Time `json:"-" gorm:"autoUpdateTime"`
}
// TableName 返回 Pages 项目部署源运行态表名。
func (PagesProjectSourceRuntime) TableName() string {
return "of_pages_project_source_runtime"
}
internal/model/openflare_pages_source_test.go
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package model
import (
"encoding/json"
"strings"
"testing"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func TestPagesSourceModelsMatchMigrationSchema(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, gormDB.AutoMigrate(
&PagesProject{},
&PagesDeployment{},
&PagesProjectSource{},
&PagesProjectSourceRuntime{},
))
assert.Equal(t, "of_pages_project_sources", (PagesProjectSource{}).TableName())
assert.Equal(t, "of_pages_project_source_runtime", (PagesProjectSourceRuntime{}).TableName())
assert.True(t, gormDB.Migrator().HasColumn(&PagesProjectSource{}, "github_repository"))
assert.False(t, gormDB.Migrator().HasColumn(&PagesProjectSource{}, "git_hub_repository"))
assert.True(t, gormDB.Migrator().HasColumn(&PagesProjectSourceRuntime{}, "etag"))
assert.False(t, gormDB.Migrator().HasColumn(&PagesProjectSourceRuntime{}, "e_tag"))
var indexSQL string
require.NoError(t, gormDB.Raw(
"SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?",
"idx_of_pages_deployments_source_revision",
).Scan(&indexSQL).Error)
assert.Contains(t, strings.ToUpper(indexSQL), "WHERE SOURCE_IDENTITY IS NOT NULL AND SOURCE_REVISION IS NOT NULL")
}
func TestPagesSourceModelsDoNotSerializeSecretsOrFencingState(t *testing.T) {
sourceJSON, err := json.Marshal(PagesProjectSource{
ID: 1,
ProjectID: 2,
RemoteURL: "https://example.com/site.zip?token=secret",
ConfigVersion: 3,
SourceIdentity: strings.Repeat("a", 64),
})
require.NoError(t, err)
assert.JSONEq(t, `{}`, string(sourceJSON))
runtimeJSON, err := json.Marshal(PagesProjectSourceRuntime{
SourceID: 1,
ETag: `"secret-etag"`,
LeaseToken: "secret-lease",
})
require.NoError(t, err)
assert.JSONEq(t, `{}`, string(runtimeJSON))
identity := strings.Repeat("b", 64)
revision := strings.Repeat("c", 64)
deploymentJSON, err := json.Marshal(PagesDeployment{
SourceType: "remote_url",
SourceIdentity: &identity,
SourceRevision: &revision,
SourceLabel: "site.zip",
SourceMeta: `{"provider":"remote_url","private":"secret"}`,
TriggerType: "manual_sync",
})
require.NoError(t, err)
assert.NotContains(t, string(deploymentJSON), identity)
assert.NotContains(t, string(deploymentJSON), revision)
assert.NotContains(t, string(deploymentJSON), "private")
assert.Contains(t, string(deploymentJSON), `"source_type":"remote_url"`)
assert.Contains(t, string(deploymentJSON), `"source_label":"site.zip"`)
assert.Contains(t, string(deploymentJSON), `"trigger_type":"manual_sync"`)
}
internal/router/v1/openflare/register_pages.go
+5
View File
@@ -18,6 +18,11 @@ func registerPagesRoutes(apiGroup *gin.RouterGroup) {
apiutil.RegisterCollection(pagesRoute, "POST", pages.CreateProjectHandler)
pagesRoute.POST("/:id/update", pages.UpdateProjectHandler)
pagesRoute.POST("/:id/delete", pages.DeleteProjectHandler)
pagesRoute.GET("/:id/source", pages.GetSourceHandler)
pagesRoute.POST("/:id/source/update", pages.UpdateSourceHandler)
pagesRoute.POST("/:id/source/delete", pages.DeleteSourceHandler)
pagesRoute.POST("/:id/source/check", pages.CheckSourceHandler)
pagesRoute.POST("/:id/source/sync", pages.SyncSourceHandler)
pagesRoute.GET("/:id/deployments", pages.ListDeploymentsHandler)
pagesRoute.POST("/:id/deployments/upload", pages.UploadDeploymentHandler)
pagesRoute.POST("/:id/deployments/upload-from-url", pages.UploadDeploymentFromURLHandler)
internal/task/executor.go
+5 -1
View File
@@ -428,7 +428,7 @@ func notifyTaskCompleted(ctx context.Context, execution *model.TaskExecution, re
}
func shouldFlushTaskExecutionLog(ctx context.Context, execErr error) bool {
if execErr == nil {
if isTerminalTaskExecutionError(execErr) {
return true
}
@@ -440,6 +440,10 @@ func shouldFlushTaskExecutionLog(ctx context.Context, execErr error) bool {
return retryCount >= maxRetry
}
func isTerminalTaskExecutionError(execErr error) bool {
return execErr == nil || errors.Is(execErr, asynq.SkipRetry)
}
func handleFailedTask(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, execErr error, span trace.Span) {
execution.Status = model.TaskExecutionStatusFailed
execution.ErrorMessage = execErr.Error()
internal/task/executor_test.go
+51
View File
@@ -6,10 +6,12 @@ package task
import (
"context"
"errors"
"fmt"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/hibiken/asynq"
@@ -284,6 +286,55 @@ func TestCompleteTaskExecutionFlushesLog(t *testing.T) {
assert.Contains(t, found.Log, "任务执行成功")
}
func TestCompleteTaskExecutionFlushesPermanentFailureLog(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
execution := &model.TaskExecution{
TaskID: "complete_permanent_flush_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusRunning,
Retryable: true,
MaxRetry: 3,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
ctx = withTaskID(ctx, execution.TaskID)
AppendLog(ctx, "永久失败前的日志")
execErr := PermanentError("来源配置无效")
finishTime := time.Now()
completeTaskExecution(
ctx,
execution,
asynq.NewTask(testTaskType, nil),
100*time.Millisecond,
finishTime,
nil,
execErr,
trace.SpanFromContext(ctx),
)
found, err := model.GetTaskExecutionByTaskID(ctx, execution.TaskID)
require.NoError(t, err)
assert.Equal(t, model.TaskExecutionStatusFailed, found.Status)
assert.Equal(t, "来源配置无效", found.ErrorMessage)
assert.Contains(t, found.Log, "永久失败前的日志")
assert.Contains(t, found.Log, "任务执行失败")
keys, err := db.Redis.Keys(ctx, "*"+execution.TaskID+"*").Result()
require.NoError(t, err)
assert.Empty(t, keys)
}
func TestPermanentErrorIsTerminalForLogFlush(t *testing.T) {
assert.True(t, isTerminalTaskExecutionError(PermanentError("配置无效")))
assert.False(t, isTerminalTaskExecutionError(errors.New("temporary failure")))
}
func TestRetryTask(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
internal/task/handlers/register.go
+5
View File
@@ -8,6 +8,7 @@ package handlers
import (
"github.com/Rain-kl/Wavelet/internal/apps/admin/push"
"github.com/Rain-kl/Wavelet/internal/apps/openflare"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/pages"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/tls"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/apps/user"
@@ -51,6 +52,10 @@ func Register() {
task.RegisterHandler(openflare.UptimeKumaSyncTask, &openflare.UptimeKumaSyncHandler{})
task.RegisterTaskMeta(openflare.UptimeKumaSyncMeta)
// pages source actions are only dispatched by the Pages domain API/scanner.
task.RegisterHandler(pages.PagesSourceActionTask, &pages.SourceActionHandler{})
task.RegisterTaskMeta(pages.PagesSourceActionMeta)
// tls single renew
task.RegisterHandler(tls.SSLSingleRenewTask, &tls.SSLSingleRenewHandler{})
task.RegisterTaskMeta(tls.SSLSingleRenewMeta)
internal/task/meta.go
+9 -3
View File
@@ -31,6 +31,7 @@ type TaskMeta struct {
MaxRetry int `json:"max_retry"`
Queue string `json:"queue"`
Retryable bool `json:"retryable"` // 是否支持手动重试
InternalOnly bool `json:"-"` // 是否仅允许内部业务入口调度
Params []TaskParam `json:"params,omitempty"`
}
@@ -51,13 +52,18 @@ func RegisterTaskMeta(meta TaskMeta) {
dispatchableTasks = append(dispatchableTasks, meta)
}
// GetDispatchableTasks 获取所有已注册的元数据列表（返回副本以避免并发并发读写冲突）
// GetDispatchableTasks 获取允许通过通用 Admin 入口调度的元数据列表。
func GetDispatchableTasks() []TaskMeta {
dispatchableTasksMutex.RLock()
defer dispatchableTasksMutex.RUnlock()
metas := make([]TaskMeta, len(dispatchableTasks))
copy(metas, dispatchableTasks)
metas := make([]TaskMeta, 0, len(dispatchableTasks))
for _, meta := range dispatchableTasks {
if meta.InternalOnly {
continue
}
metas = append(metas, meta)
}
return metas
}
internal/task/meta_test.go
+25
View File
@@ -29,3 +29,28 @@ func TestDuplicateTaskMeta(t *testing.T) {
}
}
}
func TestInternalOnlyTaskMetaIsHiddenFromDispatchableTasks(t *testing.T) {
const taskType = "test_internal_only_meta"
meta := task.TaskMeta{
Type: taskType,
AsynqTask: "test:internal_only_meta",
Name: "内部测试任务",
InternalOnly: true,
}
task.RegisterTaskMeta(meta)
registered := task.GetTaskMeta(taskType)
if registered == nil {
t.Fatal("GetTaskMeta() did not return internal-only metadata")
}
if !registered.InternalOnly {
t.Fatal("GetTaskMeta() lost InternalOnly flag")
}
for _, dispatchable := range task.GetDispatchableTasks() {
if dispatchable.Type == taskType {
t.Fatalf("GetDispatchableTasks() exposed internal-only task %q", taskType)
}
}
}
internal/task/permanent_error.go
+35
View File
@@ -0,0 +1,35 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"strings"
"github.com/hibiken/asynq"
)
const defaultPermanentErrorMessage = "任务无法继续执行"
type permanentTaskError struct {
message string
}
// PermanentError marks a safe domain message as a non-retryable task failure.
// It intentionally accepts no underlying error so Error never exposes provider,
// URL, header, response-body, or other sensitive implementation details.
func PermanentError(message string) error {
message = strings.TrimSpace(message)
if message == "" {
message = defaultPermanentErrorMessage
}
return &permanentTaskError{message: message}
}
func (e *permanentTaskError) Error() string {
return e.message
}
func (e *permanentTaskError) Unwrap() error {
return asynq.SkipRetry
}
internal/task/permanent_error_test.go
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"errors"
"testing"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/assert"
)
func TestPermanentErrorSkipsRetryWithoutExposingAsynqMessage(t *testing.T) {
err := PermanentError(" 来源配置无效 ")
assert.True(t, errors.Is(err, asynq.SkipRetry))
assert.Equal(t, "来源配置无效", err.Error())
assert.NotContains(t, err.Error(), asynq.SkipRetry.Error())
}
func TestPermanentErrorUsesSafeFallbackForBlankMessage(t *testing.T) {
err := PermanentError(" ")
assert.True(t, errors.Is(err, asynq.SkipRetry))
assert.Equal(t, defaultPermanentErrorMessage, err.Error())
}