mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-10 17:26:38 +08:00
refactor(backend): rename OpenFlare directory to lowercase openflare
This commit is contained in:
ryan
@@ -0,0 +1,56 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"Wavelet/openflare/share/pagesarchive"
|
||||
)
|
||||
|
||||
// downloadPagesPackageFromURL is the deprecated one-shot URL adapter. It uses
|
||||
// the same bounded downloader as persisted sources and allows insecure TLS for
|
||||
// operator-managed internal artifact services.
|
||||
func downloadPagesPackageFromURL(
|
||||
ctx context.Context,
|
||||
rawURL string,
|
||||
maxPackageBytes int64,
|
||||
) (tempPath string, checksum string, size int64, format pagesarchive.Format, fileName string, err error) {
|
||||
if _, err := parseAndValidatePagesDownloadURL(rawURL); err != nil {
|
||||
return "", "", 0, "", "", err
|
||||
}
|
||||
candidate, err := FetchRemoteSource(ctx, RemoteSourceRequest{
|
||||
URL: strings.TrimSpace(rawURL),
|
||||
AllowInsecure: true,
|
||||
MaxPackageBytes: maxPackageBytes,
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), errPagesSourceRemoteURLInvalid) {
|
||||
return "", "", 0, "", "", errors.New(errPagesPackageURLInvalid)
|
||||
}
|
||||
return "", "", 0, "", "", err
|
||||
}
|
||||
// Ownership transfers to the existing one-shot caller, which removes the
|
||||
// temporary file after the candidate deployment has been created.
|
||||
return candidate.TempPath, candidate.Checksum, candidate.PackageSize, candidate.Format, candidate.SafeLabel, nil
|
||||
}
|
||||
|
||||
func parseAndValidatePagesDownloadURL(raw string) (*url.URL, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return nil, errors.New(errPagesPackageURLRequired)
|
||||
}
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || parsed.User != nil || parsed.Fragment != "" || parsed.Opaque != "" {
|
||||
return nil, errors.New(errPagesPackageURLInvalid)
|
||||
}
|
||||
scheme := strings.ToLower(strings.TrimSpace(parsed.Scheme))
|
||||
if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(parsed.Hostname()) == "" {
|
||||
return nil, errors.New(errPagesPackageURLInvalid)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestParseAndValidatePagesDownloadURL(t *testing.T) {
|
||||
_, err := parseAndValidatePagesDownloadURL("")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "填写")
|
||||
|
||||
_, err = parseAndValidatePagesDownloadURL("ftp://example.com/a.zip")
|
||||
require.Error(t, err)
|
||||
|
||||
// Private / local hosts are allowed (operator-controlled artifact hosts).
|
||||
for _, raw := range []string{
|
||||
"http://127.0.0.1/a.zip",
|
||||
"https://localhost/a.zip",
|
||||
"http://192.168.1.10:8080/site.tar.gz",
|
||||
"https://example.com/dist/site.tar.gz?x=1",
|
||||
} {
|
||||
parsed, parseErr := parseAndValidatePagesDownloadURL(raw)
|
||||
require.NoError(t, parseErr, raw)
|
||||
assert.NotEmpty(t, parsed.Hostname(), raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPagesPackageFromURLAllowsPrivateHost(t *testing.T) {
|
||||
var body bytes.Buffer
|
||||
zw := zip.NewWriter(&body)
|
||||
w, err := zw.Create("index.html")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("ok"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
zipBytes := body.Bytes()
|
||||
|
||||
var sawProviderUA bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("User-Agent") == remoteSourceUserAgent {
|
||||
sawProviderUA = true
|
||||
}
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="remote-site.zip"`)
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(zipBytes)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
tempPath, checksum, size, format, fileName, err := downloadPagesPackageFromURL(
|
||||
context.Background(),
|
||||
server.URL+"/pkg.zip",
|
||||
10*1024*1024,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = os.Remove(tempPath) })
|
||||
assert.True(t, sawProviderUA)
|
||||
assert.NotEmpty(t, checksum)
|
||||
assert.Positive(t, size)
|
||||
assert.Equal(t, "zip", string(format))
|
||||
assert.Equal(t, "pkg.zip", fileName)
|
||||
}
|
||||
|
||||
func TestUploadDeploymentFromURLPrivateHost(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
var body bytes.Buffer
|
||||
zw := zip.NewWriter(&body)
|
||||
w, err := zw.Create("index.html")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("from-url"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="from-url.zip"`)
|
||||
_, _ = w.Write(body.Bytes())
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
project, err := CreateProject(ctx, Input{Name: "URL Site", Slug: "url-site", Enabled: true})
|
||||
require.NoError(t, err)
|
||||
|
||||
deployment, err := UploadDeploymentFromURL(ctx, project.ID, server.URL+"/from-url.zip", "root")
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, deployment.ID)
|
||||
assert.Equal(t, 1, deployment.FileCount)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package pages provides logics and management for OpenFlare static page deployments.
|
||||
package pages
|
||||
|
||||
const (
|
||||
errPagesProjectNotFound = "pages 项目不存在"
|
||||
errPagesSlugExists = "pages 项目标识已存在"
|
||||
errPagesNameRequired = "pages 项目名称不能为空"
|
||||
errPagesSlugInvalid = "pages 项目标识只能包含小写字母、数字和连字符"
|
||||
errPagesDeleteReferenced = "pages 项目已被规则引用，不能删除"
|
||||
errPagesDeploymentNotFound = "pages 部署不存在"
|
||||
errPagesDeploymentMismatch = "pages 部署不属于该项目"
|
||||
errPagesDeleteActiveDeploy = "不能删除当前激活的 Pages 部署"
|
||||
errPagesPackageMissing = "缺少 Pages 部署包"
|
||||
errPagesPackageURLRequired = "请填写部署包下载链接"
|
||||
errPagesPackageURLInvalid = "部署包下载链接无效，仅支持 http/https"
|
||||
errPagesPackageURLDownloadFailed = "从链接下载部署包失败"
|
||||
errPagesPackageURLTooLarge = "链接指向的部署包超过大小限制"
|
||||
errPagesPackageNotZip = "pages 部署包必须是 .zip 文件" // legacy alias kept for tests
|
||||
errPagesPackageUnsupported = "pages 部署包仅支持 zip、tar.gz、tar.xz、tar.bz2、tar、7z 格式"
|
||||
errPagesPackageInvalidZip = "pages 部署包不是有效 zip 文件" // legacy alias
|
||||
errPagesPackageInvalid = "pages 部署包不是有效的压缩文件"
|
||||
errPagesPackageEmpty = "pages 部署包不能为空"
|
||||
errPagesPackageExtractedTooLarge = "pages 部署包展开后体积超过限制"
|
||||
errPagesPackageFileTooLarge = "pages 部署包内文件过大"
|
||||
errPagesAPIProxyPathRequired = "启用 API 反代时，匹配路径不能为空"
|
||||
errPagesAPIProxyPathPrefix = "API 反代匹配路径必须以 '/' 开头"
|
||||
errPagesAPIProxyPassRequired = "启用 API 反代时，后端服务地址不能为空" //nolint:gosec // false positive: this is an error message, not hardcoded credentials
|
||||
errPagesAPIProxyPassInvalid = "API 反代后端服务地址必须是有效的 HTTP/HTTPS URL" //nolint:gosec // false positive: this is an error message, not hardcoded credentials
|
||||
errPagesPackagePathEmpty = "pages 部署包路径为空"
|
||||
errPagesPackageUploadMissing = "pages 部署包上传记录不存在"
|
||||
errPagesPackageNotInActiveConfig = "pages 部署尚未进入激活配置"
|
||||
errPagesDeploymentHashMissing = "pages 部署包哈希缺失"
|
||||
errPagesInvalidSnapshotFormat = "配置快照格式无效"
|
||||
errPagesActorMissing = "无法识别当前用户"
|
||||
errPagesEntryFileMissing = "当前激活部署中不存在指定入口文件"
|
||||
errPagesSourceNotFound = "pages 部署源不存在"
|
||||
errPagesSourceTypeRequired = "请选择 pages 部署源类型"
|
||||
errPagesSourceTypeUnsupported = "pages 部署源类型不受支持"
|
||||
errPagesSourceRemoteFields = "远程地址来源不能包含 GitHub 或自动更新配置"
|
||||
errPagesSourceRemoteURLRequired = "请提供远程部署包地址"
|
||||
errPagesSourceRemoteURLInvalid = "远程部署包地址无效，仅支持不含用户信息和片段的 http/https 地址"
|
||||
errPagesSourceGitHubFields = "GitHub Release 来源不能包含远程地址配置"
|
||||
errPagesSourceRepositoryInvalid = "GitHub 仓库地址无效，仅支持 https://github.com/{owner}/{repo}"
|
||||
errPagesSourceSelectorInvalid = "GitHub Release 选择方式无效"
|
||||
errPagesSourceAssetNameInvalid = "GitHub Release 资源名称必须是安全的文件名"
|
||||
errPagesSourceCheckInterval = "GitHub latest 检查间隔必须在 5 到 1440 分钟之间"
|
||||
errPagesSourceAutoNotAvailable = "自动更新将在后续阶段开放，当前必须保持关闭"
|
||||
errPagesSourceReleaseNotFound = "未找到符合配置的 GitHub Release 资源"
|
||||
errPagesSourceDigestInvalid = "GitHub Release 资源摘要格式无效"
|
||||
errPagesSourceDigestMismatch = "GitHub Release 资源摘要校验失败"
|
||||
errPagesSourceConfirmationNeeded = "检测到同一 Release 的资源已被替换，请刷新并确认当前版本"
|
||||
errPagesSourceConfirmationStale = "确认的版本已变化，请刷新后重新确认"
|
||||
errPagesSourceInitialCheckWarning = "部署源已保存，但首次检查任务入队失败，请稍后手动检查"
|
||||
errPagesSourceCheckUnsupported = "远程地址来源不支持检查更新，请使用立即同步"
|
||||
errPagesSourceActionBusy = "pages 部署源任务正在执行"
|
||||
errPagesSourceActionInvalid = "pages 部署源任务参数无效"
|
||||
errPagesSourceActionStale = "pages 部署源配置已变化，本次任务已跳过"
|
||||
errPagesSourceLeaseLost = "pages 部署源任务执行权已失效"
|
||||
errPagesSourceLeaseExpired = "上次 pages 部署源任务租约已过期"
|
||||
errPagesSourceSyncFailed = "pages 部署源同步失败"
|
||||
errPagesSourceTaskDispatchFailed = "pages 部署源任务入队失败"
|
||||
errPagesSourceInternal = "pages 部署源操作失败，请稍后重试"
|
||||
)
|
||||
@@ -0,0 +1,339 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/pkg/logger"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
githubReleaseSelectorLatest = "latest"
|
||||
githubReleaseSelectorTag = "tag"
|
||||
githubSourceIdentityDomain = "openflare:pages:github-release:v2"
|
||||
initialCheckRetryDelay = 5 * time.Minute
|
||||
githubRepositoryPathParts = 2
|
||||
githubCheckJitterRange = 301
|
||||
githubCheckJitterCenter = 150
|
||||
)
|
||||
|
||||
var (
|
||||
githubOwnerPattern = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$`)
|
||||
githubRepoPattern = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
|
||||
)
|
||||
|
||||
type githubSourceConfig struct {
|
||||
Repository string
|
||||
Selector string
|
||||
Tag string
|
||||
AssetName string
|
||||
AutoUpdate bool
|
||||
CheckInterval int
|
||||
SourceIdentity string
|
||||
}
|
||||
|
||||
func validateGitHubSourceInput(input SourceUpdateInput) error {
|
||||
if strings.TrimSpace(input.SourceType) != PagesSourceTypeGitHubRelease {
|
||||
return errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
if strings.TrimSpace(input.RemoteURL) != "" || input.AllowInsecure {
|
||||
return errors.New(errPagesSourceGitHubFields)
|
||||
}
|
||||
if _, err := normalizeGitHubRepositoryURL(input.RepositoryURL); err != nil {
|
||||
return err
|
||||
}
|
||||
selector := strings.TrimSpace(input.ReleaseSelector)
|
||||
if selector == "" {
|
||||
selector = githubReleaseSelectorLatest
|
||||
}
|
||||
assetName := input.AssetName
|
||||
if assetName == "" {
|
||||
assetName = defaultGitHubAssetName
|
||||
}
|
||||
if !validGitHubAssetName(assetName) {
|
||||
return errors.New(errPagesSourceAssetNameInvalid)
|
||||
}
|
||||
switch selector {
|
||||
case githubReleaseSelectorLatest:
|
||||
if input.ReleaseTag != "" {
|
||||
return errors.New(errPagesSourceSelectorInvalid)
|
||||
}
|
||||
interval := input.CheckIntervalMinutes
|
||||
if interval != 0 && (interval < minimumCheckInterval || interval > maximumCheckInterval) {
|
||||
return errors.New(errPagesSourceCheckInterval)
|
||||
}
|
||||
case githubReleaseSelectorTag:
|
||||
if !validGitHubReleaseTagConfig(input.ReleaseTag) || input.AutoUpdateEnabled || input.CheckIntervalMinutes != 0 {
|
||||
return errors.New(errPagesSourceSelectorInvalid)
|
||||
}
|
||||
default:
|
||||
return errors.New(errPagesSourceSelectorInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildGitHubSourceConfig(input SourceUpdateInput) (githubSourceConfig, error) {
|
||||
repository, err := normalizeGitHubRepositoryURL(input.RepositoryURL)
|
||||
if err != nil {
|
||||
return githubSourceConfig{}, err
|
||||
}
|
||||
selector := strings.TrimSpace(input.ReleaseSelector)
|
||||
if selector == "" {
|
||||
selector = githubReleaseSelectorLatest
|
||||
}
|
||||
tag := input.ReleaseTag
|
||||
assetName := input.AssetName
|
||||
if assetName == "" {
|
||||
assetName = defaultGitHubAssetName
|
||||
}
|
||||
interval := input.CheckIntervalMinutes
|
||||
if selector == githubReleaseSelectorLatest && interval == 0 {
|
||||
interval = defaultCheckInterval
|
||||
}
|
||||
autoUpdate := input.AutoUpdateEnabled
|
||||
if selector == githubReleaseSelectorTag {
|
||||
autoUpdate = false
|
||||
interval = 0
|
||||
}
|
||||
return githubSourceConfig{
|
||||
Repository: repository,
|
||||
Selector: selector,
|
||||
Tag: tag,
|
||||
AssetName: assetName,
|
||||
AutoUpdate: autoUpdate,
|
||||
CheckInterval: interval,
|
||||
SourceIdentity: buildGitHubSourceIdentity(repository, selector, tag, assetName),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildGitHubSourceIdentity(repository, selector, tag, assetName string) string {
|
||||
fields := [...]string{repository, selector, tag, assetName}
|
||||
encoded := make([]byte, 0, len(githubSourceIdentityDomain)+len(fields)*8+
|
||||
len(repository)+len(selector)+len(tag)+len(assetName))
|
||||
encoded = append(encoded, githubSourceIdentityDomain...)
|
||||
var fieldLength [8]byte
|
||||
for _, field := range fields {
|
||||
// Go strings hold the validated UTF-8 bytes used by GitHub. Prefixing each
|
||||
// field with its byte length prevents delimiter characters from creating
|
||||
// ambiguous identities across field boundaries.
|
||||
binary.BigEndian.PutUint64(fieldLength[:], uint64(len(field)))
|
||||
encoded = append(encoded, fieldLength[:]...)
|
||||
encoded = append(encoded, field...)
|
||||
}
|
||||
identityHash := sha256.Sum256(encoded)
|
||||
return hex.EncodeToString(identityHash[:])
|
||||
}
|
||||
|
||||
func normalizeGitHubRepositoryURL(raw string) (string, error) {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.Scheme != "https" || !strings.EqualFold(parsed.Host, "github.com") ||
|
||||
parsed.User != nil || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" ||
|
||||
strings.Contains(raw, "#") ||
|
||||
parsed.EscapedPath() != parsed.Path || !strings.HasPrefix(parsed.Path, "/") ||
|
||||
strings.HasPrefix(parsed.Path, "//") || strings.HasSuffix(parsed.Path, "/") {
|
||||
return "", errors.New(errPagesSourceRepositoryInvalid)
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(parsed.Path, "/"), "/")
|
||||
if len(parts) != githubRepositoryPathParts {
|
||||
return "", errors.New(errPagesSourceRepositoryInvalid)
|
||||
}
|
||||
owner := parts[0]
|
||||
repository := parts[1]
|
||||
repository = strings.TrimSuffix(repository, ".git")
|
||||
if !githubOwnerPattern.MatchString(owner) || !githubRepoPattern.MatchString(repository) ||
|
||||
len(repository) > 100 || repository == "." || repository == ".." {
|
||||
return "", errors.New(errPagesSourceRepositoryInvalid)
|
||||
}
|
||||
return owner + "/" + repository, nil
|
||||
}
|
||||
|
||||
func validGitHubReleaseTagConfig(value string) bool {
|
||||
if !validGitHubReleaseDisplayTag(value) ||
|
||||
strings.ContainsAny(value, " ~^:?*[\\") || strings.HasPrefix(value, "/") ||
|
||||
strings.HasSuffix(value, "/") || strings.HasSuffix(value, ".") ||
|
||||
strings.Contains(value, "//") || strings.Contains(value, "..") || strings.Contains(value, "@{") {
|
||||
return false
|
||||
}
|
||||
for component := range strings.SplitSeq(value, "/") {
|
||||
if strings.HasPrefix(component, ".") || strings.HasSuffix(component, ".lock") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validGitHubReleaseDisplayTag(value string) bool {
|
||||
if value == "" || len(value) > 255 || !utf8.ValidString(value) {
|
||||
return false
|
||||
}
|
||||
for _, character := range value {
|
||||
if unsafeGitHubInputRune(character) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validGitHubAssetName(value string) bool {
|
||||
if value == "" || len(value) > 255 || !utf8.ValidString(value) ||
|
||||
path.Base(value) != value || strings.Contains(value, "\\") ||
|
||||
value == "." || value == ".." {
|
||||
return false
|
||||
}
|
||||
for _, character := range value {
|
||||
if unsafeGitHubInputRune(character) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func unsafeGitHubInputRune(character rune) bool {
|
||||
return unicode.IsControl(character) || character == '\u2028' || character == '\u2029' ||
|
||||
character == '\u061c' || character == '\u200e' || character == '\u200f' ||
|
||||
(character >= '\u202a' && character <= '\u202e') ||
|
||||
(character >= '\u2066' && character <= '\u2069')
|
||||
}
|
||||
|
||||
func updateGitHubSourceTx(tx *gorm.DB, projectID uint, input SourceUpdateInput) (bool, error) {
|
||||
if _, err := repository.LockPagesProjectByIDTx(tx, projectID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
existing, hasExisting, err := loadProjectSourceForUpdate(tx, projectID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
config, err := buildGitHubSourceConfig(input)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !hasExisting {
|
||||
return true, createGitHubSourceTx(tx, projectID, config)
|
||||
}
|
||||
if !githubSourceConfigChanged(existing, config) {
|
||||
return false, nil
|
||||
}
|
||||
runtime, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, existing.ID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
identityChanged := existing.SourceIdentity != config.SourceIdentity
|
||||
if err := repository.UpdatePagesProjectSourceTx(tx, existing, githubSourceUpdates(config, existing.ConfigVersion+1)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := resetRuntimeAfterGitHubUpdate(tx, runtime, config, identityChanged); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func createGitHubSourceTx(tx *gorm.DB, projectID uint, config githubSourceConfig) error {
|
||||
source := &model.PagesProjectSource{
|
||||
ProjectID: projectID,
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
GitHubRepository: config.Repository,
|
||||
ReleaseSelector: config.Selector,
|
||||
ReleaseTag: config.Tag,
|
||||
AssetName: config.AssetName,
|
||||
AutoUpdateEnabled: config.AutoUpdate,
|
||||
CheckIntervalMinutes: config.CheckInterval,
|
||||
ConfigVersion: 1,
|
||||
SourceIdentity: config.SourceIdentity,
|
||||
}
|
||||
if err := repository.CreatePagesProjectSourceTx(tx, source); err != nil {
|
||||
return err
|
||||
}
|
||||
runtime := &model.PagesProjectSourceRuntime{SourceID: source.ID, SyncStatus: pagesSourceStatusIdle}
|
||||
if config.Selector == githubReleaseSelectorLatest {
|
||||
next := nextGitHubCheckAt(time.Now(), source.ID, config.CheckInterval)
|
||||
runtime.NextCheckAt = &next
|
||||
}
|
||||
return repository.CreatePagesProjectSourceRuntimeTx(tx, runtime)
|
||||
}
|
||||
|
||||
func githubSourceUpdates(config githubSourceConfig, version int) map[string]any {
|
||||
return map[string]any{
|
||||
"source_type": PagesSourceTypeGitHubRelease,
|
||||
"remote_url": "",
|
||||
"allow_insecure": false,
|
||||
"github_repository": config.Repository,
|
||||
"release_selector": config.Selector,
|
||||
"release_tag": config.Tag,
|
||||
"asset_name": config.AssetName,
|
||||
sourceColumnAutoUpdateEnabled: config.AutoUpdate,
|
||||
"check_interval_minutes": config.CheckInterval,
|
||||
sourceColumnConfigVersion: version,
|
||||
"source_identity": config.SourceIdentity,
|
||||
}
|
||||
}
|
||||
|
||||
func githubSourceConfigChanged(existing *model.PagesProjectSource, config githubSourceConfig) bool {
|
||||
return existing.SourceType != PagesSourceTypeGitHubRelease || existing.RemoteURL != "" ||
|
||||
existing.AllowInsecure || existing.GitHubRepository != config.Repository ||
|
||||
existing.ReleaseSelector != config.Selector || existing.ReleaseTag != config.Tag ||
|
||||
existing.AssetName != config.AssetName || existing.AutoUpdateEnabled != config.AutoUpdate ||
|
||||
existing.CheckIntervalMinutes != config.CheckInterval
|
||||
}
|
||||
|
||||
func resetRuntimeAfterGitHubUpdate(
|
||||
tx *gorm.DB,
|
||||
runtime *model.PagesProjectSourceRuntime,
|
||||
config githubSourceConfig,
|
||||
identityChanged bool,
|
||||
) error {
|
||||
if err := resetRuntimeAfterSourceUpdate(tx, runtime, identityChanged); err != nil {
|
||||
return err
|
||||
}
|
||||
var nextCheckAt any
|
||||
if config.Selector == githubReleaseSelectorLatest {
|
||||
next := nextGitHubCheckAt(time.Now(), runtime.SourceID, config.CheckInterval)
|
||||
nextCheckAt = &next
|
||||
}
|
||||
return repository.UpdatePagesProjectSourceRuntimeFieldTx(tx, runtime, "next_check_at", nextCheckAt)
|
||||
}
|
||||
|
||||
func nextGitHubCheckAt(now time.Time, sourceID uint, intervalMinutes int) time.Time {
|
||||
// A stable, bounded offset avoids a thundering herd without persisting
|
||||
// another scheduling field. Scanner Phase 3 reuses this calculation.
|
||||
jitterSeconds := int64(sourceID%githubCheckJitterRange) - githubCheckJitterCenter
|
||||
return now.Add(time.Duration(intervalMinutes)*time.Minute + time.Duration(jitterSeconds)*time.Second)
|
||||
}
|
||||
|
||||
func markInitialCheckDispatchFailed(ctx context.Context, sourceID uint, configVersion int) {
|
||||
updates := map[string]any{
|
||||
sourceRuntimeColumnSyncStatus: pagesSourceStatusFailed,
|
||||
sourceRuntimeColumnLastError: errPagesSourceInitialCheckWarning,
|
||||
}
|
||||
source, err := repository.GetPagesProjectSourceByIDAndConfigVersion(ctx, sourceID, configVersion)
|
||||
if err != nil {
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
logger.ErrorF(ctx, "[PagesSource] load initial check source snapshot failed: source_id=%d error=%v", sourceID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if source.ReleaseSelector == githubReleaseSelectorLatest {
|
||||
next := time.Now().Add(initialCheckRetryDelay)
|
||||
updates["next_check_at"] = &next
|
||||
}
|
||||
now := time.Now()
|
||||
if _, err := repository.MarkPagesSourceInitialCheckDispatchFailed(ctx, sourceID, configVersion, now, updates); err != nil {
|
||||
logger.ErrorF(ctx, "[PagesSource] mark initial check dispatch failure: source_id=%d error=%v", sourceID, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,753 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/githubrelease"
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/openflare/plugins/server/kernel/task"
|
||||
"Wavelet/openflare/share/pagesarchive"
|
||||
"Wavelet/pkg/logger"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const githubSourceDetailProvider = "github"
|
||||
|
||||
var githubDigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
type githubSourceProviderDomainError struct {
|
||||
message string
|
||||
permanent bool
|
||||
retryAt *time.Time
|
||||
statusCode int
|
||||
}
|
||||
|
||||
func (domainError *githubSourceProviderDomainError) Error() string {
|
||||
return domainError.message
|
||||
}
|
||||
|
||||
type githubReleaseAPI interface {
|
||||
Resolve(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error)
|
||||
Download(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error)
|
||||
}
|
||||
|
||||
var newGitHubReleaseClient = func() githubReleaseAPI {
|
||||
return githubrelease.NewClient()
|
||||
}
|
||||
|
||||
type githubSourceTarget struct {
|
||||
Revision string
|
||||
Detail sourceDetail
|
||||
DetailJSON string
|
||||
Release githubrelease.Release
|
||||
Asset githubrelease.Asset
|
||||
RetryAt *time.Time
|
||||
}
|
||||
|
||||
type githubCheckTaskResult struct {
|
||||
Message string
|
||||
Detail string
|
||||
Revision string
|
||||
Status string
|
||||
RetryAt *time.Time
|
||||
Stale bool
|
||||
}
|
||||
|
||||
type preparedGitHubSource struct {
|
||||
target *githubSourceTarget
|
||||
download *githubrelease.DownloadResult
|
||||
format pagesarchive.Format
|
||||
manifest *deploymentManifest
|
||||
ingestState *sourceIngestState
|
||||
limits pagesLimits
|
||||
}
|
||||
|
||||
func checkGitHubSource(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
) (*githubCheckTaskResult, error) {
|
||||
if snapshot == nil || snapshot.SourceType != PagesSourceTypeGitHubRelease {
|
||||
return nil, errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
task.AppendLog(ctx, "[check] 正在检查 GitHub Release：repo=%s asset=%s", snapshot.GitHubRepository, snapshot.AssetName)
|
||||
client := newGitHubReleaseClient()
|
||||
result, err := client.Resolve(ctx, githubrelease.ResolveRequest{
|
||||
Repository: snapshot.GitHubRepository,
|
||||
Selector: githubrelease.Selector(snapshot.ReleaseSelector),
|
||||
Tag: snapshot.ReleaseTag,
|
||||
AssetName: snapshot.AssetName,
|
||||
ETag: snapshot.ETag,
|
||||
})
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] GitHub resolve failed: source_id=%d repo=%s error=%v", snapshot.SourceID, snapshot.GitHubRepository, err)
|
||||
retryAt, _ := githubrelease.RetryAt(err)
|
||||
domainErr := githubSourceDomainError(err)
|
||||
if failErr := failGitHubCheckLease(ctx, snapshot, domainErr.Error(), retryAt); failErr != nil {
|
||||
if errors.Is(failErr, errSourceFinalFence) {
|
||||
return &githubCheckTaskResult{Message: errPagesSourceActionStale, Stale: true}, nil
|
||||
}
|
||||
return nil, failErr
|
||||
}
|
||||
return nil, domainErr
|
||||
}
|
||||
if result.NotModified {
|
||||
revision, status, err := finishGitHubCheckNotModified(ctx, snapshot, result)
|
||||
if err != nil {
|
||||
if errors.Is(err, errSourceFinalFence) {
|
||||
return &githubCheckTaskResult{Message: errPagesSourceActionStale, Stale: true}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
detail, _ := json.Marshal(map[string]string{"revision": revision, pagesDeploymentColumnStatus: status})
|
||||
return &githubCheckTaskResult{
|
||||
Message: "GitHub Release 检查完成，内容未变化",
|
||||
Detail: string(detail),
|
||||
Revision: revision,
|
||||
Status: status,
|
||||
RetryAt: result.RetryAt,
|
||||
}, nil
|
||||
}
|
||||
target, err := buildGitHubSourceTarget(result.Release, result.Asset, result.RetryAt)
|
||||
if err != nil {
|
||||
retryAt := time.Time{}
|
||||
if result.RetryAt != nil {
|
||||
retryAt = result.RetryAt.UTC()
|
||||
}
|
||||
if failErr := failGitHubCheckLease(ctx, snapshot, err.Error(), retryAt); failErr != nil {
|
||||
if errors.Is(failErr, errSourceFinalFence) {
|
||||
return &githubCheckTaskResult{Message: errPagesSourceActionStale, Stale: true}, nil
|
||||
}
|
||||
return nil, failErr
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
status, err := finishGitHubCheckTarget(ctx, snapshot, result, target)
|
||||
if err != nil {
|
||||
if errors.Is(err, errSourceFinalFence) {
|
||||
return &githubCheckTaskResult{Message: errPagesSourceActionStale, Stale: true}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
detail, _ := json.Marshal(map[string]string{"revision": target.Revision, pagesDeploymentColumnStatus: status})
|
||||
message := "GitHub Release 检查完成"
|
||||
switch status {
|
||||
case pagesSourceStatusUpdateAvailable:
|
||||
message = "发现新的 GitHub Release 部署包"
|
||||
case pagesSourceStatusAttention:
|
||||
message = "检测到同一 Release 的资源被替换，需要确认"
|
||||
}
|
||||
return &githubCheckTaskResult{
|
||||
Message: message, Detail: string(detail), Revision: target.Revision,
|
||||
Status: status, RetryAt: result.RetryAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildGitHubSourceTarget(
|
||||
release githubrelease.Release,
|
||||
asset githubrelease.Asset,
|
||||
retryAt *time.Time,
|
||||
) (*githubSourceTarget, error) {
|
||||
digest := strings.ToLower(strings.TrimSpace(asset.Digest))
|
||||
if digest != "" && !githubDigestPattern.MatchString(digest) {
|
||||
return nil, errors.New(errPagesSourceDigestInvalid)
|
||||
}
|
||||
if strings.TrimSpace(release.ID) == "" || strings.TrimSpace(asset.ID) == "" ||
|
||||
!validGitHubReleaseDisplayTag(release.Tag) || !validGitHubAssetName(asset.Name) ||
|
||||
asset.State != "uploaded" || asset.UpdatedAt.IsZero() {
|
||||
return nil, errors.New(errPagesSourceReleaseNotFound)
|
||||
}
|
||||
updatedAt := asset.UpdatedAt.UTC().Format(time.RFC3339Nano)
|
||||
rawRevision := "github:" + release.ID + ":" + asset.ID + ":" + updatedAt + ":" + digest
|
||||
sum := sha256.Sum256([]byte(rawRevision))
|
||||
detail := sourceDetail{
|
||||
Provider: githubSourceDetailProvider,
|
||||
Tag: release.Tag,
|
||||
AssetName: asset.Name,
|
||||
ReleaseID: release.ID,
|
||||
AssetID: asset.ID,
|
||||
AssetUpdatedAt: updatedAt,
|
||||
Digest: digest,
|
||||
}
|
||||
detailJSON, err := json.Marshal(detail)
|
||||
if err != nil {
|
||||
return nil, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
return &githubSourceTarget{
|
||||
Revision: hex.EncodeToString(sum[:]),
|
||||
Detail: detail,
|
||||
DetailJSON: string(detailJSON),
|
||||
Release: release,
|
||||
Asset: asset,
|
||||
RetryAt: retryAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func finishGitHubCheckNotModified(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
result githubrelease.ResolveResult,
|
||||
) (string, string, error) {
|
||||
var revision string
|
||||
var status string
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
runtime, now, err := lockOwnedSourceRuntime(tx, snapshot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
revision = runtime.LastSeenRevision
|
||||
status = normalizedSourceRuntimeStatus(runtime)
|
||||
updates := githubCheckTerminalUpdates(snapshot, now, result.RetryAt)
|
||||
updates["etag"] = result.ETag
|
||||
updates[sourceRuntimeColumnSyncStatus] = status
|
||||
return repository.UpdatePagesProjectSourceRuntimeTx(tx, runtime, updates)
|
||||
})
|
||||
return revision, status, err
|
||||
}
|
||||
|
||||
func finishGitHubCheckTarget(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
result githubrelease.ResolveResult,
|
||||
target *githubSourceTarget,
|
||||
) (string, error) {
|
||||
status := pagesSourceStatusIdle
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
runtime, now, err := lockOwnedSourceRuntime(tx, snapshot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status = targetRuntimeStatus(target, runtime.LastAppliedRevision, runtime.LastAppliedDetail)
|
||||
updates := githubCheckTerminalUpdates(snapshot, now, result.RetryAt)
|
||||
updates["etag"] = result.ETag
|
||||
updates["last_seen_revision"] = target.Revision
|
||||
updates["last_seen_detail"] = target.DetailJSON
|
||||
updates[sourceRuntimeColumnSyncStatus] = status
|
||||
return repository.UpdatePagesProjectSourceRuntimeTx(tx, runtime, updates)
|
||||
})
|
||||
return status, err
|
||||
}
|
||||
|
||||
func githubCheckTerminalUpdates(
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
now time.Time,
|
||||
retryAt *time.Time,
|
||||
) map[string]any {
|
||||
updates := map[string]any{
|
||||
sourceRuntimeColumnLastError: "",
|
||||
sourceRuntimeColumnLastCheckedAt: &now,
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
}
|
||||
updates[sourceRuntimeColumnNextCheckAt] = nextCheckAfterGitHubResponse(snapshot, now, retryAt)
|
||||
return updates
|
||||
}
|
||||
|
||||
func nextCheckAfterGitHubResponse(
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
now time.Time,
|
||||
retryAt *time.Time,
|
||||
) any {
|
||||
if snapshot.ReleaseSelector != githubReleaseSelectorLatest {
|
||||
return nil
|
||||
}
|
||||
next := nextGitHubCheckAt(now, snapshot.SourceID, snapshot.CheckIntervalMinutes)
|
||||
if retryAt != nil && retryAt.After(next) {
|
||||
next = retryAt.In(now.Location())
|
||||
}
|
||||
return &next
|
||||
}
|
||||
|
||||
func lockOwnedSourceRuntime(
|
||||
tx *gorm.DB,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
) (*model.PagesProjectSourceRuntime, time.Time, error) {
|
||||
runtime, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, snapshot.SourceID)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
now := time.Now()
|
||||
if runtime.LeaseToken != snapshot.LeaseToken || runtime.LeaseExpiresAt == nil ||
|
||||
!runtime.LeaseExpiresAt.After(now) {
|
||||
return nil, time.Time{}, errSourceFinalFence
|
||||
}
|
||||
return runtime, now, nil
|
||||
}
|
||||
|
||||
func failGitHubCheckLease(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
message string,
|
||||
retryAt time.Time,
|
||||
) error {
|
||||
now := time.Now()
|
||||
next := now.Add(initialCheckRetryDelay)
|
||||
if retryAt.After(next) {
|
||||
next = retryAt.In(now.Location())
|
||||
}
|
||||
updates := map[string]any{
|
||||
sourceRuntimeColumnSyncStatus: pagesSourceStatusFailed,
|
||||
sourceRuntimeColumnLastError: safeSourceRuntimeError(message),
|
||||
sourceRuntimeColumnLastCheckedAt: &now,
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
}
|
||||
if snapshot.ReleaseSelector == githubReleaseSelectorLatest {
|
||||
updates[sourceRuntimeColumnNextCheckAt] = &next
|
||||
} else {
|
||||
updates[sourceRuntimeColumnNextCheckAt] = nil
|
||||
}
|
||||
rows, err := repository.UpdatePagesSourceRuntimeByActiveLease(
|
||||
ctx, snapshot.SourceID, snapshot.LeaseToken, now, updates,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows != 1 {
|
||||
return errSourceFinalFence
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func targetRuntimeStatus(
|
||||
target *githubSourceTarget,
|
||||
appliedRevision string,
|
||||
appliedDetail string,
|
||||
) string {
|
||||
if target == nil || target.Revision == appliedRevision {
|
||||
return pagesSourceStatusIdle
|
||||
}
|
||||
applied := sourceDetail{}
|
||||
if unmarshalSourceDetail(appliedDetail, &applied) == nil && target.Detail.ReleaseID != "" &&
|
||||
target.Detail.ReleaseID == applied.ReleaseID {
|
||||
return pagesSourceStatusAttention
|
||||
}
|
||||
return pagesSourceStatusUpdateAvailable
|
||||
}
|
||||
|
||||
func preflightGitHubSyncConfirmation(ctx context.Context, sourceID uint, confirmedRevision string) error {
|
||||
runtime, err := repository.GetPagesProjectSourceRuntimeBySourceID(ctx, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
replacement := sourceHasSameReleaseReplacement(runtime)
|
||||
if replacement && confirmedRevision == "" {
|
||||
return errors.New(errPagesSourceConfirmationNeeded)
|
||||
}
|
||||
if confirmedRevision != "" && (!replacement || confirmedRevision != runtime.LastSeenRevision) {
|
||||
return errors.New(errPagesSourceConfirmationStale)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func syncGitHubSourceWithTrigger(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
actor string,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
triggerType string,
|
||||
) (outcome *sourceSyncOutcome, resultErr error) {
|
||||
if snapshot == nil || snapshot.SourceType != PagesSourceTypeGitHubRelease || !validPagesSourceActor(actor) {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
if !validSourceDeploymentTrigger(triggerType) {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
defer func() {
|
||||
resultErr = finalizeGitHubSyncFailure(ctx, snapshot, resultErr)
|
||||
}()
|
||||
workCtx, heartbeat, err := startSourceLeaseHeartbeat(
|
||||
ctx, snapshot, pagesSourceSyncLeaseDuration, pagesSourceHeartbeatInterval,
|
||||
)
|
||||
if err != nil {
|
||||
return sourceHeartbeatOutcome(err)
|
||||
}
|
||||
defer func() { _ = heartbeat.stop() }()
|
||||
|
||||
client := newGitHubReleaseClient()
|
||||
target, guardedOutcome, err := resolveAndGuardGitHubSync(
|
||||
workCtx, client, snapshot, targetRevision, confirmedRevision,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if guardedOutcome != nil {
|
||||
return guardedOutcome, nil
|
||||
}
|
||||
prepared, err := prepareGitHubSyncPackage(workCtx, client, snapshot, target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if cleanupErr := prepared.download.Cleanup(); cleanupErr != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] cleanup GitHub package failed: source_id=%d error=%v", snapshot.SourceID, cleanupErr)
|
||||
}
|
||||
}()
|
||||
defer compensateSourceIngest(ctx, snapshot, prepared.ingestState)
|
||||
if heartbeatErr := heartbeat.stop(); heartbeatErr != nil {
|
||||
return sourceHeartbeatOutcome(heartbeatErr)
|
||||
}
|
||||
renewed, err := renewSourceLease(ctx, snapshot, pagesSourceSyncLeaseDuration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !renewed {
|
||||
return &sourceSyncOutcome{Stale: true}, nil
|
||||
}
|
||||
return activatePreparedGitHubSource(ctx, snapshot, actor, triggerType, prepared)
|
||||
}
|
||||
|
||||
func finalizeGitHubSyncFailure(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
resultErr error,
|
||||
) error {
|
||||
if resultErr == nil {
|
||||
return nil
|
||||
}
|
||||
cleanupCtx, cancel := sourceCleanupContext(ctx)
|
||||
defer cancel()
|
||||
finalizerErr := persistGitHubSyncFailure(cleanupCtx, snapshot, resultErr)
|
||||
if finalizerErr == nil {
|
||||
return resultErr
|
||||
}
|
||||
logger.WarnF(
|
||||
cleanupCtx,
|
||||
"[PagesSource] finalize GitHub sync failure failed: source_id=%d source_error=%s error=%v",
|
||||
snapshot.SourceID, safeGitHubSourceError(resultErr), finalizerErr,
|
||||
)
|
||||
// final fence 丢失表示已有新任务接管 runtime，不应覆盖；数据库
|
||||
// finalizer 失败则保持可重试，避免继承永久错误或 provider deadline 分类。
|
||||
if errors.Is(finalizerErr, errSourceFinalFence) {
|
||||
return resultErr
|
||||
}
|
||||
return errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
|
||||
func persistGitHubSyncFailure(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
resultErr error,
|
||||
) error {
|
||||
var domainError *githubSourceProviderDomainError
|
||||
if errors.As(resultErr, &domainError) && domainError.retryAt != nil {
|
||||
return failGitHubCheckLease(ctx, snapshot, domainError.message, *domainError.retryAt)
|
||||
}
|
||||
return failSourceLease(ctx, snapshot, safeGitHubSourceError(resultErr))
|
||||
}
|
||||
|
||||
func activatePreparedGitHubSource(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
actor string,
|
||||
triggerType string,
|
||||
prepared *preparedGitHubSource,
|
||||
) (*sourceSyncOutcome, error) {
|
||||
task.AppendLog(ctx, "[activate] 正在原子切换 GitHub Release 部署")
|
||||
deployment, reused, referenced, err := commitSourceDeploymentWithTrigger(
|
||||
ctx, snapshot, prepared.target.Revision, prepared.download.SHA256,
|
||||
prepared.target.Detail, prepared.target.DetailJSON, actor, triggerType, prepared.manifest,
|
||||
prepared.ingestState.Result, prepared.ingestState.HasIngest, prepared.target.RetryAt,
|
||||
)
|
||||
prepared.ingestState.Referenced = referenced
|
||||
if errors.Is(err, errSourceFinalFence) {
|
||||
return &sourceSyncOutcome{Stale: true}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prepared.ingestState.Referenced = prepared.ingestState.HasIngest && deployment.UploadID == prepared.ingestState.Result.Upload.ID
|
||||
if pruneErr := pruneProjectDeploymentHistory(ctx, snapshot.ProjectID, prepared.limits.HistoryCount, 0); pruneErr != nil {
|
||||
logger.ErrorF(ctx, "[PagesSource] strict prune failed after GitHub sync: project_id=%d source_id=%d error=%v", snapshot.ProjectID, snapshot.SourceID, pruneErr)
|
||||
}
|
||||
view := buildDeploymentView(deployment)
|
||||
return &sourceSyncOutcome{Deployment: &view, Reused: reused}, nil
|
||||
}
|
||||
|
||||
func resolveAndGuardGitHubSync(
|
||||
ctx context.Context,
|
||||
client githubReleaseAPI,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
) (*githubSourceTarget, *sourceSyncOutcome, error) {
|
||||
task.AppendLog(ctx, "[resolve] 正在解析 GitHub Release：repo=%s asset=%s", snapshot.GitHubRepository, snapshot.AssetName)
|
||||
resolved, err := client.Resolve(ctx, githubrelease.ResolveRequest{
|
||||
Repository: snapshot.GitHubRepository,
|
||||
Selector: githubrelease.Selector(snapshot.ReleaseSelector),
|
||||
Tag: snapshot.ReleaseTag,
|
||||
AssetName: snapshot.AssetName,
|
||||
})
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] GitHub resolve failed: source_id=%d repo=%s error=%v", snapshot.SourceID, snapshot.GitHubRepository, err)
|
||||
return nil, nil, githubSourceDomainError(err)
|
||||
}
|
||||
if resolved.NotModified {
|
||||
return nil, nil, errors.New(errPagesSourceReleaseNotFound)
|
||||
}
|
||||
target, err := buildGitHubSourceTarget(resolved.Release, resolved.Asset, resolved.RetryAt)
|
||||
if err != nil {
|
||||
return nil, nil, &githubSourceProviderDomainError{
|
||||
message: safeGitHubSourceError(err),
|
||||
permanent: isPermanentSourceSyncError(err),
|
||||
retryAt: resolved.RetryAt,
|
||||
}
|
||||
}
|
||||
guardedOutcome, err := guardGitHubSyncTarget(ctx, snapshot, target, targetRevision, confirmedRevision)
|
||||
return target, guardedOutcome, err
|
||||
}
|
||||
|
||||
func guardGitHubSyncTarget(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
target *githubSourceTarget,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
) (*sourceSyncOutcome, error) {
|
||||
status := targetRuntimeStatus(target, snapshot.LastAppliedRevision, snapshot.LastAppliedDetail)
|
||||
if targetRevision != "" && targetRevision != target.Revision {
|
||||
return releaseGuardedGitHubTarget(ctx, snapshot, target, status, "", true, true)
|
||||
}
|
||||
if confirmedRevision != "" && (confirmedRevision != snapshot.LastSeenRevision || confirmedRevision != target.Revision) {
|
||||
return releaseGuardedGitHubTarget(ctx, snapshot, target, status, errPagesSourceConfirmationStale, false, false)
|
||||
}
|
||||
if status == pagesSourceStatusAttention && confirmedRevision != target.Revision {
|
||||
return releaseGuardedGitHubTarget(ctx, snapshot, target, status, errPagesSourceConfirmationNeeded, false, false)
|
||||
}
|
||||
if confirmedRevision != "" && status != pagesSourceStatusAttention {
|
||||
return nil, errors.New(errPagesSourceConfirmationStale)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func releaseGuardedGitHubTarget(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
target *githubSourceTarget,
|
||||
status string,
|
||||
lastError string,
|
||||
expedite bool,
|
||||
staleSuccess bool,
|
||||
) (*sourceSyncOutcome, error) {
|
||||
err := releaseGitHubSyncWithoutActivation(ctx, snapshot, target, status, lastError, expedite, target.RetryAt)
|
||||
if errors.Is(err, errSourceFinalFence) || (err == nil && staleSuccess) {
|
||||
return &sourceSyncOutcome{Stale: true}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, errors.New(lastError)
|
||||
}
|
||||
|
||||
func prepareGitHubSyncPackage(
|
||||
ctx context.Context,
|
||||
client githubReleaseAPI,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
target *githubSourceTarget,
|
||||
) (*preparedGitHubSource, error) {
|
||||
limits := resolvePagesLimits(ctx)
|
||||
task.AppendLog(ctx, "[download] 正在下载 GitHub Release asset：repo=%s asset=%s", snapshot.GitHubRepository, snapshot.AssetName)
|
||||
download, err := client.Download(ctx, githubrelease.DownloadRequest{
|
||||
Repository: snapshot.GitHubRepository,
|
||||
Asset: target.Asset,
|
||||
MaxBytes: limits.PackageBytes,
|
||||
})
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] GitHub download failed: source_id=%d repo=%s asset=%s error=%v", snapshot.SourceID, snapshot.GitHubRepository, snapshot.AssetName, err)
|
||||
return nil, githubSourceDomainError(err)
|
||||
}
|
||||
prepared, err := inspectAndIngestGitHubPackage(ctx, snapshot, target, download, limits)
|
||||
if err != nil {
|
||||
if cleanupErr := download.Cleanup(); cleanupErr != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] cleanup GitHub package after preparation failure failed: source_id=%d error=%v", snapshot.SourceID, cleanupErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func inspectAndIngestGitHubPackage(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
target *githubSourceTarget,
|
||||
download *githubrelease.DownloadResult,
|
||||
limits pagesLimits,
|
||||
) (*preparedGitHubSource, error) {
|
||||
if download.SHA256 == "" || download.Path == "" {
|
||||
return nil, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
if target.Detail.Digest != "" && "sha256:"+download.SHA256 != target.Detail.Digest {
|
||||
return nil, errors.New(errPagesSourceDigestMismatch)
|
||||
}
|
||||
format, ok := pagesarchive.DetectFormatFromName(target.Asset.Name)
|
||||
var err error
|
||||
if !ok {
|
||||
format, _, err = detectRemoteSourceFormat(download.Path, target.Asset.Name, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rootDir, err := validateAndNormalizePagesRootDir(snapshot.RootDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entryFile, err := validateAndNormalizePagesEntryFile(snapshot.EntryFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
task.AppendLog(ctx, "[verify] 正在校验 GitHub Release 归档与入口")
|
||||
manifest, err := inspectPagesPackage(download.Path, format, rootDir, entryFile, limits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ingestState, err := resolveGitHubSourceIngest(ctx, snapshot, target, download, format)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &preparedGitHubSource{
|
||||
target: target, download: download, format: format,
|
||||
manifest: manifest, ingestState: ingestState, limits: limits,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveGitHubSourceIngest(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
target *githubSourceTarget,
|
||||
download *githubrelease.DownloadResult,
|
||||
format pagesarchive.Format,
|
||||
) (*sourceIngestState, error) {
|
||||
if _, err := findSourceDeployment(ctx, snapshot.ProjectID, snapshot.SourceIdentity, target.Revision); err == nil {
|
||||
return &sourceIngestState{}, nil
|
||||
} else if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
task.AppendLog(ctx, "[ingest] 正在保存 GitHub Release 部署包")
|
||||
result, err := ingestPagesDeploymentPackageWithSource(
|
||||
ctx, download.Path, download.SHA256, snapshot.ProjectID, snapshot.SourceID, target.Asset.Name, format,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sourceIngestState{Result: result, HasIngest: true}, nil
|
||||
}
|
||||
|
||||
func releaseGitHubSyncWithoutActivation(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
target *githubSourceTarget,
|
||||
status string,
|
||||
lastError string,
|
||||
expedite bool,
|
||||
retryAt *time.Time,
|
||||
) error {
|
||||
now := time.Now()
|
||||
nextCheckAt := nextCheckAfterGitHubResponse(snapshot, now, retryAt)
|
||||
if expedite && snapshot.ReleaseSelector == githubReleaseSelectorLatest {
|
||||
next := now.Add(initialCheckRetryDelay)
|
||||
if retryAt != nil && retryAt.After(next) {
|
||||
next = retryAt.In(now.Location())
|
||||
}
|
||||
nextCheckAt = &next
|
||||
}
|
||||
updates := map[string]any{
|
||||
"last_seen_revision": target.Revision,
|
||||
"last_seen_detail": target.DetailJSON,
|
||||
sourceRuntimeColumnSyncStatus: status,
|
||||
sourceRuntimeColumnLastError: lastError,
|
||||
sourceRuntimeColumnLastCheckedAt: &now,
|
||||
sourceRuntimeColumnNextCheckAt: nextCheckAt,
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
}
|
||||
rows, err := repository.UpdatePagesSourceRuntimeByActiveLease(
|
||||
ctx, snapshot.SourceID, snapshot.LeaseToken, now, updates,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows != 1 {
|
||||
return errSourceFinalFence
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safeGitHubSourceError(err error) string {
|
||||
if err == nil {
|
||||
return errPagesSourceSyncFailed
|
||||
}
|
||||
message := strings.TrimSpace(err.Error())
|
||||
for _, safeMessage := range []string{
|
||||
errPagesSourceSyncFailed,
|
||||
errPagesSourceReleaseNotFound,
|
||||
errPagesSourceDigestInvalid,
|
||||
errPagesSourceDigestMismatch,
|
||||
errPagesSourceConfirmationNeeded,
|
||||
errPagesSourceConfirmationStale,
|
||||
errPagesPackageURLTooLarge,
|
||||
errPagesPackageEmpty,
|
||||
errPagesPackageUnsupported,
|
||||
errPagesPackageInvalid,
|
||||
errPagesPackageExtractedTooLarge,
|
||||
errPagesPackageFileTooLarge,
|
||||
errPagesEntryFileMissing,
|
||||
} {
|
||||
if message == safeMessage {
|
||||
return safeMessage
|
||||
}
|
||||
}
|
||||
return errPagesSourceSyncFailed
|
||||
}
|
||||
|
||||
func githubSourceDomainError(err error) error {
|
||||
message := errPagesSourceSyncFailed
|
||||
statusCode := 0
|
||||
var providerError *githubrelease.Error
|
||||
if errors.As(err, &providerError) {
|
||||
statusCode = providerError.StatusCode
|
||||
}
|
||||
retryAt, hasRetryAt := githubrelease.RetryAt(err)
|
||||
var retryDeadline *time.Time
|
||||
if hasRetryAt {
|
||||
retryDeadline = &retryAt
|
||||
}
|
||||
if err == nil {
|
||||
return &githubSourceProviderDomainError{message: message, permanent: false, statusCode: statusCode}
|
||||
}
|
||||
if githubrelease.IsDigestError(err) {
|
||||
message = errPagesSourceDigestMismatch
|
||||
return &githubSourceProviderDomainError{message: message, permanent: true, statusCode: statusCode}
|
||||
}
|
||||
if githubrelease.IsNotFound(err) {
|
||||
message = errPagesSourceReleaseNotFound
|
||||
return &githubSourceProviderDomainError{message: message, permanent: true, statusCode: statusCode}
|
||||
}
|
||||
if errors.Is(err, githubrelease.ErrAssetTooLarge) {
|
||||
message = errPagesPackageURLTooLarge
|
||||
return &githubSourceProviderDomainError{message: message, permanent: true, statusCode: statusCode}
|
||||
}
|
||||
if errors.Is(err, githubrelease.ErrEmptyAsset) {
|
||||
message = errPagesPackageEmpty
|
||||
return &githubSourceProviderDomainError{message: message, permanent: true, statusCode: statusCode}
|
||||
}
|
||||
return &githubSourceProviderDomainError{
|
||||
message: message, permanent: !githubrelease.IsRetryable(err), retryAt: retryDeadline, statusCode: statusCode,
|
||||
}
|
||||
}
|
||||
|
||||
func shouldSkipGitHubActionRetry(err error) bool {
|
||||
var domainError *githubSourceProviderDomainError
|
||||
return errors.As(err, &domainError) && (domainError.permanent || domainError.retryAt != nil)
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
func TestGitHubSourceIdentityLengthPrefixesFieldsAndResetsRuntime(t *testing.T) {
|
||||
firstInput := SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/OpenFlare/site",
|
||||
ReleaseSelector: githubReleaseSelectorTag,
|
||||
ReleaseTag: "release|foo",
|
||||
AssetName: "bar.zip",
|
||||
}
|
||||
secondInput := SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/OpenFlare/site",
|
||||
ReleaseSelector: githubReleaseSelectorTag,
|
||||
ReleaseTag: "release",
|
||||
AssetName: "foo|bar.zip",
|
||||
}
|
||||
firstConfig, err := buildGitHubSourceConfig(firstInput)
|
||||
if err != nil {
|
||||
t.Fatalf("buildGitHubSourceConfig(first) error = %v, want nil", err)
|
||||
}
|
||||
secondConfig, err := buildGitHubSourceConfig(secondInput)
|
||||
if err != nil {
|
||||
t.Fatalf("buildGitHubSourceConfig(second) error = %v, want nil", err)
|
||||
}
|
||||
legacyIdentityInput := func(config githubSourceConfig) string {
|
||||
return "github|" + config.Repository + "|" + config.Selector + "|" +
|
||||
config.Tag + "|" + config.AssetName
|
||||
}
|
||||
if firstLegacy, secondLegacy := legacyIdentityInput(firstConfig), legacyIdentityInput(secondConfig); firstLegacy != secondLegacy {
|
||||
t.Fatalf("legacy identity inputs differ: %q != %q; collision fixture is invalid", firstLegacy, secondLegacy)
|
||||
}
|
||||
if firstConfig.SourceIdentity == secondConfig.SourceIdentity {
|
||||
t.Fatalf("length-prefixed identities collide: %q", firstConfig.SourceIdentity)
|
||||
}
|
||||
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-identity-collision")
|
||||
firstSource, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, firstInput)
|
||||
if got, want := firstSource.SourceIdentity, firstConfig.SourceIdentity; got != want {
|
||||
t.Fatalf("first source identity = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
checkedAt := time.Now().Add(-time.Minute)
|
||||
syncedAt := time.Now().Add(-30 * time.Second)
|
||||
nextCheckAt := time.Now().Add(time.Hour)
|
||||
leaseExpiresAt := time.Now().Add(time.Minute)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", firstSource.ID).
|
||||
Updates(map[string]any{
|
||||
"etag": `"old-etag"`,
|
||||
"last_seen_revision": strings.Repeat("a", 64),
|
||||
"last_seen_detail": `{"provider":"github_release","tag":"release|foo"}`,
|
||||
"last_applied_revision": strings.Repeat("b", 64),
|
||||
"last_applied_detail": `{"provider":"github_release","tag":"older"}`,
|
||||
"sync_status": pagesSourceStatusSyncing,
|
||||
"last_error": "old error",
|
||||
"last_checked_at": &checkedAt,
|
||||
"last_synced_at": &syncedAt,
|
||||
"next_check_at": &nextCheckAt,
|
||||
"lease_expires_at": &leaseExpiresAt,
|
||||
"lease_token": "old-lease",
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed runtime cursors error = %v, want nil", err)
|
||||
}
|
||||
|
||||
secondSource, runtime := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, secondInput)
|
||||
if secondSource.ID != firstSource.ID {
|
||||
t.Errorf("updated source ID = %d, want unchanged %d", secondSource.ID, firstSource.ID)
|
||||
}
|
||||
if got, want := secondSource.SourceIdentity, secondConfig.SourceIdentity; got != want {
|
||||
t.Errorf("updated source identity = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := secondSource.ConfigVersion, firstSource.ConfigVersion+1; got != want {
|
||||
t.Errorf("updated source config version = %d, want %d", got, want)
|
||||
}
|
||||
if runtime.ETag != "" || runtime.LastSeenRevision != "" || runtime.LastSeenDetail != "" ||
|
||||
runtime.LastAppliedRevision != "" || runtime.LastAppliedDetail != "" {
|
||||
t.Errorf("identity change retained runtime cursors: %+v", runtime)
|
||||
}
|
||||
if runtime.LastCheckedAt != nil || runtime.LastSyncedAt != nil || runtime.NextCheckAt != nil {
|
||||
t.Errorf(
|
||||
"identity change retained runtime timestamps: checked=%v synced=%v next=%v",
|
||||
runtime.LastCheckedAt,
|
||||
runtime.LastSyncedAt,
|
||||
runtime.NextCheckAt,
|
||||
)
|
||||
}
|
||||
if runtime.SyncStatus != pagesSourceStatusIdle || runtime.LastError != "" ||
|
||||
runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil {
|
||||
t.Errorf(
|
||||
"identity change retained runtime state: status=%q error=%q lease=(%q, %v)",
|
||||
runtime.SyncStatus,
|
||||
runtime.LastError,
|
||||
runtime.LeaseToken,
|
||||
runtime.LeaseExpiresAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,896 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/githubrelease"
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/task"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type fakeGitHubReleaseClient struct {
|
||||
resolve func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error)
|
||||
download func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error)
|
||||
}
|
||||
|
||||
func (client *fakeGitHubReleaseClient) Resolve(
|
||||
ctx context.Context,
|
||||
request githubrelease.ResolveRequest,
|
||||
) (githubrelease.ResolveResult, error) {
|
||||
return client.resolve(ctx, request)
|
||||
}
|
||||
|
||||
func (client *fakeGitHubReleaseClient) Download(
|
||||
ctx context.Context,
|
||||
request githubrelease.DownloadRequest,
|
||||
) (*githubrelease.DownloadResult, error) {
|
||||
return client.download(ctx, request)
|
||||
}
|
||||
|
||||
func useFakeGitHubReleaseClient(t *testing.T, client githubReleaseAPI) {
|
||||
t.Helper()
|
||||
previous := newGitHubReleaseClient
|
||||
newGitHubReleaseClient = func() githubReleaseAPI { return client }
|
||||
t.Cleanup(func() { newGitHubReleaseClient = previous })
|
||||
}
|
||||
|
||||
func mustConfigureGitHubSourceWithoutDispatch(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
input SourceUpdateInput,
|
||||
) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime) {
|
||||
t.Helper()
|
||||
if err := validateGitHubSourceInput(input); err != nil {
|
||||
t.Fatalf("validateGitHubSourceInput(%+v) error = %v, want nil", input, err)
|
||||
}
|
||||
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
_, err := updateGitHubSourceTx(tx, projectID, input)
|
||||
return err
|
||||
}); err != nil {
|
||||
t.Fatalf("updateGitHubSourceTx(project=%d) error = %v, want nil", projectID, err)
|
||||
}
|
||||
return mustLoadPagesSource(t, ctx, projectID)
|
||||
}
|
||||
|
||||
func mustLoadPagesSource(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime) {
|
||||
t.Helper()
|
||||
var source model.PagesProjectSource
|
||||
if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil {
|
||||
t.Fatalf("load source for project %d error = %v, want nil", projectID, err)
|
||||
}
|
||||
var runtime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
|
||||
t.Fatalf("load runtime for source %d error = %v, want nil", source.ID, err)
|
||||
}
|
||||
return &source, &runtime
|
||||
}
|
||||
|
||||
func TestGitHubSourceValidationNormalizationAndProviderSwitch(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
setupPagesSourceDispatchTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-config")
|
||||
input := SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/OpenFlare/site.git",
|
||||
}
|
||||
result, err := UpdateSourceAs(ctx, project.ID, input, "user:42")
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateSourceAs(GitHub) error = %v, want nil", err)
|
||||
}
|
||||
if result.CheckTask == nil || result.CheckTask.Action != sourceActionCheck || result.Warning != "" {
|
||||
t.Errorf("UpdateSourceAs(GitHub) result = %+v, want initial check receipt without warning", result)
|
||||
}
|
||||
execution, err := repository.GetTaskExecutionByTaskID(ctx, result.CheckTask.TaskID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskExecutionByTaskID(%q) error = %v, want nil", result.CheckTask.TaskID, err)
|
||||
}
|
||||
var actionPayload SourceActionPayload
|
||||
if err := json.Unmarshal([]byte(execution.Payload), &actionPayload); err != nil {
|
||||
t.Fatalf("json.Unmarshal(initial check payload) error = %v, want nil", err)
|
||||
}
|
||||
if actionPayload.Actor != "user:42" || actionPayload.Action != sourceActionCheck ||
|
||||
actionPayload.TargetRevision != "" || actionPayload.ConfirmedRevision != "" {
|
||||
t.Errorf("initial check payload = %+v, want real actor and credential-free check", actionPayload)
|
||||
}
|
||||
source, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if got, want := source.GitHubRepository, "OpenFlare/site"; got != want {
|
||||
t.Errorf("GitHubRepository = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.ReleaseSelector, githubReleaseSelectorLatest; got != want {
|
||||
t.Errorf("ReleaseSelector = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.AssetName, defaultGitHubAssetName; got != want {
|
||||
t.Errorf("AssetName = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := source.CheckIntervalMinutes, defaultCheckInterval; got != want {
|
||||
t.Errorf("CheckIntervalMinutes = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := source.SourceIdentity, "dbbd25307aaa3b88bc25353476940a049428655bd8421ac63045fdcb5fb23c9d"; got != want {
|
||||
t.Errorf("SourceIdentity = %q, want %q", got, want)
|
||||
}
|
||||
if runtime.NextCheckAt == nil {
|
||||
t.Error("GitHub latest NextCheckAt = nil, want scheduled value")
|
||||
}
|
||||
|
||||
var taskCount int64
|
||||
if err := db.DB(ctx).Model(&model.TaskExecution{}).Count(&taskCount).Error; err != nil {
|
||||
t.Fatalf("count initial checks error = %v, want nil", err)
|
||||
}
|
||||
if _, err := UpdateSourceAs(ctx, project.ID, input, "user:42"); err != nil {
|
||||
t.Fatalf("UpdateSourceAs(GitHub no-op) error = %v, want nil", err)
|
||||
}
|
||||
var noOpTaskCount int64
|
||||
if err := db.DB(ctx).Model(&model.TaskExecution{}).Count(&noOpTaskCount).Error; err != nil {
|
||||
t.Fatalf("count no-op checks error = %v, want nil", err)
|
||||
}
|
||||
if noOpTaskCount != taskCount {
|
||||
t.Errorf("no-op initial check count = %d, want unchanged %d", noOpTaskCount, taskCount)
|
||||
}
|
||||
|
||||
secret := "provider-switch-secret"
|
||||
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: "https://artifacts.example.com/site.zip?token=" + secret,
|
||||
AllowInsecure: false,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateSource(GitHub to Remote) error = %v, want nil", err)
|
||||
}
|
||||
remote, _ := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if remote.GitHubRepository != "" || remote.ReleaseSelector != "" || remote.AssetName != "" ||
|
||||
remote.AutoUpdateEnabled || remote.CheckIntervalMinutes != 0 {
|
||||
t.Errorf("Remote switched source retained GitHub fields: %+v", remote)
|
||||
}
|
||||
if _, err := UpdateSourceAs(ctx, project.ID, input, "user:42"); err != nil {
|
||||
t.Fatalf("UpdateSourceAs(Remote to GitHub) error = %v, want nil", err)
|
||||
}
|
||||
github, _ := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if github.RemoteURL != "" || github.AllowInsecure {
|
||||
t.Errorf("GitHub switched source retained Remote fields: URL=%q allow_insecure=%v", github.RemoteURL, github.AllowInsecure)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubSourceSaveSurvivesInitialCheckDispatchFailure(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
// Isolate from other tests that may leave a global Asynq client registered.
|
||||
task.SetService(nil)
|
||||
t.Cleanup(func() { task.SetService(nil) })
|
||||
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-dispatch-warning")
|
||||
result, err := UpdateSourceAs(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
}, "user:9")
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateSourceAs(dispatch failure) error = %v, want saved source with warning", err)
|
||||
}
|
||||
if result.CheckTask != nil || result.Warning != errPagesSourceInitialCheckWarning {
|
||||
t.Errorf("UpdateSourceAs(dispatch failure) result = %+v, want warning and nil check task", result)
|
||||
}
|
||||
source, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if source.GitHubRepository != "a/b" || runtime.SyncStatus != pagesSourceStatusFailed ||
|
||||
runtime.LastError != errPagesSourceInitialCheckWarning {
|
||||
t.Errorf("saved source/runtime = repo:%q status:%q error:%q", source.GitHubRepository, runtime.SyncStatus, runtime.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubSourceRejectsUnsafeOrModeIncompatibleFields(t *testing.T) {
|
||||
tests := []SourceUpdateInput{
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "http://github.com/a/b"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a%20b/repo"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b/extra"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com//a/b"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b/"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b?"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b#"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", AssetName: "dist\n.zip"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", AssetName: "dist\u202e.zip"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", AssetName: "dir/dist.zip"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: "v1", CheckIntervalMinutes: 60},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: "v1", AutoUpdateEnabled: true},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: " v1"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: "v1\n"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: "v1\u2028draft"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: `v1\draft`},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: "release//v1"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: "release/.draft"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "tag", ReleaseTag: "release/v1.lock"},
|
||||
{SourceType: PagesSourceTypeGitHubRelease, RepositoryURL: "https://github.com/a/b", ReleaseSelector: "latest", ReleaseTag: "v1"},
|
||||
}
|
||||
for _, input := range tests {
|
||||
if err := validateGitHubSourceInput(input); err == nil {
|
||||
t.Errorf("validateGitHubSourceInput(%+v) error = nil, want non-nil", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubSourceAcceptsLegalAssetAndTagCharacters(t *testing.T) {
|
||||
tests := []SourceUpdateInput{
|
||||
{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
AssetName: "dist?channel=stable&part#1.zip",
|
||||
},
|
||||
{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
AssetName: " dist.zip ",
|
||||
},
|
||||
{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
ReleaseSelector: "tag",
|
||||
ReleaseTag: "release/v1#stable&build=1",
|
||||
AssetName: "dist.zip",
|
||||
},
|
||||
{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b.git",
|
||||
ReleaseSelector: "tag",
|
||||
ReleaseTag: "@",
|
||||
AssetName: "dist.zip",
|
||||
},
|
||||
{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
ReleaseSelector: "tag",
|
||||
ReleaseTag: "release/v1.LOCK",
|
||||
AssetName: "dist.zip",
|
||||
},
|
||||
}
|
||||
for _, input := range tests {
|
||||
if err := validateGitHubSourceInput(input); err != nil {
|
||||
t.Errorf("validateGitHubSourceInput(%+v) error = %v, want nil", input, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitialCheckFailureUsesExactConfigFence(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-initial-fence")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
staleVersion := source.ConfigVersion
|
||||
if err := db.DB(ctx).Model(source).Update("config_version", staleVersion+1).Error; err != nil {
|
||||
t.Fatalf("increment source config version error = %v, want nil", err)
|
||||
}
|
||||
markInitialCheckDispatchFailed(ctx, source.ID, staleVersion)
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.SyncStatus != pagesSourceStatusIdle || runtime.LastError != "" {
|
||||
t.Errorf("stale initial failure runtime = status:%q error:%q, want unchanged idle", runtime.SyncStatus, runtime.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubCheckUsesETagAndDetectsSameReleaseReplacement(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-check")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
appliedRevision := strings.Repeat("a", 64)
|
||||
appliedDetail := `{"provider":"github","release_id":"100","asset_id":"1","tag":"release/v1","asset_name":"dist.zip"}`
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).Where("source_id = ?", source.ID).Updates(map[string]any{
|
||||
"etag": `"old-etag"`,
|
||||
"last_applied_revision": appliedRevision,
|
||||
"last_applied_detail": appliedDetail,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed GitHub runtime error = %v, want nil", err)
|
||||
}
|
||||
updatedAt := time.Date(2026, 7, 19, 10, 0, 0, 0, time.UTC)
|
||||
var gotETag string
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(_ context.Context, request githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
gotETag = request.ETag
|
||||
return githubrelease.ResolveResult{
|
||||
ETag: `"new-etag"`,
|
||||
Release: githubrelease.Release{ID: "100", Tag: "release/v1"},
|
||||
Asset: githubrelease.Asset{ID: "2", Name: "dist.zip", State: "uploaded", UpdatedAt: updatedAt},
|
||||
}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("Download called during check, want resolve only")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionCheck)
|
||||
if err != nil || outcome != sourceLeaseAcquired {
|
||||
t.Fatalf("acquire check lease = (%+v, %q, %v), want acquired", snapshot, outcome, err)
|
||||
}
|
||||
result, err := checkGitHubSource(ctx, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("checkGitHubSource() error = %v, want nil", err)
|
||||
}
|
||||
if result.Stale {
|
||||
t.Error("checkGitHubSource() stale = true, want false")
|
||||
}
|
||||
if got, want := gotETag, `"old-etag"`; got != want {
|
||||
t.Errorf("Resolve ETag = %q, want %q", got, want)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.SyncStatus != pagesSourceStatusAttention || runtime.LastSeenRevision == "" {
|
||||
t.Errorf("replacement runtime = status:%q seen:%q, want attention with revision", runtime.SyncStatus, runtime.LastSeenRevision)
|
||||
}
|
||||
view, err := GetSource(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSource() error = %v, want nil", err)
|
||||
}
|
||||
if view.LastSeen == nil || view.LastSeen.Label != "release/v1" {
|
||||
t.Errorf("LastSeen = %+v, want full tag with slash", view.LastSeen)
|
||||
}
|
||||
if err := preflightGitHubSyncConfirmation(ctx, source.ID, ""); err == nil || err.Error() != errPagesSourceConfirmationNeeded {
|
||||
t.Errorf("preflight without confirmation error = %v, want %q", err, errPagesSourceConfirmationNeeded)
|
||||
}
|
||||
if err := preflightGitHubSyncConfirmation(ctx, source.ID, runtime.LastSeenRevision); err != nil {
|
||||
t.Errorf("preflight exact confirmation error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubCheckNotModifiedRefreshesRuntimeWithoutDeployment(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-304")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(_ context.Context, request githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{NotModified: true, ETag: `"same"`}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("Download called for 304 check")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionCheck)
|
||||
if _, err := checkGitHubSource(ctx, snapshot); err != nil {
|
||||
t.Fatalf("checkGitHubSource(304) error = %v, want nil", err)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.ETag != `"same"` || runtime.LastCheckedAt == nil || runtime.NextCheckAt == nil || runtime.LeaseToken != "" {
|
||||
t.Errorf("304 runtime = %+v, want refreshed timestamps/etag and released lease", runtime)
|
||||
}
|
||||
var deployments int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("project_id = ?", project.ID).Count(&deployments).Error; err != nil {
|
||||
t.Fatalf("count deployments error = %v, want nil", err)
|
||||
}
|
||||
if deployments != 0 {
|
||||
t.Errorf("deployments after check = %d, want 0", deployments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubTargetMismatchPreservesAttentionAndExpeditesRecheck(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-target-mismatch")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
appliedRevision := strings.Repeat("a", 64)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).Where("source_id = ?", source.ID).Updates(map[string]any{
|
||||
"last_applied_revision": appliedRevision,
|
||||
"last_applied_detail": `{"provider":"github","release_id":"100","asset_id":"1","tag":"v1","asset_name":"dist.zip"}`,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed applied runtime error = %v, want nil", err)
|
||||
}
|
||||
retryAt := time.Now().Add(2 * time.Hour).UTC()
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{
|
||||
Release: githubrelease.Release{ID: "100", Tag: "v1"},
|
||||
Asset: githubrelease.Asset{
|
||||
ID: "2", Name: "dist.zip", State: "uploaded",
|
||||
UpdatedAt: time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC),
|
||||
},
|
||||
RetryAt: &retryAt,
|
||||
}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("Download called after target mismatch")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
outcome, err := syncGitHubSource(ctx, snapshot, pagesSourceCreatedBySystem, strings.Repeat("b", 64), "")
|
||||
if err != nil {
|
||||
t.Fatalf("syncGitHubSource(target mismatch) error = %v, want nil stale outcome", err)
|
||||
}
|
||||
if outcome == nil || !outcome.Stale {
|
||||
t.Errorf("syncGitHubSource(target mismatch) = %+v, want stale", outcome)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.SyncStatus != pagesSourceStatusAttention {
|
||||
t.Errorf("target mismatch SyncStatus = %q, want %q", runtime.SyncStatus, pagesSourceStatusAttention)
|
||||
}
|
||||
if runtime.NextCheckAt == nil || runtime.NextCheckAt.Before(retryAt) {
|
||||
t.Errorf("target mismatch NextCheckAt = %v, want server deadline >= %v", runtime.NextCheckAt, retryAt)
|
||||
}
|
||||
var deployments int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("project_id = ?", project.ID).Count(&deployments).Error; err != nil {
|
||||
t.Fatalf("count mismatch deployments error = %v, want nil", err)
|
||||
}
|
||||
if deployments != 0 {
|
||||
t.Errorf("target mismatch deployments = %d, want 0", deployments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubCheckLostLeaseReturnsStaleWithoutOverwritingRuntime(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-check-fence")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{}, errors.New("transient provider failure")
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
return nil, errors.New("unexpected")
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionCheck)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).Where("source_id = ?", source.ID).Updates(map[string]any{
|
||||
"lease_token": "new-owner",
|
||||
"lease_expires_at": time.Now().Add(time.Minute),
|
||||
"sync_status": pagesSourceStatusSyncing,
|
||||
"last_error": "new-owner-state",
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("replace lease owner error = %v, want nil", err)
|
||||
}
|
||||
result, err := checkGitHubSource(ctx, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("checkGitHubSource(lost lease) error = %v, want stale no-op", err)
|
||||
}
|
||||
if result == nil || !result.Stale {
|
||||
t.Errorf("checkGitHubSource(lost lease) = %+v, want stale", result)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.LeaseToken != "new-owner" || runtime.LastError != "new-owner-state" || runtime.SyncStatus != pagesSourceStatusSyncing {
|
||||
t.Errorf("lost lease runtime = token:%q error:%q status:%q, want new owner state", runtime.LeaseToken, runtime.LastError, runtime.SyncStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubCheckRateLimitUsesServerDeadlineAndSuppressesFastRetry(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-rate-limit")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
retryAt := time.Now().Add(2 * time.Hour).UTC()
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{}, &githubrelease.Error{
|
||||
Kind: githubrelease.ErrMetadata, StatusCode: 429, RetryAt: &retryAt,
|
||||
}
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
return nil, errors.New("unexpected")
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionCheck)
|
||||
_, err := checkGitHubSource(ctx, snapshot)
|
||||
if err == nil || err.Error() != errPagesSourceSyncFailed {
|
||||
t.Fatalf("checkGitHubSource(rate limit) error = %v, want safe sync failure", err)
|
||||
}
|
||||
if !shouldSkipGitHubActionRetry(err) {
|
||||
t.Error("shouldSkipGitHubActionRetry(rate limit) = false, want true")
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.NextCheckAt == nil || runtime.NextCheckAt.Before(retryAt) || runtime.SyncStatus != pagesSourceStatusFailed {
|
||||
t.Errorf("rate limit runtime = next:%v status:%q, want deadline >= %v and failed", runtime.NextCheckAt, runtime.SyncStatus, retryAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubCheckInvalidResolvedTargetUsesServerDeadline(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-invalid-check-target")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
retryAt := time.Now().Add(2 * time.Hour).UTC()
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{
|
||||
Release: githubrelease.Release{ID: "1", Tag: "v1"},
|
||||
Asset: githubrelease.Asset{
|
||||
ID: "2", Name: "dist.zip", State: "uploaded",
|
||||
UpdatedAt: time.Now().UTC(), Digest: "sha256:invalid",
|
||||
},
|
||||
RetryAt: &retryAt,
|
||||
}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("Download called after invalid check target")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionCheck)
|
||||
_, err := checkGitHubSource(ctx, snapshot)
|
||||
if err == nil || err.Error() != errPagesSourceDigestInvalid {
|
||||
t.Fatalf("checkGitHubSource(invalid target) error = %v, want %q", err, errPagesSourceDigestInvalid)
|
||||
}
|
||||
if !isPermanentSourceSyncError(err) {
|
||||
t.Error("invalid check target classification = retryable, want permanent")
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.SyncStatus != pagesSourceStatusFailed || runtime.LastError != errPagesSourceDigestInvalid ||
|
||||
runtime.NextCheckAt == nil || runtime.NextCheckAt.Before(retryAt) || runtime.LeaseToken != "" {
|
||||
t.Errorf(
|
||||
"invalid check target runtime = status:%q error:%q next:%v lease:%q, want failed/%q/deadline >= %v/cleared",
|
||||
runtime.SyncStatus, runtime.LastError, runtime.NextCheckAt, runtime.LeaseToken,
|
||||
errPagesSourceDigestInvalid, retryAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubSyncInvalidResolvedTargetUsesServerDeadline(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-invalid-sync-target")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
retryAt := time.Now().Add(2 * time.Hour).UTC()
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{
|
||||
Release: githubrelease.Release{ID: "1", Tag: "v1"},
|
||||
Asset: githubrelease.Asset{
|
||||
ID: "2", Name: "dist.zip", State: "uploaded",
|
||||
UpdatedAt: time.Now().UTC(), Digest: "sha256:invalid",
|
||||
},
|
||||
RetryAt: &retryAt,
|
||||
}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("Download called after invalid sync target")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
_, err := syncGitHubSource(ctx, snapshot, "user:7", "", "")
|
||||
if err == nil || err.Error() != errPagesSourceDigestInvalid {
|
||||
t.Fatalf("syncGitHubSource(invalid target) error = %v, want %q", err, errPagesSourceDigestInvalid)
|
||||
}
|
||||
if !isPermanentSourceSyncError(err) {
|
||||
t.Error("invalid sync target classification = retryable, want permanent")
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.SyncStatus != pagesSourceStatusFailed || runtime.LastError != errPagesSourceDigestInvalid ||
|
||||
runtime.NextCheckAt == nil || runtime.NextCheckAt.Before(retryAt) || runtime.LeaseToken != "" {
|
||||
t.Errorf(
|
||||
"invalid sync target runtime = status:%q error:%q next:%v lease:%q, want failed/%q/deadline >= %v/cleared",
|
||||
runtime.SyncStatus, runtime.LastError, runtime.NextCheckAt, runtime.LeaseToken,
|
||||
errPagesSourceDigestInvalid, retryAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubCheckHandlerSkipsProviderFastRetry(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status int
|
||||
retryDate bool
|
||||
}{
|
||||
{name: "bad request", status: http.StatusBadRequest},
|
||||
{name: "rate limited forbidden", status: http.StatusForbidden, retryDate: true},
|
||||
{name: "too many requests", status: http.StatusTooManyRequests, retryDate: true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-handler-"+strings.ReplaceAll(test.name, " ", "-"))
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
var retryAt *time.Time
|
||||
if test.retryDate {
|
||||
deadline := time.Now().Add(2 * time.Hour).UTC()
|
||||
retryAt = &deadline
|
||||
}
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{}, &githubrelease.Error{
|
||||
Kind: githubrelease.ErrMetadata, StatusCode: test.status, RetryAt: retryAt,
|
||||
}
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("Download called after provider check failure")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
raw, err := json.Marshal(SourceActionPayload{
|
||||
SourceID: source.ID, ConfigVersion: source.ConfigVersion,
|
||||
Action: sourceActionCheck, Actor: "user:7",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(check payload) error = %v, want nil", err)
|
||||
}
|
||||
result, err := (&SourceActionHandler{}).Execute(ctx, raw)
|
||||
if result != nil || err == nil || !errors.Is(err, asynq.SkipRetry) {
|
||||
t.Fatalf("SourceActionHandler.Execute(status %d) = result:%+v error:%v, want SkipRetry", test.status, result, err)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.SyncStatus != pagesSourceStatusFailed || runtime.NextCheckAt == nil || runtime.LeaseToken != "" {
|
||||
t.Errorf("provider failure runtime = status:%q next:%v lease:%q", runtime.SyncStatus, runtime.NextCheckAt, runtime.LeaseToken)
|
||||
}
|
||||
if retryAt != nil && (runtime.NextCheckAt == nil || runtime.NextCheckAt.Before(*retryAt)) {
|
||||
t.Errorf("provider failure NextCheckAt = %v, want deadline >= %v", runtime.NextCheckAt, *retryAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubSyncActivatesWithMetadataRevisionAndPackageChecksum(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-sync")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
AssetName: "dist.zip",
|
||||
})
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "github-v1"})
|
||||
packageHash := sha256.Sum256(packageBytes)
|
||||
updatedAt := time.Date(2026, 7, 19, 11, 0, 0, 0, time.UTC)
|
||||
retryAt := time.Now().Add(2 * time.Hour).UTC()
|
||||
release := githubrelease.Release{ID: "200", Tag: "release/v2"}
|
||||
asset := githubrelease.Asset{ID: "10", Name: "dist.zip", State: "uploaded", UpdatedAt: updatedAt}
|
||||
client := &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{Release: release, Asset: asset, RetryAt: &retryAt}, nil
|
||||
},
|
||||
download: func(_ context.Context, request githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
path := filepath.Join(t.TempDir(), "download")
|
||||
if err := os.WriteFile(path, packageBytes, 0o600); err != nil {
|
||||
t.Fatalf("os.WriteFile(download) error = %v, want nil", err)
|
||||
}
|
||||
return &githubrelease.DownloadResult{
|
||||
Path: path, Size: int64(len(packageBytes)), SHA256: hex.EncodeToString(packageHash[:]),
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
useFakeGitHubReleaseClient(t, client)
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
outcome, err := syncGitHubSource(ctx, snapshot, "user:7", "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("syncGitHubSource() error = %v, want nil", err)
|
||||
}
|
||||
if outcome == nil || outcome.Deployment == nil || outcome.Stale {
|
||||
t.Fatalf("syncGitHubSource() = %+v, want active deployment", outcome)
|
||||
}
|
||||
deployment, err := repository.GetPagesDeploymentByID(ctx, outcome.Deployment.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", outcome.Deployment.ID, err)
|
||||
}
|
||||
if got, want := deployment.Checksum, hex.EncodeToString(packageHash[:]); got != want {
|
||||
t.Errorf("deployment Checksum = %q, want package hash %q", got, want)
|
||||
}
|
||||
if deployment.SourceRevision == nil || *deployment.SourceRevision == deployment.Checksum {
|
||||
t.Errorf("deployment SourceRevision = %v, want metadata revision distinct from package checksum", deployment.SourceRevision)
|
||||
}
|
||||
if got, want := deployment.SourceLabel, "release/v2"; got != want {
|
||||
t.Errorf("deployment SourceLabel = %q, want %q", got, want)
|
||||
}
|
||||
if deployment.SourceType != PagesSourceTypeGitHubRelease || deployment.TriggerType != pagesSourceTriggerManualSync ||
|
||||
deployment.CreatedBy != "user:7" {
|
||||
t.Errorf("deployment provenance = type:%q trigger:%q actor:%q", deployment.SourceType, deployment.TriggerType, deployment.CreatedBy)
|
||||
}
|
||||
if strings.Contains(deployment.SourceMeta, "http") || strings.Contains(deployment.SourceMeta, "token") {
|
||||
t.Errorf("deployment SourceMeta = %q, want no URL or token", deployment.SourceMeta)
|
||||
}
|
||||
if !strings.Contains(deployment.SourceMeta, `"tag":"release/v2"`) || strings.Contains(deployment.SourceMeta, `"label"`) {
|
||||
t.Errorf("deployment SourceMeta = %q, want provider-specific tag field", deployment.SourceMeta)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.NextCheckAt == nil || runtime.NextCheckAt.Before(retryAt) {
|
||||
t.Errorf("sync runtime NextCheckAt = %v, want server deadline >= %v", runtime.NextCheckAt, retryAt)
|
||||
}
|
||||
|
||||
secondSnapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
second, err := syncGitHubSource(ctx, secondSnapshot, "user:7", "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("syncGitHubSource(idempotent) error = %v, want nil", err)
|
||||
}
|
||||
if second == nil || !second.Reused || second.Deployment == nil || second.Deployment.ID != outcome.Deployment.ID {
|
||||
t.Errorf("syncGitHubSource(idempotent) = %+v, want reused deployment %d", second, outcome.Deployment.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubSyncActivatesExactConfirmedReplacement(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-confirm-replacement")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
AssetName: "dist.zip",
|
||||
})
|
||||
release := githubrelease.Release{ID: "300", Tag: "v3"}
|
||||
asset := githubrelease.Asset{
|
||||
ID: "12", Name: "dist.zip", State: "uploaded",
|
||||
UpdatedAt: time.Date(2026, 7, 19, 13, 0, 0, 0, time.UTC),
|
||||
}
|
||||
target, err := buildGitHubSourceTarget(release, asset, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("buildGitHubSourceTarget() error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).Where("source_id = ?", source.ID).Updates(map[string]any{
|
||||
"last_seen_revision": target.Revision,
|
||||
"last_seen_detail": target.DetailJSON,
|
||||
"last_applied_revision": strings.Repeat("a", 64),
|
||||
"last_applied_detail": `{"provider":"github","release_id":"300","asset_id":"11","tag":"v3","asset_name":"dist.zip"}`,
|
||||
"sync_status": pagesSourceStatusAttention,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed replacement cursor error = %v, want nil", err)
|
||||
}
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "confirmed-v3"})
|
||||
packageHash := sha256.Sum256(packageBytes)
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{Release: release, Asset: asset}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
path := filepath.Join(t.TempDir(), "confirmed.zip")
|
||||
if err := os.WriteFile(path, packageBytes, 0o600); err != nil {
|
||||
t.Fatalf("os.WriteFile(confirmed package) error = %v, want nil", err)
|
||||
}
|
||||
return &githubrelease.DownloadResult{
|
||||
Path: path, Size: int64(len(packageBytes)), SHA256: hex.EncodeToString(packageHash[:]),
|
||||
}, nil
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
outcome, err := syncGitHubSource(ctx, snapshot, "user:9", "", target.Revision)
|
||||
if err != nil {
|
||||
t.Fatalf("syncGitHubSource(confirmed replacement) error = %v, want nil", err)
|
||||
}
|
||||
if outcome == nil || outcome.Deployment == nil || outcome.Stale {
|
||||
t.Fatalf("syncGitHubSource(confirmed replacement) = %+v, want active deployment", outcome)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.SyncStatus != pagesSourceStatusIdle || runtime.LastAppliedRevision != target.Revision {
|
||||
t.Errorf("confirmed replacement runtime = status:%q applied:%q, want idle/%q", runtime.SyncStatus, runtime.LastAppliedRevision, target.Revision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceActionPayloadSeparatesSystemTargetAndUserConfirmation(t *testing.T) {
|
||||
handler := &SourceActionHandler{}
|
||||
revision := strings.Repeat("a", 64)
|
||||
invalid := []SourceActionPayload{
|
||||
{SourceID: 1, ConfigVersion: 1, Action: sourceActionSync, Actor: "user:1", TargetRevision: revision},
|
||||
{SourceID: 1, ConfigVersion: 1, Action: sourceActionSync, Actor: pagesSourceCreatedBySystem, TriggerType: pagesSourceTriggerManualSync},
|
||||
{SourceID: 1, ConfigVersion: 1, Action: sourceActionSync, Actor: pagesSourceCreatedBySystem, ConfirmedRevision: revision},
|
||||
{SourceID: 1, ConfigVersion: 1, Action: sourceActionSync, Actor: pagesSourceCreatedBySystem, TargetRevision: revision, ConfirmedRevision: revision},
|
||||
}
|
||||
for _, payload := range invalid {
|
||||
raw, _ := json.Marshal(payload)
|
||||
if normalized, err := handler.ValidatePayload(raw); err == nil {
|
||||
t.Errorf("ValidatePayload(%+v) = %s, nil; want error", payload, normalized)
|
||||
}
|
||||
}
|
||||
valid := []SourceActionPayload{
|
||||
{SourceID: 1, ConfigVersion: 1, Action: sourceActionSync, Actor: pagesSourceCreatedBySystem, TriggerType: pagesSourceTriggerScheduledAutoUpdate, TargetRevision: revision},
|
||||
{SourceID: 1, ConfigVersion: 1, Action: sourceActionSync, Actor: "user:1", TriggerType: pagesSourceTriggerManualSync, ConfirmedRevision: revision},
|
||||
}
|
||||
for _, payload := range valid {
|
||||
raw, _ := json.Marshal(payload)
|
||||
if _, err := handler.ValidatePayload(raw); err != nil {
|
||||
t.Errorf("ValidatePayload(%+v) error = %v, want nil", payload, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubProviderErrorsMapToSafeRetryClassification(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
provider error
|
||||
want string
|
||||
permanent bool
|
||||
skipRetry bool
|
||||
}{
|
||||
{
|
||||
name: "asset missing", provider: &githubrelease.Error{Kind: githubrelease.ErrAssetNotFound, StatusCode: 200},
|
||||
want: errPagesSourceReleaseNotFound, permanent: true, skipRetry: true,
|
||||
},
|
||||
{
|
||||
name: "digest mismatch", provider: &githubrelease.Error{Kind: githubrelease.ErrDigestMismatch, StatusCode: 200},
|
||||
want: errPagesSourceDigestMismatch, permanent: true, skipRetry: true,
|
||||
},
|
||||
{
|
||||
name: "rate limit", provider: &githubrelease.Error{
|
||||
Kind: githubrelease.ErrMetadata, StatusCode: 429,
|
||||
RetryAt: func() *time.Time { value := time.Now().Add(time.Hour); return &value }(),
|
||||
},
|
||||
want: errPagesSourceSyncFailed, permanent: false, skipRetry: true,
|
||||
},
|
||||
{
|
||||
name: "network", provider: &githubrelease.Error{Kind: githubrelease.ErrDownload},
|
||||
want: errPagesSourceSyncFailed, permanent: false, skipRetry: false,
|
||||
},
|
||||
{
|
||||
name: "forbidden without retry", provider: &githubrelease.Error{Kind: githubrelease.ErrMetadata, StatusCode: 403},
|
||||
want: errPagesSourceSyncFailed, permanent: true, skipRetry: true,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
domainErr := githubSourceDomainError(test.provider)
|
||||
if got := domainErr.Error(); got != test.want {
|
||||
t.Errorf("githubSourceDomainError() = %q, want %q", got, test.want)
|
||||
}
|
||||
var typedDomainError *githubSourceProviderDomainError
|
||||
if !errors.As(domainErr, &typedDomainError) {
|
||||
t.Fatalf("githubSourceDomainError() type = %T, want *githubSourceProviderDomainError", domainErr)
|
||||
}
|
||||
if got := typedDomainError.permanent; got != test.permanent {
|
||||
t.Errorf("githubSourceProviderDomainError.permanent = %t, want %t", got, test.permanent)
|
||||
}
|
||||
if got := shouldSkipGitHubActionRetry(domainErr); got != test.skipRetry {
|
||||
t.Errorf("shouldSkipGitHubActionRetry() = %t, want %t", got, test.skipRetry)
|
||||
}
|
||||
if strings.Contains(domainErr.Error(), "status=") || strings.Contains(domainErr.Error(), "repo=") {
|
||||
t.Errorf("githubSourceDomainError() = %q, want stable Pages message", domainErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubSyncRejectsStaleConfirmationWithoutChangingActive(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "github-confirm-stale")
|
||||
oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/a/b",
|
||||
})
|
||||
asset := githubrelease.Asset{ID: "2", Name: "dist.zip", State: "uploaded", UpdatedAt: time.Now().UTC()}
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{Release: githubrelease.Release{ID: "1", Tag: "v1"}, Asset: asset}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("Download called for stale confirmation")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
snapshot, _, _ := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
_, err := syncGitHubSource(ctx, snapshot, "user:1", "", strings.Repeat("f", 64))
|
||||
if err == nil || err.Error() != errPagesSourceConfirmationStale {
|
||||
t.Errorf("syncGitHubSource(stale confirmation) error = %v, want %q", err, errPagesSourceConfirmationStale)
|
||||
}
|
||||
storedProject, loadErr := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
if loadErr != nil {
|
||||
t.Fatalf("GetPagesProjectByID() error = %v, want nil", loadErr)
|
||||
}
|
||||
if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != oldActive.ID {
|
||||
t.Errorf("ActiveDeploymentID = %v, want old active %d", storedProject.ActiveDeploymentID, oldActive.ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/openflare/share/pagesarchive"
|
||||
"Wavelet/pkg/logger"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const (
|
||||
pagesMaxDeploymentFiles = 1000
|
||||
defaultPagesMaxPackageSizeMB = 100
|
||||
maxPagesMaxPackageSizeMB = 2048
|
||||
defaultPagesMaxHistoryCount = 20
|
||||
defaultPagesEntryFile = "index.html"
|
||||
defaultPagesFallbackPath = "/index.html"
|
||||
pagesIngestMarkerKey = "pages_ingest_marker"
|
||||
pagesIngestMarkerV2 = "pages_deployment_v2"
|
||||
pagesProjectIDMetadataKey = "pages_project_id"
|
||||
pagesSourceIDMetadataKey = "pages_source_id"
|
||||
pagesMaxPathLength = 512
|
||||
bytesPerMiB = 1024 * 1024
|
||||
pagesExtractedSizeMultiplier = 4
|
||||
pagesMinExtractedSizeBytes = 100 * bytesPerMiB
|
||||
pagesRowLockStrength = "UPDATE"
|
||||
)
|
||||
|
||||
var pagesSlugPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,126}[a-z0-9]$|^[a-z0-9]$`)
|
||||
|
||||
type deploymentManifest struct {
|
||||
Files []model.PagesDeploymentFile
|
||||
FileCount int
|
||||
TotalSize int64
|
||||
EntryFile string
|
||||
Format pagesarchive.Format
|
||||
}
|
||||
|
||||
type pagesLimits struct {
|
||||
PackageBytes int64
|
||||
ExtractedBytes int64
|
||||
MaxFiles int
|
||||
HistoryCount int
|
||||
}
|
||||
|
||||
func isUniqueConstraintError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(err.Error()), "unique")
|
||||
}
|
||||
|
||||
func resolvePagesLimits(ctx context.Context) pagesLimits {
|
||||
packageMB := defaultPagesMaxPackageSizeMB
|
||||
if value, err := repository.GetIntByKey(ctx, model.ConfigKeyPagesMaxPackageSizeMB); err == nil && value > 0 {
|
||||
packageMB = value
|
||||
}
|
||||
if packageMB > maxPagesMaxPackageSizeMB {
|
||||
packageMB = maxPagesMaxPackageSizeMB
|
||||
}
|
||||
|
||||
historyCount := defaultPagesMaxHistoryCount
|
||||
if value, err := repository.GetIntByKey(ctx, model.ConfigKeyPagesMaxHistoryCount); err == nil {
|
||||
historyCount = max(value, 0)
|
||||
}
|
||||
|
||||
packageBytes := int64(packageMB) * bytesPerMiB
|
||||
extractedBytes := max(packageBytes*pagesExtractedSizeMultiplier, pagesMinExtractedSizeBytes)
|
||||
|
||||
return pagesLimits{
|
||||
PackageBytes: packageBytes,
|
||||
ExtractedBytes: extractedBytes,
|
||||
MaxFiles: pagesMaxDeploymentFiles,
|
||||
HistoryCount: historyCount,
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePagesSlug(raw string) string {
|
||||
value := strings.ToLower(strings.TrimSpace(raw))
|
||||
var builder strings.Builder
|
||||
lastDash := false
|
||||
for _, r := range value {
|
||||
valid := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
|
||||
if valid {
|
||||
builder.WriteRune(r)
|
||||
lastDash = false
|
||||
continue
|
||||
}
|
||||
if !lastDash {
|
||||
builder.WriteByte('-')
|
||||
lastDash = true
|
||||
}
|
||||
}
|
||||
return strings.Trim(builder.String(), "-")
|
||||
}
|
||||
|
||||
func validateAndNormalizePagesRootDir(raw string) (string, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if len(value) > pagesMaxPathLength {
|
||||
return "", errors.New("pages 根目录长度不能超过 512")
|
||||
}
|
||||
normalized, err := pagesarchive.NormalizeLogicalPath(value, true)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pages 根目录不合法: %w", err)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func normalizePagesFallbackPath(raw string) (string, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
value = defaultPagesFallbackPath
|
||||
}
|
||||
if len(value) > pagesMaxPathLength {
|
||||
return "", errors.New("spa fallback 回退路径长度不能超过 512")
|
||||
}
|
||||
if !strings.HasPrefix(value, "/") {
|
||||
return "", errors.New("spa fallback 回退路径必须以 / 开头")
|
||||
}
|
||||
if value == "/" || strings.HasSuffix(value, "/") {
|
||||
return "", errors.New("spa fallback 回退路径必须指向具体文件")
|
||||
}
|
||||
if strings.Contains(value, "\\") || strings.ContainsAny(value, "\"';") {
|
||||
return "", errors.New("spa fallback 回退路径包含不支持的字符")
|
||||
}
|
||||
for _, r := range value {
|
||||
if r <= 0x20 || r == 0x7f {
|
||||
return "", errors.New("spa fallback 回退路径不能包含空白或控制字符")
|
||||
}
|
||||
}
|
||||
for segment := range strings.SplitSeq(value, "/") {
|
||||
if segment == "." || segment == ".." {
|
||||
return "", errors.New("spa fallback 回退路径不能包含 . 或 .. 路径段")
|
||||
}
|
||||
}
|
||||
cleaned := path.Clean(value)
|
||||
if cleaned == "." || !strings.HasPrefix(cleaned, "/") {
|
||||
return "", errors.New("spa fallback 回退路径不合法")
|
||||
}
|
||||
if cleaned == "/" || strings.HasSuffix(cleaned, "/") {
|
||||
return "", errors.New("spa fallback 回退路径必须指向具体文件")
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
func normalizeStoredPagesFallbackPath(value string) string {
|
||||
normalized, err := normalizePagesFallbackPath(value)
|
||||
if err != nil {
|
||||
return defaultPagesFallbackPath
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func validateAndNormalizePagesEntryFile(raw string) (string, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
value = defaultPagesEntryFile
|
||||
}
|
||||
if len(value) > pagesMaxPathLength {
|
||||
return "", errors.New("pages 入口文件长度不能超过 512")
|
||||
}
|
||||
normalized, err := pagesarchive.NormalizeLogicalPath(value, false)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pages 入口文件不合法: %w", err)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func persistPagesUploadTemp(fileHeader *multipart.FileHeader, maxPackageBytes int64) (string, string, int64, pagesarchive.Format, error) {
|
||||
format, ok := pagesarchive.DetectFormatFromName(fileHeader.Filename)
|
||||
if !ok {
|
||||
return "", "", 0, "", errors.New(errPagesPackageUnsupported)
|
||||
}
|
||||
file, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
return "", "", 0, "", err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
temp, err := os.CreateTemp("", "openflare-pages-*."+safeTempSuffix(format))
|
||||
if err != nil {
|
||||
return "", "", 0, "", err
|
||||
}
|
||||
defer func() { _ = temp.Close() }()
|
||||
hash := sha256.New()
|
||||
limited := io.LimitReader(file, maxPackageBytes+1)
|
||||
written, err := io.Copy(io.MultiWriter(temp, hash), limited)
|
||||
if err != nil {
|
||||
_ = os.Remove(temp.Name())
|
||||
return "", "", 0, "", err
|
||||
}
|
||||
if written > maxPackageBytes {
|
||||
_ = os.Remove(temp.Name())
|
||||
return "", "", 0, "", fmt.Errorf("pages 部署包不能超过 %d MiB", maxPackageBytes/bytesPerMiB)
|
||||
}
|
||||
return temp.Name(), hex.EncodeToString(hash.Sum(nil)), written, format, nil
|
||||
}
|
||||
|
||||
func safeTempSuffix(format pagesarchive.Format) string {
|
||||
switch format {
|
||||
case pagesarchive.FormatTarGz:
|
||||
return "tar.gz"
|
||||
case pagesarchive.FormatTarXz:
|
||||
return "tar.xz"
|
||||
case pagesarchive.FormatTarBz2:
|
||||
return "tar.bz2"
|
||||
case pagesarchive.FormatSevenZip:
|
||||
return "7z"
|
||||
case pagesarchive.FormatTar:
|
||||
return "tar"
|
||||
case pagesarchive.FormatZip:
|
||||
return "zip"
|
||||
default:
|
||||
return "zip"
|
||||
}
|
||||
}
|
||||
|
||||
func pagesLegacyRelativeCandidates(project *model.PagesProject, deployment *model.PagesDeployment) []string {
|
||||
if project == nil || deployment == nil {
|
||||
return nil
|
||||
}
|
||||
slug := strings.TrimSpace(project.Slug)
|
||||
checksum := strings.TrimSpace(deployment.Checksum)
|
||||
if slug == "" || checksum == "" {
|
||||
return nil
|
||||
}
|
||||
// Legacy artifacts were always stored as .zip.
|
||||
fileName := checksum + ".zip"
|
||||
return []string{
|
||||
filepath.Join("artifacts", slug, fileName),
|
||||
filepath.Join("pages", "artifacts", slug, fileName),
|
||||
filepath.Join("data", "pages", "artifacts", slug, fileName),
|
||||
}
|
||||
}
|
||||
|
||||
func ingestPagesDeploymentPackage(
|
||||
ctx context.Context,
|
||||
localPath string,
|
||||
checksum string,
|
||||
projectID uint,
|
||||
fileName string,
|
||||
format pagesarchive.Format,
|
||||
) (ofupload.IngestResult, error) {
|
||||
return ingestPagesDeploymentPackageWithSource(ctx, localPath, checksum, projectID, 0, fileName, format)
|
||||
}
|
||||
|
||||
func ingestPagesDeploymentPackageWithSource(
|
||||
ctx context.Context,
|
||||
localPath string,
|
||||
checksum string,
|
||||
projectID uint,
|
||||
sourceID uint,
|
||||
fileName string,
|
||||
format pagesarchive.Format,
|
||||
) (ofupload.IngestResult, error) {
|
||||
systemUser := repository.GetSystemUser(ctx)
|
||||
accessMode := 0
|
||||
extension := pagesarchive.NormalizeNameExtension(fileName, format)
|
||||
extra := map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10),
|
||||
}
|
||||
if sourceID != 0 {
|
||||
extra[pagesSourceIDMetadataKey] = strconv.FormatUint(uint64(sourceID), 10)
|
||||
}
|
||||
return ofupload.IngestFromLocalPath(ctx, localPath, ofupload.IngestRequest{
|
||||
UserID: systemUser.ID,
|
||||
FileName: fileName,
|
||||
MimeType: pagesarchive.MIMEType(format),
|
||||
Extension: extension,
|
||||
Hash: checksum,
|
||||
Type: ofupload.ReservedPagesDeploymentType,
|
||||
AccessMode: &accessMode,
|
||||
SkipExtensionCheck: true,
|
||||
Policy: ofupload.PolicyDedupNewRecord,
|
||||
Metadata: model.UploadMetadata{
|
||||
Extra: extra,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func removeDeploymentArtifact(ctx context.Context, projectID uint, deployment *model.PagesDeployment) {
|
||||
if deployment == nil {
|
||||
return
|
||||
}
|
||||
if deployment.UploadID == 0 {
|
||||
return
|
||||
}
|
||||
if err := removePagesUploadIfUnreferenced(ctx, projectID, deployment.UploadID); err != nil {
|
||||
logger.WarnF(ctx,
|
||||
"[Pages] remove deployment artifact failed: deployment_id=%d upload_id=%d error=%v",
|
||||
deployment.ID, deployment.UploadID, err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// removePagesUploadIfUnreferenced soft-deletes a reserved Pages upload only
|
||||
// after locking its project (when present), locking the upload, and rechecking
|
||||
// deployment references in the same transaction.
|
||||
func removePagesUploadIfUnreferenced(ctx context.Context, projectID uint, uploadID uint64) error {
|
||||
if uploadID == 0 {
|
||||
return nil
|
||||
}
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
if projectID != 0 {
|
||||
if _, projectErr := repository.LockPagesProjectByIDTx(tx, projectID); projectErr != nil &&
|
||||
!errors.Is(projectErr, gorm.ErrRecordNotFound) {
|
||||
return projectErr
|
||||
}
|
||||
}
|
||||
|
||||
var uploadRecord model.Upload
|
||||
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
|
||||
Where("id = ?", uploadID).
|
||||
First(&uploadRecord).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if uploadRecord.Type != ofupload.ReservedPagesDeploymentType {
|
||||
return fmt.Errorf("pages 部署包上传类型不匹配: %s", uploadRecord.Type)
|
||||
}
|
||||
|
||||
var references int64
|
||||
if err := tx.Model(&model.PagesDeployment{}).
|
||||
Where("upload_id = ?", uploadID).
|
||||
Count(&references).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if references > 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := ofupload.RemoveLockedTx(tx, &uploadRecord)
|
||||
return err
|
||||
})
|
||||
// Always invalidate after transaction completion, including idempotent no-op,
|
||||
// so a prior post-commit cache interruption can heal on retry.
|
||||
ofupload.InvalidateUploadMetaCache(ctx, uploadID)
|
||||
return err
|
||||
}
|
||||
|
||||
func inspectPagesPackage(packagePath string, format pagesarchive.Format, rootDir string, entryFile string, limits pagesLimits) (*deploymentManifest, error) {
|
||||
archiveManifest, err := pagesarchive.InspectFile(packagePath, format, pagesarchive.InspectOptions{
|
||||
RootDir: rootDir,
|
||||
EntryFile: entryFile,
|
||||
VerifySizes: true,
|
||||
Limits: pagesarchive.Limits{
|
||||
MaxFiles: limits.MaxFiles,
|
||||
MaxFileBytes: limits.ExtractedBytes,
|
||||
MaxTotalBytes: limits.ExtractedBytes,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, mapPagesArchiveError(err)
|
||||
}
|
||||
manifest := &deploymentManifest{
|
||||
Files: make([]model.PagesDeploymentFile, 0, len(archiveManifest.Files)),
|
||||
FileCount: archiveManifest.FileCount,
|
||||
TotalSize: archiveManifest.TotalSize,
|
||||
EntryFile: entryFile,
|
||||
Format: format,
|
||||
}
|
||||
for _, file := range archiveManifest.Files {
|
||||
manifest.Files = append(manifest.Files, model.PagesDeploymentFile{
|
||||
Path: file.Path,
|
||||
Size: file.Size,
|
||||
Checksum: file.Checksum,
|
||||
})
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func mapPagesArchiveError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
message := err.Error()
|
||||
switch {
|
||||
case strings.Contains(message, "unsupported pages package format"):
|
||||
return errors.New(errPagesPackageUnsupported)
|
||||
case strings.Contains(message, "open zip"), strings.Contains(message, "open gzip"),
|
||||
strings.Contains(message, "open xz"), strings.Contains(message, "open 7z"),
|
||||
strings.Contains(message, "read tar"):
|
||||
return errors.New(errPagesPackageInvalid)
|
||||
case strings.Contains(message, "empty"):
|
||||
return errors.New(errPagesPackageEmpty)
|
||||
case strings.Contains(message, "missing entry file"):
|
||||
return err
|
||||
case strings.Contains(message, "file count exceeds"):
|
||||
return fmt.Errorf("pages 部署文件数不能超过 %d", pagesMaxDeploymentFiles)
|
||||
case strings.Contains(message, "extracted size exceeds"):
|
||||
return errors.New(errPagesPackageExtractedTooLarge)
|
||||
case strings.Contains(message, "file too large"), strings.Contains(message, "size out of bounds"):
|
||||
return errors.New(errPagesPackageFileTooLarge)
|
||||
case strings.Contains(message, "symlink"):
|
||||
return err
|
||||
case strings.Contains(message, "absolute path"), strings.Contains(message, "escapes directory"):
|
||||
return err
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func packageDownloadName(deploymentID uint, fileName string, contentType string) string {
|
||||
if format, ok := pagesarchive.DetectFormatFromName(fileName); ok {
|
||||
return fmt.Sprintf("pages-deployment-%d.%s", deploymentID, pagesarchive.Extension(format))
|
||||
}
|
||||
// Fall back by content type.
|
||||
switch strings.ToLower(strings.TrimSpace(contentType)) {
|
||||
case "application/gzip", "application/x-gzip":
|
||||
return fmt.Sprintf("pages-deployment-%d.tar.gz", deploymentID)
|
||||
case "application/x-xz":
|
||||
return fmt.Sprintf("pages-deployment-%d.tar.xz", deploymentID)
|
||||
case "application/x-bzip2":
|
||||
return fmt.Sprintf("pages-deployment-%d.tar.bz2", deploymentID)
|
||||
case "application/x-7z-compressed":
|
||||
return fmt.Sprintf("pages-deployment-%d.7z", deploymentID)
|
||||
case "application/x-tar":
|
||||
return fmt.Sprintf("pages-deployment-%d.tar", deploymentID)
|
||||
default:
|
||||
return fmt.Sprintf("pages-deployment-%d.zip", deploymentID)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,794 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
oftask "Wavelet/openflare/plugins/server/kernel/task"
|
||||
"Wavelet/openflare/plugins/server/kernel/testhelper"
|
||||
"Wavelet/pkg/idgen"
|
||||
uploadshared "Wavelet/plugins/domain/upload/shared"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupPagesTestDB(t *testing.T) func() {
|
||||
t.Helper()
|
||||
|
||||
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sqliteDB.AutoMigrate(
|
||||
&model.User{},
|
||||
&model.Upload{},
|
||||
&model.UploadStat{},
|
||||
&model.TaskExecution{},
|
||||
&model.PagesProject{},
|
||||
&model.PagesDeployment{},
|
||||
&model.PagesDeploymentFile{},
|
||||
&model.PagesProjectSource{},
|
||||
&model.PagesProjectSourceRuntime{},
|
||||
&model.ConfigVersion{},
|
||||
&model.SystemConfig{},
|
||||
))
|
||||
require.NoError(t, sqliteDB.Create(&model.User{
|
||||
ID: 999,
|
||||
Username: "system",
|
||||
Password: "*",
|
||||
Nickname: "系统",
|
||||
IsActive: true,
|
||||
}).Error)
|
||||
require.NoError(t, sqliteDB.Create([]model.SystemConfig{
|
||||
{
|
||||
Key: model.ConfigKeyPagesMaxPackageSizeMB,
|
||||
Value: "100",
|
||||
Type: "business",
|
||||
Description: "Pages 部署包上传大小上限（MiB）",
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyPagesMaxHistoryCount,
|
||||
Value: "0", // unlimited for existing tests
|
||||
Type: "business",
|
||||
Description: "Pages 每个项目最大历史部署保留数（0 表示不限制）",
|
||||
},
|
||||
}).Error)
|
||||
|
||||
db.SetDB(sqliteDB)
|
||||
require.NoError(t, idgen.Init(1))
|
||||
oftask.SetService(&testhelper.NoopTaskService{})
|
||||
mockStorage := uploadshared.NewMockStorageService()
|
||||
uploadshared.SetDBService(db.NewService(sqliteDB))
|
||||
uploadshared.SetStorageService(mockStorage)
|
||||
ofupload.SetStorage(mockStorage)
|
||||
_ = repository.InvalidateSystemConfigCache(context.Background(), model.ConfigKeyPagesMaxPackageSizeMB)
|
||||
_ = repository.InvalidateSystemConfigCache(context.Background(), model.ConfigKeyPagesMaxHistoryCount)
|
||||
return func() {
|
||||
ofupload.SetStorage(nil)
|
||||
uploadshared.ResetServices()
|
||||
db.SetDB(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func setupPagesStorageMock(t *testing.T) (restore func(), disable func()) {
|
||||
t.Helper()
|
||||
mock := uploadshared.NewMockStorageService()
|
||||
uploadshared.SetStorageService(mock)
|
||||
ofupload.SetStorage(mock)
|
||||
restore = func() {
|
||||
ofupload.SetStorage(nil)
|
||||
uploadshared.ResetServices()
|
||||
}
|
||||
disable = restore
|
||||
return restore, disable
|
||||
}
|
||||
|
||||
func TestCreateProject(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Marketing Site",
|
||||
Slug: "marketing-site",
|
||||
Description: "public site",
|
||||
Enabled: true,
|
||||
SPAFallbackEnabled: true,
|
||||
SPAFallbackPath: "/index.html",
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, project.ID)
|
||||
assert.Equal(t, "Marketing Site", project.Name)
|
||||
assert.Equal(t, "marketing-site", project.Slug)
|
||||
assert.Equal(t, "public site", project.Description)
|
||||
assert.True(t, project.Enabled)
|
||||
assert.True(t, project.SPAFallbackEnabled)
|
||||
assert.Equal(t, "/index.html", project.SPAFallbackPath)
|
||||
assert.Equal(t, "index.html", project.EntryFile)
|
||||
assert.Equal(t, int64(0), project.DeploymentCount)
|
||||
|
||||
_, err = CreateProject(ctx, Input{
|
||||
Name: "Duplicate Slug",
|
||||
Slug: "marketing-site",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, errPagesSlugExists, err.Error())
|
||||
}
|
||||
|
||||
func TestCreateProjectRejectsUnsafeFallbackPath(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := CreateProject(ctx, Input{
|
||||
Name: "Unsafe Fallback",
|
||||
Slug: "unsafe-fallback",
|
||||
Enabled: true,
|
||||
SPAFallbackEnabled: true,
|
||||
SPAFallbackPath: "/index.html; proxy_pass http://evil",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "回退路径")
|
||||
}
|
||||
|
||||
func TestCreateProjectRejectsUnsafeContentPaths(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
rootDirs := []string{"/public", "public/../dist", "C:/public", `public\\dist`, "./public", "public\x00dist"}
|
||||
for index, rootDir := range rootDirs {
|
||||
_, err := CreateProject(ctx, Input{
|
||||
Name: fmt.Sprintf("Unsafe Root %d", index),
|
||||
Slug: fmt.Sprintf("unsafe-root-%d", index),
|
||||
RootDir: rootDir,
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
require.Error(t, err, rootDir)
|
||||
}
|
||||
|
||||
entryFiles := []string{"/index.html", "../index.html", "C:/index.html", `public\\index.html`, "./index.html", "index.html;bad"}
|
||||
for index, entryFile := range entryFiles {
|
||||
_, err := CreateProject(ctx, Input{
|
||||
Name: fmt.Sprintf("Unsafe Entry %d", index),
|
||||
Slug: fmt.Sprintf("unsafe-entry-%d", index),
|
||||
EntryFile: entryFile,
|
||||
})
|
||||
require.Error(t, err, entryFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateProjectValidatesActiveDeploymentEntry(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Content Root",
|
||||
Slug: "content-root",
|
||||
Enabled: true,
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
deployment, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "site.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "root",
|
||||
"dist/index.html": "dist",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
staleCandidate, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "stale.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "stale",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
_, err = ActivateDeployment(ctx, project.ID, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
updated, err := UpdateProject(ctx, project.ID, Input{
|
||||
Name: project.Name,
|
||||
Slug: project.Slug,
|
||||
Enabled: true,
|
||||
RootDir: "dist",
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "dist", updated.RootDir)
|
||||
|
||||
_, err = ActivateDeployment(ctx, project.ID, staleCandidate.ID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), errPagesEntryFileMissing)
|
||||
|
||||
_, err = UpdateProject(ctx, project.ID, Input{
|
||||
Name: project.Name,
|
||||
Slug: project.Slug,
|
||||
Enabled: true,
|
||||
RootDir: "missing",
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), errPagesEntryFileMissing)
|
||||
|
||||
stored, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "dist", stored.RootDir)
|
||||
assert.Equal(t, "index.html", stored.EntryFile)
|
||||
}
|
||||
|
||||
func TestUploadDeploymentAcceptsZeroByteFiles(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Zero Byte Site",
|
||||
Slug: "zero-byte-site",
|
||||
Enabled: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
deployment, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "site.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "ok",
|
||||
".gitkeep": "",
|
||||
})), "root")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, deployment.FileCount)
|
||||
}
|
||||
|
||||
func TestUploadDeploymentStoresPackageInUploadFramework(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Upload Framework Site",
|
||||
Slug: "upload-framework-site",
|
||||
Enabled: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
deployment, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "site.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "ok",
|
||||
})), "root")
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, deployment.UploadID)
|
||||
|
||||
storedDeployment, err := repository.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, storedDeployment.UploadID)
|
||||
assert.Empty(t, storedDeployment.ArtifactPath)
|
||||
|
||||
var uploadCount int64
|
||||
require.NoError(t, db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error)
|
||||
assert.Equal(t, int64(1), uploadCount)
|
||||
var uploadRecord model.Upload
|
||||
require.NoError(t, db.DB(ctx).First(&uploadRecord, storedDeployment.UploadID).Error)
|
||||
assert.Equal(t, ofupload.ReservedPagesDeploymentType, uploadRecord.Type)
|
||||
assert.Equal(t, pagesIngestMarkerV2, uploadRecord.Metadata.Extra[pagesIngestMarkerKey])
|
||||
assert.Equal(t, fmt.Sprint(project.ID), uploadRecord.Metadata.Extra[pagesProjectIDMetadataKey])
|
||||
assert.NotContains(t, uploadRecord.Metadata.Extra, "project_slug")
|
||||
assert.NotContains(t, uploadRecord.Metadata.Extra, "format")
|
||||
}
|
||||
|
||||
func TestOpenDeploymentPackageHydratesLegacyArtifactPath(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Legacy Site",
|
||||
Slug: "openspeedtest",
|
||||
Enabled: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
artifactDir := filepath.Join(t.TempDir(), "pages", "artifacts", project.Slug)
|
||||
require.NoError(t, os.MkdirAll(artifactDir, 0o755))
|
||||
artifactPath := filepath.Join(artifactDir, "legacy-checksum.zip")
|
||||
require.NoError(t, os.WriteFile(artifactPath, testPagesZip(t, map[string]string{"index.html": "legacy"}), 0o644))
|
||||
|
||||
deployment := &model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 1,
|
||||
Checksum: "legacy-checksum",
|
||||
Status: model.PagesDeploymentStatusUploaded,
|
||||
ArtifactPath: artifactPath,
|
||||
FileCount: 1,
|
||||
TotalSize: 10,
|
||||
CreatedBy: "test",
|
||||
}
|
||||
require.NoError(t, db.DB(ctx).Create(deployment).Error)
|
||||
require.NoError(t, db.DB(ctx).Create(&model.PagesDeploymentFile{
|
||||
DeploymentID: deployment.ID,
|
||||
Path: "index.html",
|
||||
Size: 6,
|
||||
Checksum: "legacy-checksum",
|
||||
}).Error)
|
||||
|
||||
_, err = ActivateDeployment(ctx, project.ID, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, db.DB(ctx).Create(&model.ConfigVersion{
|
||||
Version: "v2026-legacy",
|
||||
SnapshotJSON: fmt.Sprintf(`{"routes":[{"upstream_type":"pages","pages_deployment":{"deployment_id":%d}}]}`, deployment.ID),
|
||||
MainConfig: "",
|
||||
RenderedConfig: "",
|
||||
SupportFilesJSON: "[]",
|
||||
Checksum: "legacy-config-checksum",
|
||||
IsActive: true,
|
||||
CreatedBy: "test",
|
||||
}).Error)
|
||||
|
||||
packageObj, err := OpenDeploymentPackage(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
defer packageObj.Body.Close()
|
||||
assert.Equal(t, fmt.Sprintf("pages-deployment-%d.zip", deployment.ID), packageObj.FileName)
|
||||
|
||||
body, err := io.ReadAll(packageObj.Body)
|
||||
require.NoError(t, err)
|
||||
reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, reader.File, 1)
|
||||
assert.Equal(t, "index.html", reader.File[0].Name)
|
||||
|
||||
storedDeployment, err := repository.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, storedDeployment.UploadID)
|
||||
assert.Empty(t, storedDeployment.ArtifactPath)
|
||||
|
||||
var uploadCount int64
|
||||
require.NoError(t, db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error)
|
||||
assert.Equal(t, int64(1), uploadCount)
|
||||
|
||||
packageObj2, err := OpenDeploymentPackage(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
defer packageObj2.Body.Close()
|
||||
body2, err := io.ReadAll(packageObj2.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, body, body2)
|
||||
}
|
||||
|
||||
func TestOpenDeploymentPackageRequiresActiveConfigSnapshot(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Published Site",
|
||||
Slug: "published-site",
|
||||
Enabled: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
deployment, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "site.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "ok",
|
||||
})), "root")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = ActivateDeployment(ctx, project.ID, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = OpenDeploymentPackage(ctx, deployment.ID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "激活配置")
|
||||
|
||||
require.NoError(t, db.DB(ctx).Create(&model.ConfigVersion{
|
||||
Version: "v2026-001",
|
||||
SnapshotJSON: fmt.Sprintf(`{"routes":[{"upstream_type":"pages","pages_deployment":{"deployment_id":%d}}]}`, deployment.ID),
|
||||
MainConfig: "",
|
||||
RenderedConfig: "",
|
||||
SupportFilesJSON: "[]",
|
||||
Checksum: "test-checksum",
|
||||
IsActive: true,
|
||||
CreatedBy: "test",
|
||||
}).Error)
|
||||
|
||||
packageObj, err := OpenDeploymentPackage(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
defer packageObj.Body.Close()
|
||||
assert.Equal(t, fmt.Sprintf("pages-deployment-%d.zip", deployment.ID), packageObj.FileName)
|
||||
|
||||
// Latest-by-project resolves the active package once the project is on active config.
|
||||
depID, hash, err := GetProjectLatestPackageHash(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, deployment.ID, depID)
|
||||
assert.NotEmpty(t, hash)
|
||||
|
||||
latestPkg, err := OpenProjectLatestPackage(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
defer latestPkg.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(packageObj.Body)
|
||||
require.NoError(t, err)
|
||||
reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, reader.File, 1)
|
||||
assert.Equal(t, "index.html", reader.File[0].Name)
|
||||
}
|
||||
|
||||
func TestProjectLatestRejectsWhenNotOnActiveConfigOrNotActive(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{Name: "Gate", Slug: "gate", Enabled: true})
|
||||
require.NoError(t, err)
|
||||
d1, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "a.zip", testPagesZip(t, map[string]string{"index.html": "a"})), "root")
|
||||
require.NoError(t, err)
|
||||
d2, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "b.zip", testPagesZip(t, map[string]string{"index.html": "b"})), "root")
|
||||
require.NoError(t, err)
|
||||
_, err = ActivateDeployment(ctx, project.ID, d2.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// No active main config → reject.
|
||||
_, _, err = GetProjectLatestPackageHash(ctx, project.ID)
|
||||
require.Error(t, err)
|
||||
|
||||
require.NoError(t, db.DB(ctx).Create(&model.ConfigVersion{
|
||||
Version: "v-gate",
|
||||
SnapshotJSON: fmt.Sprintf(
|
||||
`{"routes":[{"upstream_type":"pages","pages_project_id":%d,"pages_deployment":{"project_id":%d,"deployment_id":%d}}]}`,
|
||||
project.ID, project.ID, d2.ID,
|
||||
),
|
||||
SupportFilesJSON: "[]",
|
||||
Checksum: "c-gate",
|
||||
IsActive: true,
|
||||
CreatedBy: "test",
|
||||
}).Error)
|
||||
|
||||
// Non-active historical deployment must not be downloadable.
|
||||
_, err = OpenDeploymentPackage(ctx, d1.ID)
|
||||
require.Error(t, err)
|
||||
|
||||
// Active latest works.
|
||||
_, hash, err := GetProjectLatestPackageHash(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, hash)
|
||||
|
||||
// Disabled project rejects.
|
||||
require.NoError(t, db.DB(ctx).Model(&model.PagesProject{}).Where("id = ?", project.ID).Update("enabled", false).Error)
|
||||
_, _, err = GetProjectLatestPackageHash(ctx, project.ID)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func testPagesZip(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buffer bytes.Buffer
|
||||
writer := zip.NewWriter(&buffer)
|
||||
for name, content := range files {
|
||||
file, err := writer.Create(name)
|
||||
require.NoError(t, err)
|
||||
_, err = file.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func TestUploadDeploymentAcceptsTarGz(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "TarGz Site",
|
||||
Slug: "tar-gz-site",
|
||||
Enabled: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
deployment, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "site.tar.gz", testPagesTarGz(t, map[string]string{
|
||||
"index.html": "tar-ok",
|
||||
"app.js": "1",
|
||||
})), "root")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, deployment.FileCount)
|
||||
assert.NotZero(t, deployment.UploadID)
|
||||
}
|
||||
|
||||
func TestSelectDeploymentsToPruneKeepsActiveAndNewest(t *testing.T) {
|
||||
// ids 4(newest) ... 1(oldest); active is oldest id=1; keep=2 → keep {1,4}, prune {3,2}
|
||||
deployments := []model.PagesDeployment{
|
||||
{ID: 4, ProjectID: 1},
|
||||
{ID: 3, ProjectID: 1},
|
||||
{ID: 2, ProjectID: 1},
|
||||
{ID: 1, ProjectID: 1},
|
||||
}
|
||||
toDelete := selectDeploymentsToPrune(deployments, 1, 0, 2)
|
||||
require.Len(t, toDelete, 2)
|
||||
assert.Equal(t, uint(3), toDelete[0].ID)
|
||||
assert.Equal(t, uint(2), toDelete[1].ID)
|
||||
|
||||
// active is newest; keep=2 → keep {4,3}, prune {2,1}
|
||||
toDelete = selectDeploymentsToPrune(deployments, 4, 0, 2)
|
||||
require.Len(t, toDelete, 2)
|
||||
assert.Equal(t, uint(2), toDelete[0].ID)
|
||||
assert.Equal(t, uint(1), toDelete[1].ID)
|
||||
|
||||
// no active; keep=2 → keep {4,3}
|
||||
toDelete = selectDeploymentsToPrune(deployments, 0, 0, 2)
|
||||
require.Len(t, toDelete, 2)
|
||||
assert.Equal(t, uint(2), toDelete[0].ID)
|
||||
assert.Equal(t, uint(1), toDelete[1].ID)
|
||||
|
||||
// keep=1 with active → only active, prune the rest
|
||||
toDelete = selectDeploymentsToPrune(deployments, 2, 0, 1)
|
||||
require.Len(t, toDelete, 3)
|
||||
for _, item := range toDelete {
|
||||
assert.NotEqual(t, uint(2), item.ID)
|
||||
}
|
||||
|
||||
// already within limit
|
||||
assert.Nil(t, selectDeploymentsToPrune(deployments[:2], 4, 0, 2))
|
||||
// unlimited
|
||||
assert.Nil(t, selectDeploymentsToPrune(deployments, 1, 0, 0))
|
||||
|
||||
// history=1 temporarily preserves active plus the freshly uploaded candidate.
|
||||
toDelete = selectDeploymentsToPrune(deployments, 2, 4, 1)
|
||||
require.Len(t, toDelete, 2)
|
||||
assert.Equal(t, uint(3), toDelete[0].ID)
|
||||
assert.Equal(t, uint(1), toDelete[1].ID)
|
||||
assert.Equal(t, uint(4), resolveLatestCandidateID(deployments, 2, true))
|
||||
assert.Zero(t, resolveLatestCandidateID(deployments, 2, false))
|
||||
}
|
||||
|
||||
func TestPruneProjectDeploymentHistory(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, db.DB(ctx).Model(&model.SystemConfig{}).
|
||||
Where("key = ?", model.ConfigKeyPagesMaxHistoryCount).
|
||||
Update("value", "2").Error)
|
||||
require.NoError(t, repository.InvalidateSystemConfigCache(ctx, model.ConfigKeyPagesMaxHistoryCount))
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "History Site",
|
||||
Slug: "history-site",
|
||||
Enabled: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
var ids []uint
|
||||
for i := 0; i < 3; i++ {
|
||||
deployment, uploadErr := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "site.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": fmt.Sprintf("v%d", i),
|
||||
})), "root")
|
||||
require.NoError(t, uploadErr)
|
||||
ids = append(ids, deployment.ID)
|
||||
}
|
||||
// After 3 uploads with keep=2 and no active: only 2 newest remain.
|
||||
deployments, err := repository.ListPagesDeployments(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 2)
|
||||
assert.Equal(t, ids[2], deployments[0].ID)
|
||||
assert.Equal(t, ids[1], deployments[1].ID)
|
||||
|
||||
// Activate the older of the remaining two, then upload again.
|
||||
_, err = ActivateDeployment(ctx, project.ID, ids[1])
|
||||
require.NoError(t, err)
|
||||
|
||||
latest, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "site.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "v-latest",
|
||||
})), "root")
|
||||
require.NoError(t, err)
|
||||
|
||||
deployments, err = repository.ListPagesDeployments(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 2, "must be at most N=2, not active+N newest")
|
||||
|
||||
storedProject, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, storedProject.ActiveDeploymentID)
|
||||
assert.Equal(t, ids[1], *storedProject.ActiveDeploymentID)
|
||||
|
||||
kept := map[uint]struct{}{}
|
||||
for _, item := range deployments {
|
||||
kept[item.ID] = struct{}{}
|
||||
}
|
||||
_, hasActive := kept[ids[1]]
|
||||
_, hasLatest := kept[latest.ID]
|
||||
assert.True(t, hasActive, "active deployment must be retained")
|
||||
assert.True(t, hasLatest, "newest deployment must fill remaining slot")
|
||||
}
|
||||
|
||||
func TestHistoryCountOnePreservesFreshCandidateUntilActivation(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, db.DB(ctx).Model(&model.SystemConfig{}).
|
||||
Where("key = ?", model.ConfigKeyPagesMaxHistoryCount).
|
||||
Update("value", "1").Error)
|
||||
require.NoError(t, repository.InvalidateSystemConfigCache(ctx, model.ConfigKeyPagesMaxHistoryCount))
|
||||
|
||||
project, err := CreateProject(ctx, Input{Name: "Single History", Slug: "single-history", Enabled: true})
|
||||
require.NoError(t, err)
|
||||
active, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "v1.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "v1",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
_, err = ActivateDeployment(ctx, project.ID, active.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
oldCandidate, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "v2.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "v2",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
deployments, err := repository.ListPagesDeployments(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 2)
|
||||
|
||||
newCandidate, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "v3.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "v3",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
deployments, err = repository.ListPagesDeployments(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 2)
|
||||
kept := map[uint]bool{}
|
||||
for _, deployment := range deployments {
|
||||
kept[deployment.ID] = true
|
||||
}
|
||||
assert.True(t, kept[active.ID])
|
||||
assert.True(t, kept[newCandidate.ID])
|
||||
assert.False(t, kept[oldCandidate.ID])
|
||||
var removedUpload model.Upload
|
||||
require.NoError(t, db.DB(ctx).First(&removedUpload, oldCandidate.UploadID).Error)
|
||||
assert.Equal(t, model.UploadStatusDeleted, removedUpload.Status)
|
||||
|
||||
_, err = ActivateDeployment(ctx, project.ID, newCandidate.ID)
|
||||
require.NoError(t, err)
|
||||
deployments, err = repository.ListPagesDeployments(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 1)
|
||||
assert.Equal(t, newCandidate.ID, deployments[0].ID)
|
||||
}
|
||||
|
||||
func TestPruneUsesLockTimeNewestCandidateInsteadOfStaleCaller(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{Name: "Concurrent Candidate", Slug: "concurrent-candidate", Enabled: true})
|
||||
require.NoError(t, err)
|
||||
active, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "v1.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "v1",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
_, err = ActivateDeployment(ctx, project.ID, active.ID)
|
||||
require.NoError(t, err)
|
||||
staleCandidate, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "v2.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "v2",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
newCandidate, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "v3.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "v3",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
|
||||
deleted, err := pruneProjectDeploymentHistoryOnce(ctx, project.ID, 1, staleCandidate.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, deleted)
|
||||
deployments, err := repository.ListPagesDeployments(ctx, project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 2)
|
||||
kept := map[uint]bool{}
|
||||
for _, deployment := range deployments {
|
||||
kept[deployment.ID] = true
|
||||
}
|
||||
assert.True(t, kept[active.ID])
|
||||
assert.True(t, kept[newCandidate.ID])
|
||||
assert.False(t, kept[staleCandidate.ID])
|
||||
}
|
||||
|
||||
func TestDeleteDeploymentAndProjectSoftDeleteUnreferencedArtifacts(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{Name: "Delete Artifacts", Slug: "delete-artifacts", Enabled: true})
|
||||
require.NoError(t, err)
|
||||
first, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "first.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "first",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
second, err := UploadDeployment(ctx, project.ID, testPagesMultipartFile(t, "second.zip", testPagesZip(t, map[string]string{
|
||||
"index.html": "second",
|
||||
})), "user:1")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, DeleteDeployment(ctx, project.ID, second.ID))
|
||||
var secondUpload model.Upload
|
||||
require.NoError(t, db.DB(ctx).First(&secondUpload, second.UploadID).Error)
|
||||
assert.Equal(t, model.UploadStatusDeleted, secondUpload.Status)
|
||||
|
||||
require.NoError(t, DeleteProject(ctx, project.ID))
|
||||
var firstUpload model.Upload
|
||||
require.NoError(t, db.DB(ctx).First(&firstUpload, first.UploadID).Error)
|
||||
assert.Equal(t, model.UploadStatusDeleted, firstUpload.Status)
|
||||
_, err = repository.GetPagesProjectByID(ctx, project.ID)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func testPagesTarGz(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buffer bytes.Buffer
|
||||
gzWriter := gzip.NewWriter(&buffer)
|
||||
tarWriter := tar.NewWriter(gzWriter)
|
||||
for name, content := range files {
|
||||
require.NoError(t, tarWriter.WriteHeader(&tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}))
|
||||
_, err := tarWriter.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tarWriter.Close())
|
||||
require.NoError(t, gzWriter.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func testPagesMultipartFile(t *testing.T, fileName string, content []byte) *multipart.FileHeader {
|
||||
t.Helper()
|
||||
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
part, err := writer.CreateFormFile("package", fileName)
|
||||
require.NoError(t, err)
|
||||
_, err = part.Write(content)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
req := httptest.NewRequest("POST", "/", &body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
require.NoError(t, req.ParseMultipartForm(int64(len(content))+1024))
|
||||
|
||||
file, header, err := req.FormFile("package")
|
||||
require.NoError(t, err)
|
||||
file.Close()
|
||||
return header
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
)
|
||||
|
||||
// ProjectLatestPackageMetadata describes the active package limits published to Agents.
|
||||
type ProjectLatestPackageMetadata struct {
|
||||
DeploymentID uint
|
||||
Hash string
|
||||
PackageSize int64
|
||||
FileCount int
|
||||
TotalSize int64
|
||||
}
|
||||
|
||||
// GetProjectLatestPackageMetadata returns one coherent metadata snapshot for a
|
||||
// project's currently active deployment.
|
||||
func GetProjectLatestPackageMetadata(ctx context.Context, projectID uint) (*ProjectLatestPackageMetadata, error) {
|
||||
deployment, err := resolveProjectActiveDeploymentForAgent(ctx, projectID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deployment.UploadID == 0 {
|
||||
if err := ensureDeploymentUploadRecord(ctx, deployment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deployment, err = repository.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if deployment.UploadID == 0 {
|
||||
return nil, errors.New(errPagesDeploymentNotFound)
|
||||
}
|
||||
|
||||
uploadRecord, err := ofupload.GetActiveUpload(ctx, deployment.UploadID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pages 部署包不存在: %w", err)
|
||||
}
|
||||
hash := strings.TrimSpace(uploadRecord.Hash)
|
||||
if hash == "" {
|
||||
hash = strings.TrimSpace(deployment.Checksum)
|
||||
}
|
||||
if hash == "" {
|
||||
return nil, errors.New(errPagesDeploymentHashMissing)
|
||||
}
|
||||
|
||||
return &ProjectLatestPackageMetadata{
|
||||
DeploymentID: deployment.ID,
|
||||
Hash: hash,
|
||||
PackageSize: uploadRecord.FileSize,
|
||||
FileCount: deployment.FileCount,
|
||||
TotalSize: deployment.TotalSize,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
func TestGetProjectLatestPackageMetadataPreservesZeroTotalSize(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
|
||||
ctx := context.Background()
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Empty Files",
|
||||
Slug: "empty-files",
|
||||
Enabled: true,
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateProject() error = %v", err)
|
||||
}
|
||||
packageBytes := testPagesZip(t, map[string]string{
|
||||
"index.html": "",
|
||||
".gitkeep": "",
|
||||
})
|
||||
deployment, err := UploadDeployment(
|
||||
ctx,
|
||||
project.ID,
|
||||
testPagesMultipartFile(t, "empty-files.zip", packageBytes),
|
||||
"test",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UploadDeployment() error = %v", err)
|
||||
}
|
||||
if _, err := ActivateDeployment(ctx, project.ID, deployment.ID); err != nil {
|
||||
t.Fatalf("ActivateDeployment() error = %v", err)
|
||||
}
|
||||
if err := db.DB(ctx).Create(&model.ConfigVersion{
|
||||
Version: "v-package-metadata",
|
||||
SnapshotJSON: fmt.Sprintf(
|
||||
`{"routes":[{"upstream_type":"pages","pages_project_id":%d}]}`,
|
||||
project.ID,
|
||||
),
|
||||
SupportFilesJSON: "[]",
|
||||
Checksum: "package-metadata-config",
|
||||
IsActive: true,
|
||||
CreatedBy: "test",
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create active ConfigVersion error = %v", err)
|
||||
}
|
||||
|
||||
got, err := GetProjectLatestPackageMetadata(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetProjectLatestPackageMetadata(%d) error = %v", project.ID, err)
|
||||
}
|
||||
wantHashBytes := sha256.Sum256(packageBytes)
|
||||
wantHash := hex.EncodeToString(wantHashBytes[:])
|
||||
if got.DeploymentID != deployment.ID || got.Hash != wantHash {
|
||||
t.Errorf("GetProjectLatestPackageMetadata(%d) identity = (%d, %q), want (%d, %q)",
|
||||
project.ID, got.DeploymentID, got.Hash, deployment.ID, wantHash)
|
||||
}
|
||||
if got.PackageSize != int64(len(packageBytes)) {
|
||||
t.Errorf("GetProjectLatestPackageMetadata(%d).PackageSize = %d, want %d",
|
||||
project.ID, got.PackageSize, len(packageBytes))
|
||||
}
|
||||
if got.FileCount != 2 || got.TotalSize != 0 {
|
||||
t.Errorf("GetProjectLatestPackageMetadata(%d) content = (%d files, %d bytes), want (2 files, 0 bytes)",
|
||||
project.ID, got.FileCount, got.TotalSize)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
openrestyrender "Wavelet/openflare/share/render/openresty"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// RebindSnapshotPagesToCurrentActive rewrites pages_deployment fields so every
|
||||
// pages route points at the project's current active deployment.
|
||||
//
|
||||
// Main config versions and Pages deployments are independent. Rolling back a
|
||||
// main config version must not require old Pages packages; Agents always follow
|
||||
// the live active deployment for each referenced project.
|
||||
//
|
||||
// Returns the original JSON unchanged when there are no pages routes. Does not
|
||||
// mutate stored config_versions rows. Non-pages route fields are preserved.
|
||||
func RebindSnapshotPagesToCurrentActive(ctx context.Context, snapshotJSON string) (string, error) {
|
||||
text := strings.TrimSpace(snapshotJSON)
|
||||
if text == "" {
|
||||
return snapshotJSON, nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(text, "[") {
|
||||
var routes []map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(text), &routes); err != nil {
|
||||
return "", fmt.Errorf("parse pages snapshot routes: %w", err)
|
||||
}
|
||||
changed, err := rebindPagesRouteMaps(ctx, routes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !changed {
|
||||
return snapshotJSON, nil
|
||||
}
|
||||
encoded, err := json.Marshal(routes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(encoded), nil
|
||||
}
|
||||
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(text), &raw); err != nil {
|
||||
return "", fmt.Errorf("parse pages snapshot document: %w", err)
|
||||
}
|
||||
routesRaw, ok := raw["routes"]
|
||||
if !ok || len(routesRaw) == 0 {
|
||||
return snapshotJSON, nil
|
||||
}
|
||||
var routes []map[string]json.RawMessage
|
||||
if err := json.Unmarshal(routesRaw, &routes); err != nil {
|
||||
return "", fmt.Errorf("parse pages snapshot routes: %w", err)
|
||||
}
|
||||
changed, err := rebindPagesRouteMaps(ctx, routes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !changed {
|
||||
return snapshotJSON, nil
|
||||
}
|
||||
encodedRoutes, err := json.Marshal(routes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw["routes"] = encodedRoutes
|
||||
encoded, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(encoded), nil
|
||||
}
|
||||
|
||||
func rebindPagesRouteMaps(ctx context.Context, routes []map[string]json.RawMessage) (bool, error) {
|
||||
changed := false
|
||||
for index := range routes {
|
||||
route := routes[index]
|
||||
if route == nil {
|
||||
continue
|
||||
}
|
||||
upstreamType := rawJSONString(route["upstream_type"])
|
||||
if !strings.EqualFold(strings.TrimSpace(upstreamType), "pages") {
|
||||
continue
|
||||
}
|
||||
siteName := rawJSONString(route["site_name"])
|
||||
projectID, err := resolveProjectIDFromRouteMap(route)
|
||||
if err != nil {
|
||||
if siteName == "" {
|
||||
siteName = "pages"
|
||||
}
|
||||
return false, fmt.Errorf("路由 %s %w", siteName, err)
|
||||
}
|
||||
project, activeDeployment, err := loadActivePagesProject(ctx, projectID, siteName)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
deployment, err := buildLivePagesDeployment(project, activeDeployment)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
projectIDCopy := project.ID
|
||||
originURL := fmt.Sprintf("openflare-pages://project/%d", project.ID)
|
||||
|
||||
if err := putJSON(route, "pages_project_id", projectIDCopy); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := putJSON(route, "pages_deployment", deployment); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := putJSON(route, "origin_url", originURL); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := putJSON(route, "upstreams", []string{originURL}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
routes[index] = route
|
||||
changed = true
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
const jsonNullLiteral = "null"
|
||||
|
||||
func isPresentJSON(raw json.RawMessage) bool {
|
||||
return len(raw) > 0 && string(raw) != jsonNullLiteral
|
||||
}
|
||||
|
||||
func resolveProjectIDFromRouteMap(route map[string]json.RawMessage) (uint, error) {
|
||||
if raw, ok := route["pages_project_id"]; ok && isPresentJSON(raw) {
|
||||
var projectID uint
|
||||
if err := json.Unmarshal(raw, &projectID); err == nil && projectID != 0 {
|
||||
return projectID, nil
|
||||
}
|
||||
}
|
||||
if raw, ok := route["pages_deployment"]; ok && isPresentJSON(raw) {
|
||||
var deployment struct {
|
||||
ProjectID uint `json:"project_id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &deployment); err == nil && deployment.ProjectID != 0 {
|
||||
return deployment.ProjectID, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("pages 配置无效: 缺少 pages_project_id")
|
||||
}
|
||||
|
||||
func loadActivePagesProject(ctx context.Context, projectID uint, siteName string) (*model.PagesProject, *model.PagesDeployment, error) {
|
||||
if siteName == "" {
|
||||
siteName = "pages"
|
||||
}
|
||||
project, err := repository.GetPagesProjectByID(ctx, projectID)
|
||||
if err != nil {
|
||||
if errorsIsNotFound(err) {
|
||||
return nil, nil, fmt.Errorf("路由 %s Pages 配置无效: pages 项目不存在", siteName)
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
if !project.Enabled {
|
||||
return nil, nil, fmt.Errorf("路由 %s Pages 配置无效: pages 项目未启用", siteName)
|
||||
}
|
||||
if project.ActiveDeploymentID == nil || *project.ActiveDeploymentID == 0 {
|
||||
return nil, nil, fmt.Errorf("路由 %s Pages 配置无效: pages 项目没有激活部署", siteName)
|
||||
}
|
||||
activeDeployment, err := repository.GetPagesDeploymentByID(ctx, *project.ActiveDeploymentID)
|
||||
if err != nil {
|
||||
if errorsIsNotFound(err) {
|
||||
return nil, nil, fmt.Errorf("路由 %s Pages 配置无效: pages 激活部署不存在", siteName)
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
if activeDeployment.ProjectID != project.ID {
|
||||
return nil, nil, fmt.Errorf("路由 %s Pages 配置无效: pages 激活部署不匹配", siteName)
|
||||
}
|
||||
if strings.TrimSpace(activeDeployment.Checksum) == "" {
|
||||
return nil, nil, fmt.Errorf("路由 %s Pages 配置无效: pages 部署校验和缺失", siteName)
|
||||
}
|
||||
return project, activeDeployment, nil
|
||||
}
|
||||
|
||||
func buildLivePagesDeployment(
|
||||
project *model.PagesProject,
|
||||
active *model.PagesDeployment,
|
||||
) (*openrestyrender.PagesDeployment, error) {
|
||||
rootDir, err := validateAndNormalizePagesRootDir(project.RootDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entryFile, err := validateAndNormalizePagesEntryFile(project.EntryFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fallbackPath := strings.TrimSpace(project.SPAFallbackPath)
|
||||
if fallbackPath == "" {
|
||||
fallbackPath = defaultPagesFallbackPath
|
||||
}
|
||||
localRoot := openrestyrender.PagesProjectLocalRoot(project.ID)
|
||||
if rootDir != "" {
|
||||
localRoot = path.Join(localRoot, rootDir)
|
||||
}
|
||||
return &openrestyrender.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
ProjectSlug: strings.TrimSpace(project.Slug),
|
||||
DeploymentID: active.ID,
|
||||
DeploymentNumber: active.DeploymentNumber,
|
||||
Checksum: strings.TrimSpace(active.Checksum),
|
||||
EntryFile: entryFile,
|
||||
SPAFallbackEnabled: project.SPAFallbackEnabled,
|
||||
SPAFallbackPath: fallbackPath,
|
||||
APIProxyEnabled: project.APIProxyEnabled,
|
||||
APIProxyPath: strings.TrimSpace(project.APIProxyPath),
|
||||
APIProxyPass: strings.TrimSpace(project.APIProxyPass),
|
||||
APIProxyRewrite: strings.TrimSpace(project.APIProxyRewrite),
|
||||
LocalRoot: localRoot,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func rawJSONString(raw json.RawMessage) string {
|
||||
if !isPresentJSON(raw) {
|
||||
return ""
|
||||
}
|
||||
var value string
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return ""
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func putJSON(route map[string]json.RawMessage, key string, value any) error {
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
route[key] = encoded
|
||||
return nil
|
||||
}
|
||||
|
||||
func errorsIsNotFound(err error) bool {
|
||||
return err != nil && (errors.Is(err, gorm.ErrRecordNotFound) || strings.Contains(strings.ToLower(err.Error()), "record not found"))
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRebindSnapshotPagesToCurrentActive(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Rebind Site",
|
||||
Slug: "rebind-site",
|
||||
Enabled: true,
|
||||
RootDir: "public/site",
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
old := &model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 1,
|
||||
Checksum: "old-checksum",
|
||||
Status: model.PagesDeploymentStatusUploaded,
|
||||
FileCount: 1,
|
||||
}
|
||||
require.NoError(t, db.DB(ctx).Create(old).Error)
|
||||
active := &model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 2,
|
||||
Checksum: "new-checksum",
|
||||
Status: model.PagesDeploymentStatusActive,
|
||||
FileCount: 1,
|
||||
}
|
||||
require.NoError(t, db.DB(ctx).Create(active).Error)
|
||||
require.NoError(t, db.DB(ctx).Model(&model.PagesProject{}).
|
||||
Where("id = ?", project.ID).
|
||||
Update("active_deployment_id", active.ID).Error)
|
||||
|
||||
// Frozen snapshot still points at the old deployment (simulates old main config).
|
||||
frozen := map[string]any{
|
||||
"routes": []map[string]any{
|
||||
{
|
||||
"site_name": "rebind",
|
||||
"origin_url": "openflare-pages://project/1",
|
||||
"enabled": true,
|
||||
"upstream_type": "pages",
|
||||
"pages_project_id": project.ID,
|
||||
"pages_deployment": map[string]any{
|
||||
"project_id": project.ID,
|
||||
"deployment_id": old.ID,
|
||||
"checksum": "old-checksum",
|
||||
"local_root": "__OPENFLARE_PAGES_DIR__/projects/1/current",
|
||||
},
|
||||
"extra_keep_me": "yes",
|
||||
},
|
||||
},
|
||||
"waf": map[string]any{"rule_groups": []any{}},
|
||||
}
|
||||
frozenJSON, err := json.Marshal(frozen)
|
||||
require.NoError(t, err)
|
||||
|
||||
reboundJSON, err := RebindSnapshotPagesToCurrentActive(ctx, string(frozenJSON))
|
||||
require.NoError(t, err)
|
||||
|
||||
var rebound map[string]any
|
||||
require.NoError(t, json.Unmarshal([]byte(reboundJSON), &rebound))
|
||||
_, hasWAF := rebound["waf"]
|
||||
assert.True(t, hasWAF)
|
||||
|
||||
routes := rebound["routes"].([]any)
|
||||
require.Len(t, routes, 1)
|
||||
route := routes[0].(map[string]any)
|
||||
assert.Equal(t, "yes", route["extra_keep_me"])
|
||||
deployment := route["pages_deployment"].(map[string]any)
|
||||
assert.EqualValues(t, active.ID, deployment["deployment_id"])
|
||||
assert.Equal(t, "new-checksum", deployment["checksum"])
|
||||
assert.Equal(t, "__OPENFLARE_PAGES_DIR__/projects/1/current/public/site", deployment["local_root"])
|
||||
}
|
||||
@@ -0,0 +1,610 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/openflare/plugins/server/kernel/apiutil"
|
||||
"Wavelet/pkg/logger"
|
||||
"Wavelet/pkg/response"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func handleLogicError(c *gin.Context, err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return apiutil.AbortNotFoundIfMissing(c, err, errPagesProjectNotFound)
|
||||
}
|
||||
|
||||
func handleSourceLogicError(c *gin.Context, err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) || err.Error() == errPagesSourceNotFound {
|
||||
response.AbortNotFound(c, errPagesSourceNotFound)
|
||||
return true
|
||||
}
|
||||
switch err.Error() {
|
||||
case errPagesSourceActionBusy:
|
||||
response.AbortConflict(c, errPagesSourceActionBusy)
|
||||
case errPagesSourceTypeRequired,
|
||||
errPagesSourceTypeUnsupported,
|
||||
errPagesSourceRemoteFields,
|
||||
errPagesSourceRemoteURLRequired,
|
||||
errPagesSourceRemoteURLInvalid,
|
||||
errPagesSourceGitHubFields,
|
||||
errPagesSourceRepositoryInvalid,
|
||||
errPagesSourceSelectorInvalid,
|
||||
errPagesSourceAssetNameInvalid,
|
||||
errPagesSourceCheckInterval,
|
||||
errPagesSourceAutoNotAvailable,
|
||||
errPagesSourceReleaseNotFound,
|
||||
errPagesSourceDigestInvalid,
|
||||
errPagesSourceDigestMismatch,
|
||||
errPagesSourceConfirmationNeeded,
|
||||
errPagesSourceConfirmationStale,
|
||||
errPagesSourceCheckUnsupported,
|
||||
errPagesSourceActionInvalid:
|
||||
response.AbortBadRequest(c, err.Error())
|
||||
case errPagesSourceTaskDispatchFailed:
|
||||
response.AbortInternal(c, errPagesSourceInternal)
|
||||
default:
|
||||
logger.ErrorF(c.Request.Context(), "[PagesSource] API operation failed: error=%v", err)
|
||||
response.AbortInternal(c, errPagesSourceInternal)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func decodeStrictJSON(c *gin.Context, target any, allowEmpty bool) bool {
|
||||
decoder := json.NewDecoder(c.Request.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
if allowEmpty && errors.Is(err, io.EOF) {
|
||||
return true
|
||||
}
|
||||
response.AbortBadRequest(c, errPagesSourceActionInvalid)
|
||||
return false
|
||||
}
|
||||
if err := ensureJSONEOF(decoder); err != nil {
|
||||
response.AbortBadRequest(c, errPagesSourceActionInvalid)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func deploymentIDParam(c *gin.Context) (uint, bool) {
|
||||
raw := c.Param("deployment_id")
|
||||
if raw == "" {
|
||||
response.AbortBadRequest(c, "无效的 ID")
|
||||
return 0, false
|
||||
}
|
||||
id64, err := strconv.ParseUint(raw, 10, 64)
|
||||
if err != nil || id64 == 0 {
|
||||
response.AbortBadRequest(c, "无效的 ID")
|
||||
return 0, false
|
||||
}
|
||||
return uint(id64), true
|
||||
}
|
||||
|
||||
func currentPagesActor(c *gin.Context) (string, bool) {
|
||||
if raw, ok := c.Get(contracts.AuthUserObjKey); ok {
|
||||
switch user := raw.(type) {
|
||||
case *contracts.UserDTO:
|
||||
if user != nil && user.ID != 0 {
|
||||
return fmt.Sprintf("user:%d", user.ID), true
|
||||
}
|
||||
case contracts.UserDTO:
|
||||
if user.ID != 0 {
|
||||
return fmt.Sprintf("user:%d", user.ID), true
|
||||
}
|
||||
}
|
||||
}
|
||||
if raw, ok := c.Get(contracts.AuthUserIDKey); ok {
|
||||
switch id := raw.(type) {
|
||||
case uint64:
|
||||
if id != 0 {
|
||||
return fmt.Sprintf("user:%d", id), true
|
||||
}
|
||||
case int:
|
||||
if id > 0 {
|
||||
return fmt.Sprintf("user:%d", uint64(id)), true
|
||||
}
|
||||
}
|
||||
}
|
||||
response.AbortUnauthorized(c, errPagesActorMissing)
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ListProjectsHandler 列出全部 Pages 项目。
|
||||
// @Summary 列出 Pages 项目
|
||||
// @Description 返回全部 OpenFlare Pages 项目，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} response.Any{data=[]pages.View} "Pages 项目列表"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages [get]
|
||||
func ListProjectsHandler(c *gin.Context) {
|
||||
projects, err := ListProjects(c.Request.Context())
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(projects))
|
||||
}
|
||||
|
||||
// GetProjectHandler 获取 Pages 项目详情。
|
||||
// @Summary 获取 Pages 项目详情
|
||||
// @Description 按 ID 返回 Pages 项目详情，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Success 200 {object} response.Any{data=pages.View} "Pages 项目详情"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id} [get]
|
||||
func GetProjectHandler(c *gin.Context) {
|
||||
id, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
project, err := GetProject(c.Request.Context(), id)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(project))
|
||||
}
|
||||
|
||||
// CreateProjectHandler 创建 Pages 项目。
|
||||
// @Summary 创建 Pages 项目
|
||||
// @Description 创建新的 OpenFlare Pages 项目，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param request body pages.Input true "项目参数"
|
||||
// @Success 200 {object} response.Any{data=pages.View} "创建成功的项目"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages [post]
|
||||
func CreateProjectHandler(c *gin.Context) {
|
||||
var input Input
|
||||
if !apiutil.BindJSON(c, &input) {
|
||||
return
|
||||
}
|
||||
project, err := CreateProject(c.Request.Context(), input)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(project))
|
||||
}
|
||||
|
||||
// UpdateProjectHandler 更新 Pages 项目。
|
||||
// @Summary 更新 Pages 项目
|
||||
// @Description 按 ID 更新 OpenFlare Pages 项目，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Param request body pages.Input true "项目参数"
|
||||
// @Success 200 {object} response.Any{data=pages.View} "更新后的项目"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/update [post]
|
||||
func UpdateProjectHandler(c *gin.Context) {
|
||||
id, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var input Input
|
||||
if !apiutil.BindJSON(c, &input) {
|
||||
return
|
||||
}
|
||||
project, err := UpdateProject(c.Request.Context(), id, input)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(project))
|
||||
}
|
||||
|
||||
// DeleteProjectHandler 删除 Pages 项目。
|
||||
// @Summary 删除 Pages 项目
|
||||
// @Description 按 ID 删除 OpenFlare Pages 项目，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Success 200 {object} response.Any "删除成功"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/delete [post]
|
||||
func DeleteProjectHandler(c *gin.Context) {
|
||||
id, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := DeleteProject(c.Request.Context(), id); handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OKNil())
|
||||
}
|
||||
|
||||
// GetSourceHandler 获取 Pages 项目的部署源。
|
||||
// @Summary 获取 Pages 部署源
|
||||
// @Description 返回项目部署源配置与运行状态，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Success 200 {object} response.Any{data=pages.SourceView} "部署源"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "项目或部署源不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/source [get]
|
||||
func GetSourceHandler(c *gin.Context) {
|
||||
projectID, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
source, err := GetSource(c.Request.Context(), projectID)
|
||||
if handleSourceLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(source))
|
||||
}
|
||||
|
||||
// UpdateSourceHandler 创建或更新 Pages 项目部署源。
|
||||
// @Summary 更新 Pages 部署源
|
||||
// @Description 支持 Remote URL 与公开 GitHub Release 来源；敏感地址仅写入，不会在响应中返回
|
||||
// @Tags openflare-pages
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Param request body pages.SourceUpdateInput true "部署源配置"
|
||||
// @Success 200 {object} response.Any{data=pages.SourceUpdateResult} "更新结果"
|
||||
// @Failure 400 {object} response.Any "配置无效"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/source/update [post]
|
||||
func UpdateSourceHandler(c *gin.Context) {
|
||||
projectID, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var input SourceUpdateInput
|
||||
if !decodeStrictJSON(c, &input, false) {
|
||||
return
|
||||
}
|
||||
actor, ok := currentPagesActor(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
result, err := UpdateSourceAs(c.Request.Context(), projectID, input, actor)
|
||||
if handleSourceLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(result))
|
||||
}
|
||||
|
||||
// DeleteSourceHandler 将 Pages 项目切换回手动部署模式。
|
||||
// @Summary 删除 Pages 部署源
|
||||
// @Description 幂等删除持久部署源；已有部署历史与当前生产部署保持不变
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Success 200 {object} response.Any{data=pages.SourceView} "手动来源视图"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/source/delete [post]
|
||||
func DeleteSourceHandler(c *gin.Context) {
|
||||
projectID, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
source, err := DeleteSource(c.Request.Context(), projectID)
|
||||
if handleSourceLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(source))
|
||||
}
|
||||
|
||||
// CheckSourceHandler 请求检查 Pages 部署源。
|
||||
// @Summary 检查 Pages 部署源
|
||||
// @Description 异步检查 GitHub Release 来源；Remote URL 来源不支持检查更新
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Success 200 {object} response.Any{data=pages.SourceActionReceipt} "任务回执"
|
||||
// @Failure 400 {object} response.Any "当前来源不支持检查"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "部署源不存在"
|
||||
// @Failure 409 {object} response.Any "来源任务正在执行"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/source/check [post]
|
||||
func CheckSourceHandler(c *gin.Context) {
|
||||
projectID, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
actor, ok := currentPagesActor(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
receipt, err := DispatchSourceAction(c.Request.Context(), projectID, sourceActionCheck, actor, "")
|
||||
if handleSourceLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(receipt))
|
||||
}
|
||||
|
||||
// SourceSyncInput is the optional source sync action payload.
|
||||
type SourceSyncInput struct {
|
||||
ConfirmedRevision string `json:"confirmed_revision"`
|
||||
}
|
||||
|
||||
// SyncSourceHandler 请求同步并发布 Pages 部署源。
|
||||
// @Summary 同步并发布 Pages 部署源
|
||||
// @Description 异步下载、校验并原子激活来源部署包；空请求体与空 JSON 对象均有效
|
||||
// @Tags openflare-pages
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Param request body pages.SourceSyncInput false "同步参数"
|
||||
// @Success 200 {object} response.Any{data=pages.SourceActionReceipt} "任务回执"
|
||||
// @Failure 400 {object} response.Any "参数或来源类型无效"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "部署源不存在"
|
||||
// @Failure 409 {object} response.Any "来源任务正在执行"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/source/sync [post]
|
||||
func SyncSourceHandler(c *gin.Context) {
|
||||
projectID, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var input SourceSyncInput
|
||||
if !decodeStrictJSON(c, &input, true) {
|
||||
return
|
||||
}
|
||||
actor, ok := currentPagesActor(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
receipt, err := DispatchSourceAction(
|
||||
c.Request.Context(),
|
||||
projectID,
|
||||
sourceActionSync,
|
||||
actor,
|
||||
input.ConfirmedRevision,
|
||||
)
|
||||
if handleSourceLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(receipt))
|
||||
}
|
||||
|
||||
// ListDeploymentsHandler 列出项目的全部部署。
|
||||
// @Summary 列出 Pages 部署
|
||||
// @Description 返回指定项目的全部部署记录，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Success 200 {object} response.Any{data=[]pages.DeploymentView} "部署列表"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/deployments [get]
|
||||
func ListDeploymentsHandler(c *gin.Context) {
|
||||
id, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
deployments, err := ListProjectDeployments(c.Request.Context(), id)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(deployments))
|
||||
}
|
||||
|
||||
// UploadDeploymentHandler 上传 Pages 部署包。
|
||||
// @Summary 上传 Pages 部署包
|
||||
// @Description 为指定项目上传静态资源压缩包（zip/tar.gz/tar.xz/tar.bz2/tar/7z），需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Param package formData file true "部署包文件"
|
||||
// @Success 200 {object} response.Any{data=pages.DeploymentView} "部署记录"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/deployments/upload [post]
|
||||
func UploadDeploymentHandler(c *gin.Context) {
|
||||
id, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
file, err := c.FormFile("package")
|
||||
if err != nil {
|
||||
response.AbortBadRequest(c, errPagesPackageMissing)
|
||||
return
|
||||
}
|
||||
actor, ok := currentPagesActor(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
deployment, err := UploadDeployment(c.Request.Context(), id, file, actor)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(deployment))
|
||||
}
|
||||
|
||||
// UploadDeploymentFromURLHandler 从 URL 下载并创建 Pages 部署。
|
||||
// @Summary 从 URL 导入 Pages 部署包
|
||||
// @Description 已弃用的一次性 URL 导入；使用 trusted_internal 策略兼容内网与自签名证书，不创建持久部署源
|
||||
// @Deprecated
|
||||
// @Tags openflare-pages
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Param request body pages.UploadFromURLInput true "下载链接"
|
||||
// @Success 200 {object} response.Any{data=pages.DeploymentView} "部署记录"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/deployments/upload-from-url [post]
|
||||
func UploadDeploymentFromURLHandler(c *gin.Context) {
|
||||
id, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req UploadFromURLInput
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
response.AbortBadRequest(c, errPagesPackageURLRequired)
|
||||
return
|
||||
}
|
||||
actor, ok := currentPagesActor(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
deployment, err := UploadDeploymentFromURL(c.Request.Context(), id, req.URL, actor)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(deployment))
|
||||
}
|
||||
|
||||
// ActivateDeploymentHandler 激活 Pages 部署。
|
||||
// @Summary 激活 Pages 部署
|
||||
// @Description 将指定部署设为项目当前生效版本，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Param deployment_id path int true "部署 ID"
|
||||
// @Success 200 {object} response.Any{data=pages.View} "激活后的项目"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目或部署不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/deployments/{deployment_id}/activate [post]
|
||||
func ActivateDeploymentHandler(c *gin.Context) {
|
||||
projectID, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
deploymentID, ok := deploymentIDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
actor, ok := currentPagesActor(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
project, err := ActivateDeploymentAs(c.Request.Context(), projectID, deploymentID, actor)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(project))
|
||||
}
|
||||
|
||||
// DeleteDeploymentHandler 删除 Pages 部署。
|
||||
// @Summary 删除 Pages 部署
|
||||
// @Description 删除指定项目的部署记录，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param id path int true "项目 ID"
|
||||
// @Param deployment_id path int true "部署 ID"
|
||||
// @Success 200 {object} response.Any "删除成功"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "项目或部署不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/{id}/deployments/{deployment_id}/delete [post]
|
||||
func DeleteDeploymentHandler(c *gin.Context) {
|
||||
projectID, ok := apiutil.IDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
deploymentID, ok := deploymentIDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := DeleteDeployment(c.Request.Context(), projectID, deploymentID); handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OKNil())
|
||||
}
|
||||
|
||||
// ListDeploymentFilesHandler 列出部署文件清单。
|
||||
// @Summary 列出 Pages 部署文件
|
||||
// @Description 返回指定部署包含的文件清单，需要管理员权限
|
||||
// @Tags openflare-pages
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Param deployment_id path int true "部署 ID"
|
||||
// @Success 200 {object} response.Any{data=[]pages.DeploymentFileView} "部署文件列表"
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 404 {object} response.Any "无权限或不存在"
|
||||
// @Failure 404 {object} response.Any "部署不存在"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/d/pages/deployments/{deployment_id}/files [get]
|
||||
func ListDeploymentFilesHandler(c *gin.Context) {
|
||||
deploymentID, ok := deploymentIDParam(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
files, err := ListDeploymentFiles(c.Request.Context(), deploymentID)
|
||||
if handleLogicError(c, err) {
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(files))
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/testhelper"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type sourceHandlerEnvelope struct {
|
||||
ErrorMsg string `json:"error_msg"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
|
||||
func newPagesSourceTestRouter(userID uint64) *gin.Engine {
|
||||
router := testhelper.NewTestGinEngine(func(ctx *gin.Context) {
|
||||
ctx.Set(contracts.AuthUserIDKey, userID)
|
||||
ctx.Next()
|
||||
})
|
||||
router.GET("/api/v1/d/pages/:id/source", GetSourceHandler)
|
||||
router.POST("/api/v1/d/pages/:id/source/update", UpdateSourceHandler)
|
||||
router.POST("/api/v1/d/pages/:id/source/delete", DeleteSourceHandler)
|
||||
router.POST("/api/v1/d/pages/:id/source/check", CheckSourceHandler)
|
||||
router.POST("/api/v1/d/pages/:id/source/sync", SyncSourceHandler)
|
||||
return router
|
||||
}
|
||||
|
||||
func performPagesSourceRequest(
|
||||
t *testing.T,
|
||||
router http.Handler,
|
||||
method string,
|
||||
path string,
|
||||
body []byte,
|
||||
) (int, sourceHandlerEnvelope) {
|
||||
t.Helper()
|
||||
request := httptest.NewRequest(method, path, bytes.NewReader(body))
|
||||
if body != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
router.ServeHTTP(recorder, request)
|
||||
var envelope sourceHandlerEnvelope
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil {
|
||||
t.Fatalf("json.Unmarshal(%s %s response %q) error = %v, want nil", method, path, recorder.Body.String(), err)
|
||||
}
|
||||
return recorder.Code, envelope
|
||||
}
|
||||
|
||||
func setupPagesSourceDispatchTest(t *testing.T) {
|
||||
t.Helper()
|
||||
}
|
||||
|
||||
func TestPagesSourceHandlersReturnStableActionErrors(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
router := newPagesSourceTestRouter(42)
|
||||
|
||||
manualProject := mustCreatePagesSourceProject(t, ctx, "handler-no-source")
|
||||
code, envelope := performPagesSourceRequest(
|
||||
t,
|
||||
router,
|
||||
http.MethodPost,
|
||||
fmt.Sprintf("/api/v1/d/pages/%d/source/sync", manualProject.ID),
|
||||
nil,
|
||||
)
|
||||
if got, want := code, http.StatusNotFound; got != want {
|
||||
t.Errorf("POST source/sync without source status = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := envelope.ErrorMsg, errPagesSourceNotFound; got != want {
|
||||
t.Errorf("POST source/sync without source error = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
remoteProject := mustCreatePagesSourceProject(t, ctx, "handler-check")
|
||||
_, _ = mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
remoteProject.ID,
|
||||
"https://example.com/site.zip?token=handler-secret",
|
||||
false,
|
||||
)
|
||||
code, envelope = performPagesSourceRequest(
|
||||
t,
|
||||
router,
|
||||
http.MethodPost,
|
||||
fmt.Sprintf("/api/v1/d/pages/%d/source/check", remoteProject.ID),
|
||||
nil,
|
||||
)
|
||||
if got, want := code, http.StatusBadRequest; got != want {
|
||||
t.Errorf("POST remote source/check status = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := envelope.ErrorMsg, errPagesSourceCheckUnsupported; got != want {
|
||||
t.Errorf("POST remote source/check error = %q, want %q", got, want)
|
||||
}
|
||||
if strings.Contains(string(envelope.Data), "handler-secret") || strings.Contains(envelope.ErrorMsg, "handler-secret") {
|
||||
t.Errorf("POST remote source/check response = %+v, want no URL secret", envelope)
|
||||
}
|
||||
|
||||
busyProject := mustCreatePagesSourceProject(t, ctx, "handler-busy")
|
||||
busySource, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
busyProject.ID,
|
||||
"https://example.com/site.zip",
|
||||
false,
|
||||
)
|
||||
future := time.Now().Add(time.Minute)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", busySource.ID).
|
||||
Updates(map[string]any{
|
||||
"sync_status": pagesSourceStatusSyncing,
|
||||
"lease_token": "busy-owner",
|
||||
"lease_expires_at": &future,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed busy source runtime error = %v, want nil", err)
|
||||
}
|
||||
code, envelope = performPagesSourceRequest(
|
||||
t,
|
||||
router,
|
||||
http.MethodPost,
|
||||
fmt.Sprintf("/api/v1/d/pages/%d/source/sync", busyProject.ID),
|
||||
[]byte(`{}`),
|
||||
)
|
||||
if got, want := code, http.StatusConflict; got != want {
|
||||
t.Errorf("POST busy source/sync status = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := envelope.ErrorMsg, errPagesSourceActionBusy; got != want {
|
||||
t.Errorf("POST busy source/sync error = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncSourceHandlerAcceptsEmptyBodyAndEmptyObject(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
setupPagesSourceDispatchTest(t)
|
||||
router := newPagesSourceTestRouter(77)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "handler-empty-sync")
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip?token=dispatch-secret",
|
||||
false,
|
||||
)
|
||||
path := fmt.Sprintf("/api/v1/d/pages/%d/source/sync", project.ID)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
}{
|
||||
{name: "empty body", body: nil},
|
||||
{name: "empty object", body: []byte(`{}`)},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
code, envelope := performPagesSourceRequest(t, router, http.MethodPost, path, test.body)
|
||||
if got, want := code, http.StatusOK; got != want {
|
||||
t.Fatalf("POST source/sync (%s) status = %d, want %d; error=%q", test.name, got, want, envelope.ErrorMsg)
|
||||
}
|
||||
if envelope.ErrorMsg != "" {
|
||||
t.Errorf("POST source/sync (%s) error = %q, want empty", test.name, envelope.ErrorMsg)
|
||||
}
|
||||
var receipt SourceActionReceipt
|
||||
if err := json.Unmarshal(envelope.Data, &receipt); err != nil {
|
||||
t.Fatalf("json.Unmarshal(source/sync %s receipt) error = %v, want nil", test.name, err)
|
||||
}
|
||||
if receipt.TaskID == "" || receipt.ExecutionID == "" || receipt.Action != sourceActionSync {
|
||||
t.Errorf("POST source/sync (%s) receipt = %+v, want task/execution IDs and action %q", test.name, receipt, sourceActionSync)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var executions []model.TaskExecution
|
||||
if err := db.DB(ctx).Where("task_type = ?", TaskTypePagesSourceAction).Order("id asc").Find(&executions).Error; err != nil {
|
||||
t.Fatalf("list Pages source task executions error = %v, want nil", err)
|
||||
}
|
||||
if got, want := len(executions), 2; got != want {
|
||||
t.Fatalf("Pages source task execution count = %d, want %d", got, want)
|
||||
}
|
||||
for _, execution := range executions {
|
||||
if strings.Contains(execution.Payload, "dispatch-secret") || strings.Contains(execution.Payload, "http") {
|
||||
t.Errorf("task execution %q payload = %s, want no Remote URL secret", execution.TaskID, execution.Payload)
|
||||
}
|
||||
var payload SourceActionPayload
|
||||
if err := json.Unmarshal([]byte(execution.Payload), &payload); err != nil {
|
||||
t.Errorf("json.Unmarshal(task execution %q payload) error = %v, want nil", execution.TaskID, err)
|
||||
continue
|
||||
}
|
||||
if payload.SourceID != source.ID || payload.ConfigVersion != source.ConfigVersion || payload.Actor != "user:77" {
|
||||
t.Errorf("task execution %q payload = %+v, want source=%d config=%d actor=user:77", execution.TaskID, payload, source.ID, source.ConfigVersion)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestUploadDeploymentHandlerRecordsCurrentUserActor(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
|
||||
project, err := CreateProject(t.Context(), Input{Name: "Actor Upload", Slug: "actor-upload", Enabled: true})
|
||||
require.NoError(t, err)
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "ok"})
|
||||
|
||||
var requestBody bytes.Buffer
|
||||
writer := multipart.NewWriter(&requestBody)
|
||||
part, err := writer.CreateFormFile("package", "site.zip")
|
||||
require.NoError(t, err)
|
||||
_, err = part.Write(packageBytes)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/d/pages/1/deployments/upload", &requestBody)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
c.Request = req
|
||||
c.Params = gin.Params{{Key: "id", Value: strconv.FormatUint(uint64(project.ID), 10)}}
|
||||
c.Set(contracts.AuthUserIDKey, uint64(42))
|
||||
|
||||
UploadDeploymentHandler(c)
|
||||
assert.Equal(t, http.StatusOK, recorder.Code)
|
||||
deployments, err := repository.ListPagesDeployments(t.Context(), project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 1)
|
||||
assert.Equal(t, "user:42", deployments[0].CreatedBy)
|
||||
}
|
||||
|
||||
func TestUploadDeploymentFromURLHandlerRecordsCurrentUserActor(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
|
||||
project, err := CreateProject(t.Context(), Input{Name: "Actor URL", Slug: "actor-url", Enabled: true})
|
||||
require.NoError(t, err)
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "ok"})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="site.zip"`)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(packageBytes)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
body, err := json.Marshal(UploadFromURLInput{URL: server.URL + "/site.zip"})
|
||||
require.NoError(t, err)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/d/pages/1/deployments/upload-from-url", bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
c.Request = req
|
||||
c.Params = gin.Params{{Key: "id", Value: fmt.Sprint(project.ID)}}
|
||||
c.Set(contracts.AuthUserIDKey, uint64(77))
|
||||
|
||||
UploadDeploymentFromURLHandler(c)
|
||||
assert.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String())
|
||||
deployments, err := repository.ListPagesDeployments(t.Context(), project.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deployments, 1)
|
||||
assert.Equal(t, "user:77", deployments[0].CreatedBy)
|
||||
}
|
||||
|
||||
func TestCurrentPagesActorRejectsMissingUser(t *testing.T) {
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
|
||||
actor, ok := currentPagesActor(c)
|
||||
assert.False(t, ok)
|
||||
assert.Empty(t, actor)
|
||||
assert.True(t, c.IsAborted())
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
// PagesSourceTypeManual represents projects without a persisted source row.
|
||||
PagesSourceTypeManual = "manual"
|
||||
// PagesSourceTypeRemoteURL represents a persisted artifact URL.
|
||||
PagesSourceTypeRemoteURL = "remote_url"
|
||||
// PagesSourceTypeGitHubRelease represents a public GitHub Release asset.
|
||||
PagesSourceTypeGitHubRelease = "github_release"
|
||||
|
||||
pagesSourceStatusIdle = "idle"
|
||||
pagesSourceStatusChecking = "checking"
|
||||
pagesSourceStatusUpdateAvailable = "update_available"
|
||||
pagesSourceStatusSyncing = "syncing"
|
||||
pagesSourceStatusFailed = "failed"
|
||||
pagesSourceStatusAttention = "attention"
|
||||
|
||||
defaultRemoteAssetLabel = "pages-package"
|
||||
defaultGitHubAssetName = "dist.zip"
|
||||
defaultCheckInterval = 1440
|
||||
minimumCheckInterval = 5
|
||||
maximumCheckInterval = 1440
|
||||
)
|
||||
|
||||
// SourceUpdateInput is the discriminated source configuration payload.
|
||||
// GitHub fields are accepted by the decoder so mode-incompatible values can be
|
||||
// rejected deterministically.
|
||||
type SourceUpdateInput struct {
|
||||
SourceType string `json:"source_type"`
|
||||
RemoteURL string `json:"remote_url"`
|
||||
AllowInsecure bool `json:"allow_insecure"`
|
||||
RepositoryURL string `json:"repository_url"`
|
||||
ReleaseSelector string `json:"release_selector"`
|
||||
ReleaseTag string `json:"release_tag"`
|
||||
AssetName string `json:"asset_name"`
|
||||
AutoUpdateEnabled bool `json:"auto_update_enabled"`
|
||||
CheckIntervalMinutes int `json:"check_interval_minutes"`
|
||||
}
|
||||
|
||||
// SourceRevisionView is a source cursor shown to the console.
|
||||
type SourceRevisionView struct {
|
||||
Revision string `json:"revision"`
|
||||
Label string `json:"label"`
|
||||
AssetName string `json:"asset_name,omitempty"`
|
||||
}
|
||||
|
||||
// SourceView is the discriminated source view returned to the console.
|
||||
type SourceView struct {
|
||||
SourceType string `json:"source_type"`
|
||||
RemoteURL string `json:"remote_url,omitempty"`
|
||||
AllowInsecure bool `json:"allow_insecure,omitempty"`
|
||||
GitHubRepository string `json:"github_repository,omitempty"`
|
||||
ReleaseSelector string `json:"release_selector,omitempty"`
|
||||
ReleaseTag string `json:"release_tag,omitempty"`
|
||||
AssetName string `json:"asset_name,omitempty"`
|
||||
AutoUpdateEnabled *bool `json:"auto_update_enabled,omitempty"`
|
||||
CheckIntervalMinutes int `json:"check_interval_minutes,omitempty"`
|
||||
SyncStatus string `json:"sync_status,omitempty"`
|
||||
UpdateAvailable bool `json:"update_available,omitempty"`
|
||||
LastSeen *SourceRevisionView `json:"last_seen,omitempty"`
|
||||
LastApplied *SourceRevisionView `json:"last_applied,omitempty"`
|
||||
LastCheckedAt *time.Time `json:"last_checked_at,omitempty"`
|
||||
LastSyncedAt *time.Time `json:"last_synced_at,omitempty"`
|
||||
NextCheckAt *time.Time `json:"next_check_at,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
// SourceActionReceipt identifies the internal task execution created by an action API.
|
||||
type SourceActionReceipt struct {
|
||||
TaskID string `json:"task_id"`
|
||||
ExecutionID string `json:"execution_id"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
|
||||
// SourceUpdateResult is returned after persisting source configuration.
|
||||
type SourceUpdateResult struct {
|
||||
Source *SourceView `json:"source"`
|
||||
CheckTask *SourceActionReceipt `json:"check_task"`
|
||||
Warning string `json:"warning"`
|
||||
}
|
||||
|
||||
type sourceDetail struct {
|
||||
Provider string `json:"provider"`
|
||||
DisplayName string `json:"display_name,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
LegacyLabel string `json:"label,omitempty"`
|
||||
AssetName string `json:"asset_name,omitempty"`
|
||||
ReleaseID string `json:"release_id,omitempty"`
|
||||
AssetID string `json:"asset_id,omitempty"`
|
||||
AssetUpdatedAt string `json:"asset_updated_at,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
}
|
||||
|
||||
type remoteSourceConfig struct {
|
||||
URL string
|
||||
AllowInsecure bool
|
||||
Identity string
|
||||
}
|
||||
|
||||
// GetSource returns the current persisted source or a manual discriminator.
|
||||
func GetSource(ctx context.Context, projectID uint) (*SourceView, error) {
|
||||
if _, err := repository.GetPagesProjectByID(ctx, projectID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
source, runtime, err := loadSourceByProject(ctx, projectID)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return &SourceView{SourceType: PagesSourceTypeManual}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildSourceView(source, runtime)
|
||||
}
|
||||
|
||||
// UpdateSource creates or updates a source. Direct callers use the system actor;
|
||||
// HTTP handlers should call UpdateSourceAs so the initial check is auditable.
|
||||
func UpdateSource(ctx context.Context, projectID uint, input SourceUpdateInput) (*SourceUpdateResult, error) {
|
||||
return UpdateSourceAs(ctx, projectID, input, pagesSourceCreatedBySystem)
|
||||
}
|
||||
|
||||
// UpdateSourceAs persists source configuration and queues the first GitHub check
|
||||
// after commit when the GitHub configuration was materially changed.
|
||||
func UpdateSourceAs(
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
input SourceUpdateInput,
|
||||
actor string,
|
||||
) (*SourceUpdateResult, error) {
|
||||
if !validPagesSourceActor(actor) {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
if err := validateSourceUpdateInput(input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
changed := false
|
||||
var persistedSource model.PagesProjectSource
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
var err error
|
||||
switch strings.TrimSpace(input.SourceType) {
|
||||
case PagesSourceTypeRemoteURL:
|
||||
changed, err = updateRemoteSourceTx(tx, projectID, input)
|
||||
case PagesSourceTypeGitHubRelease:
|
||||
changed, err = updateGitHubSourceTx(tx, projectID, input)
|
||||
default:
|
||||
err = errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
if err != nil || !changed || strings.TrimSpace(input.SourceType) != PagesSourceTypeGitHubRelease {
|
||||
return err
|
||||
}
|
||||
source, loadErr := repository.LockPagesProjectSourceByProjectIDTx(tx, projectID)
|
||||
if loadErr != nil {
|
||||
return loadErr
|
||||
}
|
||||
persistedSource = *source
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
view, err := GetSource(ctx, projectID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := &SourceUpdateResult{Source: view, Warning: ""}
|
||||
if changed && strings.TrimSpace(input.SourceType) == PagesSourceTypeGitHubRelease {
|
||||
receipt, dispatchErr := dispatchSourceActionSnapshot(ctx, persistedSource, sourceActionCheck, actor, "", "", "manual")
|
||||
if dispatchErr != nil {
|
||||
result.Warning = errPagesSourceInitialCheckWarning
|
||||
markInitialCheckDispatchFailed(ctx, persistedSource.ID, persistedSource.ConfigVersion)
|
||||
} else {
|
||||
result.CheckTask = receipt
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func updateRemoteSourceTx(tx *gorm.DB, projectID uint, input SourceUpdateInput) (bool, error) {
|
||||
if _, err := repository.LockPagesProjectByIDTx(tx, projectID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
existing, hasExisting, err := loadProjectSourceForUpdate(tx, projectID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
config, err := buildRemoteSourceConfig(existing, hasExisting, input)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !hasExisting {
|
||||
return true, createRemoteSourceTx(tx, projectID, config)
|
||||
}
|
||||
return updateExistingRemoteSourceTx(tx, existing, config)
|
||||
}
|
||||
|
||||
func loadProjectSourceForUpdate(tx *gorm.DB, projectID uint) (*model.PagesProjectSource, bool, error) {
|
||||
source, err := repository.LockPagesProjectSourceByProjectIDTx(tx, projectID)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return &model.PagesProjectSource{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return source, true, nil
|
||||
}
|
||||
|
||||
func buildRemoteSourceConfig(
|
||||
_ *model.PagesProjectSource,
|
||||
_ bool,
|
||||
input SourceUpdateInput,
|
||||
) (remoteSourceConfig, error) {
|
||||
remoteURL := strings.TrimSpace(input.RemoteURL)
|
||||
if remoteURL == "" {
|
||||
return remoteSourceConfig{}, errors.New(errPagesSourceRemoteURLRequired)
|
||||
}
|
||||
parsedURL, err := parseRemoteSourceURL(remoteURL)
|
||||
if err != nil {
|
||||
return remoteSourceConfig{}, err
|
||||
}
|
||||
return remoteSourceConfig{
|
||||
URL: remoteURL,
|
||||
AllowInsecure: input.AllowInsecure,
|
||||
Identity: remoteSourceIdentity(parsedURL),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func createRemoteSourceTx(tx *gorm.DB, projectID uint, config remoteSourceConfig) error {
|
||||
source := &model.PagesProjectSource{
|
||||
ProjectID: projectID,
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: config.URL,
|
||||
AllowInsecure: config.AllowInsecure,
|
||||
AutoUpdateEnabled: false,
|
||||
CheckIntervalMinutes: 0,
|
||||
ConfigVersion: 1,
|
||||
SourceIdentity: config.Identity,
|
||||
}
|
||||
if err := repository.CreatePagesProjectSourceTx(tx, source); err != nil {
|
||||
return err
|
||||
}
|
||||
return repository.CreatePagesProjectSourceRuntimeTx(tx, &model.PagesProjectSourceRuntime{
|
||||
SourceID: source.ID,
|
||||
SyncStatus: pagesSourceStatusIdle,
|
||||
})
|
||||
}
|
||||
|
||||
func updateExistingRemoteSourceTx(
|
||||
tx *gorm.DB,
|
||||
existing *model.PagesProjectSource,
|
||||
config remoteSourceConfig,
|
||||
) (bool, error) {
|
||||
if !remoteSourceConfigChanged(existing, config) {
|
||||
return false, nil
|
||||
}
|
||||
runtime, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, existing.ID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
identityChanged := existing.SourceIdentity != config.Identity
|
||||
if err := repository.UpdatePagesProjectSourceTx(tx, existing, map[string]any{
|
||||
"source_type": PagesSourceTypeRemoteURL,
|
||||
"remote_url": config.URL,
|
||||
"allow_insecure": config.AllowInsecure,
|
||||
"github_repository": "",
|
||||
"release_selector": "",
|
||||
"release_tag": "",
|
||||
"asset_name": "",
|
||||
sourceColumnAutoUpdateEnabled: false,
|
||||
"check_interval_minutes": 0,
|
||||
sourceColumnConfigVersion: existing.ConfigVersion + 1,
|
||||
"source_identity": config.Identity,
|
||||
}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, resetRuntimeAfterSourceUpdate(tx, runtime, identityChanged)
|
||||
}
|
||||
|
||||
func remoteSourceConfigChanged(existing *model.PagesProjectSource, config remoteSourceConfig) bool {
|
||||
return existing.SourceType != PagesSourceTypeRemoteURL ||
|
||||
existing.RemoteURL != config.URL ||
|
||||
existing.AllowInsecure != config.AllowInsecure ||
|
||||
existing.GitHubRepository != "" ||
|
||||
existing.ReleaseSelector != "" ||
|
||||
existing.ReleaseTag != "" ||
|
||||
existing.AssetName != "" ||
|
||||
existing.AutoUpdateEnabled ||
|
||||
existing.CheckIntervalMinutes != 0
|
||||
}
|
||||
|
||||
// DeleteSource idempotently switches a project back to manual mode.
|
||||
func DeleteSource(ctx context.Context, projectID uint) (*SourceView, error) {
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
if _, err := repository.LockPagesProjectByIDTx(tx, projectID); err != nil {
|
||||
return err
|
||||
}
|
||||
source, err := repository.LockPagesProjectSourceByProjectIDTx(tx, projectID)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, source.ID); err != nil &&
|
||||
!errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return err
|
||||
}
|
||||
if err := repository.DeletePagesProjectSourceRuntimeBySourceIDTx(tx, source.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
return repository.DeletePagesProjectSourceTx(tx, source)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &SourceView{SourceType: PagesSourceTypeManual}, nil
|
||||
}
|
||||
|
||||
func validateRemoteSourceInput(input SourceUpdateInput) error {
|
||||
sourceType := strings.TrimSpace(input.SourceType)
|
||||
if sourceType == "" {
|
||||
return errors.New(errPagesSourceTypeRequired)
|
||||
}
|
||||
if sourceType != PagesSourceTypeRemoteURL {
|
||||
return errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
if strings.TrimSpace(input.RepositoryURL) != "" || strings.TrimSpace(input.ReleaseSelector) != "" ||
|
||||
strings.TrimSpace(input.ReleaseTag) != "" || strings.TrimSpace(input.AssetName) != "" ||
|
||||
input.AutoUpdateEnabled || input.CheckIntervalMinutes != 0 {
|
||||
return errors.New(errPagesSourceRemoteFields)
|
||||
}
|
||||
if strings.TrimSpace(input.RemoteURL) == "" {
|
||||
return errors.New(errPagesSourceRemoteURLRequired)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateSourceUpdateInput(input SourceUpdateInput) error {
|
||||
switch strings.TrimSpace(input.SourceType) {
|
||||
case PagesSourceTypeRemoteURL:
|
||||
return validateRemoteSourceInput(input)
|
||||
case PagesSourceTypeGitHubRelease:
|
||||
return validateGitHubSourceInput(input)
|
||||
case "":
|
||||
return errors.New(errPagesSourceTypeRequired)
|
||||
default:
|
||||
return errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
}
|
||||
|
||||
func parseRemoteSourceURL(raw string) (*url.URL, error) {
|
||||
parsed, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
|
||||
return nil, errors.New(errPagesSourceRemoteURLInvalid)
|
||||
}
|
||||
parsed.Scheme = strings.ToLower(parsed.Scheme)
|
||||
if parsed.Scheme != remoteSourceSchemeHTTP && parsed.Scheme != remoteSourceSchemeHTTPS {
|
||||
return nil, errors.New(errPagesSourceRemoteURLInvalid)
|
||||
}
|
||||
if strings.TrimSpace(parsed.Hostname()) == "" {
|
||||
return nil, errors.New(errPagesSourceRemoteURLInvalid)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func remoteSourceIdentity(parsed *url.URL) string {
|
||||
hostname := strings.ToLower(parsed.Hostname())
|
||||
port := parsed.Port()
|
||||
if (parsed.Scheme == "https" && port == "443") || (parsed.Scheme == "http" && port == "80") {
|
||||
port = ""
|
||||
}
|
||||
host := hostname
|
||||
if port != "" {
|
||||
host = net.JoinHostPort(hostname, port)
|
||||
} else if strings.Contains(hostname, ":") {
|
||||
host = "[" + hostname + "]"
|
||||
}
|
||||
canonicalPath := parsed.EscapedPath()
|
||||
if canonicalPath == "" {
|
||||
canonicalPath = "/"
|
||||
}
|
||||
canonicalPath = path.Clean("/" + strings.TrimPrefix(canonicalPath, "/"))
|
||||
canonical := parsed.Scheme + "://" + host + canonicalPath
|
||||
sum := sha256.Sum256([]byte("remote_url|" + canonical))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func loadSourceByProject(ctx context.Context, projectID uint) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime, error) {
|
||||
return repository.GetPagesProjectSourceAndRuntimeByProjectID(ctx, projectID)
|
||||
}
|
||||
|
||||
func buildSourceView(source *model.PagesProjectSource, runtime *model.PagesProjectSourceRuntime) (*SourceView, error) {
|
||||
if source == nil || runtime == nil {
|
||||
return nil, errors.New(errPagesSourceNotFound)
|
||||
}
|
||||
view := &SourceView{
|
||||
SourceType: source.SourceType,
|
||||
SyncStatus: runtime.SyncStatus,
|
||||
LastSyncedAt: runtime.LastSyncedAt,
|
||||
LastError: runtime.LastError,
|
||||
}
|
||||
if runtime.LastAppliedRevision != "" {
|
||||
view.LastApplied = revisionView(runtime.LastAppliedRevision, runtime.LastAppliedDetail)
|
||||
}
|
||||
switch source.SourceType {
|
||||
case PagesSourceTypeRemoteURL:
|
||||
view.RemoteURL = source.RemoteURL
|
||||
view.AllowInsecure = source.AllowInsecure
|
||||
case PagesSourceTypeGitHubRelease:
|
||||
view.LastCheckedAt = runtime.LastCheckedAt
|
||||
view.NextCheckAt = runtime.NextCheckAt
|
||||
view.UpdateAvailable = runtime.LastSeenRevision != "" && runtime.LastSeenRevision != runtime.LastAppliedRevision
|
||||
if runtime.LastSeenRevision != "" {
|
||||
view.LastSeen = revisionView(runtime.LastSeenRevision, runtime.LastSeenDetail)
|
||||
}
|
||||
view.GitHubRepository = source.GitHubRepository
|
||||
view.ReleaseSelector = source.ReleaseSelector
|
||||
view.ReleaseTag = source.ReleaseTag
|
||||
view.AssetName = source.AssetName
|
||||
autoUpdateEnabled := source.AutoUpdateEnabled
|
||||
view.AutoUpdateEnabled = &autoUpdateEnabled
|
||||
view.CheckIntervalMinutes = source.CheckIntervalMinutes
|
||||
default:
|
||||
return nil, errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
return view, nil
|
||||
}
|
||||
|
||||
func revisionView(revision string, detailJSON string) *SourceRevisionView {
|
||||
detail := sourceDetail{}
|
||||
_ = unmarshalSourceDetail(detailJSON, &detail)
|
||||
label := sourceDetailLabel(detail)
|
||||
if label == "" {
|
||||
label = defaultRemoteAssetLabel
|
||||
}
|
||||
return &SourceRevisionView{
|
||||
Revision: revision,
|
||||
Label: label,
|
||||
AssetName: detail.AssetName,
|
||||
}
|
||||
}
|
||||
|
||||
func sourceDetailLabel(detail sourceDetail) string {
|
||||
if detail.Provider == githubSourceDetailProvider {
|
||||
if label := strings.TrimSpace(detail.Tag); label != "" {
|
||||
return label
|
||||
}
|
||||
return strings.TrimSpace(detail.LegacyLabel)
|
||||
}
|
||||
if label := strings.TrimSpace(detail.DisplayName); label != "" {
|
||||
return label
|
||||
}
|
||||
return strings.TrimSpace(detail.LegacyLabel)
|
||||
}
|
||||
|
||||
func unmarshalSourceDetail(raw string, detail *sourceDetail) error {
|
||||
if detail == nil || strings.TrimSpace(raw) == "" {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal([]byte(raw), detail)
|
||||
}
|
||||
|
||||
func resetRuntimeAfterSourceUpdate(tx *gorm.DB, runtime *model.PagesProjectSourceRuntime, identityChanged bool) error {
|
||||
updates := map[string]any{
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
sourceRuntimeColumnLastError: "",
|
||||
}
|
||||
if identityChanged {
|
||||
updates["etag"] = ""
|
||||
updates["last_seen_revision"] = ""
|
||||
updates["last_seen_detail"] = ""
|
||||
updates["last_applied_revision"] = ""
|
||||
updates["last_applied_detail"] = ""
|
||||
updates["last_checked_at"] = nil
|
||||
updates["last_synced_at"] = nil
|
||||
updates["next_check_at"] = nil
|
||||
updates[sourceRuntimeColumnSyncStatus] = pagesSourceStatusIdle
|
||||
} else {
|
||||
updates[sourceRuntimeColumnSyncStatus] = normalizedSourceRuntimeStatus(runtime)
|
||||
}
|
||||
return repository.UpdatePagesProjectSourceRuntimeTx(tx, runtime, updates)
|
||||
}
|
||||
|
||||
func normalizedSourceRuntimeStatus(runtime *model.PagesProjectSourceRuntime) string {
|
||||
if runtime == nil {
|
||||
return pagesSourceStatusIdle
|
||||
}
|
||||
if sourceHasSameReleaseReplacement(runtime) {
|
||||
return pagesSourceStatusAttention
|
||||
}
|
||||
if runtime.LastSeenRevision != "" && runtime.LastSeenRevision != runtime.LastAppliedRevision {
|
||||
return pagesSourceStatusUpdateAvailable
|
||||
}
|
||||
return pagesSourceStatusIdle
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
)
|
||||
|
||||
func syncRemoteSource(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
actor string,
|
||||
) (*sourceSyncOutcome, error) {
|
||||
return syncRemoteSourceWithTrigger(ctx, snapshot, actor, pagesSourceTriggerManualSync)
|
||||
}
|
||||
|
||||
func syncGitHubSource(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
actor string,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
) (*sourceSyncOutcome, error) {
|
||||
return syncGitHubSourceWithTrigger(
|
||||
ctx, snapshot, actor, targetRevision, confirmedRevision, pagesSourceTriggerManualSync,
|
||||
)
|
||||
}
|
||||
|
||||
func commitSourceDeployment(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
revision string,
|
||||
packageChecksum string,
|
||||
detail sourceDetail,
|
||||
detailJSON string,
|
||||
actor string,
|
||||
manifest *deploymentManifest,
|
||||
ingestResult ofupload.IngestResult,
|
||||
hasIngest bool,
|
||||
nextCheckNotBefore *time.Time,
|
||||
) (*model.PagesDeployment, bool, bool, error) {
|
||||
return commitSourceDeploymentWithTrigger(
|
||||
ctx, snapshot, revision, packageChecksum, detail, detailJSON, actor,
|
||||
pagesSourceTriggerManualSync, manifest, ingestResult, hasIngest, nextCheckNotBefore,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/pkg/logger"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const pagesOrphanUploadIsolation = 2 * time.Hour
|
||||
|
||||
// PagesOrphanCleanupSummary describes one bounded delayed compensation pass.
|
||||
// Every candidate is counted exactly once in one outcome field.
|
||||
//
|
||||
//nolint:revive // Keep the domain-qualified exported name for scanner/task result clarity.
|
||||
type PagesOrphanCleanupSummary struct {
|
||||
Candidates int `json:"candidates"`
|
||||
Reconciled int `json:"reconciled"`
|
||||
Referenced int `json:"referenced"`
|
||||
LeaseBusy int `json:"lease_busy"`
|
||||
InvalidMarker int `json:"invalid_marker"`
|
||||
Skipped int `json:"skipped"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
type pagesOrphanMarker struct {
|
||||
ProjectID uint
|
||||
SourceID *uint
|
||||
}
|
||||
|
||||
type pagesOrphanCleanupOutcome uint8
|
||||
|
||||
const (
|
||||
pagesOrphanCleanupSkipped pagesOrphanCleanupOutcome = iota
|
||||
pagesOrphanCleanupReconciled
|
||||
pagesOrphanCleanupReferenced
|
||||
pagesOrphanCleanupLeaseBusy
|
||||
pagesOrphanCleanupInvalidMarker
|
||||
)
|
||||
|
||||
// ReconcilePagesOrphanUploads performs one bounded delayed compensation pass.
|
||||
// Individual candidate failures are counted and logged so they do not prevent
|
||||
// the scanner from continuing with source checks.
|
||||
func ReconcilePagesOrphanUploads(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
) (PagesOrphanCleanupSummary, error) {
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
cutoff := now.UTC().Add(-pagesOrphanUploadIsolation)
|
||||
systemUser := repository.GetSystemUser(ctx)
|
||||
candidates, err := repository.ListPagesOrphanUploadCandidates(ctx, model.PagesOrphanUploadCandidateQuery{
|
||||
SystemUserID: systemUser.ID,
|
||||
UploadType: ofupload.ReservedPagesDeploymentType,
|
||||
Marker: pagesIngestMarkerV2,
|
||||
CreatedBefore: cutoff,
|
||||
})
|
||||
if err != nil {
|
||||
return PagesOrphanCleanupSummary{}, err
|
||||
}
|
||||
|
||||
summary := PagesOrphanCleanupSummary{Candidates: len(candidates)}
|
||||
for index := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return summary, err
|
||||
}
|
||||
candidate := &candidates[index]
|
||||
marker, err := parsePagesOrphanMarker(candidate.Metadata)
|
||||
if err != nil {
|
||||
summary.InvalidMarker++
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload marker invalid: upload_id=%d error=%v", candidate.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
outcome, err := reconcilePagesOrphanUploadCandidate(
|
||||
ctx,
|
||||
candidate,
|
||||
marker,
|
||||
systemUser.ID,
|
||||
cutoff,
|
||||
)
|
||||
if err != nil {
|
||||
summary.Failed++
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload reconciliation failed: upload_id=%d error=%v", candidate.ID, err)
|
||||
continue
|
||||
}
|
||||
summary.add(outcome)
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (summary *PagesOrphanCleanupSummary) add(outcome pagesOrphanCleanupOutcome) {
|
||||
switch outcome {
|
||||
case pagesOrphanCleanupReconciled:
|
||||
summary.Reconciled++
|
||||
case pagesOrphanCleanupReferenced:
|
||||
summary.Referenced++
|
||||
case pagesOrphanCleanupLeaseBusy:
|
||||
summary.LeaseBusy++
|
||||
case pagesOrphanCleanupInvalidMarker:
|
||||
summary.InvalidMarker++
|
||||
case pagesOrphanCleanupSkipped:
|
||||
summary.Skipped++
|
||||
default:
|
||||
summary.Skipped++
|
||||
}
|
||||
}
|
||||
|
||||
func reconcilePagesOrphanUploadCandidate(
|
||||
ctx context.Context,
|
||||
candidate *model.Upload,
|
||||
marker pagesOrphanMarker,
|
||||
systemUserID uint64,
|
||||
cutoff time.Time,
|
||||
) (pagesOrphanCleanupOutcome, error) {
|
||||
if candidate == nil || candidate.ID == 0 {
|
||||
return pagesOrphanCleanupSkipped, nil
|
||||
}
|
||||
|
||||
outcome := pagesOrphanCleanupSkipped
|
||||
uploadLocked := false
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
scopeOutcome, proceed, err := lockPagesOrphanCleanupScope(ctx, tx, candidate.ID, marker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !proceed {
|
||||
outcome = scopeOutcome
|
||||
return nil
|
||||
}
|
||||
lockedOutcome, locked, err := reconcileLockedPagesOrphanUpload(
|
||||
ctx,
|
||||
tx,
|
||||
candidate.ID,
|
||||
marker,
|
||||
systemUserID,
|
||||
cutoff,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outcome = lockedOutcome
|
||||
uploadLocked = locked
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return pagesOrphanCleanupSkipped, err
|
||||
}
|
||||
if uploadLocked {
|
||||
// Also heal a prior post-commit cache invalidation interruption when the
|
||||
// status transition was an idempotent no-op.
|
||||
ofupload.InvalidateUploadMetaCache(ctx, candidate.ID)
|
||||
}
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
func lockPagesOrphanCleanupScope(
|
||||
ctx context.Context,
|
||||
tx *gorm.DB,
|
||||
uploadID uint64,
|
||||
marker pagesOrphanMarker,
|
||||
) (pagesOrphanCleanupOutcome, bool, error) {
|
||||
var project model.PagesProject
|
||||
if _, err := lockOptionalPagesCleanupRecord(tx, &project, "id = ?", marker.ProjectID); err != nil {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
if marker.SourceID == nil {
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
|
||||
source, err := repository.LockPagesProjectSourceByIDTx(tx, *marker.SourceID)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
if source.ProjectID != marker.ProjectID {
|
||||
logger.WarnF(ctx,
|
||||
"[PagesSource] orphan upload source ownership mismatch: upload_id=%d project_id=%d source_id=%d source_project_id=%d",
|
||||
uploadID,
|
||||
marker.ProjectID,
|
||||
*marker.SourceID,
|
||||
source.ProjectID,
|
||||
)
|
||||
return pagesOrphanCleanupInvalidMarker, false, nil
|
||||
}
|
||||
|
||||
runtime, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, source.ID)
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
// Read the real clock only after obtaining the runtime row lock. The scanner
|
||||
// snapshot time is only an isolation cutoff and may be stale after lock wait.
|
||||
leaseCheckedAt := time.Now()
|
||||
if err == nil && runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(leaseCheckedAt) {
|
||||
return pagesOrphanCleanupLeaseBusy, false, nil
|
||||
}
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
|
||||
func reconcileLockedPagesOrphanUpload(
|
||||
ctx context.Context,
|
||||
tx *gorm.DB,
|
||||
uploadID uint64,
|
||||
marker pagesOrphanMarker,
|
||||
systemUserID uint64,
|
||||
cutoff time.Time,
|
||||
) (pagesOrphanCleanupOutcome, bool, error) {
|
||||
var lockedUpload model.Upload
|
||||
found, err := lockOptionalPagesCleanupRecord(tx, &lockedUpload, "id = ?", uploadID)
|
||||
if err != nil || !found {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
|
||||
lockedMarker, err := parsePagesOrphanMarker(lockedUpload.Metadata)
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload marker changed or invalid: upload_id=%d error=%v", uploadID, err)
|
||||
return pagesOrphanCleanupInvalidMarker, true, nil
|
||||
}
|
||||
if lockedUpload.Status != model.UploadStatusUsed ||
|
||||
lockedUpload.UserID != systemUserID ||
|
||||
lockedUpload.Type != ofupload.ReservedPagesDeploymentType ||
|
||||
!lockedUpload.CreatedAt.Before(cutoff) {
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
if lockedMarker.ProjectID != marker.ProjectID || !sameOptionalPagesSourceID(lockedMarker.SourceID, marker.SourceID) {
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload marker changed during reconciliation: upload_id=%d", uploadID)
|
||||
return pagesOrphanCleanupInvalidMarker, true, nil
|
||||
}
|
||||
|
||||
var references int64
|
||||
if err := tx.Model(&model.PagesDeployment{}).
|
||||
Where("upload_id = ?", lockedUpload.ID).
|
||||
Count(&references).Error; err != nil {
|
||||
return pagesOrphanCleanupSkipped, true, err
|
||||
}
|
||||
if references > 0 {
|
||||
return pagesOrphanCleanupReferenced, true, nil
|
||||
}
|
||||
|
||||
transitioned, err := ofupload.RemoveLockedTx(tx, &lockedUpload)
|
||||
if err != nil {
|
||||
return pagesOrphanCleanupSkipped, true, err
|
||||
}
|
||||
if transitioned {
|
||||
return pagesOrphanCleanupReconciled, true, nil
|
||||
}
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
|
||||
func lockOptionalPagesCleanupRecord(
|
||||
tx *gorm.DB,
|
||||
value any,
|
||||
query string,
|
||||
args ...any,
|
||||
) (bool, error) {
|
||||
err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
|
||||
Where(query, args...).
|
||||
First(value).Error
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func parsePagesOrphanMarker(metadata model.UploadMetadata) (pagesOrphanMarker, error) {
|
||||
if metadata.Extra == nil {
|
||||
return pagesOrphanMarker{}, errors.New("pages marker metadata missing")
|
||||
}
|
||||
marker, ok := metadata.Extra[pagesIngestMarkerKey].(string)
|
||||
if !ok || marker != pagesIngestMarkerV2 {
|
||||
return pagesOrphanMarker{}, errors.New("pages marker version invalid")
|
||||
}
|
||||
projectID, err := parsePagesOrphanMetadataID(metadata.Extra, pagesProjectIDMetadataKey)
|
||||
if err != nil {
|
||||
return pagesOrphanMarker{}, err
|
||||
}
|
||||
result := pagesOrphanMarker{ProjectID: projectID}
|
||||
if _, exists := metadata.Extra[pagesSourceIDMetadataKey]; exists {
|
||||
sourceID, err := parsePagesOrphanMetadataID(metadata.Extra, pagesSourceIDMetadataKey)
|
||||
if err != nil {
|
||||
return pagesOrphanMarker{}, err
|
||||
}
|
||||
result.SourceID = &sourceID
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func parsePagesOrphanMetadataID(extra map[string]any, key string) (uint, error) {
|
||||
raw, exists := extra[key]
|
||||
if !exists {
|
||||
return 0, fmt.Errorf("pages marker %s missing", key)
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok || value == "" {
|
||||
return 0, fmt.Errorf("pages marker %s must be a decimal string", key)
|
||||
}
|
||||
parsed, err := strconv.ParseUint(value, 10, 64)
|
||||
maxModelID := uint64(^uint(0) >> 1)
|
||||
if err != nil || parsed == 0 || parsed > maxModelID || strconv.FormatUint(parsed, 10) != value {
|
||||
return 0, fmt.Errorf("pages marker %s is not a canonical non-zero decimal ID", key)
|
||||
}
|
||||
return uint(parsed), nil
|
||||
}
|
||||
|
||||
func sameOptionalPagesSourceID(left, right *uint) bool {
|
||||
if left == nil || right == nil {
|
||||
return left == nil && right == nil
|
||||
}
|
||||
return *left == *right
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
func TestParsePagesOrphanMarker(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extra map[string]any
|
||||
wantProject uint
|
||||
wantSource uint
|
||||
wantSourceOK bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "manual source marker",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
},
|
||||
wantProject: 12,
|
||||
},
|
||||
{
|
||||
name: "persistent source marker",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
pagesSourceIDMetadataKey: "34",
|
||||
},
|
||||
wantProject: 12,
|
||||
wantSource: 34,
|
||||
wantSourceOK: true,
|
||||
},
|
||||
{
|
||||
name: "project ID must be canonical decimal",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "012",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "source ID must be a string",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
pagesSourceIDMetadataKey: float64(34),
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "zero ID rejected",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "0",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong marker version rejected",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: "pages_deployment_v1",
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := parsePagesOrphanMarker(model.UploadMetadata{Extra: test.extra})
|
||||
if gotErr := err != nil; gotErr != test.wantErr {
|
||||
t.Fatalf("parsePagesOrphanMarker(%v) error = %v, want error presence = %t", test.extra, err, test.wantErr)
|
||||
}
|
||||
if test.wantErr {
|
||||
return
|
||||
}
|
||||
if got.ProjectID != test.wantProject {
|
||||
t.Errorf("parsePagesOrphanMarker(%v).ProjectID = %d, want %d", test.extra, got.ProjectID, test.wantProject)
|
||||
}
|
||||
if gotSourceOK := got.SourceID != nil; gotSourceOK != test.wantSourceOK {
|
||||
t.Fatalf("parsePagesOrphanMarker(%v).SourceID presence = %t, want %t", test.extra, gotSourceOK, test.wantSourceOK)
|
||||
}
|
||||
if got.SourceID != nil && *got.SourceID != test.wantSource {
|
||||
t.Errorf("parsePagesOrphanMarker(%v).SourceID = %d, want %d", test.extra, *got.SourceID, test.wantSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsDeletesEligibleUploadOnce(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
project := createPagesOrphanProject(t, ctx, "eligible-orphan")
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||
if err := ofupload.RebuildUploadStats(ctx); err != nil {
|
||||
t.Fatalf("RebuildUploadStats() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.Candidates != 1 || summary.Reconciled != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one reconciled candidate", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusDeleted)
|
||||
assertPagesCleanupTotalStat(t, ctx, 0)
|
||||
|
||||
second, err := ReconcilePagesOrphanUploads(ctx, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("second ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if second.Candidates != 0 || cleanupOutcomeTotal(second) != 0 {
|
||||
t.Errorf("second ReconcilePagesOrphanUploads() summary = %+v, want empty", second)
|
||||
}
|
||||
assertPagesCleanupTotalStat(t, ctx, 0)
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsAllowsDeletedProjectAndSource(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
missingSourceID := uint(9876)
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), 8765, &missingSourceID)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.Candidates != 1 || summary.Reconciled != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want deleted project/source treated as one orphan", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusDeleted)
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsSkipsBusyLeaseAndSourceMismatch(t *testing.T) {
|
||||
t.Run("unexpired source lease", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
realNow := time.Now().UTC()
|
||||
// A deliberately future scanner snapshot proves lease freshness uses the
|
||||
// real clock after the runtime lock, not this isolation-cutoff input.
|
||||
scannerNow := realNow.Add(24 * time.Hour)
|
||||
project := createPagesOrphanProject(t, ctx, "busy-orphan")
|
||||
source := createPagesOrphanSource(t, ctx, project.ID)
|
||||
future := realNow.Add(time.Hour)
|
||||
if err := db.DB(ctx).Create(&model.PagesProjectSourceRuntime{
|
||||
SourceID: source.ID,
|
||||
LeaseToken: "busy-worker",
|
||||
LeaseExpiresAt: &future,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create busy source runtime error = %v, want nil", err)
|
||||
}
|
||||
candidate := createPagesOrphanUpload(t, ctx, realNow.Add(-3*time.Hour), project.ID, &source.ID)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, scannerNow)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.LeaseBusy != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one lease-busy candidate", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
})
|
||||
|
||||
t.Run("source belongs to another project", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
markerProject := createPagesOrphanProject(t, ctx, "marker-project")
|
||||
actualProject := createPagesOrphanProject(t, ctx, "actual-project")
|
||||
source := createPagesOrphanSource(t, ctx, actualProject.ID)
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), markerProject.ID, &source.ID)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.InvalidMarker != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one ownership mismatch", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
})
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsRejectsMalformedMarker(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), 1, nil)
|
||||
metadata := candidate.Metadata
|
||||
metadata.Extra[pagesProjectIDMetadataKey] = "01"
|
||||
candidate.Metadata = metadata
|
||||
if err := db.DB(ctx).Save(candidate).Error; err != nil {
|
||||
t.Fatalf("seed malformed candidate marker error = %v, want nil", err)
|
||||
}
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.InvalidMarker != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one invalid marker", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
}
|
||||
|
||||
func TestPagesOrphanCleanupAndDeploymentCommitInterleavings(t *testing.T) {
|
||||
t.Run("deployment reference commits first", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
project := createPagesOrphanProject(t, ctx, "deployment-first")
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||
marker, err := parsePagesOrphanMarker(candidate.Metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("parsePagesOrphanMarker() error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Create(&model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 1,
|
||||
Checksum: "deployment-first",
|
||||
Status: model.PagesDeploymentStatusUploaded,
|
||||
UploadID: candidate.ID,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create deployment reference error = %v, want nil", err)
|
||||
}
|
||||
|
||||
outcome, err := reconcilePagesOrphanUploadCandidate(ctx, candidate, marker, 999, now.Add(-2*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("reconcilePagesOrphanUploadCandidate() error = %v, want nil", err)
|
||||
}
|
||||
if outcome != pagesOrphanCleanupReferenced {
|
||||
t.Errorf("reconcilePagesOrphanUploadCandidate() outcome = %d, want %d", outcome, pagesOrphanCleanupReferenced)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
})
|
||||
|
||||
t.Run("cleanup commits first", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
project := createPagesOrphanProject(t, ctx, "cleanup-first")
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.Reconciled != 1 {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() summary = %+v, want one reconciled candidate", summary)
|
||||
}
|
||||
|
||||
target := &model.PagesDeployment{ProjectID: project.ID, UploadID: candidate.ID}
|
||||
err = db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var lockedProject model.PagesProject
|
||||
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&lockedProject, project.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return lockSourceDeploymentUploadsTx(tx, target, ofupload.IngestResult{}, false)
|
||||
})
|
||||
if !errors.Is(err, errSourceFinalFence) {
|
||||
t.Errorf("final deployment upload lock after cleanup error = %v, want %v", err, errSourceFinalFence)
|
||||
}
|
||||
var references int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("upload_id = ?", candidate.ID).Count(&references).Error; err != nil {
|
||||
t.Fatalf("count deployment references error = %v, want nil", err)
|
||||
}
|
||||
if references != 0 {
|
||||
t.Errorf("deployment references after cleanup-first interleaving = %d, want 0", references)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func cleanupOutcomeTotal(summary PagesOrphanCleanupSummary) int {
|
||||
return summary.Reconciled + summary.Referenced + summary.LeaseBusy + summary.InvalidMarker + summary.Skipped + summary.Failed
|
||||
}
|
||||
|
||||
func createPagesOrphanProject(t *testing.T, ctx context.Context, slug string) *model.PagesProject {
|
||||
t.Helper()
|
||||
project := &model.PagesProject{Name: slug, Slug: slug, Enabled: true}
|
||||
if err := db.DB(ctx).Create(project).Error; err != nil {
|
||||
t.Fatalf("create Pages orphan project %q error = %v, want nil", slug, err)
|
||||
}
|
||||
return project
|
||||
}
|
||||
|
||||
func createPagesOrphanSource(t *testing.T, ctx context.Context, projectID uint) *model.PagesProjectSource {
|
||||
t.Helper()
|
||||
source := &model.PagesProjectSource{
|
||||
ProjectID: projectID,
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
ConfigVersion: 1,
|
||||
SourceIdentity: "orphan-source-identity",
|
||||
}
|
||||
if err := db.DB(ctx).Create(source).Error; err != nil {
|
||||
t.Fatalf("create Pages orphan source for project %d error = %v, want nil", projectID, err)
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
func createPagesOrphanUpload(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
createdAt time.Time,
|
||||
projectID uint,
|
||||
sourceID *uint,
|
||||
) *model.Upload {
|
||||
t.Helper()
|
||||
extra := map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10),
|
||||
}
|
||||
if sourceID != nil {
|
||||
extra[pagesSourceIDMetadataKey] = strconv.FormatUint(uint64(*sourceID), 10)
|
||||
}
|
||||
candidate := &model.Upload{
|
||||
UserID: 999,
|
||||
FileName: "site.zip",
|
||||
FilePath: "pages/orphan-site.zip",
|
||||
FileSize: 64,
|
||||
MimeType: "application/zip",
|
||||
Extension: "zip",
|
||||
Hash: "orphan-checksum",
|
||||
Type: ofupload.ReservedPagesDeploymentType,
|
||||
Status: model.UploadStatusUsed,
|
||||
AccessMode: 0,
|
||||
Metadata: model.UploadMetadata{Extra: extra},
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}
|
||||
if err := db.DB(ctx).Create(candidate).Error; err != nil {
|
||||
t.Fatalf("create Pages orphan upload error = %v, want nil", err)
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
|
||||
func assertPagesCleanupUploadStatus(t *testing.T, ctx context.Context, uploadID uint64, want model.UploadStatus) {
|
||||
t.Helper()
|
||||
var got model.Upload
|
||||
if err := db.DB(ctx).First(&got, uploadID).Error; err != nil {
|
||||
t.Fatalf("load upload %d error = %v, want nil", uploadID, err)
|
||||
}
|
||||
if got.Status != want {
|
||||
t.Errorf("upload %d status = %q, want %q", uploadID, got.Status, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPagesCleanupTotalStat(t *testing.T, ctx context.Context, want int64) {
|
||||
t.Helper()
|
||||
var stat model.UploadStat
|
||||
if err := db.DB(ctx).Where("dimension = ? AND stat_key = ?", model.UploadStatDimensionTotal, "").First(&stat).Error; err != nil {
|
||||
t.Fatalf("load total upload stat error = %v, want nil", err)
|
||||
}
|
||||
if stat.FileCount != want {
|
||||
t.Errorf("total upload stat FileCount = %d, want %d", stat.FileCount, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/share/pagesarchive"
|
||||
"Wavelet/pkg/httppool"
|
||||
)
|
||||
|
||||
const (
|
||||
remoteSourceDownloadTimeout = 10 * time.Minute
|
||||
remoteSourceResponseHeaderTimeout = 30 * time.Second
|
||||
remoteSourceDialTimeout = 30 * time.Second
|
||||
remoteSourceDialKeepAlive = 30 * time.Second
|
||||
remoteSourceMaxRedirects = 5
|
||||
remoteSourceMagicSniffBytes = 512
|
||||
remoteSourceMaxSafeLabelBytes = 255
|
||||
remoteSourceFallbackLabel = "package"
|
||||
remoteSourceUserAgent = "OpenFlare Pages Source/2"
|
||||
remoteSourceSchemeHTTP = "http"
|
||||
remoteSourceSchemeHTTPS = "https"
|
||||
)
|
||||
|
||||
type remoteProviderError string
|
||||
|
||||
func (providerError remoteProviderError) Error() string {
|
||||
return string(providerError)
|
||||
}
|
||||
|
||||
const (
|
||||
errRemoteProviderInvalidLimit remoteProviderError = "远程来源部署包大小限制无效"
|
||||
errRemoteProviderRedirectLimit remoteProviderError = "远程来源重定向次数超过限制"
|
||||
errRemoteProviderDownloadFailed remoteProviderError = errPagesPackageURLDownloadFailed
|
||||
errRemoteProviderTooLarge remoteProviderError = errPagesPackageURLTooLarge
|
||||
errRemoteProviderEmpty remoteProviderError = errPagesPackageEmpty
|
||||
errRemoteProviderUnsupported remoteProviderError = errPagesPackageUnsupported
|
||||
errRemoteProviderCleanupFailed remoteProviderError = "清理远程来源临时文件失败"
|
||||
)
|
||||
|
||||
// RemoteSourceRequest describes one immutable Remote URL package fetch.
|
||||
type RemoteSourceRequest struct {
|
||||
URL string
|
||||
AllowInsecure bool
|
||||
MaxPackageBytes int64
|
||||
}
|
||||
|
||||
// SourceCandidate is a constrained, immutable archive downloaded to a
|
||||
// provider-owned temporary file. The caller owns the file after a successful
|
||||
// fetch and must call Cleanup when processing finishes.
|
||||
type SourceCandidate struct {
|
||||
TempPath string
|
||||
Checksum string
|
||||
PackageSize int64
|
||||
Format pagesarchive.Format
|
||||
SafeLabel string
|
||||
}
|
||||
|
||||
// Cleanup removes the candidate temporary file. It is safe to call repeatedly.
|
||||
func (candidate *SourceCandidate) Cleanup() error {
|
||||
if candidate == nil || candidate.TempPath == "" {
|
||||
return nil
|
||||
}
|
||||
tempPath := candidate.TempPath
|
||||
err := os.Remove(tempPath)
|
||||
if err == nil || errors.Is(err, os.ErrNotExist) {
|
||||
candidate.TempPath = ""
|
||||
return nil
|
||||
}
|
||||
return errRemoteProviderCleanupFailed
|
||||
}
|
||||
|
||||
type remoteSourceResolver interface {
|
||||
LookupNetIP(context.Context, string, string) ([]netip.Addr, error)
|
||||
}
|
||||
|
||||
type remoteSourceDependencies struct {
|
||||
resolver remoteSourceResolver
|
||||
dialContext func(context.Context, string, string) (net.Conn, error)
|
||||
createTemp func(string, string) (*os.File, error)
|
||||
}
|
||||
|
||||
// FetchRemoteSource downloads a Remote URL package without writing deployment
|
||||
// state. Errors are reduced to safe domain messages and never contain the raw
|
||||
// URL, query, response headers or response body.
|
||||
func FetchRemoteSource(ctx context.Context, request RemoteSourceRequest) (*SourceCandidate, error) {
|
||||
dialer := &net.Dialer{
|
||||
Timeout: remoteSourceDialTimeout,
|
||||
KeepAlive: remoteSourceDialKeepAlive,
|
||||
}
|
||||
dependencies := remoteSourceDependencies{
|
||||
resolver: net.DefaultResolver,
|
||||
dialContext: dialer.DialContext,
|
||||
createTemp: os.CreateTemp,
|
||||
}
|
||||
return fetchRemoteSource(ctx, request, dependencies)
|
||||
}
|
||||
|
||||
func fetchRemoteSource(ctx context.Context, request RemoteSourceRequest, dependencies remoteSourceDependencies) (*SourceCandidate, error) {
|
||||
if request.MaxPackageBytes <= 0 {
|
||||
return nil, errRemoteProviderInvalidLimit
|
||||
}
|
||||
if dependencies.dialContext == nil || dependencies.createTemp == nil {
|
||||
return nil, errRemoteProviderDownloadFailed
|
||||
}
|
||||
parsed, err := parseRemoteSourceURL(request.URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateRemoteSourceTarget(ctx, parsed); err != nil {
|
||||
return nil, sanitizeRemoteProviderError(ctx, err)
|
||||
}
|
||||
|
||||
safeLabel, namedFormat := remoteSourceLabel(parsed)
|
||||
client := newRemoteSourceClient(request.AllowInsecure, dependencies)
|
||||
defer client.CloseIdleConnections()
|
||||
response, err := requestRemoteSource(ctx, client, parsed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
|
||||
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
||||
return nil, fmt.Errorf("%w: HTTP %d", errRemoteProviderDownloadFailed, response.StatusCode)
|
||||
}
|
||||
if response.ContentLength > request.MaxPackageBytes {
|
||||
return nil, errRemoteProviderTooLarge
|
||||
}
|
||||
|
||||
tempPath, checksum, packageSize, err := streamRemoteSourcePackage(
|
||||
response.Body,
|
||||
request.MaxPackageBytes,
|
||||
dependencies.createTemp,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, sanitizeRemoteProviderError(ctx, err)
|
||||
}
|
||||
format, safeLabel, err := detectRemoteSourceFormat(tempPath, safeLabel, namedFormat)
|
||||
if err != nil {
|
||||
if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
|
||||
return nil, errRemoteProviderCleanupFailed
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &SourceCandidate{
|
||||
TempPath: tempPath,
|
||||
Checksum: checksum,
|
||||
PackageSize: packageSize,
|
||||
Format: format,
|
||||
SafeLabel: safeLabel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newRemoteSourceClient(allowInsecure bool, dependencies remoteSourceDependencies) *http.Client {
|
||||
tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
|
||||
if allowInsecure {
|
||||
// Explicit administrator choice for self-signed or private CA endpoints.
|
||||
tlsConfig.InsecureSkipVerify = true //nolint:gosec // required allow_insecure semantics
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: remoteSourceDownloadTimeout,
|
||||
Transport: httppool.NewTransport(httppool.TransportOptions{
|
||||
Proxy: nil,
|
||||
DialContext: dependencies.dialContext,
|
||||
TLSClientConfig: tlsConfig,
|
||||
ResponseHeaderTimeout: remoteSourceResponseHeaderTimeout,
|
||||
TraceFilter: remoteSourceTraceFilter,
|
||||
}),
|
||||
}
|
||||
client.CheckRedirect = func(next *http.Request, previous []*http.Request) error {
|
||||
if len(previous) > remoteSourceMaxRedirects {
|
||||
return errRemoteProviderRedirectLimit
|
||||
}
|
||||
stripRemoteSourceRedirectHeaders(next)
|
||||
if err := validateRemoteSourceTarget(next.Context(), next.URL); err != nil {
|
||||
return err
|
||||
}
|
||||
applyRemoteSourceHeaders(next)
|
||||
return nil
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func requestRemoteSource(ctx context.Context, client *http.Client, parsed *url.URL) (*http.Response, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||
if err != nil {
|
||||
return nil, errors.New(errPagesSourceRemoteURLInvalid)
|
||||
}
|
||||
applyRemoteSourceHeaders(request)
|
||||
response, err := client.Do(request) //nolint:gosec // scheme and every dial target are validated above
|
||||
if err == nil {
|
||||
return response, nil
|
||||
}
|
||||
if response != nil && response.Body != nil {
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
return nil, sanitizeRemoteProviderError(ctx, err)
|
||||
}
|
||||
|
||||
func applyRemoteSourceHeaders(request *http.Request) {
|
||||
request.Header.Set("User-Agent", remoteSourceUserAgent)
|
||||
request.Header.Set("Accept", "application/octet-stream,application/zip,application/x-tar,application/gzip,*/*;q=0.1")
|
||||
// Preserve the artifact bytes exactly as stored. Automatic HTTP gzip
|
||||
// decompression would change the checksum, size and archive format.
|
||||
request.Header.Set("Accept-Encoding", "identity")
|
||||
}
|
||||
|
||||
func stripRemoteSourceRedirectHeaders(request *http.Request) {
|
||||
request.Header.Del("Authorization")
|
||||
request.Header.Del("Cookie")
|
||||
request.Header.Del("Proxy-Authorization")
|
||||
request.Header.Del("Referer")
|
||||
}
|
||||
|
||||
func remoteSourceTraceFilter(request *http.Request) bool {
|
||||
// otelhttp records url.full. Signed query strings must never enter traces.
|
||||
return request.URL == nil || request.URL.RawQuery == ""
|
||||
}
|
||||
|
||||
func validateRemoteSourceTarget(_ context.Context, target *url.URL) error {
|
||||
if target == nil || target.User != nil || target.Fragment != "" || target.Opaque != "" {
|
||||
return errors.New(errPagesSourceRemoteURLInvalid)
|
||||
}
|
||||
scheme := strings.ToLower(strings.TrimSpace(target.Scheme))
|
||||
if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(target.Hostname()) == "" {
|
||||
return errors.New(errPagesSourceRemoteURLInvalid)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func streamRemoteSourcePackage(
|
||||
body io.Reader,
|
||||
maxPackageBytes int64,
|
||||
createTemp func(string, string) (*os.File, error),
|
||||
) (tempPath string, checksum string, packageSize int64, err error) {
|
||||
if createTemp == nil {
|
||||
return "", "", 0, errRemoteProviderDownloadFailed
|
||||
}
|
||||
tempFile, err := createTemp("", "openflare-pages-source-*")
|
||||
if err != nil {
|
||||
return "", "", 0, errRemoteProviderDownloadFailed
|
||||
}
|
||||
createdTempPath := tempFile.Name()
|
||||
tempPath = createdTempPath
|
||||
defer func() {
|
||||
closeErr := tempFile.Close()
|
||||
if err == nil && closeErr != nil {
|
||||
err = errRemoteProviderDownloadFailed
|
||||
}
|
||||
if err != nil {
|
||||
if removeErr := os.Remove(createdTempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
|
||||
err = errRemoteProviderCleanupFailed
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
hasher := sha256.New()
|
||||
readLimit := maxPackageBytes
|
||||
if readLimit < math.MaxInt64 {
|
||||
readLimit++
|
||||
}
|
||||
packageSize, err = io.Copy(io.MultiWriter(tempFile, hasher), io.LimitReader(body, readLimit))
|
||||
if err != nil {
|
||||
return "", "", 0, errRemoteProviderDownloadFailed
|
||||
}
|
||||
if packageSize > maxPackageBytes {
|
||||
return "", "", 0, errRemoteProviderTooLarge
|
||||
}
|
||||
if packageSize == 0 {
|
||||
return "", "", 0, errRemoteProviderEmpty
|
||||
}
|
||||
checksum = hex.EncodeToString(hasher.Sum(nil))
|
||||
return tempPath, checksum, packageSize, nil
|
||||
}
|
||||
|
||||
func detectRemoteSourceFormat(
|
||||
tempPath string,
|
||||
safeLabel string,
|
||||
namedFormat pagesarchive.Format,
|
||||
) (pagesarchive.Format, string, error) {
|
||||
if namedFormat != "" {
|
||||
return namedFormat, safeLabel, nil
|
||||
}
|
||||
tempFile, err := os.Open(tempPath) //nolint:gosec // path is a provider-created temporary file
|
||||
if err != nil {
|
||||
return "", safeLabel, errRemoteProviderDownloadFailed
|
||||
}
|
||||
defer func() { _ = tempFile.Close() }()
|
||||
|
||||
head := make([]byte, remoteSourceMagicSniffBytes)
|
||||
readBytes, readErr := io.ReadFull(tempFile, head)
|
||||
if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) {
|
||||
return "", safeLabel, errRemoteProviderDownloadFailed
|
||||
}
|
||||
format, ok := pagesarchive.DetectFormatFromBytes(head[:readBytes])
|
||||
if !ok {
|
||||
return "", safeLabel, errRemoteProviderUnsupported
|
||||
}
|
||||
return format, appendRemoteSourceLabelExtension(safeLabel, format), nil
|
||||
}
|
||||
|
||||
func remoteSourceLabel(parsed *url.URL) (string, pagesarchive.Format) {
|
||||
baseName := path.Base(parsed.Path)
|
||||
if baseName == "" || baseName == "." || baseName == "/" {
|
||||
baseName = remoteSourceFallbackLabel
|
||||
}
|
||||
safeLabel := sanitizeRemoteSourceLabel(baseName)
|
||||
format, _ := pagesarchive.DetectFormatFromName(safeLabel)
|
||||
return limitRemoteSourceLabel(safeLabel, format), format
|
||||
}
|
||||
|
||||
func sanitizeRemoteSourceLabel(label string) string {
|
||||
var builder strings.Builder
|
||||
lastReplacement := false
|
||||
for _, character := range label {
|
||||
if isRemoteSourceLabelCharacter(character) {
|
||||
builder.WriteRune(character)
|
||||
lastReplacement = false
|
||||
continue
|
||||
}
|
||||
if !lastReplacement {
|
||||
builder.WriteByte('-')
|
||||
lastReplacement = true
|
||||
}
|
||||
}
|
||||
safeLabel := strings.TrimSpace(builder.String())
|
||||
if safeLabel == "" || strings.Trim(safeLabel, "._-") == "" {
|
||||
return remoteSourceFallbackLabel
|
||||
}
|
||||
return safeLabel
|
||||
}
|
||||
|
||||
func isRemoteSourceLabelCharacter(character rune) bool {
|
||||
return character >= 'a' && character <= 'z' ||
|
||||
character >= 'A' && character <= 'Z' ||
|
||||
character >= '0' && character <= '9' ||
|
||||
character == '.' || character == '-' || character == '_'
|
||||
}
|
||||
|
||||
func limitRemoteSourceLabel(label string, format pagesarchive.Format) string {
|
||||
if len(label) <= remoteSourceMaxSafeLabelBytes {
|
||||
return label
|
||||
}
|
||||
if format == "" {
|
||||
return strings.TrimRight(label[:remoteSourceMaxSafeLabelBytes], ".-_")
|
||||
}
|
||||
extension := "." + pagesarchive.Extension(format)
|
||||
prefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
|
||||
prefix := strings.TrimRight(label[:prefixLength], ".-_")
|
||||
if prefix == "" {
|
||||
prefix = remoteSourceFallbackLabel
|
||||
}
|
||||
return prefix + extension
|
||||
}
|
||||
|
||||
func appendRemoteSourceLabelExtension(label string, format pagesarchive.Format) string {
|
||||
extension := "." + pagesarchive.Extension(format)
|
||||
maxPrefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
|
||||
if len(label) > maxPrefixLength {
|
||||
label = strings.TrimRight(label[:maxPrefixLength], ".-_")
|
||||
}
|
||||
if label == "" {
|
||||
label = remoteSourceFallbackLabel
|
||||
}
|
||||
return label + extension
|
||||
}
|
||||
|
||||
func sanitizeRemoteProviderError(ctx context.Context, err error) error {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return fmt.Errorf("%w: %w", errRemoteProviderDownloadFailed, ctxErr)
|
||||
}
|
||||
for _, safeError := range []error{
|
||||
errRemoteProviderInvalidLimit,
|
||||
errRemoteProviderRedirectLimit,
|
||||
errRemoteProviderTooLarge,
|
||||
errRemoteProviderEmpty,
|
||||
errRemoteProviderUnsupported,
|
||||
errRemoteProviderCleanupFailed,
|
||||
} {
|
||||
if errors.Is(err, safeError) {
|
||||
return safeError
|
||||
}
|
||||
}
|
||||
return errRemoteProviderDownloadFailed
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type remoteSourceResolverFunc func(context.Context, string, string) ([]netip.Addr, error)
|
||||
|
||||
func (function remoteSourceResolverFunc) LookupNetIP(
|
||||
ctx context.Context,
|
||||
network string,
|
||||
host string,
|
||||
) ([]netip.Addr, error) {
|
||||
return function(ctx, network, host)
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourceTrustedInternalSelfSignedAndSafeLabel(t *testing.T) {
|
||||
packageBytes := makeRemoteSourceZIP(t)
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Query().Get("token") != "source-secret" {
|
||||
t.Error("signed query did not reach the artifact server")
|
||||
}
|
||||
if request.Header.Get("Accept-Encoding") != "identity" {
|
||||
t.Error("artifact request must disable automatic HTTP decompression")
|
||||
}
|
||||
writer.Header().Set("Content-Disposition", `attachment; filename="redirected.tar.gz"`)
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
server.Config.ErrorLog = log.New(io.Discard, "", 0)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: server.URL + "/original/site.zip?token=source-secret",
|
||||
AllowInsecure: true,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchRemoteSource() error = %v", err)
|
||||
}
|
||||
if candidate.Format != "zip" {
|
||||
t.Fatalf("Format = %q, want zip", candidate.Format)
|
||||
}
|
||||
if candidate.SafeLabel != "site.zip" {
|
||||
t.Fatalf("SafeLabel = %q, want original path basename", candidate.SafeLabel)
|
||||
}
|
||||
if candidate.PackageSize != int64(len(packageBytes)) {
|
||||
t.Fatalf("PackageSize = %d, want %d", candidate.PackageSize, len(packageBytes))
|
||||
}
|
||||
wantChecksum := sha256.Sum256(packageBytes)
|
||||
if candidate.Checksum != hex.EncodeToString(wantChecksum[:]) {
|
||||
t.Fatalf("Checksum = %q, want SHA-256", candidate.Checksum)
|
||||
}
|
||||
downloaded, err := os.ReadFile(candidate.TempPath) //nolint:gosec // provider-owned test temp file
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile() error = %v", err)
|
||||
}
|
||||
if !bytes.Equal(downloaded, packageBytes) {
|
||||
t.Fatal("downloaded package differs from response body")
|
||||
}
|
||||
tempPath := candidate.TempPath
|
||||
if err := candidate.Cleanup(); err != nil {
|
||||
t.Fatalf("Cleanup() error = %v", err)
|
||||
}
|
||||
if err := candidate.Cleanup(); err != nil {
|
||||
t.Fatalf("second Cleanup() error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(tempPath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("temporary file still exists: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourceKeepsOriginalLabelAcrossRedirect(t *testing.T) {
|
||||
packageBytes := makeRemoteSourceZIP(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path == "/original/site.zip" {
|
||||
writer.Header().Set("Location", "/delivery/final.tar.gz?token=redirect-secret")
|
||||
writer.WriteHeader(http.StatusFound)
|
||||
return
|
||||
}
|
||||
if request.Header.Get("Referer") != "" {
|
||||
t.Error("redirect must not forward a signed source URL as Referer")
|
||||
}
|
||||
writer.Header().Set("Content-Disposition", `attachment; filename="response.7z"`)
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: server.URL + "/original/site.zip?token=initial-secret",
|
||||
AllowInsecure: true,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchRemoteSource() error = %v", err)
|
||||
}
|
||||
defer func() { _ = candidate.Cleanup() }()
|
||||
if candidate.SafeLabel != "site.zip" || candidate.Format != "zip" {
|
||||
t.Fatalf("candidate = label %q format %q, want original site.zip", candidate.SafeLabel, candidate.Format)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourcePublicAllowsPrivateAddresses(t *testing.T) {
|
||||
packageBytes := makeRemoteSourceZIP(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: server.URL + "/site.zip?token=private-secret",
|
||||
AllowInsecure: false,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchRemoteSource() error = %v", err)
|
||||
}
|
||||
defer func() { _ = candidate.Cleanup() }()
|
||||
if candidate.Format != "zip" {
|
||||
t.Fatalf("candidate format = %q, want zip", candidate.Format)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourcePublicUsesDirectDialer(t *testing.T) {
|
||||
packageBytes := makeRemoteSourceZIP(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
var dialedAddress string
|
||||
dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver())
|
||||
dependencies.dialContext = func(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
dialedAddress = address
|
||||
dialer := &net.Dialer{}
|
||||
return dialer.DialContext(ctx, network, server.Listener.Addr().String())
|
||||
}
|
||||
candidate, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: "http://artifact.example/site.zip",
|
||||
AllowInsecure: false,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
}, dependencies)
|
||||
if err != nil {
|
||||
t.Fatalf("fetchRemoteSource() error = %v", err)
|
||||
}
|
||||
defer func() { _ = candidate.Cleanup() }()
|
||||
if dialedAddress != "artifact.example:80" {
|
||||
t.Fatalf("direct dial address = %q, want hostname dial", dialedAddress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourcePublicAllowsPrivateRedirect(t *testing.T) {
|
||||
packageBytes := makeRemoteSourceZIP(t)
|
||||
var privateServer *httptest.Server
|
||||
privateServer = httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
t.Cleanup(privateServer.Close)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Location", privateServer.URL+"/private.zip?token=redirect-secret")
|
||||
writer.WriteHeader(http.StatusFound)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: server.URL + "/start.zip?token=initial-secret",
|
||||
AllowInsecure: false,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchRemoteSource() error = %v", err)
|
||||
}
|
||||
defer func() { _ = candidate.Cleanup() }()
|
||||
if candidate.Format != "zip" {
|
||||
t.Fatalf("candidate format = %q, want zip", candidate.Format)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourcePublicRejectsSelfSignedTLS(t *testing.T) {
|
||||
packageBytes := makeRemoteSourceZIP(t)
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
server.Config.ErrorLog = log.New(io.Discard, "", 0)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver())
|
||||
rawURL := "https://artifact.example/site.zip?signature=tls-secret"
|
||||
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: rawURL,
|
||||
AllowInsecure: false,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
}, dependencies)
|
||||
if !errors.Is(err, errRemoteProviderDownloadFailed) {
|
||||
t.Fatalf("fetchRemoteSource() error = %v, want strict TLS failure", err)
|
||||
}
|
||||
assertRemoteSourceErrorRedacted(t, err, rawURL, "tls-secret", "signature=")
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourceRejectsChunkedBodyOverLimitAndCleansTemp(t *testing.T) {
|
||||
const maxPackageBytes = int64(64)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(bytes.Repeat([]byte{'x'}, int(maxPackageBytes)))
|
||||
if flusher, ok := writer.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
_, _ = writer.Write([]byte("overflow"))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
tempDir := t.TempDir()
|
||||
dependencies := defaultRemoteSourceDependenciesForTest()
|
||||
dependencies.createTemp = func(_ string, pattern string) (*os.File, error) {
|
||||
return os.CreateTemp(tempDir, pattern)
|
||||
}
|
||||
rawURL := server.URL + "/site.zip?token=chunk-secret"
|
||||
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: rawURL,
|
||||
AllowInsecure: true,
|
||||
MaxPackageBytes: maxPackageBytes,
|
||||
}, dependencies)
|
||||
if !errors.Is(err, errRemoteProviderTooLarge) {
|
||||
t.Fatalf("fetchRemoteSource() error = %v, want actual stream limit", err)
|
||||
}
|
||||
assertRemoteSourceTempDirEmpty(t, tempDir)
|
||||
assertRemoteSourceErrorRedacted(t, err, rawURL, "chunk-secret", "token=")
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourceRejectsContentLengthBeforeCreatingTemp(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Content-Length", "4096")
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
var createCount atomic.Int32
|
||||
dependencies := defaultRemoteSourceDependenciesForTest()
|
||||
dependencies.createTemp = func(directory string, pattern string) (*os.File, error) {
|
||||
createCount.Add(1)
|
||||
return os.CreateTemp(directory, pattern)
|
||||
}
|
||||
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: server.URL + "/site.zip",
|
||||
AllowInsecure: true,
|
||||
MaxPackageBytes: 1024,
|
||||
}, dependencies)
|
||||
if !errors.Is(err, errRemoteProviderTooLarge) {
|
||||
t.Fatalf("fetchRemoteSource() error = %v, want Content-Length rejection", err)
|
||||
}
|
||||
if createCount.Load() != 0 {
|
||||
t.Fatalf("CreateTemp called %d times before Content-Length rejection", createCount.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourceSniffsAtLeast512BytesForTar(t *testing.T) {
|
||||
packageBytes := make([]byte, remoteSourceMagicSniffBytes)
|
||||
copy(packageBytes[257:], []byte("ustar"))
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: server.URL + "/download",
|
||||
AllowInsecure: true,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchRemoteSource() error = %v", err)
|
||||
}
|
||||
defer func() { _ = candidate.Cleanup() }()
|
||||
if candidate.Format != "tar" {
|
||||
t.Fatalf("Format = %q, want tar detected at byte 257", candidate.Format)
|
||||
}
|
||||
if candidate.SafeLabel != "download.tar" {
|
||||
t.Fatalf("SafeLabel = %q, want download.tar", candidate.SafeLabel)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourceRedactsURLHeadersAndBodyFromErrors(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("X-Artifact-Secret", "header-secret")
|
||||
_, _ = writer.Write([]byte("response-body-secret"))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
tempDir := t.TempDir()
|
||||
dependencies := defaultRemoteSourceDependenciesForTest()
|
||||
dependencies.createTemp = func(_ string, pattern string) (*os.File, error) {
|
||||
return os.CreateTemp(tempDir, pattern)
|
||||
}
|
||||
rawURL := server.URL + "/download?token=query-secret"
|
||||
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: rawURL,
|
||||
AllowInsecure: true,
|
||||
MaxPackageBytes: 1024,
|
||||
}, dependencies)
|
||||
if !errors.Is(err, errRemoteProviderUnsupported) {
|
||||
t.Fatalf("fetchRemoteSource() error = %v, want unsupported archive", err)
|
||||
}
|
||||
assertRemoteSourceTempDirEmpty(t, tempDir)
|
||||
assertRemoteSourceErrorRedacted(
|
||||
t,
|
||||
err,
|
||||
rawURL,
|
||||
"query-secret",
|
||||
"header-secret",
|
||||
"response-body-secret",
|
||||
"token=",
|
||||
)
|
||||
}
|
||||
|
||||
func TestFetchRemoteSourceAllowsFiveRedirectsOnly(t *testing.T) {
|
||||
packageBytes := makeRemoteSourceZIP(t)
|
||||
var requestCount atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
requestCount.Add(1)
|
||||
redirectNumber, _ := strconv.Atoi(strings.TrimPrefix(request.URL.Path, "/"))
|
||||
if redirectNumber < remoteSourceMaxRedirects+1 {
|
||||
writer.Header().Set("Location", "/"+strconv.Itoa(redirectNumber+1))
|
||||
writer.WriteHeader(http.StatusFound)
|
||||
return
|
||||
}
|
||||
_, _ = writer.Write(packageBytes)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver())
|
||||
_, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{
|
||||
URL: "http://artifact.example/0",
|
||||
AllowInsecure: false,
|
||||
MaxPackageBytes: int64(len(packageBytes) + 1),
|
||||
}, dependencies)
|
||||
if !errors.Is(err, errRemoteProviderRedirectLimit) {
|
||||
t.Fatalf("fetchRemoteSource() error = %v, want redirect limit", err)
|
||||
}
|
||||
if requestCount.Load() != remoteSourceMaxRedirects+1 {
|
||||
t.Fatalf("request count = %d, want initial plus five redirects", requestCount.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func staticPublicRemoteSourceResolver() remoteSourceResolver {
|
||||
return remoteSourceResolverFunc(func(context.Context, string, string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("93.184.216.34")}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func defaultRemoteSourceDependenciesForTest() remoteSourceDependencies {
|
||||
dialer := &net.Dialer{}
|
||||
return remoteSourceDependencies{
|
||||
resolver: net.DefaultResolver,
|
||||
dialContext: dialer.DialContext,
|
||||
createTemp: os.CreateTemp,
|
||||
}
|
||||
}
|
||||
|
||||
func mappedRemoteSourceDependencies(targetAddress string, resolver remoteSourceResolver) remoteSourceDependencies {
|
||||
dialer := &net.Dialer{}
|
||||
return remoteSourceDependencies{
|
||||
resolver: resolver,
|
||||
dialContext: func(ctx context.Context, network string, _ string) (net.Conn, error) {
|
||||
return dialer.DialContext(ctx, network, targetAddress)
|
||||
},
|
||||
createTemp: os.CreateTemp,
|
||||
}
|
||||
}
|
||||
|
||||
func makeRemoteSourceZIP(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
archive := zip.NewWriter(&buffer)
|
||||
file, err := archive.Create("index.html")
|
||||
if err != nil {
|
||||
t.Fatalf("zip.Create() error = %v", err)
|
||||
}
|
||||
if _, err := file.Write([]byte("<h1>OpenFlare</h1>")); err != nil {
|
||||
t.Fatalf("zip entry Write() error = %v", err)
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
t.Fatalf("zip.Close() error = %v", err)
|
||||
}
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func assertRemoteSourceTempDirEmpty(t *testing.T, directory string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir() error = %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("temporary directory contains %d leaked files", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func assertRemoteSourceErrorRedacted(t *testing.T, err error, sensitiveValues ...string) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
message := err.Error()
|
||||
for _, sensitiveValue := range sensitiveValues {
|
||||
if sensitiveValue != "" && strings.Contains(message, sensitiveValue) {
|
||||
t.Fatalf("error %q contains sensitive value %q", message, sensitiveValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
pagesSourceCheckLeaseDuration = 2 * time.Minute
|
||||
pagesSourceSyncLeaseDuration = 15 * time.Minute
|
||||
sourceLeaseTokenBytes = 32
|
||||
sourceRuntimeErrorMaxBytes = 512
|
||||
sourceRevisionHexLength = 64
|
||||
|
||||
sourceColumnAutoUpdateEnabled = "auto_update_enabled"
|
||||
sourceColumnConfigVersion = "config_version"
|
||||
sourceRuntimeColumnSyncStatus = "sync_status"
|
||||
sourceRuntimeColumnLastError = "last_error"
|
||||
sourceRuntimeColumnLastCheckedAt = "last_checked_at"
|
||||
sourceRuntimeColumnNextCheckAt = "next_check_at"
|
||||
sourceRuntimeColumnLeaseToken = "lease_token"
|
||||
sourceRuntimeColumnLeaseExpiresAt = "lease_expires_at"
|
||||
pagesDeploymentColumnStatus = "status"
|
||||
)
|
||||
|
||||
type sourceLeaseOutcome string
|
||||
|
||||
const (
|
||||
sourceLeaseAcquired sourceLeaseOutcome = "acquired"
|
||||
sourceLeaseBusy sourceLeaseOutcome = "busy"
|
||||
sourceLeaseStale sourceLeaseOutcome = "stale"
|
||||
)
|
||||
|
||||
// sourceExecutionSnapshot captures every mutable value that can affect archive
|
||||
// validation or the atomic activation decision. The queued payload deliberately
|
||||
// does not carry project content configuration.
|
||||
type sourceExecutionSnapshot struct {
|
||||
ProjectID uint
|
||||
SourceID uint
|
||||
SourceConfigVersion int
|
||||
ContentConfigVersion int
|
||||
SourceType string
|
||||
SourceIdentity string
|
||||
RemoteURL string
|
||||
AllowInsecure bool
|
||||
GitHubRepository string
|
||||
ReleaseSelector string
|
||||
ReleaseTag string
|
||||
AssetName string
|
||||
AutoUpdateEnabled bool
|
||||
CheckIntervalMinutes int
|
||||
ETag string
|
||||
LastSeenRevision string
|
||||
LastSeenDetail string
|
||||
LastAppliedRevision string
|
||||
LastAppliedDetail string
|
||||
RootDir string
|
||||
EntryFile string
|
||||
LeaseToken string
|
||||
LeaseExpiresAt time.Time
|
||||
}
|
||||
|
||||
func acquireSourceLease(
|
||||
ctx context.Context,
|
||||
sourceID uint,
|
||||
expectedConfigVersion int,
|
||||
action string,
|
||||
) (*sourceExecutionSnapshot, sourceLeaseOutcome, error) {
|
||||
leaseDuration, status, err := sourceLeaseParameters(action)
|
||||
if err != nil {
|
||||
return nil, sourceLeaseStale, err
|
||||
}
|
||||
token, err := newSourceLeaseToken()
|
||||
if err != nil {
|
||||
return nil, sourceLeaseStale, err
|
||||
}
|
||||
now := time.Now()
|
||||
expiresAt := now.Add(leaseDuration)
|
||||
|
||||
rows, err := repository.TryAcquirePagesSourceRuntimeLease(ctx, sourceID, expectedConfigVersion, now, map[string]any{
|
||||
sourceRuntimeColumnLeaseToken: token,
|
||||
sourceRuntimeColumnLeaseExpiresAt: expiresAt,
|
||||
sourceRuntimeColumnSyncStatus: status,
|
||||
sourceRuntimeColumnLastError: "",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, sourceLeaseStale, err
|
||||
}
|
||||
if rows == 0 {
|
||||
outcome, inspectErr := inspectSourceLeaseMiss(ctx, sourceID, expectedConfigVersion, now)
|
||||
return nil, outcome, inspectErr
|
||||
}
|
||||
|
||||
snapshot, err := loadSourceExecutionSnapshot(ctx, sourceID, token)
|
||||
if err != nil {
|
||||
if errors.Is(err, errSourceLeaseSnapshotStale) {
|
||||
return nil, sourceLeaseStale, nil
|
||||
}
|
||||
return nil, sourceLeaseStale, err
|
||||
}
|
||||
return snapshot, sourceLeaseAcquired, nil
|
||||
}
|
||||
|
||||
var errSourceLeaseSnapshotStale = errors.New("source lease snapshot stale")
|
||||
|
||||
func loadSourceExecutionSnapshot(
|
||||
ctx context.Context,
|
||||
sourceID uint,
|
||||
token string,
|
||||
) (*sourceExecutionSnapshot, error) {
|
||||
var snapshot sourceExecutionSnapshot
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
source, err := repository.GetPagesProjectSourceByIDTx(tx, sourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return errSourceLeaseSnapshotStale
|
||||
}
|
||||
return err
|
||||
}
|
||||
project, err := repository.LockPagesProjectByIDTx(tx, source.ProjectID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return errSourceLeaseSnapshotStale
|
||||
}
|
||||
return err
|
||||
}
|
||||
source, err = repository.LockPagesProjectSourceByIDTx(tx, sourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return errSourceLeaseSnapshotStale
|
||||
}
|
||||
return err
|
||||
}
|
||||
runtime, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, source.ID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return errSourceLeaseSnapshotStale
|
||||
}
|
||||
return err
|
||||
}
|
||||
// 必须在 runtime 行锁拿到后重新取时间，避免锁等待跨过
|
||||
// lease expiry 时仍使用事务开始前的旧时间继续执行。
|
||||
now := time.Now()
|
||||
if runtime.LeaseToken != token || runtime.LeaseExpiresAt == nil || !runtime.LeaseExpiresAt.After(now) {
|
||||
return errSourceLeaseSnapshotStale
|
||||
}
|
||||
snapshot = sourceExecutionSnapshot{
|
||||
ProjectID: project.ID,
|
||||
SourceID: source.ID,
|
||||
SourceConfigVersion: source.ConfigVersion,
|
||||
ContentConfigVersion: project.ContentConfigVersion,
|
||||
SourceType: source.SourceType,
|
||||
SourceIdentity: source.SourceIdentity,
|
||||
RemoteURL: source.RemoteURL,
|
||||
AllowInsecure: source.AllowInsecure,
|
||||
GitHubRepository: source.GitHubRepository,
|
||||
ReleaseSelector: source.ReleaseSelector,
|
||||
ReleaseTag: source.ReleaseTag,
|
||||
AssetName: source.AssetName,
|
||||
AutoUpdateEnabled: source.AutoUpdateEnabled,
|
||||
CheckIntervalMinutes: source.CheckIntervalMinutes,
|
||||
ETag: runtime.ETag,
|
||||
LastSeenRevision: runtime.LastSeenRevision,
|
||||
LastSeenDetail: runtime.LastSeenDetail,
|
||||
LastAppliedRevision: runtime.LastAppliedRevision,
|
||||
LastAppliedDetail: runtime.LastAppliedDetail,
|
||||
RootDir: project.RootDir,
|
||||
EntryFile: project.EntryFile,
|
||||
LeaseToken: token,
|
||||
LeaseExpiresAt: *runtime.LeaseExpiresAt,
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &snapshot, nil
|
||||
}
|
||||
|
||||
func inspectSourceLeaseMiss(
|
||||
ctx context.Context,
|
||||
sourceID uint,
|
||||
expectedConfigVersion int,
|
||||
now time.Time,
|
||||
) (sourceLeaseOutcome, error) {
|
||||
source, err := repository.GetPagesProjectSourceByID(ctx, sourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return sourceLeaseStale, nil
|
||||
}
|
||||
return sourceLeaseStale, err
|
||||
}
|
||||
if source.ConfigVersion != expectedConfigVersion {
|
||||
return sourceLeaseStale, nil
|
||||
}
|
||||
runtime, err := repository.GetPagesProjectSourceRuntimeBySourceID(ctx, sourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return sourceLeaseStale, nil
|
||||
}
|
||||
return sourceLeaseStale, err
|
||||
}
|
||||
if runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(now) {
|
||||
return sourceLeaseBusy, nil
|
||||
}
|
||||
return sourceLeaseStale, nil
|
||||
}
|
||||
|
||||
func sourceLeaseParameters(action string) (time.Duration, string, error) {
|
||||
switch action {
|
||||
case sourceActionCheck:
|
||||
return pagesSourceCheckLeaseDuration, pagesSourceStatusChecking, nil
|
||||
case sourceActionSync:
|
||||
return pagesSourceSyncLeaseDuration, pagesSourceStatusSyncing, nil
|
||||
default:
|
||||
return 0, "", errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
}
|
||||
|
||||
func newSourceLeaseToken() (string, error) {
|
||||
value := make([]byte, sourceLeaseTokenBytes)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(value), nil
|
||||
}
|
||||
|
||||
func renewSourceLease(ctx context.Context, snapshot *sourceExecutionSnapshot, duration time.Duration) (bool, error) {
|
||||
if snapshot == nil || snapshot.SourceID == 0 || snapshot.LeaseToken == "" || duration <= 0 {
|
||||
return false, errors.New(errPagesSourceLeaseLost)
|
||||
}
|
||||
now := time.Now()
|
||||
expiresAt := now.Add(duration)
|
||||
rows, err := repository.RenewPagesSourceRuntimeLease(ctx, snapshot.SourceID, snapshot.LeaseToken, now, expiresAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if rows == 0 {
|
||||
return false, nil
|
||||
}
|
||||
snapshot.LeaseExpiresAt = expiresAt
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func failSourceLease(ctx context.Context, snapshot *sourceExecutionSnapshot, message string) error {
|
||||
if snapshot == nil || snapshot.SourceID == 0 || snapshot.LeaseToken == "" {
|
||||
return nil
|
||||
}
|
||||
message = safeSourceRuntimeError(message)
|
||||
now := time.Now()
|
||||
_, err := repository.UpdatePagesSourceRuntimeByActiveLease(
|
||||
ctx,
|
||||
snapshot.SourceID,
|
||||
snapshot.LeaseToken,
|
||||
now,
|
||||
map[string]any{
|
||||
sourceRuntimeColumnSyncStatus: pagesSourceStatusFailed,
|
||||
sourceRuntimeColumnLastError: message,
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
},
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func safeSourceRuntimeError(message string) string {
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
return errPagesSourceSyncFailed
|
||||
}
|
||||
if len(message) > sourceRuntimeErrorMaxBytes {
|
||||
message = message[:sourceRuntimeErrorMaxBytes]
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func sourceLeaseIsBusy(ctx context.Context, sourceID uint) (bool, error) {
|
||||
runtime, err := repository.GetPagesProjectSourceRuntimeBySourceID(ctx, sourceID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(time.Now()), nil
|
||||
}
|
||||
|
||||
// recoverExpiredSourceLease clears one exact expired lease owner. Matching the
|
||||
// token, observed expiry and status prevents a scanner from overwriting a
|
||||
// worker that renewed or was replaced after the candidate query.
|
||||
func recoverExpiredSourceLease(
|
||||
ctx context.Context,
|
||||
sourceID uint,
|
||||
token string,
|
||||
expiresAt time.Time,
|
||||
status string,
|
||||
now time.Time,
|
||||
nextCheckAt *time.Time,
|
||||
) (bool, error) {
|
||||
if sourceID == 0 || token == "" ||
|
||||
(status != pagesSourceStatusChecking && status != pagesSourceStatusSyncing) {
|
||||
return false, nil
|
||||
}
|
||||
updates := map[string]any{
|
||||
sourceRuntimeColumnSyncStatus: pagesSourceStatusFailed,
|
||||
sourceRuntimeColumnLastError: errPagesSourceLeaseExpired,
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
sourceRuntimeColumnNextCheckAt: nextCheckAt,
|
||||
}
|
||||
rows, err := repository.RecoverExpiredPagesSourceRuntimeLease(
|
||||
ctx, sourceID, token, expiresAt, status, now, updates,
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return rows == 1, nil
|
||||
}
|
||||
|
||||
// fenceAndNormalizeRuntime invalidates in-flight work while preserving safe
|
||||
// seen/applied cursors. The caller must already hold the source row lock.
|
||||
func fenceAndNormalizeRuntime(tx *gorm.DB, sourceID uint) error {
|
||||
runtime, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, sourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return repository.UpdatePagesProjectSourceRuntimeTx(tx, runtime, map[string]any{
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
sourceRuntimeColumnSyncStatus: normalizedSourceRuntimeStatus(runtime),
|
||||
})
|
||||
}
|
||||
|
||||
func sourceHasSameReleaseReplacement(runtime *model.PagesProjectSourceRuntime) bool {
|
||||
if runtime == nil || runtime.LastSeenRevision == "" || runtime.LastSeenRevision == runtime.LastAppliedRevision {
|
||||
return false
|
||||
}
|
||||
seen := sourceDetail{}
|
||||
applied := sourceDetail{}
|
||||
if unmarshalSourceDetail(runtime.LastSeenDetail, &seen) != nil ||
|
||||
unmarshalSourceDetail(runtime.LastAppliedDetail, &applied) != nil {
|
||||
return false
|
||||
}
|
||||
return seen.ReleaseID != "" && seen.ReleaseID == applied.ReleaseID
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
func TestSourceLeaseHeartbeatRenewsAndCancelsOnOwnershipLoss(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "lease-heartbeat")
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip",
|
||||
false,
|
||||
)
|
||||
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
if err != nil || outcome != sourceLeaseAcquired || snapshot == nil {
|
||||
t.Fatalf("acquireSourceLease(heartbeat) = (%+v, %q, %v), want acquired", snapshot, outcome, err)
|
||||
}
|
||||
|
||||
workCtx, heartbeat, err := startSourceLeaseHeartbeat(ctx, snapshot, 500*time.Millisecond, 20*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatalf("startSourceLeaseHeartbeat() error = %v, want nil", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = heartbeat.stop() })
|
||||
|
||||
var initial model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&initial).Error; err != nil {
|
||||
t.Fatalf("load initial heartbeat runtime error = %v, want nil", err)
|
||||
}
|
||||
if initial.LeaseExpiresAt == nil {
|
||||
t.Fatal("initial heartbeat expiry = nil, want non-nil")
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
var renewedRuntime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&renewedRuntime).Error; err != nil {
|
||||
t.Fatalf("load renewed heartbeat runtime error = %v, want nil", err)
|
||||
}
|
||||
if renewedRuntime.LeaseExpiresAt != nil && renewedRuntime.LeaseExpiresAt.After(*initial.LeaseExpiresAt) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("heartbeat did not extend lease before deadline")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", source.ID).
|
||||
Update("lease_token", "replacement-owner").Error; err != nil {
|
||||
t.Fatalf("replace heartbeat lease owner error = %v, want nil", err)
|
||||
}
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("heartbeat work context was not canceled after ownership loss")
|
||||
}
|
||||
if err := heartbeat.stop(); !errors.Is(err, errSourceLeaseHeartbeatLost) {
|
||||
t.Fatalf("heartbeat.stop() error = %v, want %v", err, errSourceLeaseHeartbeatLost)
|
||||
}
|
||||
if err := heartbeat.stop(); !errors.Is(err, errSourceLeaseHeartbeatLost) {
|
||||
t.Fatalf("heartbeat.stop() second error = %v, want stable %v", err, errSourceLeaseHeartbeatLost)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireSourceLeaseConcurrentOnlyOneOwner(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "lease-concurrent")
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip",
|
||||
false,
|
||||
)
|
||||
type leaseResult struct {
|
||||
snapshot *sourceExecutionSnapshot
|
||||
outcome sourceLeaseOutcome
|
||||
err error
|
||||
}
|
||||
results := make(chan leaseResult, 2)
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(2)
|
||||
for range 2 {
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
results <- leaseResult{snapshot: snapshot, outcome: outcome, err: err}
|
||||
}()
|
||||
}
|
||||
workers.Wait()
|
||||
close(results)
|
||||
|
||||
acquired := 0
|
||||
busy := 0
|
||||
for result := range results {
|
||||
if result.err != nil {
|
||||
t.Errorf("acquireSourceLease(concurrent) error = %v, want nil", result.err)
|
||||
continue
|
||||
}
|
||||
switch result.outcome {
|
||||
case sourceLeaseAcquired:
|
||||
acquired++
|
||||
if result.snapshot == nil || result.snapshot.LeaseToken == "" {
|
||||
t.Errorf("acquireSourceLease(concurrent acquired) snapshot = %+v, want token-bearing snapshot", result.snapshot)
|
||||
}
|
||||
case sourceLeaseBusy:
|
||||
busy++
|
||||
if result.snapshot != nil {
|
||||
t.Errorf("acquireSourceLease(concurrent busy) snapshot = %+v, want nil", result.snapshot)
|
||||
}
|
||||
default:
|
||||
t.Errorf("acquireSourceLease(concurrent) outcome = %q, want acquired or busy", result.outcome)
|
||||
}
|
||||
}
|
||||
if acquired != 1 || busy != 1 {
|
||||
t.Errorf("concurrent lease outcomes = acquired:%d busy:%d, want 1 and 1", acquired, busy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireSourceLeaseMutualExclusionExpiryAndTerminalOwnership(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "lease-cas")
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip",
|
||||
false,
|
||||
)
|
||||
|
||||
first, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
if err != nil {
|
||||
t.Fatalf("acquireSourceLease(first) error = %v, want nil", err)
|
||||
}
|
||||
if got, want := outcome, sourceLeaseAcquired; got != want {
|
||||
t.Fatalf("acquireSourceLease(first) outcome = %q, want %q", got, want)
|
||||
}
|
||||
if first == nil || first.LeaseToken == "" {
|
||||
t.Fatalf("acquireSourceLease(first) snapshot = %+v, want token-bearing snapshot", first)
|
||||
}
|
||||
|
||||
second, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
if err != nil {
|
||||
t.Fatalf("acquireSourceLease(duplicate) error = %v, want nil", err)
|
||||
}
|
||||
if got, want := outcome, sourceLeaseBusy; got != want {
|
||||
t.Errorf("acquireSourceLease(duplicate) outcome = %q, want %q", got, want)
|
||||
}
|
||||
if second != nil {
|
||||
t.Errorf("acquireSourceLease(duplicate) snapshot = %+v, want nil", second)
|
||||
}
|
||||
|
||||
past := time.Now().Add(-time.Second)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", source.ID).
|
||||
Update("lease_expires_at", &past).Error; err != nil {
|
||||
t.Fatalf("expire first lease error = %v, want nil", err)
|
||||
}
|
||||
takeover, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
if err != nil {
|
||||
t.Fatalf("acquireSourceLease(takeover) error = %v, want nil", err)
|
||||
}
|
||||
if got, want := outcome, sourceLeaseAcquired; got != want {
|
||||
t.Fatalf("acquireSourceLease(takeover) outcome = %q, want %q", got, want)
|
||||
}
|
||||
if takeover == nil {
|
||||
t.Fatal("acquireSourceLease(takeover) snapshot = nil, want non-nil")
|
||||
}
|
||||
if takeover.LeaseToken == "" || takeover.LeaseToken == first.LeaseToken {
|
||||
t.Fatalf("takeover LeaseToken = %q, want non-empty token distinct from %q", takeover.LeaseToken, first.LeaseToken)
|
||||
}
|
||||
|
||||
renewed, err := renewSourceLease(ctx, first, pagesSourceSyncLeaseDuration)
|
||||
if err != nil {
|
||||
t.Fatalf("renewSourceLease(expired owner) error = %v, want nil", err)
|
||||
}
|
||||
if renewed {
|
||||
t.Error("renewSourceLease(expired owner) = true, want false")
|
||||
}
|
||||
if err := failSourceLease(ctx, first, "stale worker must not win"); err != nil {
|
||||
t.Fatalf("failSourceLease(expired owner) error = %v, want nil", err)
|
||||
}
|
||||
var runtime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
|
||||
t.Fatalf("load runtime after takeover error = %v, want nil", err)
|
||||
}
|
||||
if got, want := runtime.LeaseToken, takeover.LeaseToken; got != want {
|
||||
t.Errorf("runtime LeaseToken after stale terminal write = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := runtime.SyncStatus, pagesSourceStatusSyncing; got != want {
|
||||
t.Errorf("runtime SyncStatus after stale terminal write = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
renewed, err = renewSourceLease(ctx, takeover, pagesSourceSyncLeaseDuration)
|
||||
if err != nil {
|
||||
t.Fatalf("renewSourceLease(current owner) error = %v, want nil", err)
|
||||
}
|
||||
if !renewed {
|
||||
t.Error("renewSourceLease(current owner) = false, want true")
|
||||
}
|
||||
if err := failSourceLease(ctx, takeover, errPagesSourceSyncFailed); err != nil {
|
||||
t.Fatalf("failSourceLease(current owner) error = %v, want nil", err)
|
||||
}
|
||||
var failedRuntime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&failedRuntime).Error; err != nil {
|
||||
t.Fatalf("load failed runtime error = %v, want nil", err)
|
||||
}
|
||||
if got, want := failedRuntime.SyncStatus, pagesSourceStatusFailed; got != want {
|
||||
t.Errorf("failed runtime SyncStatus = %q, want %q", got, want)
|
||||
}
|
||||
if failedRuntime.LeaseToken != "" || failedRuntime.LeaseExpiresAt != nil {
|
||||
t.Errorf("failed runtime lease = (%q, %v), want cleared", failedRuntime.LeaseToken, failedRuntime.LeaseExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceConfigAndProjectContentChangesFenceLease(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "lease-fence")
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip?token=first",
|
||||
false,
|
||||
)
|
||||
|
||||
configSnapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
if err != nil || outcome != sourceLeaseAcquired {
|
||||
t.Fatalf("acquireSourceLease(config fence) = (%+v, %q, %v), want acquired", configSnapshot, outcome, err)
|
||||
}
|
||||
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: "https://example.com/site.zip?token=second",
|
||||
AllowInsecure: false,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateSource(config fence) error = %v, want nil", err)
|
||||
}
|
||||
renewed, err := renewSourceLease(ctx, configSnapshot, pagesSourceSyncLeaseDuration)
|
||||
if err != nil {
|
||||
t.Fatalf("renewSourceLease(after source update) error = %v, want nil", err)
|
||||
}
|
||||
if renewed {
|
||||
t.Error("renewSourceLease(after source update) = true, want false")
|
||||
}
|
||||
var updatedSource model.PagesProjectSource
|
||||
if err := db.DB(ctx).Where("id = ?", source.ID).First(&updatedSource).Error; err != nil {
|
||||
t.Fatalf("load updated source error = %v, want nil", err)
|
||||
}
|
||||
if got, want := updatedSource.ConfigVersion, source.ConfigVersion+1; got != want {
|
||||
t.Errorf("updated source ConfigVersion = %d, want %d", got, want)
|
||||
}
|
||||
if snapshot, staleOutcome, staleErr := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync); staleErr != nil || staleOutcome != sourceLeaseStale || snapshot != nil {
|
||||
t.Errorf("acquireSourceLease(old config) = (%+v, %q, %v), want (nil, %q, nil)", snapshot, staleOutcome, staleErr, sourceLeaseStale)
|
||||
}
|
||||
|
||||
contentSnapshot, outcome, err := acquireSourceLease(ctx, source.ID, updatedSource.ConfigVersion, sourceActionSync)
|
||||
if err != nil || outcome != sourceLeaseAcquired {
|
||||
t.Fatalf("acquireSourceLease(content fence) = (%+v, %q, %v), want acquired", contentSnapshot, outcome, err)
|
||||
}
|
||||
if _, err := UpdateProject(ctx, project.ID, Input{
|
||||
Name: project.Name,
|
||||
Slug: project.Slug,
|
||||
Enabled: true,
|
||||
RootDir: "dist",
|
||||
EntryFile: "index.html",
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateProject(content fence) error = %v, want nil", err)
|
||||
}
|
||||
renewed, err = renewSourceLease(ctx, contentSnapshot, pagesSourceSyncLeaseDuration)
|
||||
if err != nil {
|
||||
t.Fatalf("renewSourceLease(after content update) error = %v, want nil", err)
|
||||
}
|
||||
if renewed {
|
||||
t.Error("renewSourceLease(after content update) = true, want false")
|
||||
}
|
||||
storedProject, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if got, want := storedProject.ContentConfigVersion, project.ContentConfigVersion+1; got != want {
|
||||
t.Errorf("ContentConfigVersion after RootDir update = %d, want %d", got, want)
|
||||
}
|
||||
var runtime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
|
||||
t.Fatalf("load fenced runtime error = %v, want nil", err)
|
||||
}
|
||||
if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil {
|
||||
t.Errorf("content-fenced runtime lease = (%q, %v), want cleared", runtime.LeaseToken, runtime.LeaseExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceRuntimeUsesOnlySixDocumentedStates(t *testing.T) {
|
||||
states := []string{
|
||||
pagesSourceStatusIdle,
|
||||
pagesSourceStatusChecking,
|
||||
pagesSourceStatusUpdateAvailable,
|
||||
pagesSourceStatusSyncing,
|
||||
pagesSourceStatusFailed,
|
||||
pagesSourceStatusAttention,
|
||||
}
|
||||
seen := make(map[string]struct{}, len(states))
|
||||
for _, state := range states {
|
||||
if strings.TrimSpace(state) == "" {
|
||||
t.Errorf("documented source state = %q, want non-empty", state)
|
||||
}
|
||||
if _, exists := seen[state]; exists {
|
||||
t.Errorf("documented source state %q is duplicated", state)
|
||||
}
|
||||
seen[state] = struct{}{}
|
||||
}
|
||||
if got, want := len(seen), 6; got != want {
|
||||
t.Errorf("unique source states = %d, want %d", got, want)
|
||||
}
|
||||
|
||||
updateRuntime := &model.PagesProjectSourceRuntime{
|
||||
LastSeenRevision: strings.Repeat("a", 64),
|
||||
LastAppliedRevision: strings.Repeat("b", 64),
|
||||
LastSeenDetail: `{"release_id":"new"}`,
|
||||
LastAppliedDetail: `{"release_id":"old"}`,
|
||||
}
|
||||
if got, want := normalizedSourceRuntimeStatus(updateRuntime), pagesSourceStatusUpdateAvailable; got != want {
|
||||
t.Errorf("normalizedSourceRuntimeStatus(update) = %q, want %q", got, want)
|
||||
}
|
||||
attentionRuntime := &model.PagesProjectSourceRuntime{
|
||||
LastSeenRevision: strings.Repeat("a", 64),
|
||||
LastAppliedRevision: strings.Repeat("b", 64),
|
||||
LastSeenDetail: `{"release_id":"same"}`,
|
||||
LastAppliedDetail: `{"release_id":"same"}`,
|
||||
}
|
||||
if got, want := normalizedSourceRuntimeStatus(attentionRuntime), pagesSourceStatusAttention; got != want {
|
||||
t.Errorf("normalizedSourceRuntimeStatus(attention) = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := normalizedSourceRuntimeStatus(&model.PagesProjectSourceRuntime{}), pagesSourceStatusIdle; got != want {
|
||||
t.Errorf("normalizedSourceRuntimeStatus(idle) = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/openflare/plugins/server/kernel/task"
|
||||
"Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
// PagesSourceScanTask is the private Asynq task type for the periodic scanner.
|
||||
PagesSourceScanTask = "openflare:pages_source_scan"
|
||||
// TaskTypePagesSourceScan is the internal task meta type seeded in w_schedules.
|
||||
TaskTypePagesSourceScan = "of_pages_source_scan"
|
||||
|
||||
pagesSourceScanBatchSize = 20
|
||||
)
|
||||
|
||||
// PagesSourceScanMeta describes the periodic Pages source scanner schedule.
|
||||
var PagesSourceScanMeta = task.TaskMeta{
|
||||
Type: TaskTypePagesSourceScan,
|
||||
AsynqTask: PagesSourceScanTask,
|
||||
Name: "OpenFlare Pages 部署源扫描",
|
||||
Description: "补偿孤儿部署包、恢复过期执行权并串行检查到期的 GitHub latest 部署源",
|
||||
SupportsTime: false,
|
||||
MaxRetry: 0,
|
||||
Queue: task.QueueDefault,
|
||||
Retryable: false,
|
||||
}
|
||||
|
||||
type pagesSourceScanPayload struct{}
|
||||
|
||||
type pagesSourceScanSummary struct {
|
||||
ExpiredCandidates int `json:"expired_candidates"`
|
||||
RecoveredLeases int `json:"recovered_leases"`
|
||||
OrphanCleanup PagesOrphanCleanupSummary `json:"orphan_cleanup"`
|
||||
DueSources int `json:"due_sources"`
|
||||
SelectedSources int `json:"selected_sources"`
|
||||
CheckedSources int `json:"checked_sources"`
|
||||
UpdatesFound int `json:"updates_found"`
|
||||
AttentionSources int `json:"attention_sources"`
|
||||
DispatchedSyncs int `json:"dispatched_syncs"`
|
||||
FailedDispatches int `json:"failed_dispatches"`
|
||||
BusySources int `json:"busy_sources"`
|
||||
StaleSources int `json:"stale_sources"`
|
||||
FailedSources int `json:"failed_sources"`
|
||||
Backlog int `json:"backlog"`
|
||||
ProviderBackoffs []pagesSourceProviderBackoff `json:"provider_backoffs,omitempty"`
|
||||
}
|
||||
|
||||
type pagesSourceProviderBackoff struct {
|
||||
SourceID uint `json:"source_id"`
|
||||
StatusCode int `json:"status_code"`
|
||||
RetryAt string `json:"retry_at"`
|
||||
}
|
||||
|
||||
var (
|
||||
pagesSourceScanNow = time.Now
|
||||
reconcilePagesSourceOrphans = ReconcilePagesOrphanUploads
|
||||
dispatchPagesSourceAutoSync = func(
|
||||
ctx context.Context,
|
||||
source model.PagesProjectSource,
|
||||
targetRevision string,
|
||||
) (*SourceActionReceipt, error) {
|
||||
return dispatchSourceActionSnapshotWithTrigger(
|
||||
ctx,
|
||||
source,
|
||||
sourceActionSync,
|
||||
pagesSourceCreatedBySystem,
|
||||
pagesSourceTriggerScheduledAutoUpdate,
|
||||
targetRevision,
|
||||
"",
|
||||
"system",
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
// SourceScanHandler serializes provider checks inside one scheduled task. A
|
||||
// source-level lease still permits overlapping scanner executions safely.
|
||||
type SourceScanHandler struct{}
|
||||
|
||||
// ValidatePayload accepts only an empty object; the scanner has no user input.
|
||||
func (handler *SourceScanHandler) ValidatePayload(payload []byte) ([]byte, error) {
|
||||
if len(bytes.TrimSpace(payload)) == 0 {
|
||||
payload = []byte("{}")
|
||||
}
|
||||
var input pagesSourceScanPayload
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&input); err != nil {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
if err := ensureJSONEOF(decoder); err != nil {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
|
||||
// Execute recovers expired leases and checks at most 20 due latest sources in
|
||||
// stable order. Provider and dispatch failures are isolated per source.
|
||||
func (handler *SourceScanHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
|
||||
if _, err := handler.ValidatePayload(payload); err != nil {
|
||||
return nil, task.PermanentError(errPagesSourceActionInvalid)
|
||||
}
|
||||
|
||||
now := pagesSourceScanNow()
|
||||
summary := pagesSourceScanSummary{}
|
||||
if err := recoverExpiredPagesSourceLeases(ctx, now, &summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
orphanSummary, err := reconcilePagesSourceOrphans(ctx, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
summary.OrphanCleanup = orphanSummary
|
||||
task.AppendLog(
|
||||
ctx,
|
||||
"[cleanup] orphan 候选=%d，已补偿=%d，仍被引用=%d，lease busy=%d，非法 marker=%d，跳过=%d，失败=%d",
|
||||
orphanSummary.Candidates,
|
||||
orphanSummary.Reconciled,
|
||||
orphanSummary.Referenced,
|
||||
orphanSummary.LeaseBusy,
|
||||
orphanSummary.InvalidMarker,
|
||||
orphanSummary.Skipped,
|
||||
orphanSummary.Failed,
|
||||
)
|
||||
if err := scanDueGitHubSources(ctx, now, &summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
detail, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
message := fmt.Sprintf(
|
||||
"Pages 部署源扫描完成：恢复 %d 个租约，补偿 %d 个孤儿记录，检查 %d 个来源，投递 %d 个自动更新，积压 %d 个",
|
||||
summary.RecoveredLeases,
|
||||
summary.OrphanCleanup.Reconciled,
|
||||
summary.CheckedSources,
|
||||
summary.DispatchedSyncs,
|
||||
summary.Backlog,
|
||||
)
|
||||
return &task.TaskResult{Message: message, Detail: string(detail)}, nil
|
||||
}
|
||||
|
||||
func recoverExpiredPagesSourceLeases(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
summary *pagesSourceScanSummary,
|
||||
) error {
|
||||
candidates, err := repository.ListExpiredPagesSourceLeaseCandidates(
|
||||
ctx,
|
||||
now,
|
||||
[]string{pagesSourceStatusChecking, pagesSourceStatusSyncing},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summary.ExpiredCandidates = len(candidates)
|
||||
for _, candidate := range candidates {
|
||||
var nextCheckAt *time.Time
|
||||
if candidate.SourceType == PagesSourceTypeGitHubRelease &&
|
||||
candidate.ReleaseSelector == githubReleaseSelectorLatest {
|
||||
next := nextGitHubCheckAt(now, candidate.SourceID, minimumCheckInterval)
|
||||
nextCheckAt = &next
|
||||
}
|
||||
recovered, recoverErr := recoverExpiredSourceLease(
|
||||
ctx,
|
||||
candidate.SourceID,
|
||||
candidate.LeaseToken,
|
||||
candidate.LeaseExpiresAt,
|
||||
candidate.SyncStatus,
|
||||
now,
|
||||
nextCheckAt,
|
||||
)
|
||||
if recoverErr != nil {
|
||||
summary.FailedSources++
|
||||
logger.WarnF(
|
||||
ctx,
|
||||
"[PagesSourceScan] recover expired lease failed: source_id=%d error=%v",
|
||||
candidate.SourceID,
|
||||
recoverErr,
|
||||
)
|
||||
continue
|
||||
}
|
||||
if recovered {
|
||||
summary.RecoveredLeases++
|
||||
task.AppendLog(ctx, "[recover] 已恢复过期来源租约：source_id=%d", candidate.SourceID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanDueGitHubSources(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
summary *pagesSourceScanSummary,
|
||||
) error {
|
||||
dueCount, err := repository.CountDueGitHubPagesSourceChecks(
|
||||
ctx, now, PagesSourceTypeGitHubRelease, githubReleaseSelectorLatest,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summary.DueSources = int(dueCount)
|
||||
|
||||
candidates, err := repository.ListDueGitHubPagesSourceChecks(
|
||||
ctx, now, PagesSourceTypeGitHubRelease, githubReleaseSelectorLatest, pagesSourceScanBatchSize,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summary.SelectedSources = len(candidates)
|
||||
task.AppendLog(
|
||||
ctx,
|
||||
"[scan] 到期来源=%d，本批=%d",
|
||||
summary.DueSources,
|
||||
summary.SelectedSources,
|
||||
)
|
||||
|
||||
for _, candidate := range candidates {
|
||||
scanOneDueGitHubSource(ctx, candidate, summary)
|
||||
}
|
||||
remainingDue, err := repository.CountDueGitHubPagesSourceChecks(
|
||||
ctx, now, PagesSourceTypeGitHubRelease, githubReleaseSelectorLatest,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summary.Backlog = int(remainingDue)
|
||||
task.AppendLog(ctx, "[scan] 本批处理后仍到期来源=%d", summary.Backlog)
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanOneDueGitHubSource(
|
||||
ctx context.Context,
|
||||
candidate model.PagesDueGitHubSourceCandidate,
|
||||
summary *pagesSourceScanSummary,
|
||||
) {
|
||||
snapshot, outcome, err := acquireSourceLease(
|
||||
ctx,
|
||||
candidate.SourceID,
|
||||
candidate.ConfigVersion,
|
||||
sourceActionCheck,
|
||||
)
|
||||
if err != nil {
|
||||
summary.FailedSources++
|
||||
logger.WarnF(ctx, "[PagesSourceScan] acquire check lease failed: source_id=%d error=%v", candidate.SourceID, err)
|
||||
return
|
||||
}
|
||||
switch outcome {
|
||||
case sourceLeaseBusy:
|
||||
summary.BusySources++
|
||||
task.AppendLog(ctx, "[check] 来源正在执行其它任务，跳过：source_id=%d", candidate.SourceID)
|
||||
return
|
||||
case sourceLeaseStale:
|
||||
summary.StaleSources++
|
||||
return
|
||||
case sourceLeaseAcquired:
|
||||
// 获取执行权成功，继续执行扫描。
|
||||
}
|
||||
if snapshot == nil || snapshot.SourceType != PagesSourceTypeGitHubRelease ||
|
||||
snapshot.ReleaseSelector != githubReleaseSelectorLatest {
|
||||
summary.StaleSources++
|
||||
if snapshot != nil {
|
||||
if finalizeErr := failSourceLease(ctx, snapshot, errPagesSourceActionStale); finalizeErr != nil {
|
||||
logger.WarnF(
|
||||
ctx,
|
||||
"[PagesSourceScan] finalize stale source failed: source_id=%d error=%v",
|
||||
snapshot.SourceID,
|
||||
finalizeErr,
|
||||
)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
checkResult, checkErr := checkGitHubSource(ctx, snapshot)
|
||||
if checkErr != nil {
|
||||
summary.FailedSources++
|
||||
recordPagesSourceProviderBackoff(ctx, candidate.SourceID, checkErr, summary)
|
||||
logger.WarnF(
|
||||
ctx,
|
||||
"[PagesSourceScan] source check failed: source_id=%d error=%s",
|
||||
candidate.SourceID,
|
||||
safeGitHubSourceError(checkErr),
|
||||
)
|
||||
return
|
||||
}
|
||||
if checkResult == nil || checkResult.Stale {
|
||||
summary.StaleSources++
|
||||
return
|
||||
}
|
||||
handleCheckedGitHubSource(ctx, snapshot, checkResult, summary)
|
||||
}
|
||||
|
||||
func recordPagesSourceProviderBackoff(
|
||||
ctx context.Context,
|
||||
sourceID uint,
|
||||
checkErr error,
|
||||
summary *pagesSourceScanSummary,
|
||||
) {
|
||||
var domainError *githubSourceProviderDomainError
|
||||
if !errors.As(checkErr, &domainError) ||
|
||||
(domainError.statusCode != 403 && domainError.statusCode != 429) {
|
||||
return
|
||||
}
|
||||
|
||||
retryAt := domainError.retryAt
|
||||
runtime, err := repository.GetPagesProjectSourceRuntimeBySourceID(ctx, sourceID)
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[PagesSourceScan] load provider backoff deadline failed: source_id=%d error=%v", sourceID, err)
|
||||
} else if runtime.NextCheckAt != nil {
|
||||
retryAt = runtime.NextCheckAt
|
||||
}
|
||||
|
||||
retryAtText := "unknown"
|
||||
if retryAt != nil {
|
||||
retryAtText = retryAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
summary.ProviderBackoffs = append(summary.ProviderBackoffs, pagesSourceProviderBackoff{
|
||||
SourceID: sourceID, StatusCode: domainError.statusCode, RetryAt: retryAtText,
|
||||
})
|
||||
task.AppendLog(
|
||||
ctx,
|
||||
"[check] GitHub provider 退避：source_id=%d status=%d retry_at=%s",
|
||||
sourceID,
|
||||
domainError.statusCode,
|
||||
retryAtText,
|
||||
)
|
||||
}
|
||||
|
||||
func handleCheckedGitHubSource(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
checkResult *githubCheckTaskResult,
|
||||
summary *pagesSourceScanSummary,
|
||||
) {
|
||||
summary.CheckedSources++
|
||||
switch checkResult.Status {
|
||||
case pagesSourceStatusUpdateAvailable:
|
||||
summary.UpdatesFound++
|
||||
case pagesSourceStatusAttention:
|
||||
summary.AttentionSources++
|
||||
}
|
||||
if !snapshot.AutoUpdateEnabled || checkResult.Status != pagesSourceStatusUpdateAvailable ||
|
||||
!validOptionalSourceRevision(checkResult.Revision) || checkResult.Revision == "" {
|
||||
return
|
||||
}
|
||||
|
||||
source := model.PagesProjectSource{
|
||||
ID: snapshot.SourceID,
|
||||
ProjectID: snapshot.ProjectID,
|
||||
ConfigVersion: snapshot.SourceConfigVersion,
|
||||
}
|
||||
receipt, dispatchErr := dispatchPagesSourceAutoSync(ctx, source, checkResult.Revision)
|
||||
if dispatchErr == nil {
|
||||
summary.DispatchedSyncs++
|
||||
if receipt != nil {
|
||||
task.AppendLog(
|
||||
ctx,
|
||||
"[dispatch] 已投递自动更新：source_id=%d execution_id=%s revision=%s",
|
||||
snapshot.SourceID,
|
||||
receipt.ExecutionID,
|
||||
checkResult.Revision,
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
summary.FailedSources++
|
||||
summary.FailedDispatches++
|
||||
logger.WarnF(
|
||||
ctx,
|
||||
"[PagesSourceScan] dispatch auto sync failed: source_id=%d revision=%s error=%v",
|
||||
snapshot.SourceID,
|
||||
checkResult.Revision,
|
||||
dispatchErr,
|
||||
)
|
||||
updated, recordErr := recordPagesSourceAutoDispatchFailure(
|
||||
ctx,
|
||||
snapshot,
|
||||
checkResult.Revision,
|
||||
checkResult.RetryAt,
|
||||
)
|
||||
if recordErr != nil {
|
||||
logger.WarnF(
|
||||
ctx,
|
||||
"[PagesSourceScan] record auto sync dispatch failure failed: source_id=%d error=%v",
|
||||
snapshot.SourceID,
|
||||
recordErr,
|
||||
)
|
||||
} else if !updated {
|
||||
summary.StaleSources++
|
||||
}
|
||||
}
|
||||
|
||||
func recordPagesSourceAutoDispatchFailure(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
revision string,
|
||||
retryAt *time.Time,
|
||||
) (bool, error) {
|
||||
if snapshot == nil || revision == "" {
|
||||
return false, nil
|
||||
}
|
||||
now := pagesSourceScanNow()
|
||||
next := nextGitHubCheckAt(now, snapshot.SourceID, minimumCheckInterval)
|
||||
if retryAt != nil && retryAt.After(next) {
|
||||
next = retryAt.In(now.Location())
|
||||
}
|
||||
rows, err := repository.RecordPagesSourceAutoDispatchFailure(
|
||||
ctx,
|
||||
snapshot.SourceID,
|
||||
snapshot.SourceConfigVersion,
|
||||
PagesSourceTypeGitHubRelease,
|
||||
githubReleaseSelectorLatest,
|
||||
revision,
|
||||
pagesSourceStatusUpdateAvailable,
|
||||
now,
|
||||
map[string]any{
|
||||
sourceRuntimeColumnSyncStatus: pagesSourceStatusUpdateAvailable,
|
||||
sourceRuntimeColumnLastError: errPagesSourceTaskDispatchFailed,
|
||||
sourceRuntimeColumnNextCheckAt: &next,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return rows == 1, nil
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/githubrelease"
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type scannerDispatchedSync struct {
|
||||
SourceID uint
|
||||
Revision string
|
||||
}
|
||||
|
||||
func TestGitHubLatestAutoConfigPreservesIdentityAndRuntimeCursor(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "scanner-auto-config")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/scanner/auto-config",
|
||||
AutoUpdateEnabled: false,
|
||||
CheckIntervalMinutes: 60,
|
||||
})
|
||||
identity := source.SourceIdentity
|
||||
seenRevision := strings.Repeat("a", sourceRevisionHexLength)
|
||||
appliedRevision := strings.Repeat("b", sourceRevisionHexLength)
|
||||
future := time.Now().Add(time.Hour)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", source.ID).
|
||||
Updates(map[string]any{
|
||||
"etag": `"cursor-etag"`,
|
||||
"last_seen_revision": seenRevision,
|
||||
"last_seen_detail": `{"provider":"github","release_id":"2","asset_id":"2","tag":"v2","asset_name":"dist.zip"}`,
|
||||
"last_applied_revision": appliedRevision,
|
||||
"last_applied_detail": `{"provider":"github","release_id":"1","asset_id":"1","tag":"v1","asset_name":"dist.zip"}`,
|
||||
"sync_status": pagesSourceStatusSyncing,
|
||||
"last_error": "old error",
|
||||
"lease_token": "in-flight",
|
||||
"lease_expires_at": &future,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed runtime error = %v, want nil", err)
|
||||
}
|
||||
|
||||
input := SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/scanner/auto-config",
|
||||
AutoUpdateEnabled: true,
|
||||
CheckIntervalMinutes: 15,
|
||||
}
|
||||
if err := validateGitHubSourceInput(input); err != nil {
|
||||
t.Fatalf("validateGitHubSourceInput(auto latest) error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
changed, err := updateGitHubSourceTx(tx, project.ID, input)
|
||||
if err == nil && !changed {
|
||||
return errors.New("auto config update was treated as no-op")
|
||||
}
|
||||
return err
|
||||
}); err != nil {
|
||||
t.Fatalf("updateGitHubSourceTx(auto latest) error = %v, want nil", err)
|
||||
}
|
||||
|
||||
updated, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
if updated.SourceIdentity != identity || updated.ConfigVersion != source.ConfigVersion+1 {
|
||||
t.Errorf(
|
||||
"updated source = identity:%q version:%d, want identity:%q version:%d",
|
||||
updated.SourceIdentity,
|
||||
updated.ConfigVersion,
|
||||
identity,
|
||||
source.ConfigVersion+1,
|
||||
)
|
||||
}
|
||||
if !updated.AutoUpdateEnabled || updated.CheckIntervalMinutes != 15 {
|
||||
t.Errorf("updated auto config = enabled:%t interval:%d, want true/15", updated.AutoUpdateEnabled, updated.CheckIntervalMinutes)
|
||||
}
|
||||
if runtime.ETag != `"cursor-etag"` || runtime.LastSeenRevision != seenRevision ||
|
||||
runtime.LastAppliedRevision != appliedRevision {
|
||||
t.Errorf("runtime cursor changed after auto-only update: %+v", runtime)
|
||||
}
|
||||
if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil || runtime.LastError != "" ||
|
||||
runtime.SyncStatus != pagesSourceStatusUpdateAvailable || runtime.NextCheckAt == nil {
|
||||
t.Errorf(
|
||||
"runtime fence = token:%q expiry:%v error:%q status:%q next:%v",
|
||||
runtime.LeaseToken,
|
||||
runtime.LeaseExpiresAt,
|
||||
runtime.LastError,
|
||||
runtime.SyncStatus,
|
||||
runtime.NextCheckAt,
|
||||
)
|
||||
}
|
||||
|
||||
tagConfig, err := buildGitHubSourceConfig(SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/scanner/auto-config",
|
||||
ReleaseSelector: githubReleaseSelectorTag,
|
||||
ReleaseTag: "v1",
|
||||
AutoUpdateEnabled: true,
|
||||
CheckIntervalMinutes: 60,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("buildGitHubSourceConfig(tag) error = %v, want nil", err)
|
||||
}
|
||||
if tagConfig.AutoUpdate || tagConfig.CheckInterval != 0 {
|
||||
t.Errorf("tag config auto/interval = %t/%d, want false/0", tagConfig.AutoUpdate, tagConfig.CheckInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverExpiredPagesSourceLeaseUsesExactCASAndStableJitter(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "scanner-expired-lease")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/scanner/expired-lease",
|
||||
})
|
||||
now := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC)
|
||||
usePagesSourceScannerClock(t, now)
|
||||
expiredAt := now.Add(-time.Minute)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", source.ID).
|
||||
Updates(map[string]any{
|
||||
"sync_status": pagesSourceStatusChecking,
|
||||
"lease_token": "expired-owner",
|
||||
"lease_expires_at": &expiredAt,
|
||||
"next_check_at": &expiredAt,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed expired lease error = %v, want nil", err)
|
||||
}
|
||||
|
||||
summary := pagesSourceScanSummary{}
|
||||
if err := recoverExpiredPagesSourceLeases(ctx, now, &summary); err != nil {
|
||||
t.Fatalf("recoverExpiredPagesSourceLeases() error = %v, want nil", err)
|
||||
}
|
||||
if summary.ExpiredCandidates != 1 || summary.RecoveredLeases != 1 || summary.FailedSources != 0 {
|
||||
t.Errorf("recovery summary = %+v, want one recovered lease", summary)
|
||||
}
|
||||
_, runtime := mustLoadPagesSource(t, ctx, project.ID)
|
||||
wantNext := nextGitHubCheckAt(now, source.ID, minimumCheckInterval)
|
||||
if runtime.SyncStatus != pagesSourceStatusFailed || runtime.LastError != errPagesSourceLeaseExpired ||
|
||||
runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil || runtime.NextCheckAt == nil ||
|
||||
runtime.NextCheckAt.Sub(wantNext) != 0 {
|
||||
t.Errorf(
|
||||
"recovered runtime = status:%q error:%q token:%q expiry:%v next:%v, want next %v",
|
||||
runtime.SyncStatus,
|
||||
runtime.LastError,
|
||||
runtime.LeaseToken,
|
||||
runtime.LeaseExpiresAt,
|
||||
runtime.NextCheckAt,
|
||||
wantNext,
|
||||
)
|
||||
}
|
||||
|
||||
renewedExpiry := now.Add(time.Minute)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", source.ID).
|
||||
Updates(map[string]any{
|
||||
"sync_status": pagesSourceStatusSyncing,
|
||||
"lease_token": "renewed-owner",
|
||||
"lease_expires_at": &renewedExpiry,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed renewed lease error = %v, want nil", err)
|
||||
}
|
||||
recovered, err := recoverExpiredSourceLease(
|
||||
ctx,
|
||||
source.ID,
|
||||
"renewed-owner",
|
||||
expiredAt,
|
||||
pagesSourceStatusSyncing,
|
||||
now,
|
||||
&wantNext,
|
||||
)
|
||||
if err != nil || recovered {
|
||||
t.Fatalf("recoverExpiredSourceLease(stale expiry) = %t, %v; want false, nil", recovered, err)
|
||||
}
|
||||
_, runtime = mustLoadPagesSource(t, ctx, project.ID)
|
||||
if runtime.LeaseToken != "renewed-owner" || runtime.LeaseExpiresAt == nil ||
|
||||
runtime.LeaseExpiresAt.Sub(renewedExpiry) != 0 || runtime.SyncStatus != pagesSourceStatusSyncing {
|
||||
t.Errorf("stale recovery overwrote renewed lease: %+v", runtime)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledAutoSyncPersistsExplicitDeploymentTrigger(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "scanner-scheduled-trigger")
|
||||
source, _ := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/scanner/scheduled-trigger",
|
||||
AutoUpdateEnabled: true,
|
||||
})
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "scheduled-v1"})
|
||||
packageHash := sha256.Sum256(packageBytes)
|
||||
release := githubrelease.Release{ID: "scheduled-release", Tag: "v1"}
|
||||
asset := githubrelease.Asset{
|
||||
ID: "scheduled-asset", Name: defaultGitHubAssetName, State: "uploaded",
|
||||
UpdatedAt: time.Date(2026, 7, 19, 12, 30, 0, 0, time.UTC),
|
||||
}
|
||||
target, err := buildGitHubSourceTarget(release, asset, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("buildGitHubSourceTarget() error = %v, want nil", err)
|
||||
}
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(context.Context, githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
return githubrelease.ResolveResult{Release: release, Asset: asset}, nil
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
path := filepath.Join(t.TempDir(), "scheduled.zip")
|
||||
if err := os.WriteFile(path, packageBytes, 0o600); err != nil {
|
||||
t.Fatalf("os.WriteFile(scheduled package) error = %v", err)
|
||||
}
|
||||
return &githubrelease.DownloadResult{
|
||||
Path: path, Size: int64(len(packageBytes)), SHA256: hex.EncodeToString(packageHash[:]),
|
||||
}, nil
|
||||
},
|
||||
})
|
||||
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
if err != nil || outcome != sourceLeaseAcquired {
|
||||
t.Fatalf("acquireSourceLease() = %+v, %q, %v; want acquired", snapshot, outcome, err)
|
||||
}
|
||||
synced, err := syncGitHubSourceWithTrigger(
|
||||
ctx,
|
||||
snapshot,
|
||||
pagesSourceCreatedBySystem,
|
||||
target.Revision,
|
||||
"",
|
||||
pagesSourceTriggerScheduledAutoUpdate,
|
||||
)
|
||||
if err != nil || synced == nil || synced.Deployment == nil || synced.Stale {
|
||||
t.Fatalf("syncGitHubSourceWithTrigger() = %+v, %v; want active deployment", synced, err)
|
||||
}
|
||||
deployment, err := repository.GetPagesDeploymentByID(ctx, synced.Deployment.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesDeploymentByID(%d) error = %v", synced.Deployment.ID, err)
|
||||
}
|
||||
if deployment.TriggerType != pagesSourceTriggerScheduledAutoUpdate ||
|
||||
deployment.CreatedBy != pagesSourceCreatedBySystem {
|
||||
t.Errorf(
|
||||
"scheduled provenance = trigger:%q actor:%q, want %q/%q",
|
||||
deployment.TriggerType,
|
||||
deployment.CreatedBy,
|
||||
pagesSourceTriggerScheduledAutoUpdate,
|
||||
pagesSourceCreatedBySystem,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagesSourceScannerSerialBatchIsolation304AndActualBacklog(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
now := time.Now().Truncate(time.Second)
|
||||
usePagesSourceScannerClock(t, now)
|
||||
dueAt := now.Add(-time.Hour)
|
||||
|
||||
type fixture struct {
|
||||
source *model.PagesProjectSource
|
||||
runtime *model.PagesProjectSourceRuntime
|
||||
repository string
|
||||
}
|
||||
fixtures := make([]fixture, 0, 22)
|
||||
byRepository := make(map[string]int, 22)
|
||||
for index := 1; index <= 22; index++ {
|
||||
project := mustCreatePagesSourceProject(t, ctx, fmt.Sprintf("scanner-batch-%02d", index))
|
||||
repository := fmt.Sprintf("scanner/source-%02d", index)
|
||||
source, runtime := mustConfigureGitHubSourceWithoutDispatch(t, ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeGitHubRelease,
|
||||
RepositoryURL: "https://github.com/" + repository,
|
||||
AutoUpdateEnabled: index != 4,
|
||||
CheckIntervalMinutes: 60,
|
||||
})
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", source.ID).
|
||||
Update("next_check_at", &dueAt).Error; err != nil {
|
||||
t.Fatalf("mark source %d due error = %v, want nil", source.ID, err)
|
||||
}
|
||||
fixtures = append(fixtures, fixture{source: source, runtime: runtime, repository: repository})
|
||||
byRepository[repository] = index
|
||||
}
|
||||
|
||||
busyUntil := now.Add(time.Hour)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", fixtures[0].source.ID).
|
||||
Updates(map[string]any{
|
||||
"sync_status": pagesSourceStatusChecking,
|
||||
"lease_token": "busy-owner",
|
||||
"lease_expires_at": &busyUntil,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed busy source error = %v, want nil", err)
|
||||
}
|
||||
|
||||
stored304Revision := strings.Repeat("3", sourceRevisionHexLength)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", fixtures[2].source.ID).
|
||||
Updates(map[string]any{
|
||||
"etag": `"stored-etag"`,
|
||||
"last_seen_revision": stored304Revision,
|
||||
"last_seen_detail": `{"provider":"github","release_id":"release-3","asset_id":"3","tag":"v3","asset_name":"dist.zip"}`,
|
||||
"sync_status": pagesSourceStatusUpdateAvailable,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed 304 cursor error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", fixtures[4].source.ID).
|
||||
Updates(map[string]any{
|
||||
"last_applied_revision": strings.Repeat("a", sourceRevisionHexLength),
|
||||
"last_applied_detail": `{"provider":"github","release_id":"shared-release","asset_id":"old","tag":"v5","asset_name":"dist.zip"}`,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed replacement cursor error = %v, want nil", err)
|
||||
}
|
||||
|
||||
retryAt := now.Add(2 * time.Hour)
|
||||
calledRepositories := make([]string, 0, pagesSourceScanBatchSize)
|
||||
useFakeGitHubReleaseClient(t, &fakeGitHubReleaseClient{
|
||||
resolve: func(_ context.Context, request githubrelease.ResolveRequest) (githubrelease.ResolveResult, error) {
|
||||
index := byRepository[request.Repository]
|
||||
calledRepositories = append(calledRepositories, request.Repository)
|
||||
switch index {
|
||||
case 2:
|
||||
return githubrelease.ResolveResult{}, &githubrelease.Error{
|
||||
Kind: githubrelease.ErrMetadata, StatusCode: 429, RetryAt: &retryAt,
|
||||
}
|
||||
case 3:
|
||||
if request.ETag != `"stored-etag"` {
|
||||
t.Errorf("304 source ETag = %q, want stored ETag", request.ETag)
|
||||
}
|
||||
return githubrelease.ResolveResult{NotModified: true, ETag: request.ETag}, nil
|
||||
default:
|
||||
releaseID := fmt.Sprintf("release-%d", index)
|
||||
if index == 5 {
|
||||
releaseID = "shared-release"
|
||||
}
|
||||
result := githubrelease.ResolveResult{
|
||||
ETag: fmt.Sprintf(`"etag-%d"`, index),
|
||||
Release: githubrelease.Release{ID: releaseID, Tag: fmt.Sprintf("v%d", index)},
|
||||
Asset: githubrelease.Asset{
|
||||
ID: fmt.Sprintf("asset-%d", index),
|
||||
Name: defaultGitHubAssetName,
|
||||
State: "uploaded",
|
||||
UpdatedAt: now.Add(time.Duration(index) * time.Minute),
|
||||
},
|
||||
}
|
||||
if index == 6 {
|
||||
result.RetryAt = &retryAt
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
},
|
||||
download: func(context.Context, githubrelease.DownloadRequest) (*githubrelease.DownloadResult, error) {
|
||||
t.Fatal("scanner downloaded an asset; want check-only behavior")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
|
||||
dispatched := make([]scannerDispatchedSync, 0, pagesSourceScanBatchSize)
|
||||
previousDispatch := dispatchPagesSourceAutoSync
|
||||
dispatchPagesSourceAutoSync = func(
|
||||
_ context.Context,
|
||||
source model.PagesProjectSource,
|
||||
revision string,
|
||||
) (*SourceActionReceipt, error) {
|
||||
if source.ID == fixtures[5].source.ID {
|
||||
return nil, errors.New("injected dispatch failure")
|
||||
}
|
||||
dispatched = append(dispatched, scannerDispatchedSync{SourceID: source.ID, Revision: revision})
|
||||
return &SourceActionReceipt{ExecutionID: fmt.Sprintf("%d", source.ID), Action: sourceActionSync}, nil
|
||||
}
|
||||
t.Cleanup(func() { dispatchPagesSourceAutoSync = previousDispatch })
|
||||
|
||||
result, err := (&SourceScanHandler{}).Execute(ctx, []byte("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("SourceScanHandler.Execute() error = %v, want nil", err)
|
||||
}
|
||||
var summary pagesSourceScanSummary
|
||||
if err := json.Unmarshal([]byte(result.Detail), &summary); err != nil {
|
||||
t.Fatalf("json.Unmarshal(scan detail) error = %v, want nil", err)
|
||||
}
|
||||
if summary.DueSources != 22 || summary.SelectedSources != pagesSourceScanBatchSize ||
|
||||
summary.CheckedSources != 18 || summary.UpdatesFound != 17 || summary.AttentionSources != 1 ||
|
||||
summary.DispatchedSyncs != 15 || summary.FailedDispatches != 1 ||
|
||||
summary.BusySources != 1 || summary.FailedSources != 2 ||
|
||||
summary.Backlog != 3 {
|
||||
t.Errorf("scan summary = %+v, want due=22 selected=20 checked=18 updates=17 attention=1 dispatched=15 dispatch_failed=1 busy=1 failed=2 backlog=3", summary)
|
||||
}
|
||||
if len(summary.ProviderBackoffs) != 1 ||
|
||||
summary.ProviderBackoffs[0].SourceID != fixtures[1].source.ID ||
|
||||
summary.ProviderBackoffs[0].StatusCode != 429 ||
|
||||
summary.ProviderBackoffs[0].RetryAt != retryAt.UTC().Format(time.RFC3339) {
|
||||
t.Errorf("scan provider backoffs = %+v, want source=%d status=429 retry_at=%s", summary.ProviderBackoffs, fixtures[1].source.ID, retryAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
if len(calledRepositories) != 19 {
|
||||
t.Fatalf("Resolve calls = %d, want 19 (one busy source in selected batch)", len(calledRepositories))
|
||||
}
|
||||
for index, repository := range calledRepositories {
|
||||
want := fixtures[index+1].repository
|
||||
if repository != want {
|
||||
t.Fatalf("Resolve order[%d] = %q, want %q", index, repository, want)
|
||||
}
|
||||
}
|
||||
|
||||
if !containsDispatchedSource(dispatched, fixtures[2].source.ID, stored304Revision) {
|
||||
t.Errorf("304 stored revision was not dispatched: %+v", dispatched)
|
||||
}
|
||||
if containsDispatchedSource(dispatched, fixtures[3].source.ID, "") {
|
||||
t.Errorf("auto=false source was dispatched: %+v", dispatched)
|
||||
}
|
||||
if containsDispatchedSource(dispatched, fixtures[4].source.ID, "") {
|
||||
t.Errorf("attention source was dispatched: %+v", dispatched)
|
||||
}
|
||||
|
||||
_, dispatchFailedRuntime := mustLoadPagesSource(t, ctx, fixtures[5].source.ProjectID)
|
||||
if dispatchFailedRuntime.SyncStatus != pagesSourceStatusUpdateAvailable ||
|
||||
dispatchFailedRuntime.LastError != errPagesSourceTaskDispatchFailed ||
|
||||
dispatchFailedRuntime.NextCheckAt == nil || dispatchFailedRuntime.NextCheckAt.Before(retryAt) ||
|
||||
dispatchFailedRuntime.LastSeenRevision == "" {
|
||||
t.Errorf(
|
||||
"dispatch failure runtime = status:%q error:%q next:%v seen:%q, want preserved update and provider deadline >= %v",
|
||||
dispatchFailedRuntime.SyncStatus,
|
||||
dispatchFailedRuntime.LastError,
|
||||
dispatchFailedRuntime.NextCheckAt,
|
||||
dispatchFailedRuntime.LastSeenRevision,
|
||||
retryAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagesSourceScannerIncludesOrphanCleanupSummary(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
now := time.Now().Truncate(time.Second)
|
||||
usePagesSourceScannerClock(t, now)
|
||||
|
||||
previousReconcile := reconcilePagesSourceOrphans
|
||||
reconcilePagesSourceOrphans = func(
|
||||
_ context.Context,
|
||||
gotNow time.Time,
|
||||
) (PagesOrphanCleanupSummary, error) {
|
||||
if !gotNow.Equal(now) {
|
||||
t.Errorf("orphan cleanup now = %v, want %v", gotNow, now)
|
||||
}
|
||||
return PagesOrphanCleanupSummary{
|
||||
Candidates: 7,
|
||||
Reconciled: 1,
|
||||
Referenced: 2,
|
||||
LeaseBusy: 1,
|
||||
InvalidMarker: 1,
|
||||
Skipped: 1,
|
||||
Failed: 1,
|
||||
}, nil
|
||||
}
|
||||
t.Cleanup(func() { reconcilePagesSourceOrphans = previousReconcile })
|
||||
|
||||
result, err := (&SourceScanHandler{}).Execute(ctx, []byte("{}"))
|
||||
if err != nil {
|
||||
t.Fatalf("SourceScanHandler.Execute() error = %v, want nil", err)
|
||||
}
|
||||
var summary pagesSourceScanSummary
|
||||
if err := json.Unmarshal([]byte(result.Detail), &summary); err != nil {
|
||||
t.Fatalf("json.Unmarshal(scan detail) error = %v, want nil", err)
|
||||
}
|
||||
if summary.OrphanCleanup.Candidates != 7 || summary.OrphanCleanup.Reconciled != 1 ||
|
||||
summary.OrphanCleanup.Referenced != 2 || summary.OrphanCleanup.LeaseBusy != 1 ||
|
||||
summary.OrphanCleanup.InvalidMarker != 1 || summary.OrphanCleanup.Skipped != 1 ||
|
||||
summary.OrphanCleanup.Failed != 1 {
|
||||
t.Errorf("orphan cleanup summary = %+v, want injected result", summary.OrphanCleanup)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagesSourceScanPayloadAndMeta(t *testing.T) {
|
||||
handler := &SourceScanHandler{}
|
||||
if normalized, err := handler.ValidatePayload(nil); err != nil || string(normalized) != "{}" {
|
||||
t.Errorf("ValidatePayload(nil) = %s, %v; want {}, nil", normalized, err)
|
||||
}
|
||||
if _, err := handler.ValidatePayload([]byte(`{"unexpected":true}`)); err == nil {
|
||||
t.Error("ValidatePayload(unknown field) error = nil, want non-nil")
|
||||
}
|
||||
if PagesSourceScanMeta.InternalOnly || PagesSourceScanMeta.Type != TaskTypePagesSourceScan ||
|
||||
PagesSourceScanMeta.AsynqTask != PagesSourceScanTask || PagesSourceScanMeta.MaxRetry != 0 {
|
||||
t.Errorf("PagesSourceScanMeta = %+v, want public bounded scheduled scanner", PagesSourceScanMeta)
|
||||
}
|
||||
if PagesSourceScanMeta.SupportsTime {
|
||||
t.Error("PagesSourceScanMeta.SupportsTime = true, want empty scanner payload")
|
||||
}
|
||||
}
|
||||
|
||||
func usePagesSourceScannerClock(t *testing.T, now time.Time) {
|
||||
t.Helper()
|
||||
previous := pagesSourceScanNow
|
||||
pagesSourceScanNow = func() time.Time { return now }
|
||||
t.Cleanup(func() { pagesSourceScanNow = previous })
|
||||
}
|
||||
|
||||
func containsDispatchedSource(dispatched []scannerDispatchedSync, sourceID uint, revision string) bool {
|
||||
for _, item := range dispatched {
|
||||
if item.SourceID == sourceID && (revision == "" || item.Revision == revision) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,735 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/openflare/plugins/server/kernel/task"
|
||||
"Wavelet/pkg/logger"
|
||||
"Wavelet/pkg/util"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const (
|
||||
pagesSourceTriggerManualSync = "manual_sync"
|
||||
pagesSourceTriggerScheduledAutoUpdate = "scheduled_auto_update"
|
||||
pagesSourceCreatedBySystem = "system:pages-source-sync"
|
||||
pagesSourceHeartbeatInterval = pagesSourceSyncLeaseDuration / 3
|
||||
pagesSourceCleanupTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
errSourceFinalFence = errors.New("pages source final fence rejected")
|
||||
errSourceLeaseHeartbeatLost = errors.New("pages source lease heartbeat lost")
|
||||
sourceCommitNow = time.Now
|
||||
)
|
||||
|
||||
type sourceSyncOutcome struct {
|
||||
Deployment *DeploymentView
|
||||
Reused bool
|
||||
Stale bool
|
||||
}
|
||||
|
||||
type preparedRemoteSource struct {
|
||||
Candidate *SourceCandidate
|
||||
Manifest *deploymentManifest
|
||||
Detail sourceDetail
|
||||
DetailJSON string
|
||||
}
|
||||
|
||||
type sourceIngestState struct {
|
||||
Result ofupload.IngestResult
|
||||
HasIngest bool
|
||||
Referenced bool
|
||||
}
|
||||
|
||||
type sourceCommitState struct {
|
||||
Project *model.PagesProject
|
||||
Source *model.PagesProjectSource
|
||||
Runtime *model.PagesProjectSourceRuntime
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
type sourceLeaseHeartbeat struct {
|
||||
cancel context.CancelFunc
|
||||
done <-chan error
|
||||
stopOnce sync.Once
|
||||
stopErr error
|
||||
}
|
||||
|
||||
func startSourceLeaseHeartbeat(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
leaseDuration time.Duration,
|
||||
interval time.Duration,
|
||||
) (context.Context, *sourceLeaseHeartbeat, error) {
|
||||
if snapshot == nil || leaseDuration <= 0 || interval <= 0 || interval >= leaseDuration {
|
||||
return nil, nil, errors.New(errPagesSourceLeaseLost)
|
||||
}
|
||||
renewed, err := renewSourceLease(ctx, snapshot, leaseDuration)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !renewed {
|
||||
return nil, nil, errSourceLeaseHeartbeatLost
|
||||
}
|
||||
|
||||
workCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
heartbeat := &sourceLeaseHeartbeat{cancel: cancel, done: done}
|
||||
util.Go(func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-workCtx.Done():
|
||||
done <- nil
|
||||
return
|
||||
case <-ticker.C:
|
||||
renewed, renewErr := renewSourceLease(workCtx, snapshot, leaseDuration)
|
||||
if renewErr != nil {
|
||||
if workCtx.Err() != nil {
|
||||
done <- nil
|
||||
return
|
||||
}
|
||||
done <- renewErr
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
if !renewed {
|
||||
done <- errSourceLeaseHeartbeatLost
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
return workCtx, heartbeat, nil
|
||||
}
|
||||
|
||||
func (heartbeat *sourceLeaseHeartbeat) stop() error {
|
||||
if heartbeat == nil {
|
||||
return nil
|
||||
}
|
||||
heartbeat.stopOnce.Do(func() {
|
||||
heartbeat.cancel()
|
||||
heartbeat.stopErr = <-heartbeat.done
|
||||
})
|
||||
return heartbeat.stopErr
|
||||
}
|
||||
|
||||
func sourceHeartbeatOutcome(err error) (*sourceSyncOutcome, error) {
|
||||
if errors.Is(err, errSourceLeaseHeartbeatLost) {
|
||||
return &sourceSyncOutcome{Stale: true}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func recordSourceLeaseFailure(ctx context.Context, snapshot *sourceExecutionSnapshot) {
|
||||
cleanupCtx, cancel := sourceCleanupContext(ctx)
|
||||
defer cancel()
|
||||
if err := failSourceLease(cleanupCtx, snapshot, errPagesSourceSyncFailed); err != nil {
|
||||
var sourceID uint
|
||||
if snapshot != nil {
|
||||
sourceID = snapshot.SourceID
|
||||
}
|
||||
logger.WarnF(cleanupCtx, "[PagesSource] record failed runtime state failed: source_id=%d error=%v", sourceID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func sourceCleanupContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), pagesSourceCleanupTimeout)
|
||||
}
|
||||
|
||||
func syncRemoteSourceWithTrigger(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
actor string,
|
||||
triggerType string,
|
||||
) (outcome *sourceSyncOutcome, resultErr error) {
|
||||
if snapshot == nil || snapshot.SourceType != PagesSourceTypeRemoteURL {
|
||||
return nil, errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
actor = strings.TrimSpace(actor)
|
||||
if actor == "" || !validSourceDeploymentTrigger(triggerType) {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
defer func() {
|
||||
if resultErr != nil {
|
||||
recordSourceLeaseFailure(ctx, snapshot)
|
||||
}
|
||||
}()
|
||||
|
||||
workCtx, heartbeat, err := startSourceLeaseHeartbeat(
|
||||
ctx,
|
||||
snapshot,
|
||||
pagesSourceSyncLeaseDuration,
|
||||
pagesSourceHeartbeatInterval,
|
||||
)
|
||||
if err != nil {
|
||||
return sourceHeartbeatOutcome(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = heartbeat.stop()
|
||||
}()
|
||||
|
||||
limits := resolvePagesLimits(workCtx)
|
||||
prepared, err := prepareRemoteSource(workCtx, snapshot, limits)
|
||||
if err != nil {
|
||||
if heartbeatErr := heartbeat.stop(); heartbeatErr != nil {
|
||||
return sourceHeartbeatOutcome(heartbeatErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if cleanupErr := prepared.Candidate.Cleanup(); cleanupErr != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] cleanup temporary package failed: source_id=%d error=%v", snapshot.SourceID, cleanupErr)
|
||||
}
|
||||
}()
|
||||
|
||||
ingestState, err := resolveSourceIngest(workCtx, snapshot, prepared)
|
||||
if err != nil {
|
||||
if heartbeatErr := heartbeat.stop(); heartbeatErr != nil {
|
||||
return sourceHeartbeatOutcome(heartbeatErr)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
compensateSourceIngest(ctx, snapshot, ingestState)
|
||||
}()
|
||||
|
||||
if heartbeatErr := heartbeat.stop(); heartbeatErr != nil {
|
||||
return sourceHeartbeatOutcome(heartbeatErr)
|
||||
}
|
||||
renewed, err := renewSourceLease(ctx, snapshot, pagesSourceSyncLeaseDuration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !renewed {
|
||||
return &sourceSyncOutcome{Stale: true}, nil
|
||||
}
|
||||
|
||||
task.AppendLog(ctx, "[activate] 正在原子切换生产部署")
|
||||
deployment, reused, referenced, err := commitSourceDeploymentWithTrigger(
|
||||
ctx,
|
||||
snapshot,
|
||||
prepared.Candidate.Checksum,
|
||||
prepared.Candidate.Checksum,
|
||||
prepared.Detail,
|
||||
prepared.DetailJSON,
|
||||
actor,
|
||||
triggerType,
|
||||
prepared.Manifest,
|
||||
ingestState.Result,
|
||||
ingestState.HasIngest,
|
||||
nil,
|
||||
)
|
||||
ingestState.Referenced = referenced
|
||||
if errors.Is(err, errSourceFinalFence) {
|
||||
return &sourceSyncOutcome{Stale: true}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ingestState.Referenced = ingestState.HasIngest && deployment.UploadID == ingestState.Result.Upload.ID
|
||||
|
||||
if pruneErr := pruneProjectDeploymentHistory(ctx, snapshot.ProjectID, limits.HistoryCount, 0); pruneErr != nil {
|
||||
logger.ErrorF(ctx,
|
||||
"[PagesSource] strict prune failed: project_id=%d source_id=%d keep=%d error=%v",
|
||||
snapshot.ProjectID,
|
||||
snapshot.SourceID,
|
||||
limits.HistoryCount,
|
||||
pruneErr,
|
||||
)
|
||||
}
|
||||
view := buildDeploymentView(deployment)
|
||||
return &sourceSyncOutcome{Deployment: &view, Reused: reused}, nil
|
||||
}
|
||||
|
||||
func prepareRemoteSource(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
limits pagesLimits,
|
||||
) (*preparedRemoteSource, error) {
|
||||
task.AppendLog(ctx, "[download] 正在获取远程部署包")
|
||||
candidate, err := FetchRemoteSource(ctx, RemoteSourceRequest{
|
||||
URL: snapshot.RemoteURL,
|
||||
AllowInsecure: snapshot.AllowInsecure,
|
||||
MaxPackageBytes: limits.PackageBytes,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if candidate == nil || candidate.TempPath == "" || candidate.Checksum == "" || candidate.Format == "" {
|
||||
return nil, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
rootDir, err := validateAndNormalizePagesRootDir(snapshot.RootDir)
|
||||
if err != nil {
|
||||
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
|
||||
return nil, err
|
||||
}
|
||||
entryFile, err := validateAndNormalizePagesEntryFile(snapshot.EntryFile)
|
||||
if err != nil {
|
||||
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
|
||||
return nil, err
|
||||
}
|
||||
task.AppendLog(ctx, "[verify] 正在校验归档结构与入口文件")
|
||||
manifest, err := inspectPagesPackage(candidate.TempPath, candidate.Format, rootDir, entryFile, limits)
|
||||
if err != nil {
|
||||
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
|
||||
return nil, err
|
||||
}
|
||||
detail := sourceDetail{Provider: PagesSourceTypeRemoteURL, DisplayName: safeRemoteSourceLabel(candidate.SafeLabel)}
|
||||
detailJSON, err := json.Marshal(detail)
|
||||
if err != nil {
|
||||
cleanupFailedRemoteCandidate(ctx, snapshot, candidate)
|
||||
return nil, err
|
||||
}
|
||||
return &preparedRemoteSource{
|
||||
Candidate: candidate,
|
||||
Manifest: manifest,
|
||||
Detail: detail,
|
||||
DetailJSON: string(detailJSON),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cleanupFailedRemoteCandidate(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
candidate *SourceCandidate,
|
||||
) {
|
||||
if err := candidate.Cleanup(); err != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] cleanup failed preparation package: source_id=%d error=%v", snapshot.SourceID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func resolveSourceIngest(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
prepared *preparedRemoteSource,
|
||||
) (*sourceIngestState, error) {
|
||||
_, err := findSourceDeployment(
|
||||
ctx,
|
||||
snapshot.ProjectID,
|
||||
snapshot.SourceIdentity,
|
||||
prepared.Candidate.Checksum,
|
||||
)
|
||||
if err == nil {
|
||||
return &sourceIngestState{}, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
task.AppendLog(ctx, "[ingest] 正在保存受管部署包")
|
||||
result, err := ingestPagesDeploymentPackageWithSource(
|
||||
ctx,
|
||||
prepared.Candidate.TempPath,
|
||||
prepared.Candidate.Checksum,
|
||||
snapshot.ProjectID,
|
||||
snapshot.SourceID,
|
||||
sourceDetailLabel(prepared.Detail),
|
||||
prepared.Candidate.Format,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sourceIngestState{Result: result, HasIngest: true}, nil
|
||||
}
|
||||
|
||||
func compensateSourceIngest(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
state *sourceIngestState,
|
||||
) {
|
||||
if state == nil || !state.HasIngest || !state.Result.Created || state.Referenced {
|
||||
return
|
||||
}
|
||||
cleanupCtx, cancel := sourceCleanupContext(ctx)
|
||||
defer cancel()
|
||||
task.AppendLog(cleanupCtx, "[cleanup] 正在补偿未引用的部署包记录")
|
||||
if err := removePagesUploadIfUnreferenced(cleanupCtx, snapshot.ProjectID, state.Result.Upload.ID); err != nil {
|
||||
logger.ErrorF(cleanupCtx,
|
||||
"[PagesSource] compensate upload failed: project_id=%d source_id=%d upload_id=%d error=%v",
|
||||
snapshot.ProjectID, snapshot.SourceID, state.Result.Upload.ID, err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func findSourceDeployment(
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
sourceIdentity string,
|
||||
revision string,
|
||||
) (*model.PagesDeployment, error) {
|
||||
return repository.GetPagesDeploymentBySourceRevision(ctx, projectID, sourceIdentity, revision)
|
||||
}
|
||||
|
||||
func commitSourceDeploymentWithTrigger(
|
||||
ctx context.Context,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
revision string,
|
||||
packageChecksum string,
|
||||
detail sourceDetail,
|
||||
detailJSON string,
|
||||
actor string,
|
||||
triggerType string,
|
||||
manifest *deploymentManifest,
|
||||
ingestResult ofupload.IngestResult,
|
||||
hasIngest bool,
|
||||
nextCheckNotBefore *time.Time,
|
||||
) (*model.PagesDeployment, bool, bool, error) {
|
||||
if snapshot == nil || manifest == nil {
|
||||
return nil, false, false, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
if !validSourceDeploymentTrigger(triggerType) {
|
||||
return nil, false, false, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
var committed model.PagesDeployment
|
||||
reused := false
|
||||
ingestReferenced := false
|
||||
err := repository.WithPagesTx(ctx, func(tx *gorm.DB) error {
|
||||
state, err := lockSourceCommitState(tx, snapshot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target, targetReused, err := resolveSourceDeploymentTx(
|
||||
tx, state, revision, packageChecksum, detail, detailJSON, actor, triggerType,
|
||||
manifest, ingestResult, hasIngest,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := lockSourceDeploymentUploadsTx(tx, target, ingestResult, hasIngest); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureDeploymentEntry(tx, target.ID, state.Project.RootDir, state.Project.EntryFile); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := refreshSourceCommitLease(state, snapshot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := activateSourceDeploymentTx(tx, state, target, revision, detailJSON, nextCheckNotBefore); err != nil {
|
||||
return err
|
||||
}
|
||||
committed = *target
|
||||
reused = targetReused
|
||||
ingestReferenced = hasIngest && target.UploadID == ingestResult.Upload.ID
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
return &committed, reused, ingestReferenced, nil
|
||||
}
|
||||
|
||||
func lockSourceCommitState(tx *gorm.DB, snapshot *sourceExecutionSnapshot) (*sourceCommitState, error) {
|
||||
state := &sourceCommitState{}
|
||||
project, err := repository.LockPagesProjectByIDTx(tx, snapshot.ProjectID)
|
||||
if err != nil {
|
||||
return nil, sourceFenceRecordError(err)
|
||||
}
|
||||
if project.ContentConfigVersion != snapshot.ContentConfigVersion {
|
||||
return nil, errSourceFinalFence
|
||||
}
|
||||
source, err := repository.LockPagesProjectSourceByIDAndProjectIDTx(tx, snapshot.SourceID, snapshot.ProjectID)
|
||||
if err != nil {
|
||||
return nil, sourceFenceRecordError(err)
|
||||
}
|
||||
if source.ConfigVersion != snapshot.SourceConfigVersion ||
|
||||
source.SourceIdentity != snapshot.SourceIdentity ||
|
||||
source.SourceType != snapshot.SourceType {
|
||||
return nil, errSourceFinalFence
|
||||
}
|
||||
runtime, err := repository.LockPagesProjectSourceRuntimeBySourceIDTx(tx, source.ID)
|
||||
if err != nil {
|
||||
return nil, sourceFenceRecordError(err)
|
||||
}
|
||||
state.Project = project
|
||||
state.Source = source
|
||||
state.Runtime = runtime
|
||||
if err := refreshSourceCommitLease(state, snapshot); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func refreshSourceCommitLease(state *sourceCommitState, snapshot *sourceExecutionSnapshot) error {
|
||||
if state == nil || state.Runtime == nil || snapshot == nil {
|
||||
return errSourceFinalFence
|
||||
}
|
||||
now := sourceCommitNow()
|
||||
if state.Runtime.LeaseToken != snapshot.LeaseToken || state.Runtime.LeaseExpiresAt == nil ||
|
||||
!state.Runtime.LeaseExpiresAt.After(now) {
|
||||
return errSourceFinalFence
|
||||
}
|
||||
state.Now = now
|
||||
return nil
|
||||
}
|
||||
|
||||
func sourceFenceRecordError(err error) error {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return errSourceFinalFence
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func resolveSourceDeploymentTx(
|
||||
tx *gorm.DB,
|
||||
state *sourceCommitState,
|
||||
revision string,
|
||||
packageChecksum string,
|
||||
detail sourceDetail,
|
||||
detailJSON string,
|
||||
actor string,
|
||||
triggerType string,
|
||||
manifest *deploymentManifest,
|
||||
ingestResult ofupload.IngestResult,
|
||||
hasIngest bool,
|
||||
) (*model.PagesDeployment, bool, error) {
|
||||
var target model.PagesDeployment
|
||||
err := tx.Where(
|
||||
"project_id = ? AND source_identity = ? AND source_revision = ?",
|
||||
state.Project.ID,
|
||||
state.Source.SourceIdentity,
|
||||
revision,
|
||||
).First(&target).Error
|
||||
if err == nil {
|
||||
return &target, true, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, false, err
|
||||
}
|
||||
if !hasIngest {
|
||||
return nil, false, errSourceFinalFence
|
||||
}
|
||||
return createSourceDeploymentTx(
|
||||
tx, state, revision, packageChecksum, detail, detailJSON, actor, triggerType, manifest, ingestResult,
|
||||
)
|
||||
}
|
||||
|
||||
func createSourceDeploymentTx(
|
||||
tx *gorm.DB,
|
||||
state *sourceCommitState,
|
||||
revision string,
|
||||
packageChecksum string,
|
||||
detail sourceDetail,
|
||||
detailJSON string,
|
||||
actor string,
|
||||
triggerType string,
|
||||
manifest *deploymentManifest,
|
||||
ingestResult ofupload.IngestResult,
|
||||
) (*model.PagesDeployment, bool, error) {
|
||||
var maxNumber int
|
||||
if err := tx.Model(&model.PagesDeployment{}).
|
||||
Where("project_id = ?", state.Project.ID).
|
||||
Select("COALESCE(MAX(deployment_number), 0)").
|
||||
Scan(&maxNumber).Error; err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
identity := state.Source.SourceIdentity
|
||||
revisionValue := revision
|
||||
target := &model.PagesDeployment{
|
||||
ProjectID: state.Project.ID,
|
||||
DeploymentNumber: maxNumber + 1,
|
||||
Checksum: packageChecksum,
|
||||
Status: model.PagesDeploymentStatusUploaded,
|
||||
UploadID: ingestResult.Upload.ID,
|
||||
FileCount: manifest.FileCount,
|
||||
TotalSize: manifest.TotalSize,
|
||||
CreatedBy: actor,
|
||||
SourceType: state.Source.SourceType,
|
||||
SourceIdentity: &identity,
|
||||
SourceRevision: &revisionValue,
|
||||
SourceLabel: sourceDetailLabel(detail),
|
||||
SourceMeta: detailJSON,
|
||||
TriggerType: triggerType,
|
||||
}
|
||||
result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(target)
|
||||
if result.Error != nil {
|
||||
return nil, false, result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return reloadSourceDeploymentTx(tx, state.Project.ID, identity, revision)
|
||||
}
|
||||
if err := createSourceDeploymentFilesTx(tx, target.ID, manifest.Files); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return target, false, nil
|
||||
}
|
||||
|
||||
func validSourceDeploymentTrigger(triggerType string) bool {
|
||||
return triggerType == pagesSourceTriggerManualSync || triggerType == pagesSourceTriggerScheduledAutoUpdate
|
||||
}
|
||||
|
||||
func reloadSourceDeploymentTx(
|
||||
tx *gorm.DB,
|
||||
projectID uint,
|
||||
identity string,
|
||||
revision string,
|
||||
) (*model.PagesDeployment, bool, error) {
|
||||
var target model.PagesDeployment
|
||||
err := tx.Where(
|
||||
"project_id = ? AND source_identity = ? AND source_revision = ?",
|
||||
projectID,
|
||||
identity,
|
||||
revision,
|
||||
).First(&target).Error
|
||||
return &target, true, err
|
||||
}
|
||||
|
||||
func createSourceDeploymentFilesTx(tx *gorm.DB, deploymentID uint, files []model.PagesDeploymentFile) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
for index := range files {
|
||||
files[index].DeploymentID = deploymentID
|
||||
}
|
||||
return tx.Create(&files).Error
|
||||
}
|
||||
|
||||
func lockSourceDeploymentUploadsTx(
|
||||
tx *gorm.DB,
|
||||
target *model.PagesDeployment,
|
||||
ingestResult ofupload.IngestResult,
|
||||
hasIngest bool,
|
||||
) error {
|
||||
uploadIDs := []uint64{target.UploadID}
|
||||
if hasIngest && ingestResult.Upload.ID != 0 && ingestResult.Upload.ID != target.UploadID {
|
||||
uploadIDs = append(uploadIDs, ingestResult.Upload.ID)
|
||||
}
|
||||
slices.Sort(uploadIDs)
|
||||
var records []model.Upload
|
||||
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
|
||||
Where("id IN ?", uploadIDs).
|
||||
Order("id asc").
|
||||
Find(&records).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for index := range records {
|
||||
if records[index].ID != target.UploadID {
|
||||
continue
|
||||
}
|
||||
if records[index].Status == model.UploadStatusUsed && records[index].Type == ofupload.ReservedPagesDeploymentType {
|
||||
return nil
|
||||
}
|
||||
break
|
||||
}
|
||||
return errSourceFinalFence
|
||||
}
|
||||
|
||||
func activateSourceDeploymentTx(
|
||||
tx *gorm.DB,
|
||||
state *sourceCommitState,
|
||||
target *model.PagesDeployment,
|
||||
revision string,
|
||||
detailJSON string,
|
||||
nextCheckNotBefore *time.Time,
|
||||
) error {
|
||||
if err := tx.Model(&model.PagesDeployment{}).
|
||||
Where("project_id = ?", state.Project.ID).
|
||||
Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(target).Updates(map[string]any{
|
||||
pagesDeploymentColumnStatus: model.PagesDeploymentStatusActive,
|
||||
"activated_at": &state.Now,
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Model(state.Project).Update("active_deployment_id", target.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
finishedAt := sourceCommitNow()
|
||||
var nextCheckAt any
|
||||
if state.Source.SourceType == PagesSourceTypeGitHubRelease &&
|
||||
state.Source.ReleaseSelector == githubReleaseSelectorLatest {
|
||||
next := nextGitHubCheckAt(finishedAt, state.Source.ID, state.Source.CheckIntervalMinutes)
|
||||
if nextCheckNotBefore != nil && nextCheckNotBefore.After(next) {
|
||||
next = nextCheckNotBefore.In(finishedAt.Location())
|
||||
}
|
||||
nextCheckAt = &next
|
||||
}
|
||||
rows, err := repository.UpdatePagesSourceRuntimeByActiveLeaseTx(
|
||||
tx,
|
||||
state.Runtime.SourceID,
|
||||
state.Runtime.LeaseToken,
|
||||
finishedAt,
|
||||
map[string]any{
|
||||
"last_seen_revision": revision,
|
||||
"last_seen_detail": detailJSON,
|
||||
"last_applied_revision": revision,
|
||||
"last_applied_detail": detailJSON,
|
||||
sourceRuntimeColumnSyncStatus: pagesSourceStatusIdle,
|
||||
sourceRuntimeColumnLastError: "",
|
||||
sourceRuntimeColumnLastCheckedAt: &finishedAt,
|
||||
"last_synced_at": &finishedAt,
|
||||
"next_check_at": nextCheckAt,
|
||||
sourceRuntimeColumnLeaseToken: "",
|
||||
sourceRuntimeColumnLeaseExpiresAt: nil,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows != 1 {
|
||||
return errSourceFinalFence
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safeRemoteSourceLabel(raw string) string {
|
||||
value := strings.ReplaceAll(strings.ToValidUTF8(raw, ""), "\\", "/")
|
||||
value = path.Base(strings.TrimSpace(value))
|
||||
if value == "." || value == "/" {
|
||||
value = ""
|
||||
}
|
||||
var builder strings.Builder
|
||||
for _, character := range value {
|
||||
if character >= 0x20 && character != 0x7f {
|
||||
builder.WriteRune(character)
|
||||
}
|
||||
}
|
||||
value = strings.TrimSpace(builder.String())
|
||||
if value == "" {
|
||||
value = defaultRemoteAssetLabel
|
||||
}
|
||||
if len(value) > remoteSourceMaxSafeLabelBytes {
|
||||
value = value[:remoteSourceMaxSafeLabelBytes]
|
||||
for !utf8.ValidString(value) {
|
||||
_, size := utf8.DecodeLastRuneInString(value)
|
||||
value = value[:len(value)-size]
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func sourceSyncResultDetail(outcome *sourceSyncOutcome) string {
|
||||
if outcome == nil || outcome.Deployment == nil {
|
||||
return ""
|
||||
}
|
||||
payload := map[string]any{
|
||||
"deployment_id": outcome.Deployment.ID,
|
||||
"reused": outcome.Reused,
|
||||
}
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Sprintf(`{"deployment_id":%d}`, outcome.Deployment.ID)
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
@@ -0,0 +1,611 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/ofupload"
|
||||
"Wavelet/openflare/share/pagesarchive"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupPagesSourceSyncTest(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
ctx := setupPagesSourceTest(t)
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
t.Cleanup(disableStorage)
|
||||
return ctx
|
||||
}
|
||||
|
||||
func mustAcquireRemoteSyncLease(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
source *model.PagesProjectSource,
|
||||
) *sourceExecutionSnapshot {
|
||||
t.Helper()
|
||||
snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync)
|
||||
if err != nil {
|
||||
t.Fatalf("acquireSourceLease(source=%d) error = %v, want nil", source.ID, err)
|
||||
}
|
||||
if got, want := outcome, sourceLeaseAcquired; got != want {
|
||||
t.Fatalf("acquireSourceLease(source=%d) outcome = %q, want %q", source.ID, got, want)
|
||||
}
|
||||
if snapshot == nil {
|
||||
t.Fatalf("acquireSourceLease(source=%d) snapshot = nil, want non-nil", source.ID)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func mustCreateActiveManualDeployment(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
content string,
|
||||
) *model.PagesDeployment {
|
||||
t.Helper()
|
||||
view, err := UploadDeployment(
|
||||
ctx,
|
||||
projectID,
|
||||
testPagesMultipartFile(t, "manual.zip", testPagesZip(t, map[string]string{"index.html": content})),
|
||||
"user:1",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UploadDeployment(project=%d) error = %v, want nil", projectID, err)
|
||||
}
|
||||
if _, err := ActivateDeploymentAs(ctx, projectID, view.ID, "user:1"); err != nil {
|
||||
t.Fatalf("ActivateDeploymentAs(project=%d, deployment=%d) error = %v, want nil", projectID, view.ID, err)
|
||||
}
|
||||
deployment, err := repository.GetPagesDeploymentByID(ctx, view.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", view.ID, err)
|
||||
}
|
||||
return deployment
|
||||
}
|
||||
|
||||
func newPagesArchiveServer(t *testing.T, status int, body []byte, beforeWrite func() error) *httptest.Server {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
if beforeWrite != nil {
|
||||
if err := beforeWrite(); err != nil {
|
||||
writer.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/zip")
|
||||
writer.Header().Set("Content-Disposition", `attachment; filename="site.zip"`)
|
||||
writer.WriteHeader(status)
|
||||
_, _ = writer.Write(body)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return server
|
||||
}
|
||||
|
||||
func TestSyncRemoteSourceAtomicallyActivatesAndReusesChecksum(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "sync-success")
|
||||
packageBytes := testPagesZip(t, map[string]string{
|
||||
"index.html": "remote-v1",
|
||||
"assets/app.js": "console.log('ok')",
|
||||
})
|
||||
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil)
|
||||
secret := "sync-query-secret"
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
server.URL+"/site.zip?token="+secret,
|
||||
true,
|
||||
)
|
||||
|
||||
firstSnapshot := mustAcquireRemoteSyncLease(t, ctx, source)
|
||||
first, err := syncRemoteSource(ctx, firstSnapshot, "user:42")
|
||||
if err != nil {
|
||||
t.Fatalf("syncRemoteSource(first) error = %v, want nil", err)
|
||||
}
|
||||
if first == nil || first.Stale || first.Reused || first.Deployment == nil {
|
||||
t.Fatalf("syncRemoteSource(first) = %+v, want new active deployment", first)
|
||||
}
|
||||
storedProject, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != first.Deployment.ID {
|
||||
t.Fatalf("project ActiveDeploymentID = %v, want %d", storedProject.ActiveDeploymentID, first.Deployment.ID)
|
||||
}
|
||||
deployment, err := repository.GetPagesDeploymentByID(ctx, first.Deployment.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", first.Deployment.ID, err)
|
||||
}
|
||||
expectedHash := sha256.Sum256(packageBytes)
|
||||
if got, want := deployment.Checksum, hex.EncodeToString(expectedHash[:]); got != want {
|
||||
t.Errorf("deployment Checksum = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := deployment.Status, model.PagesDeploymentStatusActive; got != want {
|
||||
t.Errorf("deployment Status = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := deployment.SourceType, PagesSourceTypeRemoteURL; got != want {
|
||||
t.Errorf("deployment SourceType = %q, want %q", got, want)
|
||||
}
|
||||
if deployment.SourceIdentity == nil || *deployment.SourceIdentity != source.SourceIdentity {
|
||||
t.Errorf("deployment SourceIdentity = %v, want %q", deployment.SourceIdentity, source.SourceIdentity)
|
||||
}
|
||||
if deployment.SourceRevision == nil || *deployment.SourceRevision != deployment.Checksum {
|
||||
t.Errorf("deployment SourceRevision = %v, want %q", deployment.SourceRevision, deployment.Checksum)
|
||||
}
|
||||
if got, want := deployment.CreatedBy, "user:42"; got != want {
|
||||
t.Errorf("deployment CreatedBy = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := deployment.TriggerType, pagesSourceTriggerManualSync; got != want {
|
||||
t.Errorf("deployment TriggerType = %q, want %q", got, want)
|
||||
}
|
||||
if strings.Contains(deployment.SourceMeta, secret) || strings.Contains(deployment.SourceLabel, secret) {
|
||||
t.Errorf("deployment provenance = label:%q meta:%q, want no query secret", deployment.SourceLabel, deployment.SourceMeta)
|
||||
}
|
||||
var runtime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
|
||||
t.Fatalf("load source runtime error = %v, want nil", err)
|
||||
}
|
||||
if got, want := runtime.SyncStatus, pagesSourceStatusIdle; got != want {
|
||||
t.Errorf("runtime SyncStatus = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := runtime.LastAppliedRevision, deployment.Checksum; got != want {
|
||||
t.Errorf("runtime LastAppliedRevision = %q, want %q", got, want)
|
||||
}
|
||||
if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil {
|
||||
t.Errorf("runtime lease = (%q, %v), want cleared", runtime.LeaseToken, runtime.LeaseExpiresAt)
|
||||
}
|
||||
var uploadRecord model.Upload
|
||||
if err := db.DB(ctx).First(&uploadRecord, deployment.UploadID).Error; err != nil {
|
||||
t.Fatalf("load deployment upload %d error = %v, want nil", deployment.UploadID, err)
|
||||
}
|
||||
if got, want := uploadRecord.Status, model.UploadStatusUsed; got != want {
|
||||
t.Errorf("deployment upload Status = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := uploadRecord.Type, ofupload.ReservedPagesDeploymentType; got != want {
|
||||
t.Errorf("deployment upload Type = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := fmt.Sprint(uploadRecord.Metadata.Extra[pagesSourceIDMetadataKey]), fmt.Sprint(source.ID); got != want {
|
||||
t.Errorf("deployment upload pages_source_id = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
secondSnapshot := mustAcquireRemoteSyncLease(t, ctx, source)
|
||||
second, err := syncRemoteSource(ctx, secondSnapshot, "user:42")
|
||||
if err != nil {
|
||||
t.Fatalf("syncRemoteSource(second) error = %v, want nil", err)
|
||||
}
|
||||
if second == nil || second.Stale || !second.Reused || second.Deployment == nil {
|
||||
t.Fatalf("syncRemoteSource(second) = %+v, want reused active deployment", second)
|
||||
}
|
||||
if got, want := second.Deployment.ID, first.Deployment.ID; got != want {
|
||||
t.Errorf("reused deployment ID = %d, want %d", got, want)
|
||||
}
|
||||
var deploymentCount, uploadCount int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("project_id = ?", project.ID).Count(&deploymentCount).Error; err != nil {
|
||||
t.Fatalf("count source deployments error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error; err != nil {
|
||||
t.Fatalf("count source uploads error = %v, want nil", err)
|
||||
}
|
||||
if got, want := deploymentCount, int64(1); got != want {
|
||||
t.Errorf("deployment count after identical sync = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := uploadCount, int64(1); got != want {
|
||||
t.Errorf("upload count after identical sync = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncRemoteSourceDownloadFailureKeepsOldActive(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "sync-download-fail")
|
||||
oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active")
|
||||
server := newPagesArchiveServer(t, http.StatusBadGateway, []byte("upstream failed"), nil)
|
||||
secret := "download-failure-secret"
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
server.URL+"/site.zip?token="+secret,
|
||||
true,
|
||||
)
|
||||
|
||||
_, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:2")
|
||||
if err == nil {
|
||||
t.Fatal("syncRemoteSource(download failure) error = nil, want non-nil")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Errorf("syncRemoteSource(download failure) error = %q, want no query secret", err)
|
||||
}
|
||||
assertPagesSyncFailureState(t, ctx, project.ID, source.ID, oldActive.ID, 1)
|
||||
}
|
||||
|
||||
func TestSyncRemoteSourceArchiveFailureKeepsOldActive(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "sync-archive-fail")
|
||||
oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active")
|
||||
server := newPagesArchiveServer(t, http.StatusOK, []byte("not-a-valid-zip"), nil)
|
||||
secret := "archive-failure-secret"
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
server.URL+"/site.zip?token="+secret,
|
||||
true,
|
||||
)
|
||||
|
||||
_, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:3")
|
||||
if err == nil {
|
||||
t.Fatal("syncRemoteSource(archive failure) error = nil, want non-nil")
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Errorf("syncRemoteSource(archive failure) error = %q, want no query secret", err)
|
||||
}
|
||||
assertPagesSyncFailureState(t, ctx, project.ID, source.ID, oldActive.ID, 1)
|
||||
}
|
||||
|
||||
func TestSyncRemoteSourceFinalFenceCompensatesIngest(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "sync-final-fence")
|
||||
oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active")
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "never-activate"})
|
||||
mutationResult := make(chan error, 1)
|
||||
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, func() error {
|
||||
err := db.DB(context.Background()).Model(&model.PagesProject{}).
|
||||
Where("id = ?", project.ID).
|
||||
Update("content_config_version", gorm.Expr("content_config_version + 1")).Error
|
||||
mutationResult <- err
|
||||
return err
|
||||
})
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
server.URL+"/site.zip?token=final-fence-secret",
|
||||
true,
|
||||
)
|
||||
|
||||
outcome, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:4")
|
||||
if err != nil {
|
||||
t.Fatalf("syncRemoteSource(final fence) error = %v, want nil stale outcome", err)
|
||||
}
|
||||
select {
|
||||
case mutationErr := <-mutationResult:
|
||||
if mutationErr != nil {
|
||||
t.Fatalf("content version mutation error = %v, want nil", mutationErr)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("content version mutation was not observed")
|
||||
}
|
||||
if outcome == nil || !outcome.Stale || outcome.Deployment != nil {
|
||||
t.Fatalf("syncRemoteSource(final fence) = %+v, want stale outcome without deployment", outcome)
|
||||
}
|
||||
storedProject, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != oldActive.ID {
|
||||
t.Errorf("ActiveDeploymentID after final fence = %v, want %d", storedProject.ActiveDeploymentID, oldActive.ID)
|
||||
}
|
||||
var deployments []model.PagesDeployment
|
||||
if err := db.DB(ctx).Where("project_id = ?", project.ID).Find(&deployments).Error; err != nil {
|
||||
t.Fatalf("list deployments after final fence error = %v, want nil", err)
|
||||
}
|
||||
if got, want := len(deployments), 1; got != want {
|
||||
t.Errorf("deployment count after final fence = %d, want %d", got, want)
|
||||
}
|
||||
var uploads []model.Upload
|
||||
if err := db.DB(ctx).Order("id asc").Find(&uploads).Error; err != nil {
|
||||
t.Fatalf("list uploads after final fence error = %v, want nil", err)
|
||||
}
|
||||
var compensated *model.Upload
|
||||
for index := range uploads {
|
||||
if fmt.Sprint(uploads[index].Metadata.Extra[pagesSourceIDMetadataKey]) == fmt.Sprint(source.ID) {
|
||||
compensated = &uploads[index]
|
||||
break
|
||||
}
|
||||
}
|
||||
if compensated == nil {
|
||||
t.Fatalf("source upload after final fence = nil, want compensated upload record")
|
||||
}
|
||||
if got, want := compensated.Status, model.UploadStatusDeleted; got != want {
|
||||
t.Errorf("compensated upload Status = %q, want %q", got, want)
|
||||
}
|
||||
var danglingCount int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
|
||||
Where("upload_id = ?", compensated.ID).
|
||||
Count(&danglingCount).Error; err != nil {
|
||||
t.Fatalf("count compensated upload references error = %v, want nil", err)
|
||||
}
|
||||
if got, want := danglingCount, int64(0); got != want {
|
||||
t.Errorf("deployments referencing compensated upload = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommitSourceDeploymentRechecksLeaseAfterUploadLocks(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "sync-expiry-recheck")
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "expiry-recheck"})
|
||||
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil)
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
server.URL+"/site.zip",
|
||||
true,
|
||||
)
|
||||
first, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:5")
|
||||
if err != nil || first == nil || first.Deployment == nil {
|
||||
t.Fatalf("syncRemoteSource(seed) = (%+v, %v), want deployment", first, err)
|
||||
}
|
||||
deployment, err := repository.GetPagesDeploymentByID(ctx, first.Deployment.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", first.Deployment.ID, err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.PagesProject{}).
|
||||
Where("id = ?", project.ID).
|
||||
Update("active_deployment_id", nil).Error; err != nil {
|
||||
t.Fatalf("clear active deployment error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
|
||||
Where("id = ?", deployment.ID).
|
||||
Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil {
|
||||
t.Fatalf("reset deployment status error = %v, want nil", err)
|
||||
}
|
||||
|
||||
snapshot := mustAcquireRemoteSyncLease(t, ctx, source)
|
||||
expiresAt := time.Now().Add(time.Hour)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ? AND lease_token = ?", source.ID, snapshot.LeaseToken).
|
||||
Update("lease_expires_at", &expiresAt).Error; err != nil {
|
||||
t.Fatalf("set deterministic lease expiry error = %v, want nil", err)
|
||||
}
|
||||
originalNow := sourceCommitNow
|
||||
nowCalls := 0
|
||||
sourceCommitNow = func() time.Time {
|
||||
nowCalls++
|
||||
if nowCalls == 1 {
|
||||
return expiresAt.Add(-time.Second)
|
||||
}
|
||||
return expiresAt.Add(time.Second)
|
||||
}
|
||||
t.Cleanup(func() { sourceCommitNow = originalNow })
|
||||
|
||||
_, _, _, err = commitSourceDeployment(
|
||||
ctx,
|
||||
snapshot,
|
||||
deployment.Checksum,
|
||||
deployment.Checksum,
|
||||
sourceDetail{Provider: PagesSourceTypeRemoteURL, DisplayName: deployment.SourceLabel},
|
||||
deployment.SourceMeta,
|
||||
"user:5",
|
||||
&deploymentManifest{},
|
||||
ofupload.IngestResult{},
|
||||
false,
|
||||
nil,
|
||||
)
|
||||
if !errors.Is(err, errSourceFinalFence) {
|
||||
t.Fatalf("commitSourceDeployment(expired after upload lock) error = %v, want %v", err, errSourceFinalFence)
|
||||
}
|
||||
if nowCalls != 2 {
|
||||
t.Fatalf("sourceCommitNow calls = %d, want runtime-lock and post-upload checks", nowCalls)
|
||||
}
|
||||
storedProject, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if storedProject.ActiveDeploymentID != nil {
|
||||
t.Errorf("ActiveDeploymentID after expiry recheck = %v, want nil", storedProject.ActiveDeploymentID)
|
||||
}
|
||||
storedDeployment, err := repository.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesDeploymentByID(%d) after expiry error = %v, want nil", deployment.ID, err)
|
||||
}
|
||||
if got, want := storedDeployment.Status, model.PagesDeploymentStatusUploaded; got != want {
|
||||
t.Errorf("deployment status after expiry recheck = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompensateSourceIngestSurvivesCanceledParentContext(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "sync-canceled-compensation")
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip",
|
||||
false,
|
||||
)
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "cancel-compensation"})
|
||||
packagePath := filepath.Join(t.TempDir(), "site.zip")
|
||||
if err := os.WriteFile(packagePath, packageBytes, 0o600); err != nil {
|
||||
t.Fatalf("write test package error = %v, want nil", err)
|
||||
}
|
||||
digest := sha256.Sum256(packageBytes)
|
||||
result, err := ingestPagesDeploymentPackageWithSource(
|
||||
ctx,
|
||||
packagePath,
|
||||
hex.EncodeToString(digest[:]),
|
||||
project.ID,
|
||||
source.ID,
|
||||
"site.zip",
|
||||
pagesarchive.FormatZip,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("ingestPagesDeploymentPackageWithSource() error = %v, want nil", err)
|
||||
}
|
||||
if !result.Created {
|
||||
t.Fatal("ingest result Created = false, want a compensatable record")
|
||||
}
|
||||
|
||||
canceledCtx, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
compensateSourceIngest(canceledCtx, &sourceExecutionSnapshot{
|
||||
ProjectID: project.ID,
|
||||
SourceID: source.ID,
|
||||
}, &sourceIngestState{
|
||||
Result: result,
|
||||
HasIngest: true,
|
||||
})
|
||||
|
||||
var uploadRecord model.Upload
|
||||
if err := db.DB(ctx).Where("id = ?", result.Upload.ID).First(&uploadRecord).Error; err != nil {
|
||||
t.Fatalf("load compensated upload error = %v, want nil", err)
|
||||
}
|
||||
if got, want := uploadRecord.Status, model.UploadStatusDeleted; got != want {
|
||||
t.Errorf("compensated upload status = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPagesSyncFailureState(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
sourceID uint,
|
||||
oldActiveID uint,
|
||||
wantDeploymentCount int64,
|
||||
) {
|
||||
t.Helper()
|
||||
project, err := repository.GetPagesProjectByID(ctx, projectID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", projectID, err)
|
||||
}
|
||||
if project.ActiveDeploymentID == nil || *project.ActiveDeploymentID != oldActiveID {
|
||||
t.Errorf("project %d ActiveDeploymentID = %v, want %d", projectID, project.ActiveDeploymentID, oldActiveID)
|
||||
}
|
||||
var deploymentCount int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
|
||||
Where("project_id = ?", projectID).
|
||||
Count(&deploymentCount).Error; err != nil {
|
||||
t.Fatalf("count project %d deployments error = %v, want nil", projectID, err)
|
||||
}
|
||||
if got, want := deploymentCount, wantDeploymentCount; got != want {
|
||||
t.Errorf("project %d deployment count = %d, want %d", projectID, got, want)
|
||||
}
|
||||
var runtime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil {
|
||||
t.Fatalf("load source %d runtime error = %v, want nil", sourceID, err)
|
||||
}
|
||||
if got, want := runtime.SyncStatus, pagesSourceStatusFailed; got != want {
|
||||
t.Errorf("source %d runtime SyncStatus = %q, want %q", sourceID, got, want)
|
||||
}
|
||||
if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil {
|
||||
t.Errorf("source %d runtime lease = (%q, %v), want cleared", sourceID, runtime.LeaseToken, runtime.LeaseExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommitSourceDeploymentRejectsDeletedTargetUpload(t *testing.T) {
|
||||
ctx := setupPagesSourceSyncTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "sync-deleted-upload")
|
||||
packageBytes := testPagesZip(t, map[string]string{"index.html": "content"})
|
||||
server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil)
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
server.URL+"/site.zip",
|
||||
true,
|
||||
)
|
||||
snapshot := mustAcquireRemoteSyncLease(t, ctx, source)
|
||||
|
||||
// A pre-existing source deployment whose upload was removed must never be
|
||||
// reactivated into a dangling active pointer.
|
||||
identity := source.SourceIdentity
|
||||
revision := strings.Repeat("d", 64)
|
||||
uploadRecord := &model.Upload{
|
||||
ID: 987654321,
|
||||
UserID: 999,
|
||||
FileName: "deleted.zip",
|
||||
FilePath: "deleted.zip",
|
||||
FileSize: 1,
|
||||
MimeType: "application/zip",
|
||||
Extension: "zip",
|
||||
Hash: revision,
|
||||
Type: ofupload.ReservedPagesDeploymentType,
|
||||
Status: model.UploadStatusDeleted,
|
||||
AccessMode: 0,
|
||||
}
|
||||
if err := db.DB(ctx).Create(uploadRecord).Error; err != nil {
|
||||
t.Fatalf("create deleted upload error = %v, want nil", err)
|
||||
}
|
||||
deployment := &model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 1,
|
||||
Checksum: revision,
|
||||
Status: model.PagesDeploymentStatusUploaded,
|
||||
UploadID: uploadRecord.ID,
|
||||
FileCount: 1,
|
||||
TotalSize: 1,
|
||||
CreatedBy: "user:1",
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
SourceIdentity: &identity,
|
||||
SourceRevision: &revision,
|
||||
SourceLabel: "deleted.zip",
|
||||
SourceMeta: `{"provider":"remote_url","display_name":"deleted.zip"}`,
|
||||
TriggerType: pagesSourceTriggerManualSync,
|
||||
}
|
||||
if err := db.DB(ctx).Create(deployment).Error; err != nil {
|
||||
t.Fatalf("create source deployment error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Create(&model.PagesDeploymentFile{
|
||||
DeploymentID: deployment.ID,
|
||||
Path: "index.html",
|
||||
Size: 1,
|
||||
Checksum: revision,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create source deployment file error = %v, want nil", err)
|
||||
}
|
||||
manifest := &deploymentManifest{
|
||||
FileCount: 1,
|
||||
TotalSize: 1,
|
||||
EntryFile: "index.html",
|
||||
}
|
||||
_, _, _, err := commitSourceDeployment(
|
||||
ctx,
|
||||
snapshot,
|
||||
revision,
|
||||
revision,
|
||||
sourceDetail{Provider: PagesSourceTypeRemoteURL, DisplayName: "deleted.zip"},
|
||||
`{"provider":"remote_url","display_name":"deleted.zip"}`,
|
||||
"user:1",
|
||||
manifest,
|
||||
ofupload.IngestResult{},
|
||||
false,
|
||||
nil,
|
||||
)
|
||||
if !errors.Is(err, errSourceFinalFence) {
|
||||
t.Errorf("commitSourceDeployment(deleted upload) error = %v, want %v", err, errSourceFinalFence)
|
||||
}
|
||||
storedProject, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if storedProject.ActiveDeploymentID != nil {
|
||||
t.Errorf("ActiveDeploymentID after deleted upload rejection = %v, want nil", storedProject.ActiveDeploymentID)
|
||||
}
|
||||
var activeCount int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).
|
||||
Where("project_id = ? AND status = ?", project.ID, model.PagesDeploymentStatusActive).
|
||||
Count(&activeCount).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
t.Fatalf("count active deployments error = %v, want nil", err)
|
||||
}
|
||||
if got, want := activeCount, int64(0); got != want {
|
||||
t.Errorf("active deployment count after deleted upload rejection = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
"Wavelet/openflare/plugins/server/kernel/task"
|
||||
"Wavelet/pkg/logger"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
// PagesSourceActionTask is the private Asynq task type for source actions.
|
||||
PagesSourceActionTask = "openflare:pages_source_action"
|
||||
// TaskTypePagesSourceAction is the internal task meta type.
|
||||
TaskTypePagesSourceAction = "of_pages_source_action"
|
||||
|
||||
sourceActionCheck = "check"
|
||||
sourceActionSync = "sync"
|
||||
)
|
||||
|
||||
var errUnexpectedJSONTrailingValue = errors.New("unexpected trailing JSON value")
|
||||
|
||||
// PagesSourceActionMeta describes check/sync work for Pages deployment sources.
|
||||
var PagesSourceActionMeta = task.TaskMeta{
|
||||
Type: TaskTypePagesSourceAction,
|
||||
AsynqTask: PagesSourceActionTask,
|
||||
Name: "OpenFlare Pages 部署源操作",
|
||||
Description: "检查或同步 Pages 项目部署源",
|
||||
SupportsTime: false,
|
||||
MaxRetry: 2,
|
||||
Queue: task.QueueDefault,
|
||||
Retryable: false,
|
||||
}
|
||||
|
||||
// SourceActionPayload is the credential-free internal queue contract.
|
||||
type SourceActionPayload struct {
|
||||
SourceID uint `json:"source_id"`
|
||||
ConfigVersion int `json:"config_version"`
|
||||
Action string `json:"action"`
|
||||
Actor string `json:"actor"`
|
||||
TriggerType string `json:"trigger_type"`
|
||||
TargetRevision string `json:"target_revision"`
|
||||
ConfirmedRevision string `json:"confirmed_revision"`
|
||||
}
|
||||
|
||||
// SourceActionHandler executes a validated source action.
|
||||
type SourceActionHandler struct{}
|
||||
|
||||
// ValidatePayload rejects unknown keys and normalizes the internal contract.
|
||||
func (h *SourceActionHandler) ValidatePayload(payload []byte) ([]byte, error) {
|
||||
var input SourceActionPayload
|
||||
decoder := json.NewDecoder(bytes.NewReader(payload))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&input); err != nil {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
if err := ensureJSONEOF(decoder); err != nil {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
input.Action = strings.TrimSpace(input.Action)
|
||||
input.Actor = strings.TrimSpace(input.Actor)
|
||||
input.TriggerType = strings.TrimSpace(input.TriggerType)
|
||||
input.TargetRevision = strings.TrimSpace(input.TargetRevision)
|
||||
input.ConfirmedRevision = strings.TrimSpace(input.ConfirmedRevision)
|
||||
if input.Action == sourceActionSync && input.TriggerType == "" {
|
||||
// Keep already queued Phase 2 payloads valid while making every new
|
||||
// dispatch carry an explicit deployment trigger.
|
||||
input.TriggerType = pagesSourceTriggerManualSync
|
||||
}
|
||||
if !validSourceActionPayload(input) {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
return json.Marshal(input)
|
||||
}
|
||||
|
||||
func validSourceActionPayload(input SourceActionPayload) bool {
|
||||
if input.SourceID == 0 || input.ConfigVersion <= 0 {
|
||||
return false
|
||||
}
|
||||
if input.Action != sourceActionCheck && input.Action != sourceActionSync {
|
||||
return false
|
||||
}
|
||||
if !validPagesSourceActor(input.Actor) {
|
||||
return false
|
||||
}
|
||||
if !validOptionalSourceRevision(input.TargetRevision) ||
|
||||
!validOptionalSourceRevision(input.ConfirmedRevision) {
|
||||
return false
|
||||
}
|
||||
if input.Action == sourceActionCheck {
|
||||
return input.TriggerType == "" && input.TargetRevision == "" && input.ConfirmedRevision == ""
|
||||
}
|
||||
return validSourceSyncPayload(input)
|
||||
}
|
||||
|
||||
func validSourceSyncPayload(input SourceActionPayload) bool {
|
||||
if !validSourceDeploymentTrigger(input.TriggerType) ||
|
||||
(input.TargetRevision != "" && input.ConfirmedRevision != "") {
|
||||
return false
|
||||
}
|
||||
switch input.TriggerType {
|
||||
case pagesSourceTriggerScheduledAutoUpdate:
|
||||
return input.Actor == pagesSourceCreatedBySystem &&
|
||||
input.TargetRevision != "" && input.ConfirmedRevision == ""
|
||||
case pagesSourceTriggerManualSync:
|
||||
if input.TargetRevision != "" || !strings.HasPrefix(input.Actor, "user:") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Execute validates again inside the worker and performs the source action.
|
||||
func (h *SourceActionHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
|
||||
normalized, err := h.ValidatePayload(payload)
|
||||
if err != nil {
|
||||
return nil, task.PermanentError(errPagesSourceActionInvalid)
|
||||
}
|
||||
var input SourceActionPayload
|
||||
if err := json.Unmarshal(normalized, &input); err != nil {
|
||||
return nil, task.PermanentError(errPagesSourceActionInvalid)
|
||||
}
|
||||
|
||||
source, err := repository.GetPagesProjectSourceByID(ctx, input.SourceID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
task.AppendLog(ctx, "[resolve] 来源已不存在，本次任务跳过")
|
||||
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
|
||||
}
|
||||
logger.ErrorF(ctx, "[PagesSource] load source failed: source_id=%d error=%v", input.SourceID, err)
|
||||
return nil, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
if source.ConfigVersion != input.ConfigVersion {
|
||||
task.AppendLog(ctx, "[resolve] 来源配置已变化，本次任务跳过")
|
||||
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
|
||||
}
|
||||
if input.Action == sourceActionCheck && source.SourceType == PagesSourceTypeRemoteURL {
|
||||
return nil, task.PermanentError(errPagesSourceCheckUnsupported)
|
||||
}
|
||||
if source.SourceType != PagesSourceTypeRemoteURL && source.SourceType != PagesSourceTypeGitHubRelease {
|
||||
return nil, task.PermanentError(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
if (source.SourceType == PagesSourceTypeRemoteURL || input.Action == sourceActionCheck) &&
|
||||
(input.TargetRevision != "" || input.ConfirmedRevision != "") {
|
||||
return nil, task.PermanentError(errPagesSourceActionInvalid)
|
||||
}
|
||||
|
||||
task.AppendLog(ctx, "[resolve] 正在获取来源执行权")
|
||||
snapshot, outcome, err := acquireSourceLease(ctx, input.SourceID, input.ConfigVersion, input.Action)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[PagesSource] acquire lease failed: source_id=%d error=%v", input.SourceID, err)
|
||||
return nil, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
switch outcome {
|
||||
case sourceLeaseBusy:
|
||||
task.AppendLog(ctx, "[resolve] 已有来源任务正在执行，本次任务跳过")
|
||||
return &task.TaskResult{Message: errPagesSourceActionBusy}, nil
|
||||
case sourceLeaseStale:
|
||||
task.AppendLog(ctx, "[resolve] 来源配置或执行权已变化，本次任务跳过")
|
||||
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
|
||||
case sourceLeaseAcquired:
|
||||
// 已获取执行权，继续执行。
|
||||
}
|
||||
|
||||
if input.Action == sourceActionCheck {
|
||||
return executeGitHubCheckAction(ctx, snapshot)
|
||||
}
|
||||
return executeSourceSyncAction(ctx, source, snapshot, input)
|
||||
}
|
||||
|
||||
func executeGitHubCheckAction(ctx context.Context, snapshot *sourceExecutionSnapshot) (*task.TaskResult, error) {
|
||||
checkResult, checkErr := checkGitHubSource(ctx, snapshot)
|
||||
if checkErr != nil {
|
||||
logger.ErrorF(ctx, "[PagesSource] check failed: project_id=%d source_id=%d error=%v", snapshot.ProjectID, snapshot.SourceID, checkErr)
|
||||
if isPermanentSourceSyncError(checkErr) || shouldSkipGitHubActionRetry(checkErr) {
|
||||
return nil, task.PermanentError(checkErr.Error())
|
||||
}
|
||||
return nil, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
if checkResult == nil || checkResult.Stale {
|
||||
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
|
||||
}
|
||||
return &task.TaskResult{Message: checkResult.Message, Detail: checkResult.Detail}, nil
|
||||
}
|
||||
|
||||
func executeSourceSyncAction(
|
||||
ctx context.Context,
|
||||
source *model.PagesProjectSource,
|
||||
snapshot *sourceExecutionSnapshot,
|
||||
input SourceActionPayload,
|
||||
) (*task.TaskResult, error) {
|
||||
var result *sourceSyncOutcome
|
||||
var err error
|
||||
if source.SourceType == PagesSourceTypeGitHubRelease {
|
||||
result, err = syncGitHubSourceWithTrigger(
|
||||
ctx, snapshot, input.Actor, input.TargetRevision, input.ConfirmedRevision, input.TriggerType,
|
||||
)
|
||||
} else {
|
||||
result, err = syncRemoteSourceWithTrigger(ctx, snapshot, input.Actor, input.TriggerType)
|
||||
}
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[PagesSource] sync failed: project_id=%d source_id=%d error=%v", snapshot.ProjectID, snapshot.SourceID, err)
|
||||
if isPermanentSourceSyncError(err) || shouldSkipGitHubActionRetry(err) {
|
||||
return nil, task.PermanentError(errPagesSourceSyncFailed)
|
||||
}
|
||||
return nil, errors.New(errPagesSourceSyncFailed)
|
||||
}
|
||||
if result == nil || result.Stale {
|
||||
task.AppendLog(ctx, "[activate] 来源配置或执行权已变化，本次任务未切换部署")
|
||||
return &task.TaskResult{Message: errPagesSourceActionStale}, nil
|
||||
}
|
||||
message := "Pages 部署源同步并发布成功"
|
||||
if result.Reused {
|
||||
message = "Pages 部署源内容未变化，已重新激活现有部署"
|
||||
}
|
||||
return &task.TaskResult{Message: message, Detail: sourceSyncResultDetail(result)}, nil
|
||||
}
|
||||
|
||||
func ensureJSONEOF(decoder *json.Decoder) error {
|
||||
var trailing any
|
||||
err := decoder.Decode(&trailing)
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
if err == nil {
|
||||
return errUnexpectedJSONTrailingValue
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func validPagesSourceActor(actor string) bool {
|
||||
if actor == pagesSourceCreatedBySystem {
|
||||
return true
|
||||
}
|
||||
if !strings.HasPrefix(actor, "user:") {
|
||||
return false
|
||||
}
|
||||
id, err := strconv.ParseUint(strings.TrimPrefix(actor, "user:"), 10, 64)
|
||||
return err == nil && id > 0
|
||||
}
|
||||
|
||||
func validOptionalSourceRevision(value string) bool {
|
||||
if value == "" {
|
||||
return true
|
||||
}
|
||||
if len(value) != sourceRevisionHexLength {
|
||||
return false
|
||||
}
|
||||
decoded, err := hex.DecodeString(value)
|
||||
return err == nil && len(decoded) == 32
|
||||
}
|
||||
|
||||
func isPermanentSourceSyncError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
message := err.Error()
|
||||
return strings.Contains(message, errPagesPackageUnsupported) ||
|
||||
strings.Contains(message, errPagesPackageURLTooLarge) ||
|
||||
strings.Contains(message, errPagesPackageInvalid) ||
|
||||
strings.Contains(message, errPagesPackageEmpty) ||
|
||||
strings.Contains(message, errPagesPackageExtractedTooLarge) ||
|
||||
strings.Contains(message, errPagesPackageFileTooLarge) ||
|
||||
strings.Contains(message, errPagesEntryFileMissing) ||
|
||||
strings.Contains(message, errPagesSourceRemoteURLInvalid) ||
|
||||
strings.Contains(message, errPagesSourceReleaseNotFound) ||
|
||||
strings.Contains(message, errPagesSourceDigestInvalid) ||
|
||||
strings.Contains(message, errPagesSourceDigestMismatch) ||
|
||||
strings.Contains(message, errPagesSourceConfirmationNeeded) ||
|
||||
strings.Contains(message, errPagesSourceConfirmationStale)
|
||||
}
|
||||
|
||||
// DispatchSourceAction performs API preflight and enqueues a credential-free action.
|
||||
func DispatchSourceAction(
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
action string,
|
||||
actor string,
|
||||
confirmedRevision string,
|
||||
) (*SourceActionReceipt, error) {
|
||||
return dispatchSourceActionByProject(ctx, projectID, action, actor, "", confirmedRevision)
|
||||
}
|
||||
|
||||
func dispatchSourceActionByProject(
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
action string,
|
||||
actor string,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
) (*SourceActionReceipt, error) {
|
||||
action = strings.TrimSpace(action)
|
||||
targetRevision = strings.TrimSpace(targetRevision)
|
||||
confirmedRevision = strings.TrimSpace(confirmedRevision)
|
||||
if action != sourceActionCheck && action != sourceActionSync {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
if !validPagesSourceActor(actor) {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
|
||||
source, err := repository.GetPagesProjectSourceByProjectID(ctx, projectID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New(errPagesSourceNotFound)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if err := validateSourceActionPreflight(ctx, source, action, targetRevision, confirmedRevision); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
busy, err := sourceLeaseIsBusy(ctx, source.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if busy {
|
||||
return nil, errors.New(errPagesSourceActionBusy)
|
||||
}
|
||||
return dispatchSourceActionSnapshot(ctx, *source, action, actor, targetRevision, confirmedRevision, "manual")
|
||||
}
|
||||
|
||||
func validateSourceActionPreflight(
|
||||
ctx context.Context,
|
||||
source *model.PagesProjectSource,
|
||||
action string,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
) error {
|
||||
if source == nil {
|
||||
return errors.New(errPagesSourceNotFound)
|
||||
}
|
||||
if source.SourceType != PagesSourceTypeRemoteURL && source.SourceType != PagesSourceTypeGitHubRelease {
|
||||
return errors.New(errPagesSourceTypeUnsupported)
|
||||
}
|
||||
if action == sourceActionCheck && source.SourceType == PagesSourceTypeRemoteURL {
|
||||
return errors.New(errPagesSourceCheckUnsupported)
|
||||
}
|
||||
if source.SourceType == PagesSourceTypeRemoteURL && (targetRevision != "" || confirmedRevision != "") {
|
||||
return errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
if action == sourceActionCheck && (targetRevision != "" || confirmedRevision != "") {
|
||||
return errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
if source.SourceType == PagesSourceTypeGitHubRelease && action == sourceActionSync {
|
||||
if err := preflightGitHubSyncConfirmation(ctx, source.ID, confirmedRevision); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dispatchSourceActionSnapshot(
|
||||
ctx context.Context,
|
||||
source model.PagesProjectSource,
|
||||
action string,
|
||||
actor string,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
triggeredBy string,
|
||||
) (*SourceActionReceipt, error) {
|
||||
triggerType := ""
|
||||
if action == sourceActionSync {
|
||||
triggerType = pagesSourceTriggerManualSync
|
||||
}
|
||||
return dispatchSourceActionSnapshotWithTrigger(
|
||||
ctx, source, action, actor, triggerType, targetRevision, confirmedRevision, triggeredBy,
|
||||
)
|
||||
}
|
||||
|
||||
func dispatchSourceActionSnapshotWithTrigger(
|
||||
ctx context.Context,
|
||||
source model.PagesProjectSource,
|
||||
action string,
|
||||
actor string,
|
||||
triggerType string,
|
||||
targetRevision string,
|
||||
confirmedRevision string,
|
||||
triggeredBy string,
|
||||
) (*SourceActionReceipt, error) {
|
||||
handler := &SourceActionHandler{}
|
||||
rawPayload, err := json.Marshal(SourceActionPayload{
|
||||
SourceID: source.ID,
|
||||
ConfigVersion: source.ConfigVersion,
|
||||
Action: action,
|
||||
Actor: actor,
|
||||
TriggerType: triggerType,
|
||||
TargetRevision: targetRevision,
|
||||
ConfirmedRevision: confirmedRevision,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.New(errPagesSourceActionInvalid)
|
||||
}
|
||||
payload, err := handler.ValidatePayload(rawPayload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskID, err := task.DispatchTask(ctx, TaskTypePagesSourceAction, payload, triggeredBy)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[PagesSource] dispatch action failed: project_id=%d source_id=%d action=%s error=%v", source.ProjectID, source.ID, action, err)
|
||||
return nil, errors.New(errPagesSourceTaskDispatchFailed)
|
||||
}
|
||||
execution, err := repository.GetTaskExecutionByTaskID(ctx, taskID)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[PagesSource] load dispatched execution failed: source_id=%d task_id=%s error=%v", source.ID, taskID, err)
|
||||
return nil, errors.New(errPagesSourceTaskDispatchFailed)
|
||||
}
|
||||
return &SourceActionReceipt{
|
||||
TaskID: taskID,
|
||||
ExecutionID: strconv.FormatUint(execution.ID, 10),
|
||||
Action: action,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
func TestSourceActionPayloadValidationIsStrictAndCredentialFree(t *testing.T) {
|
||||
handler := &SourceActionHandler{}
|
||||
valid := SourceActionPayload{
|
||||
SourceID: 7,
|
||||
ConfigVersion: 3,
|
||||
Action: sourceActionSync,
|
||||
Actor: "user:42",
|
||||
TriggerType: pagesSourceTriggerManualSync,
|
||||
}
|
||||
raw, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(valid payload) error = %v, want nil", err)
|
||||
}
|
||||
normalized, err := handler.ValidatePayload(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidatePayload(valid) error = %v, want nil", err)
|
||||
}
|
||||
var got SourceActionPayload
|
||||
if err := json.Unmarshal(normalized, &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal(normalized payload) error = %v, want nil", err)
|
||||
}
|
||||
if got != valid {
|
||||
t.Errorf("ValidatePayload(valid) = %+v, want %+v", got, valid)
|
||||
}
|
||||
legacy := valid
|
||||
legacy.TriggerType = ""
|
||||
legacyRaw, err := json.Marshal(legacy)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(legacy payload) error = %v, want nil", err)
|
||||
}
|
||||
legacyNormalized, err := handler.ValidatePayload(legacyRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidatePayload(legacy payload) error = %v, want nil", err)
|
||||
}
|
||||
var legacyGot SourceActionPayload
|
||||
if err := json.Unmarshal(legacyNormalized, &legacyGot); err != nil {
|
||||
t.Fatalf("json.Unmarshal(legacy normalized payload) error = %v, want nil", err)
|
||||
}
|
||||
if legacyGot.TriggerType != pagesSourceTriggerManualSync {
|
||||
t.Errorf("legacy payload trigger_type = %q, want %q", legacyGot.TriggerType, pagesSourceTriggerManualSync)
|
||||
}
|
||||
for _, forbidden := range []string{"remote_url", "content_config_version", "expected_revision", "lease_token", "etag"} {
|
||||
if strings.Contains(string(normalized), forbidden) {
|
||||
t.Errorf("normalized payload = %s, want no forbidden field %q", normalized, forbidden)
|
||||
}
|
||||
}
|
||||
|
||||
invalidPayloads := []struct {
|
||||
name string
|
||||
raw string
|
||||
}{
|
||||
{
|
||||
name: "unknown remote URL field",
|
||||
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42","remote_url":"https://example.com/site.zip?token=secret"}`,
|
||||
},
|
||||
{
|
||||
name: "unknown content version field",
|
||||
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42","content_config_version":9}`,
|
||||
},
|
||||
{
|
||||
name: "empty actor",
|
||||
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":""}`,
|
||||
},
|
||||
{
|
||||
name: "untrusted system actor",
|
||||
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"system"}`,
|
||||
},
|
||||
{
|
||||
name: "zero user actor",
|
||||
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:0"}`,
|
||||
},
|
||||
{
|
||||
name: "invalid action",
|
||||
raw: `{"source_id":7,"config_version":3,"action":"activate","actor":"user:42"}`,
|
||||
},
|
||||
{
|
||||
name: "multiple JSON values",
|
||||
raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42"} {}`,
|
||||
},
|
||||
}
|
||||
for _, test := range invalidPayloads {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
normalized, err := handler.ValidatePayload([]byte(test.raw))
|
||||
if err == nil {
|
||||
t.Errorf("ValidatePayload(%s) = %s, nil; want non-nil error", test.raw, normalized)
|
||||
}
|
||||
if err != nil && strings.Contains(err.Error(), "secret") {
|
||||
t.Errorf("ValidatePayload(%s) error = %q, want credential-free error", test.name, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceActionPayloadAcceptsOnlyRealActors(t *testing.T) {
|
||||
tests := []struct {
|
||||
actor string
|
||||
want bool
|
||||
}{
|
||||
{actor: "user:1", want: true},
|
||||
{actor: "user:18446744073709551615", want: true},
|
||||
{actor: pagesSourceCreatedBySystem, want: true},
|
||||
{actor: "", want: false},
|
||||
{actor: "user:0", want: false},
|
||||
{actor: "user:-1", want: false},
|
||||
{actor: "user:not-a-number", want: false},
|
||||
{actor: "system", want: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
if got := validPagesSourceActor(test.actor); got != test.want {
|
||||
t.Errorf("validPagesSourceActor(%q) = %t, want %t", test.actor, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteCheckActionIsPermanentWithoutExposingURL(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "task-remote-check")
|
||||
secret := "task-query-secret"
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip?token="+secret,
|
||||
false,
|
||||
)
|
||||
raw, err := json.Marshal(SourceActionPayload{
|
||||
SourceID: source.ID,
|
||||
ConfigVersion: source.ConfigVersion,
|
||||
Action: sourceActionCheck,
|
||||
Actor: "user:9",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(check payload) error = %v, want nil", err)
|
||||
}
|
||||
|
||||
result, err := (&SourceActionHandler{}).Execute(ctx, raw)
|
||||
if result != nil {
|
||||
t.Errorf("SourceActionHandler.Execute(remote check) result = %+v, want nil", result)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("SourceActionHandler.Execute(remote check) error = nil, want permanent error")
|
||||
}
|
||||
if !errors.Is(err, asynq.SkipRetry) {
|
||||
t.Errorf("SourceActionHandler.Execute(remote check) error = %v, want errors.Is(asynq.SkipRetry)", err)
|
||||
}
|
||||
if got, want := err.Error(), errPagesSourceCheckUnsupported; got != want {
|
||||
t.Errorf("SourceActionHandler.Execute(remote check) error = %q, want %q", got, want)
|
||||
}
|
||||
if strings.Contains(err.Error(), secret) || strings.Contains(string(raw), secret) {
|
||||
t.Errorf("remote check result error/payload = %q / %s, want no URL secret", err, raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagesSourceActionMeta(t *testing.T) {
|
||||
if PagesSourceActionMeta.InternalOnly {
|
||||
t.Error("PagesSourceActionMeta.InternalOnly = true, want false")
|
||||
}
|
||||
if PagesSourceActionMeta.Type != TaskTypePagesSourceAction {
|
||||
t.Errorf("PagesSourceActionMeta.Type = %q, want %q", PagesSourceActionMeta.Type, TaskTypePagesSourceAction)
|
||||
}
|
||||
if PagesSourceActionMeta.AsynqTask != PagesSourceActionTask {
|
||||
t.Errorf("PagesSourceActionMeta.AsynqTask = %q, want %q", PagesSourceActionMeta.AsynqTask, PagesSourceActionTask)
|
||||
}
|
||||
if PagesSourceActionMeta.Retryable {
|
||||
t.Error("PagesSourceActionMeta.Retryable = true, want false for manual retry API")
|
||||
}
|
||||
if PagesSourceActionMeta.MaxRetry <= 0 {
|
||||
t.Errorf("PagesSourceActionMeta.MaxRetry = %d, want bounded transient retries", PagesSourceActionMeta.MaxRetry)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/repository"
|
||||
|
||||
"Wavelet/openflare/plugins/server/kernel/model"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
func setupPagesSourceTest(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
cleanup := setupPagesTestDB(t)
|
||||
t.Cleanup(cleanup)
|
||||
sqlDB, err := db.DB(t.Context()).DB()
|
||||
if err != nil {
|
||||
t.Fatalf("db.DB().DB() error = %v, want nil", err)
|
||||
}
|
||||
// SQLite :memory: is scoped to one connection. Keeping one connection also
|
||||
// makes lease tests exercise the production CAS without creating empty
|
||||
// per-connection databases.
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
return t.Context()
|
||||
}
|
||||
|
||||
func TestRevisionViewReadsLegacySourceDetailLabel(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
detail string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "remote",
|
||||
detail: `{"provider":"remote_url","label":"legacy.zip"}`,
|
||||
want: "legacy.zip",
|
||||
},
|
||||
{
|
||||
name: "github",
|
||||
detail: `{"provider":"github","label":"v1.2.3","asset_name":"dist.zip"}`,
|
||||
want: "v1.2.3",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
view := revisionView(strings.Repeat("a", 64), test.detail)
|
||||
if view.Label != test.want {
|
||||
t.Errorf("revisionView(%s).Label = %q, want %q", test.name, view.Label, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mustCreatePagesSourceProject(t *testing.T, ctx context.Context, slug string) *model.PagesProject {
|
||||
t.Helper()
|
||||
view, err := CreateProject(ctx, Input{
|
||||
Name: "Source " + slug,
|
||||
Slug: slug,
|
||||
Enabled: true,
|
||||
EntryFile: "index.html",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateProject(%q) error = %v, want nil", slug, err)
|
||||
}
|
||||
project, err := repository.GetPagesProjectByID(ctx, view.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", view.ID, err)
|
||||
}
|
||||
return project
|
||||
}
|
||||
|
||||
func mustConfigureRemoteSource(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
projectID uint,
|
||||
remoteURL string,
|
||||
allowInsecure bool,
|
||||
) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime) {
|
||||
t.Helper()
|
||||
_, err := UpdateSource(ctx, projectID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: remoteURL,
|
||||
AllowInsecure: allowInsecure,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateSource(%d, %q) error = %v, want nil", projectID, remoteURL, err)
|
||||
}
|
||||
var source model.PagesProjectSource
|
||||
if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil {
|
||||
t.Fatalf("load source for project %d error = %v, want nil", projectID, err)
|
||||
}
|
||||
var runtime model.PagesProjectSourceRuntime
|
||||
if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil {
|
||||
t.Fatalf("load runtime for source %d error = %v, want nil", source.ID, err)
|
||||
}
|
||||
return &source, &runtime
|
||||
}
|
||||
|
||||
func TestValidateRemoteSourceInputRejectsModeIncompatibleFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input SourceUpdateInput
|
||||
}{
|
||||
{
|
||||
name: "missing source type",
|
||||
input: SourceUpdateInput{
|
||||
RemoteURL: "https://example.com/site.zip",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "remote rejects repository field",
|
||||
input: SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: "https://example.com/site.zip",
|
||||
RepositoryURL: "https://github.com/example/site",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "remote rejects automatic updates",
|
||||
input: SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: "https://example.com/site.zip",
|
||||
AutoUpdateEnabled: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing remote url",
|
||||
input: SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := validateRemoteSourceInput(test.input); err == nil {
|
||||
t.Errorf("validateRemoteSourceInput(%+v) error = nil, want non-nil", test.input)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateSourceNewRemoteRequiresExplicitURL(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "remote-requires-url")
|
||||
_, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("UpdateSource(new remote without URL) error = nil, want non-nil")
|
||||
}
|
||||
if got, want := err.Error(), errPagesSourceRemoteURLRequired; got != want {
|
||||
t.Errorf("UpdateSource(new remote without URL) error = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteSourceCRUDPreservesSecretAndResetsRuntimeByIdentity(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "remote-crud")
|
||||
firstURL := "https://Artifacts.Example.com:443/dist/site.zip?token=first-secret&expires=1"
|
||||
source, runtime := mustConfigureRemoteSource(t, ctx, project.ID, firstURL, false)
|
||||
|
||||
if got, want := source.ConfigVersion, 1; got != want {
|
||||
t.Errorf("new source ConfigVersion = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := runtime.SyncStatus, pagesSourceStatusIdle; got != want {
|
||||
t.Errorf("new runtime SyncStatus = %q, want %q", got, want)
|
||||
}
|
||||
view, err := GetSource(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSource(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if got, want := view.RemoteURL, firstURL; got != want {
|
||||
t.Errorf("GetSource(%d).RemoteURL = %q, want %q", project.ID, got, want)
|
||||
}
|
||||
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: firstURL,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateSource(%d, no-op) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
var unchangedSource model.PagesProjectSource
|
||||
if err := db.DB(ctx).Where("id = ?", source.ID).First(&unchangedSource).Error; err != nil {
|
||||
t.Fatalf("load no-op source error = %v, want nil", err)
|
||||
}
|
||||
if got, want := unchangedSource.ConfigVersion, source.ConfigVersion; got != want {
|
||||
t.Errorf("no-op source ConfigVersion = %d, want unchanged %d", got, want)
|
||||
}
|
||||
|
||||
seenRevision := strings.Repeat("a", 64)
|
||||
appliedRevision := strings.Repeat("b", 64)
|
||||
future := time.Now().Add(time.Hour)
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
|
||||
Where("source_id = ?", source.ID).
|
||||
Updates(map[string]any{
|
||||
"last_seen_revision": seenRevision,
|
||||
"last_seen_detail": `{"provider":"remote_url","display_name":"new.zip"}`,
|
||||
"last_applied_revision": appliedRevision,
|
||||
"last_applied_detail": `{"provider":"remote_url","display_name":"old.zip"}`,
|
||||
"sync_status": pagesSourceStatusSyncing,
|
||||
"lease_token": "in-flight",
|
||||
"lease_expires_at": &future,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed source runtime error = %v, want nil", err)
|
||||
}
|
||||
|
||||
// Keep the same URL while enabling insecure TLS. The identity and cursor must
|
||||
// survive, while the in-flight lease is fenced.
|
||||
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: firstURL,
|
||||
AllowInsecure: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateSource(%d, preserve URL) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
preservedSource, preservedRuntime, err := loadSourceByProject(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("loadSourceByProject(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if got, want := preservedSource.RemoteURL, firstURL; got != want {
|
||||
t.Errorf("preserved RemoteURL = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := preservedSource.ConfigVersion, 2; got != want {
|
||||
t.Errorf("preserved source ConfigVersion = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := preservedSource.SourceIdentity, source.SourceIdentity; got != want {
|
||||
t.Errorf("preserved source identity = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := preservedRuntime.LastSeenRevision, seenRevision; got != want {
|
||||
t.Errorf("preserved LastSeenRevision = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := preservedRuntime.SyncStatus, pagesSourceStatusUpdateAvailable; got != want {
|
||||
t.Errorf("preserved runtime SyncStatus = %q, want %q", got, want)
|
||||
}
|
||||
if preservedRuntime.LeaseToken != "" || preservedRuntime.LeaseExpiresAt != nil {
|
||||
t.Errorf("preserved runtime lease = (%q, %v), want cleared", preservedRuntime.LeaseToken, preservedRuntime.LeaseExpiresAt)
|
||||
}
|
||||
|
||||
// Replacing only the query secret keeps the canonical identity and cursors.
|
||||
queryReplacementURL := "https://artifacts.example.com/dist/site.zip?token=second-secret"
|
||||
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: queryReplacementURL,
|
||||
AllowInsecure: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateSource(%d, query replacement) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
querySource, queryRuntime, err := loadSourceByProject(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("loadSourceByProject(%d) after query replacement error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if got, want := querySource.SourceIdentity, source.SourceIdentity; got != want {
|
||||
t.Errorf("query replacement identity = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := queryRuntime.LastSeenRevision, seenRevision; got != want {
|
||||
t.Errorf("query replacement LastSeenRevision = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
// Replacing the path changes identity and clears all remote cursors.
|
||||
pathReplacementURL := "https://artifacts.example.com/dist/other.zip?token=third-secret"
|
||||
if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
RemoteURL: pathReplacementURL,
|
||||
AllowInsecure: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("UpdateSource(%d, path replacement) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
pathSource, pathRuntime, err := loadSourceByProject(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("loadSourceByProject(%d) after path replacement error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if pathSource.SourceIdentity == source.SourceIdentity {
|
||||
t.Errorf("path replacement identity = %q, want a new identity", pathSource.SourceIdentity)
|
||||
}
|
||||
if pathRuntime.LastSeenRevision != "" || pathRuntime.LastAppliedRevision != "" {
|
||||
t.Errorf("path replacement cursors = (%q, %q), want empty", pathRuntime.LastSeenRevision, pathRuntime.LastAppliedRevision)
|
||||
}
|
||||
if got, want := pathRuntime.SyncStatus, pagesSourceStatusIdle; got != want {
|
||||
t.Errorf("path replacement SyncStatus = %q, want %q", got, want)
|
||||
}
|
||||
pathView, err := GetSource(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSource(%d) after path replacement error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if got, want := pathView.RemoteURL, pathReplacementURL; got != want {
|
||||
t.Errorf("GetSource(%d).RemoteURL = %q, want %q", project.ID, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteSourceIdentityIgnoresQueryAndNormalizesDefaultPort(t *testing.T) {
|
||||
first, err := parseRemoteSourceURL("HTTPS://Artifacts.Example.com:443/dist/../dist/site.zip?token=one")
|
||||
if err != nil {
|
||||
t.Fatalf("parseRemoteSourceURL(first) error = %v, want nil", err)
|
||||
}
|
||||
second, err := parseRemoteSourceURL("https://artifacts.example.com/dist/site.zip?token=two")
|
||||
if err != nil {
|
||||
t.Fatalf("parseRemoteSourceURL(second) error = %v, want nil", err)
|
||||
}
|
||||
if got, want := remoteSourceIdentity(first), remoteSourceIdentity(second); got != want {
|
||||
t.Errorf("remoteSourceIdentity(first) = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSourceIsIdempotentAndKeepsDeploymentState(t *testing.T) {
|
||||
ctx := setupPagesSourceTest(t)
|
||||
project := mustCreatePagesSourceProject(t, ctx, "source-delete")
|
||||
source, _ := mustConfigureRemoteSource(
|
||||
t,
|
||||
ctx,
|
||||
project.ID,
|
||||
"https://example.com/site.zip?token=delete-secret",
|
||||
false,
|
||||
)
|
||||
deployment := &model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 1,
|
||||
Checksum: strings.Repeat("c", 64),
|
||||
Status: model.PagesDeploymentStatusActive,
|
||||
CreatedBy: "user:1",
|
||||
SourceType: "manual_upload",
|
||||
TriggerType: "manual_upload",
|
||||
}
|
||||
if err := db.DB(ctx).Create(deployment).Error; err != nil {
|
||||
t.Fatalf("create deployment error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.PagesProject{}).
|
||||
Where("id = ?", project.ID).
|
||||
Update("active_deployment_id", deployment.ID).Error; err != nil {
|
||||
t.Fatalf("set active deployment error = %v, want nil", err)
|
||||
}
|
||||
|
||||
for attempt := 1; attempt <= 2; attempt++ {
|
||||
view, err := DeleteSource(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteSource(%d), attempt %d error = %v, want nil", project.ID, attempt, err)
|
||||
}
|
||||
if got, want := view.SourceType, PagesSourceTypeManual; got != want {
|
||||
t.Errorf("DeleteSource(%d), attempt %d SourceType = %q, want %q", project.ID, attempt, got, want)
|
||||
}
|
||||
}
|
||||
var sourceCount, runtimeCount, deploymentCount int64
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSource{}).Where("id = ?", source.ID).Count(&sourceCount).Error; err != nil {
|
||||
t.Fatalf("count source error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).Where("source_id = ?", source.ID).Count(&runtimeCount).Error; err != nil {
|
||||
t.Fatalf("count runtime error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("id = ?", deployment.ID).Count(&deploymentCount).Error; err != nil {
|
||||
t.Fatalf("count deployment error = %v, want nil", err)
|
||||
}
|
||||
if sourceCount != 0 || runtimeCount != 0 || deploymentCount != 1 {
|
||||
t.Errorf("DeleteSource counts = source:%d runtime:%d deployment:%d, want 0, 0, 1", sourceCount, runtimeCount, deploymentCount)
|
||||
}
|
||||
storedProject, err := repository.GetPagesProjectByID(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != deployment.ID {
|
||||
t.Errorf("active deployment = %v, want %d", storedProject.ActiveDeploymentID, deployment.ID)
|
||||
}
|
||||
|
||||
manual, err := GetSource(ctx, project.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSource(%d) after delete error = %v, want nil", project.ID, err)
|
||||
}
|
||||
if got, want := fmt.Sprint(manual.SourceType), PagesSourceTypeManual; got != want {
|
||||
t.Errorf("GetSource(%d).SourceType = %q, want %q", project.ID, got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user