fix(openflare): Pages 部署包经 upload 存储下载

Agent 下载 Pages 包时统一通过 upload_id 走文件存储 API;legacy
artifact_path 仅用于一次性回填 upload 并清空路径。部署视图暴露
upload_id,并补充回归测试与 changelog。
This commit is contained in:
ryan
2026-06-20 19:21:12 +08:00
parent e077b12328
commit 6c46f5d24f
4 changed files with 251 additions and 25 deletions
+2
View File
@@ -18,6 +18,8 @@ sidebar: false
### 修复
- 修复 Agent 启动时 Pages 部署包下载失败:Pages 部署包统一下载走 upload 文件存储框架,部署记录持久化 `upload_id`,legacy `artifact_path` 仅用于一次性回填 upload。
- 修复登录 Cap 人机验证:前端 `cap-solver` 与 Cap 路由测试对齐 `b3a55d4` 之后的统一 API 信封 `{ error_msg, data }`,避免 `challenge` 解构失败。
- 修复 Agent CI/Docker 构建:将 `GeoLite2-Country.mmdb` 提交至仓库作为兜底,构建前优先尝试 `scripts/fetch-agent-geoip-mmdb.sh` 拉取最新库,远程失败时回退使用已提交文件。
+87 -1
View File
@@ -22,10 +22,13 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
platformstorage "github.com/Rain-kl/Wavelet/internal/storage"
)
const (
pagesMaxDeploymentFiles = 1000
pagesLegacyArtifactCandidateCapacity = 8
pagesLegacyArtifactRootCapacity = 4
pagesMaxDeploymentFiles = 1000
pagesMaxDeploymentBytes = 100 * 1024 * 1024
defaultPagesEntryFile = "index.html"
defaultPagesFallbackPath = "/index.html"
@@ -212,6 +215,89 @@ func ingestPagesDeploymentPackage(
})
}
func legacyArtifactCandidatePaths(ctx context.Context, project *model.PagesProject, deployment *model.PagesDeployment) []string {
if deployment == nil {
return nil
}
seen := make(map[string]struct{})
candidates := make([]string, 0, pagesLegacyArtifactCandidateCapacity)
add := func(raw string) {
value := strings.TrimSpace(raw)
if value == "" {
return
}
if _, ok := seen[value]; ok {
return
}
info, statErr := os.Stat(value)
if statErr != nil || info.IsDir() {
return
}
seen[value] = struct{}{}
candidates = append(candidates, value)
}
storedPath := strings.TrimSpace(deployment.ArtifactPath)
add(storedPath)
if storedPath != "" {
add(filepath.Clean(storedPath))
add(strings.ReplaceAll(storedPath, "/data/data/", "/data/"))
add(strings.ReplaceAll(filepath.Clean(storedPath), string(filepath.Separator)+string(filepath.Separator), string(filepath.Separator)))
}
slug := ""
if project != nil {
slug = strings.TrimSpace(project.Slug)
}
checksum := strings.TrimSpace(deployment.Checksum)
if slug != "" && checksum != "" {
add(filepath.Join("artifacts", slug, checksum+".zip"))
add(filepath.Join("pages", "artifacts", slug, checksum+".zip"))
add(filepath.Join("data", "pages", "artifacts", slug, checksum+".zip"))
}
roots := make([]string, 0, pagesLegacyArtifactRootCapacity)
if cfg, err := platformstorage.LoadConfig(ctx); err == nil {
root := strings.TrimSpace(cfg.Local.Root)
if root != "" {
roots = append(roots, root)
}
}
for _, root := range roots {
if storedPath != "" && !filepath.IsAbs(storedPath) {
add(filepath.Join(root, storedPath))
}
if slug != "" && checksum != "" {
add(filepath.Join(root, "artifacts", slug, checksum+".zip"))
add(filepath.Join(root, "pages", "artifacts", slug, checksum+".zip"))
add(filepath.Join(root, "data", "pages", "artifacts", slug, checksum+".zip"))
}
}
return candidates
}
func openLegacyDeploymentArtifact(ctx context.Context, project *model.PagesProject, deployment *model.PagesDeployment) (string, *os.File, os.FileInfo, error) {
for _, candidate := range legacyArtifactCandidatePaths(ctx, project, deployment) {
file, err := os.Open(candidate) //nolint:gosec // candidate is resolved from managed legacy artifact metadata
if err != nil {
continue
}
info, statErr := file.Stat()
if statErr != nil {
_ = file.Close()
continue
}
if info.IsDir() {
_ = file.Close()
continue
}
return candidate, file, info, nil
}
return "", nil, nil, os.ErrNotExist
}
func removeDeploymentArtifact(ctx context.Context, deployment *model.PagesDeployment) {
if deployment == nil {
return
+80 -24
View File
@@ -47,6 +47,7 @@ type DeploymentView struct {
DeploymentNumber int `json:"deployment_number"`
Checksum string `json:"checksum"`
Status string `json:"status"`
UploadID uint64 `json:"upload_id,string"`
FileCount int `json:"file_count"`
TotalSize int64 `json:"total_size"`
CreatedBy string `json:"created_by"`
@@ -363,37 +364,91 @@ func OpenDeploymentPackage(ctx context.Context, deploymentID uint) (*storage.Obj
return nil, "", err
}
fileName := fmt.Sprintf("pages-deployment-%d.zip", deployment.ID)
if deployment.UploadID == 0 {
if err := ensureDeploymentUploadRecord(ctx, deployment); err != nil {
return nil, "", err
}
}
return openDeploymentPackageFromUpload(ctx, deployment, fileName)
}
func openDeploymentPackageFromUpload(ctx context.Context, deployment *model.PagesDeployment, fileName string) (*storage.Object, string, error) {
uploadRecord, err := repository.GetActiveUploadByID(ctx, deployment.UploadID)
if err != nil {
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
}
return openDeploymentPackageFromUploadRecord(ctx, &uploadRecord, fileName)
}
func openDeploymentPackageFromUploadRecord(ctx context.Context, uploadRecord *model.Upload, fileName string) (*storage.Object, string, error) {
obj, err := uploadstorage.OpenStoredObject(ctx, uploadRecord)
if err != nil {
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
}
if obj.ContentType == "" {
obj.ContentType = mimeTypeApplicationZip
}
return obj, fileName, nil
}
func ensureDeploymentUploadRecord(ctx context.Context, deployment *model.PagesDeployment) error {
if deployment == nil {
return errors.New(errPagesDeploymentNotFound)
}
if deployment.UploadID > 0 {
return nil
}
project, err := model.GetPagesProjectByID(ctx, deployment.ProjectID)
if err != nil {
return err
}
artifactPath, _, info, err := openLegacyDeploymentArtifact(ctx, project, deployment)
if err != nil {
return fmt.Errorf("pages 部署包不存在: %w", err)
}
if _, err := hydrateLegacyDeploymentUpload(ctx, deployment, project, artifactPath, info.Size()); err != nil {
return err
}
return nil
}
func hydrateLegacyDeploymentUpload(
ctx context.Context,
deployment *model.PagesDeployment,
project *model.PagesProject,
artifactPath string,
size int64,
) (*model.Upload, error) {
if deployment == nil || project == nil || strings.TrimSpace(artifactPath) == "" {
return nil, errors.New(errPagesPackagePathEmpty)
}
if deployment.UploadID > 0 {
uploadRecord, err := repository.GetActiveUploadByID(ctx, deployment.UploadID)
if err != nil {
return nil, "", errors.New(errPagesPackageUploadMissing)
if err == nil {
return &uploadRecord, nil
}
obj, err := uploadstorage.OpenStoredObject(ctx, &uploadRecord)
if err != nil {
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
}
if obj.ContentType == "" {
obj.ContentType = mimeTypeApplicationZip
}
return obj, fileName, nil
}
if strings.TrimSpace(deployment.ArtifactPath) == "" {
return nil, "", errors.New(errPagesPackagePathEmpty)
}
file, err := os.Open(deployment.ArtifactPath)
ingestResult, err := ingestPagesDeploymentPackage(
ctx,
artifactPath,
deployment.Checksum,
size,
project.Slug,
fmt.Sprintf("pages-deployment-%d.zip", deployment.ID),
)
if err != nil {
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
return nil, err
}
info, err := file.Stat()
if err != nil {
_ = file.Close()
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
if err := db.DB(ctx).Model(deployment).Updates(map[string]any{
"upload_id": ingestResult.Upload.ID,
"artifact_path": "",
}).Error; err != nil {
return nil, err
}
return &storage.Object{
Body: file,
ContentLength: info.Size(),
ContentType: mimeTypeApplicationZip,
}, fileName, nil
deployment.UploadID = ingestResult.Upload.ID
deployment.ArtifactPath = ""
return &ingestResult.Upload, nil
}
func ensureDeploymentInActiveSnapshot(ctx context.Context, deploymentID uint) error {
@@ -584,6 +639,7 @@ func buildDeploymentView(deployment *model.PagesDeployment) DeploymentView {
DeploymentNumber: deployment.DeploymentNumber,
Checksum: deployment.Checksum,
Status: deployment.Status,
UploadID: deployment.UploadID,
FileCount: deployment.FileCount,
TotalSize: deployment.TotalSize,
CreatedBy: deployment.CreatedBy,
@@ -12,6 +12,7 @@ import (
"mime/multipart"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/Rain-kl/Wavelet/internal/db"
@@ -159,6 +160,7 @@ func TestUploadDeploymentStoresPackageInUploadFramework(t *testing.T) {
"index.html": "ok",
})), "root")
require.NoError(t, err)
assert.NotZero(t, deployment.UploadID)
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
require.NoError(t, err)
@@ -170,6 +172,86 @@ func TestUploadDeploymentStoresPackageInUploadFramework(t *testing.T) {
assert.Equal(t, int64(1), uploadCount)
}
func TestOpenDeploymentPackageHydratesLegacyArtifactPath(t *testing.T) {
cleanup := setupPagesTestDB(t)
defer cleanup()
_, disableStorage := setupPagesStorageMock(t)
defer disableStorage()
ctx := context.Background()
project, err := CreateProject(ctx, Input{
Name: "Legacy Site",
Slug: "openspeedtest",
Enabled: true,
})
require.NoError(t, err)
artifactDir := filepath.Join(t.TempDir(), "pages", "artifacts", project.Slug)
require.NoError(t, os.MkdirAll(artifactDir, 0o755))
artifactPath := filepath.Join(artifactDir, "legacy-checksum.zip")
require.NoError(t, os.WriteFile(artifactPath, testPagesZip(t, map[string]string{"index.html": "legacy"}), 0o644))
deployment := &model.PagesDeployment{
ProjectID: project.ID,
DeploymentNumber: 1,
Checksum: "legacy-checksum",
Status: model.PagesDeploymentStatusUploaded,
ArtifactPath: artifactPath,
FileCount: 1,
TotalSize: 10,
CreatedBy: "test",
}
require.NoError(t, db.DB(ctx).Create(deployment).Error)
require.NoError(t, db.DB(ctx).Create(&model.PagesDeploymentFile{
DeploymentID: deployment.ID,
Path: "index.html",
Size: 6,
Checksum: "legacy-checksum",
}).Error)
_, err = ActivateDeployment(ctx, project.ID, deployment.ID)
require.NoError(t, err)
require.NoError(t, db.DB(ctx).Create(&model.ConfigVersion{
Version: "v2026-legacy",
SnapshotJSON: fmt.Sprintf(`{"routes":[{"upstream_type":"pages","pages_deployment":{"deployment_id":%d}}]}`, deployment.ID),
MainConfig: "",
RenderedConfig: "",
SupportFilesJSON: "[]",
Checksum: "legacy-config-checksum",
IsActive: true,
CreatedBy: "test",
}).Error)
packageObj, fileName, err := OpenDeploymentPackage(ctx, deployment.ID)
require.NoError(t, err)
defer packageObj.Body.Close()
assert.Equal(t, fmt.Sprintf("pages-deployment-%d.zip", deployment.ID), fileName)
body, err := io.ReadAll(packageObj.Body)
require.NoError(t, err)
reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
require.NoError(t, err)
require.Len(t, reader.File, 1)
assert.Equal(t, "index.html", reader.File[0].Name)
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
require.NoError(t, err)
assert.NotZero(t, storedDeployment.UploadID)
assert.Empty(t, storedDeployment.ArtifactPath)
var uploadCount int64
require.NoError(t, db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error)
assert.Equal(t, int64(1), uploadCount)
packageObj2, _, err := OpenDeploymentPackage(ctx, deployment.ID)
require.NoError(t, err)
defer packageObj2.Body.Close()
body2, err := io.ReadAll(packageObj2.Body)
require.NoError(t, err)
assert.Equal(t, body, body2)
}
func TestOpenDeploymentPackageRequiresActiveConfigSnapshot(t *testing.T) {
cleanup := setupPagesTestDB(t)
defer cleanup()