mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-09-29 05:56:38 +08:00
fix(openflare): Pages 部署包经 upload 存储下载
Agent 下载 Pages 包时统一通过 upload_id 走文件存储 API;legacy artifact_path 仅用于一次性回填 upload 并清空路径。部署视图暴露 upload_id,并补充回归测试与 changelog。
This commit is contained in:
@@ -18,6 +18,8 @@ sidebar: false
|
||||
|
||||
### 修复
|
||||
|
||||
- 修复 Agent 启动时 Pages 部署包下载失败:Pages 部署包统一下载走 upload 文件存储框架,部署记录持久化 `upload_id`,legacy `artifact_path` 仅用于一次性回填 upload。
|
||||
|
||||
- 修复登录 Cap 人机验证:前端 `cap-solver` 与 Cap 路由测试对齐 `b3a55d4` 之后的统一 API 信封 `{ error_msg, data }`,避免 `challenge` 解构失败。
|
||||
|
||||
- 修复 Agent CI/Docker 构建:将 `GeoLite2-Country.mmdb` 提交至仓库作为兜底,构建前优先尝试 `scripts/fetch-agent-geoip-mmdb.sh` 拉取最新库,远程失败时回退使用已提交文件。
|
||||
|
||||
@@ -22,10 +22,13 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
platformstorage "github.com/Rain-kl/Wavelet/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
pagesMaxDeploymentFiles = 1000
|
||||
pagesLegacyArtifactCandidateCapacity = 8
|
||||
pagesLegacyArtifactRootCapacity = 4
|
||||
pagesMaxDeploymentFiles = 1000
|
||||
pagesMaxDeploymentBytes = 100 * 1024 * 1024
|
||||
defaultPagesEntryFile = "index.html"
|
||||
defaultPagesFallbackPath = "/index.html"
|
||||
@@ -212,6 +215,89 @@ func ingestPagesDeploymentPackage(
|
||||
})
|
||||
}
|
||||
|
||||
func legacyArtifactCandidatePaths(ctx context.Context, project *model.PagesProject, deployment *model.PagesDeployment) []string {
|
||||
if deployment == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
candidates := make([]string, 0, pagesLegacyArtifactCandidateCapacity)
|
||||
add := func(raw string) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
return
|
||||
}
|
||||
info, statErr := os.Stat(value)
|
||||
if statErr != nil || info.IsDir() {
|
||||
return
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
candidates = append(candidates, value)
|
||||
}
|
||||
|
||||
storedPath := strings.TrimSpace(deployment.ArtifactPath)
|
||||
add(storedPath)
|
||||
if storedPath != "" {
|
||||
add(filepath.Clean(storedPath))
|
||||
add(strings.ReplaceAll(storedPath, "/data/data/", "/data/"))
|
||||
add(strings.ReplaceAll(filepath.Clean(storedPath), string(filepath.Separator)+string(filepath.Separator), string(filepath.Separator)))
|
||||
}
|
||||
|
||||
slug := ""
|
||||
if project != nil {
|
||||
slug = strings.TrimSpace(project.Slug)
|
||||
}
|
||||
checksum := strings.TrimSpace(deployment.Checksum)
|
||||
if slug != "" && checksum != "" {
|
||||
add(filepath.Join("artifacts", slug, checksum+".zip"))
|
||||
add(filepath.Join("pages", "artifacts", slug, checksum+".zip"))
|
||||
add(filepath.Join("data", "pages", "artifacts", slug, checksum+".zip"))
|
||||
}
|
||||
|
||||
roots := make([]string, 0, pagesLegacyArtifactRootCapacity)
|
||||
if cfg, err := platformstorage.LoadConfig(ctx); err == nil {
|
||||
root := strings.TrimSpace(cfg.Local.Root)
|
||||
if root != "" {
|
||||
roots = append(roots, root)
|
||||
}
|
||||
}
|
||||
for _, root := range roots {
|
||||
if storedPath != "" && !filepath.IsAbs(storedPath) {
|
||||
add(filepath.Join(root, storedPath))
|
||||
}
|
||||
if slug != "" && checksum != "" {
|
||||
add(filepath.Join(root, "artifacts", slug, checksum+".zip"))
|
||||
add(filepath.Join(root, "pages", "artifacts", slug, checksum+".zip"))
|
||||
add(filepath.Join(root, "data", "pages", "artifacts", slug, checksum+".zip"))
|
||||
}
|
||||
}
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
func openLegacyDeploymentArtifact(ctx context.Context, project *model.PagesProject, deployment *model.PagesDeployment) (string, *os.File, os.FileInfo, error) {
|
||||
for _, candidate := range legacyArtifactCandidatePaths(ctx, project, deployment) {
|
||||
file, err := os.Open(candidate) //nolint:gosec // candidate is resolved from managed legacy artifact metadata
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := file.Stat()
|
||||
if statErr != nil {
|
||||
_ = file.Close()
|
||||
continue
|
||||
}
|
||||
if info.IsDir() {
|
||||
_ = file.Close()
|
||||
continue
|
||||
}
|
||||
return candidate, file, info, nil
|
||||
}
|
||||
return "", nil, nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func removeDeploymentArtifact(ctx context.Context, deployment *model.PagesDeployment) {
|
||||
if deployment == nil {
|
||||
return
|
||||
|
||||
@@ -47,6 +47,7 @@ type DeploymentView struct {
|
||||
DeploymentNumber int `json:"deployment_number"`
|
||||
Checksum string `json:"checksum"`
|
||||
Status string `json:"status"`
|
||||
UploadID uint64 `json:"upload_id,string"`
|
||||
FileCount int `json:"file_count"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
@@ -363,37 +364,91 @@ func OpenDeploymentPackage(ctx context.Context, deploymentID uint) (*storage.Obj
|
||||
return nil, "", err
|
||||
}
|
||||
fileName := fmt.Sprintf("pages-deployment-%d.zip", deployment.ID)
|
||||
if deployment.UploadID == 0 {
|
||||
if err := ensureDeploymentUploadRecord(ctx, deployment); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
return openDeploymentPackageFromUpload(ctx, deployment, fileName)
|
||||
}
|
||||
|
||||
func openDeploymentPackageFromUpload(ctx context.Context, deployment *model.PagesDeployment, fileName string) (*storage.Object, string, error) {
|
||||
uploadRecord, err := repository.GetActiveUploadByID(ctx, deployment.UploadID)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
||||
}
|
||||
return openDeploymentPackageFromUploadRecord(ctx, &uploadRecord, fileName)
|
||||
}
|
||||
|
||||
func openDeploymentPackageFromUploadRecord(ctx context.Context, uploadRecord *model.Upload, fileName string) (*storage.Object, string, error) {
|
||||
obj, err := uploadstorage.OpenStoredObject(ctx, uploadRecord)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
||||
}
|
||||
if obj.ContentType == "" {
|
||||
obj.ContentType = mimeTypeApplicationZip
|
||||
}
|
||||
return obj, fileName, nil
|
||||
}
|
||||
|
||||
func ensureDeploymentUploadRecord(ctx context.Context, deployment *model.PagesDeployment) error {
|
||||
if deployment == nil {
|
||||
return errors.New(errPagesDeploymentNotFound)
|
||||
}
|
||||
if deployment.UploadID > 0 {
|
||||
return nil
|
||||
}
|
||||
project, err := model.GetPagesProjectByID(ctx, deployment.ProjectID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
artifactPath, _, info, err := openLegacyDeploymentArtifact(ctx, project, deployment)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pages 部署包不存在: %w", err)
|
||||
}
|
||||
if _, err := hydrateLegacyDeploymentUpload(ctx, deployment, project, artifactPath, info.Size()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func hydrateLegacyDeploymentUpload(
|
||||
ctx context.Context,
|
||||
deployment *model.PagesDeployment,
|
||||
project *model.PagesProject,
|
||||
artifactPath string,
|
||||
size int64,
|
||||
) (*model.Upload, error) {
|
||||
if deployment == nil || project == nil || strings.TrimSpace(artifactPath) == "" {
|
||||
return nil, errors.New(errPagesPackagePathEmpty)
|
||||
}
|
||||
if deployment.UploadID > 0 {
|
||||
uploadRecord, err := repository.GetActiveUploadByID(ctx, deployment.UploadID)
|
||||
if err != nil {
|
||||
return nil, "", errors.New(errPagesPackageUploadMissing)
|
||||
if err == nil {
|
||||
return &uploadRecord, nil
|
||||
}
|
||||
obj, err := uploadstorage.OpenStoredObject(ctx, &uploadRecord)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
||||
}
|
||||
if obj.ContentType == "" {
|
||||
obj.ContentType = mimeTypeApplicationZip
|
||||
}
|
||||
return obj, fileName, nil
|
||||
}
|
||||
if strings.TrimSpace(deployment.ArtifactPath) == "" {
|
||||
return nil, "", errors.New(errPagesPackagePathEmpty)
|
||||
}
|
||||
file, err := os.Open(deployment.ArtifactPath)
|
||||
|
||||
ingestResult, err := ingestPagesDeploymentPackage(
|
||||
ctx,
|
||||
artifactPath,
|
||||
deployment.Checksum,
|
||||
size,
|
||||
project.Slug,
|
||||
fmt.Sprintf("pages-deployment-%d.zip", deployment.ID),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
||||
if err := db.DB(ctx).Model(deployment).Updates(map[string]any{
|
||||
"upload_id": ingestResult.Upload.ID,
|
||||
"artifact_path": "",
|
||||
}).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &storage.Object{
|
||||
Body: file,
|
||||
ContentLength: info.Size(),
|
||||
ContentType: mimeTypeApplicationZip,
|
||||
}, fileName, nil
|
||||
deployment.UploadID = ingestResult.Upload.ID
|
||||
deployment.ArtifactPath = ""
|
||||
return &ingestResult.Upload, nil
|
||||
}
|
||||
|
||||
func ensureDeploymentInActiveSnapshot(ctx context.Context, deploymentID uint) error {
|
||||
@@ -584,6 +639,7 @@ func buildDeploymentView(deployment *model.PagesDeployment) DeploymentView {
|
||||
DeploymentNumber: deployment.DeploymentNumber,
|
||||
Checksum: deployment.Checksum,
|
||||
Status: deployment.Status,
|
||||
UploadID: deployment.UploadID,
|
||||
FileCount: deployment.FileCount,
|
||||
TotalSize: deployment.TotalSize,
|
||||
CreatedBy: deployment.CreatedBy,
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
@@ -159,6 +160,7 @@ func TestUploadDeploymentStoresPackageInUploadFramework(t *testing.T) {
|
||||
"index.html": "ok",
|
||||
})), "root")
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, deployment.UploadID)
|
||||
|
||||
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
@@ -170,6 +172,86 @@ func TestUploadDeploymentStoresPackageInUploadFramework(t *testing.T) {
|
||||
assert.Equal(t, int64(1), uploadCount)
|
||||
}
|
||||
|
||||
func TestOpenDeploymentPackageHydratesLegacyArtifactPath(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
_, disableStorage := setupPagesStorageMock(t)
|
||||
defer disableStorage()
|
||||
ctx := context.Background()
|
||||
|
||||
project, err := CreateProject(ctx, Input{
|
||||
Name: "Legacy Site",
|
||||
Slug: "openspeedtest",
|
||||
Enabled: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
artifactDir := filepath.Join(t.TempDir(), "pages", "artifacts", project.Slug)
|
||||
require.NoError(t, os.MkdirAll(artifactDir, 0o755))
|
||||
artifactPath := filepath.Join(artifactDir, "legacy-checksum.zip")
|
||||
require.NoError(t, os.WriteFile(artifactPath, testPagesZip(t, map[string]string{"index.html": "legacy"}), 0o644))
|
||||
|
||||
deployment := &model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 1,
|
||||
Checksum: "legacy-checksum",
|
||||
Status: model.PagesDeploymentStatusUploaded,
|
||||
ArtifactPath: artifactPath,
|
||||
FileCount: 1,
|
||||
TotalSize: 10,
|
||||
CreatedBy: "test",
|
||||
}
|
||||
require.NoError(t, db.DB(ctx).Create(deployment).Error)
|
||||
require.NoError(t, db.DB(ctx).Create(&model.PagesDeploymentFile{
|
||||
DeploymentID: deployment.ID,
|
||||
Path: "index.html",
|
||||
Size: 6,
|
||||
Checksum: "legacy-checksum",
|
||||
}).Error)
|
||||
|
||||
_, err = ActivateDeployment(ctx, project.ID, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, db.DB(ctx).Create(&model.ConfigVersion{
|
||||
Version: "v2026-legacy",
|
||||
SnapshotJSON: fmt.Sprintf(`{"routes":[{"upstream_type":"pages","pages_deployment":{"deployment_id":%d}}]}`, deployment.ID),
|
||||
MainConfig: "",
|
||||
RenderedConfig: "",
|
||||
SupportFilesJSON: "[]",
|
||||
Checksum: "legacy-config-checksum",
|
||||
IsActive: true,
|
||||
CreatedBy: "test",
|
||||
}).Error)
|
||||
|
||||
packageObj, fileName, err := OpenDeploymentPackage(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
defer packageObj.Body.Close()
|
||||
assert.Equal(t, fmt.Sprintf("pages-deployment-%d.zip", deployment.ID), fileName)
|
||||
|
||||
body, err := io.ReadAll(packageObj.Body)
|
||||
require.NoError(t, err)
|
||||
reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, reader.File, 1)
|
||||
assert.Equal(t, "index.html", reader.File[0].Name)
|
||||
|
||||
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, storedDeployment.UploadID)
|
||||
assert.Empty(t, storedDeployment.ArtifactPath)
|
||||
|
||||
var uploadCount int64
|
||||
require.NoError(t, db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error)
|
||||
assert.Equal(t, int64(1), uploadCount)
|
||||
|
||||
packageObj2, _, err := OpenDeploymentPackage(ctx, deployment.ID)
|
||||
require.NoError(t, err)
|
||||
defer packageObj2.Body.Close()
|
||||
body2, err := io.ReadAll(packageObj2.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, body, body2)
|
||||
}
|
||||
|
||||
func TestOpenDeploymentPackageRequiresActiveConfigSnapshot(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
Reference in New Issue
Block a user