mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-11 01:36:37 +08:00
fix(openflare): Pages 部署包经 upload 存储下载
Agent 下载 Pages 包时统一通过 upload_id 走文件存储 API;legacy artifact_path 仅用于一次性回填 upload 并清空路径。部署视图暴露 upload_id,并补充回归测试与 changelog。
This commit is contained in:
@@ -18,6 +18,8 @@ sidebar: false
|
|||||||
|
|
||||||
### 修复
|
### 修复
|
||||||
|
|
||||||
|
- 修复 Agent 启动时 Pages 部署包下载失败:Pages 部署包统一下载走 upload 文件存储框架,部署记录持久化 `upload_id`,legacy `artifact_path` 仅用于一次性回填 upload。
|
||||||
|
|
||||||
- 修复登录 Cap 人机验证:前端 `cap-solver` 与 Cap 路由测试对齐 `b3a55d4` 之后的统一 API 信封 `{ error_msg, data }`,避免 `challenge` 解构失败。
|
- 修复登录 Cap 人机验证:前端 `cap-solver` 与 Cap 路由测试对齐 `b3a55d4` 之后的统一 API 信封 `{ error_msg, data }`,避免 `challenge` 解构失败。
|
||||||
|
|
||||||
- 修复 Agent CI/Docker 构建:将 `GeoLite2-Country.mmdb` 提交至仓库作为兜底,构建前优先尝试 `scripts/fetch-agent-geoip-mmdb.sh` 拉取最新库,远程失败时回退使用已提交文件。
|
- 修复 Agent CI/Docker 构建:将 `GeoLite2-Country.mmdb` 提交至仓库作为兜底,构建前优先尝试 `scripts/fetch-agent-geoip-mmdb.sh` 拉取最新库,远程失败时回退使用已提交文件。
|
||||||
|
|||||||
@@ -22,10 +22,13 @@ import (
|
|||||||
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||||
"github.com/Rain-kl/Wavelet/internal/model"
|
"github.com/Rain-kl/Wavelet/internal/model"
|
||||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||||
|
platformstorage "github.com/Rain-kl/Wavelet/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
pagesMaxDeploymentFiles = 1000
|
pagesLegacyArtifactCandidateCapacity = 8
|
||||||
|
pagesLegacyArtifactRootCapacity = 4
|
||||||
|
pagesMaxDeploymentFiles = 1000
|
||||||
pagesMaxDeploymentBytes = 100 * 1024 * 1024
|
pagesMaxDeploymentBytes = 100 * 1024 * 1024
|
||||||
defaultPagesEntryFile = "index.html"
|
defaultPagesEntryFile = "index.html"
|
||||||
defaultPagesFallbackPath = "/index.html"
|
defaultPagesFallbackPath = "/index.html"
|
||||||
@@ -212,6 +215,89 @@ func ingestPagesDeploymentPackage(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func legacyArtifactCandidatePaths(ctx context.Context, project *model.PagesProject, deployment *model.PagesDeployment) []string {
|
||||||
|
if deployment == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
candidates := make([]string, 0, pagesLegacyArtifactCandidateCapacity)
|
||||||
|
add := func(raw string) {
|
||||||
|
value := strings.TrimSpace(raw)
|
||||||
|
if value == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, ok := seen[value]; ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
info, statErr := os.Stat(value)
|
||||||
|
if statErr != nil || info.IsDir() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[value] = struct{}{}
|
||||||
|
candidates = append(candidates, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
storedPath := strings.TrimSpace(deployment.ArtifactPath)
|
||||||
|
add(storedPath)
|
||||||
|
if storedPath != "" {
|
||||||
|
add(filepath.Clean(storedPath))
|
||||||
|
add(strings.ReplaceAll(storedPath, "/data/data/", "/data/"))
|
||||||
|
add(strings.ReplaceAll(filepath.Clean(storedPath), string(filepath.Separator)+string(filepath.Separator), string(filepath.Separator)))
|
||||||
|
}
|
||||||
|
|
||||||
|
slug := ""
|
||||||
|
if project != nil {
|
||||||
|
slug = strings.TrimSpace(project.Slug)
|
||||||
|
}
|
||||||
|
checksum := strings.TrimSpace(deployment.Checksum)
|
||||||
|
if slug != "" && checksum != "" {
|
||||||
|
add(filepath.Join("artifacts", slug, checksum+".zip"))
|
||||||
|
add(filepath.Join("pages", "artifacts", slug, checksum+".zip"))
|
||||||
|
add(filepath.Join("data", "pages", "artifacts", slug, checksum+".zip"))
|
||||||
|
}
|
||||||
|
|
||||||
|
roots := make([]string, 0, pagesLegacyArtifactRootCapacity)
|
||||||
|
if cfg, err := platformstorage.LoadConfig(ctx); err == nil {
|
||||||
|
root := strings.TrimSpace(cfg.Local.Root)
|
||||||
|
if root != "" {
|
||||||
|
roots = append(roots, root)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, root := range roots {
|
||||||
|
if storedPath != "" && !filepath.IsAbs(storedPath) {
|
||||||
|
add(filepath.Join(root, storedPath))
|
||||||
|
}
|
||||||
|
if slug != "" && checksum != "" {
|
||||||
|
add(filepath.Join(root, "artifacts", slug, checksum+".zip"))
|
||||||
|
add(filepath.Join(root, "pages", "artifacts", slug, checksum+".zip"))
|
||||||
|
add(filepath.Join(root, "data", "pages", "artifacts", slug, checksum+".zip"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
|
||||||
|
func openLegacyDeploymentArtifact(ctx context.Context, project *model.PagesProject, deployment *model.PagesDeployment) (string, *os.File, os.FileInfo, error) {
|
||||||
|
for _, candidate := range legacyArtifactCandidatePaths(ctx, project, deployment) {
|
||||||
|
file, err := os.Open(candidate) //nolint:gosec // candidate is resolved from managed legacy artifact metadata
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
info, statErr := file.Stat()
|
||||||
|
if statErr != nil {
|
||||||
|
_ = file.Close()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
_ = file.Close()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return candidate, file, info, nil
|
||||||
|
}
|
||||||
|
return "", nil, nil, os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
func removeDeploymentArtifact(ctx context.Context, deployment *model.PagesDeployment) {
|
func removeDeploymentArtifact(ctx context.Context, deployment *model.PagesDeployment) {
|
||||||
if deployment == nil {
|
if deployment == nil {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ type DeploymentView struct {
|
|||||||
DeploymentNumber int `json:"deployment_number"`
|
DeploymentNumber int `json:"deployment_number"`
|
||||||
Checksum string `json:"checksum"`
|
Checksum string `json:"checksum"`
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
|
UploadID uint64 `json:"upload_id,string"`
|
||||||
FileCount int `json:"file_count"`
|
FileCount int `json:"file_count"`
|
||||||
TotalSize int64 `json:"total_size"`
|
TotalSize int64 `json:"total_size"`
|
||||||
CreatedBy string `json:"created_by"`
|
CreatedBy string `json:"created_by"`
|
||||||
@@ -363,37 +364,91 @@ func OpenDeploymentPackage(ctx context.Context, deploymentID uint) (*storage.Obj
|
|||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
fileName := fmt.Sprintf("pages-deployment-%d.zip", deployment.ID)
|
fileName := fmt.Sprintf("pages-deployment-%d.zip", deployment.ID)
|
||||||
|
if deployment.UploadID == 0 {
|
||||||
|
if err := ensureDeploymentUploadRecord(ctx, deployment); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return openDeploymentPackageFromUpload(ctx, deployment, fileName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func openDeploymentPackageFromUpload(ctx context.Context, deployment *model.PagesDeployment, fileName string) (*storage.Object, string, error) {
|
||||||
|
uploadRecord, err := repository.GetActiveUploadByID(ctx, deployment.UploadID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
||||||
|
}
|
||||||
|
return openDeploymentPackageFromUploadRecord(ctx, &uploadRecord, fileName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func openDeploymentPackageFromUploadRecord(ctx context.Context, uploadRecord *model.Upload, fileName string) (*storage.Object, string, error) {
|
||||||
|
obj, err := uploadstorage.OpenStoredObject(ctx, uploadRecord)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
||||||
|
}
|
||||||
|
if obj.ContentType == "" {
|
||||||
|
obj.ContentType = mimeTypeApplicationZip
|
||||||
|
}
|
||||||
|
return obj, fileName, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureDeploymentUploadRecord(ctx context.Context, deployment *model.PagesDeployment) error {
|
||||||
|
if deployment == nil {
|
||||||
|
return errors.New(errPagesDeploymentNotFound)
|
||||||
|
}
|
||||||
|
if deployment.UploadID > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
project, err := model.GetPagesProjectByID(ctx, deployment.ProjectID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
artifactPath, _, info, err := openLegacyDeploymentArtifact(ctx, project, deployment)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("pages 部署包不存在: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := hydrateLegacyDeploymentUpload(ctx, deployment, project, artifactPath, info.Size()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hydrateLegacyDeploymentUpload(
|
||||||
|
ctx context.Context,
|
||||||
|
deployment *model.PagesDeployment,
|
||||||
|
project *model.PagesProject,
|
||||||
|
artifactPath string,
|
||||||
|
size int64,
|
||||||
|
) (*model.Upload, error) {
|
||||||
|
if deployment == nil || project == nil || strings.TrimSpace(artifactPath) == "" {
|
||||||
|
return nil, errors.New(errPagesPackagePathEmpty)
|
||||||
|
}
|
||||||
if deployment.UploadID > 0 {
|
if deployment.UploadID > 0 {
|
||||||
uploadRecord, err := repository.GetActiveUploadByID(ctx, deployment.UploadID)
|
uploadRecord, err := repository.GetActiveUploadByID(ctx, deployment.UploadID)
|
||||||
if err != nil {
|
if err == nil {
|
||||||
return nil, "", errors.New(errPagesPackageUploadMissing)
|
return &uploadRecord, nil
|
||||||
}
|
}
|
||||||
obj, err := uploadstorage.OpenStoredObject(ctx, &uploadRecord)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
|
||||||
}
|
|
||||||
if obj.ContentType == "" {
|
|
||||||
obj.ContentType = mimeTypeApplicationZip
|
|
||||||
}
|
|
||||||
return obj, fileName, nil
|
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(deployment.ArtifactPath) == "" {
|
|
||||||
return nil, "", errors.New(errPagesPackagePathEmpty)
|
ingestResult, err := ingestPagesDeploymentPackage(
|
||||||
}
|
ctx,
|
||||||
file, err := os.Open(deployment.ArtifactPath)
|
artifactPath,
|
||||||
|
deployment.Checksum,
|
||||||
|
size,
|
||||||
|
project.Slug,
|
||||||
|
fmt.Sprintf("pages-deployment-%d.zip", deployment.ID),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
return nil, err
|
||||||
}
|
}
|
||||||
info, err := file.Stat()
|
if err := db.DB(ctx).Model(deployment).Updates(map[string]any{
|
||||||
if err != nil {
|
"upload_id": ingestResult.Upload.ID,
|
||||||
_ = file.Close()
|
"artifact_path": "",
|
||||||
return nil, "", fmt.Errorf("pages 部署包不存在: %w", err)
|
}).Error; err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
return &storage.Object{
|
deployment.UploadID = ingestResult.Upload.ID
|
||||||
Body: file,
|
deployment.ArtifactPath = ""
|
||||||
ContentLength: info.Size(),
|
return &ingestResult.Upload, nil
|
||||||
ContentType: mimeTypeApplicationZip,
|
|
||||||
}, fileName, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureDeploymentInActiveSnapshot(ctx context.Context, deploymentID uint) error {
|
func ensureDeploymentInActiveSnapshot(ctx context.Context, deploymentID uint) error {
|
||||||
@@ -584,6 +639,7 @@ func buildDeploymentView(deployment *model.PagesDeployment) DeploymentView {
|
|||||||
DeploymentNumber: deployment.DeploymentNumber,
|
DeploymentNumber: deployment.DeploymentNumber,
|
||||||
Checksum: deployment.Checksum,
|
Checksum: deployment.Checksum,
|
||||||
Status: deployment.Status,
|
Status: deployment.Status,
|
||||||
|
UploadID: deployment.UploadID,
|
||||||
FileCount: deployment.FileCount,
|
FileCount: deployment.FileCount,
|
||||||
TotalSize: deployment.TotalSize,
|
TotalSize: deployment.TotalSize,
|
||||||
CreatedBy: deployment.CreatedBy,
|
CreatedBy: deployment.CreatedBy,
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/Rain-kl/Wavelet/internal/db"
|
"github.com/Rain-kl/Wavelet/internal/db"
|
||||||
@@ -159,6 +160,7 @@ func TestUploadDeploymentStoresPackageInUploadFramework(t *testing.T) {
|
|||||||
"index.html": "ok",
|
"index.html": "ok",
|
||||||
})), "root")
|
})), "root")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
assert.NotZero(t, deployment.UploadID)
|
||||||
|
|
||||||
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
|
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -170,6 +172,86 @@ func TestUploadDeploymentStoresPackageInUploadFramework(t *testing.T) {
|
|||||||
assert.Equal(t, int64(1), uploadCount)
|
assert.Equal(t, int64(1), uploadCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenDeploymentPackageHydratesLegacyArtifactPath(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
_, disableStorage := setupPagesStorageMock(t)
|
||||||
|
defer disableStorage()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
project, err := CreateProject(ctx, Input{
|
||||||
|
Name: "Legacy Site",
|
||||||
|
Slug: "openspeedtest",
|
||||||
|
Enabled: true,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
artifactDir := filepath.Join(t.TempDir(), "pages", "artifacts", project.Slug)
|
||||||
|
require.NoError(t, os.MkdirAll(artifactDir, 0o755))
|
||||||
|
artifactPath := filepath.Join(artifactDir, "legacy-checksum.zip")
|
||||||
|
require.NoError(t, os.WriteFile(artifactPath, testPagesZip(t, map[string]string{"index.html": "legacy"}), 0o644))
|
||||||
|
|
||||||
|
deployment := &model.PagesDeployment{
|
||||||
|
ProjectID: project.ID,
|
||||||
|
DeploymentNumber: 1,
|
||||||
|
Checksum: "legacy-checksum",
|
||||||
|
Status: model.PagesDeploymentStatusUploaded,
|
||||||
|
ArtifactPath: artifactPath,
|
||||||
|
FileCount: 1,
|
||||||
|
TotalSize: 10,
|
||||||
|
CreatedBy: "test",
|
||||||
|
}
|
||||||
|
require.NoError(t, db.DB(ctx).Create(deployment).Error)
|
||||||
|
require.NoError(t, db.DB(ctx).Create(&model.PagesDeploymentFile{
|
||||||
|
DeploymentID: deployment.ID,
|
||||||
|
Path: "index.html",
|
||||||
|
Size: 6,
|
||||||
|
Checksum: "legacy-checksum",
|
||||||
|
}).Error)
|
||||||
|
|
||||||
|
_, err = ActivateDeployment(ctx, project.ID, deployment.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, db.DB(ctx).Create(&model.ConfigVersion{
|
||||||
|
Version: "v2026-legacy",
|
||||||
|
SnapshotJSON: fmt.Sprintf(`{"routes":[{"upstream_type":"pages","pages_deployment":{"deployment_id":%d}}]}`, deployment.ID),
|
||||||
|
MainConfig: "",
|
||||||
|
RenderedConfig: "",
|
||||||
|
SupportFilesJSON: "[]",
|
||||||
|
Checksum: "legacy-config-checksum",
|
||||||
|
IsActive: true,
|
||||||
|
CreatedBy: "test",
|
||||||
|
}).Error)
|
||||||
|
|
||||||
|
packageObj, fileName, err := OpenDeploymentPackage(ctx, deployment.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer packageObj.Body.Close()
|
||||||
|
assert.Equal(t, fmt.Sprintf("pages-deployment-%d.zip", deployment.ID), fileName)
|
||||||
|
|
||||||
|
body, err := io.ReadAll(packageObj.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, reader.File, 1)
|
||||||
|
assert.Equal(t, "index.html", reader.File[0].Name)
|
||||||
|
|
||||||
|
storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotZero(t, storedDeployment.UploadID)
|
||||||
|
assert.Empty(t, storedDeployment.ArtifactPath)
|
||||||
|
|
||||||
|
var uploadCount int64
|
||||||
|
require.NoError(t, db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error)
|
||||||
|
assert.Equal(t, int64(1), uploadCount)
|
||||||
|
|
||||||
|
packageObj2, _, err := OpenDeploymentPackage(ctx, deployment.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer packageObj2.Body.Close()
|
||||||
|
body2, err := io.ReadAll(packageObj2.Body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, body, body2)
|
||||||
|
}
|
||||||
|
|
||||||
func TestOpenDeploymentPackageRequiresActiveConfigSnapshot(t *testing.T) {
|
func TestOpenDeploymentPackageRequiresActiveConfigSnapshot(t *testing.T) {
|
||||||
cleanup := setupPagesTestDB(t)
|
cleanup := setupPagesTestDB(t)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|||||||
Reference in New Issue
Block a user