mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 07:26:36 +08:00
fix(pages): 增加部署包孤儿补偿
按项目、来源、运行时与上传记录锁序补偿异常中断遗留的部署包。\n同时隐藏并保护系统内部排程，避免通用任务管理入口修改 scanner。
This commit is contained in:
deqiying
@@ -202,7 +202,7 @@ func RetryTask(c *gin.Context) {
|
||||
|
||||
// ListSchedules 获取定时任务列表
|
||||
// @Summary 获取定时任务列表
|
||||
// @Description 返回系统所有的定时任务配置列表，包括名称、关联的异步任务类型、Cron 表达式和启用状态，需要管理员权限
|
||||
// @Description 返回管理员可管理的定时任务配置列表，包括名称、关联的异步任务类型、Cron 表达式和启用状态；系统内部排程不会暴露，需要管理员权限
|
||||
// @Tags admin
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
@@ -216,7 +216,15 @@ func ListSchedules(c *gin.Context) {
|
||||
response.AbortInternal(c, err.Error())
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(schedules))
|
||||
visible := make([]model.Schedule, 0, len(schedules))
|
||||
for _, schedule := range schedules {
|
||||
meta := task.GetTaskMeta(schedule.TaskType)
|
||||
if meta != nil && meta.InternalOnly {
|
||||
continue
|
||||
}
|
||||
visible = append(visible, schedule)
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(visible))
|
||||
}
|
||||
|
||||
// CreateScheduleRequest 创建定时任务请求
|
||||
@@ -405,6 +413,7 @@ func getAdminTaskMeta(taskType string) *task.TaskMeta {
|
||||
// @Failure 400 {object} response.Any "参数错误"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 403 {object} response.Any "无管理员权限"
|
||||
// @Failure 404 {object} response.Any "定时任务不存在"
|
||||
// @Failure 500 {object} response.Any "删除定时任务失败"
|
||||
// @Router /api/v1/admin/tasks/schedules/{id} [delete]
|
||||
func DeleteSchedule(c *gin.Context) {
|
||||
@@ -413,6 +422,15 @@ func DeleteSchedule(c *gin.Context) {
|
||||
response.AbortBadRequest(c, "无效的定时任务ID")
|
||||
return
|
||||
}
|
||||
schedule, err := model.GetScheduleByID(c.Request.Context(), id)
|
||||
if err != nil {
|
||||
response.AbortNotFound(c, ScheduleNotFound)
|
||||
return
|
||||
}
|
||||
if meta := task.GetTaskMeta(schedule.TaskType); meta != nil && meta.InternalOnly {
|
||||
response.AbortBadRequest(c, InvalidTaskType)
|
||||
return
|
||||
}
|
||||
|
||||
if err := model.DeleteSchedule(c.Request.Context(), id); err != nil {
|
||||
response.AbortInternal(c, fmt.Sprintf("%s: %v", ScheduleDeleteFailed, err))
|
||||
|
||||
@@ -75,8 +75,10 @@ func setupTestRouter(authUser *model.User) *gin.Engine {
|
||||
adminGroup.GET("/tasks/executions", ListTaskExecutions)
|
||||
adminGroup.GET("/tasks/executions/:id", GetTaskExecution)
|
||||
adminGroup.POST("/tasks/executions/:id/retry", RetryTask)
|
||||
adminGroup.GET("/tasks/schedules", ListSchedules)
|
||||
adminGroup.POST("/tasks/schedules", CreateSchedule)
|
||||
adminGroup.PUT("/tasks/schedules/:id", UpdateSchedule)
|
||||
adminGroup.DELETE("/tasks/schedules/:id", DeleteSchedule)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -137,6 +139,39 @@ func TestInternalOnlyTaskAdminBoundaries(t *testing.T) {
|
||||
router := setupTestRouter(adminUser)
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("list hides internal-only schedule", func(t *testing.T) {
|
||||
internalSchedule := &model.Schedule{
|
||||
Name: "隐藏的系统内部排程",
|
||||
TaskType: testInternalOnlyTaskType,
|
||||
Cron: "*/5 * * * *",
|
||||
Payload: "{}",
|
||||
IsActive: true,
|
||||
}
|
||||
publicSchedule := &model.Schedule{
|
||||
Name: "可见的公开排程",
|
||||
TaskType: uploadtask.TaskTypeSystemCleanup,
|
||||
Cron: "0 * * * *",
|
||||
Payload: "{}",
|
||||
IsActive: true,
|
||||
}
|
||||
require.NoError(t, model.CreateSchedule(ctx, internalSchedule))
|
||||
require.NoError(t, model.CreateSchedule(ctx, publicSchedule))
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/tasks/schedules", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
var resp response.Any
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
data, err := json.Marshal(resp.Data)
|
||||
require.NoError(t, err)
|
||||
var schedules []model.Schedule
|
||||
require.NoError(t, json.Unmarshal(data, &schedules))
|
||||
assert.NotContains(t, scheduleIDs(schedules), internalSchedule.ID)
|
||||
assert.Contains(t, scheduleIDs(schedules), publicSchedule.ID)
|
||||
})
|
||||
|
||||
t.Run("dispatch rejects internal-only task", func(t *testing.T) {
|
||||
body, err := json.Marshal(DispatchTaskRequest{TaskType: testInternalOnlyTaskType})
|
||||
require.NoError(t, err)
|
||||
@@ -239,6 +274,74 @@ func TestInternalOnlyTaskAdminBoundaries(t *testing.T) {
|
||||
assert.Equal(t, "公开排程", unchanged.Name)
|
||||
assert.Equal(t, uploadtask.TaskTypeSystemCleanup, unchanged.TaskType)
|
||||
})
|
||||
|
||||
t.Run("delete rejects internal-only schedule", func(t *testing.T) {
|
||||
schedule := &model.Schedule{
|
||||
Name: "不可删除的系统内部排程",
|
||||
TaskType: testInternalOnlyTaskType,
|
||||
Cron: "*/5 * * * *",
|
||||
IsActive: true,
|
||||
}
|
||||
require.NoError(t, model.CreateSchedule(ctx, schedule))
|
||||
req := httptest.NewRequest(
|
||||
http.MethodDelete,
|
||||
fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID),
|
||||
nil,
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||
var resp response.Any
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
assert.Equal(t, InvalidTaskType, resp.ErrorMsg)
|
||||
preserved, err := model.GetScheduleByID(ctx, schedule.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testInternalOnlyTaskType, preserved.TaskType)
|
||||
})
|
||||
|
||||
t.Run("delete missing schedule returns not found", func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/v1/admin/tasks/schedules/999999", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
||||
var resp response.Any
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
assert.Equal(t, ScheduleNotFound, resp.ErrorMsg)
|
||||
})
|
||||
|
||||
t.Run("delete public schedule remains allowed", func(t *testing.T) {
|
||||
schedule := &model.Schedule{
|
||||
Name: "可删除的公开排程",
|
||||
TaskType: uploadtask.TaskTypeSystemCleanup,
|
||||
Cron: "0 * * * *",
|
||||
IsActive: false,
|
||||
}
|
||||
require.NoError(t, model.CreateSchedule(ctx, schedule))
|
||||
req := httptest.NewRequest(
|
||||
http.MethodDelete,
|
||||
fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID),
|
||||
nil,
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
_, err := model.GetScheduleByID(ctx, schedule.ID)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func scheduleIDs(schedules []model.Schedule) []uint64 {
|
||||
ids := make([]uint64, 0, len(schedules))
|
||||
for _, schedule := range schedules {
|
||||
ids = append(ids, schedule.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func TestDispatchTask(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const pagesOrphanUploadIsolation = 2 * time.Hour
|
||||
|
||||
// PagesOrphanCleanupSummary describes one bounded delayed compensation pass.
|
||||
// Every candidate is counted exactly once in one outcome field.
|
||||
//
|
||||
//nolint:revive // Keep the domain-qualified exported name for scanner/task result clarity.
|
||||
type PagesOrphanCleanupSummary struct {
|
||||
Candidates int `json:"candidates"`
|
||||
Reconciled int `json:"reconciled"`
|
||||
Referenced int `json:"referenced"`
|
||||
LeaseBusy int `json:"lease_busy"`
|
||||
InvalidMarker int `json:"invalid_marker"`
|
||||
Skipped int `json:"skipped"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
type pagesOrphanMarker struct {
|
||||
ProjectID uint
|
||||
SourceID *uint
|
||||
}
|
||||
|
||||
type pagesOrphanCleanupOutcome uint8
|
||||
|
||||
const (
|
||||
pagesOrphanCleanupSkipped pagesOrphanCleanupOutcome = iota
|
||||
pagesOrphanCleanupReconciled
|
||||
pagesOrphanCleanupReferenced
|
||||
pagesOrphanCleanupLeaseBusy
|
||||
pagesOrphanCleanupInvalidMarker
|
||||
)
|
||||
|
||||
// ReconcilePagesOrphanUploads performs one bounded delayed compensation pass.
|
||||
// Individual candidate failures are counted and logged so they do not prevent
|
||||
// the scanner from continuing with source checks.
|
||||
func ReconcilePagesOrphanUploads(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
) (PagesOrphanCleanupSummary, error) {
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
cutoff := now.UTC().Add(-pagesOrphanUploadIsolation)
|
||||
systemUser := repository.GetSystemUser(ctx)
|
||||
candidates, err := model.ListPagesOrphanUploadCandidates(ctx, model.PagesOrphanUploadCandidateQuery{
|
||||
SystemUserID: systemUser.ID,
|
||||
UploadType: upload.ReservedPagesDeploymentType,
|
||||
Marker: pagesIngestMarkerV2,
|
||||
CreatedBefore: cutoff,
|
||||
})
|
||||
if err != nil {
|
||||
return PagesOrphanCleanupSummary{}, err
|
||||
}
|
||||
|
||||
summary := PagesOrphanCleanupSummary{Candidates: len(candidates)}
|
||||
for index := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return summary, err
|
||||
}
|
||||
candidate := &candidates[index]
|
||||
marker, err := parsePagesOrphanMarker(candidate.Metadata)
|
||||
if err != nil {
|
||||
summary.InvalidMarker++
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload marker invalid: upload_id=%d error=%v", candidate.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
outcome, err := reconcilePagesOrphanUploadCandidate(
|
||||
ctx,
|
||||
candidate,
|
||||
marker,
|
||||
systemUser.ID,
|
||||
cutoff,
|
||||
)
|
||||
if err != nil {
|
||||
summary.Failed++
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload reconciliation failed: upload_id=%d error=%v", candidate.ID, err)
|
||||
continue
|
||||
}
|
||||
summary.add(outcome)
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (summary *PagesOrphanCleanupSummary) add(outcome pagesOrphanCleanupOutcome) {
|
||||
switch outcome {
|
||||
case pagesOrphanCleanupReconciled:
|
||||
summary.Reconciled++
|
||||
case pagesOrphanCleanupReferenced:
|
||||
summary.Referenced++
|
||||
case pagesOrphanCleanupLeaseBusy:
|
||||
summary.LeaseBusy++
|
||||
case pagesOrphanCleanupInvalidMarker:
|
||||
summary.InvalidMarker++
|
||||
default:
|
||||
summary.Skipped++
|
||||
}
|
||||
}
|
||||
|
||||
func reconcilePagesOrphanUploadCandidate(
|
||||
ctx context.Context,
|
||||
candidate *model.Upload,
|
||||
marker pagesOrphanMarker,
|
||||
systemUserID uint64,
|
||||
cutoff time.Time,
|
||||
) (pagesOrphanCleanupOutcome, error) {
|
||||
if candidate == nil || candidate.ID == 0 {
|
||||
return pagesOrphanCleanupSkipped, nil
|
||||
}
|
||||
|
||||
outcome := pagesOrphanCleanupSkipped
|
||||
uploadLocked := false
|
||||
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
scopeOutcome, proceed, err := lockPagesOrphanCleanupScope(ctx, tx, candidate.ID, marker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !proceed {
|
||||
outcome = scopeOutcome
|
||||
return nil
|
||||
}
|
||||
lockedOutcome, locked, err := reconcileLockedPagesOrphanUpload(
|
||||
ctx,
|
||||
tx,
|
||||
candidate.ID,
|
||||
marker,
|
||||
systemUserID,
|
||||
cutoff,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outcome = lockedOutcome
|
||||
uploadLocked = locked
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return pagesOrphanCleanupSkipped, err
|
||||
}
|
||||
if uploadLocked {
|
||||
// Also heal a prior post-commit cache invalidation interruption when the
|
||||
// status transition was an idempotent no-op.
|
||||
upload.InvalidateUploadMetaCache(ctx, candidate.ID)
|
||||
}
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
func lockPagesOrphanCleanupScope(
|
||||
ctx context.Context,
|
||||
tx *gorm.DB,
|
||||
uploadID uint64,
|
||||
marker pagesOrphanMarker,
|
||||
) (pagesOrphanCleanupOutcome, bool, error) {
|
||||
var project model.PagesProject
|
||||
if _, err := lockOptionalPagesCleanupRecord(tx, &project, "id = ?", marker.ProjectID); err != nil {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
if marker.SourceID == nil {
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
|
||||
var source model.PagesProjectSource
|
||||
sourceExists, err := lockOptionalPagesCleanupRecord(tx, &source, "id = ?", *marker.SourceID)
|
||||
if err != nil {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
if !sourceExists {
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
if source.ProjectID != marker.ProjectID {
|
||||
logger.WarnF(ctx,
|
||||
"[PagesSource] orphan upload source ownership mismatch: upload_id=%d project_id=%d source_id=%d source_project_id=%d",
|
||||
uploadID,
|
||||
marker.ProjectID,
|
||||
*marker.SourceID,
|
||||
source.ProjectID,
|
||||
)
|
||||
return pagesOrphanCleanupInvalidMarker, false, nil
|
||||
}
|
||||
|
||||
var runtime model.PagesProjectSourceRuntime
|
||||
runtimeExists, err := lockOptionalPagesCleanupRecord(tx, &runtime, "source_id = ?", source.ID)
|
||||
if err != nil {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
// Read the real clock only after obtaining the runtime row lock. The scanner
|
||||
// snapshot time is only an isolation cutoff and may be stale after lock wait.
|
||||
leaseCheckedAt := time.Now()
|
||||
if runtimeExists && runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(leaseCheckedAt) {
|
||||
return pagesOrphanCleanupLeaseBusy, false, nil
|
||||
}
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
|
||||
func reconcileLockedPagesOrphanUpload(
|
||||
ctx context.Context,
|
||||
tx *gorm.DB,
|
||||
uploadID uint64,
|
||||
marker pagesOrphanMarker,
|
||||
systemUserID uint64,
|
||||
cutoff time.Time,
|
||||
) (pagesOrphanCleanupOutcome, bool, error) {
|
||||
var lockedUpload model.Upload
|
||||
found, err := lockOptionalPagesCleanupRecord(tx, &lockedUpload, "id = ?", uploadID)
|
||||
if err != nil || !found {
|
||||
return pagesOrphanCleanupSkipped, false, err
|
||||
}
|
||||
|
||||
lockedMarker, err := parsePagesOrphanMarker(lockedUpload.Metadata)
|
||||
if err != nil {
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload marker changed or invalid: upload_id=%d error=%v", uploadID, err)
|
||||
return pagesOrphanCleanupInvalidMarker, true, nil
|
||||
}
|
||||
if lockedUpload.Status != model.UploadStatusUsed ||
|
||||
lockedUpload.UserID != systemUserID ||
|
||||
lockedUpload.Type != upload.ReservedPagesDeploymentType ||
|
||||
!lockedUpload.CreatedAt.Before(cutoff) {
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
if lockedMarker.ProjectID != marker.ProjectID || !sameOptionalPagesSourceID(lockedMarker.SourceID, marker.SourceID) {
|
||||
logger.WarnF(ctx, "[PagesSource] orphan upload marker changed during reconciliation: upload_id=%d", uploadID)
|
||||
return pagesOrphanCleanupInvalidMarker, true, nil
|
||||
}
|
||||
|
||||
var references int64
|
||||
if err := tx.Model(&model.PagesDeployment{}).
|
||||
Where("upload_id = ?", lockedUpload.ID).
|
||||
Count(&references).Error; err != nil {
|
||||
return pagesOrphanCleanupSkipped, true, err
|
||||
}
|
||||
if references > 0 {
|
||||
return pagesOrphanCleanupReferenced, true, nil
|
||||
}
|
||||
|
||||
transitioned, err := upload.RemoveLockedTx(tx, &lockedUpload)
|
||||
if err != nil {
|
||||
return pagesOrphanCleanupSkipped, true, err
|
||||
}
|
||||
if transitioned {
|
||||
return pagesOrphanCleanupReconciled, true, nil
|
||||
}
|
||||
return pagesOrphanCleanupSkipped, true, nil
|
||||
}
|
||||
|
||||
func lockOptionalPagesCleanupRecord(
|
||||
tx *gorm.DB,
|
||||
value any,
|
||||
query string,
|
||||
args ...any,
|
||||
) (bool, error) {
|
||||
err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
|
||||
Where(query, args...).
|
||||
First(value).Error
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func parsePagesOrphanMarker(metadata model.UploadMetadata) (pagesOrphanMarker, error) {
|
||||
if metadata.Extra == nil {
|
||||
return pagesOrphanMarker{}, errors.New("pages marker metadata missing")
|
||||
}
|
||||
marker, ok := metadata.Extra[pagesIngestMarkerKey].(string)
|
||||
if !ok || marker != pagesIngestMarkerV2 {
|
||||
return pagesOrphanMarker{}, errors.New("pages marker version invalid")
|
||||
}
|
||||
projectID, err := parsePagesOrphanMetadataID(metadata.Extra, pagesProjectIDMetadataKey)
|
||||
if err != nil {
|
||||
return pagesOrphanMarker{}, err
|
||||
}
|
||||
result := pagesOrphanMarker{ProjectID: projectID}
|
||||
if _, exists := metadata.Extra[pagesSourceIDMetadataKey]; exists {
|
||||
sourceID, err := parsePagesOrphanMetadataID(metadata.Extra, pagesSourceIDMetadataKey)
|
||||
if err != nil {
|
||||
return pagesOrphanMarker{}, err
|
||||
}
|
||||
result.SourceID = &sourceID
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func parsePagesOrphanMetadataID(extra map[string]any, key string) (uint, error) {
|
||||
raw, exists := extra[key]
|
||||
if !exists {
|
||||
return 0, fmt.Errorf("pages marker %s missing", key)
|
||||
}
|
||||
value, ok := raw.(string)
|
||||
if !ok || value == "" {
|
||||
return 0, fmt.Errorf("pages marker %s must be a decimal string", key)
|
||||
}
|
||||
parsed, err := strconv.ParseUint(value, 10, 64)
|
||||
maxModelID := uint64(^uint(0) >> 1)
|
||||
if err != nil || parsed == 0 || parsed > maxModelID || strconv.FormatUint(parsed, 10) != value {
|
||||
return 0, fmt.Errorf("pages marker %s is not a canonical non-zero decimal ID", key)
|
||||
}
|
||||
return uint(parsed), nil
|
||||
}
|
||||
|
||||
func sameOptionalPagesSourceID(left, right *uint) bool {
|
||||
if left == nil || right == nil {
|
||||
return left == nil && right == nil
|
||||
}
|
||||
return *left == *right
|
||||
}
|
||||
@@ -0,0 +1,381 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
func TestParsePagesOrphanMarker(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extra map[string]any
|
||||
wantProject uint
|
||||
wantSource uint
|
||||
wantSourceOK bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "manual source marker",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
},
|
||||
wantProject: 12,
|
||||
},
|
||||
{
|
||||
name: "persistent source marker",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
pagesSourceIDMetadataKey: "34",
|
||||
},
|
||||
wantProject: 12,
|
||||
wantSource: 34,
|
||||
wantSourceOK: true,
|
||||
},
|
||||
{
|
||||
name: "project ID must be canonical decimal",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "012",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "source ID must be a string",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
pagesSourceIDMetadataKey: float64(34),
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "zero ID rejected",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: "0",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong marker version rejected",
|
||||
extra: map[string]any{
|
||||
pagesIngestMarkerKey: "pages_deployment_v1",
|
||||
pagesProjectIDMetadataKey: "12",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := parsePagesOrphanMarker(model.UploadMetadata{Extra: test.extra})
|
||||
if gotErr := err != nil; gotErr != test.wantErr {
|
||||
t.Fatalf("parsePagesOrphanMarker(%v) error = %v, want error presence = %t", test.extra, err, test.wantErr)
|
||||
}
|
||||
if test.wantErr {
|
||||
return
|
||||
}
|
||||
if got.ProjectID != test.wantProject {
|
||||
t.Errorf("parsePagesOrphanMarker(%v).ProjectID = %d, want %d", test.extra, got.ProjectID, test.wantProject)
|
||||
}
|
||||
if gotSourceOK := got.SourceID != nil; gotSourceOK != test.wantSourceOK {
|
||||
t.Fatalf("parsePagesOrphanMarker(%v).SourceID presence = %t, want %t", test.extra, gotSourceOK, test.wantSourceOK)
|
||||
}
|
||||
if got.SourceID != nil && *got.SourceID != test.wantSource {
|
||||
t.Errorf("parsePagesOrphanMarker(%v).SourceID = %d, want %d", test.extra, *got.SourceID, test.wantSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsDeletesEligibleUploadOnce(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
project := createPagesOrphanProject(t, ctx, "eligible-orphan")
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||
if err := upload.RebuildUploadStats(ctx); err != nil {
|
||||
t.Fatalf("RebuildUploadStats() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.Candidates != 1 || summary.Reconciled != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one reconciled candidate", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusDeleted)
|
||||
assertPagesCleanupTotalStat(t, ctx, 0)
|
||||
|
||||
second, err := ReconcilePagesOrphanUploads(ctx, now.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("second ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if second.Candidates != 0 || cleanupOutcomeTotal(second) != 0 {
|
||||
t.Errorf("second ReconcilePagesOrphanUploads() summary = %+v, want empty", second)
|
||||
}
|
||||
assertPagesCleanupTotalStat(t, ctx, 0)
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsAllowsDeletedProjectAndSource(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
missingSourceID := uint(9876)
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), 8765, &missingSourceID)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.Candidates != 1 || summary.Reconciled != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want deleted project/source treated as one orphan", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusDeleted)
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsSkipsBusyLeaseAndSourceMismatch(t *testing.T) {
|
||||
t.Run("unexpired source lease", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
realNow := time.Now().UTC()
|
||||
// A deliberately future scanner snapshot proves lease freshness uses the
|
||||
// real clock after the runtime lock, not this isolation-cutoff input.
|
||||
scannerNow := realNow.Add(24 * time.Hour)
|
||||
project := createPagesOrphanProject(t, ctx, "busy-orphan")
|
||||
source := createPagesOrphanSource(t, ctx, project.ID)
|
||||
future := realNow.Add(time.Hour)
|
||||
if err := db.DB(ctx).Create(&model.PagesProjectSourceRuntime{
|
||||
SourceID: source.ID,
|
||||
LeaseToken: "busy-worker",
|
||||
LeaseExpiresAt: &future,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create busy source runtime error = %v, want nil", err)
|
||||
}
|
||||
candidate := createPagesOrphanUpload(t, ctx, realNow.Add(-3*time.Hour), project.ID, &source.ID)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, scannerNow)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.LeaseBusy != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one lease-busy candidate", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
})
|
||||
|
||||
t.Run("source belongs to another project", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
markerProject := createPagesOrphanProject(t, ctx, "marker-project")
|
||||
actualProject := createPagesOrphanProject(t, ctx, "actual-project")
|
||||
source := createPagesOrphanSource(t, ctx, actualProject.ID)
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), markerProject.ID, &source.ID)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.InvalidMarker != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one ownership mismatch", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
})
|
||||
}
|
||||
|
||||
func TestReconcilePagesOrphanUploadsRejectsMalformedMarker(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), 1, nil)
|
||||
metadata := candidate.Metadata
|
||||
metadata.Extra[pagesProjectIDMetadataKey] = "01"
|
||||
candidate.Metadata = metadata
|
||||
if err := db.DB(ctx).Save(candidate).Error; err != nil {
|
||||
t.Fatalf("seed malformed candidate marker error = %v, want nil", err)
|
||||
}
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.InvalidMarker != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one invalid marker", summary)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
}
|
||||
|
||||
func TestPagesOrphanCleanupAndDeploymentCommitInterleavings(t *testing.T) {
|
||||
t.Run("deployment reference commits first", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
project := createPagesOrphanProject(t, ctx, "deployment-first")
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||
marker, err := parsePagesOrphanMarker(candidate.Metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("parsePagesOrphanMarker() error = %v, want nil", err)
|
||||
}
|
||||
if err := db.DB(ctx).Create(&model.PagesDeployment{
|
||||
ProjectID: project.ID,
|
||||
DeploymentNumber: 1,
|
||||
Checksum: "deployment-first",
|
||||
Status: model.PagesDeploymentStatusUploaded,
|
||||
UploadID: candidate.ID,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create deployment reference error = %v, want nil", err)
|
||||
}
|
||||
|
||||
outcome, err := reconcilePagesOrphanUploadCandidate(ctx, candidate, marker, 999, now.Add(-2*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("reconcilePagesOrphanUploadCandidate() error = %v, want nil", err)
|
||||
}
|
||||
if outcome != pagesOrphanCleanupReferenced {
|
||||
t.Errorf("reconcilePagesOrphanUploadCandidate() outcome = %d, want %d", outcome, pagesOrphanCleanupReferenced)
|
||||
}
|
||||
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||
})
|
||||
|
||||
t.Run("cleanup commits first", func(t *testing.T) {
|
||||
cleanup := setupPagesTestDB(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
project := createPagesOrphanProject(t, ctx, "cleanup-first")
|
||||
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||
|
||||
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||
if err != nil {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||
}
|
||||
if summary.Reconciled != 1 {
|
||||
t.Fatalf("ReconcilePagesOrphanUploads() summary = %+v, want one reconciled candidate", summary)
|
||||
}
|
||||
|
||||
target := &model.PagesDeployment{ProjectID: project.ID, UploadID: candidate.ID}
|
||||
err = db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
var lockedProject model.PagesProject
|
||||
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&lockedProject, project.ID).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return lockSourceDeploymentUploadsTx(tx, target, upload.IngestResult{}, false)
|
||||
})
|
||||
if !errors.Is(err, errSourceFinalFence) {
|
||||
t.Errorf("final deployment upload lock after cleanup error = %v, want %v", err, errSourceFinalFence)
|
||||
}
|
||||
var references int64
|
||||
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("upload_id = ?", candidate.ID).Count(&references).Error; err != nil {
|
||||
t.Fatalf("count deployment references error = %v, want nil", err)
|
||||
}
|
||||
if references != 0 {
|
||||
t.Errorf("deployment references after cleanup-first interleaving = %d, want 0", references)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func cleanupOutcomeTotal(summary PagesOrphanCleanupSummary) int {
|
||||
return summary.Reconciled + summary.Referenced + summary.LeaseBusy + summary.InvalidMarker + summary.Skipped + summary.Failed
|
||||
}
|
||||
|
||||
func createPagesOrphanProject(t *testing.T, ctx context.Context, slug string) *model.PagesProject {
|
||||
t.Helper()
|
||||
project := &model.PagesProject{Name: slug, Slug: slug, Enabled: true}
|
||||
if err := db.DB(ctx).Create(project).Error; err != nil {
|
||||
t.Fatalf("create Pages orphan project %q error = %v, want nil", slug, err)
|
||||
}
|
||||
return project
|
||||
}
|
||||
|
||||
func createPagesOrphanSource(t *testing.T, ctx context.Context, projectID uint) *model.PagesProjectSource {
|
||||
t.Helper()
|
||||
source := &model.PagesProjectSource{
|
||||
ProjectID: projectID,
|
||||
SourceType: PagesSourceTypeRemoteURL,
|
||||
ConfigVersion: 1,
|
||||
SourceIdentity: "orphan-source-identity",
|
||||
}
|
||||
if err := db.DB(ctx).Create(source).Error; err != nil {
|
||||
t.Fatalf("create Pages orphan source for project %d error = %v, want nil", projectID, err)
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
func createPagesOrphanUpload(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
createdAt time.Time,
|
||||
projectID uint,
|
||||
sourceID *uint,
|
||||
) *model.Upload {
|
||||
t.Helper()
|
||||
extra := map[string]any{
|
||||
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||
pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10),
|
||||
}
|
||||
if sourceID != nil {
|
||||
extra[pagesSourceIDMetadataKey] = strconv.FormatUint(uint64(*sourceID), 10)
|
||||
}
|
||||
candidate := &model.Upload{
|
||||
UserID: 999,
|
||||
FileName: "site.zip",
|
||||
FilePath: "pages/orphan-site.zip",
|
||||
FileSize: 64,
|
||||
MimeType: "application/zip",
|
||||
Extension: "zip",
|
||||
Hash: "orphan-checksum",
|
||||
Type: upload.ReservedPagesDeploymentType,
|
||||
Status: model.UploadStatusUsed,
|
||||
AccessMode: 0,
|
||||
Metadata: model.UploadMetadata{Extra: extra},
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: createdAt,
|
||||
}
|
||||
if err := db.DB(ctx).Create(candidate).Error; err != nil {
|
||||
t.Fatalf("create Pages orphan upload error = %v, want nil", err)
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
|
||||
func assertPagesCleanupUploadStatus(t *testing.T, ctx context.Context, uploadID uint64, want model.UploadStatus) {
|
||||
t.Helper()
|
||||
var got model.Upload
|
||||
if err := db.DB(ctx).First(&got, uploadID).Error; err != nil {
|
||||
t.Fatalf("load upload %d error = %v, want nil", uploadID, err)
|
||||
}
|
||||
if got.Status != want {
|
||||
t.Errorf("upload %d status = %q, want %q", uploadID, got.Status, want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertPagesCleanupTotalStat(t *testing.T, ctx context.Context, want int64) {
|
||||
t.Helper()
|
||||
var stat model.UploadStat
|
||||
if err := db.DB(ctx).Where("dimension = ? AND stat_key = ?", model.UploadStatDimensionTotal, "").First(&stat).Error; err != nil {
|
||||
t.Fatalf("load total upload stat error = %v, want nil", err)
|
||||
}
|
||||
if stat.FileCount != want {
|
||||
t.Errorf("total upload stat FileCount = %d, want %d", stat.FileCount, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user