mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-08 16:46:37 +08:00
fix(pages): 增加部署包孤儿补偿
按项目、来源、运行时与上传记录锁序补偿异常中断遗留的部署包。\n同时隐藏并保护系统内部排程,避免通用任务管理入口修改 scanner。
This commit is contained in:
+7
-1
@@ -2595,7 +2595,7 @@ const docTemplate = `{
|
|||||||
"SessionCookie": []
|
"SessionCookie": []
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"description": "返回系统所有的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态,需要管理员权限",
|
"description": "返回管理员可管理的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态;系统内部排程不会暴露,需要管理员权限",
|
||||||
"produces": [
|
"produces": [
|
||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
@@ -2860,6 +2860,12 @@ const docTemplate = `{
|
|||||||
"$ref": "#/definitions/response.Any"
|
"$ref": "#/definitions/response.Any"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"404": {
|
||||||
|
"description": "定时任务不存在",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/response.Any"
|
||||||
|
}
|
||||||
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "删除定时任务失败",
|
"description": "删除定时任务失败",
|
||||||
"schema": {
|
"schema": {
|
||||||
|
|||||||
+7
-1
@@ -2588,7 +2588,7 @@
|
|||||||
"SessionCookie": []
|
"SessionCookie": []
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"description": "返回系统所有的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态,需要管理员权限",
|
"description": "返回管理员可管理的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态;系统内部排程不会暴露,需要管理员权限",
|
||||||
"produces": [
|
"produces": [
|
||||||
"application/json"
|
"application/json"
|
||||||
],
|
],
|
||||||
@@ -2853,6 +2853,12 @@
|
|||||||
"$ref": "#/definitions/response.Any"
|
"$ref": "#/definitions/response.Any"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"404": {
|
||||||
|
"description": "定时任务不存在",
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/definitions/response.Any"
|
||||||
|
}
|
||||||
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "删除定时任务失败",
|
"description": "删除定时任务失败",
|
||||||
"schema": {
|
"schema": {
|
||||||
|
|||||||
+5
-1
@@ -5717,7 +5717,7 @@ paths:
|
|||||||
- admin
|
- admin
|
||||||
/api/v1/admin/tasks/schedules:
|
/api/v1/admin/tasks/schedules:
|
||||||
get:
|
get:
|
||||||
description: 返回系统所有的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态,需要管理员权限
|
description: 返回管理员可管理的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态;系统内部排程不会暴露,需要管理员权限
|
||||||
produces:
|
produces:
|
||||||
- application/json
|
- application/json
|
||||||
responses:
|
responses:
|
||||||
@@ -5822,6 +5822,10 @@ paths:
|
|||||||
description: 无管理员权限
|
description: 无管理员权限
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/definitions/response.Any'
|
$ref: '#/definitions/response.Any'
|
||||||
|
"404":
|
||||||
|
description: 定时任务不存在
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/response.Any'
|
||||||
"500":
|
"500":
|
||||||
description: 删除定时任务失败
|
description: 删除定时任务失败
|
||||||
schema:
|
schema:
|
||||||
|
|||||||
@@ -202,7 +202,7 @@ func RetryTask(c *gin.Context) {
|
|||||||
|
|
||||||
// ListSchedules 获取定时任务列表
|
// ListSchedules 获取定时任务列表
|
||||||
// @Summary 获取定时任务列表
|
// @Summary 获取定时任务列表
|
||||||
// @Description 返回系统所有的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态,需要管理员权限
|
// @Description 返回管理员可管理的定时任务配置列表,包括名称、关联的异步任务类型、Cron 表达式和启用状态;系统内部排程不会暴露,需要管理员权限
|
||||||
// @Tags admin
|
// @Tags admin
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Security SessionCookie
|
// @Security SessionCookie
|
||||||
@@ -216,7 +216,15 @@ func ListSchedules(c *gin.Context) {
|
|||||||
response.AbortInternal(c, err.Error())
|
response.AbortInternal(c, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.JSON(http.StatusOK, response.OK(schedules))
|
visible := make([]model.Schedule, 0, len(schedules))
|
||||||
|
for _, schedule := range schedules {
|
||||||
|
meta := task.GetTaskMeta(schedule.TaskType)
|
||||||
|
if meta != nil && meta.InternalOnly {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
visible = append(visible, schedule)
|
||||||
|
}
|
||||||
|
c.JSON(http.StatusOK, response.OK(visible))
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateScheduleRequest 创建定时任务请求
|
// CreateScheduleRequest 创建定时任务请求
|
||||||
@@ -405,6 +413,7 @@ func getAdminTaskMeta(taskType string) *task.TaskMeta {
|
|||||||
// @Failure 400 {object} response.Any "参数错误"
|
// @Failure 400 {object} response.Any "参数错误"
|
||||||
// @Failure 401 {object} response.Any "未登录"
|
// @Failure 401 {object} response.Any "未登录"
|
||||||
// @Failure 403 {object} response.Any "无管理员权限"
|
// @Failure 403 {object} response.Any "无管理员权限"
|
||||||
|
// @Failure 404 {object} response.Any "定时任务不存在"
|
||||||
// @Failure 500 {object} response.Any "删除定时任务失败"
|
// @Failure 500 {object} response.Any "删除定时任务失败"
|
||||||
// @Router /api/v1/admin/tasks/schedules/{id} [delete]
|
// @Router /api/v1/admin/tasks/schedules/{id} [delete]
|
||||||
func DeleteSchedule(c *gin.Context) {
|
func DeleteSchedule(c *gin.Context) {
|
||||||
@@ -413,6 +422,15 @@ func DeleteSchedule(c *gin.Context) {
|
|||||||
response.AbortBadRequest(c, "无效的定时任务ID")
|
response.AbortBadRequest(c, "无效的定时任务ID")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
schedule, err := model.GetScheduleByID(c.Request.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
response.AbortNotFound(c, ScheduleNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if meta := task.GetTaskMeta(schedule.TaskType); meta != nil && meta.InternalOnly {
|
||||||
|
response.AbortBadRequest(c, InvalidTaskType)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if err := model.DeleteSchedule(c.Request.Context(), id); err != nil {
|
if err := model.DeleteSchedule(c.Request.Context(), id); err != nil {
|
||||||
response.AbortInternal(c, fmt.Sprintf("%s: %v", ScheduleDeleteFailed, err))
|
response.AbortInternal(c, fmt.Sprintf("%s: %v", ScheduleDeleteFailed, err))
|
||||||
|
|||||||
@@ -75,8 +75,10 @@ func setupTestRouter(authUser *model.User) *gin.Engine {
|
|||||||
adminGroup.GET("/tasks/executions", ListTaskExecutions)
|
adminGroup.GET("/tasks/executions", ListTaskExecutions)
|
||||||
adminGroup.GET("/tasks/executions/:id", GetTaskExecution)
|
adminGroup.GET("/tasks/executions/:id", GetTaskExecution)
|
||||||
adminGroup.POST("/tasks/executions/:id/retry", RetryTask)
|
adminGroup.POST("/tasks/executions/:id/retry", RetryTask)
|
||||||
|
adminGroup.GET("/tasks/schedules", ListSchedules)
|
||||||
adminGroup.POST("/tasks/schedules", CreateSchedule)
|
adminGroup.POST("/tasks/schedules", CreateSchedule)
|
||||||
adminGroup.PUT("/tasks/schedules/:id", UpdateSchedule)
|
adminGroup.PUT("/tasks/schedules/:id", UpdateSchedule)
|
||||||
|
adminGroup.DELETE("/tasks/schedules/:id", DeleteSchedule)
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,6 +139,39 @@ func TestInternalOnlyTaskAdminBoundaries(t *testing.T) {
|
|||||||
router := setupTestRouter(adminUser)
|
router := setupTestRouter(adminUser)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("list hides internal-only schedule", func(t *testing.T) {
|
||||||
|
internalSchedule := &model.Schedule{
|
||||||
|
Name: "隐藏的系统内部排程",
|
||||||
|
TaskType: testInternalOnlyTaskType,
|
||||||
|
Cron: "*/5 * * * *",
|
||||||
|
Payload: "{}",
|
||||||
|
IsActive: true,
|
||||||
|
}
|
||||||
|
publicSchedule := &model.Schedule{
|
||||||
|
Name: "可见的公开排程",
|
||||||
|
TaskType: uploadtask.TaskTypeSystemCleanup,
|
||||||
|
Cron: "0 * * * *",
|
||||||
|
Payload: "{}",
|
||||||
|
IsActive: true,
|
||||||
|
}
|
||||||
|
require.NoError(t, model.CreateSchedule(ctx, internalSchedule))
|
||||||
|
require.NoError(t, model.CreateSchedule(ctx, publicSchedule))
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/tasks/schedules", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusOK, w.Code)
|
||||||
|
var resp response.Any
|
||||||
|
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||||
|
data, err := json.Marshal(resp.Data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var schedules []model.Schedule
|
||||||
|
require.NoError(t, json.Unmarshal(data, &schedules))
|
||||||
|
assert.NotContains(t, scheduleIDs(schedules), internalSchedule.ID)
|
||||||
|
assert.Contains(t, scheduleIDs(schedules), publicSchedule.ID)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("dispatch rejects internal-only task", func(t *testing.T) {
|
t.Run("dispatch rejects internal-only task", func(t *testing.T) {
|
||||||
body, err := json.Marshal(DispatchTaskRequest{TaskType: testInternalOnlyTaskType})
|
body, err := json.Marshal(DispatchTaskRequest{TaskType: testInternalOnlyTaskType})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -239,6 +274,74 @@ func TestInternalOnlyTaskAdminBoundaries(t *testing.T) {
|
|||||||
assert.Equal(t, "公开排程", unchanged.Name)
|
assert.Equal(t, "公开排程", unchanged.Name)
|
||||||
assert.Equal(t, uploadtask.TaskTypeSystemCleanup, unchanged.TaskType)
|
assert.Equal(t, uploadtask.TaskTypeSystemCleanup, unchanged.TaskType)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("delete rejects internal-only schedule", func(t *testing.T) {
|
||||||
|
schedule := &model.Schedule{
|
||||||
|
Name: "不可删除的系统内部排程",
|
||||||
|
TaskType: testInternalOnlyTaskType,
|
||||||
|
Cron: "*/5 * * * *",
|
||||||
|
IsActive: true,
|
||||||
|
}
|
||||||
|
require.NoError(t, model.CreateSchedule(ctx, schedule))
|
||||||
|
req := httptest.NewRequest(
|
||||||
|
http.MethodDelete,
|
||||||
|
fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID),
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
router.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||||
|
var resp response.Any
|
||||||
|
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||||
|
assert.Equal(t, InvalidTaskType, resp.ErrorMsg)
|
||||||
|
preserved, err := model.GetScheduleByID(ctx, schedule.ID)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, testInternalOnlyTaskType, preserved.TaskType)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("delete missing schedule returns not found", func(t *testing.T) {
|
||||||
|
req := httptest.NewRequest(http.MethodDelete, "/api/v1/admin/tasks/schedules/999999", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
router.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusNotFound, w.Code)
|
||||||
|
var resp response.Any
|
||||||
|
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||||
|
assert.Equal(t, ScheduleNotFound, resp.ErrorMsg)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("delete public schedule remains allowed", func(t *testing.T) {
|
||||||
|
schedule := &model.Schedule{
|
||||||
|
Name: "可删除的公开排程",
|
||||||
|
TaskType: uploadtask.TaskTypeSystemCleanup,
|
||||||
|
Cron: "0 * * * *",
|
||||||
|
IsActive: false,
|
||||||
|
}
|
||||||
|
require.NoError(t, model.CreateSchedule(ctx, schedule))
|
||||||
|
req := httptest.NewRequest(
|
||||||
|
http.MethodDelete,
|
||||||
|
fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID),
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
router.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusOK, w.Code)
|
||||||
|
_, err := model.GetScheduleByID(ctx, schedule.ID)
|
||||||
|
assert.Error(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func scheduleIDs(schedules []model.Schedule) []uint64 {
|
||||||
|
ids := make([]uint64, 0, len(schedules))
|
||||||
|
for _, schedule := range schedules {
|
||||||
|
ids = append(ids, schedule.ID)
|
||||||
|
}
|
||||||
|
return ids
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDispatchTask(t *testing.T) {
|
func TestDispatchTask(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,328 @@
|
|||||||
|
// Copyright 2026 Arctel.net
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package pages
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/db"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/model"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||||
|
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
)
|
||||||
|
|
||||||
|
const pagesOrphanUploadIsolation = 2 * time.Hour
|
||||||
|
|
||||||
|
// PagesOrphanCleanupSummary describes one bounded delayed compensation pass.
|
||||||
|
// Every candidate is counted exactly once in one outcome field.
|
||||||
|
//
|
||||||
|
//nolint:revive // Keep the domain-qualified exported name for scanner/task result clarity.
|
||||||
|
type PagesOrphanCleanupSummary struct {
|
||||||
|
Candidates int `json:"candidates"`
|
||||||
|
Reconciled int `json:"reconciled"`
|
||||||
|
Referenced int `json:"referenced"`
|
||||||
|
LeaseBusy int `json:"lease_busy"`
|
||||||
|
InvalidMarker int `json:"invalid_marker"`
|
||||||
|
Skipped int `json:"skipped"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type pagesOrphanMarker struct {
|
||||||
|
ProjectID uint
|
||||||
|
SourceID *uint
|
||||||
|
}
|
||||||
|
|
||||||
|
type pagesOrphanCleanupOutcome uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
pagesOrphanCleanupSkipped pagesOrphanCleanupOutcome = iota
|
||||||
|
pagesOrphanCleanupReconciled
|
||||||
|
pagesOrphanCleanupReferenced
|
||||||
|
pagesOrphanCleanupLeaseBusy
|
||||||
|
pagesOrphanCleanupInvalidMarker
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReconcilePagesOrphanUploads performs one bounded delayed compensation pass.
|
||||||
|
// Individual candidate failures are counted and logged so they do not prevent
|
||||||
|
// the scanner from continuing with source checks.
|
||||||
|
func ReconcilePagesOrphanUploads(
|
||||||
|
ctx context.Context,
|
||||||
|
now time.Time,
|
||||||
|
) (PagesOrphanCleanupSummary, error) {
|
||||||
|
if now.IsZero() {
|
||||||
|
now = time.Now()
|
||||||
|
}
|
||||||
|
cutoff := now.UTC().Add(-pagesOrphanUploadIsolation)
|
||||||
|
systemUser := repository.GetSystemUser(ctx)
|
||||||
|
candidates, err := model.ListPagesOrphanUploadCandidates(ctx, model.PagesOrphanUploadCandidateQuery{
|
||||||
|
SystemUserID: systemUser.ID,
|
||||||
|
UploadType: upload.ReservedPagesDeploymentType,
|
||||||
|
Marker: pagesIngestMarkerV2,
|
||||||
|
CreatedBefore: cutoff,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return PagesOrphanCleanupSummary{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := PagesOrphanCleanupSummary{Candidates: len(candidates)}
|
||||||
|
for index := range candidates {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return summary, err
|
||||||
|
}
|
||||||
|
candidate := &candidates[index]
|
||||||
|
marker, err := parsePagesOrphanMarker(candidate.Metadata)
|
||||||
|
if err != nil {
|
||||||
|
summary.InvalidMarker++
|
||||||
|
logger.WarnF(ctx, "[PagesSource] orphan upload marker invalid: upload_id=%d error=%v", candidate.ID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome, err := reconcilePagesOrphanUploadCandidate(
|
||||||
|
ctx,
|
||||||
|
candidate,
|
||||||
|
marker,
|
||||||
|
systemUser.ID,
|
||||||
|
cutoff,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
summary.Failed++
|
||||||
|
logger.WarnF(ctx, "[PagesSource] orphan upload reconciliation failed: upload_id=%d error=%v", candidate.ID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
summary.add(outcome)
|
||||||
|
}
|
||||||
|
return summary, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (summary *PagesOrphanCleanupSummary) add(outcome pagesOrphanCleanupOutcome) {
|
||||||
|
switch outcome {
|
||||||
|
case pagesOrphanCleanupReconciled:
|
||||||
|
summary.Reconciled++
|
||||||
|
case pagesOrphanCleanupReferenced:
|
||||||
|
summary.Referenced++
|
||||||
|
case pagesOrphanCleanupLeaseBusy:
|
||||||
|
summary.LeaseBusy++
|
||||||
|
case pagesOrphanCleanupInvalidMarker:
|
||||||
|
summary.InvalidMarker++
|
||||||
|
default:
|
||||||
|
summary.Skipped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func reconcilePagesOrphanUploadCandidate(
|
||||||
|
ctx context.Context,
|
||||||
|
candidate *model.Upload,
|
||||||
|
marker pagesOrphanMarker,
|
||||||
|
systemUserID uint64,
|
||||||
|
cutoff time.Time,
|
||||||
|
) (pagesOrphanCleanupOutcome, error) {
|
||||||
|
if candidate == nil || candidate.ID == 0 {
|
||||||
|
return pagesOrphanCleanupSkipped, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome := pagesOrphanCleanupSkipped
|
||||||
|
uploadLocked := false
|
||||||
|
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||||
|
scopeOutcome, proceed, err := lockPagesOrphanCleanupScope(ctx, tx, candidate.ID, marker)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !proceed {
|
||||||
|
outcome = scopeOutcome
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
lockedOutcome, locked, err := reconcileLockedPagesOrphanUpload(
|
||||||
|
ctx,
|
||||||
|
tx,
|
||||||
|
candidate.ID,
|
||||||
|
marker,
|
||||||
|
systemUserID,
|
||||||
|
cutoff,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
outcome = lockedOutcome
|
||||||
|
uploadLocked = locked
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return pagesOrphanCleanupSkipped, err
|
||||||
|
}
|
||||||
|
if uploadLocked {
|
||||||
|
// Also heal a prior post-commit cache invalidation interruption when the
|
||||||
|
// status transition was an idempotent no-op.
|
||||||
|
upload.InvalidateUploadMetaCache(ctx, candidate.ID)
|
||||||
|
}
|
||||||
|
return outcome, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lockPagesOrphanCleanupScope(
|
||||||
|
ctx context.Context,
|
||||||
|
tx *gorm.DB,
|
||||||
|
uploadID uint64,
|
||||||
|
marker pagesOrphanMarker,
|
||||||
|
) (pagesOrphanCleanupOutcome, bool, error) {
|
||||||
|
var project model.PagesProject
|
||||||
|
if _, err := lockOptionalPagesCleanupRecord(tx, &project, "id = ?", marker.ProjectID); err != nil {
|
||||||
|
return pagesOrphanCleanupSkipped, false, err
|
||||||
|
}
|
||||||
|
if marker.SourceID == nil {
|
||||||
|
return pagesOrphanCleanupSkipped, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var source model.PagesProjectSource
|
||||||
|
sourceExists, err := lockOptionalPagesCleanupRecord(tx, &source, "id = ?", *marker.SourceID)
|
||||||
|
if err != nil {
|
||||||
|
return pagesOrphanCleanupSkipped, false, err
|
||||||
|
}
|
||||||
|
if !sourceExists {
|
||||||
|
return pagesOrphanCleanupSkipped, true, nil
|
||||||
|
}
|
||||||
|
if source.ProjectID != marker.ProjectID {
|
||||||
|
logger.WarnF(ctx,
|
||||||
|
"[PagesSource] orphan upload source ownership mismatch: upload_id=%d project_id=%d source_id=%d source_project_id=%d",
|
||||||
|
uploadID,
|
||||||
|
marker.ProjectID,
|
||||||
|
*marker.SourceID,
|
||||||
|
source.ProjectID,
|
||||||
|
)
|
||||||
|
return pagesOrphanCleanupInvalidMarker, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var runtime model.PagesProjectSourceRuntime
|
||||||
|
runtimeExists, err := lockOptionalPagesCleanupRecord(tx, &runtime, "source_id = ?", source.ID)
|
||||||
|
if err != nil {
|
||||||
|
return pagesOrphanCleanupSkipped, false, err
|
||||||
|
}
|
||||||
|
// Read the real clock only after obtaining the runtime row lock. The scanner
|
||||||
|
// snapshot time is only an isolation cutoff and may be stale after lock wait.
|
||||||
|
leaseCheckedAt := time.Now()
|
||||||
|
if runtimeExists && runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(leaseCheckedAt) {
|
||||||
|
return pagesOrphanCleanupLeaseBusy, false, nil
|
||||||
|
}
|
||||||
|
return pagesOrphanCleanupSkipped, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func reconcileLockedPagesOrphanUpload(
|
||||||
|
ctx context.Context,
|
||||||
|
tx *gorm.DB,
|
||||||
|
uploadID uint64,
|
||||||
|
marker pagesOrphanMarker,
|
||||||
|
systemUserID uint64,
|
||||||
|
cutoff time.Time,
|
||||||
|
) (pagesOrphanCleanupOutcome, bool, error) {
|
||||||
|
var lockedUpload model.Upload
|
||||||
|
found, err := lockOptionalPagesCleanupRecord(tx, &lockedUpload, "id = ?", uploadID)
|
||||||
|
if err != nil || !found {
|
||||||
|
return pagesOrphanCleanupSkipped, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
lockedMarker, err := parsePagesOrphanMarker(lockedUpload.Metadata)
|
||||||
|
if err != nil {
|
||||||
|
logger.WarnF(ctx, "[PagesSource] orphan upload marker changed or invalid: upload_id=%d error=%v", uploadID, err)
|
||||||
|
return pagesOrphanCleanupInvalidMarker, true, nil
|
||||||
|
}
|
||||||
|
if lockedUpload.Status != model.UploadStatusUsed ||
|
||||||
|
lockedUpload.UserID != systemUserID ||
|
||||||
|
lockedUpload.Type != upload.ReservedPagesDeploymentType ||
|
||||||
|
!lockedUpload.CreatedAt.Before(cutoff) {
|
||||||
|
return pagesOrphanCleanupSkipped, true, nil
|
||||||
|
}
|
||||||
|
if lockedMarker.ProjectID != marker.ProjectID || !sameOptionalPagesSourceID(lockedMarker.SourceID, marker.SourceID) {
|
||||||
|
logger.WarnF(ctx, "[PagesSource] orphan upload marker changed during reconciliation: upload_id=%d", uploadID)
|
||||||
|
return pagesOrphanCleanupInvalidMarker, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var references int64
|
||||||
|
if err := tx.Model(&model.PagesDeployment{}).
|
||||||
|
Where("upload_id = ?", lockedUpload.ID).
|
||||||
|
Count(&references).Error; err != nil {
|
||||||
|
return pagesOrphanCleanupSkipped, true, err
|
||||||
|
}
|
||||||
|
if references > 0 {
|
||||||
|
return pagesOrphanCleanupReferenced, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
transitioned, err := upload.RemoveLockedTx(tx, &lockedUpload)
|
||||||
|
if err != nil {
|
||||||
|
return pagesOrphanCleanupSkipped, true, err
|
||||||
|
}
|
||||||
|
if transitioned {
|
||||||
|
return pagesOrphanCleanupReconciled, true, nil
|
||||||
|
}
|
||||||
|
return pagesOrphanCleanupSkipped, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lockOptionalPagesCleanupRecord(
|
||||||
|
tx *gorm.DB,
|
||||||
|
value any,
|
||||||
|
query string,
|
||||||
|
args ...any,
|
||||||
|
) (bool, error) {
|
||||||
|
err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
|
||||||
|
Where(query, args...).
|
||||||
|
First(value).Error
|
||||||
|
if err == nil {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePagesOrphanMarker(metadata model.UploadMetadata) (pagesOrphanMarker, error) {
|
||||||
|
if metadata.Extra == nil {
|
||||||
|
return pagesOrphanMarker{}, errors.New("pages marker metadata missing")
|
||||||
|
}
|
||||||
|
marker, ok := metadata.Extra[pagesIngestMarkerKey].(string)
|
||||||
|
if !ok || marker != pagesIngestMarkerV2 {
|
||||||
|
return pagesOrphanMarker{}, errors.New("pages marker version invalid")
|
||||||
|
}
|
||||||
|
projectID, err := parsePagesOrphanMetadataID(metadata.Extra, pagesProjectIDMetadataKey)
|
||||||
|
if err != nil {
|
||||||
|
return pagesOrphanMarker{}, err
|
||||||
|
}
|
||||||
|
result := pagesOrphanMarker{ProjectID: projectID}
|
||||||
|
if _, exists := metadata.Extra[pagesSourceIDMetadataKey]; exists {
|
||||||
|
sourceID, err := parsePagesOrphanMetadataID(metadata.Extra, pagesSourceIDMetadataKey)
|
||||||
|
if err != nil {
|
||||||
|
return pagesOrphanMarker{}, err
|
||||||
|
}
|
||||||
|
result.SourceID = &sourceID
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePagesOrphanMetadataID(extra map[string]any, key string) (uint, error) {
|
||||||
|
raw, exists := extra[key]
|
||||||
|
if !exists {
|
||||||
|
return 0, fmt.Errorf("pages marker %s missing", key)
|
||||||
|
}
|
||||||
|
value, ok := raw.(string)
|
||||||
|
if !ok || value == "" {
|
||||||
|
return 0, fmt.Errorf("pages marker %s must be a decimal string", key)
|
||||||
|
}
|
||||||
|
parsed, err := strconv.ParseUint(value, 10, 64)
|
||||||
|
maxModelID := uint64(^uint(0) >> 1)
|
||||||
|
if err != nil || parsed == 0 || parsed > maxModelID || strconv.FormatUint(parsed, 10) != value {
|
||||||
|
return 0, fmt.Errorf("pages marker %s is not a canonical non-zero decimal ID", key)
|
||||||
|
}
|
||||||
|
return uint(parsed), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameOptionalPagesSourceID(left, right *uint) bool {
|
||||||
|
if left == nil || right == nil {
|
||||||
|
return left == nil && right == nil
|
||||||
|
}
|
||||||
|
return *left == *right
|
||||||
|
}
|
||||||
@@ -0,0 +1,381 @@
|
|||||||
|
// Copyright 2026 Arctel.net
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package pages
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/db"
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/model"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParsePagesOrphanMarker(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
extra map[string]any
|
||||||
|
wantProject uint
|
||||||
|
wantSource uint
|
||||||
|
wantSourceOK bool
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "manual source marker",
|
||||||
|
extra: map[string]any{
|
||||||
|
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||||
|
pagesProjectIDMetadataKey: "12",
|
||||||
|
},
|
||||||
|
wantProject: 12,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "persistent source marker",
|
||||||
|
extra: map[string]any{
|
||||||
|
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||||
|
pagesProjectIDMetadataKey: "12",
|
||||||
|
pagesSourceIDMetadataKey: "34",
|
||||||
|
},
|
||||||
|
wantProject: 12,
|
||||||
|
wantSource: 34,
|
||||||
|
wantSourceOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "project ID must be canonical decimal",
|
||||||
|
extra: map[string]any{
|
||||||
|
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||||
|
pagesProjectIDMetadataKey: "012",
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "source ID must be a string",
|
||||||
|
extra: map[string]any{
|
||||||
|
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||||
|
pagesProjectIDMetadataKey: "12",
|
||||||
|
pagesSourceIDMetadataKey: float64(34),
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero ID rejected",
|
||||||
|
extra: map[string]any{
|
||||||
|
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||||
|
pagesProjectIDMetadataKey: "0",
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "wrong marker version rejected",
|
||||||
|
extra: map[string]any{
|
||||||
|
pagesIngestMarkerKey: "pages_deployment_v1",
|
||||||
|
pagesProjectIDMetadataKey: "12",
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := parsePagesOrphanMarker(model.UploadMetadata{Extra: test.extra})
|
||||||
|
if gotErr := err != nil; gotErr != test.wantErr {
|
||||||
|
t.Fatalf("parsePagesOrphanMarker(%v) error = %v, want error presence = %t", test.extra, err, test.wantErr)
|
||||||
|
}
|
||||||
|
if test.wantErr {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if got.ProjectID != test.wantProject {
|
||||||
|
t.Errorf("parsePagesOrphanMarker(%v).ProjectID = %d, want %d", test.extra, got.ProjectID, test.wantProject)
|
||||||
|
}
|
||||||
|
if gotSourceOK := got.SourceID != nil; gotSourceOK != test.wantSourceOK {
|
||||||
|
t.Fatalf("parsePagesOrphanMarker(%v).SourceID presence = %t, want %t", test.extra, gotSourceOK, test.wantSourceOK)
|
||||||
|
}
|
||||||
|
if got.SourceID != nil && *got.SourceID != test.wantSource {
|
||||||
|
t.Errorf("parsePagesOrphanMarker(%v).SourceID = %d, want %d", test.extra, *got.SourceID, test.wantSource)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilePagesOrphanUploadsDeletesEligibleUploadOnce(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
project := createPagesOrphanProject(t, ctx, "eligible-orphan")
|
||||||
|
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||||
|
if err := upload.RebuildUploadStats(ctx); err != nil {
|
||||||
|
t.Fatalf("RebuildUploadStats() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if summary.Candidates != 1 || summary.Reconciled != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||||
|
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one reconciled candidate", summary)
|
||||||
|
}
|
||||||
|
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusDeleted)
|
||||||
|
assertPagesCleanupTotalStat(t, ctx, 0)
|
||||||
|
|
||||||
|
second, err := ReconcilePagesOrphanUploads(ctx, now.Add(time.Minute))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if second.Candidates != 0 || cleanupOutcomeTotal(second) != 0 {
|
||||||
|
t.Errorf("second ReconcilePagesOrphanUploads() summary = %+v, want empty", second)
|
||||||
|
}
|
||||||
|
assertPagesCleanupTotalStat(t, ctx, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilePagesOrphanUploadsAllowsDeletedProjectAndSource(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
missingSourceID := uint(9876)
|
||||||
|
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), 8765, &missingSourceID)
|
||||||
|
|
||||||
|
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if summary.Candidates != 1 || summary.Reconciled != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||||
|
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want deleted project/source treated as one orphan", summary)
|
||||||
|
}
|
||||||
|
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusDeleted)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilePagesOrphanUploadsSkipsBusyLeaseAndSourceMismatch(t *testing.T) {
|
||||||
|
t.Run("unexpired source lease", func(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
ctx := context.Background()
|
||||||
|
realNow := time.Now().UTC()
|
||||||
|
// A deliberately future scanner snapshot proves lease freshness uses the
|
||||||
|
// real clock after the runtime lock, not this isolation-cutoff input.
|
||||||
|
scannerNow := realNow.Add(24 * time.Hour)
|
||||||
|
project := createPagesOrphanProject(t, ctx, "busy-orphan")
|
||||||
|
source := createPagesOrphanSource(t, ctx, project.ID)
|
||||||
|
future := realNow.Add(time.Hour)
|
||||||
|
if err := db.DB(ctx).Create(&model.PagesProjectSourceRuntime{
|
||||||
|
SourceID: source.ID,
|
||||||
|
LeaseToken: "busy-worker",
|
||||||
|
LeaseExpiresAt: &future,
|
||||||
|
}).Error; err != nil {
|
||||||
|
t.Fatalf("create busy source runtime error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
candidate := createPagesOrphanUpload(t, ctx, realNow.Add(-3*time.Hour), project.ID, &source.ID)
|
||||||
|
|
||||||
|
summary, err := ReconcilePagesOrphanUploads(ctx, scannerNow)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if summary.LeaseBusy != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||||
|
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one lease-busy candidate", summary)
|
||||||
|
}
|
||||||
|
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("source belongs to another project", func(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
markerProject := createPagesOrphanProject(t, ctx, "marker-project")
|
||||||
|
actualProject := createPagesOrphanProject(t, ctx, "actual-project")
|
||||||
|
source := createPagesOrphanSource(t, ctx, actualProject.ID)
|
||||||
|
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), markerProject.ID, &source.ID)
|
||||||
|
|
||||||
|
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if summary.InvalidMarker != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||||
|
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one ownership mismatch", summary)
|
||||||
|
}
|
||||||
|
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcilePagesOrphanUploadsRejectsMalformedMarker(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), 1, nil)
|
||||||
|
metadata := candidate.Metadata
|
||||||
|
metadata.Extra[pagesProjectIDMetadataKey] = "01"
|
||||||
|
candidate.Metadata = metadata
|
||||||
|
if err := db.DB(ctx).Save(candidate).Error; err != nil {
|
||||||
|
t.Fatalf("seed malformed candidate marker error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if summary.InvalidMarker != 1 || cleanupOutcomeTotal(summary) != 1 {
|
||||||
|
t.Errorf("ReconcilePagesOrphanUploads() summary = %+v, want one invalid marker", summary)
|
||||||
|
}
|
||||||
|
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPagesOrphanCleanupAndDeploymentCommitInterleavings(t *testing.T) {
|
||||||
|
t.Run("deployment reference commits first", func(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
project := createPagesOrphanProject(t, ctx, "deployment-first")
|
||||||
|
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||||
|
marker, err := parsePagesOrphanMarker(candidate.Metadata)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parsePagesOrphanMarker() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := db.DB(ctx).Create(&model.PagesDeployment{
|
||||||
|
ProjectID: project.ID,
|
||||||
|
DeploymentNumber: 1,
|
||||||
|
Checksum: "deployment-first",
|
||||||
|
Status: model.PagesDeploymentStatusUploaded,
|
||||||
|
UploadID: candidate.ID,
|
||||||
|
}).Error; err != nil {
|
||||||
|
t.Fatalf("create deployment reference error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
outcome, err := reconcilePagesOrphanUploadCandidate(ctx, candidate, marker, 999, now.Add(-2*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reconcilePagesOrphanUploadCandidate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if outcome != pagesOrphanCleanupReferenced {
|
||||||
|
t.Errorf("reconcilePagesOrphanUploadCandidate() outcome = %d, want %d", outcome, pagesOrphanCleanupReferenced)
|
||||||
|
}
|
||||||
|
assertPagesCleanupUploadStatus(t, ctx, candidate.ID, model.UploadStatusUsed)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("cleanup commits first", func(t *testing.T) {
|
||||||
|
cleanup := setupPagesTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
project := createPagesOrphanProject(t, ctx, "cleanup-first")
|
||||||
|
candidate := createPagesOrphanUpload(t, ctx, now.Add(-3*time.Hour), project.ID, nil)
|
||||||
|
|
||||||
|
summary, err := ReconcilePagesOrphanUploads(ctx, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReconcilePagesOrphanUploads() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if summary.Reconciled != 1 {
|
||||||
|
t.Fatalf("ReconcilePagesOrphanUploads() summary = %+v, want one reconciled candidate", summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
target := &model.PagesDeployment{ProjectID: project.ID, UploadID: candidate.ID}
|
||||||
|
err = db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||||
|
var lockedProject model.PagesProject
|
||||||
|
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&lockedProject, project.ID).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return lockSourceDeploymentUploadsTx(tx, target, upload.IngestResult{}, false)
|
||||||
|
})
|
||||||
|
if !errors.Is(err, errSourceFinalFence) {
|
||||||
|
t.Errorf("final deployment upload lock after cleanup error = %v, want %v", err, errSourceFinalFence)
|
||||||
|
}
|
||||||
|
var references int64
|
||||||
|
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("upload_id = ?", candidate.ID).Count(&references).Error; err != nil {
|
||||||
|
t.Fatalf("count deployment references error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if references != 0 {
|
||||||
|
t.Errorf("deployment references after cleanup-first interleaving = %d, want 0", references)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanupOutcomeTotal(summary PagesOrphanCleanupSummary) int {
|
||||||
|
return summary.Reconciled + summary.Referenced + summary.LeaseBusy + summary.InvalidMarker + summary.Skipped + summary.Failed
|
||||||
|
}
|
||||||
|
|
||||||
|
func createPagesOrphanProject(t *testing.T, ctx context.Context, slug string) *model.PagesProject {
|
||||||
|
t.Helper()
|
||||||
|
project := &model.PagesProject{Name: slug, Slug: slug, Enabled: true}
|
||||||
|
if err := db.DB(ctx).Create(project).Error; err != nil {
|
||||||
|
t.Fatalf("create Pages orphan project %q error = %v, want nil", slug, err)
|
||||||
|
}
|
||||||
|
return project
|
||||||
|
}
|
||||||
|
|
||||||
|
func createPagesOrphanSource(t *testing.T, ctx context.Context, projectID uint) *model.PagesProjectSource {
|
||||||
|
t.Helper()
|
||||||
|
source := &model.PagesProjectSource{
|
||||||
|
ProjectID: projectID,
|
||||||
|
SourceType: PagesSourceTypeRemoteURL,
|
||||||
|
ConfigVersion: 1,
|
||||||
|
SourceIdentity: "orphan-source-identity",
|
||||||
|
}
|
||||||
|
if err := db.DB(ctx).Create(source).Error; err != nil {
|
||||||
|
t.Fatalf("create Pages orphan source for project %d error = %v, want nil", projectID, err)
|
||||||
|
}
|
||||||
|
return source
|
||||||
|
}
|
||||||
|
|
||||||
|
func createPagesOrphanUpload(
|
||||||
|
t *testing.T,
|
||||||
|
ctx context.Context,
|
||||||
|
createdAt time.Time,
|
||||||
|
projectID uint,
|
||||||
|
sourceID *uint,
|
||||||
|
) *model.Upload {
|
||||||
|
t.Helper()
|
||||||
|
extra := map[string]any{
|
||||||
|
pagesIngestMarkerKey: pagesIngestMarkerV2,
|
||||||
|
pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10),
|
||||||
|
}
|
||||||
|
if sourceID != nil {
|
||||||
|
extra[pagesSourceIDMetadataKey] = strconv.FormatUint(uint64(*sourceID), 10)
|
||||||
|
}
|
||||||
|
candidate := &model.Upload{
|
||||||
|
UserID: 999,
|
||||||
|
FileName: "site.zip",
|
||||||
|
FilePath: "pages/orphan-site.zip",
|
||||||
|
FileSize: 64,
|
||||||
|
MimeType: "application/zip",
|
||||||
|
Extension: "zip",
|
||||||
|
Hash: "orphan-checksum",
|
||||||
|
Type: upload.ReservedPagesDeploymentType,
|
||||||
|
Status: model.UploadStatusUsed,
|
||||||
|
AccessMode: 0,
|
||||||
|
Metadata: model.UploadMetadata{Extra: extra},
|
||||||
|
CreatedAt: createdAt,
|
||||||
|
UpdatedAt: createdAt,
|
||||||
|
}
|
||||||
|
if err := db.DB(ctx).Create(candidate).Error; err != nil {
|
||||||
|
t.Fatalf("create Pages orphan upload error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertPagesCleanupUploadStatus(t *testing.T, ctx context.Context, uploadID uint64, want model.UploadStatus) {
|
||||||
|
t.Helper()
|
||||||
|
var got model.Upload
|
||||||
|
if err := db.DB(ctx).First(&got, uploadID).Error; err != nil {
|
||||||
|
t.Fatalf("load upload %d error = %v, want nil", uploadID, err)
|
||||||
|
}
|
||||||
|
if got.Status != want {
|
||||||
|
t.Errorf("upload %d status = %q, want %q", uploadID, got.Status, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertPagesCleanupTotalStat(t *testing.T, ctx context.Context, want int64) {
|
||||||
|
t.Helper()
|
||||||
|
var stat model.UploadStat
|
||||||
|
if err := db.DB(ctx).Where("dimension = ? AND stat_key = ?", model.UploadStatDimensionTotal, "").First(&stat).Error; err != nil {
|
||||||
|
t.Fatalf("load total upload stat error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if stat.FileCount != want {
|
||||||
|
t.Errorf("total upload stat FileCount = %d, want %d", stat.FileCount, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
// Copyright 2026 Arctel.net
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// PagesOrphanUploadCandidateLimit bounds one delayed Pages upload cleanup pass.
|
||||||
|
PagesOrphanUploadCandidateLimit = 100
|
||||||
|
|
||||||
|
pagesOrphanMarkerPredicatePostgres = "w_uploads.metadata #>> '{extra,pages_ingest_marker}' = ?"
|
||||||
|
pagesOrphanMarkerPredicateSQLite = "CASE WHEN json_valid(w_uploads.metadata) THEN json_extract(w_uploads.metadata, '$.extra.pages_ingest_marker') ELSE NULL END = ?"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PagesOrphanUploadCandidateQuery describes the fail-closed SQL candidate set
|
||||||
|
// for delayed Pages upload compensation.
|
||||||
|
type PagesOrphanUploadCandidateQuery struct {
|
||||||
|
SystemUserID uint64
|
||||||
|
UploadType string
|
||||||
|
Marker string
|
||||||
|
CreatedBefore time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListPagesOrphanUploadCandidates returns at most 100 unreferenced, isolated
|
||||||
|
// Pages V2 upload records. Callers must still lock and recheck every condition
|
||||||
|
// before deleting a candidate.
|
||||||
|
func ListPagesOrphanUploadCandidates(
|
||||||
|
ctx context.Context,
|
||||||
|
input PagesOrphanUploadCandidateQuery,
|
||||||
|
) ([]Upload, error) {
|
||||||
|
if input.SystemUserID == 0 || input.UploadType == "" || input.Marker == "" || input.CreatedBefore.IsZero() {
|
||||||
|
return nil, errors.New("invalid pages orphan upload candidate query")
|
||||||
|
}
|
||||||
|
markerPredicate, err := pagesOrphanMarkerPredicate(db.DB(ctx).Name())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
deploymentTable := (PagesDeployment{}).TableName()
|
||||||
|
uploadTable := (Upload{}).TableName()
|
||||||
|
var candidates []Upload
|
||||||
|
err = db.DB(ctx).
|
||||||
|
Model(&Upload{}).
|
||||||
|
Where(uploadTable+".status = ?", UploadStatusUsed).
|
||||||
|
Where(uploadTable+".user_id = ?", input.SystemUserID).
|
||||||
|
Where(uploadTable+".type = ?", input.UploadType).
|
||||||
|
Where(uploadTable+".created_at < ?", input.CreatedBefore).
|
||||||
|
Where(markerPredicate, input.Marker).
|
||||||
|
Where("NOT EXISTS (SELECT 1 FROM " + deploymentTable + " WHERE " + deploymentTable + ".upload_id = " + uploadTable + ".id)").
|
||||||
|
Order(uploadTable + ".id ASC").
|
||||||
|
Limit(PagesOrphanUploadCandidateLimit).
|
||||||
|
Find(&candidates).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return candidates, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pagesOrphanMarkerPredicate(dialect string) (string, error) {
|
||||||
|
switch dialect {
|
||||||
|
case "postgres":
|
||||||
|
return pagesOrphanMarkerPredicatePostgres, nil
|
||||||
|
case "sqlite":
|
||||||
|
return pagesOrphanMarkerPredicateSQLite, nil
|
||||||
|
default:
|
||||||
|
return "", errors.New("unsupported database dialect for Pages orphan cleanup")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
// Copyright 2026 Arctel.net
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Rain-kl/Wavelet/internal/db"
|
||||||
|
"github.com/glebarez/sqlite"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPagesOrphanMarkerPredicate(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
dialect string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "postgres jsonb path",
|
||||||
|
dialect: "postgres",
|
||||||
|
want: "metadata #>> '{extra,pages_ingest_marker}'",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "sqlite guarded json extract",
|
||||||
|
dialect: "sqlite",
|
||||||
|
want: "CASE WHEN json_valid(w_uploads.metadata) THEN json_extract",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown dialect rejected",
|
||||||
|
dialect: "mysql",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
got, err := pagesOrphanMarkerPredicate(test.dialect)
|
||||||
|
if gotErr := err != nil; gotErr != test.wantErr {
|
||||||
|
t.Fatalf("pagesOrphanMarkerPredicate(%q) error = %v, want error presence = %t", test.dialect, err, test.wantErr)
|
||||||
|
}
|
||||||
|
if test.want != "" && !strings.Contains(got, test.want) {
|
||||||
|
t.Errorf("pagesOrphanMarkerPredicate(%q) = %q, want substring %q", test.dialect, got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListPagesOrphanUploadCandidatesFiltersAndLimits(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
gormDB := setupPagesCleanupModelTestDB(t)
|
||||||
|
cutoff := time.Now().UTC().Add(-2 * time.Hour)
|
||||||
|
old := cutoff.Add(-time.Minute)
|
||||||
|
marker := UploadMetadata{Extra: map[string]any{
|
||||||
|
"pages_ingest_marker": "pages_deployment_v2",
|
||||||
|
"pages_project_id": "1",
|
||||||
|
}}
|
||||||
|
|
||||||
|
valid := make([]Upload, 0, PagesOrphanUploadCandidateLimit+1)
|
||||||
|
for index := 0; index < PagesOrphanUploadCandidateLimit+1; index++ {
|
||||||
|
valid = append(valid, pagesCleanupModelUpload(uint64(index+100), 999, "openflare_pages_deployment", UploadStatusUsed, old, marker))
|
||||||
|
}
|
||||||
|
if err := gormDB.Create(&valid).Error; err != nil {
|
||||||
|
t.Fatalf("create valid candidates error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
referenced := pagesCleanupModelUpload(1, 999, "openflare_pages_deployment", UploadStatusUsed, old, marker)
|
||||||
|
wrongOwner := pagesCleanupModelUpload(2, 1000, "openflare_pages_deployment", UploadStatusUsed, old, marker)
|
||||||
|
wrongType := pagesCleanupModelUpload(3, 999, "generic", UploadStatusUsed, old, marker)
|
||||||
|
wrongStatus := pagesCleanupModelUpload(4, 999, "openflare_pages_deployment", UploadStatusPending, old, marker)
|
||||||
|
fresh := pagesCleanupModelUpload(5, 999, "openflare_pages_deployment", UploadStatusUsed, cutoff, marker)
|
||||||
|
wrongMarker := pagesCleanupModelUpload(6, 999, "openflare_pages_deployment", UploadStatusUsed, old, UploadMetadata{Extra: map[string]any{
|
||||||
|
"pages_ingest_marker": "pages_deployment_v1",
|
||||||
|
"pages_project_id": "1",
|
||||||
|
}})
|
||||||
|
for _, upload := range []Upload{referenced, wrongOwner, wrongType, wrongStatus, fresh, wrongMarker} {
|
||||||
|
if err := gormDB.Create(&upload).Error; err != nil {
|
||||||
|
t.Fatalf("create filtered upload %d error = %v, want nil", upload.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := gormDB.Create(&PagesDeployment{
|
||||||
|
ProjectID: 1,
|
||||||
|
DeploymentNumber: 1,
|
||||||
|
Checksum: "referenced",
|
||||||
|
Status: PagesDeploymentStatusUploaded,
|
||||||
|
UploadID: referenced.ID,
|
||||||
|
}).Error; err != nil {
|
||||||
|
t.Fatalf("create referenced deployment error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidJSON := pagesCleanupModelUpload(7, 999, "openflare_pages_deployment", UploadStatusUsed, old, marker)
|
||||||
|
if err := gormDB.Create(&invalidJSON).Error; err != nil {
|
||||||
|
t.Fatalf("create invalid JSON upload error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := gormDB.Table((Upload{}).TableName()).Where("id = ?", invalidJSON.ID).
|
||||||
|
UpdateColumn("metadata", "{invalid").Error; err != nil {
|
||||||
|
t.Fatalf("corrupt upload metadata error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := ListPagesOrphanUploadCandidates(ctx, PagesOrphanUploadCandidateQuery{
|
||||||
|
SystemUserID: 999,
|
||||||
|
UploadType: "openflare_pages_deployment",
|
||||||
|
Marker: "pages_deployment_v2",
|
||||||
|
CreatedBefore: cutoff,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListPagesOrphanUploadCandidates() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(got) != PagesOrphanUploadCandidateLimit {
|
||||||
|
t.Fatalf("ListPagesOrphanUploadCandidates() count = %d, want %d", len(got), PagesOrphanUploadCandidateLimit)
|
||||||
|
}
|
||||||
|
for index, candidate := range got {
|
||||||
|
wantID := uint64(index + 100)
|
||||||
|
if candidate.ID != wantID {
|
||||||
|
t.Errorf("ListPagesOrphanUploadCandidates()[%d].ID = %d, want %d", index, candidate.ID, wantID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListPagesOrphanUploadCandidatesSkipsInvalidSQLiteJSON(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
gormDB := setupPagesCleanupModelTestDB(t)
|
||||||
|
cutoff := time.Now().UTC().Add(-2 * time.Hour)
|
||||||
|
upload := pagesCleanupModelUpload(1, 999, "openflare_pages_deployment", UploadStatusUsed, cutoff.Add(-time.Minute), UploadMetadata{})
|
||||||
|
if err := gormDB.Create(&upload).Error; err != nil {
|
||||||
|
t.Fatalf("create invalid JSON candidate error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := gormDB.Table((Upload{}).TableName()).Where("id = ?", upload.ID).
|
||||||
|
UpdateColumn("metadata", "{invalid").Error; err != nil {
|
||||||
|
t.Fatalf("corrupt upload metadata error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := ListPagesOrphanUploadCandidates(ctx, PagesOrphanUploadCandidateQuery{
|
||||||
|
SystemUserID: 999,
|
||||||
|
UploadType: "openflare_pages_deployment",
|
||||||
|
Marker: "pages_deployment_v2",
|
||||||
|
CreatedBefore: cutoff,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListPagesOrphanUploadCandidates(invalid JSON) error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Errorf("ListPagesOrphanUploadCandidates(invalid JSON) count = %d, want 0", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupPagesCleanupModelTestDB(t *testing.T) *gorm.DB {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||||
|
DisableForeignKeyConstraintWhenMigrating: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open Pages cleanup model test database error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
if err := gormDB.AutoMigrate(&Upload{}, &PagesDeployment{}); err != nil {
|
||||||
|
t.Fatalf("migrate Pages cleanup model test database error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
db.SetDB(gormDB)
|
||||||
|
t.Cleanup(func() { db.SetDB(nil) })
|
||||||
|
return gormDB
|
||||||
|
}
|
||||||
|
|
||||||
|
func pagesCleanupModelUpload(
|
||||||
|
id uint64,
|
||||||
|
userID uint64,
|
||||||
|
uploadType string,
|
||||||
|
status UploadStatus,
|
||||||
|
createdAt time.Time,
|
||||||
|
metadata UploadMetadata,
|
||||||
|
) Upload {
|
||||||
|
return Upload{
|
||||||
|
ID: id,
|
||||||
|
UserID: userID,
|
||||||
|
FileName: "site.zip",
|
||||||
|
FilePath: "pages/site.zip",
|
||||||
|
FileSize: 10,
|
||||||
|
MimeType: "application/zip",
|
||||||
|
Extension: "zip",
|
||||||
|
Hash: "checksum",
|
||||||
|
Type: uploadType,
|
||||||
|
Status: status,
|
||||||
|
AccessMode: 0,
|
||||||
|
Metadata: metadata,
|
||||||
|
CreatedAt: createdAt,
|
||||||
|
UpdatedAt: createdAt,
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user