fix(pages): 收紧部署包与 Agent 同步边界

完成 V2 Phase 0 安全与一致性前置:统一真实归档限额、流式拉取、候选裁剪、保留上传删除语义及 Pages 路由引用锁。
This commit is contained in:
deqiying
2026-07-19 16:42:45 +08:00
parent f386674464
commit 4e8ec23264
47 changed files with 4005 additions and 759 deletions
+16 -18
View File
@@ -10,16 +10,15 @@ import (
"fmt"
"time"
uploadcache "github.com/Rain-kl/Wavelet/internal/apps/upload/cache"
"github.com/Rain-kl/Wavelet/internal/apps/upload/ingest"
"github.com/Rain-kl/Wavelet/internal/apps/upload/shared"
uploadstats "github.com/Rain-kl/Wavelet/internal/apps/upload/stats"
uploadstorage "github.com/Rain-kl/Wavelet/internal/apps/upload/storage"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
"github.com/Rain-kl/Wavelet/internal/task"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const (
@@ -77,32 +76,31 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
for _, u := range unusedUploads {
totalProcessed++
transitioned := false
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&model.Upload{}).
Where("id = ? AND status = ?", u.ID, model.UploadStatusPending).
Update("status", model.UploadStatusDeleted).Error; err != nil {
var locked model.Upload
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
Where("id = ?", u.ID).
First(&locked).Error; err != nil {
return err
}
_, backend, err := storage.Active(ctx)
if err != nil {
return err
if locked.Status != model.UploadStatusPending || !locked.CreatedAt.Before(oneHourAgo) {
return nil
}
if err := backend.Delete(ctx, u.FilePath); err != nil {
return err
}
return nil
var err error
transitioned, err = ingest.RemoveLockedTx(tx, &locked)
return err
}); err != nil {
task.AppendLog(ctx, "清理上传文件失败 [ID:%d]: %v", u.ID, err)
lastID = u.ID
continue
}
uploadstats.RecordUploadStatsRemove(ctx, &u)
uploadcache.InvalidateUploadMetaCache(ctx, u.ID)
totalDeleted++
ingest.InvalidateUploadMetaCache(ctx, u.ID)
if transitioned {
totalDeleted++
}
lastID = u.ID
}
}
+28 -2
View File
@@ -19,6 +19,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/upload/filesrv"
"github.com/Rain-kl/Wavelet/internal/apps/upload/shared"
uploadstats "github.com/Rain-kl/Wavelet/internal/apps/upload/stats"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/diskcache"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -33,13 +34,17 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
// Mock S3 存储(让 DeleteObject 总是成功)
deleteCount := 0
// Mock S3 存储并记录 Delete,cleanup 不应物理删除共享对象。
storageMock := storage.MockStorage(
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
return nil
},
func(ctx context.Context, key string) (*storage.Object, error) { return nil, nil },
func(ctx context.Context, key string) error { return nil },
func(ctx context.Context, key string) error {
deleteCount++
return nil
},
)
defer storageMock()
storage.IsEnabledFunc = func() bool { return true }
@@ -86,6 +91,7 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
for _, r := range records {
err := db.DB(ctx).Create(r).Error
require.NoError(t, err)
require.NoError(t, uploadstats.ApplyUploadStatsAdd(ctx, r))
}
// 准备推送历史测试数据:1个旧的(应删除),1个新的(应保留)
@@ -147,6 +153,26 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
var usedCount int64
db.DB(ctx).Model(&model.Upload{}).Where("status = ?", model.UploadStatusUsed).Count(&usedCount)
assert.Equal(t, int64(1), usedCount, "used 状态的文件不应受影响")
assert.Equal(t, 0, deleteCount, "记录级 cleanup 不应调用 storage backend Delete")
var totalStats model.UploadStat
err = db.DB(ctx).
Where("dimension = ? AND stat_key = ?", model.UploadStatDimensionTotal, "").
First(&totalStats).Error
require.NoError(t, err)
assert.Equal(t, int64(2), totalStats.FileCount, "cleanup 后统计只应保留 used 与最近 pending 记录")
assert.Equal(t, int64(768), totalStats.FileSize, "cleanup 后统计大小应只扣减一次")
_, err = handler.Execute(ctx, nil)
require.NoError(t, err)
var statsAfterSecondRun model.UploadStat
err = db.DB(ctx).
Where("dimension = ? AND stat_key = ?", model.UploadStatDimensionTotal, "").
First(&statsAfterSecondRun).Error
require.NoError(t, err)
assert.Equal(t, totalStats.FileCount, statsAfterSecondRun.FileCount, "重复 cleanup 不应再次扣减统计")
assert.Equal(t, totalStats.FileSize, statsAfterSecondRun.FileSize, "重复 cleanup 不应再次扣减统计大小")
assert.Equal(t, 0, deleteCount, "重复 cleanup 仍不应调用 storage backend Delete")
// 验证推送历史数据状态:10天前的应被删除,今天的应保留
var pushCount int64