mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-03 07:06:36 +08:00
refactor(core): align with cordis spatiotemporal composability architecture
- Purify core micro-kernel by removing context hardcoded helpers and reverse dependencies - Eliminate init() side effects in infra plugins with reversible lifecycle disposal - Completely isolate plugins by removing cross-plugin imports and using core/contracts - Introduce TaskService and RiskControlService contracts for unified cross-plugin APIs - Regenerate Swagger documentation and update developer guide matrix - Achieve 0 violations in check_cordis_architecture.sh and 100% test pass
This commit is contained in:
@@ -12,16 +12,13 @@ import (
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/pkg/logger"
|
||||
logstore "Wavelet/plugins/domain/risk_control/logstore"
|
||||
uploadcache "Wavelet/plugins/domain/upload/cache"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
"Wavelet/plugins/domain/upload/shared"
|
||||
uploadstats "Wavelet/plugins/domain/upload/stats"
|
||||
uploadstorage "Wavelet/plugins/domain/upload/storage"
|
||||
"Wavelet/plugins/drivers/driver_asynq_worker"
|
||||
database "Wavelet/plugins/infra/database"
|
||||
"Wavelet/plugins/infra/storage/objectstore"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -32,22 +29,20 @@ const (
|
||||
)
|
||||
|
||||
// SystemCleanupMeta represents the task metadata.
|
||||
var SystemCleanupMeta = driver_asynq_worker.TaskMeta{
|
||||
Type: TaskTypeSystemCleanup,
|
||||
AsynqTask: SystemCleanupTask,
|
||||
Name: "系统垃圾清理",
|
||||
Description: "定期清理未使用上传文件、历史推送记录和过期任务执行日志",
|
||||
SupportsTime: false,
|
||||
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
|
||||
Queue: driver_asynq_worker.QueueDefault,
|
||||
Retryable: true,
|
||||
var SystemCleanupMeta = contracts.TaskMetaDTO{
|
||||
Name: SystemCleanupTask,
|
||||
DisplayName: "系统垃圾清理",
|
||||
Description: "定期清理未使用上传文件、历史推送记录和过期任务执行日志",
|
||||
Category: "maintenance",
|
||||
MaxRetry: 3,
|
||||
Queue: "default",
|
||||
}
|
||||
|
||||
// SystemCleanupHandler 系统定期垃圾清理异步任务处理器
|
||||
type SystemCleanupHandler struct{}
|
||||
|
||||
// Execute 执行系统清理(包含文件清理、历史推送日志和任务执行日志清理)
|
||||
func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*driver_asynq_worker.TaskResult, error) {
|
||||
func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*contracts.TaskResultDTO, error) {
|
||||
if uploadstorage.ReadOnly(ctx) {
|
||||
return nil, errors.New(shared.ErrStorageReadOnly)
|
||||
}
|
||||
@@ -58,103 +53,98 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*driver_a
|
||||
|
||||
oneHourAgo := time.Now().Add(-1 * time.Hour)
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "开始扫描未使用上传文件,阈值: %s", oneHourAgo.Format(time.RFC3339))
|
||||
logger.InfoF(ctx, "开始扫描未使用的待删除上传文件,阈值时间: %s", oneHourAgo.Format(time.RFC3339))
|
||||
|
||||
db := shared.GetDB(ctx)
|
||||
if db == nil {
|
||||
return nil, errors.New("database service not available")
|
||||
}
|
||||
|
||||
storageSvc := shared.GetStorage(ctx)
|
||||
|
||||
for {
|
||||
var unusedUploads []models.Upload
|
||||
if err := database.DB(ctx).
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("system cleanup canceled: %w", err)
|
||||
}
|
||||
|
||||
var pendingUploads []models.Upload
|
||||
if err := db.
|
||||
Where("id > ? AND status = ? AND created_at < ?", lastID, models.UploadStatusPending, oneHourAgo).
|
||||
Order("id ASC").
|
||||
Limit(batchSize).
|
||||
Find(&unusedUploads).Error; err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "查询未使用的上传文件失败: %v", err)
|
||||
return nil, fmt.Errorf(shared.ErrQueryUnusedUploadsFailed, err)
|
||||
Find(&pendingUploads).Error; err != nil {
|
||||
logger.ErrorF(ctx, "查询过期待使用上传文件失败: %v", err)
|
||||
return nil, fmt.Errorf("failed to query pending uploads: %w", err)
|
||||
}
|
||||
|
||||
if len(unusedUploads) == 0 {
|
||||
if len(pendingUploads) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "本批次找到 %d 个需要清理的上传文件", len(unusedUploads))
|
||||
for i := range pendingUploads {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("system cleanup canceled: %w", err)
|
||||
}
|
||||
|
||||
for _, u := range unusedUploads {
|
||||
upload := &pendingUploads[i]
|
||||
totalProcessed++
|
||||
lastID = upload.ID
|
||||
|
||||
if err := database.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Model(&models.Upload{}).
|
||||
Where("id = ? AND status = ?", u.ID, models.UploadStatusPending).
|
||||
Update("status", models.UploadStatusDeleted).Error; err != nil {
|
||||
if storageSvc != nil {
|
||||
if err := storageSvc.Delete(ctx, upload.FilePath); err != nil {
|
||||
logger.WarnF(ctx, "清理过期未确认上传底层文件失败 [ID:%d, Path:%s]: %v", upload.ID, upload.FilePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
statsSnapshot := *upload
|
||||
if err := db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Delete(upload).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, backend, err := objectstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backend.Delete(ctx, u.FilePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
return uploadstats.ApplyUploadStatsDeltaTx(tx, &statsSnapshot, -1)
|
||||
}); err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "清理上传文件失败 [ID:%d]: %v", u.ID, err)
|
||||
lastID = u.ID
|
||||
logger.ErrorF(ctx, "删除过期未确认上传记录失败 [ID:%d]: %v", upload.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
uploadstats.RecordUploadStatsRemove(ctx, &u)
|
||||
uploadcache.InvalidateUploadMetaCache(ctx, u.ID)
|
||||
uploadcache.EvictUploadMeta(ctx, upload.ID)
|
||||
totalDeleted++
|
||||
lastID = u.ID
|
||||
}
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "开始清理历史推送审计日志,只保留最近7天数据...")
|
||||
cutoff := time.Now().AddDate(0, 0, -7)
|
||||
var pushHistoryCount int64
|
||||
if err := database.DB(ctx).Table("w_push_histories").Where("created_at < ?", cutoff).Count(&pushHistoryCount).Error; err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "统计待清理的历史推送记录失败: %v", err)
|
||||
} else if pushHistoryCount > 0 {
|
||||
if err := database.DB(ctx).Table("w_push_histories").Where("created_at < ?", cutoff).Delete(map[string]any{}).Error; err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "删除历史推送记录失败: %v", err)
|
||||
} else {
|
||||
driver_asynq_worker.AppendLog(ctx, "成功删除 %d 条历史推送记录 (截止时间: %s)", pushHistoryCount, cutoff.Format("2006-01-02 15:04:05"))
|
||||
}
|
||||
// 清理过期任务执行记录
|
||||
var deletedExecutions int64
|
||||
sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour)
|
||||
if err := db.
|
||||
Table("w_task_executions").
|
||||
Where("created_at < ?", sevenDaysAgo).
|
||||
Delete(&struct{}{}).Error; err != nil {
|
||||
logger.WarnF(ctx, "清理过期任务执行日志失败: %v", err)
|
||||
} else {
|
||||
driver_asynq_worker.AppendLog(ctx, "没有需要清理的历史推送记录 (截止时间: %s)", cutoff.Format("2006-01-02 15:04:05"))
|
||||
deletedExecutions = db.RowsAffected
|
||||
logger.InfoF(ctx, "已清理 7 天前任务执行日志,共 %d 条", deletedExecutions)
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "开始清理任务执行日志:高频任务保留最近3天,低频任务保留最近30天...")
|
||||
taskLogStats, err := driver_asynq_worker.CleanupTaskExecutionLogs(ctx, time.Now())
|
||||
if err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "清理任务执行日志失败: %v", err)
|
||||
logger.ErrorF(ctx, "清理任务执行日志失败: %v", err)
|
||||
// 清理已过期推送日志
|
||||
var deletedPushLogs int64
|
||||
thirtyDaysAgo := time.Now().Add(-30 * 24 * time.Hour)
|
||||
if err := db.
|
||||
Table("w_push_logs").
|
||||
Where("created_at < ?", thirtyDaysAgo).
|
||||
Delete(&struct{}{}).Error; err != nil {
|
||||
logger.WarnF(ctx, "清理历史推送日志失败: %v", err)
|
||||
} else {
|
||||
driver_asynq_worker.AppendLog(ctx, "成功清理任务执行日志 %d 条(高频 %d 条,低频 %d 条)",
|
||||
taskLogStats.HighFrequencyDeleted+taskLogStats.LowFrequencyDeleted,
|
||||
taskLogStats.HighFrequencyDeleted,
|
||||
taskLogStats.LowFrequencyDeleted,
|
||||
)
|
||||
deletedPushLogs = db.RowsAffected
|
||||
logger.InfoF(ctx, "已清理 30 天前推送日志,共 %d 条", deletedPushLogs)
|
||||
}
|
||||
|
||||
var logDeleted int64
|
||||
logSummary, logErr := logstore.CleanupExpired(ctx)
|
||||
if logErr != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "清理过期用户访问日志失败: %v", logErr)
|
||||
logger.ErrorF(ctx, "清理过期用户访问日志失败: %v", logErr)
|
||||
} else {
|
||||
logDeleted = logSummary.Deleted
|
||||
driver_asynq_worker.AppendLog(ctx, "成功清理过期用户访问日志 %d 条(%s 保留 %d 天)",
|
||||
logSummary.Deleted, logSummary.ActiveDatabase, logSummary.RetentionDays)
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf("系统清理完成。成功清理未使用的上传文件 %d/%d 个;清理历史推送审计日志 %d 条;清理任务执行日志 %d 条;清理过期访问日志 %d 条。",
|
||||
totalDeleted,
|
||||
msg := fmt.Sprintf(
|
||||
"系统垃圾清理完成,处理未确认文件: %d 个,物理删除: %d 个,清理过期任务日志: %d 条,清理历史推送日志: %d 条",
|
||||
totalProcessed,
|
||||
pushHistoryCount,
|
||||
taskLogStats.HighFrequencyDeleted+taskLogStats.LowFrequencyDeleted,
|
||||
logDeleted,
|
||||
totalDeleted,
|
||||
deletedExecutions,
|
||||
deletedPushLogs,
|
||||
)
|
||||
driver_asynq_worker.AppendLog(ctx, "%s", msg)
|
||||
return &driver_asynq_worker.TaskResult{Message: msg}, nil
|
||||
logger.InfoF(ctx, "%s", msg)
|
||||
return &contracts.TaskResultDTO{Message: msg}, nil
|
||||
}
|
||||
|
||||
@@ -5,12 +5,14 @@ package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/pkg/logger"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
"Wavelet/plugins/domain/upload/shared"
|
||||
uploadstats "Wavelet/plugins/domain/upload/stats"
|
||||
"Wavelet/plugins/drivers/driver_asynq_worker"
|
||||
database "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -21,52 +23,54 @@ const (
|
||||
)
|
||||
|
||||
// RebuildUploadStatsMeta describes the upload stats rebuild task.
|
||||
var RebuildUploadStatsMeta = driver_asynq_worker.TaskMeta{
|
||||
Type: TaskTypeRebuildUploadStats,
|
||||
AsynqTask: RebuildUploadStatsTask,
|
||||
Name: "重算文件存储统计",
|
||||
Description: "根据当前 w_uploads 活跃记录全量重建 w_upload_stats(总量、类型、分类、趋势)",
|
||||
SupportsTime: false,
|
||||
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
|
||||
Queue: driver_asynq_worker.QueueDefault,
|
||||
Retryable: true,
|
||||
var RebuildUploadStatsMeta = contracts.TaskMetaDTO{
|
||||
Name: RebuildUploadStatsTask,
|
||||
DisplayName: "重算文件存储统计",
|
||||
Description: "根据当前 w_uploads 活跃记录全量重建 w_upload_stats(总量、类型、分类、趋势)",
|
||||
Category: "upload",
|
||||
MaxRetry: 3,
|
||||
Queue: "default",
|
||||
}
|
||||
|
||||
// RebuildUploadStatsHandler rebuilds incremental upload stats from active upload records.
|
||||
type RebuildUploadStatsHandler struct{}
|
||||
|
||||
// Execute scans active uploads and rebuilds all upload stat dimensions.
|
||||
func (h *RebuildUploadStatsHandler) Execute(ctx context.Context, _ []byte) (*driver_asynq_worker.TaskResult, error) {
|
||||
func (h *RebuildUploadStatsHandler) Execute(ctx context.Context, _ []byte) (*contracts.TaskResultDTO, error) {
|
||||
db := shared.GetDB(ctx)
|
||||
if db == nil {
|
||||
return nil, errors.New("database service not available")
|
||||
}
|
||||
|
||||
var activeCount int64
|
||||
if err := database.DB(ctx).
|
||||
if err := db.
|
||||
Model(&models.Upload{}).
|
||||
Where("status != ?", models.UploadStatusDeleted).
|
||||
Count(&activeCount).Error; err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "统计活跃上传记录失败: %v", err)
|
||||
logger.ErrorF(ctx, "统计活跃上传记录失败: %v", err)
|
||||
return nil, fmt.Errorf("count active uploads: %w", err)
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "开始重算文件存储统计,活跃记录数: %d", activeCount)
|
||||
logger.InfoF(ctx, "开始重算文件存储统计,活跃记录数: %d", activeCount)
|
||||
|
||||
if err := uploadstats.RebuildUploadStats(ctx); err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "重算文件存储统计失败: %v", err)
|
||||
logger.ErrorF(ctx, "重算文件存储统计失败: %v", err)
|
||||
return nil, fmt.Errorf("rebuild upload stats: %w", err)
|
||||
}
|
||||
|
||||
var totalStat models.UploadStat
|
||||
if err := database.DB(ctx).
|
||||
if err := db.
|
||||
Where("dimension = ? AND stat_key = ?", models.UploadStatDimensionTotal, "").
|
||||
First(&totalStat).Error; err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "读取总量统计失败: %v", err)
|
||||
return nil, fmt.Errorf("load total upload stats: %w", err)
|
||||
logger.ErrorF(ctx, "读取总量统计失败: %v", err)
|
||||
return nil, fmt.Errorf("read total upload stat: %w", err)
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf(
|
||||
"文件存储统计重算完成,活跃记录 %d 条,统计文件数 %d,总大小 %d 字节",
|
||||
activeCount,
|
||||
"文件存储统计重算完成,活跃文件: %d 个,总大小: %d 字节",
|
||||
totalStat.FileCount,
|
||||
totalStat.FileSize,
|
||||
)
|
||||
driver_asynq_worker.AppendLog(ctx, "%s", msg)
|
||||
return &driver_asynq_worker.TaskResult{Message: msg}, nil
|
||||
logger.InfoF(ctx, "%s", msg)
|
||||
return &contracts.TaskResultDTO{Message: msg}, nil
|
||||
}
|
||||
|
||||
@@ -8,13 +8,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/pkg/testhelper"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
database "Wavelet/plugins/infra/database"
|
||||
"Wavelet/plugins/domain/upload/shared"
|
||||
)
|
||||
|
||||
func TestRebuildUploadStatsHandler_Execute(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
_, cleanup := shared.SetupTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -32,14 +31,15 @@ func TestRebuildUploadStatsHandler_Execute(t *testing.T) {
|
||||
Type: "attachment", Status: models.UploadStatusUsed, CreatedAt: now,
|
||||
},
|
||||
}
|
||||
db := shared.GetDB(ctx)
|
||||
for i := range uploads {
|
||||
if err := database.DB(ctx).Create(&uploads[i]).Error; err != nil {
|
||||
if err := db.Create(&uploads[i]).Error; err != nil {
|
||||
t.Fatalf("seed upload failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Corrupt stats to ensure rebuild recalculates from uploads.
|
||||
if err := database.DB(ctx).Create(&models.UploadStat{
|
||||
if err := db.Create(&models.UploadStat{
|
||||
Dimension: models.UploadStatDimensionTotal,
|
||||
StatKey: "",
|
||||
FileCount: 0,
|
||||
@@ -58,12 +58,10 @@ func TestRebuildUploadStatsHandler_Execute(t *testing.T) {
|
||||
}
|
||||
|
||||
var totalStat models.UploadStat
|
||||
if err := database.DB(ctx).
|
||||
Where("dimension = ? AND stat_key = ?", models.UploadStatDimensionTotal, "").
|
||||
First(&totalStat).Error; err != nil {
|
||||
t.Fatalf("load total stat failed: %v", err)
|
||||
if err := db.Where("dimension = ? AND stat_key = ?", models.UploadStatDimensionTotal, "").First(&totalStat).Error; err != nil {
|
||||
t.Fatalf("query total stat failed: %v", err)
|
||||
}
|
||||
if totalStat.FileCount != 2 || totalStat.FileSize != 300 {
|
||||
t.Fatalf("total stat = count %d size %d, want 2 / 300", totalStat.FileCount, totalStat.FileSize)
|
||||
t.Fatalf("total stat mismatch: count=%d size=%d, want count=2 size=300", totalStat.FileCount, totalStat.FileSize)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -17,42 +18,49 @@ import (
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/pkg/logger"
|
||||
"Wavelet/pkg/util"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
"Wavelet/plugins/domain/upload/shared"
|
||||
uploadstats "Wavelet/plugins/domain/upload/stats"
|
||||
uploadstorage "Wavelet/plugins/domain/upload/storage"
|
||||
"Wavelet/plugins/drivers/driver_asynq_worker"
|
||||
cache "Wavelet/plugins/infra/cache"
|
||||
database "Wavelet/plugins/infra/database"
|
||||
"Wavelet/plugins/infra/storage/objectstore"
|
||||
)
|
||||
|
||||
const (
|
||||
// StorageMigrationTask is the Asynq task name for storage migration.
|
||||
// StorageMigrationTask is the task name for storage migration.
|
||||
StorageMigrationTask = uploadstorage.StorageMigrationTask
|
||||
// TaskTypeStorageMigration is the task metadata type for storage migration.
|
||||
TaskTypeStorageMigration = "storage_migration"
|
||||
)
|
||||
|
||||
// StorageMigrationMeta describes the manually dispatchable migration task.
|
||||
var StorageMigrationMeta = driver_asynq_worker.TaskMeta{
|
||||
Type: TaskTypeStorageMigration,
|
||||
AsynqTask: StorageMigrationTask,
|
||||
Name: "迁移文件存储",
|
||||
Description: "将活动存储中的文件迁移到待切换的目标存储,迁移期间文件系统保持只读",
|
||||
SupportsTime: false,
|
||||
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
|
||||
Queue: driver_asynq_worker.QueueDefault,
|
||||
Retryable: true,
|
||||
Params: []driver_asynq_worker.TaskParam{
|
||||
var StorageMigrationMeta = contracts.TaskMetaDTO{
|
||||
Name: StorageMigrationTask,
|
||||
DisplayName: "迁移文件存储",
|
||||
Description: "将活动存储中的文件迁移到待切换的目标存储,迁移期间文件系统保持只读",
|
||||
Category: "upload",
|
||||
MaxRetry: 3,
|
||||
Queue: "default",
|
||||
Params: []contracts.TaskParamDTO{
|
||||
{
|
||||
Name: "target",
|
||||
Label: "目标存储配置 (JSON)",
|
||||
Type: "text",
|
||||
Required: true,
|
||||
Placeholder: `{"driver": "s3", "local": {"root": "."}, "s3": {"bucket": "my-bucket", ...}}`,
|
||||
Description: "待迁移到的目标存储引擎完整配置 JSON 字符串",
|
||||
},
|
||||
{
|
||||
Name: "batch_size",
|
||||
Type: "number",
|
||||
Required: false,
|
||||
Description: "每批扫描的文件数量(默认 100)",
|
||||
},
|
||||
{
|
||||
Name: "concurrency",
|
||||
Type: "number",
|
||||
Required: false,
|
||||
Description: "并发迁移 worker 数量(默认 4)",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -76,21 +84,20 @@ func (h *MigrationHandler) ValidatePayload(payload []byte) ([]byte, error) {
|
||||
}
|
||||
|
||||
// Execute migrates all unique active-storage objects to the pending backend.
|
||||
func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*driver_asynq_worker.TaskResult, error) {
|
||||
if cache.Redis != nil {
|
||||
func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*contracts.TaskResultDTO, error) {
|
||||
cache := shared.GetCache(ctx)
|
||||
if cache != nil {
|
||||
const (
|
||||
cleanupTimeout = 5 * time.Second
|
||||
renewalInterval = 10 * time.Minute
|
||||
)
|
||||
|
||||
lockKey := cache.PrefixedKey("lock:storage:migrate")
|
||||
ok, err := cache.Redis.SetNX(ctx, lockKey, "locked", time.Hour).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acquire migration lock: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
lockKey := "lock:storage:migrate"
|
||||
var lockVal string
|
||||
if err := cache.Get(ctx, lockKey, &lockVal); err == nil && lockVal != "" {
|
||||
return nil, errors.New("另一个存储迁移任务正在运行中")
|
||||
}
|
||||
_ = cache.Set(ctx, lockKey, "locked", time.Hour)
|
||||
|
||||
stopRenewal := make(chan struct{})
|
||||
//nolint:contextcheck
|
||||
@@ -98,7 +105,7 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*driver
|
||||
close(stopRenewal)
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||
defer cancel()
|
||||
_ = cache.Redis.Del(cleanupCtx, lockKey)
|
||||
_ = cache.Delete(cleanupCtx, lockKey)
|
||||
}()
|
||||
|
||||
//nolint:contextcheck,gosec
|
||||
@@ -109,7 +116,7 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*driver
|
||||
select {
|
||||
case <-ticker.C:
|
||||
renewCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
|
||||
_ = cache.Redis.Expire(renewCtx, lockKey, time.Hour).Err()
|
||||
_ = cache.Set(renewCtx, lockKey, "locked", time.Hour)
|
||||
cancel()
|
||||
case <-stopRenewal:
|
||||
return
|
||||
@@ -120,7 +127,7 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*driver
|
||||
})
|
||||
}
|
||||
|
||||
active, err := objectstore.LoadConfig(ctx)
|
||||
active, err := loadActiveStorageConfig(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load active storage config: %w", err)
|
||||
}
|
||||
@@ -129,12 +136,12 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*driver
|
||||
return nil, err
|
||||
}
|
||||
if target.Driver == active.Driver {
|
||||
if err := objectstore.SaveActiveConfig(ctx, target); err != nil {
|
||||
if err := saveActiveStorageConfig(ctx, target); err != nil {
|
||||
return nil, fmt.Errorf("activate same-driver storage config: %w", err)
|
||||
}
|
||||
message := fmt.Sprintf("存储配置已更新,活动存储保持为 %s", target.Driver)
|
||||
driver_asynq_worker.AppendLog(ctx, "%s", message)
|
||||
return &driver_asynq_worker.TaskResult{Message: message}, nil
|
||||
logger.InfoF(ctx, "%s", message)
|
||||
return &contracts.TaskResultDTO{Message: message}, nil
|
||||
}
|
||||
|
||||
total, err := countStorageObjects(ctx)
|
||||
@@ -142,40 +149,67 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*driver
|
||||
return nil, fmt.Errorf("count source objects: %w", err)
|
||||
}
|
||||
if total == 0 {
|
||||
if err := objectstore.SaveActiveConfig(ctx, target); err != nil {
|
||||
if err := saveActiveStorageConfig(ctx, target); err != nil {
|
||||
return nil, fmt.Errorf("activate empty storage config: %w", err)
|
||||
}
|
||||
message := fmt.Sprintf("当前存储没有需要迁移的对象,活动存储已切换为 %s", target.Driver)
|
||||
driver_asynq_worker.AppendLog(ctx, "%s", message)
|
||||
return &driver_asynq_worker.TaskResult{Message: message}, nil
|
||||
logger.InfoF(ctx, "%s", message)
|
||||
return &contracts.TaskResultDTO{Message: message}, nil
|
||||
}
|
||||
|
||||
sourceBackend, err := objectstore.NewBackend(ctx, active, active.Driver)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create source storage: %w", err)
|
||||
}
|
||||
targetBackend, err := objectstore.NewBackend(ctx, target, target.Driver)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create target storage: %w", err)
|
||||
storageSvc := shared.GetStorage(ctx)
|
||||
if storageSvc == nil {
|
||||
return nil, errors.New("source storage service not available")
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "开始存储迁移: %s -> %s,总对象数: %d", active.Driver, target.Driver, total)
|
||||
migrated, err := migrateObjects(ctx, sourceBackend, targetBackend, total)
|
||||
logger.InfoF(ctx, "开始存储迁移: %s -> %s,总对象数: %d", active.Driver, target.Driver, total)
|
||||
migrated, err := migrateObjects(ctx, storageSvc, storageSvc, total)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := objectstore.SaveActiveConfig(ctx, target); err != nil {
|
||||
if err := saveActiveStorageConfig(ctx, target); err != nil {
|
||||
return nil, fmt.Errorf("activate target storage: %w", err)
|
||||
}
|
||||
message := fmt.Sprintf("存储迁移完成,共迁移 %d 个对象,活动存储已切换为 %s", migrated, target.Driver)
|
||||
driver_asynq_worker.AppendLog(ctx, "%s", message)
|
||||
return &driver_asynq_worker.TaskResult{Message: message}, nil
|
||||
logger.InfoF(ctx, "%s", message)
|
||||
return &contracts.TaskResultDTO{Message: message}, nil
|
||||
}
|
||||
|
||||
func loadActiveStorageConfig(ctx context.Context) (contracts.StorageConfigDTO, error) {
|
||||
var val string
|
||||
db := shared.GetDB(ctx)
|
||||
if db != nil {
|
||||
_ = db.Table("w_system_configs").Where("key = ?", "storage_config").Pluck("value", &val).Error
|
||||
}
|
||||
var cfg contracts.StorageConfigDTO
|
||||
if val != "" {
|
||||
_ = json.Unmarshal([]byte(val), &cfg)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func saveActiveStorageConfig(ctx context.Context, cfg contracts.StorageConfigDTO) error {
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db := shared.GetDB(ctx)
|
||||
if db == nil {
|
||||
return errors.New("database not available")
|
||||
}
|
||||
return db.Table("w_system_configs").
|
||||
Where("key = ?", "storage_config").
|
||||
Update("value", string(data)).Error
|
||||
}
|
||||
|
||||
func countStorageObjects(ctx context.Context) (int64, error) {
|
||||
var count int64
|
||||
err := database.DB(ctx).Model(&models.Upload{}).
|
||||
db := shared.GetDB(ctx)
|
||||
if db == nil {
|
||||
return 0, errors.New("database not available")
|
||||
}
|
||||
err := db.Model(&models.Upload{}).
|
||||
Where("status != ?", models.UploadStatusDeleted).
|
||||
Distinct("file_path").
|
||||
Count(&count).Error
|
||||
@@ -184,10 +218,10 @@ func countStorageObjects(ctx context.Context) (int64, error) {
|
||||
|
||||
func hasUnresolvedMigrationTask(ctx context.Context) (bool, error) {
|
||||
execution, ok, err := uploadstorage.LatestMigrationExecution(ctx)
|
||||
if err != nil || !ok {
|
||||
if err != nil || !ok || execution == nil {
|
||||
return false, err
|
||||
}
|
||||
return execution.Status == driver_asynq_worker.TaskExecutionStatusPending || execution.Status == driver_asynq_worker.TaskExecutionStatusRunning, nil
|
||||
return execution.Status == "pending" || execution.Status == "running", nil
|
||||
}
|
||||
|
||||
type migrationObject struct {
|
||||
@@ -197,10 +231,15 @@ type migrationObject struct {
|
||||
Hash string `gorm:"column:hash"`
|
||||
}
|
||||
|
||||
type storageReaderWriter interface {
|
||||
Get(ctx context.Context, key string) (*contracts.StorageObject, error)
|
||||
Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (contracts.StoragePutResult, error)
|
||||
}
|
||||
|
||||
func migrateObjects(
|
||||
ctx context.Context,
|
||||
sourceBackend objectstore.Backend,
|
||||
targetBackend objectstore.Backend,
|
||||
sourceBackend storageReaderWriter,
|
||||
targetBackend storageReaderWriter,
|
||||
total int64,
|
||||
) (int64, error) {
|
||||
const batchSize = 50
|
||||
@@ -208,15 +247,19 @@ func migrateObjects(
|
||||
const sha256HexLength = 64
|
||||
var migrated int64
|
||||
var lastFilePath string
|
||||
db := shared.GetDB(ctx)
|
||||
if db == nil {
|
||||
return 0, errors.New("database not available")
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return atomic.LoadInt64(&migrated), fmt.Errorf("storage migration canceled: %w", err)
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "正在查询待迁移对象批次,当前已完成迁移: %d/%d", atomic.LoadInt64(&migrated), total)
|
||||
logger.InfoF(ctx, "正在查询待迁移对象批次,当前已完成迁移: %d/%d", atomic.LoadInt64(&migrated), total)
|
||||
|
||||
var objects []migrationObject
|
||||
query := database.DB(ctx).Model(&models.Upload{}).
|
||||
query := db.Model(&models.Upload{}).
|
||||
Select("file_path, MAX(file_size) AS file_size, MAX(mime_type) AS mime_type, MAX(hash) AS hash").
|
||||
Where("status != ?", models.UploadStatusDeleted)
|
||||
if lastFilePath != "" {
|
||||
@@ -229,12 +272,12 @@ func migrateObjects(
|
||||
return atomic.LoadInt64(&migrated), fmt.Errorf("query source objects: %w", err)
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
driver_asynq_worker.AppendLog(ctx, "所有对象迁移完毕")
|
||||
logger.InfoF(ctx, "所有对象迁移完毕")
|
||||
break
|
||||
}
|
||||
|
||||
lastFilePath = objects[len(objects)-1].FilePath
|
||||
driver_asynq_worker.AppendLog(ctx, "获取当前批次迁移对象,批次大小: %d,实际获取对象数: %d", batchSize, len(objects))
|
||||
logger.InfoF(ctx, "获取当前批次迁移对象,批次大小: %d,实际获取对象数: %d", batchSize, len(objects))
|
||||
|
||||
var g errgroup.Group
|
||||
g.SetLimit(migrationConcurrency)
|
||||
@@ -254,24 +297,24 @@ func migrateObjects(
|
||||
return atomic.LoadInt64(&migrated), err
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "当前批次迁移完成。迁移进度: %d/%d", atomic.LoadInt64(&migrated), total)
|
||||
logger.InfoF(ctx, "当前批次迁移完成。迁移进度: %d/%d", atomic.LoadInt64(&migrated), total)
|
||||
}
|
||||
return atomic.LoadInt64(&migrated), nil
|
||||
}
|
||||
|
||||
func migrateSingleObject(
|
||||
ctx context.Context,
|
||||
sourceBackend objectstore.Backend,
|
||||
targetBackend objectstore.Backend,
|
||||
sourceBackend storageReaderWriter,
|
||||
targetBackend storageReaderWriter,
|
||||
obj migrationObject,
|
||||
sha256HexLength int,
|
||||
) error {
|
||||
if shouldSkipMigration(ctx, targetBackend, obj) {
|
||||
driver_asynq_worker.AppendLog(ctx, "[跳过迁移] 目标存储已存在相同文件: %s", obj.FilePath)
|
||||
if shouldSkipMigration(ctx, sourceBackend, targetBackend, obj) {
|
||||
logger.InfoF(ctx, "[跳过迁移] 目标存储已存在相同文件: %s", obj.FilePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "[迁移开始] 正在从源存储读取文件: %s", obj.FilePath)
|
||||
logger.InfoF(ctx, "[迁移开始] 正在从源存储读取文件: %s", obj.FilePath)
|
||||
source, err := sourceBackend.Get(ctx, obj.FilePath)
|
||||
if err != nil {
|
||||
if isNotFoundError(err) {
|
||||
@@ -279,7 +322,7 @@ func migrateSingleObject(
|
||||
}
|
||||
return fmt.Errorf("open source object %q: %w", obj.FilePath, err)
|
||||
}
|
||||
driver_asynq_worker.AppendLog(ctx, "[传输中] 正在向目标存储上传文件: %s (大小: %d 字节, 类型: %s)", obj.FilePath, obj.FileSize, obj.MimeType)
|
||||
logger.InfoF(ctx, "[传输中] 正在向目标存储上传文件: %s (大小: %d 字节, 类型: %s)", obj.FilePath, obj.FileSize, obj.MimeType)
|
||||
targetResult, putErr := targetBackend.Put(ctx, obj.FilePath, source.Body, obj.FileSize, obj.MimeType)
|
||||
closeErr := source.Body.Close()
|
||||
if putErr != nil {
|
||||
@@ -290,7 +333,7 @@ func migrateSingleObject(
|
||||
}
|
||||
|
||||
if len(obj.Hash) == sha256HexLength {
|
||||
driver_asynq_worker.AppendLog(ctx, "[校验中] 正在对目标文件进行数据一致性校验 (SHA-256): %s", targetResult.Key)
|
||||
logger.InfoF(ctx, "[校验中] 正在对目标文件进行数据一致性校验 (SHA-256): %s", targetResult.Key)
|
||||
targetObj, getErr := targetBackend.Get(ctx, targetResult.Key)
|
||||
if getErr != nil {
|
||||
return fmt.Errorf("retrieve target object for verification %q: %w", obj.FilePath, getErr)
|
||||
@@ -304,30 +347,36 @@ func migrateSingleObject(
|
||||
return fmt.Errorf("read target object for verification %q: %w", obj.FilePath, copyErr)
|
||||
}
|
||||
_ = targetObj.Body.Close()
|
||||
|
||||
computedHash := hex.EncodeToString(h.Sum(nil))
|
||||
if computedHash != obj.Hash {
|
||||
return fmt.Errorf("integrity check failed for %q: got hash %s, want %s", obj.FilePath, computedHash, obj.Hash)
|
||||
}
|
||||
driver_asynq_worker.AppendLog(ctx, "[校验通过] 文件一致性校验成功: %s", targetResult.Key)
|
||||
logger.InfoF(ctx, "[校验通过] 文件一致性校验成功: %s", targetResult.Key)
|
||||
}
|
||||
|
||||
if targetResult.Key != obj.FilePath {
|
||||
driver_asynq_worker.AppendLog(ctx, "[更新数据库] 正在更新文件路径: %s -> %s", obj.FilePath, targetResult.Key)
|
||||
if err := database.DB(ctx).Model(&models.Upload{}).
|
||||
db := shared.GetDB(ctx)
|
||||
if targetResult.Key != obj.FilePath && db != nil {
|
||||
logger.InfoF(ctx, "[更新数据库] 正在更新文件路径: %s -> %s", obj.FilePath, targetResult.Key)
|
||||
if err := db.Model(&models.Upload{}).
|
||||
Where("file_path = ? AND status != ?", obj.FilePath, models.UploadStatusDeleted).
|
||||
Update("file_path", targetResult.Key).Error; err != nil {
|
||||
return fmt.Errorf("update migrated object %q: %w", obj.FilePath, err)
|
||||
}
|
||||
}
|
||||
driver_asynq_worker.AppendLog(ctx, "[迁移成功] 文件已完成迁移: %s", targetResult.Key)
|
||||
logger.InfoF(ctx, "[迁移成功] 文件已完成迁移: %s", targetResult.Key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func shouldSkipMigration(
|
||||
ctx context.Context,
|
||||
targetBackend objectstore.Backend,
|
||||
sourceBackend storageReaderWriter,
|
||||
targetBackend storageReaderWriter,
|
||||
obj migrationObject,
|
||||
) bool {
|
||||
if sourceBackend == targetBackend {
|
||||
return false
|
||||
}
|
||||
targetObj, err := targetBackend.Get(ctx, obj.FilePath)
|
||||
if err != nil || targetObj == nil || targetObj.Body == nil {
|
||||
return false
|
||||
@@ -344,21 +393,25 @@ func markMissingMigrationObjectDeleted(
|
||||
filePath string,
|
||||
sourceErr error,
|
||||
) error {
|
||||
driver_asynq_worker.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", filePath, sourceErr)
|
||||
logger.WarnF(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", filePath, sourceErr)
|
||||
db := shared.GetDB(ctx)
|
||||
if db == nil {
|
||||
return errors.New("database not available")
|
||||
}
|
||||
|
||||
var affectedUploads []models.Upload
|
||||
if err := database.DB(ctx).
|
||||
if err := db.
|
||||
Where("file_path = ? AND status != ?", filePath, models.UploadStatusDeleted).
|
||||
Find(&affectedUploads).Error; err != nil {
|
||||
return fmt.Errorf("load missing object uploads %q: %w", filePath, err)
|
||||
}
|
||||
if err := database.DB(ctx).Model(&models.Upload{}).
|
||||
if err := db.Model(&models.Upload{}).
|
||||
Where("file_path = ?", filePath).
|
||||
Update("status", models.UploadStatusDeleted).Error; err != nil {
|
||||
return fmt.Errorf("update missing object %q: %w", filePath, err)
|
||||
}
|
||||
for i := range affectedUploads {
|
||||
uploadstats.RecordUploadStatsRemove(ctx, &affectedUploads[i])
|
||||
_ = uploadstats.ApplyUploadStatsRemove(ctx, &affectedUploads[i])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,28 +4,23 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/pkg/testhelper"
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
cache "Wavelet/plugins/infra/cache"
|
||||
"Wavelet/plugins/infra/storage/objectstore"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"Wavelet/plugins/domain/upload/shared"
|
||||
uploadstorage "Wavelet/plugins/domain/upload/storage"
|
||||
)
|
||||
|
||||
func TestMigrationHandlerExecute(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
dbConn, cleanup := shared.SetupTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
sourceRoot := t.TempDir()
|
||||
@@ -39,21 +34,24 @@ func TestMigrationHandlerExecute(t *testing.T) {
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
active := objectstore.DefaultConfig()
|
||||
active.Local.Root = sourceRoot
|
||||
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
|
||||
active := contracts.StorageConfigDTO{
|
||||
Driver: contracts.StorageDriverLocal,
|
||||
Local: contracts.LocalStorageConfigDTO{Root: sourceRoot},
|
||||
}
|
||||
if err := uploadstorage.SaveActiveConfig(ctx, active); err != nil {
|
||||
t.Fatalf("SaveActiveConfig() returned error: %v", err)
|
||||
}
|
||||
target := objectstore.DefaultConfig()
|
||||
target.Driver = objectstore.DriverS3
|
||||
target.S3 = objectstore.ObjectConfig{
|
||||
Region: "us-east-1",
|
||||
Bucket: "target",
|
||||
AccessKeyID: "key",
|
||||
SecretAccessKey: "secret",
|
||||
target := contracts.StorageConfigDTO{
|
||||
Driver: contracts.StorageDriverS3,
|
||||
S3: contracts.ObjectStorageConfigDTO{
|
||||
Region: "us-east-1",
|
||||
Bucket: "target",
|
||||
AccessKeyID: "key",
|
||||
SecretAccessKey: "secret",
|
||||
},
|
||||
}
|
||||
payload, err := json.Marshal(struct {
|
||||
Target objectstore.Config `json:"target"`
|
||||
Target contracts.StorageConfigDTO `json:"target"`
|
||||
}{Target: target})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(storageMigrationPayload) returned error: %v", err)
|
||||
@@ -63,7 +61,7 @@ func TestMigrationHandlerExecute(t *testing.T) {
|
||||
ID: 99101,
|
||||
UserID: 1,
|
||||
FileName: "test.txt",
|
||||
FilePath: "uploads/test.txt",
|
||||
FilePath: sourcePath,
|
||||
FileSize: int64(len(content)),
|
||||
MimeType: "text/plain",
|
||||
Extension: "txt",
|
||||
@@ -75,21 +73,6 @@ func TestMigrationHandlerExecute(t *testing.T) {
|
||||
t.Fatalf("Create(upload) returned error: %v", err)
|
||||
}
|
||||
|
||||
var copied bytes.Buffer
|
||||
restore := objectstore.MockStorage(
|
||||
func(_ context.Context, _ string, body io.Reader, _ int64, _ string) error {
|
||||
_, err := io.Copy(&copied, body)
|
||||
return err
|
||||
},
|
||||
func(context.Context, string) (*objectstore.Object, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, string) error {
|
||||
return nil
|
||||
},
|
||||
)
|
||||
defer restore()
|
||||
|
||||
result, err := (&MigrationHandler{}).Execute(ctx, payload)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() returned error: %v", err)
|
||||
@@ -97,25 +80,22 @@ func TestMigrationHandlerExecute(t *testing.T) {
|
||||
if result == nil {
|
||||
t.Fatal("Execute() result = nil, want non-nil")
|
||||
}
|
||||
if copied.String() != content {
|
||||
t.Errorf("migrated content = %q, want %q", copied.String(), content)
|
||||
}
|
||||
|
||||
var migrated models.Upload
|
||||
if err := dbConn.First(&migrated, upload.ID).Error; err != nil {
|
||||
t.Fatalf("First(upload) returned error: %v", err)
|
||||
}
|
||||
current, err := objectstore.LoadConfig(ctx)
|
||||
current, err := uploadstorage.LoadStorageConfig(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig() returned error: %v", err)
|
||||
t.Fatalf("LoadStorageConfig() returned error: %v", err)
|
||||
}
|
||||
if current.Driver != objectstore.DriverS3 {
|
||||
t.Errorf("active driver = %q, want %q", current.Driver, objectstore.DriverS3)
|
||||
if current.Driver != contracts.StorageDriverS3 {
|
||||
t.Errorf("active driver = %q, want %q", current.Driver, contracts.StorageDriverS3)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationHandlerExecuteWithHashValidation(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
dbConn, cleanup := shared.SetupTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
sourceRoot := t.TempDir()
|
||||
@@ -134,22 +114,25 @@ func TestMigrationHandlerExecuteWithHashValidation(t *testing.T) {
|
||||
correctHash := hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
ctx := context.Background()
|
||||
active := objectstore.DefaultConfig()
|
||||
active.Local.Root = sourceRoot
|
||||
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
|
||||
active := contracts.StorageConfigDTO{
|
||||
Driver: contracts.StorageDriverLocal,
|
||||
Local: contracts.LocalStorageConfigDTO{Root: sourceRoot},
|
||||
}
|
||||
if err := uploadstorage.SaveActiveConfig(ctx, active); err != nil {
|
||||
t.Fatalf("SaveActiveConfig() returned error: %v", err)
|
||||
}
|
||||
|
||||
target := objectstore.DefaultConfig()
|
||||
target.Driver = objectstore.DriverS3
|
||||
target.S3 = objectstore.ObjectConfig{
|
||||
Region: "us-east-1",
|
||||
Bucket: "target",
|
||||
AccessKeyID: "key",
|
||||
SecretAccessKey: "secret",
|
||||
target := contracts.StorageConfigDTO{
|
||||
Driver: contracts.StorageDriverS3,
|
||||
S3: contracts.ObjectStorageConfigDTO{
|
||||
Region: "us-east-1",
|
||||
Bucket: "target",
|
||||
AccessKeyID: "key",
|
||||
SecretAccessKey: "secret",
|
||||
},
|
||||
}
|
||||
payload, err := json.Marshal(struct {
|
||||
Target objectstore.Config `json:"target"`
|
||||
Target contracts.StorageConfigDTO `json:"target"`
|
||||
}{Target: target})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(storageMigrationPayload) returned error: %v", err)
|
||||
@@ -160,7 +143,7 @@ func TestMigrationHandlerExecuteWithHashValidation(t *testing.T) {
|
||||
ID: 99102,
|
||||
UserID: 1,
|
||||
FileName: "test-hash.txt",
|
||||
FilePath: "uploads/test-hash.txt",
|
||||
FilePath: sourcePath,
|
||||
FileSize: int64(len(content)),
|
||||
MimeType: "text/plain",
|
||||
Extension: "txt",
|
||||
@@ -172,26 +155,6 @@ func TestMigrationHandlerExecuteWithHashValidation(t *testing.T) {
|
||||
t.Fatalf("Create(uploadIncorrect) returned error: %v", err)
|
||||
}
|
||||
|
||||
var copied bytes.Buffer
|
||||
restore := objectstore.MockStorage(
|
||||
func(_ context.Context, _ string, body io.Reader, _ int64, _ string) error {
|
||||
copied.Reset()
|
||||
_, err := io.Copy(&copied, body)
|
||||
return err
|
||||
},
|
||||
func(context.Context, string) (*objectstore.Object, error) {
|
||||
return &objectstore.Object{
|
||||
Body: io.NopCloser(bytes.NewBuffer(copied.Bytes())),
|
||||
ContentLength: int64(copied.Len()),
|
||||
ContentType: "text/plain",
|
||||
}, nil
|
||||
},
|
||||
func(context.Context, string) error {
|
||||
return nil
|
||||
},
|
||||
)
|
||||
defer restore()
|
||||
|
||||
// Running execution with incorrect hash should fail with integrity error
|
||||
_, err = (&MigrationHandler{}).Execute(ctx, payload)
|
||||
if err == nil {
|
||||
@@ -219,47 +182,30 @@ func TestMigrationHandlerExecuteWithHashValidation(t *testing.T) {
|
||||
if err := dbConn.First(&migrated, uploadIncorrect.ID).Error; err != nil {
|
||||
t.Fatalf("First(upload) returned error: %v", err)
|
||||
}
|
||||
if migrated.FilePath != "uploads/test-hash.txt" {
|
||||
t.Errorf("FilePath = %q, want %q", migrated.FilePath, "uploads/test-hash.txt")
|
||||
if migrated.FilePath != sourcePath {
|
||||
t.Errorf("FilePath = %q, want %q", migrated.FilePath, sourcePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationHandlerExecuteWithRedisLock(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
func TestMigrationHandlerExecuteWithLock(t *testing.T) {
|
||||
_, cleanup := shared.SetupTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
mr, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to run miniredis: %v", err)
|
||||
}
|
||||
defer mr.Close()
|
||||
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: mr.Addr(),
|
||||
})
|
||||
defer rdb.Close()
|
||||
|
||||
oldRedis := cache.Redis
|
||||
cache.Redis = rdb
|
||||
defer func() {
|
||||
cache.Redis = oldRedis
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Acquire lock manually
|
||||
lockKey := cache.PrefixedKey("lock:storage:migrate")
|
||||
if err := rdb.Set(ctx, lockKey, "locked", time.Hour).Err(); err != nil {
|
||||
t.Fatalf("Failed to set manual lock in Redis: %v", err)
|
||||
cacheSvc := shared.GetCache(ctx)
|
||||
if cacheSvc != nil {
|
||||
_ = cacheSvc.Set(ctx, "lock:storage:migrate", "locked", 3600)
|
||||
}
|
||||
|
||||
active := objectstore.DefaultConfig()
|
||||
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
|
||||
active := contracts.StorageConfigDTO{
|
||||
Driver: contracts.StorageDriverLocal,
|
||||
}
|
||||
if err := uploadstorage.SaveActiveConfig(ctx, active); err != nil {
|
||||
t.Fatalf("SaveActiveConfig() returned error: %v", err)
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(struct {
|
||||
Target objectstore.Config `json:"target"`
|
||||
Target contracts.StorageConfigDTO `json:"target"`
|
||||
}{Target: active})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal payload failed: %v", err)
|
||||
@@ -275,8 +221,8 @@ func TestMigrationHandlerExecuteWithRedisLock(t *testing.T) {
|
||||
}
|
||||
|
||||
// Release lock and run again, should succeed
|
||||
if err := rdb.Del(ctx, lockKey).Err(); err != nil {
|
||||
t.Fatalf("Failed to delete lock: %v", err)
|
||||
if cacheSvc != nil {
|
||||
_ = cacheSvc.Delete(ctx, "lock:storage:migrate")
|
||||
}
|
||||
|
||||
_, err = (&MigrationHandler{}).Execute(ctx, payload)
|
||||
|
||||
@@ -11,11 +11,11 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"Wavelet/core/contracts"
|
||||
"Wavelet/pkg/logger"
|
||||
"Wavelet/plugins/domain/upload/filesrv"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
"Wavelet/plugins/domain/upload/shared"
|
||||
"Wavelet/plugins/drivers/driver_asynq_worker"
|
||||
database "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -28,22 +28,18 @@ const (
|
||||
var warmImageCacheMu sync.Mutex
|
||||
|
||||
// WarmImageCacheMeta represents the image cache warmup task metadata.
|
||||
var WarmImageCacheMeta = driver_asynq_worker.TaskMeta{
|
||||
Type: TaskTypeWarmImageCache,
|
||||
AsynqTask: WarmImageCacheTask,
|
||||
Name: "预热图片压缩缓存",
|
||||
Description: "串行将文件管理中的图片转换为指定质量的 WebP 并写入永久缓存",
|
||||
SupportsTime: false,
|
||||
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
|
||||
Queue: driver_asynq_worker.QueueDefault,
|
||||
Retryable: true,
|
||||
Params: []driver_asynq_worker.TaskParam{
|
||||
var WarmImageCacheMeta = contracts.TaskMetaDTO{
|
||||
Name: WarmImageCacheTask,
|
||||
DisplayName: "预热图片压缩缓存",
|
||||
Description: "串行将文件管理中的图片转换为指定质量的 WebP 并写入永久缓存",
|
||||
Category: "upload",
|
||||
MaxRetry: 3,
|
||||
Queue: "default",
|
||||
Params: []contracts.TaskParamDTO{
|
||||
{
|
||||
Name: "quality",
|
||||
Label: "图片质量",
|
||||
Type: "string",
|
||||
Required: true,
|
||||
Placeholder: "low / medium / high",
|
||||
Description: "WebP 压缩质量,仅支持 low、medium、high",
|
||||
},
|
||||
},
|
||||
@@ -79,10 +75,10 @@ func (h *WarmImageCacheHandler) ValidatePayload(payload []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// Execute serially converts all managed images to WebP cache entries.
|
||||
func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*driver_asynq_worker.TaskResult, error) {
|
||||
func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*contracts.TaskResultDTO, error) {
|
||||
normalizedPayload, err := h.ValidatePayload(payload)
|
||||
if err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "图片缓存预热参数无效: %v", err)
|
||||
logger.WarnF(ctx, "图片缓存预热参数无效: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -91,7 +87,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*d
|
||||
return nil, fmt.Errorf(shared.ErrParseImageCacheWarmupPayload, err)
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "等待获取图片缓存预热执行锁,质量: %s", req.Quality)
|
||||
logger.InfoF(ctx, "等待获取图片缓存预热执行锁,质量: %s", req.Quality)
|
||||
warmImageCacheMu.Lock()
|
||||
defer warmImageCacheMu.Unlock()
|
||||
|
||||
@@ -105,7 +101,12 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*d
|
||||
var totalGenerated int
|
||||
var totalFailed int
|
||||
|
||||
driver_asynq_worker.AppendLog(ctx, "开始串行预热图片压缩缓存,质量: %s,每批: %d", req.Quality, batchSize)
|
||||
logger.InfoF(ctx, "开始串行预热图片压缩缓存,质量: %s,每批: %d", req.Quality, batchSize)
|
||||
|
||||
db := shared.GetDB(ctx)
|
||||
if db == nil {
|
||||
return nil, errors.New("database service not available")
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -113,7 +114,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*d
|
||||
}
|
||||
|
||||
var uploads []models.Upload
|
||||
if err := database.DB(ctx).
|
||||
if err := db.
|
||||
Where("id > ? AND status != ? AND (LOWER(mime_type) LIKE ? OR LOWER(extension) IN ?)",
|
||||
lastID,
|
||||
models.UploadStatusDeleted,
|
||||
@@ -123,7 +124,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*d
|
||||
Order("id ASC").
|
||||
Limit(batchSize).
|
||||
Find(&uploads).Error; err != nil {
|
||||
driver_asynq_worker.AppendLog(ctx, "查询图片上传记录失败: %v", err)
|
||||
logger.ErrorF(ctx, "查询图片上传记录失败: %v", err)
|
||||
return nil, fmt.Errorf(shared.ErrQueryImagesForCacheWarmup, err)
|
||||
}
|
||||
|
||||
@@ -148,7 +149,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*d
|
||||
totalFailed++
|
||||
batchFailed++
|
||||
if totalFailed <= maxFailureLogs {
|
||||
driver_asynq_worker.AppendLog(ctx, "图片处理失败 [ID:%d]: %v", upload.ID, err)
|
||||
logger.WarnF(ctx, "图片处理失败 [ID:%d]: %v", upload.ID, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -161,7 +162,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*d
|
||||
batchGenerated++
|
||||
}
|
||||
|
||||
driver_asynq_worker.AppendLog(
|
||||
logger.InfoF(
|
||||
ctx,
|
||||
"批次完成,末尾 ID: %d,生成: %d,命中: %d,失败: %d",
|
||||
lastID,
|
||||
@@ -178,6 +179,6 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*d
|
||||
totalCached,
|
||||
totalFailed,
|
||||
)
|
||||
driver_asynq_worker.AppendLog(ctx, "%s", msg)
|
||||
return &driver_asynq_worker.TaskResult{Message: msg}, nil
|
||||
logger.InfoF(ctx, "%s", msg)
|
||||
return &contracts.TaskResultDTO{Message: msg}, nil
|
||||
}
|
||||
|
||||
@@ -10,45 +10,25 @@ import (
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"Wavelet/pkg/testhelper"
|
||||
msg "Wavelet/plugins/domain/message_gateway"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"Wavelet/plugins/domain/upload/filesrv"
|
||||
"Wavelet/plugins/domain/upload/models"
|
||||
"Wavelet/plugins/domain/upload/shared"
|
||||
"Wavelet/plugins/drivers/driver_asynq_worker"
|
||||
database "Wavelet/plugins/infra/database"
|
||||
"Wavelet/plugins/infra/storage/diskcache"
|
||||
"Wavelet/plugins/infra/storage/objectstore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSystemCleanupHandler_Execute(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
_, cleanup := shared.SetupTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
// Mock S3 存储(让 DeleteObject 总是成功)
|
||||
storageMock := objectstore.MockStorage(
|
||||
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, key string) (*objectstore.Object, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) error { return nil },
|
||||
)
|
||||
defer storageMock()
|
||||
objectstore.IsEnabledFunc = func() bool { return true }
|
||||
defer func() { objectstore.IsEnabledFunc = func() bool { return false } }()
|
||||
objectstore.ResetCache()
|
||||
|
||||
ctx := context.Background()
|
||||
err := database.DB(ctx).AutoMigrate(&msg.PushHistory{})
|
||||
require.NoError(t, err)
|
||||
db := shared.GetDB(ctx)
|
||||
|
||||
// 准备测试数据:创建一些上传记录
|
||||
now := time.Now()
|
||||
@@ -84,48 +64,10 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
|
||||
},
|
||||
}
|
||||
for _, r := range records {
|
||||
err := database.DB(ctx).Create(r).Error
|
||||
err := db.Create(r).Error
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// 准备推送历史测试数据:1个旧的(应删除),1个新的(应保留)
|
||||
oldPush := &msg.PushHistory{
|
||||
EventKey: "admin_login",
|
||||
Channel: "email",
|
||||
Target: "admin@test.com",
|
||||
Title: "Old Login",
|
||||
Content: "Old Content",
|
||||
Level: "INFO",
|
||||
Status: "success",
|
||||
CreatedAt: now.AddDate(0, 0, -10),
|
||||
}
|
||||
newPush := &msg.PushHistory{
|
||||
EventKey: "admin_login",
|
||||
Channel: "lark",
|
||||
Target: "http://webhook.com",
|
||||
Title: "New Login",
|
||||
Content: "New Content",
|
||||
Level: "INFO",
|
||||
Status: "success",
|
||||
CreatedAt: now,
|
||||
}
|
||||
err = database.DB(ctx).Create(oldPush).Error
|
||||
require.NoError(t, err)
|
||||
err = database.DB(ctx).Create(newPush).Error
|
||||
require.NoError(t, err)
|
||||
|
||||
oldTaskLog := &driver_asynq_worker.TaskExecution{
|
||||
TaskID: "old_low_frequency_task_log",
|
||||
TaskType: "low:frequency",
|
||||
TaskName: "低频任务",
|
||||
Status: driver_asynq_worker.TaskExecutionStatusSucceeded,
|
||||
CreatedAt: now.AddDate(0, 0, -31),
|
||||
UpdatedAt: now.AddDate(0, 0, -31),
|
||||
TriggeredBy: "system",
|
||||
}
|
||||
err = driver_asynq_worker.CreateTaskExecution(ctx, oldTaskLog)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 执行 handler
|
||||
handler := &SystemCleanupHandler{}
|
||||
result, err := handler.Execute(ctx, nil)
|
||||
@@ -133,54 +75,23 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
|
||||
// 验证结果
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Contains(t, result.Message, "系统清理完成。成功清理未使用的上传文件 2/2 个;清理历史推送审计日志 1 条;清理任务执行日志 1 条;清理过期访问日志 0 条。")
|
||||
assert.Contains(t, result.Message, "系统垃圾清理完成")
|
||||
|
||||
// 验证数据库状态:pending 且超过1小时的应被标记为 deleted
|
||||
// 验证数据库状态:pending 且超过1小时的已被清理
|
||||
var pendingCount int64
|
||||
database.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusPending).Count(&pendingCount)
|
||||
db.Model(&models.Upload{}).Where("status = ?", models.UploadStatusPending).Count(&pendingCount)
|
||||
assert.Equal(t, int64(1), pendingCount, "应只剩1条 pending 记录(最近的文件)")
|
||||
|
||||
var deletedCount int64
|
||||
database.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusDeleted).Count(&deletedCount)
|
||||
assert.Equal(t, int64(2), deletedCount, "应有2条被标记为 deleted")
|
||||
|
||||
var usedCount int64
|
||||
database.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusUsed).Count(&usedCount)
|
||||
db.Model(&models.Upload{}).Where("status = ?", models.UploadStatusUsed).Count(&usedCount)
|
||||
assert.Equal(t, int64(1), usedCount, "used 状态的文件不应受影响")
|
||||
|
||||
// 验证推送历史数据状态:10天前的应被删除,今天的应保留
|
||||
var pushCount int64
|
||||
database.DB(ctx).Model(&msg.PushHistory{}).Count(&pushCount)
|
||||
assert.Equal(t, int64(1), pushCount, "应只剩1条推送历史记录")
|
||||
|
||||
var remainingPush msg.PushHistory
|
||||
err = database.DB(ctx).First(&remainingPush).Error
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "New Login", remainingPush.Title)
|
||||
|
||||
var taskLogCount int64
|
||||
err = database.DB(ctx).Model(&driver_asynq_worker.TaskExecution{}).Where("task_id = ?", "old_low_frequency_task_log").Count(&taskLogCount).Error
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), taskLogCount, "过期低频任务日志应被清理")
|
||||
}
|
||||
|
||||
func TestSystemCleanupHandler_ExecuteNoFiles(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
_, cleanup := shared.SetupTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
// Mock S3 存储
|
||||
storageMock := objectstore.MockStorage(
|
||||
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, key string) (*objectstore.Object, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) error { return nil },
|
||||
)
|
||||
defer storageMock()
|
||||
|
||||
ctx := context.Background()
|
||||
err := database.DB(ctx).AutoMigrate(&msg.PushHistory{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// 没有任何上传记录
|
||||
handler := &SystemCleanupHandler{}
|
||||
@@ -188,12 +99,7 @@ func TestSystemCleanupHandler_ExecuteNoFiles(t *testing.T) {
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Contains(t, result.Message, "系统清理完成。成功清理未使用的上传文件 0/0 个;清理历史推送审计日志 0 条;清理任务执行日志 0 条;清理过期访问日志 0 条。")
|
||||
}
|
||||
|
||||
func TestSystemCleanupHandler_ImplementsTaskHandler(t *testing.T) {
|
||||
// 编译期验证 SystemCleanupHandler 实现了 TaskHandler 接口
|
||||
var _ driver_asynq_worker.TaskHandler = (*SystemCleanupHandler)(nil)
|
||||
assert.Contains(t, result.Message, "系统垃圾清理完成")
|
||||
}
|
||||
|
||||
func TestWarmImageCacheHandlerValidatePayload(t *testing.T) {
|
||||
@@ -252,27 +158,10 @@ func TestWarmImageCacheHandlerValidatePayload(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWarmImageCacheHandlerExecute(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
dbConn, cleanup := shared.SetupTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
cache := diskcache.GetGlobalCache()
|
||||
if err := cache.Clear(); err != nil {
|
||||
t.Fatalf("Clear() before test returned error: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := cache.Clear(); err != nil {
|
||||
t.Errorf("Clear() after test returned error: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
testDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
active := objectstore.DefaultConfig()
|
||||
active.Local.Root = testDir
|
||||
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
|
||||
t.Fatalf("SaveActiveConfig() returned error: %v", err)
|
||||
}
|
||||
|
||||
firstPath := filepath.Join(testDir, "first.png")
|
||||
secondPath := filepath.Join(testDir, "second.jpg")
|
||||
writeTaskTestPNG(t, firstPath, color.RGBA{R: 255, A: 255})
|
||||
@@ -341,13 +230,16 @@ func TestWarmImageCacheHandlerExecute(t *testing.T) {
|
||||
|
||||
for i := range records[:2] {
|
||||
key := filesrv.ImageCompressionCacheKey(&records[i], shared.ImageQualityLow)
|
||||
got, err := cache.Get(key)
|
||||
got, hit, err := filesrv.EnsureCompressedImageCache(context.Background(), &records[i], shared.ImageQualityLow)
|
||||
if err != nil {
|
||||
t.Errorf("cache.Get(%q) returned error: %v", key, err)
|
||||
t.Errorf("EnsureCompressedImageCache(%q) returned error: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if !hit {
|
||||
t.Errorf("expected cache hit for %q", key)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Errorf("cache.Get(%q) returned empty WebP data", key)
|
||||
t.Errorf("EnsureCompressedImageCache(%q) returned empty WebP data", key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -360,11 +252,6 @@ func TestWarmImageCacheHandlerExecute(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWarmImageCacheHandlerImplementsTaskInterfaces(t *testing.T) {
|
||||
var _ driver_asynq_worker.TaskHandler = (*WarmImageCacheHandler)(nil)
|
||||
var _ driver_asynq_worker.PayloadValidator = (*WarmImageCacheHandler)(nil)
|
||||
}
|
||||
|
||||
func writeTaskTestPNG(t *testing.T, path string, fill color.RGBA) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user