mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-08 00:26:37 +08:00
async task framework
This commit is contained in:
ryan
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2025 linux.do
|
||||
Copyright 2025-2026 linux.do
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
@@ -18,34 +18,31 @@ package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/linux-do/credit/internal/db"
|
||||
"github.com/linux-do/credit/internal/logger"
|
||||
"github.com/linux-do/credit/internal/model"
|
||||
"github.com/linux-do/credit/internal/storage"
|
||||
"github.com/linux-do/credit/internal/task"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// HandleCleanupUnusedUploads 处理清理未使用上传文件的定时任务
|
||||
func HandleCleanupUnusedUploads(ctx context.Context, t *asynq.Task) error {
|
||||
logger.InfoF(ctx, "开始清理未使用的上传文件任务")
|
||||
cleanupUnusedUploads(ctx)
|
||||
logger.InfoF(ctx, "未使用上传文件清理任务完成")
|
||||
return nil
|
||||
}
|
||||
// CleanupUnusedUploadsHandler 清理未使用上传文件的异步任务处理器
|
||||
type CleanupUnusedUploadsHandler struct{}
|
||||
|
||||
// cleanupUnusedUploads 清理超过1小时未使用的上传文件
|
||||
func cleanupUnusedUploads(ctx context.Context) {
|
||||
// Execute 执行清理未使用上传文件的业务逻辑
|
||||
func (h *CleanupUnusedUploadsHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
|
||||
const batchSize = 100 // 每批处理100个文件
|
||||
var lastID uint64 = 0
|
||||
var totalProcessed int = 0
|
||||
var totalDeleted int = 0
|
||||
var totalProcessed int
|
||||
var totalDeleted int
|
||||
|
||||
// 计算1小时前的时间
|
||||
oneHourAgo := time.Now().Add(-1 * time.Hour)
|
||||
|
||||
task.AppendLog(ctx, "开始扫描未使用上传文件，阈值: %s", oneHourAgo.Format(time.RFC3339))
|
||||
|
||||
for {
|
||||
// 使用游标分页查询未使用且超过1小时的上传记录
|
||||
var unusedUploads []model.Upload
|
||||
@@ -54,8 +51,8 @@ func cleanupUnusedUploads(ctx context.Context) {
|
||||
Order("id ASC").
|
||||
Limit(batchSize).
|
||||
Find(&unusedUploads).Error; err != nil {
|
||||
logger.ErrorF(ctx, "查询未使用的上传文件失败: %v", err)
|
||||
return
|
||||
task.AppendLog(ctx, "查询未使用的上传文件失败: %v", err)
|
||||
return nil, fmt.Errorf("查询未使用的上传文件失败: %w", err)
|
||||
}
|
||||
|
||||
// 没有更多数据，退出循环
|
||||
@@ -63,7 +60,7 @@ func cleanupUnusedUploads(ctx context.Context) {
|
||||
break
|
||||
}
|
||||
|
||||
logger.InfoF(ctx, "本批次找到 %d 个需要清理的上传文件", len(unusedUploads))
|
||||
task.AppendLog(ctx, "本批次找到 %d 个需要清理的上传文件", len(unusedUploads))
|
||||
|
||||
// 处理每个未使用的上传文件
|
||||
for _, upload := range unusedUploads {
|
||||
@@ -84,22 +81,17 @@ func cleanupUnusedUploads(ctx context.Context) {
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
logger.ErrorF(ctx, "清理上传文件失败 [ID:%d]: %v", upload.ID, err)
|
||||
task.AppendLog(ctx, "清理上传文件失败 [ID:%d]: %v", upload.ID, err)
|
||||
lastID = upload.ID
|
||||
continue
|
||||
}
|
||||
|
||||
totalDeleted++
|
||||
logger.InfoF(ctx, "成功清理上传文件 [ID:%d, Path:%s, Size:%d bytes]", upload.ID, upload.FilePath, upload.FileSize)
|
||||
|
||||
// 更新游标
|
||||
lastID = upload.ID
|
||||
}
|
||||
}
|
||||
|
||||
if totalDeleted > 0 {
|
||||
logger.InfoF(ctx, "清理任务完成，共处理 %d 个文件，成功删除 %d 个", totalProcessed, totalDeleted)
|
||||
} else {
|
||||
logger.InfoF(ctx, "没有需要清理的上传文件")
|
||||
}
|
||||
msg := fmt.Sprintf("共处理 %d 个文件，成功删除 %d 个", totalProcessed, totalDeleted)
|
||||
task.AppendLog(ctx, msg)
|
||||
return &task.TaskResult{Message: msg}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
Copyright 2025-2026 linux.do
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/linux-do/credit/internal/db"
|
||||
"github.com/linux-do/credit/internal/model"
|
||||
"github.com/linux-do/credit/internal/storage"
|
||||
"github.com/linux-do/credit/internal/task"
|
||||
"github.com/linux-do/credit/internal/testhelper"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCleanupUnusedUploadsHandler_Execute(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
// Mock S3 存储（让 DeleteObject 总是成功）
|
||||
storageMock := storage.MockStorage(
|
||||
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, key string) (*storage.ObjectInfo, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) error { return nil },
|
||||
)
|
||||
defer storageMock()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 准备测试数据：创建一些上传记录
|
||||
now := time.Now()
|
||||
twoHoursAgo := now.Add(-2 * time.Hour)
|
||||
|
||||
records := []*model.Upload{
|
||||
// 超过1小时且状态为 pending 的记录 —— 应被清理
|
||||
{
|
||||
UserID: 1001, FileName: "old_file_1.jpg", FilePath: "uploads/old_1.jpg",
|
||||
FileSize: 1024, MimeType: "image/jpeg", Extension: "jpg", Hash: "hash1",
|
||||
StorageDriver: "s3", Type: "attachment", Status: model.UploadStatusPending,
|
||||
CreatedAt: twoHoursAgo,
|
||||
},
|
||||
{
|
||||
UserID: 1001, FileName: "old_file_2.png", FilePath: "uploads/old_2.png",
|
||||
FileSize: 2048, MimeType: "image/png", Extension: "png", Hash: "hash2",
|
||||
StorageDriver: "s3", Type: "attachment", Status: model.UploadStatusPending,
|
||||
CreatedAt: twoHoursAgo,
|
||||
},
|
||||
// 状态为 used 的记录 —— 不应被清理
|
||||
{
|
||||
UserID: 1001, FileName: "used_file.jpg", FilePath: "uploads/used.jpg",
|
||||
FileSize: 512, MimeType: "image/jpeg", Extension: "jpg", Hash: "hash3",
|
||||
StorageDriver: "s3", Type: "attachment", Status: model.UploadStatusUsed,
|
||||
CreatedAt: twoHoursAgo,
|
||||
},
|
||||
// 不到1小时的 pending 记录 —— 不应被清理
|
||||
{
|
||||
UserID: 1001, FileName: "recent_file.jpg", FilePath: "uploads/recent.jpg",
|
||||
FileSize: 256, MimeType: "image/jpeg", Extension: "jpg", Hash: "hash4",
|
||||
StorageDriver: "s3", Type: "attachment", Status: model.UploadStatusPending,
|
||||
CreatedAt: now.Add(-10 * time.Minute),
|
||||
},
|
||||
}
|
||||
for _, r := range records {
|
||||
err := db.DB(ctx).Create(r).Error
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// 执行 handler
|
||||
handler := &CleanupUnusedUploadsHandler{}
|
||||
result, err := handler.Execute(ctx, nil)
|
||||
|
||||
// 验证结果
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Contains(t, result.Message, "共处理 2 个文件，成功删除 2 个")
|
||||
|
||||
// 验证数据库状态：pending 且超过1小时的应被标记为 deleted
|
||||
var pendingCount int64
|
||||
db.DB(ctx).Model(&model.Upload{}).Where("status = ?", model.UploadStatusPending).Count(&pendingCount)
|
||||
assert.Equal(t, int64(1), pendingCount, "应只剩1条 pending 记录（最近的文件）")
|
||||
|
||||
var deletedCount int64
|
||||
db.DB(ctx).Model(&model.Upload{}).Where("status = ?", model.UploadStatusDeleted).Count(&deletedCount)
|
||||
assert.Equal(t, int64(2), deletedCount, "应有2条被标记为 deleted")
|
||||
|
||||
var usedCount int64
|
||||
db.DB(ctx).Model(&model.Upload{}).Where("status = ?", model.UploadStatusUsed).Count(&usedCount)
|
||||
assert.Equal(t, int64(1), usedCount, "used 状态的文件不应受影响")
|
||||
}
|
||||
|
||||
func TestCleanupUnusedUploadsHandler_ExecuteNoFiles(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
// Mock S3 存储
|
||||
storageMock := storage.MockStorage(
|
||||
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, key string) (*storage.ObjectInfo, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) error { return nil },
|
||||
)
|
||||
defer storageMock()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 没有任何上传记录
|
||||
handler := &CleanupUnusedUploadsHandler{}
|
||||
result, err := handler.Execute(ctx, nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.Contains(t, result.Message, "共处理 0 个文件，成功删除 0 个")
|
||||
}
|
||||
|
||||
func TestCleanupUnusedUploadsHandler_ImplementsTaskHandler(t *testing.T) {
|
||||
// 编译期验证 CleanupUnusedUploadsHandler 实现了 TaskHandler 接口
|
||||
var _ task.TaskHandler = (*CleanupUnusedUploadsHandler)(nil)
|
||||
}
|
||||
Reference in New Issue
Block a user