tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-11 09:46:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(persistence): migrate all pkg/persistence imports to plugins/infra/database and plugins/infra/cache

Browse Source 
- Replace db.DB(ctx) with database.DB(ctx) from plugins/infra/database
- Replace db.Redis/db.PrefixedKey/db.GetJSON/db.SetJSON with cachepkg.* from plugins/infra/cache
- Replace pkg/persistence/idgen with pkg/idgen (already exists)
- Replace pkg/persistence/batchwriter with pkg/batchwriter (already exists)
- Replace pkg/persistence/migrator with pkg/migrator (already exists)
- Replace pkg/persistence/logstore with plugins/domain/risk_control/logstore
- Delete defunct pkg/{persistence,cap,message_gateway,push,shared,task}
- Fix vet issues: db alias in domain_test.go, driver_asynq_worker.TaskHandler reference
- Update Makefile architecture guard
- Update docs and skill references
- Update go.mod: gorilla/sessions promotion to direct dependency
This commit is contained in:
ryan
2026-08-28 10:59:24 +08:00
parent fb6a3edb89
commit 416603b616
223 changed files with 1304 additions and 10057 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/domain/upload/cache/access_cache.go
Vendored
+6 -5
View File
@@ -11,7 +11,8 @@ import (
"sync"
"time"
db "github.com/Rain-kl/Wavelet/pkg/persistence"
cachepkg "github.com/Rain-kl/Wavelet/plugins/infra/cache"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/util"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
@@ -41,8 +42,8 @@ func ResetAccessCaches() {
// PublishAccessCacheInvalidation broadcasts upload access cache eviction to all nodes.
func PublishAccessCacheInvalidation(ctx context.Context) {
if db.Redis != nil {
_ = db.Redis.Publish(ctx, fileAccessInvalidationChannel, "reset").Err()
if cachepkg.Redis != nil {
_ = cachepkg.Redis.Publish(ctx, fileAccessInvalidationChannel, "reset").Err()
}
}
@@ -51,7 +52,7 @@ func ensureAccessCacheListener() {
}
func startAccessCacheInvalidationListener() {
rdb := db.Redis
rdb := cachepkg.Redis
if rdb == nil {
return
}
@@ -114,7 +115,7 @@ func fetchFileAccessWhitelist(ctx context.Context) map[string]struct{} {
func parseFileAccessWhitelist(ctx context.Context) []string {
var sc struct{ Value string }
err := db.DB(ctx).Table("w_system_configs").Where("key = ?", "file_access_whitelist").First(&sc).Error
err := database.DB(ctx).Table("w_system_configs").Where("key = ?", "file_access_whitelist").First(&sc).Error
if err != nil || sc.Value == "" {
return []string{shared.DefaultPublicUploadType}
}
plugins/domain/upload/cache/meta_cache.go
Vendored
+14 -13
View File
@@ -10,7 +10,8 @@ import (
"sync"
"github.com/Rain-kl/Wavelet/pkg/cache/ram"
db "github.com/Rain-kl/Wavelet/pkg/persistence"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
cachepkg "github.com/Rain-kl/Wavelet/plugins/infra/cache"
"github.com/Rain-kl/Wavelet/pkg/util"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
)
@@ -42,7 +43,7 @@ func cloneUpload(u models.Upload) models.Upload {
}
func ensureUploadMetaCacheListener() {
if db.Redis == nil {
if cachepkg.Redis == nil {
return
}
uploadMetaListenerOnce.Do(startUploadMetaCacheInvalidationListener)
@@ -52,7 +53,7 @@ func startUploadMetaCacheInvalidationListener() {
uploadMetaListenerCtx, uploadMetaListenerCancel = context.WithCancel(context.Background())
uploadMetaListenerDone = make(chan struct{})
redisClient := db.Redis // 捕获当前客户端：goroutine 不读可变全局，避免与测试置空 db.Redis 竞争
redisClient := cachepkg.Redis // 捕获当前客户端：goroutine 不读可变全局，避免与测试置空 cachepkg.Redis 竞争
util.Go(func() {
defer close(uploadMetaListenerDone)
pubsub := redisClient.Subscribe(uploadMetaListenerCtx, uploadMetaInvalidationChan)
@@ -77,14 +78,14 @@ func startUploadMetaCacheInvalidationListener() {
}
func publishUploadMetaRAMInvalidation(ctx context.Context, id uint64) {
if db.Redis == nil {
if cachepkg.Redis == nil {
return
}
payload, err := json.Marshal(uploadMetaInvalidationMessage{ID: id})
if err != nil {
return
}
_ = db.Redis.Publish(ctx, uploadMetaInvalidationChan, payload).Err()
_ = cachepkg.Redis.Publish(ctx, uploadMetaInvalidationChan, payload).Err()
}
// GetUploadByID loads upload metadata from RAM, Redis, or the database.
@@ -96,16 +97,16 @@ func GetUploadByID(ctx context.Context, id uint64) (models.Upload, error) {
}
key := uploadMetaRedisKey(id)
if db.Redis != nil {
if cachepkg.Redis != nil {
var u models.Upload
if err := db.GetJSON(ctx, key, &u); err == nil {
if err := cachepkg.GetJSON(ctx, key, &u); err == nil {
uploadMetaRAM.Set(id, cloneUpload(u))
return u, nil
}
}
var u models.Upload
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("id = ? AND status IN (?, ?)", id, models.UploadStatusPending, models.UploadStatusUsed).
First(&u).Error; err != nil {
return models.Upload{}, err
@@ -125,8 +126,8 @@ func SetUploadMetaCache(ctx context.Context, u *models.Upload) {
cloned := cloneUpload(*u)
uploadMetaRAM.Set(u.ID, cloned)
if db.Redis != nil {
_ = db.SetJSON(ctx, uploadMetaRedisKey(u.ID), cloned, uploadMetaRedisCacheTTL)
if cachepkg.Redis != nil {
_ = cachepkg.SetJSON(ctx, uploadMetaRedisKey(u.ID), cloned, uploadMetaRedisCacheTTL)
}
}
@@ -135,8 +136,8 @@ func InvalidateUploadMetaCache(ctx context.Context, id uint64) {
ensureUploadMetaCacheListener()
uploadMetaRAM.Invalidate(id)
if db.Redis != nil {
_ = db.Redis.Del(ctx, db.PrefixedKey(uploadMetaRedisKey(id))).Err()
if cachepkg.Redis != nil {
_ = cachepkg.Redis.Del(ctx, cachepkg.PrefixedKey(uploadMetaRedisKey(id))).Err()
publishUploadMetaRAMInvalidation(ctx, id)
}
}
@@ -151,7 +152,7 @@ func StopUploadMetaCacheListener() {
if uploadMetaListenerCancel != nil {
uploadMetaListenerCancel()
if uploadMetaListenerDone != nil {
<-uploadMetaListenerDone // 等待 goroutine 退出，保证之后置空 db.Redis 不再竞争
<-uploadMetaListenerDone // 等待 goroutine 退出，保证之后置空 cachepkg.Redis 不再竞争
}
uploadMetaListenerCancel = nil
uploadMetaListenerDone = nil
plugins/domain/upload/cache/meta_cache_test.go
Vendored
+8 -8
View File
@@ -9,7 +9,7 @@ import (
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
cachepkg "github.com/Rain-kl/Wavelet/plugins/infra/cache"
"github.com/Rain-kl/Wavelet/pkg/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"gorm.io/gorm"
@@ -58,7 +58,7 @@ func TestGetUploadByIDLoadsFromDBAndPopulatesCache(t *testing.T) {
}
var redisUpload models.Upload
if err := db.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err != nil {
if err := cachepkg.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err != nil {
t.Fatalf("redis cache miss after DB load: %v", err)
}
if redisUpload.ID != upload.ID {
@@ -137,7 +137,7 @@ func TestInvalidateUploadMetaCacheClearsRAMAndRedis(t *testing.T) {
InvalidateUploadMetaCache(ctx, upload.ID)
var redisUpload models.Upload
if err := db.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err == nil {
if err := cachepkg.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err == nil {
t.Fatal("expected redis cache to be invalidated")
}
@@ -188,7 +188,7 @@ func TestUploadMetaInvalidationPubSubClearsPeerRAM(t *testing.T) {
if err != nil {
t.Fatalf("marshal invalidation payload: %v", err)
}
if err := db.Redis.Publish(ctx, uploadMetaInvalidationChan, string(payload)).Err(); err != nil {
if err := cachepkg.Redis.Publish(ctx, uploadMetaInvalidationChan, string(payload)).Err(); err != nil {
t.Fatalf("publish invalidation: %v", err)
}
@@ -205,7 +205,7 @@ func TestUploadMetaInvalidationPubSubClearsPeerRAM(t *testing.T) {
t.Fatal("expected peer RAM cache to be cleared by pub/sub")
}
if err := db.Redis.Del(ctx, db.PrefixedKey(uploadMetaRedisKey(upload.ID))).Err(); err != nil {
if err := cachepkg.Redis.Del(ctx, cachepkg.PrefixedKey(uploadMetaRedisKey(upload.ID))).Err(); err != nil {
t.Fatalf("delete redis cache: %v", err)
}
if _, err := GetUploadByID(ctx, upload.ID); err == nil {
@@ -243,10 +243,10 @@ func TestGetUploadByIDWorksWithRedisDisabled(t *testing.T) {
defer cleanup()
ResetUploadMetaCacheForTest()
redisClient := db.Redis
db.Redis = nil
redisClient := cachepkg.Redis
cachepkg.Redis = nil
t.Cleanup(func() {
db.Redis = redisClient
cachepkg.Redis = redisClient
StopUploadMetaCacheListener()
})
plugins/domain/upload/errs.go
+1
View File
@@ -35,4 +35,5 @@ const (
ErrS3KeyStartsWithSlash = shared.ErrS3KeyStartsWithSlash
ErrS3KeyContainsNullBytes = shared.ErrS3KeyContainsNullBytes
ErrQueryUnusedUploadsFailed = shared.ErrQueryUnusedUploadsFailed
ErrUnauthorized = shared.ErrUnauthorized
)
plugins/domain/upload/exports.go
+5 -5
View File
@@ -4,7 +4,6 @@
package upload
import (
"github.com/Rain-kl/Wavelet/pkg/task"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/handler"
@@ -12,6 +11,7 @@ import (
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
uploadtask "github.com/Rain-kl/Wavelet/plugins/domain/upload/task"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/util"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
)
// HTTP handlers
@@ -116,11 +116,11 @@ type WarmImageCachePayload = uploadtask.WarmImageCachePayload
// Ensure task handler types implement required interfaces.
var (
_ task.TaskHandler = (*MigrationHandler)(nil)
_ task.TaskHandler = (*SystemCleanupHandler)(nil)
_ task.TaskHandler = (*RebuildUploadStatsHandler)(nil)
_ driver_asynq_worker.TaskHandler = (*MigrationHandler)(nil)
_ driver_asynq_worker.TaskHandler = (*SystemCleanupHandler)(nil)
_ driver_asynq_worker.TaskHandler = (*RebuildUploadStatsHandler)(nil)
_ interface {
task.TaskHandler
driver_asynq_worker.TaskHandler
ValidatePayload([]byte) ([]byte, error)
} = (*WarmImageCacheHandler)(nil)
)
plugins/domain/upload/filesrv/file_server.go
+1 -2
View File
@@ -17,7 +17,6 @@ import (
"github.com/Rain-kl/Wavelet/core/contracts"
"github.com/Rain-kl/Wavelet/pkg/response"
appshared "github.com/Rain-kl/Wavelet/pkg/shared"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
@@ -78,7 +77,7 @@ func ServeFileByID(c *gin.Context) {
}
if err := CheckFileAccessPermission(c, upload); err != nil {
response.AbortUnauthorized(c, appshared.UnAuthorized)
response.AbortUnauthorized(c, shared.ErrUnauthorized)
return
}
plugins/domain/upload/handler/routers.go
+1 -2
View File
@@ -25,7 +25,6 @@ import (
"github.com/Rain-kl/Wavelet/core/contracts"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/pkg/response"
appshared "github.com/Rain-kl/Wavelet/pkg/shared"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/ingest"
@@ -183,7 +182,7 @@ func DownloadFile(c *gin.Context) {
}
if err := filesrv.CheckFileAccessPermission(c, upload); err != nil {
response.AbortUnauthorized(c, appshared.UnAuthorized)
response.AbortUnauthorized(c, shared.ErrUnauthorized)
return
}
plugins/domain/upload/ingest/helpers.go
+4 -4
View File
@@ -13,8 +13,8 @@ import (
"time"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/persistence/idgen"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/idgen"
uploadcache "github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/repository"
@@ -50,7 +50,7 @@ func resolveAccessMode(uploadType string, explicit *int) int {
func validateAllowedExtension(ctx context.Context, ext string) error {
var val string
err := db.DB(ctx).Table("w_system_configs").Where("key = ?", "upload_allowed_extensions").Pluck("value", &val).Error
err := database.DB(ctx).Table("w_system_configs").Where("key = ?", "upload_allowed_extensions").Pluck("value", &val).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
@@ -128,7 +128,7 @@ func persistUploadRecord(ctx context.Context, upload *models.Upload, objectKey s
}
func createUploadWithStats(ctx context.Context, upload *models.Upload) error {
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return database.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := repository.CreateUploadTx(tx, upload); err != nil {
return err
}
plugins/domain/upload/ingest/ingest_test.go
+2 -2
View File
@@ -13,7 +13,7 @@ import (
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/objectstore"
@@ -276,7 +276,7 @@ type totalStatsSnapshot struct {
func loadTotalStats(ctx context.Context) (totalStatsSnapshot, error) {
var rows []models.UploadStat
if err := db.DB(ctx).Where("dimension = ?", models.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
if err := database.DB(ctx).Where("dimension = ?", models.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
return totalStatsSnapshot{}, err
}
if len(rows) == 0 {
plugins/domain/upload/ingest/remove.go
+2 -2
View File
@@ -6,7 +6,7 @@ package ingest
import (
"context"
"github.com/Rain-kl/Wavelet/pkg/persistence"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
uploadcache "github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/repository"
@@ -45,7 +45,7 @@ func RemoveOwned(ctx context.Context, userID, uploadID uint64) (models.Upload, e
func softDeleteUploadWithStats(ctx context.Context, upload *models.Upload) error {
statsSnapshot := *upload
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := database.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := repository.SoftDeleteUploadTx(tx, upload); err != nil {
return err
}
plugins/domain/upload/repository.go
+12 -12
View File
@@ -9,8 +9,8 @@ import (
"gorm.io/gorm"
db "github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/persistence/idgen"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/idgen"
"github.com/Rain-kl/Wavelet/pkg/util"
)
@@ -28,7 +28,7 @@ type UploadListFilter struct {
// ListUploads returns paginated upload records matching the filter.
func ListUploads(ctx context.Context, filter UploadListFilter) (int64, []Upload, error) {
query := db.DB(ctx).Model(&Upload{}).
query := database.DB(ctx).Model(&Upload{}).
Where("status != ?", UploadStatusDeleted)
if filter.UserID != 0 {
@@ -60,7 +60,7 @@ func ListUploads(ctx context.Context, filter UploadListFilter) (int64, []Upload,
// GetActiveUploadByID loads a non-deleted upload by ID.
func GetActiveUploadByID(ctx context.Context, id uint64) (Upload, error) {
var upload Upload
if err := db.DB(ctx).Where("id = ? AND status != ?", id, UploadStatusDeleted).First(&upload).Error; err != nil {
if err := database.DB(ctx).Where("id = ? AND status != ?", id, UploadStatusDeleted).First(&upload).Error; err != nil {
return Upload{}, err
}
return upload, nil
@@ -69,7 +69,7 @@ func GetActiveUploadByID(ctx context.Context, id uint64) (Upload, error) {
// SoftDeleteUpload marks an upload as deleted.
// External modules must use upload.Remove or upload.RemoveOwned; only internal/apps/upload may call this.
func SoftDeleteUpload(ctx context.Context, upload *Upload) error {
return SoftDeleteUploadTx(db.DB(ctx), upload)
return SoftDeleteUploadTx(database.DB(ctx), upload)
}
// SoftDeleteUploadTx marks an upload as deleted within an existing transaction.
@@ -82,13 +82,13 @@ func UpdateUpload(ctx context.Context, upload *Upload, updates map[string]any) e
if len(updates) == 0 {
return nil
}
return db.DB(ctx).Model(upload).Updates(updates).Error
return database.DB(ctx).Model(upload).Updates(updates).Error
}
// ListDistinctUploadTypes returns all distinct non-empty upload business types.
func ListDistinctUploadTypes(ctx context.Context) ([]string, error) {
var types []string
if err := db.DB(ctx).Model(&Upload{}).
if err := database.DB(ctx).Model(&Upload{}).
Where("type IS NOT NULL AND type != ''").
Distinct().
Pluck("type", &types).Error; err != nil {
@@ -100,7 +100,7 @@ func ListDistinctUploadTypes(ctx context.Context) ([]string, error) {
// FindReusableUploadByHash finds an existing upload with the same hash and size.
func FindReusableUploadByHash(ctx context.Context, hash string, size int64) (Upload, error) {
var existing Upload
err := db.DB(ctx).
err := database.DB(ctx).
Where("hash = ? AND file_size = ? AND status IN (?, ?)", hash, size, UploadStatusPending, UploadStatusUsed).
First(&existing).Error
return existing, err
@@ -108,7 +108,7 @@ func FindReusableUploadByHash(ctx context.Context, hash string, size int64) (Upl
// CreateUpload persists a new upload record.
func CreateUpload(ctx context.Context, upload *Upload) error {
return CreateUploadTx(db.DB(ctx), upload)
return CreateUploadTx(database.DB(ctx), upload)
}
// CreateUploadTx persists a new upload record within an existing transaction.
@@ -122,7 +122,7 @@ func CreateUploadTx(tx *gorm.DB, upload *Upload) error {
// ListUploadsByIDs returns active uploads matching the given IDs.
func ListUploadsByIDs(ctx context.Context, ids []uint64) ([]Upload, error) {
var uploads []Upload
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("id IN ? AND status IN (?, ?)", ids, UploadStatusPending, UploadStatusUsed).
Find(&uploads).Error; err != nil {
return nil, err
@@ -134,13 +134,13 @@ func ListUploadsByIDs(ctx context.Context, ids []uint64) ([]Upload, error) {
//
//nolint:revive
func UploadQuery(ctx context.Context) *gorm.DB {
return db.DB(ctx).Model(&Upload{})
return database.DB(ctx).Model(&Upload{})
}
// ListUploadStats returns all upload statistics rows.
func ListUploadStats(ctx context.Context) ([]UploadStat, error) {
var stats []UploadStat
if err := db.DB(ctx).Find(&stats).Error; err != nil {
if err := database.DB(ctx).Find(&stats).Error; err != nil {
return nil, err
}
return stats, nil
plugins/domain/upload/repository/repository.go
+11 -11
View File
@@ -9,7 +9,7 @@ import (
"context"
"strings"
db "github.com/Rain-kl/Wavelet/pkg/persistence"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/util"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"gorm.io/gorm"
@@ -27,7 +27,7 @@ type UploadListFilter struct {
// ListUploads returns paginated upload records matching the filter.
func ListUploads(ctx context.Context, filter UploadListFilter) (int64, []models.Upload, error) {
query := db.DB(ctx).Model(&models.Upload{}).
query := database.DB(ctx).Model(&models.Upload{}).
Where("status != ?", models.UploadStatusDeleted)
if filter.UserID != 0 {
@@ -59,7 +59,7 @@ func ListUploads(ctx context.Context, filter UploadListFilter) (int64, []models.
// GetActiveUploadByID loads a non-deleted upload by ID.
func GetActiveUploadByID(ctx context.Context, id uint64) (models.Upload, error) {
var upload models.Upload
if err := db.DB(ctx).Where("id = ? AND status != ?", id, models.UploadStatusDeleted).First(&upload).Error; err != nil {
if err := database.DB(ctx).Where("id = ? AND status != ?", id, models.UploadStatusDeleted).First(&upload).Error; err != nil {
return models.Upload{}, err
}
return upload, nil
@@ -67,7 +67,7 @@ func GetActiveUploadByID(ctx context.Context, id uint64) (models.Upload, error)
// SoftDeleteUpload marks an upload as deleted.
func SoftDeleteUpload(ctx context.Context, upload *models.Upload) error {
return SoftDeleteUploadTx(db.DB(ctx), upload)
return SoftDeleteUploadTx(database.DB(ctx), upload)
}
// SoftDeleteUploadTx marks an upload as deleted within an existing transaction.
@@ -80,13 +80,13 @@ func UpdateUpload(ctx context.Context, upload *models.Upload, updates map[string
if len(updates) == 0 {
return nil
}
return db.DB(ctx).Model(upload).Updates(updates).Error
return database.DB(ctx).Model(upload).Updates(updates).Error
}
// ListDistinctUploadTypes returns all distinct non-empty upload business types.
func ListDistinctUploadTypes(ctx context.Context) ([]string, error) {
var types []string
if err := db.DB(ctx).Model(&models.Upload{}).
if err := database.DB(ctx).Model(&models.Upload{}).
Where("type IS NOT NULL AND type != ''").
Distinct().
Pluck("type", &types).Error; err != nil {
@@ -98,7 +98,7 @@ func ListDistinctUploadTypes(ctx context.Context) ([]string, error) {
// FindReusableUploadByHash finds an existing upload with the same hash and size.
func FindReusableUploadByHash(ctx context.Context, hash string, size int64) (models.Upload, error) {
var existing models.Upload
err := db.DB(ctx).
err := database.DB(ctx).
Where("hash = ? AND file_size = ? AND status IN (?, ?)", hash, size, models.UploadStatusPending, models.UploadStatusUsed).
First(&existing).Error
return existing, err
@@ -106,7 +106,7 @@ func FindReusableUploadByHash(ctx context.Context, hash string, size int64) (mod
// CreateUpload persists a new upload record.
func CreateUpload(ctx context.Context, upload *models.Upload) error {
return CreateUploadTx(db.DB(ctx), upload)
return CreateUploadTx(database.DB(ctx), upload)
}
// CreateUploadTx persists a new upload record within an existing transaction.
@@ -117,7 +117,7 @@ func CreateUploadTx(tx *gorm.DB, upload *models.Upload) error {
// ListUploadsByIDs returns active uploads matching the given IDs.
func ListUploadsByIDs(ctx context.Context, ids []uint64) ([]models.Upload, error) {
var uploads []models.Upload
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("id IN ? AND status IN (?, ?)", ids, models.UploadStatusPending, models.UploadStatusUsed).
Find(&uploads).Error; err != nil {
return nil, err
@@ -127,13 +127,13 @@ func ListUploadsByIDs(ctx context.Context, ids []uint64) ([]models.Upload, error
// UploadQuery returns a scoped GORM query for uploads.
func UploadQuery(ctx context.Context) *gorm.DB {
return db.DB(ctx).Model(&models.Upload{})
return database.DB(ctx).Model(&models.Upload{})
}
// ListUploadStats returns all upload statistics rows.
func ListUploadStats(ctx context.Context) ([]models.UploadStat, error) {
var stats []models.UploadStat
if err := db.DB(ctx).Find(&stats).Error; err != nil {
if err := database.DB(ctx).Find(&stats).Error; err != nil {
return nil, err
}
return stats, nil
plugins/domain/upload/shared/errs.go
+1
View File
@@ -40,4 +40,5 @@ const (
ErrQueryImagesForCacheWarmup = "查询待预热图片失败: %w"
ErrQueryTypeListFailed = "查询文件类型列表失败"
ErrUpdateFileFailed = "更新文件失败"
ErrUnauthorized = "未登录"
)
plugins/domain/upload/stats/stats_counter.go
+3 -3
View File
@@ -8,7 +8,7 @@ import (
"time"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/pkg/persistence"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -26,7 +26,7 @@ func ApplyUploadStatsRemove(ctx context.Context, upload *models.Upload) error {
// RebuildUploadStats rebuilds all incremental stats from current upload records.
func RebuildUploadStats(ctx context.Context) error {
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return database.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Where("1 = 1").Delete(&models.UploadStat{}).Error; err != nil {
return err
}
@@ -49,7 +49,7 @@ func applyUploadStatsDelta(ctx context.Context, upload *models.Upload, sign int6
if upload == nil || !isActiveUploadStatus(upload.Status) {
return nil
}
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return database.DB(ctx).Transaction(func(tx *gorm.DB) error {
return ApplyUploadStatsDeltaTx(tx, upload, sign)
})
}
plugins/domain/upload/stats/stats_counter_test.go
+3 -3
View File
@@ -8,7 +8,7 @@ import (
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"gorm.io/gorm"
@@ -29,7 +29,7 @@ func TestApplyUploadStatsDeltaTxWithinTransaction(t *testing.T) {
CreatedAt: time.Now(),
}
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := database.DB(ctx).Transaction(func(tx *gorm.DB) error {
return ApplyUploadStatsDeltaTx(tx, upload, 1)
}); err != nil {
t.Fatalf("ApplyUploadStatsDeltaTx returned error: %v", err)
@@ -90,7 +90,7 @@ type uploadStatsSnapshot struct {
func loadUploadStats(ctx context.Context) (uploadStatsSnapshot, error) {
var rows []models.UploadStat
if err := db.DB(ctx).Where("dimension = ?", models.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
if err := database.DB(ctx).Where("dimension = ?", models.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
return uploadStatsSnapshot{}, err
}
if len(rows) == 0 {
plugins/domain/upload/storage/access_state.go
+3 -4
View File
@@ -9,9 +9,8 @@ import (
"sync"
"time"
"github.com/Rain-kl/Wavelet/pkg/task"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/objectstore"
)
@@ -71,9 +70,9 @@ func buildMigrationAccessState(ctx context.Context) MigrationAccessState {
}
state := MigrationAccessState{
ReadOnly: execution.Status != task.TaskExecutionStatusSucceeded,
ReadOnly: execution.Status != driver_asynq_worker.TaskExecutionStatusSucceeded,
}
if execution.Status == task.TaskExecutionStatusSucceeded {
if execution.Status == driver_asynq_worker.TaskExecutionStatusSucceeded {
return state
}
plugins/domain/upload/storage/migration.go
+3 -4
View File
@@ -10,8 +10,7 @@ import (
"fmt"
"strings"
"github.com/Rain-kl/Wavelet/pkg/task"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/objectstore"
)
@@ -19,8 +18,8 @@ import (
const StorageMigrationTask = "storage:migrate"
// LatestMigrationExecution returns the most recent storage migration task execution.
func LatestMigrationExecution(ctx context.Context) (*task.TaskExecution, bool, error) {
return task.GetLatestTaskExecutionByTaskType(ctx, StorageMigrationTask)
func LatestMigrationExecution(ctx context.Context) (*driver_asynq_worker.TaskExecution, bool, error) {
return driver_asynq_worker.GetLatestTaskExecutionByTaskType(ctx, StorageMigrationTask)
}
// ParseMigrationTargetConfig parses and validates a storage migration target payload.
plugins/domain/upload/task/cleanup.go
+31 -31
View File
@@ -10,18 +10,18 @@ import (
"fmt"
"time"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"gorm.io/gorm"
"github.com/Rain-kl/Wavelet/pkg/persistence/logstore"
"github.com/Rain-kl/Wavelet/pkg/task"
"github.com/Rain-kl/Wavelet/pkg/logger"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
logstore "github.com/Rain-kl/Wavelet/plugins/domain/risk_control/logstore"
uploadcache "github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/objectstore"
"gorm.io/gorm"
)
const (
@@ -32,14 +32,14 @@ const (
)
// SystemCleanupMeta represents the task metadata.
var SystemCleanupMeta = task.TaskMeta{
var SystemCleanupMeta = driver_asynq_worker.TaskMeta{
Type: TaskTypeSystemCleanup,
AsynqTask: SystemCleanupTask,
Name: "系统垃圾清理",
Description: "定期清理未使用上传文件、历史推送记录和过期任务执行日志",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
Queue: driver_asynq_worker.QueueDefault,
Retryable: true,
}
@@ -47,7 +47,7 @@ var SystemCleanupMeta = task.TaskMeta{
type SystemCleanupHandler struct{}
// Execute 执行系统清理（包含文件清理、历史推送日志和任务执行日志清理）
func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*driver_asynq_worker.TaskResult, error) {
if uploadstorage.ReadOnly(ctx) {
return nil, errors.New(shared.ErrStorageReadOnly)
}
@@ -58,16 +58,16 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
oneHourAgo := time.Now().Add(-1 * time.Hour)
task.AppendLog(ctx, "开始扫描未使用上传文件，阈值: %s", oneHourAgo.Format(time.RFC3339))
driver_asynq_worker.AppendLog(ctx, "开始扫描未使用上传文件，阈值: %s", oneHourAgo.Format(time.RFC3339))
for {
var unusedUploads []models.Upload
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("id > ? AND status = ? AND created_at < ?", lastID, models.UploadStatusPending, oneHourAgo).
Order("id ASC").
Limit(batchSize).
Find(&unusedUploads).Error; err != nil {
task.AppendLog(ctx, "查询未使用的上传文件失败: %v", err)
driver_asynq_worker.AppendLog(ctx, "查询未使用的上传文件失败: %v", err)
return nil, fmt.Errorf(shared.ErrQueryUnusedUploadsFailed, err)
}
@@ -75,12 +75,12 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
break
}
task.AppendLog(ctx, "本批次找到 %d 个需要清理的上传文件", len(unusedUploads))
driver_asynq_worker.AppendLog(ctx, "本批次找到 %d 个需要清理的上传文件", len(unusedUploads))
for _, u := range unusedUploads {
totalProcessed++
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := database.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&models.Upload{}).
Where("id = ? AND status = ?", u.ID, models.UploadStatusPending).
Update("status", models.UploadStatusDeleted).Error; err != nil {
@@ -97,7 +97,7 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
return nil
}); err != nil {
task.AppendLog(ctx, "清理上传文件失败 [ID:%d]: %v", u.ID, err)
driver_asynq_worker.AppendLog(ctx, "清理上传文件失败 [ID:%d]: %v", u.ID, err)
lastID = u.ID
continue
}
@@ -109,28 +109,28 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
}
}
task.AppendLog(ctx, "开始清理历史推送审计日志，只保留最近7天数据...")
driver_asynq_worker.AppendLog(ctx, "开始清理历史推送审计日志，只保留最近7天数据...")
cutoff := time.Now().AddDate(0, 0, -7)
var pushHistoryCount int64
if err := db.DB(ctx).Table("w_push_histories").Where("created_at < ?", cutoff).Count(&pushHistoryCount).Error; err != nil {
task.AppendLog(ctx, "统计待清理的历史推送记录失败: %v", err)
if err := database.DB(ctx).Table("w_push_histories").Where("created_at < ?", cutoff).Count(&pushHistoryCount).Error; err != nil {
driver_asynq_worker.AppendLog(ctx, "统计待清理的历史推送记录失败: %v", err)
} else if pushHistoryCount > 0 {
if err := db.DB(ctx).Table("w_push_histories").Where("created_at < ?", cutoff).Delete(map[string]any{}).Error; err != nil {
task.AppendLog(ctx, "删除历史推送记录失败: %v", err)
if err := database.DB(ctx).Table("w_push_histories").Where("created_at < ?", cutoff).Delete(map[string]any{}).Error; err != nil {
driver_asynq_worker.AppendLog(ctx, "删除历史推送记录失败: %v", err)
} else {
task.AppendLog(ctx, "成功删除 %d 条历史推送记录 (截止时间: %s)", pushHistoryCount, cutoff.Format("2006-01-02 15:04:05"))
driver_asynq_worker.AppendLog(ctx, "成功删除 %d 条历史推送记录 (截止时间: %s)", pushHistoryCount, cutoff.Format("2006-01-02 15:04:05"))
}
} else {
task.AppendLog(ctx, "没有需要清理的历史推送记录 (截止时间: %s)", cutoff.Format("2006-01-02 15:04:05"))
driver_asynq_worker.AppendLog(ctx, "没有需要清理的历史推送记录 (截止时间: %s)", cutoff.Format("2006-01-02 15:04:05"))
}
task.AppendLog(ctx, "开始清理任务执行日志：高频任务保留最近3天，低频任务保留最近30天...")
taskLogStats, err := task.CleanupTaskExecutionLogs(ctx, time.Now())
driver_asynq_worker.AppendLog(ctx, "开始清理任务执行日志：高频任务保留最近3天，低频任务保留最近30天...")
taskLogStats, err := driver_asynq_worker.CleanupTaskExecutionLogs(ctx, time.Now())
if err != nil {
task.AppendLog(ctx, "清理任务执行日志失败: %v", err)
driver_asynq_worker.AppendLog(ctx, "清理任务执行日志失败: %v", err)
logger.ErrorF(ctx, "清理任务执行日志失败: %v", err)
} else {
task.AppendLog(ctx, "成功清理任务执行日志 %d 条（高频 %d 条，低频 %d 条）",
driver_asynq_worker.AppendLog(ctx, "成功清理任务执行日志 %d 条（高频 %d 条，低频 %d 条）",
taskLogStats.HighFrequencyDeleted+taskLogStats.LowFrequencyDeleted,
taskLogStats.HighFrequencyDeleted,
taskLogStats.LowFrequencyDeleted,
@@ -140,11 +140,11 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
var logDeleted int64
logSummary, logErr := logstore.CleanupExpired(ctx)
if logErr != nil {
task.AppendLog(ctx, "清理过期用户访问日志失败: %v", logErr)
driver_asynq_worker.AppendLog(ctx, "清理过期用户访问日志失败: %v", logErr)
logger.ErrorF(ctx, "清理过期用户访问日志失败: %v", logErr)
} else {
logDeleted = logSummary.Deleted
task.AppendLog(ctx, "成功清理过期用户访问日志 %d 条（%s 保留 %d 天）",
driver_asynq_worker.AppendLog(ctx, "成功清理过期用户访问日志 %d 条（%s 保留 %d 天）",
logSummary.Deleted, logSummary.ActiveDatabase, logSummary.RetentionDays)
}
@@ -155,6 +155,6 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
taskLogStats.HighFrequencyDeleted+taskLogStats.LowFrequencyDeleted,
logDeleted,
)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
driver_asynq_worker.AppendLog(ctx, "%s", msg)
return &driver_asynq_worker.TaskResult{Message: msg}, nil
}
plugins/domain/upload/task/rebuild_stats.go
+14 -14
View File
@@ -7,10 +7,10 @@ import (
"context"
"fmt"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/task"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
)
const (
@@ -21,14 +21,14 @@ const (
)
// RebuildUploadStatsMeta describes the upload stats rebuild task.
var RebuildUploadStatsMeta = task.TaskMeta{
var RebuildUploadStatsMeta = driver_asynq_worker.TaskMeta{
Type: TaskTypeRebuildUploadStats,
AsynqTask: RebuildUploadStatsTask,
Name: "重算文件存储统计",
Description: "根据当前 w_uploads 活跃记录全量重建 w_upload_stats（总量、类型、分类、趋势）",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
Queue: driver_asynq_worker.QueueDefault,
Retryable: true,
}
@@ -36,28 +36,28 @@ var RebuildUploadStatsMeta = task.TaskMeta{
type RebuildUploadStatsHandler struct{}
// Execute scans active uploads and rebuilds all upload stat dimensions.
func (h *RebuildUploadStatsHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
func (h *RebuildUploadStatsHandler) Execute(ctx context.Context, _ []byte) (*driver_asynq_worker.TaskResult, error) {
var activeCount int64
if err := db.DB(ctx).
if err := database.DB(ctx).
Model(&models.Upload{}).
Where("status != ?", models.UploadStatusDeleted).
Count(&activeCount).Error; err != nil {
task.AppendLog(ctx, "统计活跃上传记录失败: %v", err)
driver_asynq_worker.AppendLog(ctx, "统计活跃上传记录失败: %v", err)
return nil, fmt.Errorf("count active uploads: %w", err)
}
task.AppendLog(ctx, "开始重算文件存储统计，活跃记录数: %d", activeCount)
driver_asynq_worker.AppendLog(ctx, "开始重算文件存储统计，活跃记录数: %d", activeCount)
if err := uploadstats.RebuildUploadStats(ctx); err != nil {
task.AppendLog(ctx, "重算文件存储统计失败: %v", err)
driver_asynq_worker.AppendLog(ctx, "重算文件存储统计失败: %v", err)
return nil, fmt.Errorf("rebuild upload stats: %w", err)
}
var totalStat models.UploadStat
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("dimension = ? AND stat_key = ?", models.UploadStatDimensionTotal, "").
First(&totalStat).Error; err != nil {
task.AppendLog(ctx, "读取总量统计失败: %v", err)
driver_asynq_worker.AppendLog(ctx, "读取总量统计失败: %v", err)
return nil, fmt.Errorf("load total upload stats: %w", err)
}
@@ -67,6 +67,6 @@ func (h *RebuildUploadStatsHandler) Execute(ctx context.Context, _ []byte) (*tas
totalStat.FileCount,
totalStat.FileSize,
)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
driver_asynq_worker.AppendLog(ctx, "%s", msg)
return &driver_asynq_worker.TaskResult{Message: msg}, nil
}
plugins/domain/upload/task/rebuild_stats_test.go
+4 -4
View File
@@ -8,7 +8,7 @@ import (
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
)
@@ -33,13 +33,13 @@ func TestRebuildUploadStatsHandler_Execute(t *testing.T) {
},
}
for i := range uploads {
if err := db.DB(ctx).Create(&uploads[i]).Error; err != nil {
if err := database.DB(ctx).Create(&uploads[i]).Error; err != nil {
t.Fatalf("seed upload failed: %v", err)
}
}
// Corrupt stats to ensure rebuild recalculates from uploads.
if err := db.DB(ctx).Create(&models.UploadStat{
if err := database.DB(ctx).Create(&models.UploadStat{
Dimension: models.UploadStatDimensionTotal,
StatKey: "",
FileCount: 0,
@@ -58,7 +58,7 @@ func TestRebuildUploadStatsHandler_Execute(t *testing.T) {
}
var totalStat models.UploadStat
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("dimension = ? AND stat_key = ?", models.UploadStatDimensionTotal, "").
First(&totalStat).Error; err != nil {
t.Fatalf("load total stat failed: %v", err)
plugins/domain/upload/task/storage_migration.go
+40 -38
View File
@@ -15,14 +15,16 @@ import (
"sync/atomic"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/task"
"golang.org/x/sync/errgroup"
cache "github.com/Rain-kl/Wavelet/plugins/infra/cache"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/util"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/objectstore"
"golang.org/x/sync/errgroup"
)
const (
@@ -33,16 +35,16 @@ const (
)
// StorageMigrationMeta describes the manually dispatchable migration task.
var StorageMigrationMeta = task.TaskMeta{
var StorageMigrationMeta = driver_asynq_worker.TaskMeta{
Type: TaskTypeStorageMigration,
AsynqTask: StorageMigrationTask,
Name: "迁移文件存储",
Description: "将活动存储中的文件迁移到待切换的目标存储，迁移期间文件系统保持只读",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
Queue: driver_asynq_worker.QueueDefault,
Retryable: true,
Params: []task.TaskParam{
Params: []driver_asynq_worker.TaskParam{
{
Name: "target",
Label: "目标存储配置 (JSON)",
@@ -74,15 +76,15 @@ func (h *MigrationHandler) ValidatePayload(payload []byte) ([]byte, error) {
}
// Execute migrates all unique active-storage objects to the pending backend.
func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
if db.Redis != nil {
func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*driver_asynq_worker.TaskResult, error) {
if cache.Redis != nil {
const (
cleanupTimeout = 5 * time.Second
renewalInterval = 10 * time.Minute
)
lockKey := db.PrefixedKey("lock:storage:migrate")
ok, err := db.Redis.SetNX(ctx, lockKey, "locked", time.Hour).Result()
lockKey := cache.PrefixedKey("lock:storage:migrate")
ok, err := cache.Redis.SetNX(ctx, lockKey, "locked", time.Hour).Result()
if err != nil {
return nil, fmt.Errorf("acquire migration lock: %w", err)
}
@@ -96,7 +98,7 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.T
close(stopRenewal)
cleanupCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()
_ = db.Redis.Del(cleanupCtx, lockKey)
_ = cache.Redis.Del(cleanupCtx, lockKey)
}()
//nolint:contextcheck,gosec
@@ -107,7 +109,7 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.T
select {
case <-ticker.C:
renewCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
_ = db.Redis.Expire(renewCtx, lockKey, time.Hour).Err()
_ = cache.Redis.Expire(renewCtx, lockKey, time.Hour).Err()
cancel()
case <-stopRenewal:
return
@@ -131,8 +133,8 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.T
return nil, fmt.Errorf("activate same-driver storage config: %w", err)
}
message := fmt.Sprintf("存储配置已更新，活动存储保持为 %s", target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
driver_asynq_worker.AppendLog(ctx, "%s", message)
return &driver_asynq_worker.TaskResult{Message: message}, nil
}
total, err := countStorageObjects(ctx)
@@ -144,8 +146,8 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.T
return nil, fmt.Errorf("activate empty storage config: %w", err)
}
message := fmt.Sprintf("当前存储没有需要迁移的对象，活动存储已切换为 %s", target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
driver_asynq_worker.AppendLog(ctx, "%s", message)
return &driver_asynq_worker.TaskResult{Message: message}, nil
}
sourceBackend, err := objectstore.NewBackend(ctx, active, active.Driver)
@@ -157,7 +159,7 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.T
return nil, fmt.Errorf("create target storage: %w", err)
}
task.AppendLog(ctx, "开始存储迁移: %s -> %s，总对象数: %d", active.Driver, target.Driver, total)
driver_asynq_worker.AppendLog(ctx, "开始存储迁移: %s -> %s，总对象数: %d", active.Driver, target.Driver, total)
migrated, err := migrateObjects(ctx, sourceBackend, targetBackend, total)
if err != nil {
return nil, err
@@ -167,13 +169,13 @@ func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.T
return nil, fmt.Errorf("activate target storage: %w", err)
}
message := fmt.Sprintf("存储迁移完成，共迁移 %d 个对象，活动存储已切换为 %s", migrated, target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
driver_asynq_worker.AppendLog(ctx, "%s", message)
return &driver_asynq_worker.TaskResult{Message: message}, nil
}
func countStorageObjects(ctx context.Context) (int64, error) {
var count int64
err := db.DB(ctx).Model(&models.Upload{}).
err := database.DB(ctx).Model(&models.Upload{}).
Where("status != ?", models.UploadStatusDeleted).
Distinct("file_path").
Count(&count).Error
@@ -185,7 +187,7 @@ func hasUnresolvedMigrationTask(ctx context.Context) (bool, error) {
if err != nil || !ok {
return false, err
}
return execution.Status == task.TaskExecutionStatusPending || execution.Status == task.TaskExecutionStatusRunning, nil
return execution.Status == driver_asynq_worker.TaskExecutionStatusPending || execution.Status == driver_asynq_worker.TaskExecutionStatusRunning, nil
}
type migrationObject struct {
@@ -211,10 +213,10 @@ func migrateObjects(
return atomic.LoadInt64(&migrated), fmt.Errorf("storage migration canceled: %w", err)
}
task.AppendLog(ctx, "正在查询待迁移对象批次，当前已完成迁移: %d/%d", atomic.LoadInt64(&migrated), total)
driver_asynq_worker.AppendLog(ctx, "正在查询待迁移对象批次，当前已完成迁移: %d/%d", atomic.LoadInt64(&migrated), total)
var objects []migrationObject
query := db.DB(ctx).Model(&models.Upload{}).
query := database.DB(ctx).Model(&models.Upload{}).
Select("file_path, MAX(file_size) AS file_size, MAX(mime_type) AS mime_type, MAX(hash) AS hash").
Where("status != ?", models.UploadStatusDeleted)
if lastFilePath != "" {
@@ -227,12 +229,12 @@ func migrateObjects(
return atomic.LoadInt64(&migrated), fmt.Errorf("query source objects: %w", err)
}
if len(objects) == 0 {
task.AppendLog(ctx, "所有对象迁移完毕")
driver_asynq_worker.AppendLog(ctx, "所有对象迁移完毕")
break
}
lastFilePath = objects[len(objects)-1].FilePath
task.AppendLog(ctx, "获取当前批次迁移对象，批次大小: %d，实际获取对象数: %d", batchSize, len(objects))
driver_asynq_worker.AppendLog(ctx, "获取当前批次迁移对象，批次大小: %d，实际获取对象数: %d", batchSize, len(objects))
var g errgroup.Group
g.SetLimit(migrationConcurrency)
@@ -252,7 +254,7 @@ func migrateObjects(
return atomic.LoadInt64(&migrated), err
}
task.AppendLog(ctx, "当前批次迁移完成。迁移进度: %d/%d", atomic.LoadInt64(&migrated), total)
driver_asynq_worker.AppendLog(ctx, "当前批次迁移完成。迁移进度: %d/%d", atomic.LoadInt64(&migrated), total)
}
return atomic.LoadInt64(&migrated), nil
}
@@ -265,11 +267,11 @@ func migrateSingleObject(
sha256HexLength int,
) error {
if shouldSkipMigration(ctx, targetBackend, obj) {
task.AppendLog(ctx, "[跳过迁移] 目标存储已存在相同文件: %s", obj.FilePath)
driver_asynq_worker.AppendLog(ctx, "[跳过迁移] 目标存储已存在相同文件: %s", obj.FilePath)
return nil
}
task.AppendLog(ctx, "[迁移开始] 正在从源存储读取文件: %s", obj.FilePath)
driver_asynq_worker.AppendLog(ctx, "[迁移开始] 正在从源存储读取文件: %s", obj.FilePath)
source, err := sourceBackend.Get(ctx, obj.FilePath)
if err != nil {
if isNotFoundError(err) {
@@ -277,7 +279,7 @@ func migrateSingleObject(
}
return fmt.Errorf("open source object %q: %w", obj.FilePath, err)
}
task.AppendLog(ctx, "[传输中] 正在向目标存储上传文件: %s (大小: %d 字节, 类型: %s)", obj.FilePath, obj.FileSize, obj.MimeType)
driver_asynq_worker.AppendLog(ctx, "[传输中] 正在向目标存储上传文件: %s (大小: %d 字节, 类型: %s)", obj.FilePath, obj.FileSize, obj.MimeType)
targetResult, putErr := targetBackend.Put(ctx, obj.FilePath, source.Body, obj.FileSize, obj.MimeType)
closeErr := source.Body.Close()
if putErr != nil {
@@ -288,7 +290,7 @@ func migrateSingleObject(
}
if len(obj.Hash) == sha256HexLength {
task.AppendLog(ctx, "[校验中] 正在对目标文件进行数据一致性校验 (SHA-256): %s", targetResult.Key)
driver_asynq_worker.AppendLog(ctx, "[校验中] 正在对目标文件进行数据一致性校验 (SHA-256): %s", targetResult.Key)
targetObj, getErr := targetBackend.Get(ctx, targetResult.Key)
if getErr != nil {
return fmt.Errorf("retrieve target object for verification %q: %w", obj.FilePath, getErr)
@@ -306,18 +308,18 @@ func migrateSingleObject(
if computedHash != obj.Hash {
return fmt.Errorf("integrity check failed for %q: got hash %s, want %s", obj.FilePath, computedHash, obj.Hash)
}
task.AppendLog(ctx, "[校验通过] 文件一致性校验成功: %s", targetResult.Key)
driver_asynq_worker.AppendLog(ctx, "[校验通过] 文件一致性校验成功: %s", targetResult.Key)
}
if targetResult.Key != obj.FilePath {
task.AppendLog(ctx, "[更新数据库] 正在更新文件路径: %s -> %s", obj.FilePath, targetResult.Key)
if err := db.DB(ctx).Model(&models.Upload{}).
driver_asynq_worker.AppendLog(ctx, "[更新数据库] 正在更新文件路径: %s -> %s", obj.FilePath, targetResult.Key)
if err := database.DB(ctx).Model(&models.Upload{}).
Where("file_path = ? AND status != ?", obj.FilePath, models.UploadStatusDeleted).
Update("file_path", targetResult.Key).Error; err != nil {
return fmt.Errorf("update migrated object %q: %w", obj.FilePath, err)
}
}
task.AppendLog(ctx, "[迁移成功] 文件已完成迁移: %s", targetResult.Key)
driver_asynq_worker.AppendLog(ctx, "[迁移成功] 文件已完成迁移: %s", targetResult.Key)
return nil
}
@@ -342,15 +344,15 @@ func markMissingMigrationObjectDeleted(
filePath string,
sourceErr error,
) error {
task.AppendLog(ctx, "警告: 源存储中物理文件不存在，标记为已删除并跳过: %s (错误: %v)", filePath, sourceErr)
driver_asynq_worker.AppendLog(ctx, "警告: 源存储中物理文件不存在，标记为已删除并跳过: %s (错误: %v)", filePath, sourceErr)
var affectedUploads []models.Upload
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("file_path = ? AND status != ?", filePath, models.UploadStatusDeleted).
Find(&affectedUploads).Error; err != nil {
return fmt.Errorf("load missing object uploads %q: %w", filePath, err)
}
if err := db.DB(ctx).Model(&models.Upload{}).
if err := database.DB(ctx).Model(&models.Upload{}).
Where("file_path = ?", filePath).
Update("status", models.UploadStatusDeleted).Error; err != nil {
return fmt.Errorf("update missing object %q: %w", filePath, err)
plugins/domain/upload/task/storage_migration_task_test.go
+5 -5
View File
@@ -16,7 +16,7 @@ import (
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
cache "github.com/Rain-kl/Wavelet/plugins/infra/cache"
"github.com/Rain-kl/Wavelet/pkg/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/objectstore"
@@ -239,16 +239,16 @@ func TestMigrationHandlerExecuteWithRedisLock(t *testing.T) {
})
defer rdb.Close()
oldRedis := db.Redis
db.Redis = rdb
oldRedis := cache.Redis
cache.Redis = rdb
defer func() {
db.Redis = oldRedis
cache.Redis = oldRedis
}()
ctx := context.Background()
// Acquire lock manually
lockKey := db.PrefixedKey("lock:storage:migrate")
lockKey := cache.PrefixedKey("lock:storage:migrate")
if err := rdb.Set(ctx, lockKey, "locked", time.Hour).Err(); err != nil {
t.Fatalf("Failed to set manual lock in Redis: %v", err)
}
plugins/domain/upload/task/tasks.go
+16 -16
View File
@@ -12,11 +12,11 @@ import (
"strings"
"sync"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/task"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
)
const (
@@ -29,16 +29,16 @@ const (
var warmImageCacheMu sync.Mutex
// WarmImageCacheMeta represents the image cache warmup task metadata.
var WarmImageCacheMeta = task.TaskMeta{
var WarmImageCacheMeta = driver_asynq_worker.TaskMeta{
Type: TaskTypeWarmImageCache,
AsynqTask: WarmImageCacheTask,
Name: "预热图片压缩缓存",
Description: "串行将文件管理中的图片转换为指定质量的 WebP 并写入永久缓存",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
MaxRetry: driver_asynq_worker.DefaultMaxRetry,
Queue: driver_asynq_worker.QueueDefault,
Retryable: true,
Params: []task.TaskParam{
Params: []driver_asynq_worker.TaskParam{
{
Name: "quality",
Label: "图片质量",
@@ -80,10 +80,10 @@ func (h *WarmImageCacheHandler) ValidatePayload(payload []byte) ([]byte, error)
}
// Execute serially converts all managed images to WebP cache entries.
func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*driver_asynq_worker.TaskResult, error) {
normalizedPayload, err := h.ValidatePayload(payload)
if err != nil {
task.AppendLog(ctx, "图片缓存预热参数无效: %v", err)
driver_asynq_worker.AppendLog(ctx, "图片缓存预热参数无效: %v", err)
return nil, err
}
@@ -92,7 +92,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*t
return nil, fmt.Errorf(shared.ErrParseImageCacheWarmupPayload, err)
}
task.AppendLog(ctx, "等待获取图片缓存预热执行锁，质量: %s", req.Quality)
driver_asynq_worker.AppendLog(ctx, "等待获取图片缓存预热执行锁，质量: %s", req.Quality)
warmImageCacheMu.Lock()
defer warmImageCacheMu.Unlock()
@@ -106,7 +106,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*t
var totalGenerated int
var totalFailed int
task.AppendLog(ctx, "开始串行预热图片压缩缓存，质量: %s，每批: %d", req.Quality, batchSize)
driver_asynq_worker.AppendLog(ctx, "开始串行预热图片压缩缓存，质量: %s，每批: %d", req.Quality, batchSize)
for {
if err := ctx.Err(); err != nil {
@@ -114,7 +114,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*t
}
var uploads []models.Upload
if err := db.DB(ctx).
if err := database.DB(ctx).
Where("id > ? AND status != ? AND (LOWER(mime_type) LIKE ? OR LOWER(extension) IN ?)",
lastID,
models.UploadStatusDeleted,
@@ -124,7 +124,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*t
Order("id ASC").
Limit(batchSize).
Find(&uploads).Error; err != nil {
task.AppendLog(ctx, "查询图片上传记录失败: %v", err)
driver_asynq_worker.AppendLog(ctx, "查询图片上传记录失败: %v", err)
return nil, fmt.Errorf(shared.ErrQueryImagesForCacheWarmup, err)
}
@@ -149,7 +149,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*t
totalFailed++
batchFailed++
if totalFailed <= maxFailureLogs {
task.AppendLog(ctx, "图片处理失败 [ID:%d]: %v", upload.ID, err)
driver_asynq_worker.AppendLog(ctx, "图片处理失败 [ID:%d]: %v", upload.ID, err)
}
continue
}
@@ -162,7 +162,7 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*t
batchGenerated++
}
task.AppendLog(
driver_asynq_worker.AppendLog(
ctx,
"批次完成，末尾 ID: %d，生成: %d，命中: %d，失败: %d",
lastID,
@@ -179,6 +179,6 @@ func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*t
totalCached,
totalFailed,
)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
driver_asynq_worker.AppendLog(ctx, "%s", msg)
return &driver_asynq_worker.TaskResult{Message: msg}, nil
}
plugins/domain/upload/task/tasks_test.go
+19 -19
View File
@@ -17,13 +17,13 @@ import (
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/task"
database "github.com/Rain-kl/Wavelet/plugins/infra/database"
"github.com/Rain-kl/Wavelet/pkg/testhelper"
msg "github.com/Rain-kl/Wavelet/plugins/domain/message_gateway"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
"github.com/Rain-kl/Wavelet/plugins/drivers/driver_asynq_worker"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/diskcache"
"github.com/Rain-kl/Wavelet/plugins/infra/storage/objectstore"
"github.com/stretchr/testify/assert"
@@ -48,7 +48,7 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
objectstore.ResetCache()
ctx := context.Background()
err := db.DB(ctx).AutoMigrate(&msg.PushHistory{})
err := database.DB(ctx).AutoMigrate(&msg.PushHistory{})
require.NoError(t, err)
// 准备测试数据：创建一些上传记录
@@ -85,7 +85,7 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
},
}
for _, r := range records {
err := db.DB(ctx).Create(r).Error
err := database.DB(ctx).Create(r).Error
require.NoError(t, err)
}
@@ -110,21 +110,21 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
Status: "success",
CreatedAt: now,
}
err = db.DB(ctx).Create(oldPush).Error
err = database.DB(ctx).Create(oldPush).Error
require.NoError(t, err)
err = db.DB(ctx).Create(newPush).Error
err = database.DB(ctx).Create(newPush).Error
require.NoError(t, err)
oldTaskLog := &task.TaskExecution{
oldTaskLog := &driver_asynq_worker.TaskExecution{
TaskID: "old_low_frequency_task_log",
TaskType: "low:frequency",
TaskName: "低频任务",
Status: task.TaskExecutionStatusSucceeded,
Status: driver_asynq_worker.TaskExecutionStatusSucceeded,
CreatedAt: now.AddDate(0, 0, -31),
UpdatedAt: now.AddDate(0, 0, -31),
TriggeredBy: "system",
}
err = task.CreateTaskExecution(ctx, oldTaskLog)
err = driver_asynq_worker.CreateTaskExecution(ctx, oldTaskLog)
require.NoError(t, err)
// 执行 handler
@@ -138,29 +138,29 @@ func TestSystemCleanupHandler_Execute(t *testing.T) {
// 验证数据库状态：pending 且超过1小时的应被标记为 deleted
var pendingCount int64
db.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusPending).Count(&pendingCount)
database.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusPending).Count(&pendingCount)
assert.Equal(t, int64(1), pendingCount, "应只剩1条 pending 记录（最近的文件）")
var deletedCount int64
db.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusDeleted).Count(&deletedCount)
database.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusDeleted).Count(&deletedCount)
assert.Equal(t, int64(2), deletedCount, "应有2条被标记为 deleted")
var usedCount int64
db.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusUsed).Count(&usedCount)
database.DB(ctx).Model(&models.Upload{}).Where("status = ?", models.UploadStatusUsed).Count(&usedCount)
assert.Equal(t, int64(1), usedCount, "used 状态的文件不应受影响")
// 验证推送历史数据状态：10天前的应被删除，今天的应保留
var pushCount int64
db.DB(ctx).Model(&msg.PushHistory{}).Count(&pushCount)
database.DB(ctx).Model(&msg.PushHistory{}).Count(&pushCount)
assert.Equal(t, int64(1), pushCount, "应只剩1条推送历史记录")
var remainingPush msg.PushHistory
err = db.DB(ctx).First(&remainingPush).Error
err = database.DB(ctx).First(&remainingPush).Error
require.NoError(t, err)
assert.Equal(t, "New Login", remainingPush.Title)
var taskLogCount int64
err = db.DB(ctx).Model(&task.TaskExecution{}).Where("task_id = ?", "old_low_frequency_task_log").Count(&taskLogCount).Error
err = database.DB(ctx).Model(&driver_asynq_worker.TaskExecution{}).Where("task_id = ?", "old_low_frequency_task_log").Count(&taskLogCount).Error
require.NoError(t, err)
assert.Equal(t, int64(0), taskLogCount, "过期低频任务日志应被清理")
}
@@ -180,7 +180,7 @@ func TestSystemCleanupHandler_ExecuteNoFiles(t *testing.T) {
defer storageMock()
ctx := context.Background()
err := db.DB(ctx).AutoMigrate(&msg.PushHistory{})
err := database.DB(ctx).AutoMigrate(&msg.PushHistory{})
require.NoError(t, err)
// 没有任何上传记录
@@ -194,7 +194,7 @@ func TestSystemCleanupHandler_ExecuteNoFiles(t *testing.T) {
func TestSystemCleanupHandler_ImplementsTaskHandler(t *testing.T) {
// 编译期验证 SystemCleanupHandler 实现了 TaskHandler 接口
var _ task.TaskHandler = (*SystemCleanupHandler)(nil)
var _ driver_asynq_worker.TaskHandler = (*SystemCleanupHandler)(nil)
}
func TestWarmImageCacheHandlerValidatePayload(t *testing.T) {
@@ -362,8 +362,8 @@ func TestWarmImageCacheHandlerExecute(t *testing.T) {
}
func TestWarmImageCacheHandlerImplementsTaskInterfaces(t *testing.T) {
var _ task.TaskHandler = (*WarmImageCacheHandler)(nil)
var _ task.PayloadValidator = (*WarmImageCacheHandler)(nil)
var _ driver_asynq_worker.TaskHandler = (*WarmImageCacheHandler)(nil)
var _ driver_asynq_worker.PayloadValidator = (*WarmImageCacheHandler)(nil)
}
func writeTaskTestPNG(t *testing.T, path string, fill color.RGBA) {