perf(upload): optimize file hot paths and incremental admin stats

- Use RWMutex for disk cache reads and singleflight for WebP cache misses
- Cache migration read-only state and file access whitelist with pub/sub invalidation
- Add w_upload_stats incremental counters updated on upload/delete
- Add w_uploads composite indexes and goose backfill migrations
- Document performance analysis in docs/PERFORMANCE.md
- Fix RegisterCustomRoutes to accept apiV1Router parameter
This commit is contained in:
ryan
2026-06-17 11:33:48 +08:00
parent e057cd5de5
commit 6a8f9a7aea
26 changed files with 1370 additions and 187 deletions
@@ -11,6 +11,7 @@ import ("context"
"net/http"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
@@ -255,9 +256,15 @@ func UpdateSystemConfig(c *gin.Context) {
}
if key == model.ConfigKeyStorageConfig {
upload.ResetAccessCaches()
upload.PublishAccessCacheInvalidation(c.Request.Context())
storage.ResetCache()
storage.PublishCacheInvalidation(c.Request.Context())
}
if key == model.ConfigKeyFileAccessWhitelist {
upload.ResetAccessCaches()
upload.PublishAccessCacheInvalidation(c.Request.Context())
}
c.JSON(http.StatusOK, response.OKNil())
}
+200
View File
@@ -0,0 +1,200 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"context"
"encoding/json"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
)
const accessCacheTTL = 5 * time.Second
const fileAccessInvalidationChannel = "upload:file_access_invalidation"
type migrationAccessState struct {
readOnly bool
target storage.Config
hasTarget bool
targetErr error
loadErr error
}
var (
accessCacheOnce sync.Once
migrationAccessMu sync.RWMutex
migrationAccessCached migrationAccessState
migrationAccessValid bool
migrationAccessCheckedAt time.Time
fileAccessWhitelistMu sync.RWMutex
fileAccessWhitelistTypes map[string]struct{}
fileAccessWhitelistValid bool
fileAccessWhitelistCheckedAt time.Time
)
// ResetAccessCaches clears in-process upload access caches.
func ResetAccessCaches() {
migrationAccessMu.Lock()
migrationAccessValid = false
migrationAccessMu.Unlock()
fileAccessWhitelistMu.Lock()
fileAccessWhitelistValid = false
fileAccessWhitelistTypes = nil
fileAccessWhitelistMu.Unlock()
}
// PublishAccessCacheInvalidation broadcasts upload access cache eviction to all nodes.
func PublishAccessCacheInvalidation(ctx context.Context) {
if db.Redis != nil {
_ = db.Redis.Publish(ctx, fileAccessInvalidationChannel, "reset").Err()
}
}
func ensureAccessCacheListener() {
accessCacheOnce.Do(startAccessCacheInvalidationListener)
}
func startAccessCacheInvalidationListener() {
if db.Redis == nil {
return
}
go func() {
pubsub := db.Redis.Subscribe(
context.Background(),
storage.ConfigInvalidationChannel,
fileAccessInvalidationChannel,
)
defer func() {
_ = pubsub.Close()
}()
for range pubsub.Channel() {
ResetAccessCaches()
}
}()
}
func loadMigrationAccessState(ctx context.Context) migrationAccessState {
ensureAccessCacheListener()
migrationAccessMu.RLock()
if migrationAccessValid && time.Since(migrationAccessCheckedAt) < accessCacheTTL {
state := migrationAccessCached
migrationAccessMu.RUnlock()
return state
}
migrationAccessMu.RUnlock()
migrationAccessMu.Lock()
defer migrationAccessMu.Unlock()
if migrationAccessValid && time.Since(migrationAccessCheckedAt) < accessCacheTTL {
return migrationAccessCached
}
migrationAccessCached = buildMigrationAccessState(ctx)
migrationAccessValid = true
migrationAccessCheckedAt = time.Now()
return migrationAccessCached
}
func buildMigrationAccessState(ctx context.Context) migrationAccessState {
execution, ok, err := latestStorageMigrationExecution(ctx)
if err != nil {
return migrationAccessState{loadErr: err, readOnly: true}
}
if !ok {
return migrationAccessState{}
}
state := migrationAccessState{
readOnly: execution.Status != model.TaskExecutionStatusSucceeded,
}
if execution.Status == model.TaskExecutionStatusSucceeded {
return state
}
target, err := parseMigrationTargetConfig(ctx, []byte(execution.Payload))
if err != nil {
state.targetErr = err
return state
}
state.target = target
state.hasTarget = true
return state
}
func loadFileAccessWhitelist(ctx context.Context) map[string]struct{} {
ensureAccessCacheListener()
fileAccessWhitelistMu.RLock()
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < accessCacheTTL {
types := fileAccessWhitelistTypes
fileAccessWhitelistMu.RUnlock()
return types
}
fileAccessWhitelistMu.RUnlock()
fileAccessWhitelistMu.Lock()
defer fileAccessWhitelistMu.Unlock()
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < accessCacheTTL {
return fileAccessWhitelistTypes
}
fileAccessWhitelistTypes = fetchFileAccessWhitelist(ctx)
fileAccessWhitelistValid = true
fileAccessWhitelistCheckedAt = time.Now()
return fileAccessWhitelistTypes
}
func fetchFileAccessWhitelist(ctx context.Context) map[string]struct{} {
whitelist := parseFileAccessWhitelist(ctx)
types := make(map[string]struct{}, len(whitelist))
for _, item := range whitelist {
types[strings.ToLower(item)] = struct{}{}
}
return types
}
func parseFileAccessWhitelist(ctx context.Context) []string {
var sc model.SystemConfig
if err := sc.GetByKey(ctx, model.ConfigKeyFileAccessWhitelist); err != nil || sc.Value == "" {
return []string{defaultPublicUploadType}
}
var whitelist []string
if err := json.Unmarshal([]byte(sc.Value), &whitelist); err == nil && len(whitelist) > 0 {
return whitelist
}
whitelist = parseCommaSeparatedWhitelist(sc.Value)
if len(whitelist) == 0 {
return []string{defaultPublicUploadType}
}
return whitelist
}
func parseCommaSeparatedWhitelist(value string) []string {
parts := strings.Split(value, ",")
whitelist := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
whitelist = append(whitelist, part)
}
}
return whitelist
}
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
)
func TestLoadMigrationAccessStateCachesResult(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
first := loadMigrationAccessState(ctx)
second := loadMigrationAccessState(ctx)
if first.readOnly != second.readOnly {
t.Fatalf("readOnly mismatch: first=%v second=%v", first.readOnly, second.readOnly)
}
if first.hasTarget != second.hasTarget {
t.Fatalf("hasTarget mismatch: first=%v second=%v", first.hasTarget, second.hasTarget)
}
}
func TestIsFilePublicUsesCachedWhitelist(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
if !isFilePublic(ctx, "avatar") {
t.Fatal("expected avatar to be public by default")
}
if isFilePublic(ctx, "attachment") {
t.Fatal("expected attachment to be private by default")
}
if !isFilePublic(ctx, "AVATAR") {
t.Fatal("expected whitelist lookup to be case-insensitive")
}
}
func TestResetAccessCachesRefreshesWhitelist(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
if !isFilePublic(ctx, "avatar") {
t.Fatal("expected seeded avatar whitelist before reset")
}
var sc model.SystemConfig
if err := dbConn.Where("key = ?", model.ConfigKeyFileAccessWhitelist).First(&sc).Error; err != nil {
t.Fatalf("load whitelist config: %v", err)
}
sc.Value = `["attachment"]`
if err := dbConn.Save(&sc).Error; err != nil {
t.Fatalf("save whitelist config: %v", err)
}
if err := db.HSetJSON(ctx, model.SystemConfigRedisHashKey, model.ConfigKeyFileAccessWhitelist, &sc); err != nil {
t.Fatalf("refresh whitelist redis cache: %v", err)
}
ResetAccessCaches()
if !isFilePublic(ctx, "attachment") {
t.Fatal("expected attachment to be public after whitelist refresh")
}
if isFilePublic(ctx, "avatar") {
t.Fatal("expected avatar to be private after whitelist refresh")
}
}
func TestAccessCacheTTLExpires(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
_ = loadFileAccessWhitelist(ctx)
fileAccessWhitelistMu.Lock()
fileAccessWhitelistCheckedAt = time.Now().Add(-accessCacheTTL - time.Second)
fileAccessWhitelistMu.Unlock()
// Should still work after TTL by reloading from config.
if !isFilePublic(ctx, "avatar") {
t.Fatal("expected whitelist reload after TTL expiration")
}
}
+1
View File
@@ -110,6 +110,7 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
continue
}
recordUploadStatsRemove(ctx, &u)
totalDeleted++
lastID = u.ID
}
+3 -1
View File
@@ -14,5 +14,7 @@ const (
imageQualityMedium = "medium"
imageQualityHigh = "high"
imageQualityOrigin = "origin"
storageDriverLocal = string(storage.DriverLocal)
storageDriverLocal = string(storage.DriverLocal)
defaultPublicUploadType = "avatar"
fileStatsTrendDays = 7
)
+2
View File
@@ -140,6 +140,7 @@ func DeleteFile(c *gin.Context) {
c.JSON(http.StatusOK, response.Err(ErrDeleteFileFailed))
return
}
recordUploadStatsRemove(ctx, &upload)
c.JSON(http.StatusOK, response.OKNil())
}
@@ -290,6 +291,7 @@ func DeleteMyFile(c *gin.Context) {
c.JSON(http.StatusOK, response.Err(ErrDeleteFileFailed))
return
}
recordUploadStatsRemove(ctx, &upload)
c.JSON(http.StatusOK, response.OKNil())
}
+48 -30
View File
@@ -4,9 +4,9 @@
package upload
import ("bytes"
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -22,9 +22,18 @@ import ("bytes"
"github.com/Rain-kl/Wavelet/internal/util"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
"golang.org/x/sync/singleflight"
"gorm.io/gorm"
)
var compressedImageFlight singleflight.Group
type compressedImageCacheResult struct {
bytes []byte
cached bool
err error
}
// ServeFileByID 根据 ID 获取并提供已上传的文件
// @Summary 获取已上传文件
// @Description 根据文件 ID 获取并提供已上传的临时或正式文件,若配置了缓存则优先走本地缓存,否则从 S3 等后端存储读取并流式返回
@@ -194,21 +203,51 @@ func ensureCompressedImageCache(
return nil, false, fmt.Errorf("read compressed image cache: %w", err)
}
result, err, _ := compressedImageFlight.Do(cacheKey, func() (any, error) {
return generateCompressedImageCache(ctx, upload, quality, cacheKey)
})
if err != nil {
return nil, false, err
}
res := result.(compressedImageCacheResult)
return res.bytes, res.cached, res.err
}
func generateCompressedImageCache(
ctx context.Context,
upload *model.Upload,
quality string,
cacheKey string,
) (compressedImageCacheResult, error) {
cache := diskcache.GetGlobalCache()
webpBytes, err := cache.Get(cacheKey)
if err == nil {
return compressedImageCacheResult{bytes: webpBytes, cached: true}, nil
}
if !errors.Is(err, diskcache.ErrCacheMiss) {
return compressedImageCacheResult{}, fmt.Errorf("read compressed image cache: %w", err)
}
origBytes, err := getOriginalFileBytes(ctx, upload)
if err != nil {
return nil, false, fmt.Errorf("read original image: %w", err)
return compressedImageCacheResult{}, fmt.Errorf("read original image: %w", err)
}
webpBytes, err = CompressImageToWebP(bytes.NewReader(origBytes), quality)
if err != nil {
return nil, false, fmt.Errorf("compress image to WebP: %w", err)
return compressedImageCacheResult{}, fmt.Errorf("compress image to WebP: %w", err)
}
if err := cache.Set(cacheKey, webpBytes, diskcache.NoExpiration); err != nil {
return webpBytes, false, fmt.Errorf("write compressed image cache: %w", err)
return compressedImageCacheResult{
bytes: webpBytes,
err: fmt.Errorf("write compressed image cache: %w", err),
}, nil
}
return webpBytes, false, nil
return compressedImageCacheResult{bytes: webpBytes}, nil
}
func imageCompressionCacheKey(upload *model.Upload, quality string) string {
@@ -254,30 +293,9 @@ func getOriginalFileBytes(ctx context.Context, upload *model.Upload) ([]byte, er
// isFilePublic 校验文件类型是否在公开访问白名单中
func isFilePublic(ctx context.Context, uploadType string) bool {
var sc model.SystemConfig
var whitelist []string
if err := sc.GetByKey(ctx, model.ConfigKeyFileAccessWhitelist); err == nil && sc.Value != "" {
if err := json.Unmarshal([]byte(sc.Value), &whitelist); err != nil {
// 降级使用逗号分隔解析
parts := strings.Split(sc.Value, ",")
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
whitelist = append(whitelist, p)
}
}
}
} else {
// 默认兜底白名单为 avatar
whitelist = []string{"avatar"}
}
for _, w := range whitelist {
if strings.EqualFold(w, uploadType) {
return true
}
}
return false
whitelist := loadFileAccessWhitelist(ctx)
_, ok := whitelist[strings.ToLower(uploadType)]
return ok
}
func checkPrivateFileOwner(c *gin.Context, ownerID uint64) error {
+3 -1
View File
@@ -120,7 +120,7 @@ func UploadFile(c *gin.Context) {
accessModeStr := c.PostForm("access_mode")
var accessMode int
if accessModeStr == "" {
if uploadType == "avatar" {
if uploadType == defaultPublicUploadType {
accessMode = 1
} else {
accessMode = 0
@@ -388,6 +388,7 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
c.JSON(http.StatusOK, response.Err(ErrSaveUploadRecordFailed))
return true, err
}
recordUploadStatsAdd(ctx, &newUpload)
logger.InfoF(ctx, "文件触发秒传成功! ID: %d, Path: %s", id, existing.FilePath)
c.JSON(http.StatusOK, response.OK(newUpload))
@@ -458,5 +459,6 @@ func saveUploadRecord(ctx context.Context, upload *model.Upload, storageDriver,
}
return ErrSaveUploadRecordFailed
}
recordUploadStatsAdd(ctx, upload)
return ""
}
+3
View File
@@ -806,6 +806,9 @@ func TestGetFileStats(t *testing.T) {
t.Fatalf("failed to create upload: %v", err)
}
}
if err := RebuildUploadStats(context.Background()); err != nil {
t.Fatalf("failed to rebuild upload stats: %v", err)
}
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads/stats", nil)
w := httptest.NewRecorder()
+60 -117
View File
@@ -3,7 +3,8 @@
package upload
import ("net/http"
import (
"net/http"
"strings"
"time"
@@ -11,7 +12,8 @@ import ("net/http"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/gin-gonic/gin"
"github.com/Rain-kl/Wavelet/internal/common/response")
"github.com/Rain-kl/Wavelet/internal/common/response"
)
const (
catImage = "图片"
@@ -56,137 +58,78 @@ type fileStatsResponse struct {
func GetFileStats(c *gin.Context) {
ctx := c.Request.Context()
// 1. 获取总文件数与总文件大小
var summary struct {
TotalCount int64 `json:"total_count"`
TotalSize int64 `json:"total_size"`
}
err := db.DB(ctx).Model(&model.Upload{}).
Select("COUNT(*) as total_count, COALESCE(SUM(file_size), 0) as total_size").
Where("status != ?", model.UploadStatusDeleted).
Scan(&summary).Error
if err != nil {
var stats []model.UploadStat
if err := db.DB(ctx).Find(&stats).Error; err != nil {
c.JSON(http.StatusOK, response.Err(err.Error()))
return
}
// 2. 获取业务类型分布 (Group By type)
type rawDist struct {
Key string `gorm:"column:key"`
Count int64 `gorm:"column:count"`
Size int64 `gorm:"column:size"`
}
var typeRaw []rawDist
err = db.DB(ctx).Model(&model.Upload{}).
Select("type as key, COUNT(*) as count, COALESCE(SUM(file_size), 0) as size").
Where("status != ?", model.UploadStatusDeleted).
Group("type").
Scan(&typeRaw).Error
if err != nil {
c.JSON(http.StatusOK, response.Err(err.Error()))
return
}
types := make([]distributionItem, 0, len(typeRaw))
for _, tr := range typeRaw {
name := tr.Key
if name == "" {
name = "generic"
}
types = append(types, distributionItem{
Name: name,
Count: tr.Count,
Size: tr.Size,
})
}
// 3. 获取所有文件的大小、后缀与MIME,用于在 Go 中内存分类统计 (避免数据库中写复杂的 JSON/String 匹配逻辑)
type fileCategoryRaw struct {
Extension string `gorm:"column:extension"`
MimeType string `gorm:"column:mime_type"`
FileSize int64 `gorm:"column:file_size"`
}
var fileRaws []fileCategoryRaw
err = db.DB(ctx).Model(&model.Upload{}).
Select("extension, mime_type, file_size").
Where("status != ?", model.UploadStatusDeleted).
Scan(&fileRaws).Error
if err != nil {
c.JSON(http.StatusOK, response.Err(err.Error()))
return
}
catCount := make(map[string]int64)
catSize := make(map[string]int64)
categoriesList := []string{catImage, catVideo, catAudio, catDocument, catArchive, catOther}
for _, cat := range categoriesList {
catCount[cat] = 0
catSize[cat] = 0
}
for _, fr := range fileRaws {
cat := getFileCategory(fr.MimeType, fr.Extension)
catCount[cat]++
catSize[cat] += fr.FileSize
}
categories := make([]distributionItem, 0, len(categoriesList))
for _, cat := range categoriesList {
categories = append(categories, distributionItem{
Name: cat,
Count: catCount[cat],
Size: catSize[cat],
})
}
// 4. 获取近 7 天的新增文件趋势 (在 Go 中补全没有新增记录的日期为 0)
type fileTrendRaw struct {
CreatedAt time.Time `gorm:"column:created_at"`
FileSize int64 `gorm:"column:file_size"`
}
var trendRaws []fileTrendRaw
// 7天前 00:00:00 (即 6 天前 00:00:00 至今天)
now := time.Now()
startTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).AddDate(0, 0, -6)
err = db.DB(ctx).Model(&model.Upload{}).
Select("created_at, file_size").
Where("status != ? AND created_at >= ?", model.UploadStatusDeleted, startTime).
Scan(&trendRaws).Error
if err != nil {
c.JSON(http.StatusOK, response.Err(err.Error()))
return
trendDates := make([]string, 0, fileStatsTrendDays)
trendCountMap := make(map[string]int64, fileStatsTrendDays)
trendSizeMap := make(map[string]int64, fileStatsTrendDays)
for i := fileStatsTrendDays - 1; i >= 0; i-- {
date := now.AddDate(0, 0, -i).Format("2006-01-02")
trendDates = append(trendDates, date)
trendCountMap[date] = 0
trendSizeMap[date] = 0
}
trendCountMap := make(map[string]int64)
trendSizeMap := make(map[string]int64)
for i := 0; i < 7; i++ {
dStr := now.AddDate(0, 0, -i).Format("2006-01-02")
trendCountMap[dStr] = 0
trendSizeMap[dStr] = 0
var (
totalCount int64
totalSize int64
types []distributionItem
categories []distributionItem
)
categoriesList := []string{catImage, catVideo, catAudio, catDocument, catArchive, catOther}
categoryMap := make(map[string]distributionItem, len(categoriesList))
for _, cat := range categoriesList {
categoryMap[cat] = distributionItem{Name: cat}
}
for _, tr := range trendRaws {
dStr := tr.CreatedAt.Format("2006-01-02")
if _, exists := trendCountMap[dStr]; exists {
trendCountMap[dStr]++
trendSizeMap[dStr] += tr.FileSize
for _, stat := range stats {
switch stat.Dimension {
case model.UploadStatDimensionTotal:
totalCount = stat.FileCount
totalSize = stat.FileSize
case model.UploadStatDimensionType:
types = append(types, distributionItem{
Name: stat.StatKey,
Count: stat.FileCount,
Size: stat.FileSize,
})
case model.UploadStatDimensionCategory:
if item, ok := categoryMap[stat.StatKey]; ok {
item.Count = stat.FileCount
item.Size = stat.FileSize
categoryMap[stat.StatKey] = item
}
case model.UploadStatDimensionTrend:
if _, ok := trendCountMap[stat.StatKey]; ok {
trendCountMap[stat.StatKey] = stat.FileCount
trendSizeMap[stat.StatKey] = stat.FileSize
}
}
}
const trendDays = 7
trend := make([]trendItem, 0, trendDays)
for i := trendDays - 1; i >= 0; i-- {
dStr := now.AddDate(0, 0, -i).Format("2006-01-02")
categories = make([]distributionItem, 0, len(categoriesList))
for _, cat := range categoriesList {
categories = append(categories, categoryMap[cat])
}
trend := make([]trendItem, 0, len(trendDates))
for _, date := range trendDates {
trend = append(trend, trendItem{
Date: dStr,
Count: trendCountMap[dStr],
Size: trendSizeMap[dStr],
Date: date,
Count: trendCountMap[date],
Size: trendSizeMap[date],
})
}
c.JSON(http.StatusOK, response.OK(fileStatsResponse{
TotalCount: summary.TotalCount,
TotalSize: summary.TotalSize,
TotalCount: totalCount,
TotalSize: totalSize,
Trend: trend,
Categories: categories,
Types: types,
@@ -231,4 +174,4 @@ func isDocumentExtension(ext string) bool {
}
}
return false
}
}
+128
View File
@@ -0,0 +1,128 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"context"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// ApplyUploadStatsAdd increments incremental stats for a newly active upload record.
func ApplyUploadStatsAdd(ctx context.Context, upload *model.Upload) error {
return applyUploadStatsDelta(ctx, upload, 1)
}
// ApplyUploadStatsRemove decrements incremental stats for a removed active upload record.
func ApplyUploadStatsRemove(ctx context.Context, upload *model.Upload) error {
return applyUploadStatsDelta(ctx, upload, -1)
}
// RebuildUploadStats rebuilds all incremental stats from current upload records.
func RebuildUploadStats(ctx context.Context) error {
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Where("1 = 1").Delete(&model.UploadStat{}).Error; err != nil {
return err
}
var uploads []model.Upload
if err := tx.Where("status != ?", model.UploadStatusDeleted).Find(&uploads).Error; err != nil {
return err
}
for i := range uploads {
if err := applyUploadStatsDeltaTx(tx, &uploads[i], 1); err != nil {
return err
}
}
return nil
})
}
func applyUploadStatsDelta(ctx context.Context, upload *model.Upload, sign int64) error {
if upload == nil || !isActiveUploadStatus(upload.Status) {
return nil
}
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return applyUploadStatsDeltaTx(tx, upload, sign)
})
}
func applyUploadStatsDeltaTx(tx *gorm.DB, upload *model.Upload, sign int64) error {
if upload == nil || !isActiveUploadStatus(upload.Status) || sign == 0 {
return nil
}
countDelta := sign
sizeDelta := sign * upload.FileSize
typeKey := upload.Type
if typeKey == "" {
typeKey = "generic"
}
entries := []struct {
dimension string
key string
}{
{model.UploadStatDimensionTotal, ""},
{model.UploadStatDimensionType, typeKey},
{model.UploadStatDimensionCategory, getFileCategory(upload.MimeType, upload.Extension)},
{model.UploadStatDimensionTrend, upload.CreatedAt.Format("2006-01-02")},
}
for _, entry := range entries {
if err := upsertUploadStatDelta(tx, entry.dimension, entry.key, countDelta, sizeDelta); err != nil {
return err
}
}
return nil
}
func upsertUploadStatDelta(tx *gorm.DB, dimension, key string, countDelta, sizeDelta int64) error {
return tx.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "dimension"},
{Name: "stat_key"},
},
DoUpdates: clause.Assignments(map[string]any{
"file_count": gorm.Expr(
"CASE WHEN w_upload_stats.file_count + ? < 0 THEN 0 ELSE w_upload_stats.file_count + ? END",
countDelta,
countDelta,
),
"file_size": gorm.Expr(
"CASE WHEN w_upload_stats.file_size + ? < 0 THEN 0 ELSE w_upload_stats.file_size + ? END",
sizeDelta,
sizeDelta,
),
"updated_at": time.Now(),
}),
}).Create(&model.UploadStat{
Dimension: dimension,
StatKey: key,
FileCount: countDelta,
FileSize: sizeDelta,
}).Error
}
func recordUploadStatsAdd(ctx context.Context, upload *model.Upload) {
if err := ApplyUploadStatsAdd(ctx, upload); err != nil {
logger.WarnF(ctx, "increment upload stats failed: %v", err)
}
}
func recordUploadStatsRemove(ctx context.Context, upload *model.Upload) {
if err := ApplyUploadStatsRemove(ctx, upload); err != nil {
logger.WarnF(ctx, "decrement upload stats failed: %v", err)
}
}
func isActiveUploadStatus(status model.UploadStatus) bool {
return status == model.UploadStatusPending || status == model.UploadStatusUsed
}
@@ -0,0 +1,72 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
)
func TestApplyUploadStatsAddAndRemove(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
upload := &model.Upload{
ID: 42001,
FileSize: 128,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
if err := ApplyUploadStatsAdd(ctx, upload); err != nil {
t.Fatalf("ApplyUploadStatsAdd returned error: %v", err)
}
stats, err := loadUploadStats(ctx)
if err != nil {
t.Fatalf("loadUploadStats returned error: %v", err)
}
if stats.TotalCount != 1 || stats.TotalSize != 128 {
t.Fatalf("unexpected total stats: count=%d size=%d", stats.TotalCount, stats.TotalSize)
}
if err := ApplyUploadStatsRemove(ctx, upload); err != nil {
t.Fatalf("ApplyUploadStatsRemove returned error: %v", err)
}
stats, err = loadUploadStats(ctx)
if err != nil {
t.Fatalf("loadUploadStats after remove returned error: %v", err)
}
if stats.TotalCount != 0 || stats.TotalSize != 0 {
t.Fatalf("expected zeroed total stats, got count=%d size=%d", stats.TotalCount, stats.TotalSize)
}
}
type uploadStatsSnapshot struct {
TotalCount int64
TotalSize int64
}
func loadUploadStats(ctx context.Context) (uploadStatsSnapshot, error) {
var rows []model.UploadStat
if err := db.DB(ctx).Where("dimension = ?", model.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
return uploadStatsSnapshot{}, err
}
if len(rows) == 0 {
return uploadStatsSnapshot{}, nil
}
return uploadStatsSnapshot{
TotalCount: rows[0].FileCount,
TotalSize: rows[0].FileSize,
}, nil
}
+30 -10
View File
@@ -361,16 +361,7 @@ func migrateSingleObject(
source, err := sourceBackend.Get(ctx, obj.FilePath)
if err != nil {
if isNotFoundError(err) {
task.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", obj.FilePath, err)
if updateErr := db.DB(ctx).Model(&model.Upload{}).
Where("storage_driver = ? AND file_path = ?", sourceDriver, obj.FilePath).
Updates(map[string]any{
"status": model.UploadStatusDeleted,
colStorageDriver: targetDriver,
}).Error; updateErr != nil {
return fmt.Errorf("update missing object %q: %w", obj.FilePath, updateErr)
}
return nil
return markMissingMigrationObjectDeleted(ctx, sourceDriver, targetDriver, obj.FilePath, err)
}
return fmt.Errorf("open source object %q: %w", obj.FilePath, err)
}
@@ -441,6 +432,35 @@ func shouldSkipMigration(
return targetObj.ContentLength == obj.FileSize
}
func markMissingMigrationObjectDeleted(
ctx context.Context,
sourceDriver storage.Driver,
targetDriver storage.Driver,
filePath string,
sourceErr error,
) error {
task.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", filePath, sourceErr)
var affectedUploads []model.Upload
if err := db.DB(ctx).
Where("storage_driver = ? AND file_path = ? AND status != ?", sourceDriver, filePath, model.UploadStatusDeleted).
Find(&affectedUploads).Error; err != nil {
return fmt.Errorf("load missing object uploads %q: %w", filePath, err)
}
if err := db.DB(ctx).Model(&model.Upload{}).
Where("storage_driver = ? AND file_path = ?", sourceDriver, filePath).
Updates(map[string]any{
"status": model.UploadStatusDeleted,
colStorageDriver: targetDriver,
}).Error; err != nil {
return fmt.Errorf("update missing object %q: %w", filePath, err)
}
for i := range affectedUploads {
recordUploadStatsRemove(ctx, &affectedUploads[i])
}
return nil
}
func isNotFoundError(err error) bool {
if err == nil {
return false
+12 -16
View File
@@ -14,15 +14,12 @@ import (
// StorageReadOnly checks if the storage system is in read-only maintenance mode.
func StorageReadOnly(ctx context.Context) bool {
execution, ok, err := latestStorageMigrationExecution(ctx)
if err != nil {
logger.ErrorF(ctx, "读取存储维护状态失败: %v", err)
state := loadMigrationAccessState(ctx)
if state.loadErr != nil {
logger.ErrorF(ctx, "读取存储维护状态失败: %v", state.loadErr)
return true
}
if !ok {
return false
}
return execution.Status != model.TaskExecutionStatusSucceeded
return state.readOnly
}
func openStoredObject(ctx context.Context, upload *model.Upload) (*storage.Object, error) {
@@ -54,16 +51,15 @@ func backendForStoredDriver(ctx context.Context, driver storage.Driver) (storage
}
func currentMigrationTargetConfig(ctx context.Context) (storage.Config, bool, error) {
execution, ok, err := latestStorageMigrationExecution(ctx)
if err != nil || !ok {
return storage.Config{}, false, err
state := loadMigrationAccessState(ctx)
if state.loadErr != nil {
return storage.Config{}, false, state.loadErr
}
if execution.Status == model.TaskExecutionStatusSucceeded {
if state.targetErr != nil {
return storage.Config{}, false, state.targetErr
}
if !state.hasTarget {
return storage.Config{}, false, nil
}
target, err := parseMigrationTargetConfig(ctx, []byte(execution.Payload))
if err != nil {
return storage.Config{}, false, err
}
return target, true, nil
return state.target, true, nil
}
@@ -0,0 +1,9 @@
-- +goose Up
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
-- +goose Down
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
@@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_upload_stats (
dimension VARCHAR(32) NOT NULL,
stat_key VARCHAR(64) NOT NULL DEFAULT '',
file_count BIGINT NOT NULL DEFAULT 0,
file_size BIGINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dimension, stat_key)
);
-- +goose Down
DROP TABLE IF EXISTS w_upload_stats;
@@ -0,0 +1,67 @@
-- +goose Up
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'type',
COALESCE(NULLIF(type, ''), 'generic'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'category',
CASE
WHEN LOWER(mime_type) LIKE 'image/%'
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
OR LOWER(mime_type) LIKE '%zip%'
OR LOWER(mime_type) LIKE '%tar%'
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
OR LOWER(mime_type) LIKE 'text/%'
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
ELSE '其他'
END,
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY 2
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'trend',
TO_CHAR(created_at, 'YYYY-MM-DD'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY TO_CHAR(created_at, 'YYYY-MM-DD')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
-- +goose Down
DELETE FROM w_upload_stats;
@@ -0,0 +1,9 @@
-- +goose Up
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
-- +goose Down
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
@@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_upload_stats (
dimension VARCHAR(32) NOT NULL,
stat_key VARCHAR(64) NOT NULL DEFAULT '',
file_count BIGINT NOT NULL DEFAULT 0,
file_size BIGINT NOT NULL DEFAULT 0,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dimension, stat_key)
);
-- +goose Down
DROP TABLE IF EXISTS w_upload_stats;
@@ -0,0 +1,67 @@
-- +goose Up
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'type',
COALESCE(NULLIF(type, ''), 'generic'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'category',
CASE
WHEN LOWER(mime_type) LIKE 'image/%'
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
OR LOWER(mime_type) LIKE '%zip%'
OR LOWER(mime_type) LIKE '%tar%'
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
OR LOWER(mime_type) LIKE 'text/%'
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
ELSE '其他'
END,
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY 2
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'trend',
STRFTIME('%Y-%m-%d', created_at),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY STRFTIME('%Y-%m-%d', created_at)
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
-- +goose Down
DELETE FROM w_upload_stats;
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package model
import "time"
// Upload stats dimension keys stored in w_upload_stats.dimension.
const (
UploadStatDimensionTotal = "total"
UploadStatDimensionType = "type"
UploadStatDimensionCategory = "category"
UploadStatDimensionTrend = "trend"
)
// UploadStat stores incremental upload statistics keyed by dimension and stat_key.
type UploadStat struct {
Dimension string `json:"dimension" gorm:"primaryKey;size:32;not null"`
StatKey string `json:"stat_key" gorm:"primaryKey;size:64;not null;default:''"`
FileCount int64 `json:"file_count" gorm:"not null;default:0"`
FileSize int64 `json:"file_size" gorm:"not null;default:0"`
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
}
// TableName returns the upload stats table name.
func (UploadStat) TableName() string {
return "w_upload_stats"
}
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
// RegisterCustomRoutes registers custom business routes to keep routing clean and stable.
func RegisterCustomRoutes(_ *gin.RouterGroup) {
func RegisterCustomRoutes(apiV1Router *gin.RouterGroup) {
customRouter := apiV1Router.Group("/custom")
{
customRouter.GET("/hello", custom.Hello)
+4 -3
View File
@@ -58,7 +58,8 @@ var (
cacheMutex sync.RWMutex
)
const configInvalidationChannel = "storage:config_invalidation"
// ConfigInvalidationChannel is the Redis pub/sub channel used to evict storage caches cluster-wide.
const ConfigInvalidationChannel = "storage:config_invalidation"
var pubSubOnce sync.Once
@@ -75,7 +76,7 @@ func ResetCache() {
// PublishCacheInvalidation broadcasts cache eviction to all nodes in the cluster via Redis.
func PublishCacheInvalidation(ctx context.Context) {
if db.Redis != nil {
_ = db.Redis.Publish(ctx, configInvalidationChannel, "reset").Err()
_ = db.Redis.Publish(ctx, ConfigInvalidationChannel, "reset").Err()
}
}
@@ -85,7 +86,7 @@ func startPubSubListener() {
return
}
go func() {
pubsub := db.Redis.Subscribe(context.Background(), configInvalidationChannel)
pubsub := db.Redis.Subscribe(context.Background(), ConfigInvalidationChannel)
defer func() {
_ = pubsub.Close()
}()
+1
View File
@@ -42,6 +42,7 @@ func SetupTestEnvironment(t *testing.T) (*gorm.DB, *miniredis.Miniredis, func())
&model.ExternalAccount{},
&model.SystemConfig{},
&model.Upload{},
&model.UploadStat{},
&model.TaskExecution{},
&model.Template{},
&model.AccessToken{},