mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-02 23:06:36 +08:00
perf(upload): optimize file hot paths and incremental admin stats
- Use RWMutex for disk cache reads and singleflight for WebP cache misses - Cache migration read-only state and file access whitelist with pub/sub invalidation - Add w_upload_stats incremental counters updated on upload/delete - Add w_uploads composite indexes and goose backfill migrations - Document performance analysis in docs/PERFORMANCE.md - Fix RegisterCustomRoutes to accept apiV1Router parameter
This commit is contained in:
@@ -11,6 +11,7 @@ import ("context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
@@ -255,9 +256,15 @@ func UpdateSystemConfig(c *gin.Context) {
|
||||
}
|
||||
|
||||
if key == model.ConfigKeyStorageConfig {
|
||||
upload.ResetAccessCaches()
|
||||
upload.PublishAccessCacheInvalidation(c.Request.Context())
|
||||
storage.ResetCache()
|
||||
storage.PublishCacheInvalidation(c.Request.Context())
|
||||
}
|
||||
if key == model.ConfigKeyFileAccessWhitelist {
|
||||
upload.ResetAccessCaches()
|
||||
upload.PublishAccessCacheInvalidation(c.Request.Context())
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response.OKNil())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
)
|
||||
|
||||
const accessCacheTTL = 5 * time.Second
|
||||
|
||||
const fileAccessInvalidationChannel = "upload:file_access_invalidation"
|
||||
|
||||
type migrationAccessState struct {
|
||||
readOnly bool
|
||||
target storage.Config
|
||||
hasTarget bool
|
||||
targetErr error
|
||||
loadErr error
|
||||
}
|
||||
|
||||
var (
|
||||
accessCacheOnce sync.Once
|
||||
|
||||
migrationAccessMu sync.RWMutex
|
||||
migrationAccessCached migrationAccessState
|
||||
migrationAccessValid bool
|
||||
migrationAccessCheckedAt time.Time
|
||||
|
||||
fileAccessWhitelistMu sync.RWMutex
|
||||
fileAccessWhitelistTypes map[string]struct{}
|
||||
fileAccessWhitelistValid bool
|
||||
fileAccessWhitelistCheckedAt time.Time
|
||||
)
|
||||
|
||||
// ResetAccessCaches clears in-process upload access caches.
|
||||
func ResetAccessCaches() {
|
||||
migrationAccessMu.Lock()
|
||||
migrationAccessValid = false
|
||||
migrationAccessMu.Unlock()
|
||||
|
||||
fileAccessWhitelistMu.Lock()
|
||||
fileAccessWhitelistValid = false
|
||||
fileAccessWhitelistTypes = nil
|
||||
fileAccessWhitelistMu.Unlock()
|
||||
}
|
||||
|
||||
// PublishAccessCacheInvalidation broadcasts upload access cache eviction to all nodes.
|
||||
func PublishAccessCacheInvalidation(ctx context.Context) {
|
||||
if db.Redis != nil {
|
||||
_ = db.Redis.Publish(ctx, fileAccessInvalidationChannel, "reset").Err()
|
||||
}
|
||||
}
|
||||
|
||||
func ensureAccessCacheListener() {
|
||||
accessCacheOnce.Do(startAccessCacheInvalidationListener)
|
||||
}
|
||||
|
||||
func startAccessCacheInvalidationListener() {
|
||||
if db.Redis == nil {
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
pubsub := db.Redis.Subscribe(
|
||||
context.Background(),
|
||||
storage.ConfigInvalidationChannel,
|
||||
fileAccessInvalidationChannel,
|
||||
)
|
||||
defer func() {
|
||||
_ = pubsub.Close()
|
||||
}()
|
||||
|
||||
for range pubsub.Channel() {
|
||||
ResetAccessCaches()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func loadMigrationAccessState(ctx context.Context) migrationAccessState {
|
||||
ensureAccessCacheListener()
|
||||
|
||||
migrationAccessMu.RLock()
|
||||
if migrationAccessValid && time.Since(migrationAccessCheckedAt) < accessCacheTTL {
|
||||
state := migrationAccessCached
|
||||
migrationAccessMu.RUnlock()
|
||||
return state
|
||||
}
|
||||
migrationAccessMu.RUnlock()
|
||||
|
||||
migrationAccessMu.Lock()
|
||||
defer migrationAccessMu.Unlock()
|
||||
|
||||
if migrationAccessValid && time.Since(migrationAccessCheckedAt) < accessCacheTTL {
|
||||
return migrationAccessCached
|
||||
}
|
||||
|
||||
migrationAccessCached = buildMigrationAccessState(ctx)
|
||||
migrationAccessValid = true
|
||||
migrationAccessCheckedAt = time.Now()
|
||||
return migrationAccessCached
|
||||
}
|
||||
|
||||
func buildMigrationAccessState(ctx context.Context) migrationAccessState {
|
||||
execution, ok, err := latestStorageMigrationExecution(ctx)
|
||||
if err != nil {
|
||||
return migrationAccessState{loadErr: err, readOnly: true}
|
||||
}
|
||||
if !ok {
|
||||
return migrationAccessState{}
|
||||
}
|
||||
|
||||
state := migrationAccessState{
|
||||
readOnly: execution.Status != model.TaskExecutionStatusSucceeded,
|
||||
}
|
||||
if execution.Status == model.TaskExecutionStatusSucceeded {
|
||||
return state
|
||||
}
|
||||
|
||||
target, err := parseMigrationTargetConfig(ctx, []byte(execution.Payload))
|
||||
if err != nil {
|
||||
state.targetErr = err
|
||||
return state
|
||||
}
|
||||
|
||||
state.target = target
|
||||
state.hasTarget = true
|
||||
return state
|
||||
}
|
||||
|
||||
func loadFileAccessWhitelist(ctx context.Context) map[string]struct{} {
|
||||
ensureAccessCacheListener()
|
||||
|
||||
fileAccessWhitelistMu.RLock()
|
||||
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < accessCacheTTL {
|
||||
types := fileAccessWhitelistTypes
|
||||
fileAccessWhitelistMu.RUnlock()
|
||||
return types
|
||||
}
|
||||
fileAccessWhitelistMu.RUnlock()
|
||||
|
||||
fileAccessWhitelistMu.Lock()
|
||||
defer fileAccessWhitelistMu.Unlock()
|
||||
|
||||
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < accessCacheTTL {
|
||||
return fileAccessWhitelistTypes
|
||||
}
|
||||
|
||||
fileAccessWhitelistTypes = fetchFileAccessWhitelist(ctx)
|
||||
fileAccessWhitelistValid = true
|
||||
fileAccessWhitelistCheckedAt = time.Now()
|
||||
return fileAccessWhitelistTypes
|
||||
}
|
||||
|
||||
func fetchFileAccessWhitelist(ctx context.Context) map[string]struct{} {
|
||||
whitelist := parseFileAccessWhitelist(ctx)
|
||||
types := make(map[string]struct{}, len(whitelist))
|
||||
for _, item := range whitelist {
|
||||
types[strings.ToLower(item)] = struct{}{}
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
func parseFileAccessWhitelist(ctx context.Context) []string {
|
||||
var sc model.SystemConfig
|
||||
if err := sc.GetByKey(ctx, model.ConfigKeyFileAccessWhitelist); err != nil || sc.Value == "" {
|
||||
return []string{defaultPublicUploadType}
|
||||
}
|
||||
|
||||
var whitelist []string
|
||||
if err := json.Unmarshal([]byte(sc.Value), &whitelist); err == nil && len(whitelist) > 0 {
|
||||
return whitelist
|
||||
}
|
||||
|
||||
whitelist = parseCommaSeparatedWhitelist(sc.Value)
|
||||
if len(whitelist) == 0 {
|
||||
return []string{defaultPublicUploadType}
|
||||
}
|
||||
return whitelist
|
||||
}
|
||||
|
||||
func parseCommaSeparatedWhitelist(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
whitelist := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
whitelist = append(whitelist, part)
|
||||
}
|
||||
}
|
||||
return whitelist
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
)
|
||||
|
||||
func TestLoadMigrationAccessStateCachesResult(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
ResetAccessCaches()
|
||||
|
||||
ctx := context.Background()
|
||||
first := loadMigrationAccessState(ctx)
|
||||
second := loadMigrationAccessState(ctx)
|
||||
|
||||
if first.readOnly != second.readOnly {
|
||||
t.Fatalf("readOnly mismatch: first=%v second=%v", first.readOnly, second.readOnly)
|
||||
}
|
||||
if first.hasTarget != second.hasTarget {
|
||||
t.Fatalf("hasTarget mismatch: first=%v second=%v", first.hasTarget, second.hasTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsFilePublicUsesCachedWhitelist(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
ResetAccessCaches()
|
||||
|
||||
ctx := context.Background()
|
||||
if !isFilePublic(ctx, "avatar") {
|
||||
t.Fatal("expected avatar to be public by default")
|
||||
}
|
||||
if isFilePublic(ctx, "attachment") {
|
||||
t.Fatal("expected attachment to be private by default")
|
||||
}
|
||||
if !isFilePublic(ctx, "AVATAR") {
|
||||
t.Fatal("expected whitelist lookup to be case-insensitive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetAccessCachesRefreshesWhitelist(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
ResetAccessCaches()
|
||||
|
||||
ctx := context.Background()
|
||||
if !isFilePublic(ctx, "avatar") {
|
||||
t.Fatal("expected seeded avatar whitelist before reset")
|
||||
}
|
||||
|
||||
var sc model.SystemConfig
|
||||
if err := dbConn.Where("key = ?", model.ConfigKeyFileAccessWhitelist).First(&sc).Error; err != nil {
|
||||
t.Fatalf("load whitelist config: %v", err)
|
||||
}
|
||||
sc.Value = `["attachment"]`
|
||||
if err := dbConn.Save(&sc).Error; err != nil {
|
||||
t.Fatalf("save whitelist config: %v", err)
|
||||
}
|
||||
if err := db.HSetJSON(ctx, model.SystemConfigRedisHashKey, model.ConfigKeyFileAccessWhitelist, &sc); err != nil {
|
||||
t.Fatalf("refresh whitelist redis cache: %v", err)
|
||||
}
|
||||
|
||||
ResetAccessCaches()
|
||||
if !isFilePublic(ctx, "attachment") {
|
||||
t.Fatal("expected attachment to be public after whitelist refresh")
|
||||
}
|
||||
if isFilePublic(ctx, "avatar") {
|
||||
t.Fatal("expected avatar to be private after whitelist refresh")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccessCacheTTLExpires(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
ResetAccessCaches()
|
||||
|
||||
ctx := context.Background()
|
||||
_ = loadFileAccessWhitelist(ctx)
|
||||
|
||||
fileAccessWhitelistMu.Lock()
|
||||
fileAccessWhitelistCheckedAt = time.Now().Add(-accessCacheTTL - time.Second)
|
||||
fileAccessWhitelistMu.Unlock()
|
||||
|
||||
// Should still work after TTL by reloading from config.
|
||||
if !isFilePublic(ctx, "avatar") {
|
||||
t.Fatal("expected whitelist reload after TTL expiration")
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,7 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
|
||||
continue
|
||||
}
|
||||
|
||||
recordUploadStatsRemove(ctx, &u)
|
||||
totalDeleted++
|
||||
lastID = u.ID
|
||||
}
|
||||
|
||||
@@ -14,5 +14,7 @@ const (
|
||||
imageQualityMedium = "medium"
|
||||
imageQualityHigh = "high"
|
||||
imageQualityOrigin = "origin"
|
||||
storageDriverLocal = string(storage.DriverLocal)
|
||||
storageDriverLocal = string(storage.DriverLocal)
|
||||
defaultPublicUploadType = "avatar"
|
||||
fileStatsTrendDays = 7
|
||||
)
|
||||
|
||||
@@ -140,6 +140,7 @@ func DeleteFile(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, response.Err(ErrDeleteFileFailed))
|
||||
return
|
||||
}
|
||||
recordUploadStatsRemove(ctx, &upload)
|
||||
c.JSON(http.StatusOK, response.OKNil())
|
||||
}
|
||||
|
||||
@@ -290,6 +291,7 @@ func DeleteMyFile(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, response.Err(ErrDeleteFileFailed))
|
||||
return
|
||||
}
|
||||
recordUploadStatsRemove(ctx, &upload)
|
||||
c.JSON(http.StatusOK, response.OKNil())
|
||||
}
|
||||
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
|
||||
package upload
|
||||
|
||||
import ("bytes"
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -22,9 +22,18 @@ import ("bytes"
|
||||
"github.com/Rain-kl/Wavelet/internal/util"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
"golang.org/x/sync/singleflight"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var compressedImageFlight singleflight.Group
|
||||
|
||||
type compressedImageCacheResult struct {
|
||||
bytes []byte
|
||||
cached bool
|
||||
err error
|
||||
}
|
||||
|
||||
// ServeFileByID 根据 ID 获取并提供已上传的文件
|
||||
// @Summary 获取已上传文件
|
||||
// @Description 根据文件 ID 获取并提供已上传的临时或正式文件,若配置了缓存则优先走本地缓存,否则从 S3 等后端存储读取并流式返回
|
||||
@@ -194,21 +203,51 @@ func ensureCompressedImageCache(
|
||||
return nil, false, fmt.Errorf("read compressed image cache: %w", err)
|
||||
}
|
||||
|
||||
result, err, _ := compressedImageFlight.Do(cacheKey, func() (any, error) {
|
||||
return generateCompressedImageCache(ctx, upload, quality, cacheKey)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
res := result.(compressedImageCacheResult)
|
||||
return res.bytes, res.cached, res.err
|
||||
}
|
||||
|
||||
func generateCompressedImageCache(
|
||||
ctx context.Context,
|
||||
upload *model.Upload,
|
||||
quality string,
|
||||
cacheKey string,
|
||||
) (compressedImageCacheResult, error) {
|
||||
cache := diskcache.GetGlobalCache()
|
||||
|
||||
webpBytes, err := cache.Get(cacheKey)
|
||||
if err == nil {
|
||||
return compressedImageCacheResult{bytes: webpBytes, cached: true}, nil
|
||||
}
|
||||
if !errors.Is(err, diskcache.ErrCacheMiss) {
|
||||
return compressedImageCacheResult{}, fmt.Errorf("read compressed image cache: %w", err)
|
||||
}
|
||||
|
||||
origBytes, err := getOriginalFileBytes(ctx, upload)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("read original image: %w", err)
|
||||
return compressedImageCacheResult{}, fmt.Errorf("read original image: %w", err)
|
||||
}
|
||||
|
||||
webpBytes, err = CompressImageToWebP(bytes.NewReader(origBytes), quality)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("compress image to WebP: %w", err)
|
||||
return compressedImageCacheResult{}, fmt.Errorf("compress image to WebP: %w", err)
|
||||
}
|
||||
|
||||
if err := cache.Set(cacheKey, webpBytes, diskcache.NoExpiration); err != nil {
|
||||
return webpBytes, false, fmt.Errorf("write compressed image cache: %w", err)
|
||||
return compressedImageCacheResult{
|
||||
bytes: webpBytes,
|
||||
err: fmt.Errorf("write compressed image cache: %w", err),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return webpBytes, false, nil
|
||||
return compressedImageCacheResult{bytes: webpBytes}, nil
|
||||
}
|
||||
|
||||
func imageCompressionCacheKey(upload *model.Upload, quality string) string {
|
||||
@@ -254,30 +293,9 @@ func getOriginalFileBytes(ctx context.Context, upload *model.Upload) ([]byte, er
|
||||
|
||||
// isFilePublic 校验文件类型是否在公开访问白名单中
|
||||
func isFilePublic(ctx context.Context, uploadType string) bool {
|
||||
var sc model.SystemConfig
|
||||
var whitelist []string
|
||||
if err := sc.GetByKey(ctx, model.ConfigKeyFileAccessWhitelist); err == nil && sc.Value != "" {
|
||||
if err := json.Unmarshal([]byte(sc.Value), &whitelist); err != nil {
|
||||
// 降级使用逗号分隔解析
|
||||
parts := strings.Split(sc.Value, ",")
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
whitelist = append(whitelist, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 默认兜底白名单为 avatar
|
||||
whitelist = []string{"avatar"}
|
||||
}
|
||||
|
||||
for _, w := range whitelist {
|
||||
if strings.EqualFold(w, uploadType) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
whitelist := loadFileAccessWhitelist(ctx)
|
||||
_, ok := whitelist[strings.ToLower(uploadType)]
|
||||
return ok
|
||||
}
|
||||
|
||||
func checkPrivateFileOwner(c *gin.Context, ownerID uint64) error {
|
||||
|
||||
@@ -120,7 +120,7 @@ func UploadFile(c *gin.Context) {
|
||||
accessModeStr := c.PostForm("access_mode")
|
||||
var accessMode int
|
||||
if accessModeStr == "" {
|
||||
if uploadType == "avatar" {
|
||||
if uploadType == defaultPublicUploadType {
|
||||
accessMode = 1
|
||||
} else {
|
||||
accessMode = 0
|
||||
@@ -388,6 +388,7 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
|
||||
c.JSON(http.StatusOK, response.Err(ErrSaveUploadRecordFailed))
|
||||
return true, err
|
||||
}
|
||||
recordUploadStatsAdd(ctx, &newUpload)
|
||||
|
||||
logger.InfoF(ctx, "文件触发秒传成功! ID: %d, Path: %s", id, existing.FilePath)
|
||||
c.JSON(http.StatusOK, response.OK(newUpload))
|
||||
@@ -458,5 +459,6 @@ func saveUploadRecord(ctx context.Context, upload *model.Upload, storageDriver,
|
||||
}
|
||||
return ErrSaveUploadRecordFailed
|
||||
}
|
||||
recordUploadStatsAdd(ctx, upload)
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -806,6 +806,9 @@ func TestGetFileStats(t *testing.T) {
|
||||
t.Fatalf("failed to create upload: %v", err)
|
||||
}
|
||||
}
|
||||
if err := RebuildUploadStats(context.Background()); err != nil {
|
||||
t.Fatalf("failed to rebuild upload stats: %v", err)
|
||||
}
|
||||
|
||||
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads/stats", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
+60
-117
@@ -3,7 +3,8 @@
|
||||
|
||||
package upload
|
||||
|
||||
import ("net/http"
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -11,7 +12,8 @@ import ("net/http"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/common/response")
|
||||
"github.com/Rain-kl/Wavelet/internal/common/response"
|
||||
)
|
||||
|
||||
const (
|
||||
catImage = "图片"
|
||||
@@ -56,137 +58,78 @@ type fileStatsResponse struct {
|
||||
func GetFileStats(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
// 1. 获取总文件数与总文件大小
|
||||
var summary struct {
|
||||
TotalCount int64 `json:"total_count"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}
|
||||
err := db.DB(ctx).Model(&model.Upload{}).
|
||||
Select("COUNT(*) as total_count, COALESCE(SUM(file_size), 0) as total_size").
|
||||
Where("status != ?", model.UploadStatusDeleted).
|
||||
Scan(&summary).Error
|
||||
if err != nil {
|
||||
var stats []model.UploadStat
|
||||
if err := db.DB(ctx).Find(&stats).Error; err != nil {
|
||||
c.JSON(http.StatusOK, response.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
// 2. 获取业务类型分布 (Group By type)
|
||||
type rawDist struct {
|
||||
Key string `gorm:"column:key"`
|
||||
Count int64 `gorm:"column:count"`
|
||||
Size int64 `gorm:"column:size"`
|
||||
}
|
||||
var typeRaw []rawDist
|
||||
err = db.DB(ctx).Model(&model.Upload{}).
|
||||
Select("type as key, COUNT(*) as count, COALESCE(SUM(file_size), 0) as size").
|
||||
Where("status != ?", model.UploadStatusDeleted).
|
||||
Group("type").
|
||||
Scan(&typeRaw).Error
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, response.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
types := make([]distributionItem, 0, len(typeRaw))
|
||||
for _, tr := range typeRaw {
|
||||
name := tr.Key
|
||||
if name == "" {
|
||||
name = "generic"
|
||||
}
|
||||
types = append(types, distributionItem{
|
||||
Name: name,
|
||||
Count: tr.Count,
|
||||
Size: tr.Size,
|
||||
})
|
||||
}
|
||||
|
||||
// 3. 获取所有文件的大小、后缀与MIME,用于在 Go 中内存分类统计 (避免数据库中写复杂的 JSON/String 匹配逻辑)
|
||||
type fileCategoryRaw struct {
|
||||
Extension string `gorm:"column:extension"`
|
||||
MimeType string `gorm:"column:mime_type"`
|
||||
FileSize int64 `gorm:"column:file_size"`
|
||||
}
|
||||
var fileRaws []fileCategoryRaw
|
||||
err = db.DB(ctx).Model(&model.Upload{}).
|
||||
Select("extension, mime_type, file_size").
|
||||
Where("status != ?", model.UploadStatusDeleted).
|
||||
Scan(&fileRaws).Error
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, response.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
catCount := make(map[string]int64)
|
||||
catSize := make(map[string]int64)
|
||||
categoriesList := []string{catImage, catVideo, catAudio, catDocument, catArchive, catOther}
|
||||
for _, cat := range categoriesList {
|
||||
catCount[cat] = 0
|
||||
catSize[cat] = 0
|
||||
}
|
||||
|
||||
for _, fr := range fileRaws {
|
||||
cat := getFileCategory(fr.MimeType, fr.Extension)
|
||||
catCount[cat]++
|
||||
catSize[cat] += fr.FileSize
|
||||
}
|
||||
|
||||
categories := make([]distributionItem, 0, len(categoriesList))
|
||||
for _, cat := range categoriesList {
|
||||
categories = append(categories, distributionItem{
|
||||
Name: cat,
|
||||
Count: catCount[cat],
|
||||
Size: catSize[cat],
|
||||
})
|
||||
}
|
||||
|
||||
// 4. 获取近 7 天的新增文件趋势 (在 Go 中补全没有新增记录的日期为 0)
|
||||
type fileTrendRaw struct {
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
FileSize int64 `gorm:"column:file_size"`
|
||||
}
|
||||
var trendRaws []fileTrendRaw
|
||||
// 7天前 00:00:00 (即 6 天前 00:00:00 至今天)
|
||||
now := time.Now()
|
||||
startTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()).AddDate(0, 0, -6)
|
||||
err = db.DB(ctx).Model(&model.Upload{}).
|
||||
Select("created_at, file_size").
|
||||
Where("status != ? AND created_at >= ?", model.UploadStatusDeleted, startTime).
|
||||
Scan(&trendRaws).Error
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, response.Err(err.Error()))
|
||||
return
|
||||
trendDates := make([]string, 0, fileStatsTrendDays)
|
||||
trendCountMap := make(map[string]int64, fileStatsTrendDays)
|
||||
trendSizeMap := make(map[string]int64, fileStatsTrendDays)
|
||||
for i := fileStatsTrendDays - 1; i >= 0; i-- {
|
||||
date := now.AddDate(0, 0, -i).Format("2006-01-02")
|
||||
trendDates = append(trendDates, date)
|
||||
trendCountMap[date] = 0
|
||||
trendSizeMap[date] = 0
|
||||
}
|
||||
|
||||
trendCountMap := make(map[string]int64)
|
||||
trendSizeMap := make(map[string]int64)
|
||||
for i := 0; i < 7; i++ {
|
||||
dStr := now.AddDate(0, 0, -i).Format("2006-01-02")
|
||||
trendCountMap[dStr] = 0
|
||||
trendSizeMap[dStr] = 0
|
||||
var (
|
||||
totalCount int64
|
||||
totalSize int64
|
||||
types []distributionItem
|
||||
categories []distributionItem
|
||||
)
|
||||
|
||||
categoriesList := []string{catImage, catVideo, catAudio, catDocument, catArchive, catOther}
|
||||
categoryMap := make(map[string]distributionItem, len(categoriesList))
|
||||
for _, cat := range categoriesList {
|
||||
categoryMap[cat] = distributionItem{Name: cat}
|
||||
}
|
||||
|
||||
for _, tr := range trendRaws {
|
||||
dStr := tr.CreatedAt.Format("2006-01-02")
|
||||
if _, exists := trendCountMap[dStr]; exists {
|
||||
trendCountMap[dStr]++
|
||||
trendSizeMap[dStr] += tr.FileSize
|
||||
for _, stat := range stats {
|
||||
switch stat.Dimension {
|
||||
case model.UploadStatDimensionTotal:
|
||||
totalCount = stat.FileCount
|
||||
totalSize = stat.FileSize
|
||||
case model.UploadStatDimensionType:
|
||||
types = append(types, distributionItem{
|
||||
Name: stat.StatKey,
|
||||
Count: stat.FileCount,
|
||||
Size: stat.FileSize,
|
||||
})
|
||||
case model.UploadStatDimensionCategory:
|
||||
if item, ok := categoryMap[stat.StatKey]; ok {
|
||||
item.Count = stat.FileCount
|
||||
item.Size = stat.FileSize
|
||||
categoryMap[stat.StatKey] = item
|
||||
}
|
||||
case model.UploadStatDimensionTrend:
|
||||
if _, ok := trendCountMap[stat.StatKey]; ok {
|
||||
trendCountMap[stat.StatKey] = stat.FileCount
|
||||
trendSizeMap[stat.StatKey] = stat.FileSize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const trendDays = 7
|
||||
trend := make([]trendItem, 0, trendDays)
|
||||
for i := trendDays - 1; i >= 0; i-- {
|
||||
dStr := now.AddDate(0, 0, -i).Format("2006-01-02")
|
||||
categories = make([]distributionItem, 0, len(categoriesList))
|
||||
for _, cat := range categoriesList {
|
||||
categories = append(categories, categoryMap[cat])
|
||||
}
|
||||
|
||||
trend := make([]trendItem, 0, len(trendDates))
|
||||
for _, date := range trendDates {
|
||||
trend = append(trend, trendItem{
|
||||
Date: dStr,
|
||||
Count: trendCountMap[dStr],
|
||||
Size: trendSizeMap[dStr],
|
||||
Date: date,
|
||||
Count: trendCountMap[date],
|
||||
Size: trendSizeMap[date],
|
||||
})
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response.OK(fileStatsResponse{
|
||||
TotalCount: summary.TotalCount,
|
||||
TotalSize: summary.TotalSize,
|
||||
TotalCount: totalCount,
|
||||
TotalSize: totalSize,
|
||||
Trend: trend,
|
||||
Categories: categories,
|
||||
Types: types,
|
||||
@@ -231,4 +174,4 @@ func isDocumentExtension(ext string) bool {
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// ApplyUploadStatsAdd increments incremental stats for a newly active upload record.
|
||||
func ApplyUploadStatsAdd(ctx context.Context, upload *model.Upload) error {
|
||||
return applyUploadStatsDelta(ctx, upload, 1)
|
||||
}
|
||||
|
||||
// ApplyUploadStatsRemove decrements incremental stats for a removed active upload record.
|
||||
func ApplyUploadStatsRemove(ctx context.Context, upload *model.Upload) error {
|
||||
return applyUploadStatsDelta(ctx, upload, -1)
|
||||
}
|
||||
|
||||
// RebuildUploadStats rebuilds all incremental stats from current upload records.
|
||||
func RebuildUploadStats(ctx context.Context) error {
|
||||
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Where("1 = 1").Delete(&model.UploadStat{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var uploads []model.Upload
|
||||
if err := tx.Where("status != ?", model.UploadStatusDeleted).Find(&uploads).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := range uploads {
|
||||
if err := applyUploadStatsDeltaTx(tx, &uploads[i], 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func applyUploadStatsDelta(ctx context.Context, upload *model.Upload, sign int64) error {
|
||||
if upload == nil || !isActiveUploadStatus(upload.Status) {
|
||||
return nil
|
||||
}
|
||||
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
return applyUploadStatsDeltaTx(tx, upload, sign)
|
||||
})
|
||||
}
|
||||
|
||||
func applyUploadStatsDeltaTx(tx *gorm.DB, upload *model.Upload, sign int64) error {
|
||||
if upload == nil || !isActiveUploadStatus(upload.Status) || sign == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
countDelta := sign
|
||||
sizeDelta := sign * upload.FileSize
|
||||
typeKey := upload.Type
|
||||
if typeKey == "" {
|
||||
typeKey = "generic"
|
||||
}
|
||||
|
||||
entries := []struct {
|
||||
dimension string
|
||||
key string
|
||||
}{
|
||||
{model.UploadStatDimensionTotal, ""},
|
||||
{model.UploadStatDimensionType, typeKey},
|
||||
{model.UploadStatDimensionCategory, getFileCategory(upload.MimeType, upload.Extension)},
|
||||
{model.UploadStatDimensionTrend, upload.CreatedAt.Format("2006-01-02")},
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if err := upsertUploadStatDelta(tx, entry.dimension, entry.key, countDelta, sizeDelta); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertUploadStatDelta(tx *gorm.DB, dimension, key string, countDelta, sizeDelta int64) error {
|
||||
return tx.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{
|
||||
{Name: "dimension"},
|
||||
{Name: "stat_key"},
|
||||
},
|
||||
DoUpdates: clause.Assignments(map[string]any{
|
||||
"file_count": gorm.Expr(
|
||||
"CASE WHEN w_upload_stats.file_count + ? < 0 THEN 0 ELSE w_upload_stats.file_count + ? END",
|
||||
countDelta,
|
||||
countDelta,
|
||||
),
|
||||
"file_size": gorm.Expr(
|
||||
"CASE WHEN w_upload_stats.file_size + ? < 0 THEN 0 ELSE w_upload_stats.file_size + ? END",
|
||||
sizeDelta,
|
||||
sizeDelta,
|
||||
),
|
||||
"updated_at": time.Now(),
|
||||
}),
|
||||
}).Create(&model.UploadStat{
|
||||
Dimension: dimension,
|
||||
StatKey: key,
|
||||
FileCount: countDelta,
|
||||
FileSize: sizeDelta,
|
||||
}).Error
|
||||
}
|
||||
|
||||
func recordUploadStatsAdd(ctx context.Context, upload *model.Upload) {
|
||||
if err := ApplyUploadStatsAdd(ctx, upload); err != nil {
|
||||
logger.WarnF(ctx, "increment upload stats failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func recordUploadStatsRemove(ctx context.Context, upload *model.Upload) {
|
||||
if err := ApplyUploadStatsRemove(ctx, upload); err != nil {
|
||||
logger.WarnF(ctx, "decrement upload stats failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func isActiveUploadStatus(status model.UploadStatus) bool {
|
||||
return status == model.UploadStatusPending || status == model.UploadStatusUsed
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
)
|
||||
|
||||
func TestApplyUploadStatsAddAndRemove(t *testing.T) {
|
||||
_, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
upload := &model.Upload{
|
||||
ID: 42001,
|
||||
FileSize: 128,
|
||||
MimeType: "image/png",
|
||||
Extension: "png",
|
||||
Type: "avatar",
|
||||
Status: model.UploadStatusUsed,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
if err := ApplyUploadStatsAdd(ctx, upload); err != nil {
|
||||
t.Fatalf("ApplyUploadStatsAdd returned error: %v", err)
|
||||
}
|
||||
|
||||
stats, err := loadUploadStats(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("loadUploadStats returned error: %v", err)
|
||||
}
|
||||
if stats.TotalCount != 1 || stats.TotalSize != 128 {
|
||||
t.Fatalf("unexpected total stats: count=%d size=%d", stats.TotalCount, stats.TotalSize)
|
||||
}
|
||||
|
||||
if err := ApplyUploadStatsRemove(ctx, upload); err != nil {
|
||||
t.Fatalf("ApplyUploadStatsRemove returned error: %v", err)
|
||||
}
|
||||
|
||||
stats, err = loadUploadStats(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("loadUploadStats after remove returned error: %v", err)
|
||||
}
|
||||
if stats.TotalCount != 0 || stats.TotalSize != 0 {
|
||||
t.Fatalf("expected zeroed total stats, got count=%d size=%d", stats.TotalCount, stats.TotalSize)
|
||||
}
|
||||
}
|
||||
|
||||
type uploadStatsSnapshot struct {
|
||||
TotalCount int64
|
||||
TotalSize int64
|
||||
}
|
||||
|
||||
func loadUploadStats(ctx context.Context) (uploadStatsSnapshot, error) {
|
||||
var rows []model.UploadStat
|
||||
if err := db.DB(ctx).Where("dimension = ?", model.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
|
||||
return uploadStatsSnapshot{}, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return uploadStatsSnapshot{}, nil
|
||||
}
|
||||
return uploadStatsSnapshot{
|
||||
TotalCount: rows[0].FileCount,
|
||||
TotalSize: rows[0].FileSize,
|
||||
}, nil
|
||||
}
|
||||
@@ -361,16 +361,7 @@ func migrateSingleObject(
|
||||
source, err := sourceBackend.Get(ctx, obj.FilePath)
|
||||
if err != nil {
|
||||
if isNotFoundError(err) {
|
||||
task.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", obj.FilePath, err)
|
||||
if updateErr := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("storage_driver = ? AND file_path = ?", sourceDriver, obj.FilePath).
|
||||
Updates(map[string]any{
|
||||
"status": model.UploadStatusDeleted,
|
||||
colStorageDriver: targetDriver,
|
||||
}).Error; updateErr != nil {
|
||||
return fmt.Errorf("update missing object %q: %w", obj.FilePath, updateErr)
|
||||
}
|
||||
return nil
|
||||
return markMissingMigrationObjectDeleted(ctx, sourceDriver, targetDriver, obj.FilePath, err)
|
||||
}
|
||||
return fmt.Errorf("open source object %q: %w", obj.FilePath, err)
|
||||
}
|
||||
@@ -441,6 +432,35 @@ func shouldSkipMigration(
|
||||
return targetObj.ContentLength == obj.FileSize
|
||||
}
|
||||
|
||||
func markMissingMigrationObjectDeleted(
|
||||
ctx context.Context,
|
||||
sourceDriver storage.Driver,
|
||||
targetDriver storage.Driver,
|
||||
filePath string,
|
||||
sourceErr error,
|
||||
) error {
|
||||
task.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", filePath, sourceErr)
|
||||
|
||||
var affectedUploads []model.Upload
|
||||
if err := db.DB(ctx).
|
||||
Where("storage_driver = ? AND file_path = ? AND status != ?", sourceDriver, filePath, model.UploadStatusDeleted).
|
||||
Find(&affectedUploads).Error; err != nil {
|
||||
return fmt.Errorf("load missing object uploads %q: %w", filePath, err)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("storage_driver = ? AND file_path = ?", sourceDriver, filePath).
|
||||
Updates(map[string]any{
|
||||
"status": model.UploadStatusDeleted,
|
||||
colStorageDriver: targetDriver,
|
||||
}).Error; err != nil {
|
||||
return fmt.Errorf("update missing object %q: %w", filePath, err)
|
||||
}
|
||||
for i := range affectedUploads {
|
||||
recordUploadStatsRemove(ctx, &affectedUploads[i])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isNotFoundError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
|
||||
@@ -14,15 +14,12 @@ import (
|
||||
|
||||
// StorageReadOnly checks if the storage system is in read-only maintenance mode.
|
||||
func StorageReadOnly(ctx context.Context) bool {
|
||||
execution, ok, err := latestStorageMigrationExecution(ctx)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "读取存储维护状态失败: %v", err)
|
||||
state := loadMigrationAccessState(ctx)
|
||||
if state.loadErr != nil {
|
||||
logger.ErrorF(ctx, "读取存储维护状态失败: %v", state.loadErr)
|
||||
return true
|
||||
}
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return execution.Status != model.TaskExecutionStatusSucceeded
|
||||
return state.readOnly
|
||||
}
|
||||
|
||||
func openStoredObject(ctx context.Context, upload *model.Upload) (*storage.Object, error) {
|
||||
@@ -54,16 +51,15 @@ func backendForStoredDriver(ctx context.Context, driver storage.Driver) (storage
|
||||
}
|
||||
|
||||
func currentMigrationTargetConfig(ctx context.Context) (storage.Config, bool, error) {
|
||||
execution, ok, err := latestStorageMigrationExecution(ctx)
|
||||
if err != nil || !ok {
|
||||
return storage.Config{}, false, err
|
||||
state := loadMigrationAccessState(ctx)
|
||||
if state.loadErr != nil {
|
||||
return storage.Config{}, false, state.loadErr
|
||||
}
|
||||
if execution.Status == model.TaskExecutionStatusSucceeded {
|
||||
if state.targetErr != nil {
|
||||
return storage.Config{}, false, state.targetErr
|
||||
}
|
||||
if !state.hasTarget {
|
||||
return storage.Config{}, false, nil
|
||||
}
|
||||
target, err := parseMigrationTargetConfig(ctx, []byte(execution.Payload))
|
||||
if err != nil {
|
||||
return storage.Config{}, false, err
|
||||
}
|
||||
return target, true, nil
|
||||
return state.target, true, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS w_upload_stats (
|
||||
dimension VARCHAR(32) NOT NULL,
|
||||
stat_key VARCHAR(64) NOT NULL DEFAULT '',
|
||||
file_count BIGINT NOT NULL DEFAULT 0,
|
||||
file_size BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (dimension, stat_key)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_upload_stats;
|
||||
@@ -0,0 +1,67 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'type',
|
||||
COALESCE(NULLIF(type, ''), 'generic'),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'category',
|
||||
CASE
|
||||
WHEN LOWER(mime_type) LIKE 'image/%'
|
||||
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
|
||||
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
|
||||
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
|
||||
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
|
||||
OR LOWER(mime_type) LIKE '%zip%'
|
||||
OR LOWER(mime_type) LIKE '%tar%'
|
||||
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
|
||||
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
|
||||
OR LOWER(mime_type) LIKE 'text/%'
|
||||
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
|
||||
ELSE '其他'
|
||||
END,
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY 2
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'trend',
|
||||
TO_CHAR(created_at, 'YYYY-MM-DD'),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY TO_CHAR(created_at, 'YYYY-MM-DD')
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_upload_stats;
|
||||
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
|
||||
@@ -0,0 +1,12 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS w_upload_stats (
|
||||
dimension VARCHAR(32) NOT NULL,
|
||||
stat_key VARCHAR(64) NOT NULL DEFAULT '',
|
||||
file_count BIGINT NOT NULL DEFAULT 0,
|
||||
file_size BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (dimension, stat_key)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_upload_stats;
|
||||
@@ -0,0 +1,67 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'type',
|
||||
COALESCE(NULLIF(type, ''), 'generic'),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'category',
|
||||
CASE
|
||||
WHEN LOWER(mime_type) LIKE 'image/%'
|
||||
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
|
||||
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
|
||||
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
|
||||
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
|
||||
OR LOWER(mime_type) LIKE '%zip%'
|
||||
OR LOWER(mime_type) LIKE '%tar%'
|
||||
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
|
||||
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
|
||||
OR LOWER(mime_type) LIKE 'text/%'
|
||||
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
|
||||
ELSE '其他'
|
||||
END,
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY 2
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'trend',
|
||||
STRFTIME('%Y-%m-%d', created_at),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY STRFTIME('%Y-%m-%d', created_at)
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_upload_stats;
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// Upload stats dimension keys stored in w_upload_stats.dimension.
|
||||
const (
|
||||
UploadStatDimensionTotal = "total"
|
||||
UploadStatDimensionType = "type"
|
||||
UploadStatDimensionCategory = "category"
|
||||
UploadStatDimensionTrend = "trend"
|
||||
)
|
||||
|
||||
// UploadStat stores incremental upload statistics keyed by dimension and stat_key.
|
||||
type UploadStat struct {
|
||||
Dimension string `json:"dimension" gorm:"primaryKey;size:32;not null"`
|
||||
StatKey string `json:"stat_key" gorm:"primaryKey;size:64;not null;default:''"`
|
||||
FileCount int64 `json:"file_count" gorm:"not null;default:0"`
|
||||
FileSize int64 `json:"file_size" gorm:"not null;default:0"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the upload stats table name.
|
||||
func (UploadStat) TableName() string {
|
||||
return "w_upload_stats"
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// RegisterCustomRoutes registers custom business routes to keep routing clean and stable.
|
||||
func RegisterCustomRoutes(_ *gin.RouterGroup) {
|
||||
func RegisterCustomRoutes(apiV1Router *gin.RouterGroup) {
|
||||
customRouter := apiV1Router.Group("/custom")
|
||||
{
|
||||
customRouter.GET("/hello", custom.Hello)
|
||||
|
||||
@@ -58,7 +58,8 @@ var (
|
||||
cacheMutex sync.RWMutex
|
||||
)
|
||||
|
||||
const configInvalidationChannel = "storage:config_invalidation"
|
||||
// ConfigInvalidationChannel is the Redis pub/sub channel used to evict storage caches cluster-wide.
|
||||
const ConfigInvalidationChannel = "storage:config_invalidation"
|
||||
|
||||
var pubSubOnce sync.Once
|
||||
|
||||
@@ -75,7 +76,7 @@ func ResetCache() {
|
||||
// PublishCacheInvalidation broadcasts cache eviction to all nodes in the cluster via Redis.
|
||||
func PublishCacheInvalidation(ctx context.Context) {
|
||||
if db.Redis != nil {
|
||||
_ = db.Redis.Publish(ctx, configInvalidationChannel, "reset").Err()
|
||||
_ = db.Redis.Publish(ctx, ConfigInvalidationChannel, "reset").Err()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +86,7 @@ func startPubSubListener() {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
pubsub := db.Redis.Subscribe(context.Background(), configInvalidationChannel)
|
||||
pubsub := db.Redis.Subscribe(context.Background(), ConfigInvalidationChannel)
|
||||
defer func() {
|
||||
_ = pubsub.Close()
|
||||
}()
|
||||
|
||||
@@ -42,6 +42,7 @@ func SetupTestEnvironment(t *testing.T) (*gorm.DB, *miniredis.Miniredis, func())
|
||||
&model.ExternalAccount{},
|
||||
&model.SystemConfig{},
|
||||
&model.Upload{},
|
||||
&model.UploadStat{},
|
||||
&model.TaskExecution{},
|
||||
&model.Template{},
|
||||
&model.AccessToken{},
|
||||
|
||||
Reference in New Issue
Block a user