feat(storage): add dynamic storage config and migration

Move storage backend configuration from startup YAML to system_config-backed runtime configuration. Add local, S3-compatible, R2, MinIO, OSS, and WebDAV backend support.

Add a storage migration async task using the existing task dispatch framework. Migration target config is carried in task payload, and maintenance mode is derived from task execution state.

Split upload file management and storage operations, add the admin storage configuration tab, and update migrations and Swagger docs.
This commit is contained in:
ryan
2026-06-13 14:55:15 +08:00
parent 4bf8a4806e
commit 9a18bea324
46 changed files with 2005 additions and 743 deletions
+53 -8
View File
@@ -5,11 +5,15 @@
package system_config
import (
"encoding/json"
"errors"
"net/http"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/logger"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
"github.com/Rain-kl/Wavelet/internal/util"
mail "github.com/Rain-kl/Wavelet/internal/util/mail"
"github.com/gin-gonic/gin"
@@ -21,7 +25,7 @@ const maskedConfigValue = "******"
// CreateSystemConfigRequest 创建系统配置请求
type CreateSystemConfigRequest struct {
Key string `json:"key" binding:"required,max=64"`
Value string `json:"value" binding:"required,max=255"`
Value string `json:"value" binding:"required"`
Type string `json:"type" binding:"required,oneof=system business"`
Visibility int `json:"visibility" binding:"oneof=0 1"`
Description string `json:"description" binding:"max=255"`
@@ -29,7 +33,7 @@ type CreateSystemConfigRequest struct {
// UpdateSystemConfigRequest 更新系统配置请求
type UpdateSystemConfigRequest struct {
Value string `json:"value" binding:"required,max=255"`
Value string `json:"value" binding:"required"`
Visibility *int `json:"visibility" binding:"omitempty,oneof=0 1"`
Description string `json:"description" binding:"max=255"`
}
@@ -118,9 +122,7 @@ func ListSystemConfigs(c *gin.Context) {
}
for i := range configs {
if configs[i].Key == model.ConfigKeySMTPPassword && configs[i].Value != "" {
configs[i].Value = maskedConfigValue
}
configs[i].Value = maskSensitiveConfig(configs[i].Key, configs[i].Value)
}
c.JSON(http.StatusOK, util.OK(configs))
@@ -150,9 +152,7 @@ func GetSystemConfig(c *gin.Context) {
return
}
if config.Key == model.ConfigKeySMTPPassword && config.Value != "" {
config.Value = maskedConfigValue
}
config.Value = maskSensitiveConfig(config.Key, config.Value)
c.JSON(http.StatusOK, util.OK(config))
}
@@ -193,6 +193,14 @@ func UpdateSystemConfig(c *gin.Context) {
return
}
var originalDriver storage.Driver
if key == model.ConfigKeyStorageConfig {
var currentCfg storage.Config
if err := json.Unmarshal([]byte(config.Value), &currentCfg); err == nil {
originalDriver = currentCfg.Driver
}
}
if err := db.DB(c.Request.Context()).Transaction(func(tx *gorm.DB) error {
// 更新配置
updates := map[string]interface{}{
@@ -214,6 +222,24 @@ func UpdateSystemConfig(c *gin.Context) {
return err
}
if key == model.ConfigKeyStorageConfig && originalDriver != "" {
var newCfg storage.Config
if err := json.Unmarshal([]byte(req.Value), &newCfg); err == nil {
if newCfg.Driver == originalDriver {
// Mark failed storage:migrate task execution as succeeded
if err := tx.Model(&model.TaskExecution{}).
Where("task_type = ? AND status = ?", "storage:migrate", model.TaskExecutionStatusFailed).
Updates(map[string]any{
"status": model.TaskExecutionStatusSucceeded,
"result": "存储配置直接更新,故障迁移任务自动标记为已解决",
"finished_at": time.Now(),
}).Error; err != nil {
logger.ErrorF(c.Request.Context(), "自动更新迁移任务状态失败: %v", err)
}
}
}
}
return nil
}); err != nil {
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
@@ -288,3 +314,22 @@ func TestSMTP(c *gin.Context) {
c.JSON(http.StatusOK, util.OK(resp))
}
func maskSensitiveConfig(key, value string) string {
if value == "" {
return value
}
switch key {
case model.ConfigKeySMTPPassword:
return maskedConfigValue
case model.ConfigKeyStorageConfig:
var cfg storage.Config
if err := json.Unmarshal([]byte(value), &cfg); err == nil {
masked := storage.MaskSecrets(cfg)
if val, err := json.Marshal(masked); err == nil {
return string(val)
}
}
}
return value
}
@@ -23,7 +23,7 @@ import (
"github.com/gin-gonic/gin"
)
const expectedDefaultConfigsCount = 29
const expectedDefaultConfigsCount = 30
func setupTestRouter(authUser *model.User) *gin.Engine {
gin.SetMode(gin.TestMode)
+3 -1
View File
@@ -3,6 +3,8 @@
package upload
import "github.com/Rain-kl/Wavelet/internal/storage"
const (
maxUploadSize = 32 * 1024 * 1024 // 32MB
detectContentBytes = 512 // http.DetectContentType 需要的最小字节数
@@ -12,5 +14,5 @@ const (
imageQualityMedium = "medium"
imageQualityHigh = "high"
imageQualityOrigin = "origin"
storageDriverLocal = "local"
storageDriverLocal = string(storage.DriverLocal)
)
+1
View File
@@ -27,6 +27,7 @@ const (
ErrQueryFileCountFailed = "查询文件数量失败"
ErrQueryFileListFailed = "查询文件列表失败"
ErrDeleteFileFailed = "删除文件失败"
ErrStorageReadOnly = "存储迁移维护中,当前仅允许读取文件"
ErrS3KeyRequired = "s3 key must not be empty"
ErrS3KeyTooLongFormat = "s3 key exceeds maximum length of %d"
ErrS3KeyStartsWithSlash = "s3 key must not start with /"
+167
View File
@@ -0,0 +1,167 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"errors"
"net/http"
"sort"
"strconv"
"strings"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/util"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
type listMyFilesRequest struct {
Page int `form:"page"`
PageSize int `form:"page_size"`
Keyword string `form:"keyword"`
Type string `form:"type"`
Extension string `form:"extension"`
}
type listMyFilesResponse struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []model.Upload `json:"items"`
}
// ListMyFiles 获取当前用户上传的文件列表
// @Summary 获取我的文件列表
// @Description 分页获取当前登录用户上传的文件,支持文件名关键词、业务类型、扩展名过滤
// @Tags upload
// @Produce json
// @Param page query int false "页码(默认 1)"
// @Param page_size query int false "每页数量(默认 20,最大 100)"
// @Param keyword query string false "文件名关键词(模糊匹配)"
// @Param type query string false "业务分类过滤"
// @Param extension query string false "扩展名过滤"
// @Security SessionCookie
// @Success 200 {object} util.ResponseAny{data=listMyFilesResponse} "查询成功"
// @Failure 401 {object} util.ResponseAny "未登录"
// @Router /api/v1/upload/my [get]
func ListMyFiles(c *gin.Context) {
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
ctx := c.Request.Context()
var req listMyFilesRequest
if err := c.ShouldBindQuery(&req); err != nil {
c.JSON(http.StatusOK, util.Err(ErrInvalidParams))
return
}
if req.Page <= 0 {
req.Page = 1
}
if req.PageSize <= 0 || req.PageSize > 100 {
req.PageSize = 20
}
query := db.DB(ctx).Model(&model.Upload{}).
Where("user_id = ? AND status != ?", currUser.ID, model.UploadStatusDeleted)
if req.Keyword != "" {
query = query.Where("LOWER(file_name) LIKE ?", "%"+strings.ToLower(req.Keyword)+"%")
}
if req.Type != "" {
query = query.Where("type = ?", req.Type)
}
if req.Extension != "" {
query = query.Where("extension = ?", strings.ToLower(req.Extension))
}
var total int64
if err := query.Count(&total).Error; err != nil {
c.JSON(http.StatusOK, util.Err(ErrQueryFileCountFailed))
return
}
var items []model.Upload
offset := (req.Page - 1) * req.PageSize
if err := query.Order("created_at DESC").Offset(offset).Limit(req.PageSize).Find(&items).Error; err != nil {
c.JSON(http.StatusOK, util.Err(ErrQueryFileListFailed))
return
}
c.JSON(http.StatusOK, util.OK(listMyFilesResponse{
Total: total,
Page: req.Page,
PageSize: req.PageSize,
Items: items,
}))
}
// DeleteFile 软删除文件记录
// @Summary 删除文件
// @Description 将文件状态置为 deleted(软删除),不会立即清理底层存储对象
// @Tags upload
// @Produce json
// @Param id path string true "文件 ID"
// @Security SessionCookie
// @Success 200 {object} util.ResponseAny "删除成功"
// @Failure 403 {object} util.ResponseAny "无权操作"
// @Failure 404 {object} util.ResponseAny "文件不存在"
// @Router /api/v1/upload/{id} [delete]
func DeleteFile(c *gin.Context) {
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
ctx := c.Request.Context()
if storageReadOnly(ctx) {
c.JSON(http.StatusConflict, util.Err(ErrStorageReadOnly))
return
}
uploadID, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusOK, util.Err(ErrInvalidFileID))
return
}
var upload model.Upload
if err := db.DB(ctx).Where("id = ? AND status != ?", uploadID, model.UploadStatusDeleted).First(&upload).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.AbortWithStatus(http.StatusNotFound)
return
}
c.JSON(http.StatusOK, util.Err(ErrQueryUploadRecordFailed))
return
}
if upload.UserID != currUser.ID && !currUser.IsAdmin {
c.AbortWithStatus(http.StatusForbidden)
return
}
if err := db.DB(ctx).Model(&upload).Update("status", model.UploadStatusDeleted).Error; err != nil {
c.JSON(http.StatusOK, util.Err(ErrDeleteFileFailed))
return
}
c.JSON(http.StatusOK, util.OKNil())
}
// GetDistinctUploadTypes 获取数据库中所有已存在的文件业务类型
// @Summary 获取文件业务类型列表
// @Description 返回数据库中所有已上传文件实际拥有的业务类型列表
// @Tags admin
// @Produce json
// @Security SessionCookie
// @Success 200 {object} util.ResponseAny{data=[]string} "业务类型列表"
// @Failure 401 {object} util.ResponseAny "未登录"
// @Failure 403 {object} util.ResponseAny "无管理员权限"
// @Failure 500 {object} util.ResponseAny "内部错误"
// @Router /api/v1/admin/uploads/types [get]
func GetDistinctUploadTypes(c *gin.Context) {
var dbTypes []string
if err := db.DB(c.Request.Context()).Model(&model.Upload{}).
Where("type IS NOT NULL AND type != ''").
Distinct().
Pluck("type", &dbTypes).Error; err != nil {
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
return
}
sort.Strings(dbTypes)
c.JSON(http.StatusOK, util.OK(dbTypes))
}
+2 -31
View File
@@ -12,7 +12,6 @@ import (
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
@@ -22,7 +21,6 @@ import (
"github.com/Rain-kl/Wavelet/internal/diskcache"
"github.com/Rain-kl/Wavelet/internal/logger"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
"github.com/Rain-kl/Wavelet/internal/util"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
@@ -236,48 +234,21 @@ func normalizeImageQuality(quality string) string {
// serveOriginal 原始文件的流式响应逻辑
func serveOriginal(c *gin.Context, upload *model.Upload) {
if upload.StorageDriver == storageDriverLocal || (upload.StorageDriver == "" && !storage.IsEnabled()) {
c.File(upload.FilePath)
return
}
// Retrieve file from S3 (via CDN if configured)
obj, err := storage.GetObjectViaCache(c.Request.Context(), upload.FilePath)
obj, err := openStoredObject(c.Request.Context(), upload)
if err != nil {
c.AbortWithStatus(http.StatusNotFound)
return
}
// Cachefile
if obj.CachePath != "" {
c.File(obj.CachePath)
return
}
// Stream from CDN/S3
defer func() { _ = obj.Body.Close() }()
// Respond with the file content
c.DataFromReader(http.StatusOK, obj.ContentLength, obj.ContentType, obj.Body, nil)
}
// getOriginalFileBytes 获取原始文件所有字节
func getOriginalFileBytes(ctx context.Context, upload *model.Upload) ([]byte, error) {
if upload.StorageDriver == storageDriverLocal || (upload.StorageDriver == "" && !storage.IsEnabled()) {
return os.ReadFile(upload.FilePath)
}
// Retrieve file from S3 (via CDN if configured)
obj, err := storage.GetObjectViaCache(ctx, upload.FilePath)
obj, err := openStoredObject(ctx, upload)
if err != nil {
return nil, err
}
// Cachefile
if obj.CachePath != "" {
return os.ReadFile(obj.CachePath)
}
defer func() { _ = obj.Body.Close() }()
return io.ReadAll(obj.Body)
}
+37 -190
View File
@@ -17,16 +17,13 @@ import (
"mime/multipart"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/common"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/db/idgen"
"github.com/Rain-kl/Wavelet/internal/logger"
@@ -157,8 +154,8 @@ func UploadFile(c *gin.Context) {
id := idgen.NextUint64ID()
subPath := fmt.Sprintf("uploads/%s/%d.%s", time.Now().Format("2006/01/02"), id, ext)
// 8. 写入底层存储驱动 (优先 S3 驱动,无配置或未开启则 fallback 至本地文件)
storageDriver, subPath, errMsg := storeUploadFile(ctx, id, ext, subPath, size, mimeType, &buf, &meta)
// 8. 写入当前活动存储驱动。
storageDriver, subPath, errMsg := storeUploadFile(ctx, subPath, size, mimeType, &buf, &meta)
if errMsg != "" {
c.JSON(http.StatusOK, util.Err(errMsg))
return
@@ -323,21 +320,12 @@ func BatchDownloadFiles(c *gin.Context) {
// 打开底层文件数据源
var rc io.ReadCloser
if upload.StorageDriver == storageDriverLocal || (upload.StorageDriver == "" && !storage.IsEnabled()) {
fileSrc, err := os.Open(upload.FilePath)
if err != nil {
logger.ErrorF(ctx, "打包时读取本地文件失败: %v", err)
continue
}
rc = fileSrc
} else {
obj, err := storage.GetObject(ctx, upload.FilePath)
if err != nil {
logger.ErrorF(ctx, "打包时拉取 S3 文件失败: %v", err)
continue
}
rc = obj.Body
obj, err := openStoredObject(ctx, &upload)
if err != nil {
logger.ErrorF(ctx, "打包时读取文件失败: %v", err)
continue
}
rc = obj.Body
// 流式拷贝到 ZIP entry
_, err = io.Copy(zipFileEntry, rc)
@@ -348,131 +336,6 @@ func BatchDownloadFiles(c *gin.Context) {
}
}
type listMyFilesRequest struct {
Page int `form:"page"`
PageSize int `form:"page_size"`
Keyword string `form:"keyword"`
Type string `form:"type"`
Extension string `form:"extension"`
}
type listMyFilesResponse struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []model.Upload `json:"items"`
}
// ListMyFiles 获取当前用户上传的文件列表
// @Summary 获取我的文件列表
// @Description 分页获取当前登录用户上传的文件,支持文件名关键词、业务类型、扩展名过滤
// @Tags upload
// @Produce json
// @Param page query int false "页码(默认 1)"
// @Param page_size query int false "每页数量(默认 20,最大 100)"
// @Param keyword query string false "文件名关键词(模糊匹配)"
// @Param type query string false "业务分类过滤"
// @Param extension query string false "扩展名过滤"
// @Security SessionCookie
// @Success 200 {object} util.ResponseAny{data=listMyFilesResponse} "查询成功"
// @Failure 401 {object} util.ResponseAny "未登录"
// @Router /api/v1/upload/my [get]
func ListMyFiles(c *gin.Context) {
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
ctx := c.Request.Context()
var req listMyFilesRequest
if err := c.ShouldBindQuery(&req); err != nil {
c.JSON(http.StatusOK, util.Err(ErrInvalidParams))
return
}
if req.Page <= 0 {
req.Page = 1
}
if req.PageSize <= 0 || req.PageSize > 100 {
req.PageSize = 20
}
query := db.DB(ctx).Model(&model.Upload{}).
Where("user_id = ? AND status != ?", currUser.ID, model.UploadStatusDeleted)
if req.Keyword != "" {
query = query.Where("LOWER(file_name) LIKE ?", "%"+strings.ToLower(req.Keyword)+"%")
}
if req.Type != "" {
query = query.Where("type = ?", req.Type)
}
if req.Extension != "" {
query = query.Where("extension = ?", strings.ToLower(req.Extension))
}
var total int64
if err := query.Count(&total).Error; err != nil {
c.JSON(http.StatusOK, util.Err(ErrQueryFileCountFailed))
return
}
var items []model.Upload
offset := (req.Page - 1) * req.PageSize
if err := query.Order("created_at DESC").Offset(offset).Limit(req.PageSize).Find(&items).Error; err != nil {
c.JSON(http.StatusOK, util.Err(ErrQueryFileListFailed))
return
}
c.JSON(http.StatusOK, util.OK(listMyFilesResponse{
Total: total,
Page: req.Page,
PageSize: req.PageSize,
Items: items,
}))
}
// DeleteFile 软删除文件记录
// @Summary 删除文件
// @Description 将文件状态置为 deleted(软删除),不会立即清理底层存储对象
// @Tags upload
// @Produce json
// @Param id path string true "文件 ID"
// @Security SessionCookie
// @Success 200 {object} util.ResponseAny "删除成功"
// @Failure 403 {object} util.ResponseAny "无权操作"
// @Failure 404 {object} util.ResponseAny "文件不存在"
// @Router /api/v1/upload/{id} [delete]
func DeleteFile(c *gin.Context) {
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
ctx := c.Request.Context()
idStr := c.Param("id")
uploadID, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
c.JSON(http.StatusOK, util.Err(ErrInvalidFileID))
return
}
var upload model.Upload
if err := db.DB(ctx).Where("id = ? AND status != ?", uploadID, model.UploadStatusDeleted).First(&upload).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.AbortWithStatus(http.StatusNotFound)
return
}
c.JSON(http.StatusOK, util.Err(ErrQueryUploadRecordFailed))
return
}
// 仅允许文件所有者或管理员删除
if upload.UserID != currUser.ID && !currUser.IsAdmin {
c.AbortWithStatus(http.StatusForbidden)
return
}
if err := db.DB(ctx).Model(&upload).Update("status", model.UploadStatusDeleted).Error; err != nil {
c.JSON(http.StatusOK, util.Err(ErrDeleteFileFailed))
return
}
c.JSON(http.StatusOK, util.OKNil())
}
// validateUploadExtension 校验文件后缀是否在系统允许的上传扩展名列表中
func validateUploadExtension(ctx context.Context, ext string) string {
var sc model.SystemConfig
@@ -499,6 +362,10 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
if err != nil {
return false, err
}
if storageReadOnly(ctx) {
c.JSON(http.StatusConflict, util.Err(ErrStorageReadOnly))
return true, nil
}
id := idgen.NextUint64ID()
newUpload := model.Upload{
@@ -527,30 +394,32 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
return true, nil
}
// storeUploadFile 将文件写入底层存储驱动(S3 或本地磁盘)
func storeUploadFile(ctx context.Context, id uint64, ext, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, string, string) {
if storage.IsEnabled() {
meta.Bucket = config.Config.S3.Bucket
fullKey := storage.BuildKey(subPath)
if err := storage.PutObject(ctx, fullKey, bytes.NewReader(buf.Bytes()), size, mimeType); err != nil {
logger.ErrorF(ctx, "S3 存储上传失败: %v", err)
return "", "", ErrSaveFileFailed
}
return "s3", subPath, ""
// storeUploadFile 将文件写入当前活动存储驱动。
func storeUploadFile(ctx context.Context, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, string, string) {
if storageReadOnly(ctx) {
return "", "", ErrStorageReadOnly
}
localDir := filepath.Join("uploads", time.Now().Format("2006/01/02"))
if err := os.MkdirAll(localDir, uploadDirPerm); err != nil {
logger.ErrorF(ctx, "创建本地上传目录失败: %v", err)
driver, backend, err := storage.Active(ctx)
if err != nil {
logger.ErrorF(ctx, "初始化活动存储失败: %v", err)
return "", "", ErrSaveFileFailed
}
localPath := filepath.Join(localDir, fmt.Sprintf("%d.%s", id, ext))
if err := os.WriteFile(localPath, buf.Bytes(), uploadFilePerm); err != nil {
logger.ErrorF(ctx, "本地磁盘写入文件失败: %v", err)
storedPath, err := backend.Put(ctx, subPath, bytes.NewReader(buf.Bytes()), size, mimeType)
if err != nil {
logger.ErrorF(ctx, "写入 %s 存储失败: %v", driver, err)
return "", "", ErrSaveFileFailed
}
return storageDriverLocal, localPath, ""
switch driver {
case storage.DriverS3:
meta.Bucket = cfgBucket(ctx, driver)
case storage.DriverR2:
meta.Bucket = cfgBucket(ctx, driver)
case storage.DriverMinIO:
meta.Bucket = cfgBucket(ctx, driver)
case storage.DriverOSS:
meta.Bucket = cfgBucket(ctx, driver)
}
return string(driver), storedPath, ""
}
// isImageExtension 判断文件扩展名是否属于常见图片格式
@@ -590,35 +459,13 @@ func detectMimeType(buf *bytes.Buffer, header *multipart.FileHeader, size int64)
// saveUploadRecord 保存上传记录到数据库,失败时清理本地垃圾文件
func saveUploadRecord(ctx context.Context, upload *model.Upload, storageDriver, filePath string) string {
if err := db.DB(ctx).Create(upload).Error; err != nil {
if storageDriver == storageDriverLocal {
_ = os.Remove(filePath)
backend, backendErr := storage.ForDriver(ctx, storage.Driver(storageDriver))
if backendErr == nil {
if deleteErr := backend.Delete(ctx, filePath); deleteErr != nil {
logger.WarnF(ctx, "清理未写入数据库的上传对象失败: %v", deleteErr)
}
}
return ErrSaveUploadRecordFailed
}
return ""
}
// GetDistinctUploadTypes 获取数据库中所有已存在的文件业务类型
// @Summary 获取文件业务类型列表
// @Description 返回数据库中所有已上传文件实际拥有的业务类型列表
// @Tags admin
// @Produce json
// @Security SessionCookie
// @Success 200 {object} util.ResponseAny{data=[]string} "业务类型列表"
// @Failure 401 {object} util.ResponseAny "未登录"
// @Failure 403 {object} util.ResponseAny "无管理员权限"
// @Failure 500 {object} util.ResponseAny "内部错误"
// @Router /api/v1/admin/uploads/types [get]
func GetDistinctUploadTypes(c *gin.Context) {
var dbTypes []string
if err := db.DB(c.Request.Context()).Model(&model.Upload{}).
Where("type IS NOT NULL AND type != ''").
Distinct().
Pluck("type", &dbTypes).Error; err != nil {
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
return
}
sort.Strings(dbTypes)
c.JSON(http.StatusOK, util.OK(dbTypes))
}
+2 -3
View File
@@ -105,12 +105,12 @@ func TestUploadFile(t *testing.T) {
putCount++
return nil
},
func(ctx context.Context, key string) (*storage.ObjectInfo, error) {
func(ctx context.Context, key string) (*storage.Object, error) {
data, ok := mockFiles[key]
if !ok {
return nil, os.ErrNotExist
}
return &storage.ObjectInfo{
return &storage.Object{
Body: io.NopCloser(bytes.NewReader(data)),
ContentLength: int64(len(data)),
ContentType: "application/octet-stream",
@@ -821,4 +821,3 @@ func TestGetFileStats(t *testing.T) {
t.Errorf("expected 1 document category, got %d", categoryMap["文档"])
}
}
@@ -0,0 +1,268 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
"github.com/Rain-kl/Wavelet/internal/task"
"gorm.io/gorm"
)
const (
// StorageMigrationTask is the Asynq task name for storage migration.
StorageMigrationTask = "storage:migrate"
// TaskTypeStorageMigration is the task metadata type for storage migration.
TaskTypeStorageMigration = "storage_migration"
)
// StorageMigrationMeta describes the manually dispatchable migration task.
var StorageMigrationMeta = task.TaskMeta{
Type: TaskTypeStorageMigration,
AsynqTask: StorageMigrationTask,
Name: "迁移文件存储",
Description: "将活动存储中的文件迁移到待切换的目标存储,迁移期间文件系统保持只读",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
}
// MigrationHandler copies stored objects and activates the target backend.
type MigrationHandler struct{}
type storageMigrationPayload struct {
Target storage.Config `json:"target"`
}
// ValidatePayload rejects duplicate active migrations through the task framework.
func (h *MigrationHandler) ValidatePayload(payload []byte) ([]byte, error) {
normalized, _, err := normalizeStorageMigrationPayload(context.Background(), payload)
if err != nil {
return payload, err
}
active, err := hasUnresolvedMigrationTask(context.Background())
if err != nil {
return payload, err
}
if active {
return payload, fmt.Errorf("storage migration task is already unresolved")
}
return normalized, nil
}
// Execute migrates all unique active-storage objects to the pending backend.
func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
active, err := storage.LoadConfig(ctx)
if err != nil {
return nil, fmt.Errorf("load active storage config: %w", err)
}
target, err := parseMigrationTargetConfig(ctx, payload)
if err != nil {
return nil, err
}
if target.Driver == active.Driver {
if err := storage.SaveActiveConfig(ctx, target); err != nil {
return nil, fmt.Errorf("activate same-driver storage config: %w", err)
}
message := fmt.Sprintf("存储配置已更新,活动存储保持为 %s", target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
}
total, err := countStorageObjects(ctx, active.Driver)
if err != nil {
return nil, fmt.Errorf("count source objects: %w", err)
}
if total == 0 {
if err := storage.SaveActiveConfig(ctx, target); err != nil {
return nil, fmt.Errorf("activate empty storage config: %w", err)
}
message := fmt.Sprintf("当前存储没有需要迁移的对象,活动存储已切换为 %s", target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
}
sourceBackend, err := storage.NewBackend(ctx, active, active.Driver)
if err != nil {
return nil, fmt.Errorf("create source storage: %w", err)
}
targetBackend, err := storage.NewBackend(ctx, target, target.Driver)
if err != nil {
return nil, fmt.Errorf("create target storage: %w", err)
}
task.AppendLog(ctx, "开始存储迁移: %s -> %s,总对象数: %d", active.Driver, target.Driver, total)
migrated, err := migrateObjects(ctx, sourceBackend, targetBackend, active.Driver, target.Driver, total)
if err != nil {
return nil, err
}
if err := storage.SaveActiveConfig(ctx, target); err != nil {
return nil, fmt.Errorf("activate target storage: %w", err)
}
message := fmt.Sprintf("存储迁移完成,共迁移 %d 个对象,活动存储已切换为 %s", migrated, target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
}
func normalizeStorageMigrationPayload(ctx context.Context, payload []byte) ([]byte, storage.Config, error) {
target, err := parseMigrationTargetConfig(ctx, payload)
if err != nil {
return nil, storage.Config{}, err
}
normalized, err := json.Marshal(storageMigrationPayload{Target: target})
if err != nil {
return nil, storage.Config{}, fmt.Errorf("marshal storage migration payload: %w", err)
}
return normalized, target, nil
}
func parseMigrationTargetConfig(ctx context.Context, payload []byte) (storage.Config, error) {
if strings.TrimSpace(string(payload)) == "" {
return storage.Config{}, errors.New("storage migration target payload is required")
}
var req storageMigrationPayload
if err := json.Unmarshal(payload, &req); err != nil {
return storage.Config{}, fmt.Errorf("parse storage migration payload: %w", err)
}
current, err := storage.LoadConfig(ctx)
if err != nil {
return storage.Config{}, fmt.Errorf("load active storage config: %w", err)
}
target := storage.MergeMaskedSecrets(req.Target, current)
if err := storage.ValidateConfig(target); err != nil {
return storage.Config{}, fmt.Errorf("validate target storage config: %w", err)
}
return target, nil
}
func countStorageObjects(ctx context.Context, driver storage.Driver) (int64, error) {
var count int64
err := db.DB(ctx).Model(&model.Upload{}).
Where("storage_driver = ? AND status != ?", driver, model.UploadStatusDeleted).
Distinct("file_path").
Count(&count).Error
return count, err
}
func hasUnresolvedMigrationTask(ctx context.Context) (bool, error) {
execution, ok, err := latestStorageMigrationExecution(ctx)
if err != nil || !ok {
return false, err
}
return execution.Status == model.TaskExecutionStatusPending || execution.Status == model.TaskExecutionStatusRunning, nil
}
func latestStorageMigrationExecution(ctx context.Context) (*model.TaskExecution, bool, error) {
var execution model.TaskExecution
err := db.DB(ctx).
Where("task_type = ?", StorageMigrationTask).
Order("id DESC").
First(&execution).Error
if err == nil {
return &execution, true, nil
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, false, err
}
return nil, false, nil
}
func migrateObjects(
ctx context.Context,
sourceBackend storage.Backend,
targetBackend storage.Backend,
sourceDriver storage.Driver,
targetDriver storage.Driver,
total int64,
) (int64, error) {
const batchSize = 50
var migrated int64
for {
if err := ctx.Err(); err != nil {
return migrated, fmt.Errorf("storage migration canceled: %w", err)
}
var objects []struct {
FilePath string `gorm:"column:file_path"`
FileSize int64 `gorm:"column:file_size"`
MimeType string `gorm:"column:mime_type"`
}
if err := db.DB(ctx).Model(&model.Upload{}).
Select("file_path, MAX(file_size) AS file_size, MAX(mime_type) AS mime_type").
Where("storage_driver = ? AND status != ?", sourceDriver, model.UploadStatusDeleted).
Group("file_path").
Order("file_path ASC").
Limit(batchSize).
Scan(&objects).Error; err != nil {
return migrated, fmt.Errorf("query source objects: %w", err)
}
if len(objects) == 0 {
break
}
for _, object := range objects {
source, err := sourceBackend.Get(ctx, object.FilePath)
if err != nil {
if isNotFoundError(err) {
task.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", object.FilePath, err)
if updateErr := db.DB(ctx).Model(&model.Upload{}).
Where("storage_driver = ? AND file_path = ?", sourceDriver, object.FilePath).
Updates(map[string]any{
"status": model.UploadStatusDeleted,
"storage_driver": targetDriver,
}).Error; updateErr != nil {
return migrated, fmt.Errorf("update missing object %q: %w", object.FilePath, updateErr)
}
continue
}
return migrated, fmt.Errorf("open source object %q: %w", object.FilePath, err)
}
targetPath, putErr := targetBackend.Put(ctx, object.FilePath, source.Body, object.FileSize, object.MimeType)
closeErr := source.Body.Close()
if putErr != nil {
return migrated, fmt.Errorf("copy object %q: %w", object.FilePath, putErr)
}
if closeErr != nil {
return migrated, fmt.Errorf("close source object %q: %w", object.FilePath, closeErr)
}
if err := db.DB(ctx).Model(&model.Upload{}).
Where("storage_driver = ? AND file_path = ?", sourceDriver, object.FilePath).
Updates(map[string]any{
"storage_driver": targetDriver,
"file_path": targetPath,
}).Error; err != nil {
return migrated, fmt.Errorf("update migrated object %q: %w", object.FilePath, err)
}
migrated++
}
task.AppendLog(ctx, "迁移进度: %d/%d", migrated, total)
}
return migrated, nil
}
func isNotFoundError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, os.ErrNotExist) {
return true
}
errStr := strings.ToLower(err.Error())
for _, sub := range []string{"not found", "nosuchkey", "nosuchbucket", "404", "does not exist"} {
if strings.Contains(errStr, sub) {
return true
}
}
return false
}
@@ -0,0 +1,110 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"bytes"
"context"
"encoding/json"
"io"
"os"
"path/filepath"
"testing"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
"github.com/Rain-kl/Wavelet/internal/testhelper"
)
func TestMigrationHandlerExecute(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
sourceRoot := t.TempDir()
sourcePath := filepath.Join(sourceRoot, "uploads", "test.txt")
if err := os.MkdirAll(filepath.Dir(sourcePath), 0755); err != nil {
t.Fatalf("MkdirAll(%q) returned error: %v", sourcePath, err)
}
const content = "storage migration"
if err := os.WriteFile(sourcePath, []byte(content), 0644); err != nil {
t.Fatalf("WriteFile(%q) returned error: %v", sourcePath, err)
}
ctx := context.Background()
active := storage.DefaultConfig()
active.Local.Root = sourceRoot
if err := storage.SaveActiveConfig(ctx, active); err != nil {
t.Fatalf("SaveActiveConfig() returned error: %v", err)
}
target := storage.DefaultConfig()
target.Driver = storage.DriverS3
target.S3 = storage.ObjectConfig{
Region: "us-east-1",
Bucket: "target",
AccessKeyID: "key",
SecretAccessKey: "secret",
}
payload, err := json.Marshal(storageMigrationPayload{Target: target})
if err != nil {
t.Fatalf("Marshal(storageMigrationPayload) returned error: %v", err)
}
upload := model.Upload{
ID: 99101,
UserID: 1,
FileName: "test.txt",
FilePath: "uploads/test.txt",
FileSize: int64(len(content)),
MimeType: "text/plain",
Extension: "txt",
Hash: "hash",
StorageDriver: string(storage.DriverLocal),
Type: "attachment",
Status: model.UploadStatusUsed,
}
if err := dbConn.Create(&upload).Error; err != nil {
t.Fatalf("Create(upload) returned error: %v", err)
}
var copied bytes.Buffer
restore := storage.MockStorage(
func(_ context.Context, _ string, body io.Reader, _ int64, _ string) error {
_, err := io.Copy(&copied, body)
return err
},
func(context.Context, string) (*storage.Object, error) {
return nil, nil
},
func(context.Context, string) error {
return nil
},
)
defer restore()
result, err := (&MigrationHandler{}).Execute(ctx, payload)
if err != nil {
t.Fatalf("Execute() returned error: %v", err)
}
if result == nil {
t.Fatal("Execute() result = nil, want non-nil")
}
if copied.String() != content {
t.Errorf("migrated content = %q, want %q", copied.String(), content)
}
var migrated model.Upload
if err := dbConn.First(&migrated, upload.ID).Error; err != nil {
t.Fatalf("First(upload) returned error: %v", err)
}
if migrated.StorageDriver != string(storage.DriverS3) {
t.Errorf("StorageDriver = %q, want %q", migrated.StorageDriver, storage.DriverS3)
}
current, err := storage.LoadConfig(ctx)
if err != nil {
t.Fatalf("LoadConfig() returned error: %v", err)
}
if current.Driver != storage.DriverS3 {
t.Errorf("active driver = %q, want %q", current.Driver, storage.DriverS3)
}
}
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"context"
"fmt"
"github.com/Rain-kl/Wavelet/internal/logger"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/storage"
)
func storageReadOnly(ctx context.Context) bool {
execution, ok, err := latestStorageMigrationExecution(ctx)
if err != nil {
logger.ErrorF(ctx, "读取存储维护状态失败: %v", err)
return true
}
if !ok {
return false
}
return execution.Status != model.TaskExecutionStatusSucceeded
}
func openStoredObject(ctx context.Context, upload *model.Upload) (*storage.Object, error) {
driver := storage.Driver(upload.StorageDriver)
if driver == "" {
driver = storage.DriverLocal
}
backend, err := backendForStoredDriver(ctx, driver)
if err != nil {
return nil, err
}
return backend.Get(ctx, upload.FilePath)
}
func backendForStoredDriver(ctx context.Context, driver storage.Driver) (storage.Backend, error) {
backend, err := storage.ForDriver(ctx, driver)
if err == nil {
return backend, nil
}
target, ok, targetErr := currentMigrationTargetConfig(ctx)
if targetErr != nil {
return nil, targetErr
}
if ok && target.Driver == driver {
return storage.NewBackend(ctx, target, driver)
}
return nil, fmt.Errorf("storage configuration for driver %q is unavailable", driver)
}
func currentMigrationTargetConfig(ctx context.Context) (storage.Config, bool, error) {
execution, ok, err := latestStorageMigrationExecution(ctx)
if err != nil || !ok {
return storage.Config{}, false, err
}
if execution.Status == model.TaskExecutionStatusSucceeded {
return storage.Config{}, false, nil
}
target, err := parseMigrationTargetConfig(ctx, []byte(execution.Payload))
if err != nil {
return storage.Config{}, false, err
}
return target, true, nil
}
func cfgBucket(ctx context.Context, driver storage.Driver) string {
cfg, err := storage.LoadConfig(ctx)
if err != nil {
return ""
}
switch driver {
case storage.DriverS3:
return cfg.S3.Bucket
case storage.DriverR2:
return cfg.R2.Bucket
case storage.DriverMinIO:
return cfg.MinIO.Bucket
case storage.DriverOSS:
return cfg.OSS.Bucket
default:
return ""
}
}
+12 -2
View File
@@ -81,6 +81,9 @@ type WarmImageCacheHandler struct{}
// Execute 执行清理未使用上传文件的业务逻辑
func (h *CleanupUnusedUploadsHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
if storageReadOnly(ctx) {
return nil, errors.New(ErrStorageReadOnly)
}
const batchSize = 100 // 每批处理100个文件
var lastID uint64
var totalProcessed int
@@ -122,8 +125,15 @@ func (h *CleanupUnusedUploadsHandler) Execute(ctx context.Context, _ []byte) (*t
return err
}
// Delete from S3
if err := storage.DeleteObject(ctx, upload.FilePath); err != nil {
driver := storage.Driver(upload.StorageDriver)
if driver == "" {
driver = storage.DriverLocal
}
backend, err := storage.ForDriver(ctx, driver)
if err != nil {
return err
}
if err := backend.Delete(ctx, upload.FilePath); err != nil {
return err
}
+9 -2
View File
@@ -36,7 +36,7 @@ func TestCleanupUnusedUploadsHandler_Execute(t *testing.T) {
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
return nil
},
func(ctx context.Context, key string) (*storage.ObjectInfo, error) { return nil, nil },
func(ctx context.Context, key string) (*storage.Object, error) { return nil, nil },
func(ctx context.Context, key string) error { return nil },
)
defer storageMock()
@@ -113,7 +113,7 @@ func TestCleanupUnusedUploadsHandler_ExecuteNoFiles(t *testing.T) {
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
return nil
},
func(ctx context.Context, key string) (*storage.ObjectInfo, error) { return nil, nil },
func(ctx context.Context, key string) (*storage.Object, error) { return nil, nil },
func(ctx context.Context, key string) error { return nil },
)
defer storageMock()
@@ -204,6 +204,13 @@ func TestWarmImageCacheHandlerExecute(t *testing.T) {
})
testDir := t.TempDir()
ctx := context.Background()
active := storage.DefaultConfig()
active.Local.Root = testDir
if err := storage.SaveActiveConfig(ctx, active); err != nil {
t.Fatalf("SaveActiveConfig() returned error: %v", err)
}
firstPath := filepath.Join(testDir, "first.png")
secondPath := filepath.Join(testDir, "second.jpg")
writeTaskTestPNG(t, firstPath, color.RGBA{R: 255, A: 255})