mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-06 07:36:37 +08:00
feat(storage): add dynamic storage config and migration
Move storage backend configuration from startup YAML to system_config-backed runtime configuration. Add local, S3-compatible, R2, MinIO, OSS, and WebDAV backend support. Add a storage migration async task using the existing task dispatch framework. Migration target config is carried in task payload, and maintenance mode is derived from task execution state. Split upload file management and storage operations, add the admin storage configuration tab, and update migrations and Swagger docs.
This commit is contained in:
@@ -5,11 +5,15 @@
|
||||
package system_config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/logger"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
"github.com/Rain-kl/Wavelet/internal/util"
|
||||
mail "github.com/Rain-kl/Wavelet/internal/util/mail"
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -21,7 +25,7 @@ const maskedConfigValue = "******"
|
||||
// CreateSystemConfigRequest 创建系统配置请求
|
||||
type CreateSystemConfigRequest struct {
|
||||
Key string `json:"key" binding:"required,max=64"`
|
||||
Value string `json:"value" binding:"required,max=255"`
|
||||
Value string `json:"value" binding:"required"`
|
||||
Type string `json:"type" binding:"required,oneof=system business"`
|
||||
Visibility int `json:"visibility" binding:"oneof=0 1"`
|
||||
Description string `json:"description" binding:"max=255"`
|
||||
@@ -29,7 +33,7 @@ type CreateSystemConfigRequest struct {
|
||||
|
||||
// UpdateSystemConfigRequest 更新系统配置请求
|
||||
type UpdateSystemConfigRequest struct {
|
||||
Value string `json:"value" binding:"required,max=255"`
|
||||
Value string `json:"value" binding:"required"`
|
||||
Visibility *int `json:"visibility" binding:"omitempty,oneof=0 1"`
|
||||
Description string `json:"description" binding:"max=255"`
|
||||
}
|
||||
@@ -118,9 +122,7 @@ func ListSystemConfigs(c *gin.Context) {
|
||||
}
|
||||
|
||||
for i := range configs {
|
||||
if configs[i].Key == model.ConfigKeySMTPPassword && configs[i].Value != "" {
|
||||
configs[i].Value = maskedConfigValue
|
||||
}
|
||||
configs[i].Value = maskSensitiveConfig(configs[i].Key, configs[i].Value)
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, util.OK(configs))
|
||||
@@ -150,9 +152,7 @@ func GetSystemConfig(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if config.Key == model.ConfigKeySMTPPassword && config.Value != "" {
|
||||
config.Value = maskedConfigValue
|
||||
}
|
||||
config.Value = maskSensitiveConfig(config.Key, config.Value)
|
||||
|
||||
c.JSON(http.StatusOK, util.OK(config))
|
||||
}
|
||||
@@ -193,6 +193,14 @@ func UpdateSystemConfig(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
var originalDriver storage.Driver
|
||||
if key == model.ConfigKeyStorageConfig {
|
||||
var currentCfg storage.Config
|
||||
if err := json.Unmarshal([]byte(config.Value), ¤tCfg); err == nil {
|
||||
originalDriver = currentCfg.Driver
|
||||
}
|
||||
}
|
||||
|
||||
if err := db.DB(c.Request.Context()).Transaction(func(tx *gorm.DB) error {
|
||||
// 更新配置
|
||||
updates := map[string]interface{}{
|
||||
@@ -214,6 +222,24 @@ func UpdateSystemConfig(c *gin.Context) {
|
||||
return err
|
||||
}
|
||||
|
||||
if key == model.ConfigKeyStorageConfig && originalDriver != "" {
|
||||
var newCfg storage.Config
|
||||
if err := json.Unmarshal([]byte(req.Value), &newCfg); err == nil {
|
||||
if newCfg.Driver == originalDriver {
|
||||
// Mark failed storage:migrate task execution as succeeded
|
||||
if err := tx.Model(&model.TaskExecution{}).
|
||||
Where("task_type = ? AND status = ?", "storage:migrate", model.TaskExecutionStatusFailed).
|
||||
Updates(map[string]any{
|
||||
"status": model.TaskExecutionStatusSucceeded,
|
||||
"result": "存储配置直接更新,故障迁移任务自动标记为已解决",
|
||||
"finished_at": time.Now(),
|
||||
}).Error; err != nil {
|
||||
logger.ErrorF(c.Request.Context(), "自动更新迁移任务状态失败: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
@@ -288,3 +314,22 @@ func TestSMTP(c *gin.Context) {
|
||||
|
||||
c.JSON(http.StatusOK, util.OK(resp))
|
||||
}
|
||||
|
||||
func maskSensitiveConfig(key, value string) string {
|
||||
if value == "" {
|
||||
return value
|
||||
}
|
||||
switch key {
|
||||
case model.ConfigKeySMTPPassword:
|
||||
return maskedConfigValue
|
||||
case model.ConfigKeyStorageConfig:
|
||||
var cfg storage.Config
|
||||
if err := json.Unmarshal([]byte(value), &cfg); err == nil {
|
||||
masked := storage.MaskSecrets(cfg)
|
||||
if val, err := json.Marshal(masked); err == nil {
|
||||
return string(val)
|
||||
}
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const expectedDefaultConfigsCount = 29
|
||||
const expectedDefaultConfigsCount = 30
|
||||
|
||||
func setupTestRouter(authUser *model.User) *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
package upload
|
||||
|
||||
import "github.com/Rain-kl/Wavelet/internal/storage"
|
||||
|
||||
const (
|
||||
maxUploadSize = 32 * 1024 * 1024 // 32MB
|
||||
detectContentBytes = 512 // http.DetectContentType 需要的最小字节数
|
||||
@@ -12,5 +14,5 @@ const (
|
||||
imageQualityMedium = "medium"
|
||||
imageQualityHigh = "high"
|
||||
imageQualityOrigin = "origin"
|
||||
storageDriverLocal = "local"
|
||||
storageDriverLocal = string(storage.DriverLocal)
|
||||
)
|
||||
|
||||
@@ -27,6 +27,7 @@ const (
|
||||
ErrQueryFileCountFailed = "查询文件数量失败"
|
||||
ErrQueryFileListFailed = "查询文件列表失败"
|
||||
ErrDeleteFileFailed = "删除文件失败"
|
||||
ErrStorageReadOnly = "存储迁移维护中,当前仅允许读取文件"
|
||||
ErrS3KeyRequired = "s3 key must not be empty"
|
||||
ErrS3KeyTooLongFormat = "s3 key exceeds maximum length of %d"
|
||||
ErrS3KeyStartsWithSlash = "s3 key must not start with /"
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/util"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type listMyFilesRequest struct {
|
||||
Page int `form:"page"`
|
||||
PageSize int `form:"page_size"`
|
||||
Keyword string `form:"keyword"`
|
||||
Type string `form:"type"`
|
||||
Extension string `form:"extension"`
|
||||
}
|
||||
|
||||
type listMyFilesResponse struct {
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []model.Upload `json:"items"`
|
||||
}
|
||||
|
||||
// ListMyFiles 获取当前用户上传的文件列表
|
||||
// @Summary 获取我的文件列表
|
||||
// @Description 分页获取当前登录用户上传的文件,支持文件名关键词、业务类型、扩展名过滤
|
||||
// @Tags upload
|
||||
// @Produce json
|
||||
// @Param page query int false "页码(默认 1)"
|
||||
// @Param page_size query int false "每页数量(默认 20,最大 100)"
|
||||
// @Param keyword query string false "文件名关键词(模糊匹配)"
|
||||
// @Param type query string false "业务分类过滤"
|
||||
// @Param extension query string false "扩展名过滤"
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny{data=listMyFilesResponse} "查询成功"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Router /api/v1/upload/my [get]
|
||||
func ListMyFiles(c *gin.Context) {
|
||||
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var req listMyFilesRequest
|
||||
if err := c.ShouldBindQuery(&req); err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrInvalidParams))
|
||||
return
|
||||
}
|
||||
if req.Page <= 0 {
|
||||
req.Page = 1
|
||||
}
|
||||
if req.PageSize <= 0 || req.PageSize > 100 {
|
||||
req.PageSize = 20
|
||||
}
|
||||
|
||||
query := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("user_id = ? AND status != ?", currUser.ID, model.UploadStatusDeleted)
|
||||
|
||||
if req.Keyword != "" {
|
||||
query = query.Where("LOWER(file_name) LIKE ?", "%"+strings.ToLower(req.Keyword)+"%")
|
||||
}
|
||||
if req.Type != "" {
|
||||
query = query.Where("type = ?", req.Type)
|
||||
}
|
||||
if req.Extension != "" {
|
||||
query = query.Where("extension = ?", strings.ToLower(req.Extension))
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryFileCountFailed))
|
||||
return
|
||||
}
|
||||
|
||||
var items []model.Upload
|
||||
offset := (req.Page - 1) * req.PageSize
|
||||
if err := query.Order("created_at DESC").Offset(offset).Limit(req.PageSize).Find(&items).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryFileListFailed))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, util.OK(listMyFilesResponse{
|
||||
Total: total,
|
||||
Page: req.Page,
|
||||
PageSize: req.PageSize,
|
||||
Items: items,
|
||||
}))
|
||||
}
|
||||
|
||||
// DeleteFile 软删除文件记录
|
||||
// @Summary 删除文件
|
||||
// @Description 将文件状态置为 deleted(软删除),不会立即清理底层存储对象
|
||||
// @Tags upload
|
||||
// @Produce json
|
||||
// @Param id path string true "文件 ID"
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny "删除成功"
|
||||
// @Failure 403 {object} util.ResponseAny "无权操作"
|
||||
// @Failure 404 {object} util.ResponseAny "文件不存在"
|
||||
// @Router /api/v1/upload/{id} [delete]
|
||||
func DeleteFile(c *gin.Context) {
|
||||
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
|
||||
ctx := c.Request.Context()
|
||||
if storageReadOnly(ctx) {
|
||||
c.JSON(http.StatusConflict, util.Err(ErrStorageReadOnly))
|
||||
return
|
||||
}
|
||||
|
||||
uploadID, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrInvalidFileID))
|
||||
return
|
||||
}
|
||||
|
||||
var upload model.Upload
|
||||
if err := db.DB(ctx).Where("id = ? AND status != ?", uploadID, model.UploadStatusDeleted).First(&upload).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryUploadRecordFailed))
|
||||
return
|
||||
}
|
||||
if upload.UserID != currUser.ID && !currUser.IsAdmin {
|
||||
c.AbortWithStatus(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := db.DB(ctx).Model(&upload).Update("status", model.UploadStatusDeleted).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrDeleteFileFailed))
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, util.OKNil())
|
||||
}
|
||||
|
||||
// GetDistinctUploadTypes 获取数据库中所有已存在的文件业务类型
|
||||
// @Summary 获取文件业务类型列表
|
||||
// @Description 返回数据库中所有已上传文件实际拥有的业务类型列表
|
||||
// @Tags admin
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny{data=[]string} "业务类型列表"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Failure 403 {object} util.ResponseAny "无管理员权限"
|
||||
// @Failure 500 {object} util.ResponseAny "内部错误"
|
||||
// @Router /api/v1/admin/uploads/types [get]
|
||||
func GetDistinctUploadTypes(c *gin.Context) {
|
||||
var dbTypes []string
|
||||
if err := db.DB(c.Request.Context()).Model(&model.Upload{}).
|
||||
Where("type IS NOT NULL AND type != ''").
|
||||
Distinct().
|
||||
Pluck("type", &dbTypes).Error; err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
sort.Strings(dbTypes)
|
||||
c.JSON(http.StatusOK, util.OK(dbTypes))
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -22,7 +21,6 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/diskcache"
|
||||
"github.com/Rain-kl/Wavelet/internal/logger"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
"github.com/Rain-kl/Wavelet/internal/util"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
@@ -236,48 +234,21 @@ func normalizeImageQuality(quality string) string {
|
||||
|
||||
// serveOriginal 原始文件的流式响应逻辑
|
||||
func serveOriginal(c *gin.Context, upload *model.Upload) {
|
||||
if upload.StorageDriver == storageDriverLocal || (upload.StorageDriver == "" && !storage.IsEnabled()) {
|
||||
c.File(upload.FilePath)
|
||||
return
|
||||
}
|
||||
|
||||
// Retrieve file from S3 (via CDN if configured)
|
||||
obj, err := storage.GetObjectViaCache(c.Request.Context(), upload.FilePath)
|
||||
obj, err := openStoredObject(c.Request.Context(), upload)
|
||||
if err != nil {
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Cachefile
|
||||
if obj.CachePath != "" {
|
||||
c.File(obj.CachePath)
|
||||
return
|
||||
}
|
||||
|
||||
// Stream from CDN/S3
|
||||
defer func() { _ = obj.Body.Close() }()
|
||||
|
||||
// Respond with the file content
|
||||
c.DataFromReader(http.StatusOK, obj.ContentLength, obj.ContentType, obj.Body, nil)
|
||||
}
|
||||
|
||||
// getOriginalFileBytes 获取原始文件所有字节
|
||||
func getOriginalFileBytes(ctx context.Context, upload *model.Upload) ([]byte, error) {
|
||||
if upload.StorageDriver == storageDriverLocal || (upload.StorageDriver == "" && !storage.IsEnabled()) {
|
||||
return os.ReadFile(upload.FilePath)
|
||||
}
|
||||
|
||||
// Retrieve file from S3 (via CDN if configured)
|
||||
obj, err := storage.GetObjectViaCache(ctx, upload.FilePath)
|
||||
obj, err := openStoredObject(ctx, upload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Cachefile
|
||||
if obj.CachePath != "" {
|
||||
return os.ReadFile(obj.CachePath)
|
||||
}
|
||||
|
||||
defer func() { _ = obj.Body.Close() }()
|
||||
return io.ReadAll(obj.Body)
|
||||
}
|
||||
|
||||
+37
-190
@@ -17,16 +17,13 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
|
||||
"github.com/Rain-kl/Wavelet/internal/common"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/db/idgen"
|
||||
"github.com/Rain-kl/Wavelet/internal/logger"
|
||||
@@ -157,8 +154,8 @@ func UploadFile(c *gin.Context) {
|
||||
id := idgen.NextUint64ID()
|
||||
subPath := fmt.Sprintf("uploads/%s/%d.%s", time.Now().Format("2006/01/02"), id, ext)
|
||||
|
||||
// 8. 写入底层存储驱动 (优先 S3 驱动,无配置或未开启则 fallback 至本地文件)
|
||||
storageDriver, subPath, errMsg := storeUploadFile(ctx, id, ext, subPath, size, mimeType, &buf, &meta)
|
||||
// 8. 写入当前活动存储驱动。
|
||||
storageDriver, subPath, errMsg := storeUploadFile(ctx, subPath, size, mimeType, &buf, &meta)
|
||||
if errMsg != "" {
|
||||
c.JSON(http.StatusOK, util.Err(errMsg))
|
||||
return
|
||||
@@ -323,21 +320,12 @@ func BatchDownloadFiles(c *gin.Context) {
|
||||
|
||||
// 打开底层文件数据源
|
||||
var rc io.ReadCloser
|
||||
if upload.StorageDriver == storageDriverLocal || (upload.StorageDriver == "" && !storage.IsEnabled()) {
|
||||
fileSrc, err := os.Open(upload.FilePath)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "打包时读取本地文件失败: %v", err)
|
||||
continue
|
||||
}
|
||||
rc = fileSrc
|
||||
} else {
|
||||
obj, err := storage.GetObject(ctx, upload.FilePath)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "打包时拉取 S3 文件失败: %v", err)
|
||||
continue
|
||||
}
|
||||
rc = obj.Body
|
||||
obj, err := openStoredObject(ctx, &upload)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "打包时读取文件失败: %v", err)
|
||||
continue
|
||||
}
|
||||
rc = obj.Body
|
||||
|
||||
// 流式拷贝到 ZIP entry
|
||||
_, err = io.Copy(zipFileEntry, rc)
|
||||
@@ -348,131 +336,6 @@ func BatchDownloadFiles(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
type listMyFilesRequest struct {
|
||||
Page int `form:"page"`
|
||||
PageSize int `form:"page_size"`
|
||||
Keyword string `form:"keyword"`
|
||||
Type string `form:"type"`
|
||||
Extension string `form:"extension"`
|
||||
}
|
||||
|
||||
type listMyFilesResponse struct {
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []model.Upload `json:"items"`
|
||||
}
|
||||
|
||||
// ListMyFiles 获取当前用户上传的文件列表
|
||||
// @Summary 获取我的文件列表
|
||||
// @Description 分页获取当前登录用户上传的文件,支持文件名关键词、业务类型、扩展名过滤
|
||||
// @Tags upload
|
||||
// @Produce json
|
||||
// @Param page query int false "页码(默认 1)"
|
||||
// @Param page_size query int false "每页数量(默认 20,最大 100)"
|
||||
// @Param keyword query string false "文件名关键词(模糊匹配)"
|
||||
// @Param type query string false "业务分类过滤"
|
||||
// @Param extension query string false "扩展名过滤"
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny{data=listMyFilesResponse} "查询成功"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Router /api/v1/upload/my [get]
|
||||
func ListMyFiles(c *gin.Context) {
|
||||
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var req listMyFilesRequest
|
||||
if err := c.ShouldBindQuery(&req); err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrInvalidParams))
|
||||
return
|
||||
}
|
||||
if req.Page <= 0 {
|
||||
req.Page = 1
|
||||
}
|
||||
if req.PageSize <= 0 || req.PageSize > 100 {
|
||||
req.PageSize = 20
|
||||
}
|
||||
|
||||
query := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("user_id = ? AND status != ?", currUser.ID, model.UploadStatusDeleted)
|
||||
|
||||
if req.Keyword != "" {
|
||||
query = query.Where("LOWER(file_name) LIKE ?", "%"+strings.ToLower(req.Keyword)+"%")
|
||||
}
|
||||
if req.Type != "" {
|
||||
query = query.Where("type = ?", req.Type)
|
||||
}
|
||||
if req.Extension != "" {
|
||||
query = query.Where("extension = ?", strings.ToLower(req.Extension))
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryFileCountFailed))
|
||||
return
|
||||
}
|
||||
|
||||
var items []model.Upload
|
||||
offset := (req.Page - 1) * req.PageSize
|
||||
if err := query.Order("created_at DESC").Offset(offset).Limit(req.PageSize).Find(&items).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryFileListFailed))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, util.OK(listMyFilesResponse{
|
||||
Total: total,
|
||||
Page: req.Page,
|
||||
PageSize: req.PageSize,
|
||||
Items: items,
|
||||
}))
|
||||
}
|
||||
|
||||
// DeleteFile 软删除文件记录
|
||||
// @Summary 删除文件
|
||||
// @Description 将文件状态置为 deleted(软删除),不会立即清理底层存储对象
|
||||
// @Tags upload
|
||||
// @Produce json
|
||||
// @Param id path string true "文件 ID"
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny "删除成功"
|
||||
// @Failure 403 {object} util.ResponseAny "无权操作"
|
||||
// @Failure 404 {object} util.ResponseAny "文件不存在"
|
||||
// @Router /api/v1/upload/{id} [delete]
|
||||
func DeleteFile(c *gin.Context) {
|
||||
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
|
||||
ctx := c.Request.Context()
|
||||
|
||||
idStr := c.Param("id")
|
||||
uploadID, err := strconv.ParseUint(idStr, 10, 64)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrInvalidFileID))
|
||||
return
|
||||
}
|
||||
|
||||
var upload model.Upload
|
||||
if err := db.DB(ctx).Where("id = ? AND status != ?", uploadID, model.UploadStatusDeleted).First(&upload).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryUploadRecordFailed))
|
||||
return
|
||||
}
|
||||
|
||||
// 仅允许文件所有者或管理员删除
|
||||
if upload.UserID != currUser.ID && !currUser.IsAdmin {
|
||||
c.AbortWithStatus(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.DB(ctx).Model(&upload).Update("status", model.UploadStatusDeleted).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrDeleteFileFailed))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, util.OKNil())
|
||||
}
|
||||
|
||||
// validateUploadExtension 校验文件后缀是否在系统允许的上传扩展名列表中
|
||||
func validateUploadExtension(ctx context.Context, ext string) string {
|
||||
var sc model.SystemConfig
|
||||
@@ -499,6 +362,10 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if storageReadOnly(ctx) {
|
||||
c.JSON(http.StatusConflict, util.Err(ErrStorageReadOnly))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
id := idgen.NextUint64ID()
|
||||
newUpload := model.Upload{
|
||||
@@ -527,30 +394,32 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// storeUploadFile 将文件写入底层存储驱动(S3 或本地磁盘)
|
||||
func storeUploadFile(ctx context.Context, id uint64, ext, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, string, string) {
|
||||
if storage.IsEnabled() {
|
||||
meta.Bucket = config.Config.S3.Bucket
|
||||
fullKey := storage.BuildKey(subPath)
|
||||
if err := storage.PutObject(ctx, fullKey, bytes.NewReader(buf.Bytes()), size, mimeType); err != nil {
|
||||
logger.ErrorF(ctx, "S3 存储上传失败: %v", err)
|
||||
return "", "", ErrSaveFileFailed
|
||||
}
|
||||
return "s3", subPath, ""
|
||||
// storeUploadFile 将文件写入当前活动存储驱动。
|
||||
func storeUploadFile(ctx context.Context, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, string, string) {
|
||||
if storageReadOnly(ctx) {
|
||||
return "", "", ErrStorageReadOnly
|
||||
}
|
||||
|
||||
localDir := filepath.Join("uploads", time.Now().Format("2006/01/02"))
|
||||
if err := os.MkdirAll(localDir, uploadDirPerm); err != nil {
|
||||
logger.ErrorF(ctx, "创建本地上传目录失败: %v", err)
|
||||
driver, backend, err := storage.Active(ctx)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "初始化活动存储失败: %v", err)
|
||||
return "", "", ErrSaveFileFailed
|
||||
}
|
||||
|
||||
localPath := filepath.Join(localDir, fmt.Sprintf("%d.%s", id, ext))
|
||||
if err := os.WriteFile(localPath, buf.Bytes(), uploadFilePerm); err != nil {
|
||||
logger.ErrorF(ctx, "本地磁盘写入文件失败: %v", err)
|
||||
storedPath, err := backend.Put(ctx, subPath, bytes.NewReader(buf.Bytes()), size, mimeType)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "写入 %s 存储失败: %v", driver, err)
|
||||
return "", "", ErrSaveFileFailed
|
||||
}
|
||||
return storageDriverLocal, localPath, ""
|
||||
switch driver {
|
||||
case storage.DriverS3:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
case storage.DriverR2:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
case storage.DriverMinIO:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
case storage.DriverOSS:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
}
|
||||
return string(driver), storedPath, ""
|
||||
}
|
||||
|
||||
// isImageExtension 判断文件扩展名是否属于常见图片格式
|
||||
@@ -590,35 +459,13 @@ func detectMimeType(buf *bytes.Buffer, header *multipart.FileHeader, size int64)
|
||||
// saveUploadRecord 保存上传记录到数据库,失败时清理本地垃圾文件
|
||||
func saveUploadRecord(ctx context.Context, upload *model.Upload, storageDriver, filePath string) string {
|
||||
if err := db.DB(ctx).Create(upload).Error; err != nil {
|
||||
if storageDriver == storageDriverLocal {
|
||||
_ = os.Remove(filePath)
|
||||
backend, backendErr := storage.ForDriver(ctx, storage.Driver(storageDriver))
|
||||
if backendErr == nil {
|
||||
if deleteErr := backend.Delete(ctx, filePath); deleteErr != nil {
|
||||
logger.WarnF(ctx, "清理未写入数据库的上传对象失败: %v", deleteErr)
|
||||
}
|
||||
}
|
||||
return ErrSaveUploadRecordFailed
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetDistinctUploadTypes 获取数据库中所有已存在的文件业务类型
|
||||
// @Summary 获取文件业务类型列表
|
||||
// @Description 返回数据库中所有已上传文件实际拥有的业务类型列表
|
||||
// @Tags admin
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny{data=[]string} "业务类型列表"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Failure 403 {object} util.ResponseAny "无管理员权限"
|
||||
// @Failure 500 {object} util.ResponseAny "内部错误"
|
||||
// @Router /api/v1/admin/uploads/types [get]
|
||||
func GetDistinctUploadTypes(c *gin.Context) {
|
||||
var dbTypes []string
|
||||
if err := db.DB(c.Request.Context()).Model(&model.Upload{}).
|
||||
Where("type IS NOT NULL AND type != ''").
|
||||
Distinct().
|
||||
Pluck("type", &dbTypes).Error; err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
sort.Strings(dbTypes)
|
||||
c.JSON(http.StatusOK, util.OK(dbTypes))
|
||||
}
|
||||
|
||||
@@ -105,12 +105,12 @@ func TestUploadFile(t *testing.T) {
|
||||
putCount++
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, key string) (*storage.ObjectInfo, error) {
|
||||
func(ctx context.Context, key string) (*storage.Object, error) {
|
||||
data, ok := mockFiles[key]
|
||||
if !ok {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return &storage.ObjectInfo{
|
||||
return &storage.Object{
|
||||
Body: io.NopCloser(bytes.NewReader(data)),
|
||||
ContentLength: int64(len(data)),
|
||||
ContentType: "application/octet-stream",
|
||||
@@ -821,4 +821,3 @@ func TestGetFileStats(t *testing.T) {
|
||||
t.Errorf("expected 1 document category, got %d", categoryMap["文档"])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
"github.com/Rain-kl/Wavelet/internal/task"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
// StorageMigrationTask is the Asynq task name for storage migration.
|
||||
StorageMigrationTask = "storage:migrate"
|
||||
// TaskTypeStorageMigration is the task metadata type for storage migration.
|
||||
TaskTypeStorageMigration = "storage_migration"
|
||||
)
|
||||
|
||||
// StorageMigrationMeta describes the manually dispatchable migration task.
|
||||
var StorageMigrationMeta = task.TaskMeta{
|
||||
Type: TaskTypeStorageMigration,
|
||||
AsynqTask: StorageMigrationTask,
|
||||
Name: "迁移文件存储",
|
||||
Description: "将活动存储中的文件迁移到待切换的目标存储,迁移期间文件系统保持只读",
|
||||
SupportsTime: false,
|
||||
MaxRetry: task.DefaultMaxRetry,
|
||||
Queue: task.QueueDefault,
|
||||
Retryable: true,
|
||||
}
|
||||
|
||||
// MigrationHandler copies stored objects and activates the target backend.
|
||||
type MigrationHandler struct{}
|
||||
|
||||
type storageMigrationPayload struct {
|
||||
Target storage.Config `json:"target"`
|
||||
}
|
||||
|
||||
// ValidatePayload rejects duplicate active migrations through the task framework.
|
||||
func (h *MigrationHandler) ValidatePayload(payload []byte) ([]byte, error) {
|
||||
normalized, _, err := normalizeStorageMigrationPayload(context.Background(), payload)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
active, err := hasUnresolvedMigrationTask(context.Background())
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
if active {
|
||||
return payload, fmt.Errorf("storage migration task is already unresolved")
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
// Execute migrates all unique active-storage objects to the pending backend.
|
||||
func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
|
||||
active, err := storage.LoadConfig(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load active storage config: %w", err)
|
||||
}
|
||||
target, err := parseMigrationTargetConfig(ctx, payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if target.Driver == active.Driver {
|
||||
if err := storage.SaveActiveConfig(ctx, target); err != nil {
|
||||
return nil, fmt.Errorf("activate same-driver storage config: %w", err)
|
||||
}
|
||||
message := fmt.Sprintf("存储配置已更新,活动存储保持为 %s", target.Driver)
|
||||
task.AppendLog(ctx, "%s", message)
|
||||
return &task.TaskResult{Message: message}, nil
|
||||
}
|
||||
|
||||
total, err := countStorageObjects(ctx, active.Driver)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("count source objects: %w", err)
|
||||
}
|
||||
if total == 0 {
|
||||
if err := storage.SaveActiveConfig(ctx, target); err != nil {
|
||||
return nil, fmt.Errorf("activate empty storage config: %w", err)
|
||||
}
|
||||
message := fmt.Sprintf("当前存储没有需要迁移的对象,活动存储已切换为 %s", target.Driver)
|
||||
task.AppendLog(ctx, "%s", message)
|
||||
return &task.TaskResult{Message: message}, nil
|
||||
}
|
||||
|
||||
sourceBackend, err := storage.NewBackend(ctx, active, active.Driver)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create source storage: %w", err)
|
||||
}
|
||||
targetBackend, err := storage.NewBackend(ctx, target, target.Driver)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create target storage: %w", err)
|
||||
}
|
||||
|
||||
task.AppendLog(ctx, "开始存储迁移: %s -> %s,总对象数: %d", active.Driver, target.Driver, total)
|
||||
migrated, err := migrateObjects(ctx, sourceBackend, targetBackend, active.Driver, target.Driver, total)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := storage.SaveActiveConfig(ctx, target); err != nil {
|
||||
return nil, fmt.Errorf("activate target storage: %w", err)
|
||||
}
|
||||
message := fmt.Sprintf("存储迁移完成,共迁移 %d 个对象,活动存储已切换为 %s", migrated, target.Driver)
|
||||
task.AppendLog(ctx, "%s", message)
|
||||
return &task.TaskResult{Message: message}, nil
|
||||
}
|
||||
|
||||
func normalizeStorageMigrationPayload(ctx context.Context, payload []byte) ([]byte, storage.Config, error) {
|
||||
target, err := parseMigrationTargetConfig(ctx, payload)
|
||||
if err != nil {
|
||||
return nil, storage.Config{}, err
|
||||
}
|
||||
normalized, err := json.Marshal(storageMigrationPayload{Target: target})
|
||||
if err != nil {
|
||||
return nil, storage.Config{}, fmt.Errorf("marshal storage migration payload: %w", err)
|
||||
}
|
||||
return normalized, target, nil
|
||||
}
|
||||
|
||||
func parseMigrationTargetConfig(ctx context.Context, payload []byte) (storage.Config, error) {
|
||||
if strings.TrimSpace(string(payload)) == "" {
|
||||
return storage.Config{}, errors.New("storage migration target payload is required")
|
||||
}
|
||||
var req storageMigrationPayload
|
||||
if err := json.Unmarshal(payload, &req); err != nil {
|
||||
return storage.Config{}, fmt.Errorf("parse storage migration payload: %w", err)
|
||||
}
|
||||
current, err := storage.LoadConfig(ctx)
|
||||
if err != nil {
|
||||
return storage.Config{}, fmt.Errorf("load active storage config: %w", err)
|
||||
}
|
||||
target := storage.MergeMaskedSecrets(req.Target, current)
|
||||
if err := storage.ValidateConfig(target); err != nil {
|
||||
return storage.Config{}, fmt.Errorf("validate target storage config: %w", err)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func countStorageObjects(ctx context.Context, driver storage.Driver) (int64, error) {
|
||||
var count int64
|
||||
err := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("storage_driver = ? AND status != ?", driver, model.UploadStatusDeleted).
|
||||
Distinct("file_path").
|
||||
Count(&count).Error
|
||||
return count, err
|
||||
}
|
||||
|
||||
func hasUnresolvedMigrationTask(ctx context.Context) (bool, error) {
|
||||
execution, ok, err := latestStorageMigrationExecution(ctx)
|
||||
if err != nil || !ok {
|
||||
return false, err
|
||||
}
|
||||
return execution.Status == model.TaskExecutionStatusPending || execution.Status == model.TaskExecutionStatusRunning, nil
|
||||
}
|
||||
|
||||
func latestStorageMigrationExecution(ctx context.Context) (*model.TaskExecution, bool, error) {
|
||||
var execution model.TaskExecution
|
||||
err := db.DB(ctx).
|
||||
Where("task_type = ?", StorageMigrationTask).
|
||||
Order("id DESC").
|
||||
First(&execution).Error
|
||||
if err == nil {
|
||||
return &execution, true, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, false, err
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func migrateObjects(
|
||||
ctx context.Context,
|
||||
sourceBackend storage.Backend,
|
||||
targetBackend storage.Backend,
|
||||
sourceDriver storage.Driver,
|
||||
targetDriver storage.Driver,
|
||||
total int64,
|
||||
) (int64, error) {
|
||||
const batchSize = 50
|
||||
var migrated int64
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return migrated, fmt.Errorf("storage migration canceled: %w", err)
|
||||
}
|
||||
|
||||
var objects []struct {
|
||||
FilePath string `gorm:"column:file_path"`
|
||||
FileSize int64 `gorm:"column:file_size"`
|
||||
MimeType string `gorm:"column:mime_type"`
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.Upload{}).
|
||||
Select("file_path, MAX(file_size) AS file_size, MAX(mime_type) AS mime_type").
|
||||
Where("storage_driver = ? AND status != ?", sourceDriver, model.UploadStatusDeleted).
|
||||
Group("file_path").
|
||||
Order("file_path ASC").
|
||||
Limit(batchSize).
|
||||
Scan(&objects).Error; err != nil {
|
||||
return migrated, fmt.Errorf("query source objects: %w", err)
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, object := range objects {
|
||||
source, err := sourceBackend.Get(ctx, object.FilePath)
|
||||
if err != nil {
|
||||
if isNotFoundError(err) {
|
||||
task.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", object.FilePath, err)
|
||||
if updateErr := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("storage_driver = ? AND file_path = ?", sourceDriver, object.FilePath).
|
||||
Updates(map[string]any{
|
||||
"status": model.UploadStatusDeleted,
|
||||
"storage_driver": targetDriver,
|
||||
}).Error; updateErr != nil {
|
||||
return migrated, fmt.Errorf("update missing object %q: %w", object.FilePath, updateErr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return migrated, fmt.Errorf("open source object %q: %w", object.FilePath, err)
|
||||
}
|
||||
targetPath, putErr := targetBackend.Put(ctx, object.FilePath, source.Body, object.FileSize, object.MimeType)
|
||||
closeErr := source.Body.Close()
|
||||
if putErr != nil {
|
||||
return migrated, fmt.Errorf("copy object %q: %w", object.FilePath, putErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return migrated, fmt.Errorf("close source object %q: %w", object.FilePath, closeErr)
|
||||
}
|
||||
if err := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("storage_driver = ? AND file_path = ?", sourceDriver, object.FilePath).
|
||||
Updates(map[string]any{
|
||||
"storage_driver": targetDriver,
|
||||
"file_path": targetPath,
|
||||
}).Error; err != nil {
|
||||
return migrated, fmt.Errorf("update migrated object %q: %w", object.FilePath, err)
|
||||
}
|
||||
migrated++
|
||||
}
|
||||
task.AppendLog(ctx, "迁移进度: %d/%d", migrated, total)
|
||||
}
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func isNotFoundError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return true
|
||||
}
|
||||
errStr := strings.ToLower(err.Error())
|
||||
for _, sub := range []string{"not found", "nosuchkey", "nosuchbucket", "404", "does not exist"} {
|
||||
if strings.Contains(errStr, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
)
|
||||
|
||||
func TestMigrationHandlerExecute(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
sourceRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(sourceRoot, "uploads", "test.txt")
|
||||
if err := os.MkdirAll(filepath.Dir(sourcePath), 0755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q) returned error: %v", sourcePath, err)
|
||||
}
|
||||
const content = "storage migration"
|
||||
if err := os.WriteFile(sourcePath, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) returned error: %v", sourcePath, err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
active := storage.DefaultConfig()
|
||||
active.Local.Root = sourceRoot
|
||||
if err := storage.SaveActiveConfig(ctx, active); err != nil {
|
||||
t.Fatalf("SaveActiveConfig() returned error: %v", err)
|
||||
}
|
||||
target := storage.DefaultConfig()
|
||||
target.Driver = storage.DriverS3
|
||||
target.S3 = storage.ObjectConfig{
|
||||
Region: "us-east-1",
|
||||
Bucket: "target",
|
||||
AccessKeyID: "key",
|
||||
SecretAccessKey: "secret",
|
||||
}
|
||||
payload, err := json.Marshal(storageMigrationPayload{Target: target})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(storageMigrationPayload) returned error: %v", err)
|
||||
}
|
||||
|
||||
upload := model.Upload{
|
||||
ID: 99101,
|
||||
UserID: 1,
|
||||
FileName: "test.txt",
|
||||
FilePath: "uploads/test.txt",
|
||||
FileSize: int64(len(content)),
|
||||
MimeType: "text/plain",
|
||||
Extension: "txt",
|
||||
Hash: "hash",
|
||||
StorageDriver: string(storage.DriverLocal),
|
||||
Type: "attachment",
|
||||
Status: model.UploadStatusUsed,
|
||||
}
|
||||
if err := dbConn.Create(&upload).Error; err != nil {
|
||||
t.Fatalf("Create(upload) returned error: %v", err)
|
||||
}
|
||||
|
||||
var copied bytes.Buffer
|
||||
restore := storage.MockStorage(
|
||||
func(_ context.Context, _ string, body io.Reader, _ int64, _ string) error {
|
||||
_, err := io.Copy(&copied, body)
|
||||
return err
|
||||
},
|
||||
func(context.Context, string) (*storage.Object, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, string) error {
|
||||
return nil
|
||||
},
|
||||
)
|
||||
defer restore()
|
||||
|
||||
result, err := (&MigrationHandler{}).Execute(ctx, payload)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() returned error: %v", err)
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("Execute() result = nil, want non-nil")
|
||||
}
|
||||
if copied.String() != content {
|
||||
t.Errorf("migrated content = %q, want %q", copied.String(), content)
|
||||
}
|
||||
|
||||
var migrated model.Upload
|
||||
if err := dbConn.First(&migrated, upload.ID).Error; err != nil {
|
||||
t.Fatalf("First(upload) returned error: %v", err)
|
||||
}
|
||||
if migrated.StorageDriver != string(storage.DriverS3) {
|
||||
t.Errorf("StorageDriver = %q, want %q", migrated.StorageDriver, storage.DriverS3)
|
||||
}
|
||||
current, err := storage.LoadConfig(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig() returned error: %v", err)
|
||||
}
|
||||
if current.Driver != storage.DriverS3 {
|
||||
t.Errorf("active driver = %q, want %q", current.Driver, storage.DriverS3)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/logger"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
)
|
||||
|
||||
func storageReadOnly(ctx context.Context) bool {
|
||||
execution, ok, err := latestStorageMigrationExecution(ctx)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "读取存储维护状态失败: %v", err)
|
||||
return true
|
||||
}
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return execution.Status != model.TaskExecutionStatusSucceeded
|
||||
}
|
||||
|
||||
func openStoredObject(ctx context.Context, upload *model.Upload) (*storage.Object, error) {
|
||||
driver := storage.Driver(upload.StorageDriver)
|
||||
if driver == "" {
|
||||
driver = storage.DriverLocal
|
||||
}
|
||||
backend, err := backendForStoredDriver(ctx, driver)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return backend.Get(ctx, upload.FilePath)
|
||||
}
|
||||
|
||||
func backendForStoredDriver(ctx context.Context, driver storage.Driver) (storage.Backend, error) {
|
||||
backend, err := storage.ForDriver(ctx, driver)
|
||||
if err == nil {
|
||||
return backend, nil
|
||||
}
|
||||
|
||||
target, ok, targetErr := currentMigrationTargetConfig(ctx)
|
||||
if targetErr != nil {
|
||||
return nil, targetErr
|
||||
}
|
||||
if ok && target.Driver == driver {
|
||||
return storage.NewBackend(ctx, target, driver)
|
||||
}
|
||||
return nil, fmt.Errorf("storage configuration for driver %q is unavailable", driver)
|
||||
}
|
||||
|
||||
func currentMigrationTargetConfig(ctx context.Context) (storage.Config, bool, error) {
|
||||
execution, ok, err := latestStorageMigrationExecution(ctx)
|
||||
if err != nil || !ok {
|
||||
return storage.Config{}, false, err
|
||||
}
|
||||
if execution.Status == model.TaskExecutionStatusSucceeded {
|
||||
return storage.Config{}, false, nil
|
||||
}
|
||||
target, err := parseMigrationTargetConfig(ctx, []byte(execution.Payload))
|
||||
if err != nil {
|
||||
return storage.Config{}, false, err
|
||||
}
|
||||
return target, true, nil
|
||||
}
|
||||
|
||||
func cfgBucket(ctx context.Context, driver storage.Driver) string {
|
||||
cfg, err := storage.LoadConfig(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
switch driver {
|
||||
case storage.DriverS3:
|
||||
return cfg.S3.Bucket
|
||||
case storage.DriverR2:
|
||||
return cfg.R2.Bucket
|
||||
case storage.DriverMinIO:
|
||||
return cfg.MinIO.Bucket
|
||||
case storage.DriverOSS:
|
||||
return cfg.OSS.Bucket
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,9 @@ type WarmImageCacheHandler struct{}
|
||||
|
||||
// Execute 执行清理未使用上传文件的业务逻辑
|
||||
func (h *CleanupUnusedUploadsHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
|
||||
if storageReadOnly(ctx) {
|
||||
return nil, errors.New(ErrStorageReadOnly)
|
||||
}
|
||||
const batchSize = 100 // 每批处理100个文件
|
||||
var lastID uint64
|
||||
var totalProcessed int
|
||||
@@ -122,8 +125,15 @@ func (h *CleanupUnusedUploadsHandler) Execute(ctx context.Context, _ []byte) (*t
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete from S3
|
||||
if err := storage.DeleteObject(ctx, upload.FilePath); err != nil {
|
||||
driver := storage.Driver(upload.StorageDriver)
|
||||
if driver == "" {
|
||||
driver = storage.DriverLocal
|
||||
}
|
||||
backend, err := storage.ForDriver(ctx, driver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := backend.Delete(ctx, upload.FilePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestCleanupUnusedUploadsHandler_Execute(t *testing.T) {
|
||||
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, key string) (*storage.ObjectInfo, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) (*storage.Object, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) error { return nil },
|
||||
)
|
||||
defer storageMock()
|
||||
@@ -113,7 +113,7 @@ func TestCleanupUnusedUploadsHandler_ExecuteNoFiles(t *testing.T) {
|
||||
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, key string) (*storage.ObjectInfo, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) (*storage.Object, error) { return nil, nil },
|
||||
func(ctx context.Context, key string) error { return nil },
|
||||
)
|
||||
defer storageMock()
|
||||
@@ -204,6 +204,13 @@ func TestWarmImageCacheHandlerExecute(t *testing.T) {
|
||||
})
|
||||
|
||||
testDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
active := storage.DefaultConfig()
|
||||
active.Local.Root = testDir
|
||||
if err := storage.SaveActiveConfig(ctx, active); err != nil {
|
||||
t.Fatalf("SaveActiveConfig() returned error: %v", err)
|
||||
}
|
||||
|
||||
firstPath := filepath.Join(testDir, "first.png")
|
||||
secondPath := filepath.Join(testDir, "second.jpg")
|
||||
writeTaskTestPNG(t, firstPath, color.RGBA{R: 255, A: 255})
|
||||
|
||||
Reference in New Issue
Block a user