mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-11 09:46:37 +08:00
feat(storage): add dynamic storage config and migration
Move storage backend configuration from startup YAML to system_config-backed runtime configuration. Add local, S3-compatible, R2, MinIO, OSS, and WebDAV backend support. Add a storage migration async task using the existing task dispatch framework. Migration target config is carried in task payload, and maintenance mode is derived from task execution state. Split upload file management and storage operations, add the admin storage configuration tab, and update migrations and Swagger docs.
This commit is contained in:
ryan
+37
-190
@@ -17,16 +17,13 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
|
||||
"github.com/Rain-kl/Wavelet/internal/common"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/db/idgen"
|
||||
"github.com/Rain-kl/Wavelet/internal/logger"
|
||||
@@ -157,8 +154,8 @@ func UploadFile(c *gin.Context) {
|
||||
id := idgen.NextUint64ID()
|
||||
subPath := fmt.Sprintf("uploads/%s/%d.%s", time.Now().Format("2006/01/02"), id, ext)
|
||||
|
||||
// 8. 写入底层存储驱动 (优先 S3 驱动，无配置或未开启则 fallback 至本地文件)
|
||||
storageDriver, subPath, errMsg := storeUploadFile(ctx, id, ext, subPath, size, mimeType, &buf, &meta)
|
||||
// 8. 写入当前活动存储驱动。
|
||||
storageDriver, subPath, errMsg := storeUploadFile(ctx, subPath, size, mimeType, &buf, &meta)
|
||||
if errMsg != "" {
|
||||
c.JSON(http.StatusOK, util.Err(errMsg))
|
||||
return
|
||||
@@ -323,21 +320,12 @@ func BatchDownloadFiles(c *gin.Context) {
|
||||
|
||||
// 打开底层文件数据源
|
||||
var rc io.ReadCloser
|
||||
if upload.StorageDriver == storageDriverLocal || (upload.StorageDriver == "" && !storage.IsEnabled()) {
|
||||
fileSrc, err := os.Open(upload.FilePath)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "打包时读取本地文件失败: %v", err)
|
||||
continue
|
||||
}
|
||||
rc = fileSrc
|
||||
} else {
|
||||
obj, err := storage.GetObject(ctx, upload.FilePath)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "打包时拉取 S3 文件失败: %v", err)
|
||||
continue
|
||||
}
|
||||
rc = obj.Body
|
||||
obj, err := openStoredObject(ctx, &upload)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "打包时读取文件失败: %v", err)
|
||||
continue
|
||||
}
|
||||
rc = obj.Body
|
||||
|
||||
// 流式拷贝到 ZIP entry
|
||||
_, err = io.Copy(zipFileEntry, rc)
|
||||
@@ -348,131 +336,6 @@ func BatchDownloadFiles(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
type listMyFilesRequest struct {
|
||||
Page int `form:"page"`
|
||||
PageSize int `form:"page_size"`
|
||||
Keyword string `form:"keyword"`
|
||||
Type string `form:"type"`
|
||||
Extension string `form:"extension"`
|
||||
}
|
||||
|
||||
type listMyFilesResponse struct {
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
Items []model.Upload `json:"items"`
|
||||
}
|
||||
|
||||
// ListMyFiles 获取当前用户上传的文件列表
|
||||
// @Summary 获取我的文件列表
|
||||
// @Description 分页获取当前登录用户上传的文件，支持文件名关键词、业务类型、扩展名过滤
|
||||
// @Tags upload
|
||||
// @Produce json
|
||||
// @Param page query int false "页码（默认 1）"
|
||||
// @Param page_size query int false "每页数量（默认 20，最大 100）"
|
||||
// @Param keyword query string false "文件名关键词（模糊匹配）"
|
||||
// @Param type query string false "业务分类过滤"
|
||||
// @Param extension query string false "扩展名过滤"
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny{data=listMyFilesResponse} "查询成功"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Router /api/v1/upload/my [get]
|
||||
func ListMyFiles(c *gin.Context) {
|
||||
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var req listMyFilesRequest
|
||||
if err := c.ShouldBindQuery(&req); err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrInvalidParams))
|
||||
return
|
||||
}
|
||||
if req.Page <= 0 {
|
||||
req.Page = 1
|
||||
}
|
||||
if req.PageSize <= 0 || req.PageSize > 100 {
|
||||
req.PageSize = 20
|
||||
}
|
||||
|
||||
query := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("user_id = ? AND status != ?", currUser.ID, model.UploadStatusDeleted)
|
||||
|
||||
if req.Keyword != "" {
|
||||
query = query.Where("LOWER(file_name) LIKE ?", "%"+strings.ToLower(req.Keyword)+"%")
|
||||
}
|
||||
if req.Type != "" {
|
||||
query = query.Where("type = ?", req.Type)
|
||||
}
|
||||
if req.Extension != "" {
|
||||
query = query.Where("extension = ?", strings.ToLower(req.Extension))
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryFileCountFailed))
|
||||
return
|
||||
}
|
||||
|
||||
var items []model.Upload
|
||||
offset := (req.Page - 1) * req.PageSize
|
||||
if err := query.Order("created_at DESC").Offset(offset).Limit(req.PageSize).Find(&items).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryFileListFailed))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, util.OK(listMyFilesResponse{
|
||||
Total: total,
|
||||
Page: req.Page,
|
||||
PageSize: req.PageSize,
|
||||
Items: items,
|
||||
}))
|
||||
}
|
||||
|
||||
// DeleteFile 软删除文件记录
|
||||
// @Summary 删除文件
|
||||
// @Description 将文件状态置为 deleted（软删除），不会立即清理底层存储对象
|
||||
// @Tags upload
|
||||
// @Produce json
|
||||
// @Param id path string true "文件 ID"
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny "删除成功"
|
||||
// @Failure 403 {object} util.ResponseAny "无权操作"
|
||||
// @Failure 404 {object} util.ResponseAny "文件不存在"
|
||||
// @Router /api/v1/upload/{id} [delete]
|
||||
func DeleteFile(c *gin.Context) {
|
||||
currUser, _ := util.GetFromContext[*model.User](c, oauth.UserObjKey)
|
||||
ctx := c.Request.Context()
|
||||
|
||||
idStr := c.Param("id")
|
||||
uploadID, err := strconv.ParseUint(idStr, 10, 64)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrInvalidFileID))
|
||||
return
|
||||
}
|
||||
|
||||
var upload model.Upload
|
||||
if err := db.DB(ctx).Where("id = ? AND status != ?", uploadID, model.UploadStatusDeleted).First(&upload).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, util.Err(ErrQueryUploadRecordFailed))
|
||||
return
|
||||
}
|
||||
|
||||
// 仅允许文件所有者或管理员删除
|
||||
if upload.UserID != currUser.ID && !currUser.IsAdmin {
|
||||
c.AbortWithStatus(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.DB(ctx).Model(&upload).Update("status", model.UploadStatusDeleted).Error; err != nil {
|
||||
c.JSON(http.StatusOK, util.Err(ErrDeleteFileFailed))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, util.OKNil())
|
||||
}
|
||||
|
||||
// validateUploadExtension 校验文件后缀是否在系统允许的上传扩展名列表中
|
||||
func validateUploadExtension(ctx context.Context, ext string) string {
|
||||
var sc model.SystemConfig
|
||||
@@ -499,6 +362,10 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if storageReadOnly(ctx) {
|
||||
c.JSON(http.StatusConflict, util.Err(ErrStorageReadOnly))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
id := idgen.NextUint64ID()
|
||||
newUpload := model.Upload{
|
||||
@@ -527,30 +394,32 @@ func tryInstantUpload(ctx context.Context, c *gin.Context, currUser *model.User,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// storeUploadFile 将文件写入底层存储驱动（S3 或本地磁盘）
|
||||
func storeUploadFile(ctx context.Context, id uint64, ext, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, string, string) {
|
||||
if storage.IsEnabled() {
|
||||
meta.Bucket = config.Config.S3.Bucket
|
||||
fullKey := storage.BuildKey(subPath)
|
||||
if err := storage.PutObject(ctx, fullKey, bytes.NewReader(buf.Bytes()), size, mimeType); err != nil {
|
||||
logger.ErrorF(ctx, "S3 存储上传失败: %v", err)
|
||||
return "", "", ErrSaveFileFailed
|
||||
}
|
||||
return "s3", subPath, ""
|
||||
// storeUploadFile 将文件写入当前活动存储驱动。
|
||||
func storeUploadFile(ctx context.Context, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, string, string) {
|
||||
if storageReadOnly(ctx) {
|
||||
return "", "", ErrStorageReadOnly
|
||||
}
|
||||
|
||||
localDir := filepath.Join("uploads", time.Now().Format("2006/01/02"))
|
||||
if err := os.MkdirAll(localDir, uploadDirPerm); err != nil {
|
||||
logger.ErrorF(ctx, "创建本地上传目录失败: %v", err)
|
||||
driver, backend, err := storage.Active(ctx)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "初始化活动存储失败: %v", err)
|
||||
return "", "", ErrSaveFileFailed
|
||||
}
|
||||
|
||||
localPath := filepath.Join(localDir, fmt.Sprintf("%d.%s", id, ext))
|
||||
if err := os.WriteFile(localPath, buf.Bytes(), uploadFilePerm); err != nil {
|
||||
logger.ErrorF(ctx, "本地磁盘写入文件失败: %v", err)
|
||||
storedPath, err := backend.Put(ctx, subPath, bytes.NewReader(buf.Bytes()), size, mimeType)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "写入 %s 存储失败: %v", driver, err)
|
||||
return "", "", ErrSaveFileFailed
|
||||
}
|
||||
return storageDriverLocal, localPath, ""
|
||||
switch driver {
|
||||
case storage.DriverS3:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
case storage.DriverR2:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
case storage.DriverMinIO:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
case storage.DriverOSS:
|
||||
meta.Bucket = cfgBucket(ctx, driver)
|
||||
}
|
||||
return string(driver), storedPath, ""
|
||||
}
|
||||
|
||||
// isImageExtension 判断文件扩展名是否属于常见图片格式
|
||||
@@ -590,35 +459,13 @@ func detectMimeType(buf *bytes.Buffer, header *multipart.FileHeader, size int64)
|
||||
// saveUploadRecord 保存上传记录到数据库，失败时清理本地垃圾文件
|
||||
func saveUploadRecord(ctx context.Context, upload *model.Upload, storageDriver, filePath string) string {
|
||||
if err := db.DB(ctx).Create(upload).Error; err != nil {
|
||||
if storageDriver == storageDriverLocal {
|
||||
_ = os.Remove(filePath)
|
||||
backend, backendErr := storage.ForDriver(ctx, storage.Driver(storageDriver))
|
||||
if backendErr == nil {
|
||||
if deleteErr := backend.Delete(ctx, filePath); deleteErr != nil {
|
||||
logger.WarnF(ctx, "清理未写入数据库的上传对象失败: %v", deleteErr)
|
||||
}
|
||||
}
|
||||
return ErrSaveUploadRecordFailed
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetDistinctUploadTypes 获取数据库中所有已存在的文件业务类型
|
||||
// @Summary 获取文件业务类型列表
|
||||
// @Description 返回数据库中所有已上传文件实际拥有的业务类型列表
|
||||
// @Tags admin
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} util.ResponseAny{data=[]string} "业务类型列表"
|
||||
// @Failure 401 {object} util.ResponseAny "未登录"
|
||||
// @Failure 403 {object} util.ResponseAny "无管理员权限"
|
||||
// @Failure 500 {object} util.ResponseAny "内部错误"
|
||||
// @Router /api/v1/admin/uploads/types [get]
|
||||
func GetDistinctUploadTypes(c *gin.Context) {
|
||||
var dbTypes []string
|
||||
if err := db.DB(c.Request.Context()).Model(&model.Upload{}).
|
||||
Where("type IS NOT NULL AND type != ''").
|
||||
Distinct().
|
||||
Pluck("type", &dbTypes).Error; err != nil {
|
||||
c.JSON(http.StatusInternalServerError, util.Err(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
sort.Strings(dbTypes)
|
||||
c.JSON(http.StatusOK, util.OK(dbTypes))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user