mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-01 06:36:38 +08:00
refactor(storage): drop per-upload storage_driver, use storage_config as single source
Remove w_uploads.storage_driver and route all read/write/delete paths through storage.Active() backed by storage_config.driver. Block direct driver switches when uploads exist; require migration task instead. Simplify migration to cursor-based file_path iteration without per-row driver updates.
This commit is contained in:
@@ -11,4 +11,5 @@ const (
|
||||
ConfigKeyRequired = "配置键不能为空"
|
||||
ConfigValueRequired = "配置值不能为空"
|
||||
ConfigKeyExists = "配置键已存在"
|
||||
StorageDriverSwitchRequiresMigration = "存在存量文件,请通过存储迁移任务切换存储引擎"
|
||||
)
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
@@ -88,9 +87,6 @@ func updateSystemConfig(ctx context.Context, key string, req UpdateSystemConfigR
|
||||
if err := tx.Model(&config).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := repointUploadStorageDriversOnDriverSwitch(ctx, tx, key, originalDriver, req.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
resolveStorageMigrationTasksOnDirectDriverUpdate(ctx, tx, key, originalDriver, req.Value)
|
||||
return nil
|
||||
}); err != nil {
|
||||
@@ -101,38 +97,6 @@ func updateSystemConfig(ctx context.Context, key string, req UpdateSystemConfigR
|
||||
return nil
|
||||
}
|
||||
|
||||
func repointUploadStorageDriversOnDriverSwitch(
|
||||
ctx context.Context,
|
||||
tx *gorm.DB,
|
||||
key string,
|
||||
originalDriver storage.Driver,
|
||||
newValue string,
|
||||
) error {
|
||||
if key != model.ConfigKeyStorageConfig || originalDriver == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var newCfg storage.Config
|
||||
if err := json.Unmarshal([]byte(newValue), &newCfg); err != nil {
|
||||
return fmt.Errorf("parse storage config for driver repoint: %w", err)
|
||||
}
|
||||
if newCfg.Driver == "" || newCfg.Driver == originalDriver {
|
||||
return nil
|
||||
}
|
||||
|
||||
result := tx.Model(&model.Upload{}).
|
||||
Where("storage_driver = ? AND status != ?", string(originalDriver), model.UploadStatusDeleted).
|
||||
Update("storage_driver", string(newCfg.Driver))
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected > 0 {
|
||||
logger.InfoF(ctx, "[StorageConfig] switched driver %s -> %s, repointed %d upload records",
|
||||
originalDriver, newCfg.Driver, result.RowsAffected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveStorageMigrationTasksOnDirectDriverUpdate(
|
||||
ctx context.Context,
|
||||
tx *gorm.DB,
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/cap"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||
"github.com/Rain-kl/Wavelet/internal/common/response"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/Rain-kl/Wavelet/internal/storage"
|
||||
@@ -266,11 +267,13 @@ func TestSMTP(c *gin.Context) {
|
||||
|
||||
func isStorageConfigValidationError(err error) bool {
|
||||
msg := err.Error()
|
||||
return strings.HasPrefix(msg, "解析") ||
|
||||
return msg == StorageDriverSwitchRequiresMigration ||
|
||||
strings.HasPrefix(msg, "解析") ||
|
||||
strings.HasPrefix(msg, "验证") ||
|
||||
strings.HasPrefix(msg, "初始化测试") ||
|
||||
strings.HasPrefix(msg, "存储连通性") ||
|
||||
strings.HasPrefix(msg, "序列化")
|
||||
strings.HasPrefix(msg, "序列化") ||
|
||||
strings.HasPrefix(msg, "检查存量文件")
|
||||
}
|
||||
|
||||
func maskSensitiveConfig(key, value string) string {
|
||||
@@ -322,6 +325,15 @@ func validateAndMergeStorageConfig(ctx context.Context, value string, currentCon
|
||||
|
||||
func validateMergedStorageConfig(ctx context.Context, currentCfg, newCfg, targetCfg storage.Config) error {
|
||||
if newCfg.Driver != "" && newCfg.Driver != currentCfg.Driver {
|
||||
var uploadCount int64
|
||||
if err := db.DB(ctx).Model(&model.Upload{}).
|
||||
Where("status != ?", model.UploadStatusDeleted).
|
||||
Count(&uploadCount).Error; err != nil {
|
||||
return fmt.Errorf("检查存量文件失败: %w", err)
|
||||
}
|
||||
if uploadCount > 0 {
|
||||
return errors.New(StorageDriverSwitchRequiresMigration)
|
||||
}
|
||||
if err := validateDriverConfig(targetCfg, newCfg.Driver); err != nil {
|
||||
return fmt.Errorf("验证目标存储配置参数失败: %w", err)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
|
||||
@@ -441,7 +442,53 @@ func TestUpdateStorageConfigValidation(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reject driver switch when uploads exist", func(t *testing.T) {
|
||||
upload := model.Upload{
|
||||
ID: 88001,
|
||||
UserID: 1,
|
||||
FileName: "keep.txt",
|
||||
FilePath: "uploads/keep.txt",
|
||||
FileSize: 4,
|
||||
MimeType: "text/plain",
|
||||
Extension: "txt",
|
||||
Type: "attachment",
|
||||
Status: model.UploadStatusUsed,
|
||||
}
|
||||
if err := dbConn.Create(&upload).Error; err != nil {
|
||||
t.Fatalf("seed upload failed: %v", err)
|
||||
}
|
||||
|
||||
tempDir := t.TempDir()
|
||||
cfg := storage.DefaultConfig()
|
||||
cfg.Driver = storage.DriverS3
|
||||
cfg.S3.Endpoint = "http://127.0.0.1:19998"
|
||||
cfg.S3.Region = "us-east-1"
|
||||
cfg.S3.Bucket = "wavelet"
|
||||
cfg.S3.AccessKeyID = "test"
|
||||
cfg.S3.SecretAccessKey = "test"
|
||||
cfg.Local.Root = tempDir
|
||||
|
||||
cfgBytes, _ := json.Marshal(cfg)
|
||||
payload := UpdateSystemConfigRequest{Value: string(cfgBytes)}
|
||||
body, _ := json.Marshal(payload)
|
||||
req, _ := http.NewRequest("PUT", "/api/v1/admin/system-configs/storage_config", bytes.NewBuffer(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400 Bad Request, got %d. Body: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), StorageDriverSwitchRequiresMigration) {
|
||||
t.Fatalf("expected migration-required error, got: %s", w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("switch to local while active s3 is unreachable", func(t *testing.T) {
|
||||
if err := dbConn.Where("1 = 1").Delete(&model.Upload{}).Error; err != nil {
|
||||
t.Fatalf("clear uploads failed: %v", err)
|
||||
}
|
||||
|
||||
activeCfg := storage.DefaultConfig()
|
||||
activeCfg.Driver = storage.DriverS3
|
||||
activeCfg.S3.Endpoint = "http://127.0.0.1:9999"
|
||||
|
||||
Reference in New Issue
Block a user