refactor(storage): drop per-upload storage_driver, use storage_config as single source

Remove w_uploads.storage_driver and route all read/write/delete paths through
storage.Active() backed by storage_config.driver. Block direct driver switches
when uploads exist; require migration task instead. Simplify migration to
cursor-based file_path iteration without per-row driver updates.
This commit is contained in:
ryan
2026-06-18 13:56:31 +08:00
parent 1f391e9ec2
commit 410ff14795
25 changed files with 512 additions and 604 deletions
@@ -11,4 +11,5 @@ const (
ConfigKeyRequired = "配置键不能为空"
ConfigValueRequired = "配置值不能为空"
ConfigKeyExists = "配置键已存在"
StorageDriverSwitchRequiresMigration = "存在存量文件,请通过存储迁移任务切换存储引擎"
)
@@ -7,7 +7,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/Rain-kl/Wavelet/internal/db"
@@ -88,9 +87,6 @@ func updateSystemConfig(ctx context.Context, key string, req UpdateSystemConfigR
if err := tx.Model(&config).Updates(updates).Error; err != nil {
return err
}
if err := repointUploadStorageDriversOnDriverSwitch(ctx, tx, key, originalDriver, req.Value); err != nil {
return err
}
resolveStorageMigrationTasksOnDirectDriverUpdate(ctx, tx, key, originalDriver, req.Value)
return nil
}); err != nil {
@@ -101,38 +97,6 @@ func updateSystemConfig(ctx context.Context, key string, req UpdateSystemConfigR
return nil
}
func repointUploadStorageDriversOnDriverSwitch(
ctx context.Context,
tx *gorm.DB,
key string,
originalDriver storage.Driver,
newValue string,
) error {
if key != model.ConfigKeyStorageConfig || originalDriver == "" {
return nil
}
var newCfg storage.Config
if err := json.Unmarshal([]byte(newValue), &newCfg); err != nil {
return fmt.Errorf("parse storage config for driver repoint: %w", err)
}
if newCfg.Driver == "" || newCfg.Driver == originalDriver {
return nil
}
result := tx.Model(&model.Upload{}).
Where("storage_driver = ? AND status != ?", string(originalDriver), model.UploadStatusDeleted).
Update("storage_driver", string(newCfg.Driver))
if result.Error != nil {
return result.Error
}
if result.RowsAffected > 0 {
logger.InfoF(ctx, "[StorageConfig] switched driver %s -> %s, repointed %d upload records",
originalDriver, newCfg.Driver, result.RowsAffected)
}
return nil
}
func resolveStorageMigrationTasksOnDirectDriverUpdate(
ctx context.Context,
tx *gorm.DB,
+14 -2
View File
@@ -18,6 +18,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/cap"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/common/response"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/storage"
@@ -266,11 +267,13 @@ func TestSMTP(c *gin.Context) {
func isStorageConfigValidationError(err error) bool {
msg := err.Error()
return strings.HasPrefix(msg, "解析") ||
return msg == StorageDriverSwitchRequiresMigration ||
strings.HasPrefix(msg, "解析") ||
strings.HasPrefix(msg, "验证") ||
strings.HasPrefix(msg, "初始化测试") ||
strings.HasPrefix(msg, "存储连通性") ||
strings.HasPrefix(msg, "序列化")
strings.HasPrefix(msg, "序列化") ||
strings.HasPrefix(msg, "检查存量文件")
}
func maskSensitiveConfig(key, value string) string {
@@ -322,6 +325,15 @@ func validateAndMergeStorageConfig(ctx context.Context, value string, currentCon
func validateMergedStorageConfig(ctx context.Context, currentCfg, newCfg, targetCfg storage.Config) error {
if newCfg.Driver != "" && newCfg.Driver != currentCfg.Driver {
var uploadCount int64
if err := db.DB(ctx).Model(&model.Upload{}).
Where("status != ?", model.UploadStatusDeleted).
Count(&uploadCount).Error; err != nil {
return fmt.Errorf("检查存量文件失败: %w", err)
}
if uploadCount > 0 {
return errors.New(StorageDriverSwitchRequiresMigration)
}
if err := validateDriverConfig(targetCfg, newCfg.Driver); err != nil {
return fmt.Errorf("验证目标存储配置参数失败: %w", err)
}
@@ -13,6 +13,7 @@ import (
"net/http"
"net/http/httptest"
"net/textproto"
"strings"
"testing"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
@@ -441,7 +442,53 @@ func TestUpdateStorageConfigValidation(t *testing.T) {
}
})
t.Run("reject driver switch when uploads exist", func(t *testing.T) {
upload := model.Upload{
ID: 88001,
UserID: 1,
FileName: "keep.txt",
FilePath: "uploads/keep.txt",
FileSize: 4,
MimeType: "text/plain",
Extension: "txt",
Type: "attachment",
Status: model.UploadStatusUsed,
}
if err := dbConn.Create(&upload).Error; err != nil {
t.Fatalf("seed upload failed: %v", err)
}
tempDir := t.TempDir()
cfg := storage.DefaultConfig()
cfg.Driver = storage.DriverS3
cfg.S3.Endpoint = "http://127.0.0.1:19998"
cfg.S3.Region = "us-east-1"
cfg.S3.Bucket = "wavelet"
cfg.S3.AccessKeyID = "test"
cfg.S3.SecretAccessKey = "test"
cfg.Local.Root = tempDir
cfgBytes, _ := json.Marshal(cfg)
payload := UpdateSystemConfigRequest{Value: string(cfgBytes)}
body, _ := json.Marshal(payload)
req, _ := http.NewRequest("PUT", "/api/v1/admin/system-configs/storage_config", bytes.NewBuffer(body))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("expected 400 Bad Request, got %d. Body: %s", w.Code, w.Body.String())
}
if !strings.Contains(w.Body.String(), StorageDriverSwitchRequiresMigration) {
t.Fatalf("expected migration-required error, got: %s", w.Body.String())
}
})
t.Run("switch to local while active s3 is unreachable", func(t *testing.T) {
if err := dbConn.Where("1 = 1").Delete(&model.Upload{}).Error; err != nil {
t.Fatalf("clear uploads failed: %v", err)
}
activeCfg := storage.DefaultConfig()
activeCfg.Driver = storage.DriverS3
activeCfg.S3.Endpoint = "http://127.0.0.1:9999"