refactor(storage): drop per-upload storage_driver, use storage_config as single source

Remove w_uploads.storage_driver and route all read/write/delete paths through
storage.Active() backed by storage_config.driver. Block direct driver switches
when uploads exist; require migration task instead. Simplify migration to
cursor-based file_path iteration without per-row driver updates.
This commit is contained in:
ryan
2026-06-18 13:56:31 +08:00
parent 1f391e9ec2
commit 410ff14795
25 changed files with 512 additions and 604 deletions
@@ -29,7 +29,6 @@ func TestGetDistinctUploadTypes(t *testing.T) {
FileSize: 10,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Type: "custom_type_xyz",
Status: model.UploadStatusUsed,
}
+7 -8
View File
@@ -124,7 +124,6 @@ func createInstantUpload(ctx context.Context, existing model.Upload, input insta
MimeType: input.MimeType,
Extension: input.Extension,
Hash: input.FileHash,
StorageDriver: existing.StorageDriver,
Type: input.UploadType,
Status: model.UploadStatusUsed,
AccessMode: input.AccessMode,
@@ -142,9 +141,9 @@ func findReusableUpload(ctx context.Context, hash string, size int64) (model.Upl
return repository.FindReusableUploadByHash(ctx, hash, size)
}
func saveNewUploadRecord(ctx context.Context, upload *model.Upload, storageDriver, filePath string) error {
func saveNewUploadRecord(ctx context.Context, upload *model.Upload, filePath string) error {
if err := repository.CreateUpload(ctx, upload); err != nil {
backend, backendErr := storage.ForDriver(ctx, storage.Driver(storageDriver))
_, backend, backendErr := storage.Active(ctx)
if backendErr == nil {
if deleteErr := backend.Delete(ctx, filePath); deleteErr != nil {
logger.WarnF(ctx, "清理未写入数据库的上传对象失败: %v", deleteErr)
@@ -162,22 +161,22 @@ func loadUploadStats(ctx context.Context) ([]model.UploadStat, error) {
var errUploadForbidden = errors.New("upload forbidden")
func storeUploadObject(ctx context.Context, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, string, error) {
func storeUploadObject(ctx context.Context, subPath string, size int64, mimeType string, buf *bytes.Buffer, meta *model.UploadMetadata) (string, error) {
if uploadstorage.ReadOnly(ctx) {
return "", "", errors.New(shared.ErrStorageReadOnly)
return "", errors.New(shared.ErrStorageReadOnly)
}
driver, backend, err := storage.Active(ctx)
if err != nil {
logger.ErrorF(ctx, "初始化活动存储失败: %v", err)
return "", "", errors.New(shared.ErrSaveFileFailed)
return "", errors.New(shared.ErrSaveFileFailed)
}
result, err := backend.Put(ctx, subPath, bytes.NewReader(buf.Bytes()), size, mimeType)
if err != nil {
logger.ErrorF(ctx, "写入 %s 存储失败: %v", driver, err)
return "", "", errors.New(shared.ErrSaveFileFailed)
return "", errors.New(shared.ErrSaveFileFailed)
}
meta.Bucket = result.Bucket
return string(driver), result.Key, nil
return result.Key, nil
}
func validateUploadAllowedExtension(ctx context.Context, ext string) string {
+14 -15
View File
@@ -138,29 +138,28 @@ func UploadFile(c *gin.Context) {
id := idgen.NextUint64ID()
subPath := fmt.Sprintf("uploads/%s/%d.%s", time.Now().Format("2006/01/02"), id, ext)
storageDriver, subPath, err := storeUploadObject(ctx, subPath, size, mimeType, &buf, &meta)
subPath, err = storeUploadObject(ctx, subPath, size, mimeType, &buf, &meta)
if err != nil {
response.AbortBadRequest(c, err.Error())
return
}
newUpload := model.Upload{
ID: id,
UserID: currUser.ID,
FileName: origName,
FilePath: subPath,
FileSize: size,
MimeType: mimeType,
Extension: ext,
Hash: fileHash,
StorageDriver: storageDriver,
Type: uploadType,
Status: model.UploadStatusUsed,
AccessMode: accessMode,
Metadata: meta,
ID: id,
UserID: currUser.ID,
FileName: origName,
FilePath: subPath,
FileSize: size,
MimeType: mimeType,
Extension: ext,
Hash: fileHash,
Type: uploadType,
Status: model.UploadStatusUsed,
AccessMode: accessMode,
Metadata: meta,
}
if err := saveNewUploadRecord(ctx, &newUpload, storageDriver, subPath); err != nil {
if err := saveNewUploadRecord(ctx, &newUpload, subPath); err != nil {
response.AbortBadRequest(c, shared.ErrSaveUploadRecordFailed)
return
}
@@ -193,10 +193,6 @@ func TestUploadFile(t *testing.T) {
t.Errorf("incorrect mime type detected: %s", dbRecord.MimeType)
}
if dbRecord.StorageDriver != "s3" {
t.Errorf("expected storage driver s3, got %s", dbRecord.StorageDriver)
}
if dbRecord.Metadata.Extra["source"] != "test_runner" {
t.Errorf("expected extra meta 'source' to be 'test_runner', got %v", dbRecord.Metadata.Extra)
}
@@ -325,10 +321,6 @@ func TestUploadFile(t *testing.T) {
t.Fatalf("failed to unmarshal local upload record: %v", err)
}
if localRecord.StorageDriver != "local" {
t.Errorf("expected storage driver local, got %s", localRecord.StorageDriver)
}
// Confirm file was actually written to local disk
fileContent, err := os.ReadFile(localRecord.FilePath)
if err != nil {
@@ -358,7 +350,6 @@ func TestDownloadFile(t *testing.T) {
FileSize: 12,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Status: model.UploadStatusUsed,
}
@@ -426,7 +417,6 @@ func TestListFiles(t *testing.T) {
FileSize: 10,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Status: model.UploadStatusUsed,
},
{
@@ -437,7 +427,6 @@ func TestListFiles(t *testing.T) {
FileSize: 20,
MimeType: "image/png",
Extension: "png",
StorageDriver: "local",
Status: model.UploadStatusUsed,
},
{
@@ -448,7 +437,6 @@ func TestListFiles(t *testing.T) {
FileSize: 30,
MimeType: "text/markdown",
Extension: "md",
StorageDriver: "local",
Status: model.UploadStatusUsed,
},
{
@@ -459,7 +447,6 @@ func TestListFiles(t *testing.T) {
FileSize: 40,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Status: model.UploadStatusUsed,
},
}
@@ -582,7 +569,6 @@ func TestBatchDownloadFiles(t *testing.T) {
FileSize: 13,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Status: model.UploadStatusUsed,
},
{
@@ -593,7 +579,6 @@ func TestBatchDownloadFiles(t *testing.T) {
FileSize: 13,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Status: model.UploadStatusUsed,
},
{
@@ -604,7 +589,6 @@ func TestBatchDownloadFiles(t *testing.T) {
FileSize: 28,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Status: model.UploadStatusUsed,
},
}
@@ -773,7 +757,6 @@ func TestGetFileStats(t *testing.T) {
FileSize: 100,
MimeType: "image/png",
Extension: "png",
StorageDriver: "local",
Type: "generic",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
@@ -786,7 +769,6 @@ func TestGetFileStats(t *testing.T) {
FileSize: 500,
MimeType: "video/mp4",
Extension: "mp4",
StorageDriver: "local",
Type: "generic",
Status: model.UploadStatusUsed,
CreatedAt: time.Now().AddDate(0, 0, -2), // 2 days ago
@@ -799,7 +781,6 @@ func TestGetFileStats(t *testing.T) {
FileSize: 200,
MimeType: "application/pdf",
Extension: "pdf",
StorageDriver: "local",
Type: "avatar", // different type
Status: model.UploadStatusUsed,
CreatedAt: time.Now().AddDate(0, 0, -10), // older than 7 days
@@ -891,7 +872,6 @@ func TestUserUploadManagement(t *testing.T) {
FileSize: 100,
MimeType: "text/plain",
Extension: "txt",
StorageDriver: "local",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
@@ -903,7 +883,6 @@ func TestUserUploadManagement(t *testing.T) {
FileSize: 200,
MimeType: "image/png",
Extension: "png",
StorageDriver: "local",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}