mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-01 22:46:38 +08:00
feat(upload): implement parallel storage migration and integrity check
- Parallelize storage migration using `errgroup` with a concurrency limit of 10. - Perform post-copy SHA-256 data integrity validation to prevent silent data corruption. - Add test case verifying migration with both incorrect and correct hashes.
This commit is contained in:
@@ -6,10 +6,13 @@ package upload
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
@@ -108,3 +111,112 @@ func TestMigrationHandlerExecute(t *testing.T) {
|
||||
t.Errorf("active driver = %q, want %q", current.Driver, storage.DriverS3)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationHandlerExecuteWithHashValidation(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
sourceRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(sourceRoot, "uploads", "test-hash.txt")
|
||||
if err := os.MkdirAll(filepath.Dir(sourcePath), 0755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q) returned error: %v", sourcePath, err)
|
||||
}
|
||||
const content = "storage migration integrity check content"
|
||||
if err := os.WriteFile(sourcePath, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("WriteFile(%q) returned error: %v", sourcePath, err)
|
||||
}
|
||||
|
||||
// Calculate correct SHA-256 hash
|
||||
h := sha256.New()
|
||||
h.Write([]byte(content))
|
||||
correctHash := hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
ctx := context.Background()
|
||||
active := storage.DefaultConfig()
|
||||
active.Local.Root = sourceRoot
|
||||
if err := storage.SaveActiveConfig(ctx, active); err != nil {
|
||||
t.Fatalf("SaveActiveConfig() returned error: %v", err)
|
||||
}
|
||||
|
||||
target := storage.DefaultConfig()
|
||||
target.Driver = storage.DriverS3
|
||||
target.S3 = storage.ObjectConfig{
|
||||
Region: "us-east-1",
|
||||
Bucket: "target",
|
||||
AccessKeyID: "key",
|
||||
SecretAccessKey: "secret",
|
||||
}
|
||||
payload, err := json.Marshal(storageMigrationPayload{Target: target})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal(storageMigrationPayload) returned error: %v", err)
|
||||
}
|
||||
|
||||
// Case 1: Incorrect Hash (should fail validation)
|
||||
uploadIncorrect := model.Upload{
|
||||
ID: 99102,
|
||||
UserID: 1,
|
||||
FileName: "test-hash.txt",
|
||||
FilePath: "uploads/test-hash.txt",
|
||||
FileSize: int64(len(content)),
|
||||
MimeType: "text/plain",
|
||||
Extension: "txt",
|
||||
Hash: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // Invalid hash
|
||||
StorageDriver: string(storage.DriverLocal),
|
||||
Type: "attachment",
|
||||
Status: model.UploadStatusUsed,
|
||||
}
|
||||
if err := dbConn.Create(&uploadIncorrect).Error; err != nil {
|
||||
t.Fatalf("Create(uploadIncorrect) returned error: %v", err)
|
||||
}
|
||||
|
||||
var copied bytes.Buffer
|
||||
restore := storage.MockStorage(
|
||||
func(_ context.Context, _ string, body io.Reader, _ int64, _ string) error {
|
||||
copied.Reset()
|
||||
_, err := io.Copy(&copied, body)
|
||||
return err
|
||||
},
|
||||
func(context.Context, string) (*storage.Object, error) {
|
||||
return &storage.Object{
|
||||
Body: io.NopCloser(bytes.NewBuffer(copied.Bytes())),
|
||||
ContentLength: int64(copied.Len()),
|
||||
ContentType: "text/plain",
|
||||
}, nil
|
||||
},
|
||||
func(context.Context, string) error {
|
||||
return nil
|
||||
},
|
||||
)
|
||||
defer restore()
|
||||
|
||||
// Running execution with incorrect hash should fail with integrity error
|
||||
_, err = (&MigrationHandler{}).Execute(ctx, payload)
|
||||
if err == nil {
|
||||
t.Fatal("Execute() succeeded with incorrect hash, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "integrity check failed") {
|
||||
t.Errorf("expected integrity check failed error, got: %v", err)
|
||||
}
|
||||
|
||||
// Case 2: Correct Hash (should succeed)
|
||||
if err := dbConn.Model(&model.Upload{}).Where("id = ?", uploadIncorrect.ID).Update("hash", correctHash).Error; err != nil {
|
||||
t.Fatalf("Update hash to correct value returned error: %v", err)
|
||||
}
|
||||
|
||||
// Run execution with correct hash should succeed
|
||||
result, err := (&MigrationHandler{}).Execute(ctx, payload)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute() with correct hash failed: %v", err)
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("Execute() result = nil, want non-nil")
|
||||
}
|
||||
|
||||
var migrated model.Upload
|
||||
if err := dbConn.First(&migrated, uploadIncorrect.ID).Error; err != nil {
|
||||
t.Fatalf("First(upload) returned error: %v", err)
|
||||
}
|
||||
if migrated.StorageDriver != string(storage.DriverS3) {
|
||||
t.Errorf("StorageDriver = %q, want %q", migrated.StorageDriver, storage.DriverS3)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user