refactor(layout): consolidate backend codebase into backend/ package and clean root directory

- Moved cmd/, core/, plugins/, pkg/, downstream/, and main.go into backend/ directory
- Batch updated all Go source files to import github.com/Rain-kl/Wavelet/backend/...
- Updated Makefile, scripts/swagger.sh, architecture guards, and platform skills
- Passed all quality gates (100% tests, 0 lint issues, clean build)
This commit is contained in:
ryan
2026-08-28 12:56:02 +08:00
parent 33b38f8687
commit 43dc97e48c
319 changed files with 912 additions and 1031 deletions
+145
View File
@@ -0,0 +1,145 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package cache provides in-process upload access-control caches.
package cache
import (
"context"
"encoding/json"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/backend/pkg/util"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/storage"
cachepkg "github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
database "github.com/Rain-kl/Wavelet/backend/plugins/infra/database"
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage/objectstore"
)
const fileAccessInvalidationChannel = "upload:file_access_invalidation"
var (
accessCacheOnce sync.Once
fileAccessWhitelistMu sync.RWMutex
fileAccessWhitelistTypes map[string]struct{}
fileAccessWhitelistValid bool
fileAccessWhitelistCheckedAt time.Time
)
// ResetAccessCaches clears in-process upload access caches.
func ResetAccessCaches() {
uploadstorage.ResetMigrationAccessCache()
fileAccessWhitelistMu.Lock()
fileAccessWhitelistValid = false
fileAccessWhitelistTypes = nil
fileAccessWhitelistMu.Unlock()
}
// PublishAccessCacheInvalidation broadcasts upload access cache eviction to all nodes.
func PublishAccessCacheInvalidation(ctx context.Context) {
if cachepkg.Redis != nil {
_ = cachepkg.Redis.Publish(ctx, fileAccessInvalidationChannel, "reset").Err()
}
}
func ensureAccessCacheListener() {
accessCacheOnce.Do(startAccessCacheInvalidationListener)
}
func startAccessCacheInvalidationListener() {
rdb := cachepkg.Redis
if rdb == nil {
return
}
util.Go(func() {
pubsub := rdb.Subscribe(
context.Background(),
objectstore.ConfigInvalidationChannel,
fileAccessInvalidationChannel,
)
defer func() {
_ = pubsub.Close()
}()
for range pubsub.Channel() {
ResetAccessCaches()
}
})
}
// IsFilePublic reports whether uploadType is in the public access whitelist.
func IsFilePublic(ctx context.Context, uploadType string) bool {
whitelist := loadFileAccessWhitelist(ctx)
_, ok := whitelist[strings.ToLower(uploadType)]
return ok
}
func loadFileAccessWhitelist(ctx context.Context) map[string]struct{} {
ensureAccessCacheListener()
fileAccessWhitelistMu.RLock()
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < time.Duration(shared.AccessCacheTTL)*time.Second {
types := fileAccessWhitelistTypes
fileAccessWhitelistMu.RUnlock()
return types
}
fileAccessWhitelistMu.RUnlock()
fileAccessWhitelistMu.Lock()
defer fileAccessWhitelistMu.Unlock()
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < time.Duration(shared.AccessCacheTTL)*time.Second {
return fileAccessWhitelistTypes
}
fileAccessWhitelistTypes = fetchFileAccessWhitelist(ctx)
fileAccessWhitelistValid = true
fileAccessWhitelistCheckedAt = time.Now()
return fileAccessWhitelistTypes
}
func fetchFileAccessWhitelist(ctx context.Context) map[string]struct{} {
whitelist := parseFileAccessWhitelist(ctx)
types := make(map[string]struct{}, len(whitelist))
for _, item := range whitelist {
types[strings.ToLower(item)] = struct{}{}
}
return types
}
func parseFileAccessWhitelist(ctx context.Context) []string {
var sc struct{ Value string }
err := database.DB(ctx).Table("w_system_configs").Where("key = ?", "file_access_whitelist").First(&sc).Error
if err != nil || sc.Value == "" {
return []string{shared.DefaultPublicUploadType}
}
var whitelist []string
if err := json.Unmarshal([]byte(sc.Value), &whitelist); err == nil && len(whitelist) > 0 {
return whitelist
}
whitelist = parseCommaSeparatedWhitelist(sc.Value)
if len(whitelist) == 0 {
return []string{shared.DefaultPublicUploadType}
}
return whitelist
}
func parseCommaSeparatedWhitelist(value string) []string {
parts := strings.Split(value, ",")
whitelist := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
whitelist = append(whitelist, part)
}
}
return whitelist
}
@@ -0,0 +1,89 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cache
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/backend/pkg/testhelper"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/storage"
)
func TestLoadMigrationAccessStateCachesResult(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
first := uploadstorage.LoadMigrationAccessState(ctx)
second := uploadstorage.LoadMigrationAccessState(ctx)
if first.ReadOnly != second.ReadOnly {
t.Fatalf("readOnly mismatch: first=%v second=%v", first.ReadOnly, second.ReadOnly)
}
if first.HasTarget != second.HasTarget {
t.Fatalf("hasTarget mismatch: first=%v second=%v", first.HasTarget, second.HasTarget)
}
}
func TestIsFilePublicUsesCachedWhitelist(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
if !IsFilePublic(ctx, "avatar") {
t.Fatal("expected avatar to be public by default")
}
if IsFilePublic(ctx, "attachment") {
t.Fatal("expected attachment to be private by default")
}
if !IsFilePublic(ctx, "AVATAR") {
t.Fatal("expected whitelist lookup to be case-insensitive")
}
}
func TestResetAccessCachesRefreshesWhitelist(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
if !IsFilePublic(ctx, "avatar") {
t.Fatal("expected seeded avatar whitelist before reset")
}
if err := dbConn.Table("w_system_configs").Where("key = ?", "file_access_whitelist").Update("value", `["attachment"]`).Error; err != nil {
t.Fatalf("update whitelist config: %v", err)
}
ResetAccessCaches()
if !IsFilePublic(ctx, "attachment") {
t.Fatal("expected attachment to be public after whitelist refresh")
}
if IsFilePublic(ctx, "avatar") {
t.Fatal("expected avatar to be private after whitelist refresh")
}
}
func TestAccessCacheTTLExpires(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
_ = loadFileAccessWhitelist(ctx)
fileAccessWhitelistMu.Lock()
fileAccessWhitelistCheckedAt = time.Now().Add(-time.Duration(shared.AccessCacheTTL)*time.Second - time.Second)
fileAccessWhitelistMu.Unlock()
// Should still work after TTL by reloading from config.
if !IsFilePublic(ctx, "avatar") {
t.Fatal("expected whitelist reload after TTL expiration")
}
}
+161
View File
@@ -0,0 +1,161 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cache
import (
"context"
"encoding/json"
"fmt"
"sync"
"github.com/Rain-kl/Wavelet/backend/pkg/cache/ram"
"github.com/Rain-kl/Wavelet/backend/pkg/util"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/models"
cachepkg "github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
database "github.com/Rain-kl/Wavelet/backend/plugins/infra/database"
)
const (
uploadMetaRedisCacheTTL = 30 * 60 // seconds
uploadMetaRAMMaximumSize = 4096
uploadMetaInvalidationChan = "upload:meta_invalidation"
)
type uploadMetaInvalidationMessage struct {
ID uint64 `json:"id"`
}
var (
uploadMetaRAM = ram.MustNew[uint64, models.Upload](ram.Options{MaximumSize: uploadMetaRAMMaximumSize})
uploadMetaListenerOnce sync.Once
uploadMetaListenerCtx context.Context
uploadMetaListenerCancel context.CancelFunc
uploadMetaListenerDone chan struct{}
)
func uploadMetaRedisKey(id uint64) string {
return fmt.Sprintf("upload:meta:%d", id)
}
func cloneUpload(u models.Upload) models.Upload {
return u
}
func ensureUploadMetaCacheListener() {
if cachepkg.Redis == nil {
return
}
uploadMetaListenerOnce.Do(startUploadMetaCacheInvalidationListener)
}
func startUploadMetaCacheInvalidationListener() {
uploadMetaListenerCtx, uploadMetaListenerCancel = context.WithCancel(context.Background())
uploadMetaListenerDone = make(chan struct{})
redisClient := cachepkg.Redis // 捕获当前客户端:goroutine 不读可变全局,避免与测试置空 cachepkg.Redis 竞争
util.Go(func() {
defer close(uploadMetaListenerDone)
pubsub := redisClient.Subscribe(uploadMetaListenerCtx, uploadMetaInvalidationChan)
defer func() {
_ = pubsub.Close()
}()
util.Go(func() {
<-uploadMetaListenerCtx.Done()
_ = pubsub.Close()
})
for msg := range pubsub.Channel() {
var payload uploadMetaInvalidationMessage
if err := json.Unmarshal([]byte(msg.Payload), &payload); err != nil || payload.ID == 0 {
uploadMetaRAM.InvalidateAll()
continue
}
uploadMetaRAM.Invalidate(payload.ID)
}
})
}
func publishUploadMetaRAMInvalidation(ctx context.Context, id uint64) {
if cachepkg.Redis == nil {
return
}
payload, err := json.Marshal(uploadMetaInvalidationMessage{ID: id})
if err != nil {
return
}
_ = cachepkg.Redis.Publish(ctx, uploadMetaInvalidationChan, payload).Err()
}
// GetUploadByID loads upload metadata from RAM, Redis, or the database.
func GetUploadByID(ctx context.Context, id uint64) (models.Upload, error) {
ensureUploadMetaCacheListener()
if u, ok := uploadMetaRAM.GetIfPresent(id); ok {
return cloneUpload(u), nil
}
key := uploadMetaRedisKey(id)
if cachepkg.Redis != nil {
var u models.Upload
if err := cachepkg.GetJSON(ctx, key, &u); err == nil {
uploadMetaRAM.Set(id, cloneUpload(u))
return u, nil
}
}
var u models.Upload
if err := database.DB(ctx).
Where("id = ? AND status IN (?, ?)", id, models.UploadStatusPending, models.UploadStatusUsed).
First(&u).Error; err != nil {
return models.Upload{}, err
}
SetUploadMetaCache(ctx, &u)
return u, nil
}
// SetUploadMetaCache populates RAM and Redis upload metadata caches.
func SetUploadMetaCache(ctx context.Context, u *models.Upload) {
ensureUploadMetaCacheListener()
if u == nil {
return
}
cloned := cloneUpload(*u)
uploadMetaRAM.Set(u.ID, cloned)
if cachepkg.Redis != nil {
_ = cachepkg.SetJSON(ctx, uploadMetaRedisKey(u.ID), cloned, uploadMetaRedisCacheTTL)
}
}
// InvalidateUploadMetaCache clears RAM and Redis upload metadata caches and notifies peer nodes.
func InvalidateUploadMetaCache(ctx context.Context, id uint64) {
ensureUploadMetaCacheListener()
uploadMetaRAM.Invalidate(id)
if cachepkg.Redis != nil {
_ = cachepkg.Redis.Del(ctx, cachepkg.PrefixedKey(uploadMetaRedisKey(id))).Err()
publishUploadMetaRAMInvalidation(ctx, id)
}
}
// ResetUploadMetaCacheForTest clears the in-process upload metadata RAM cache.
func ResetUploadMetaCacheForTest() {
uploadMetaRAM.InvalidateAll()
}
// StopUploadMetaCacheListener stops the Redis Pub/Sub subscription listener and resets the sync.Once guard.
func StopUploadMetaCacheListener() {
if uploadMetaListenerCancel != nil {
uploadMetaListenerCancel()
if uploadMetaListenerDone != nil {
<-uploadMetaListenerDone // 等待 goroutine 退出,保证之后置空 cachepkg.Redis 不再竞争
}
uploadMetaListenerCancel = nil
uploadMetaListenerDone = nil
}
uploadMetaListenerOnce = sync.Once{}
}
+287
View File
@@ -0,0 +1,287 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cache
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/Rain-kl/Wavelet/backend/pkg/testhelper"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/models"
cachepkg "github.com/Rain-kl/Wavelet/backend/plugins/infra/cache"
"gorm.io/gorm"
)
func init() {
testhelper.RegisterCleanup(func() {
StopUploadMetaCacheListener()
ResetUploadMetaCacheForTest()
})
}
func seedUpload(t *testing.T, dbConn *gorm.DB, upload models.Upload) {
t.Helper()
if err := dbConn.Create(&upload).Error; err != nil {
t.Fatalf("create upload: %v", err)
}
}
func TestGetUploadByIDLoadsFromDBAndPopulatesCache(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := models.Upload{
ID: 91001,
UserID: 1,
FileName: "cached.png",
FilePath: "cached.png",
FileSize: 12,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: models.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID: %v", err)
}
if got.ID != upload.ID || got.FileName != upload.FileName {
t.Fatalf("unexpected upload: %+v", got)
}
var redisUpload models.Upload
if err := cachepkg.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err != nil {
t.Fatalf("redis cache miss after DB load: %v", err)
}
if redisUpload.ID != upload.ID {
t.Fatalf("redis upload id mismatch: got=%d want=%d", redisUpload.ID, upload.ID)
}
if err := dbConn.Delete(&models.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
gotCached, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID from RAM cache: %v", err)
}
if gotCached.ID != upload.ID {
t.Fatalf("expected RAM cache hit for upload %d", upload.ID)
}
}
func TestGetUploadByIDReadsFromRedisWhenRAMEmpty(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := models.Upload{
ID: 91002,
UserID: 1,
FileName: "redis.png",
FilePath: "redis.png",
FileSize: 8,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: models.UploadStatusPending,
AccessMode: 0,
}
seedUpload(t, dbConn, upload)
SetUploadMetaCache(ctx, &upload)
ResetUploadMetaCacheForTest()
if err := dbConn.Delete(&models.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID from redis: %v", err)
}
if got.ID != upload.ID || got.FileName != upload.FileName {
t.Fatalf("unexpected upload from redis: %+v", got)
}
}
func TestInvalidateUploadMetaCacheClearsRAMAndRedis(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := models.Upload{
ID: 91003,
UserID: 1,
FileName: "invalidate.png",
FilePath: "invalidate.png",
FileSize: 4,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: models.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
SetUploadMetaCache(ctx, &upload)
InvalidateUploadMetaCache(ctx, upload.ID)
var redisUpload models.Upload
if err := cachepkg.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err == nil {
t.Fatal("expected redis cache to be invalidated")
}
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID after invalidate should reload from DB: %v", err)
}
if got.ID != upload.ID {
t.Fatalf("unexpected upload reloaded from DB: %+v", got)
}
}
func TestUploadMetaInvalidationPubSubClearsPeerRAM(t *testing.T) {
StopUploadMetaCacheListener()
defer StopUploadMetaCacheListener()
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := models.Upload{
ID: 91006,
UserID: 1,
FileName: "pubsub.png",
FilePath: "pubsub.png",
FileSize: 4,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: models.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
if _, err := GetUploadByID(ctx, upload.ID); err != nil {
t.Fatalf("GetUploadByID: %v", err)
}
time.Sleep(50 * time.Millisecond) // allow pub/sub listener to subscribe
if err := dbConn.Delete(&models.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
if _, err := GetUploadByID(ctx, upload.ID); err != nil {
t.Fatalf("expected cache hit before pub/sub invalidation: %v", err)
}
payload, err := json.Marshal(uploadMetaInvalidationMessage{ID: upload.ID})
if err != nil {
t.Fatalf("marshal invalidation payload: %v", err)
}
if err := cachepkg.Redis.Publish(ctx, uploadMetaInvalidationChan, string(payload)).Err(); err != nil {
t.Fatalf("publish invalidation: %v", err)
}
deadline := time.Now().Add(2 * time.Second)
ramCleared := false
for time.Now().Before(deadline) {
if _, ok := uploadMetaRAM.GetIfPresent(upload.ID); !ok {
ramCleared = true
break
}
time.Sleep(20 * time.Millisecond)
}
if !ramCleared {
t.Fatal("expected peer RAM cache to be cleared by pub/sub")
}
if err := cachepkg.Redis.Del(ctx, cachepkg.PrefixedKey(uploadMetaRedisKey(upload.ID))).Err(); err != nil {
t.Fatalf("delete redis cache: %v", err)
}
if _, err := GetUploadByID(ctx, upload.ID); err == nil {
t.Fatal("expected cache miss after pub/sub RAM eviction and redis delete")
}
}
func TestGetUploadByIDSkipsDeletedUploads(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := models.Upload{
ID: 91004,
UserID: 1,
FileName: "deleted.png",
FilePath: "deleted.png",
FileSize: 4,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: models.UploadStatusDeleted,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
if _, err := GetUploadByID(ctx, upload.ID); err == nil {
t.Fatal("expected error for deleted upload")
}
}
func TestGetUploadByIDWorksWithRedisDisabled(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
redisClient := cachepkg.Redis
cachepkg.Redis = nil
t.Cleanup(func() {
cachepkg.Redis = redisClient
StopUploadMetaCacheListener()
})
ctx := context.Background()
upload := models.Upload{
ID: 91005,
UserID: 1,
FileName: "ram-only.png",
FilePath: "ram-only.png",
FileSize: 6,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: models.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID without redis: %v", err)
}
if got.ID != upload.ID {
t.Fatalf("unexpected upload: %+v", got)
}
if err := dbConn.Delete(&models.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
gotCached, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID from RAM without redis: %v", err)
}
if gotCached.ID != upload.ID {
t.Fatal("expected RAM cache hit when redis is disabled")
}
}