refactor(layout): consolidate backend codebase into backend/ package and clean root directory

- Moved cmd/, core/, plugins/, pkg/, downstream/, and main.go into backend/ directory
- Batch updated all Go source files to import github.com/Rain-kl/Wavelet/backend/...
- Updated Makefile, scripts/swagger.sh, architecture guards, and platform skills
- Passed all quality gates (100% tests, 0 lint issues, clean build)
This commit is contained in:
ryan
2026-08-28 12:56:02 +08:00
parent 33b38f8687
commit 43dc97e48c
319 changed files with 912 additions and 1031 deletions
@@ -0,0 +1,322 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package filesrv serves uploaded files with access control and image compression.
package filesrv
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"github.com/Rain-kl/Wavelet/backend/core/contracts"
"github.com/Rain-kl/Wavelet/backend/pkg/response"
pkgutil "github.com/Rain-kl/Wavelet/backend/pkg/util"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/storage"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/util"
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage/diskcache"
"github.com/Rain-kl/Wavelet/backend/pkg/logger"
"github.com/gin-gonic/gin"
"golang.org/x/sync/singleflight"
"gorm.io/gorm"
)
var compressedImageFlight singleflight.Group
type compressedImageCacheResult struct {
bytes []byte
cached bool
err error
}
type fileTypeCategory string
const (
fileTypeImage fileTypeCategory = "image"
fileTypeVideo fileTypeCategory = "video"
fileTypeAudio fileTypeCategory = "audio"
fileTypeOther fileTypeCategory = "other"
)
// ServeFileByID 根据 ID 获取并提供已上传的文件
// @Summary 获取已上传文件
// @Description 根据文件 ID 获取并提供已上传的临时或正式文件,若配置了缓存则优先走本地缓存,否则从 S3 等后端存储读取并流式返回
// @Tags upload
// @Produce octet-stream
// @Param id path string true "文件 ID"
// @Param quality query string false "图片质量 (low, medium, high, origin),默认为 origin"
// @Success 200 {file} file "成功获取文件内容"
// @Failure 400 {object} response.Any "文件 ID 格式错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "文件未找到"
// @Failure 500 {object} response.Any "服务内部错误"
// @Router /f/{id} [get]
func ServeFileByID(c *gin.Context) {
upload, err := GetUploadRecordByID(c)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
response.AbortNotFound(c, "文件记录未找到")
return
}
if _, ok := err.(*strconv.NumError); ok {
response.AbortBadRequest(c, "无效的上传ID")
return
}
response.AbortInternal(c, "服务器内部错误")
return
}
if err := CheckFileAccessPermission(c, upload); err != nil {
response.AbortUnauthorized(c, shared.ErrUnauthorized)
return
}
ServeUpload(c, upload)
}
// GetUploadRecordByID 从请求路径参数中解析文件 ID 并从数据库中检索处于 Pending 或 Used 状态的上传记录。
func GetUploadRecordByID(c *gin.Context) (*models.Upload, error) {
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Security-Policy", "sandbox")
idStr := c.Param("id")
uploadID, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
return nil, err
}
upload, err := cache.GetUploadByID(c.Request.Context(), uploadID)
if err != nil {
return nil, err
}
return &upload, nil
}
func getFileTypeCategory(upload *models.Upload) fileTypeCategory {
mime := strings.ToLower(upload.MimeType)
ext := strings.ToLower(upload.Extension)
if strings.HasPrefix(mime, "image/") || util.IsImageExtension(ext) {
return fileTypeImage
}
if strings.HasPrefix(mime, "video/") {
return fileTypeVideo
}
if strings.HasPrefix(mime, "audio/") {
return fileTypeAudio
}
return fileTypeOther
}
// ServeUpload 将已存在的文件内容读取并流式响应给客户端。
func ServeUpload(c *gin.Context, upload *models.Upload) {
setCacheHeaders(c, upload)
category := getFileTypeCategory(upload)
quality := util.NormalizeImageQuality(c.Query("quality"))
switch category {
case fileTypeImage:
if quality != shared.ImageQualityOrigin {
serveCompressedImage(c, upload, quality)
return
}
fallthrough
default:
serveOriginalWithConditionalCheck(c, upload)
}
}
func setCacheHeaders(c *gin.Context, upload *models.Upload) {
if cache.IsFilePublic(c.Request.Context(), upload.Type) {
c.Header("Cache-Control", "public, max-age=31536000")
} else {
c.Header("Cache-Control", "private, no-cache")
}
}
func serveOriginalWithConditionalCheck(c *gin.Context, upload *models.Upload) {
etag := fmt.Sprintf(`W/"%s"`, upload.Hash)
c.Header("ETag", etag)
if c.GetHeader("If-None-Match") == etag {
c.AbortWithStatus(http.StatusNotModified)
return
}
serveOriginal(c, upload)
}
func serveCompressedImage(c *gin.Context, upload *models.Upload, quality string) {
etag := fmt.Sprintf(`W/"%s-%s"`, upload.Hash, quality)
c.Header("ETag", etag)
if c.GetHeader("If-None-Match") == etag {
c.AbortWithStatus(http.StatusNotModified)
return
}
webpBytes, hit, err := EnsureCompressedImageCache(c.Request.Context(), upload, quality)
if hit {
c.Header("X-Cache", "HIT")
} else {
c.Header("X-Cache", "MISS")
}
if err != nil {
if len(webpBytes) > 0 {
logger.WarnF(c.Request.Context(), "failed to cache compressed image: %v", err)
c.Data(http.StatusOK, "image/webp", webpBytes)
return
}
logger.ErrorF(c.Request.Context(), "failed to prepare compressed image cache: %v", err)
serveOriginal(c, upload)
return
}
c.Data(http.StatusOK, "image/webp", webpBytes)
}
// EnsureCompressedImageCache returns cached or freshly generated WebP bytes for an upload.
func EnsureCompressedImageCache(
ctx context.Context,
upload *models.Upload,
quality string,
) ([]byte, bool, error) {
cacheStore := diskcache.GetGlobalCache()
cacheKey := ImageCompressionCacheKey(upload, quality)
webpBytes, err := cacheStore.Get(cacheKey)
if err == nil {
return webpBytes, true, nil
}
if !errors.Is(err, diskcache.ErrCacheMiss) {
return nil, false, fmt.Errorf("read compressed image cache: %w", err)
}
result, err, _ := compressedImageFlight.Do(cacheKey, func() (any, error) {
return generateCompressedImageCache(ctx, upload, quality, cacheKey)
})
if err != nil {
return nil, false, err
}
res := result.(compressedImageCacheResult)
return res.bytes, res.cached, res.err
}
func generateCompressedImageCache(
ctx context.Context,
upload *models.Upload,
quality string,
cacheKey string,
) (compressedImageCacheResult, error) {
cacheStore := diskcache.GetGlobalCache()
webpBytes, err := cacheStore.Get(cacheKey)
if err == nil {
return compressedImageCacheResult{bytes: webpBytes, cached: true}, nil
}
if !errors.Is(err, diskcache.ErrCacheMiss) {
return compressedImageCacheResult{}, fmt.Errorf("read compressed image cache: %w", err)
}
origBytes, err := getOriginalFileBytes(ctx, upload)
if err != nil {
return compressedImageCacheResult{}, fmt.Errorf("read original image: %w", err)
}
webpBytes, err = util.CompressImageToWebP(bytes.NewReader(origBytes), quality)
if err != nil {
return compressedImageCacheResult{}, fmt.Errorf("compress image to WebP: %w", err)
}
if err := cacheStore.Set(cacheKey, webpBytes, diskcache.NoExpiration); err != nil {
return compressedImageCacheResult{
bytes: webpBytes,
err: fmt.Errorf("write compressed image cache: %w", err),
}, nil
}
return compressedImageCacheResult{bytes: webpBytes}, nil
}
// ImageCompressionCacheKey returns the disk cache key for a compressed upload image.
func ImageCompressionCacheKey(upload *models.Upload, quality string) string {
return fmt.Sprintf(
"upload_webp_v1_%d_%d_%d_%s_%s",
upload.ID,
upload.UpdatedAt.UnixNano(),
upload.FileSize,
upload.Hash,
quality,
)
}
func serveOriginal(c *gin.Context, upload *models.Upload) {
obj, err := uploadstorage.OpenStoredObject(c.Request.Context(), upload)
if err != nil {
response.AbortNotFound(c, "文件未找到")
return
}
defer func() { _ = obj.Body.Close() }()
c.DataFromReader(http.StatusOK, obj.ContentLength, obj.ContentType, obj.Body, nil)
}
func getOriginalFileBytes(ctx context.Context, upload *models.Upload) ([]byte, error) {
obj, err := uploadstorage.OpenStoredObject(ctx, upload)
if err != nil {
return nil, err
}
defer func() { _ = obj.Body.Close() }()
return io.ReadAll(obj.Body)
}
func checkPrivateFileOwner(c *gin.Context, ownerID uint64) error {
var currUserID uint64
var isAdmin bool
if u, ok := pkgutil.GetFromContext[*contracts.UserDTO](c, contracts.AuthUserObjKey); ok && u != nil {
currUserID = u.ID
isAdmin = u.IsAdmin
} else {
u, err := auth.GetUserFromRequest(c)
if err != nil {
return err
}
currUserID = u.ID
isAdmin = u.IsAdmin
}
if isAdmin {
return nil
}
if currUserID != ownerID {
return errors.New("forbidden: cross-user access denied")
}
return nil
}
// CheckFileAccessPermission 校验文件是否可以被当前请求访问
func CheckFileAccessPermission(c *gin.Context, upload *models.Upload) error {
if upload.AccessMode == 0 {
return checkPrivateFileOwner(c, upload.UserID)
}
if !cache.IsFilePublic(c.Request.Context(), upload.Type) {
if _, ok := pkgutil.GetFromContext[*contracts.UserDTO](c, contracts.AuthUserObjKey); !ok {
if _, err := auth.GetUserFromRequest(c); err != nil {
return err
}
}
}
return nil
}
@@ -0,0 +1,380 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package filesrv
import (
"bytes"
"crypto/sha256"
"encoding/json"
"fmt"
"image"
"image/color"
"image/png"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gin-contrib/sessions"
"github.com/gin-contrib/sessions/cookie"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"github.com/Rain-kl/Wavelet/backend/core/contracts"
"github.com/Rain-kl/Wavelet/backend/pkg/response"
"github.com/Rain-kl/Wavelet/backend/pkg/testhelper"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/models"
"github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/shared"
uploadutil "github.com/Rain-kl/Wavelet/backend/plugins/domain/upload/util"
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage/diskcache"
"github.com/Rain-kl/Wavelet/backend/plugins/infra/storage/objectstore"
)
func init() {
testhelper.RegisterCleanup(cache.ResetUploadMetaCacheForTest)
}
func TestServeFileByIDAccessControl(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
cache.ResetAccessCaches()
tempDir := t.TempDir()
configureLocalStorageRoot(t, dbConn, tempDir)
// Create a user in DB
user := contracts.UserDTO{
ID: 12345,
Username: "file_test_user",
IsActive: true,
}
if err := dbConn.Table("w_users").Create(&user).Error; err != nil {
t.Fatalf("failed to create user: %v", err)
}
// Create an access token for this user
tokenStr := "test-secret-token-123"
tokenHash := fmt.Sprintf("%x", sha256.Sum256([]byte(tokenStr)))
tokenRecord := map[string]any{
"user_id": user.ID,
"name": "test_token",
"token_hash": tokenHash,
"masked_token": "test-***",
}
if err := dbConn.Table("w_access_tokens").Create(&tokenRecord).Error; err != nil {
t.Fatalf("failed to create token: %v", err)
}
// Create two files: one in whitelist (avatar), one not in whitelist (attachment)
avatarFile := models.Upload{
ID: 8001,
UserID: user.ID,
FileName: "avatar.png",
FilePath: "avatar.png",
FileSize: 5,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: models.UploadStatusUsed,
AccessMode: 1,
}
attachmentFile := models.Upload{
ID: 8002,
UserID: user.ID,
FileName: "doc.pdf",
FilePath: "doc.pdf",
FileSize: 5,
MimeType: "application/pdf",
Extension: "pdf",
Type: "attachment",
Status: models.UploadStatusUsed,
AccessMode: 1,
}
if err := os.WriteFile(filepath.Join(tempDir, "avatar.png"), []byte("image"), 0644); err != nil {
t.Fatalf("failed to write avatar file: %v", err)
}
if err := os.WriteFile(filepath.Join(tempDir, "doc.pdf"), []byte("bytes"), 0644); err != nil {
t.Fatalf("failed to write attachment file: %v", err)
}
dbConn.Create(&avatarFile)
dbConn.Create(&attachmentFile)
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(response.ErrorHandlerMiddleware())
store := cookie.NewStore([]byte("secret"))
r.Use(sessions.Sessions("wavelet_session_id", store))
r.GET("/f/:id", ServeFileByID)
t.Run("public access allowed for whitelist type (avatar)", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8001", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200 for public file, got %d", w.Code)
}
if w.Body.String() != "image" {
t.Fatalf("expected body 'image', got '%s'", w.Body.String())
}
})
t.Run("public access rejected for non-whitelist type (attachment)", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8002", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized {
t.Fatalf("expected status 401 for private file without auth, got %d", w.Code)
}
})
t.Run("authenticated access allowed for non-whitelist type (attachment)", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8002", nil)
req.Header.Set("Authorization", "Bearer "+tokenStr)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200 for authenticated request, got %d", w.Code)
}
if w.Body.String() != "bytes" {
t.Fatalf("expected body 'bytes', got '%s'", w.Body.String())
}
})
t.Run("non-existent file returns 404", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/99999", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
t.Fatalf("expected status 404 for non-existent file, got %d", w.Code)
}
})
t.Run("invalid id format returns 400", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/invalid-id", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("expected status 400 for invalid ID, got %d", w.Code)
}
})
}
func TestServeFileByIDImageCompression(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
cache.ResetAccessCaches()
tempDir := t.TempDir()
configureLocalStorageRoot(t, dbConn, tempDir)
cache := diskcache.GetGlobalCache()
if err := cache.Clear(); err != nil {
t.Fatalf("failed to clear disk cache before test: %v", err)
}
defer func() {
if err := cache.Clear(); err != nil {
t.Errorf("failed to clear disk cache after test: %v", err)
}
}()
// Create test user
user := contracts.UserDTO{
ID: 555,
Username: "compress_tester",
IsActive: true,
}
dbConn.Table("w_users").Create(&user)
// Create a 1x1 pixel PNG image
img := image.NewRGBA(image.Rect(0, 0, 1, 1))
img.Set(0, 0, color.RGBA{R: 255, G: 0, B: 0, A: 255})
var pngBuf bytes.Buffer
if err := png.Encode(&pngBuf, img); err != nil {
t.Fatalf("failed to encode test png: %v", err)
}
filePath := filepath.Join(tempDir, "test_image.png")
if err := os.WriteFile(filePath, pngBuf.Bytes(), 0644); err != nil {
t.Fatalf("failed to write test png: %v", err)
}
// Save upload record to DB
uploadRecord := models.Upload{
ID: 3001,
UserID: user.ID,
FileName: "test_image.png",
FilePath: "test_image.png",
FileSize: int64(pngBuf.Len()),
MimeType: "image/png",
Extension: "png",
Type: "avatar", // Whitelisted by default
Status: models.UploadStatusUsed,
AccessMode: 1,
}
dbConn.Create(&uploadRecord)
// Setup Router
gin.SetMode(gin.TestMode)
r := gin.New()
r.GET("/f/:id", ServeFileByID)
t.Run("serve original file without compress parameter", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
// Content-Type should be image/png (default local serving type)
if w.Header().Get("Content-Type") != "image/png" {
t.Errorf("expected Content-Type image/png, got %s", w.Header().Get("Content-Type"))
}
if w.Header().Get("X-Cache") != "" {
t.Errorf("expected no X-Cache header for original file, got %s", w.Header().Get("X-Cache"))
}
if w.Header().Get("ETag") == "" {
t.Errorf("expected ETag header for original file")
}
})
t.Run("first request with quality=medium produces cache MISS and converts to WebP", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
if w.Header().Get("Content-Type") != "image/webp" {
t.Errorf("expected Content-Type image/webp, got %s", w.Header().Get("Content-Type"))
}
if w.Header().Get("X-Cache") != "MISS" {
t.Errorf("expected X-Cache MISS on first compress request, got %s", w.Header().Get("X-Cache"))
}
if w.Header().Get("ETag") == "" {
t.Errorf("expected ETag header")
}
if len(w.Body.Bytes()) == 0 {
t.Errorf("expected non-empty body")
}
})
t.Run("second request with quality=medium produces cache HIT", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
if w.Header().Get("Content-Type") != "image/webp" {
t.Errorf("expected Content-Type image/webp, got %s", w.Header().Get("Content-Type"))
}
if w.Header().Get("X-Cache") != "HIT" {
t.Errorf("expected X-Cache HIT on second compress request, got %s", w.Header().Get("X-Cache"))
}
})
t.Run("request with quality=origin behaves like original request", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=origin", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
if w.Header().Get("Content-Type") != "image/png" {
t.Errorf("expected Content-Type image/png, got %s", w.Header().Get("Content-Type"))
}
if w.Header().Get("X-Cache") != "" {
t.Errorf("expected no X-Cache header for origin quality, got %s", w.Header().Get("X-Cache"))
}
})
t.Run("conditional GET with matching If-None-Match returns 304", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
etag := w.Header().Get("ETag")
if etag == "" {
t.Fatalf("expected ETag header from initial request")
}
// Second request with If-None-Match
req2, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
req2.Header.Set("If-None-Match", etag)
w2 := httptest.NewRecorder()
r.ServeHTTP(w2, req2)
if w2.Code != http.StatusNotModified {
t.Fatalf("expected status 304 Not Modified, got %d", w2.Code)
}
if w2.Body.Len() != 0 {
t.Errorf("expected empty body on 304 response, got %d bytes", w2.Body.Len())
}
})
}
func TestNormalizeImageQuality(t *testing.T) {
tests := []struct {
name string
quality string
want string
}{
{name: "empty quality returns origin", quality: "", want: shared.ImageQualityOrigin},
{name: "origin returns origin", quality: "origin", want: shared.ImageQualityOrigin},
{name: "ORIGIN case-insensitive returns origin", quality: "ORIGIN", want: shared.ImageQualityOrigin},
{name: "low returns low", quality: "low", want: shared.ImageQualityLow},
{name: "LOW returns low", quality: "LOW", want: shared.ImageQualityLow},
{name: "medium returns medium", quality: "medium", want: shared.ImageQualityMedium},
{name: "high returns high", quality: "high", want: shared.ImageQualityHigh},
{name: "unknown quality defaults to origin", quality: "ultra_hd", want: shared.ImageQualityOrigin},
{name: "whitespace padded quality is trimmed", quality: " medium ", want: shared.ImageQualityMedium},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := uploadutil.NormalizeImageQuality(tt.quality); got != tt.want {
t.Errorf("NormalizeImageQuality(%q) = %q, want %q", tt.quality, got, tt.want)
}
})
}
}
func configureLocalStorageRoot(t *testing.T, dbConn *gorm.DB, tempDir string) {
var sc struct {
Key string
Value string
}
if err := dbConn.Table("w_system_configs").Where("key = ?", "storage_config").First(&sc).Error; err != nil {
t.Fatalf("failed to find storage config: %v", err)
}
var cfg objectstore.Config
if err := json.Unmarshal([]byte(sc.Value), &cfg); err != nil {
t.Fatalf("failed to unmarshal storage config: %v", err)
}
cfg.Local.Root = tempDir
newVal, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("failed to marshal storage config: %v", err)
}
sc.Value = string(newVal)
if err := dbConn.Table("w_system_configs").Where("key = ?", "storage_config").Update("value", sc.Value).Error; err != nil {
t.Fatalf("failed to save storage config: %v", err)
}
objectstore.ResetCache()
}