refactor(api): unify error handling and split upload/oauth god modules

Replace c.JSON(200, response.Err) and middleware gin.H bypasses with
response.Abort* helpers so errors flow through Gin Error chain and
ErrorHandlerMiddleware for OTel trace correlation.

Split oauth/sources.go into domain-focused files and decompose upload
into handler/filesrv/stats/task/cache/storage/util subpackages with a
root facade preserving existing import paths.
This commit is contained in:
ryan
2026-06-18 10:51:25 +08:00
parent f826807cf1
commit 9af84c8ed6
62 changed files with 2040 additions and 1696 deletions
+316
View File
@@ -0,0 +1,316 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package filesrv serves uploaded files with access control and image compression.
package filesrv
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/apps/upload/cache"
"github.com/Rain-kl/Wavelet/internal/apps/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/internal/apps/upload/storage"
"github.com/Rain-kl/Wavelet/internal/apps/upload/util"
"github.com/Rain-kl/Wavelet/internal/common"
"github.com/Rain-kl/Wavelet/internal/common/response"
"github.com/Rain-kl/Wavelet/internal/db"
"github.com/Rain-kl/Wavelet/internal/diskcache"
"github.com/Rain-kl/Wavelet/internal/model"
apputil "github.com/Rain-kl/Wavelet/internal/util"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
"golang.org/x/sync/singleflight"
"gorm.io/gorm"
)
var compressedImageFlight singleflight.Group
type compressedImageCacheResult struct {
bytes []byte
cached bool
err error
}
type fileTypeCategory string
const (
fileTypeImage fileTypeCategory = "image"
fileTypeVideo fileTypeCategory = "video"
fileTypeAudio fileTypeCategory = "audio"
fileTypeOther fileTypeCategory = "other"
)
// ServeFileByID 根据 ID 获取并提供已上传的文件
// @Summary 获取已上传文件
// @Description 根据文件 ID 获取并提供已上传的临时或正式文件,若配置了缓存则优先走本地缓存,否则从 S3 等后端存储读取并流式返回
// @Tags upload
// @Produce octet-stream
// @Param id path string true "文件 ID"
// @Param quality query string false "图片质量 (low, medium, high, origin),默认为 origin"
// @Success 200 {file} file "成功获取文件内容"
// @Failure 400 {object} response.Any "文件 ID 格式错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "文件未找到"
// @Failure 500 {object} response.Any "服务内部错误"
// @Router /f/{id} [get]
func ServeFileByID(c *gin.Context) {
upload, err := GetUploadRecordByID(c)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
c.AbortWithStatus(http.StatusNotFound)
return
}
if _, ok := err.(*strconv.NumError); ok {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid upload ID"})
return
}
c.AbortWithStatus(http.StatusInternalServerError)
return
}
if err := CheckFileAccessPermission(c, upload); err != nil {
response.AbortUnauthorized(c, common.UnAuthorized)
return
}
ServeUpload(c, upload)
}
// GetUploadRecordByID 从请求路径参数中解析文件 ID 并从数据库中检索处于 Pending 或 Used 状态的上传记录。
func GetUploadRecordByID(c *gin.Context) (*model.Upload, error) {
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Security-Policy", "sandbox")
idStr := c.Param("id")
uploadID, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
return nil, err
}
var upload model.Upload
if err := db.DB(c.Request.Context()).
Where("id = ? AND status IN (?, ?)", uploadID, model.UploadStatusPending, model.UploadStatusUsed).
First(&upload).Error; err != nil {
return nil, err
}
return &upload, nil
}
func getFileTypeCategory(upload *model.Upload) fileTypeCategory {
mime := strings.ToLower(upload.MimeType)
ext := strings.ToLower(upload.Extension)
if strings.HasPrefix(mime, "image/") || util.IsImageExtension(ext) {
return fileTypeImage
}
if strings.HasPrefix(mime, "video/") {
return fileTypeVideo
}
if strings.HasPrefix(mime, "audio/") {
return fileTypeAudio
}
return fileTypeOther
}
// ServeUpload 将已存在的文件内容读取并流式响应给客户端。
func ServeUpload(c *gin.Context, upload *model.Upload) {
setCacheHeaders(c, upload)
category := getFileTypeCategory(upload)
quality := util.NormalizeImageQuality(c.Query("quality"))
switch category {
case fileTypeImage:
if quality != shared.ImageQualityOrigin {
serveCompressedImage(c, upload, quality)
return
}
fallthrough
default:
serveOriginalWithConditionalCheck(c, upload)
}
}
func setCacheHeaders(c *gin.Context, upload *model.Upload) {
if cache.IsFilePublic(c.Request.Context(), upload.Type) {
c.Header("Cache-Control", "public, max-age=31536000")
} else {
c.Header("Cache-Control", "private, no-cache")
}
}
func serveOriginalWithConditionalCheck(c *gin.Context, upload *model.Upload) {
etag := fmt.Sprintf(`W/"%s"`, upload.Hash)
c.Header("ETag", etag)
if c.GetHeader("If-None-Match") == etag {
c.AbortWithStatus(http.StatusNotModified)
return
}
serveOriginal(c, upload)
}
func serveCompressedImage(c *gin.Context, upload *model.Upload, quality string) {
etag := fmt.Sprintf(`W/"%s-%s"`, upload.Hash, quality)
c.Header("ETag", etag)
if c.GetHeader("If-None-Match") == etag {
c.AbortWithStatus(http.StatusNotModified)
return
}
webpBytes, _, err := EnsureCompressedImageCache(c.Request.Context(), upload, quality)
if err != nil {
if len(webpBytes) > 0 {
logger.WarnF(c.Request.Context(), "failed to cache compressed image: %v", err)
c.Data(http.StatusOK, "image/webp", webpBytes)
return
}
logger.ErrorF(c.Request.Context(), "failed to prepare compressed image cache: %v", err)
serveOriginal(c, upload)
return
}
c.Data(http.StatusOK, "image/webp", webpBytes)
}
// EnsureCompressedImageCache returns cached or freshly generated WebP bytes for an upload.
func EnsureCompressedImageCache(
ctx context.Context,
upload *model.Upload,
quality string,
) ([]byte, bool, error) {
cacheStore := diskcache.GetGlobalCache()
cacheKey := ImageCompressionCacheKey(upload, quality)
webpBytes, err := cacheStore.Get(cacheKey)
if err == nil {
return webpBytes, true, nil
}
if !errors.Is(err, diskcache.ErrCacheMiss) {
return nil, false, fmt.Errorf("read compressed image cache: %w", err)
}
result, err, _ := compressedImageFlight.Do(cacheKey, func() (any, error) {
return generateCompressedImageCache(ctx, upload, quality, cacheKey)
})
if err != nil {
return nil, false, err
}
res := result.(compressedImageCacheResult)
return res.bytes, res.cached, res.err
}
func generateCompressedImageCache(
ctx context.Context,
upload *model.Upload,
quality string,
cacheKey string,
) (compressedImageCacheResult, error) {
cacheStore := diskcache.GetGlobalCache()
webpBytes, err := cacheStore.Get(cacheKey)
if err == nil {
return compressedImageCacheResult{bytes: webpBytes, cached: true}, nil
}
if !errors.Is(err, diskcache.ErrCacheMiss) {
return compressedImageCacheResult{}, fmt.Errorf("read compressed image cache: %w", err)
}
origBytes, err := getOriginalFileBytes(ctx, upload)
if err != nil {
return compressedImageCacheResult{}, fmt.Errorf("read original image: %w", err)
}
webpBytes, err = util.CompressImageToWebP(bytes.NewReader(origBytes), quality)
if err != nil {
return compressedImageCacheResult{}, fmt.Errorf("compress image to WebP: %w", err)
}
if err := cacheStore.Set(cacheKey, webpBytes, diskcache.NoExpiration); err != nil {
return compressedImageCacheResult{
bytes: webpBytes,
err: fmt.Errorf("write compressed image cache: %w", err),
}, nil
}
return compressedImageCacheResult{bytes: webpBytes}, nil
}
// ImageCompressionCacheKey returns the disk cache key for a compressed upload image.
func ImageCompressionCacheKey(upload *model.Upload, quality string) string {
return fmt.Sprintf(
"upload_webp_v1_%d_%d_%d_%s_%s",
upload.ID,
upload.UpdatedAt.UnixNano(),
upload.FileSize,
upload.Hash,
quality,
)
}
func serveOriginal(c *gin.Context, upload *model.Upload) {
obj, err := uploadstorage.OpenStoredObject(c.Request.Context(), upload)
if err != nil {
c.AbortWithStatus(http.StatusNotFound)
return
}
defer func() { _ = obj.Body.Close() }()
c.DataFromReader(http.StatusOK, obj.ContentLength, obj.ContentType, obj.Body, nil)
}
func getOriginalFileBytes(ctx context.Context, upload *model.Upload) ([]byte, error) {
obj, err := uploadstorage.OpenStoredObject(ctx, upload)
if err != nil {
return nil, err
}
defer func() { _ = obj.Body.Close() }()
return io.ReadAll(obj.Body)
}
func checkPrivateFileOwner(c *gin.Context, ownerID uint64) error {
var currUser *model.User
var err error
if u, ok := apputil.GetFromContext[*model.User](c, oauth.UserObjKey); ok && u != nil {
currUser = u
} else {
currUser, err = oauth.GetUserFromRequest(c)
if err != nil {
return err
}
}
if currUser.IsAdmin {
return nil
}
if currUser.ID != ownerID {
return errors.New("forbidden: cross-user access denied")
}
return nil
}
// CheckFileAccessPermission 校验文件是否可以被当前请求访问
func CheckFileAccessPermission(c *gin.Context, upload *model.Upload) error {
if upload.AccessMode == 0 {
return checkPrivateFileOwner(c, upload.UserID)
}
if !cache.IsFilePublic(c.Request.Context(), upload.Type) {
if _, ok := apputil.GetFromContext[*model.User](c, oauth.UserObjKey); !ok {
if _, err := oauth.GetUserFromRequest(c); err != nil {
return err
}
}
}
return nil
}
@@ -0,0 +1,349 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package filesrv
import (
"bytes"
"encoding/json"
"image"
"image/color"
"image/png"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/Rain-kl/Wavelet/internal/apps/upload/cache"
"github.com/Rain-kl/Wavelet/internal/apps/upload/shared"
"github.com/Rain-kl/Wavelet/internal/apps/upload/util"
"github.com/Rain-kl/Wavelet/internal/common"
"github.com/Rain-kl/Wavelet/internal/common/response"
"github.com/Rain-kl/Wavelet/internal/diskcache"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/gin-contrib/sessions"
"github.com/gin-contrib/sessions/cookie"
"github.com/gin-gonic/gin"
)
func TestServeFileByIDAccessControl(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
cache.ResetAccessCaches()
// Ensure uploads dir is cleaned up
defer func() { _ = os.RemoveAll("uploads") }()
// Create a user in DB
user := model.User{
ID: 12345,
Username: "file_test_user",
IsActive: true,
}
if err := dbConn.Create(&user).Error; err != nil {
t.Fatalf("failed to create user: %v", err)
}
// Create an access token for this user
tokenStr := "test-secret-token-123"
tokenHash := model.HashToken(tokenStr)
tokenRecord := model.AccessToken{
UserID: user.ID,
Name: "test_token",
TokenHash: tokenHash,
}
if err := dbConn.Create(&tokenRecord).Error; err != nil {
t.Fatalf("failed to create token: %v", err)
}
// Create two files: one in whitelist (avatar), one not in whitelist (attachment)
avatarFile := model.Upload{
ID: 8001,
UserID: user.ID,
FileName: "avatar.png",
FilePath: "uploads/avatar.png",
FileSize: 5,
MimeType: "image/png",
Extension: "png",
StorageDriver: "local",
Type: "avatar",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
attachmentFile := model.Upload{
ID: 8002,
UserID: user.ID,
FileName: "doc.pdf",
FilePath: "uploads/doc.pdf",
FileSize: 5,
MimeType: "application/pdf",
Extension: "pdf",
StorageDriver: "local",
Type: "attachment",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
_ = os.MkdirAll("uploads", 0755)
_ = os.WriteFile(avatarFile.FilePath, []byte("image"), 0644)
_ = os.WriteFile(attachmentFile.FilePath, []byte("bytes"), 0644)
dbConn.Create(&avatarFile)
dbConn.Create(&attachmentFile)
// Set up router
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(response.ErrorHandlerMiddleware())
store := cookie.NewStore([]byte("secret"))
r.Use(sessions.Sessions("test_session", store))
r.GET("/f/:id", ServeFileByID)
t.Run("whitelisted file type (avatar) accessed without authentication", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8001", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("expected 200, got %d. Body: %s", w.Code, w.Body.String())
}
if w.Body.String() != "image" {
t.Errorf("expected 'image', got %q", w.Body.String())
}
})
t.Run("non-whitelisted file type (attachment) accessed without authentication returns 401", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8002", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized {
t.Errorf("expected 401, got %d. Body: %s", w.Code, w.Body.String())
}
var body map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("failed to parse JSON: %v", err)
}
if body["error_msg"] != common.UnAuthorized {
t.Errorf("expected error_msg %q, got %v", common.UnAuthorized, body["error_msg"])
}
})
t.Run("non-whitelisted file type (attachment) accessed with valid token succeeds", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8002", nil)
req.Header.Set("X-Access-Token", tokenStr)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("expected 200, got %d. Body: %s", w.Code, w.Body.String())
}
if w.Body.String() != "bytes" {
t.Errorf("expected 'bytes', got %q", w.Body.String())
}
})
t.Run("accessing non-existent file returns 404", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/9999", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
t.Errorf("expected 404, got %d", w.Code)
}
})
}
func TestImageCompression(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
cache := diskcache.GetGlobalCache()
if err := cache.Clear(); err != nil {
t.Fatalf("failed to clear disk cache before test: %v", err)
}
// Ensure uploads dir is cleaned up
defer func() {
_ = os.RemoveAll("uploads")
}()
defer func() {
if err := cache.Clear(); err != nil {
t.Errorf("failed to clear disk cache after test: %v", err)
}
}()
// Create test user
user := model.User{
ID: 555,
Username: "compress_tester",
IsActive: true,
}
dbConn.Create(&user)
// Create a 1x1 pixel PNG image
img := image.NewRGBA(image.Rect(0, 0, 1, 1))
img.Set(0, 0, color.RGBA{R: 255, G: 0, B: 0, A: 255})
var pngBuf bytes.Buffer
if err := png.Encode(&pngBuf, img); err != nil {
t.Fatalf("failed to encode test png: %v", err)
}
_ = os.MkdirAll("uploads", 0755)
filePath := "uploads/test_image.png"
if err := os.WriteFile(filePath, pngBuf.Bytes(), 0644); err != nil {
t.Fatalf("failed to write test png: %v", err)
}
// Save upload record to DB
uploadRecord := model.Upload{
ID: 3001,
UserID: user.ID,
FileName: "test_image.png",
FilePath: filePath,
FileSize: int64(pngBuf.Len()),
MimeType: "image/png",
Extension: "png",
StorageDriver: "local",
Type: "avatar", // Whitelisted by default
Status: model.UploadStatusUsed,
AccessMode: 1,
}
dbConn.Create(&uploadRecord)
// Setup Router
gin.SetMode(gin.TestMode)
r := gin.New()
r.GET("/f/:id", ServeFileByID)
t.Run("serve original file without compress parameter", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
// Content-Type should be image/png (default local serving type)
if w.Header().Get("Content-Type") != "image/png" {
t.Errorf("expected Content-Type image/png, got %s", w.Header().Get("Content-Type"))
}
if len(w.Body.Bytes()) != pngBuf.Len() {
t.Errorf("expected body size %d, got %d", pngBuf.Len(), len(w.Body.Bytes()))
}
})
t.Run("serve compressed WebP file with medium quality", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d. Body: %s", w.Code, w.Body.String())
}
// Content-Type should be image/webp
if w.Header().Get("Content-Type") != "image/webp" {
t.Errorf("expected Content-Type image/webp, got %s", w.Header().Get("Content-Type"))
}
cacheKey := ImageCompressionCacheKey(&uploadRecord, shared.ImageQualityMedium)
cachedBytes, err := cache.Get(cacheKey)
if err != nil {
t.Fatalf("disk cache Get(%q) returned error: %v", cacheKey, err)
}
if !bytes.Equal(cachedBytes, w.Body.Bytes()) {
t.Errorf("cached compressed image differs from response")
}
if err := os.Remove(filePath); err != nil {
t.Fatalf("failed to remove source image before cache-hit request: %v", err)
}
t.Cleanup(func() {
if err := os.WriteFile(filePath, pngBuf.Bytes(), 0644); err != nil {
t.Errorf("failed to restore source image: %v", err)
}
})
w2 := httptest.NewRecorder()
r.ServeHTTP(w2, req)
if w2.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w2.Code)
}
if !bytes.Equal(w2.Body.Bytes(), cachedBytes) {
t.Errorf("cache-hit response differs from cached compressed image")
}
})
t.Run("serve compressed WebP file and check cache headers and 304 Not Modified", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
etag := w.Header().Get("ETag")
if etag == "" {
t.Error("expected ETag header, got empty")
}
cacheControl := w.Header().Get("Cache-Control")
if cacheControl != "public, max-age=31536000" {
t.Errorf("expected Cache-Control 'public, max-age=31536000', got %q", cacheControl)
}
// Perform conditional GET request
reqCond, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
reqCond.Header.Set("If-None-Match", etag)
wCond := httptest.NewRecorder()
r.ServeHTTP(wCond, reqCond)
if wCond.Code != http.StatusNotModified {
t.Errorf("expected status 304, got %d", wCond.Code)
}
})
t.Run("serve original file with origin quality", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=origin", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
if w.Header().Get("Content-Type") != "image/png" {
t.Errorf("expected Content-Type image/png, got %s", w.Header().Get("Content-Type"))
}
if !bytes.Equal(w.Body.Bytes(), pngBuf.Bytes()) {
t.Errorf("origin-quality response differs from original image")
}
})
}
func TestNormalizeImageQuality(t *testing.T) {
tests := []struct {
name string
quality string
want string
}{
{name: shared.ImageQualityLow, quality: shared.ImageQualityLow, want: shared.ImageQualityLow},
{name: shared.ImageQualityMedium, quality: shared.ImageQualityMedium, want: shared.ImageQualityMedium},
{name: shared.ImageQualityHigh, quality: shared.ImageQualityHigh, want: shared.ImageQualityHigh},
{name: "origin", quality: "origin", want: "origin"},
{name: "uppercase", quality: "LOW", want: shared.ImageQualityLow},
{name: "empty", quality: "", want: "origin"},
{name: "invalid", quality: "maximum", want: "origin"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := util.NormalizeImageQuality(tt.quality); got != tt.want {
t.Errorf("NormalizeImageQuality(%q) = %q, want %q", tt.quality, got, tt.want)
}
})
}
}