refactor(core): completely decommission legacy internal/apps, internal/platform/bootstrap, and internal/router/v1

This commit is contained in:
ryan
2026-08-28 07:28:45 +08:00
parent df3c5ae756
commit dc49b72c29
182 changed files with 946 additions and 19604 deletions
@@ -0,0 +1,157 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package admin
import (
"net/http"
"strconv"
persistence "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/gin-gonic/gin"
)
// ListAuthSources lists all configured authentication sources.
func ListAuthSources(c *gin.Context) {
var sources []auth.AuthSource
gormDB := persistence.DB(c.Request.Context())
if err := gormDB.Order("id ASC").Find(&sources).Error; err != nil {
response.AbortInternal(c, "获取认证源列表失败")
return
}
views := make([]auth.AuthSourceView, len(sources))
for i := range sources {
views[i] = auth.AuthSourceView{
ID: sources[i].ID,
Name: sources[i].Name,
Type: sources[i].Type,
DisplayName: sources[i].DisplayName,
IsActive: sources[i].IsActive,
IconURL: sources[i].IconURL,
ClientSecretConfigured: sources[i].ClientSecret != "",
}
}
c.JSON(http.StatusOK, response.OK(views))
}
// CreateAuthSource creates a new authentication source.
func CreateAuthSource(c *gin.Context) {
var source auth.AuthSource
if err := c.ShouldBindJSON(&source); err != nil {
response.AbortBadRequest(c, "无效的参数")
return
}
if err := source.Validate(); err != nil {
response.AbortBadRequest(c, err.Error())
return
}
gormDB := persistence.DB(c.Request.Context())
if err := gormDB.Create(&source).Error; err != nil {
response.AbortBadRequest(c, "创建认证源失败: "+err.Error())
return
}
source.Sanitize()
c.JSON(http.StatusOK, response.OK(source))
}
// UpdateAuthSource updates an authentication source.
func UpdateAuthSource(c *gin.Context) {
idStr := c.Param("id")
id, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
response.AbortBadRequest(c, "无效的认证源 ID")
return
}
gormDB := persistence.DB(c.Request.Context())
var existing auth.AuthSource
if err := gormDB.First(&existing, id).Error; err != nil {
response.AbortNotFound(c, "认证源不存在")
return
}
var req auth.AuthSource
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, "无效的参数")
return
}
existing.DisplayName = req.DisplayName
existing.ClientID = req.ClientID
if req.ClientSecret != "" {
existing.ClientSecret = req.ClientSecret
}
existing.OpenIDDiscoveryURL = req.OpenIDDiscoveryURL
existing.Scopes = req.Scopes
existing.IconURL = req.IconURL
if err := existing.Validate(); err != nil {
response.AbortBadRequest(c, err.Error())
return
}
if err := gormDB.Save(&existing).Error; err != nil {
response.AbortInternal(c, "更新认证源失败")
return
}
existing.Sanitize()
c.JSON(http.StatusOK, response.OK(existing))
}
// ToggleAuthSource toggles the active state of an auth source.
func ToggleAuthSource(c *gin.Context) {
idStr := c.Param("id")
id, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
response.AbortBadRequest(c, "无效的认证源 ID")
return
}
gormDB := persistence.DB(c.Request.Context())
var existing auth.AuthSource
if err := gormDB.First(&existing, id).Error; err != nil {
response.AbortNotFound(c, "认证源不存在")
return
}
existing.IsActive = !existing.IsActive
if existing.IsActive {
if err := existing.Validate(); err != nil {
response.AbortBadRequest(c, err.Error())
return
}
}
if err := gormDB.Model(&existing).Update("is_active", existing.IsActive).Error; err != nil {
response.AbortInternal(c, "切换认证源状态失败")
return
}
c.JSON(http.StatusOK, response.OK(gin.H{"is_active": existing.IsActive}))
}
// DeleteAuthSource deletes an authentication source.
func DeleteAuthSource(c *gin.Context) {
idStr := c.Param("id")
id, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
response.AbortBadRequest(c, "无效的认证源 ID")
return
}
gormDB := persistence.DB(c.Request.Context())
if err := gormDB.Delete(&auth.AuthSource{}, id).Error; err != nil {
response.AbortInternal(c, "删除认证源失败")
return
}
c.JSON(http.StatusOK, response.OKNil())
}
+4 -5
View File
@@ -13,11 +13,6 @@ import (
"strings"
"time"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"github.com/Rain-kl/Wavelet/internal/apps/cap"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -25,6 +20,10 @@ import (
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger"
mail "github.com/Rain-kl/Wavelet/pkg/mail"
"github.com/Rain-kl/Wavelet/plugins/domain/cap"
"github.com/Rain-kl/Wavelet/plugins/domain/upload"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
const maskedConfigValue = "******"
+3 -4
View File
@@ -15,10 +15,6 @@ import (
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/Rain-kl/Wavelet/internal/apps/risk_control"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -27,6 +23,9 @@ import (
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/pkg/util"
"github.com/Rain-kl/Wavelet/plugins/domain/risk_control"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
)
const (
+9 -9
View File
@@ -15,12 +15,12 @@ import (
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
)
const minPasswordLength = 8
@@ -243,7 +243,7 @@ func DeleteUser(c *gin.Context) {
return
}
currUser, _ := oauth.GetFromContext[*model.User](c, oauth.UserObjKey)
currUser, _ := auth.GetFromContext[*model.User](c, auth.UserObjKey)
if currUser == nil {
response.AbortUnauthorized(c, AdminRequired)
return
@@ -334,7 +334,7 @@ func UpdateUser(c *gin.Context) {
return
}
currUser, _ := oauth.GetFromContext[*model.User](c, oauth.UserObjKey)
currUser, _ := auth.GetFromContext[*model.User](c, auth.UserObjKey)
if currUser == nil {
response.AbortUnauthorized(c, AdminRequired)
return
@@ -388,10 +388,10 @@ func updateUserStatus(ctx context.Context, id uint64, active bool) error {
err = repository.UpdateUserActive(ctx, id, active)
if err == nil {
oauth.InvalidateCachedUser(ctx, id)
auth.InvalidateCachedUser(ctx, id)
if !active {
for _, token := range tokens {
oauth.InvalidateCachedToken(ctx, token.TokenHash)
auth.InvalidateCachedToken(ctx, token.TokenHash)
}
}
}
@@ -414,9 +414,9 @@ func deleteUser(ctx context.Context, currentUserID, targetID uint64) error {
err = repository.DeleteUserWithRelations(ctx, targetID)
if err == nil {
oauth.InvalidateCachedUser(ctx, targetID)
auth.InvalidateCachedUser(ctx, targetID)
for _, token := range tokens {
oauth.InvalidateCachedToken(ctx, token.TokenHash)
auth.InvalidateCachedToken(ctx, token.TokenHash)
}
}
return err
@@ -539,10 +539,10 @@ func updateUser(ctx context.Context, currentUserID uint64, param updateUserParam
err = repository.UpdateUser(ctx, &targetUser)
if err == nil {
oauth.InvalidateCachedUser(ctx, param.ID)
auth.InvalidateCachedUser(ctx, param.ID)
if needRevokeTokens {
for _, token := range tokens {
oauth.InvalidateCachedToken(ctx, token.TokenHash)
auth.InvalidateCachedToken(ctx, token.TokenHash)
}
}
}
+4 -4
View File
@@ -4,11 +4,11 @@
package admin
import (
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger"
otel_trace "github.com/Rain-kl/Wavelet/pkg/trace"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/gin-gonic/gin"
)
@@ -18,15 +18,15 @@ func LoginAdminRequired() gin.HandlerFunc {
ctx, span := otel_trace.Start(c.Request.Context(), "LoginAdminRequired")
defer span.End()
user, _ := oauth.GetFromContext[*model.User](c, oauth.UserObjKey)
user, _ := auth.GetFromContext[*model.User](c, auth.UserObjKey)
if user == nil {
response.AbortNotFound(c, AdminRequired)
return
}
// 如果是通过 Access Token 鉴权,需要检查令牌本身是否具有管理员权限
if tokenAuth, _ := oauth.GetFromContext[bool](c, oauth.TokenAuthKey); tokenAuth {
tokenAdmin, _ := oauth.GetFromContext[bool](c, oauth.TokenAdminKey)
if tokenAuth, _ := auth.GetFromContext[bool](c, auth.TokenAuthKey); tokenAuth {
tokenAdmin, _ := auth.GetFromContext[bool](c, auth.TokenAdminKey)
if !tokenAdmin {
response.AbortNotFound(c, TokenAdminRequired)
return
+52 -100
View File
@@ -1,3 +1,6 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package admin provides the system management console, diagnostics, audit logging, and configuration hot-reloading domain plugin for Cordis.
package admin
@@ -6,20 +9,7 @@ import (
"github.com/Rain-kl/Wavelet/core"
"github.com/Rain-kl/Wavelet/core/extpoints"
"github.com/Rain-kl/Wavelet/internal/apps/admin"
admin_auth_source "github.com/Rain-kl/Wavelet/internal/apps/admin/auth_source"
admin_cache "github.com/Rain-kl/Wavelet/internal/apps/admin/cache"
admin_db_manage "github.com/Rain-kl/Wavelet/internal/apps/admin/db_manage"
admin_logs "github.com/Rain-kl/Wavelet/internal/apps/admin/logs"
admin_push "github.com/Rain-kl/Wavelet/internal/apps/admin/push"
admin_status "github.com/Rain-kl/Wavelet/internal/apps/admin/status"
"github.com/Rain-kl/Wavelet/internal/apps/admin/system_config"
admin_task "github.com/Rain-kl/Wavelet/internal/apps/admin/task"
admin_template "github.com/Rain-kl/Wavelet/internal/apps/admin/template"
admin_updater "github.com/Rain-kl/Wavelet/internal/apps/admin/updater"
admin_user "github.com/Rain-kl/Wavelet/internal/apps/admin/user"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/hibiken/asynq"
)
@@ -58,153 +48,115 @@ func (p *Plugin) Manifest() core.Manifest {
// Apply registers admin routes, tasks, schedules, and settings into the Context.
func (p *Plugin) Apply(ctx *core.Context) error {
// 1. Register Admin HTTP Routes
adminRouter := ctx.Router().Group("/api/v1/admin", oauth.LoginRequired(), admin.LoginAdminRequired())
adminRouter := ctx.Router().Group("/api/v1/admin", auth.LoginRequired(), LoginAdminRequired())
{
// Status & Diagnostics
adminRouter.GET("/status", admin_status.GetSystemStatus)
adminRouter.GET("/status/log-database", admin_status.GetLogDatabaseStatus)
adminRouter.GET("/db-info", admin_status.GetDatabaseInfo)
adminRouter.GET("/db-export", admin_status.ExportDatabase)
adminRouter.GET("/status", GetSystemStatus)
adminRouter.GET("/status/log-database", GetLogDatabaseStatus)
adminRouter.GET("/db-info", GetDatabaseInfo)
adminRouter.GET("/db-export", ExportDatabase)
// DB Management
dbGroup := adminRouter.Group("/db-manage")
{
dbGroup.GET("/overview", admin_db_manage.GetDBOverview)
dbGroup.GET("/tables", admin_db_manage.ListDBTables)
dbGroup.GET("/table-data", admin_db_manage.GetDBTableData)
dbGroup.POST("/query", admin_db_manage.ExecuteSQL)
dbGroup.GET("/overview", GetDBOverview)
dbGroup.GET("/tables", ListDBTables)
dbGroup.GET("/table-data", GetDBTableData)
dbGroup.POST("/query", ExecuteSQL)
}
// Cache Management
cacheGroup := adminRouter.Group("/cache")
{
cacheGroup.GET("/status", admin_cache.GetCacheStatus)
cacheGroup.POST("/config", admin_cache.UpdateCacheConfig)
cacheGroup.POST("/clear", admin_cache.ClearCache)
cacheGroup.GET("/status", GetCacheStatus)
cacheGroup.POST("/config", UpdateCacheConfig)
cacheGroup.POST("/clear", ClearCache)
}
// Updater
updateGroup := adminRouter.Group("/update")
{
updateGroup.GET("", admin_updater.GetUpdateStatus)
updateGroup.POST("/apply", admin_updater.ApplyUpdate)
updateGroup.GET("", GetUpdateStatus)
updateGroup.POST("/apply", ApplyUpdate)
}
// Logs
logsGroup := adminRouter.Group("/logs")
{
logsGroup.GET("", admin_logs.GetLogs)
logsGroup.GET("/access", admin_logs.GetAccessLogs)
logsGroup.GET("/analytics", admin_logs.GetLogsAnalytics)
logsGroup.GET("/ws", admin_logs.HandleLogWebSocket)
logsGroup.GET("", GetLogs)
logsGroup.GET("/access", GetAccessLogs)
logsGroup.GET("/analytics", GetLogsAnalytics)
logsGroup.GET("/ws", HandleLogWebSocket)
}
// Users
usersGroup := adminRouter.Group("/users")
{
usersGroup.GET("", admin_user.ListUsers)
usersGroup.POST("", admin_user.CreateUser)
usersGroup.GET("/:id", admin_user.GetUser)
usersGroup.PUT("/:id/status", admin_user.UpdateUserStatus)
usersGroup.PUT("/:id", admin_user.UpdateUser)
usersGroup.DELETE("/:id", admin_user.DeleteUser)
usersGroup.GET("", ListUsers)
usersGroup.POST("", CreateUser)
usersGroup.GET("/:id", GetUser)
usersGroup.PUT("/:id/status", UpdateUserStatus)
usersGroup.PUT("/:id", UpdateUser)
usersGroup.DELETE("/:id", DeleteUser)
}
// Auth Sources
authSourcesGroup := adminRouter.Group("/auth-sources")
{
authSourcesGroup.GET("", admin_auth_source.ListAuthSources)
authSourcesGroup.POST("", admin_auth_source.CreateAuthSource)
authSourcesGroup.PUT("/:id", admin_auth_source.UpdateAuthSource)
authSourcesGroup.PUT("/:id/toggle", admin_auth_source.ToggleAuthSource)
authSourcesGroup.DELETE("/:id", admin_auth_source.DeleteAuthSource)
authSourcesGroup.GET("", ListAuthSources)
authSourcesGroup.POST("", CreateAuthSource)
authSourcesGroup.PUT("/:id", UpdateAuthSource)
authSourcesGroup.PUT("/:id/toggle", ToggleAuthSource)
authSourcesGroup.DELETE("/:id", DeleteAuthSource)
}
// System Configs
configGroup := adminRouter.Group("/system-configs")
{
configGroup.GET("", system_config.ListSystemConfigs)
configGroup.POST("", system_config.CreateSystemConfig)
configGroup.POST("/smtp/test", system_config.TestSMTP)
configGroup.GET("", ListSystemConfigs)
configGroup.POST("", CreateSystemConfig)
configGroup.POST("/smtp/test", TestSMTP)
keyGroup := configGroup.Group("/:key")
{
keyGroup.GET("", system_config.GetSystemConfig)
keyGroup.PUT("", system_config.UpdateSystemConfig)
keyGroup.GET("", GetSystemConfig)
keyGroup.PUT("", UpdateSystemConfig)
}
}
// Templates
templateGroup := adminRouter.Group("/templates")
{
templateGroup.GET("", admin_template.ListTemplates)
templateGroup.POST("", admin_template.CreateTemplate)
templateGroup.GET("", ListTemplates)
templateGroup.POST("", CreateTemplate)
keyGroup := templateGroup.Group("/:key")
{
keyGroup.GET("", admin_template.GetTemplate)
keyGroup.PUT("", admin_template.UpdateTemplate)
keyGroup.DELETE("", admin_template.DeleteTemplate)
keyGroup.GET("", GetTemplate)
keyGroup.PUT("", UpdateTemplate)
keyGroup.DELETE("", DeleteTemplate)
}
}
// Uploads Management
uploadGroup := adminRouter.Group("/uploads")
{
uploadGroup.GET("", upload.ListFiles)
uploadGroup.GET("/stats", upload.GetFileStats)
uploadGroup.DELETE("/:id", upload.DeleteFile)
uploadGroup.GET("/download/:id", upload.DownloadFile)
uploadGroup.POST("/download/batch", upload.BatchDownloadFiles)
uploadGroup.GET("/types", upload.GetDistinctUploadTypes)
}
// Tasks
taskGroup := adminRouter.Group("/tasks")
{
taskGroup.GET("/types", admin_task.ListTaskTypes)
taskGroup.POST("/dispatch", admin_task.DispatchTask)
taskGroup.GET("/types", ListTaskTypes)
taskGroup.POST("/dispatch", DispatchTask)
executions := taskGroup.Group("/executions")
{
executions.GET("", admin_task.ListTaskExecutions)
executions.GET("/:id", admin_task.GetTaskExecution)
executions.POST("/:id/retry", admin_task.RetryTask)
executions.GET("", ListTaskExecutions)
executions.GET("/:id", GetTaskExecution)
executions.POST("/:id/retry", RetryTask)
}
schedules := taskGroup.Group("/schedules")
{
schedules.GET("", admin_task.ListSchedules)
schedules.POST("", admin_task.CreateSchedule)
schedules.PUT("/:id", admin_task.UpdateSchedule)
schedules.DELETE("/:id", admin_task.DeleteSchedule)
}
}
// Push & Notifications
pushGroup := adminRouter.Group("/push")
{
events := pushGroup.Group("/events")
{
events.GET("", admin_push.ListEvents)
events.GET("/builtin", admin_push.ListBuiltInEvents)
events.POST("", admin_push.CreateEvent)
events.PUT("/:id", admin_push.UpdateEvent)
events.DELETE("/:id", admin_push.DeleteEvent)
events.POST("/:id/toggle", admin_push.ToggleEvent)
}
pushGroup.GET("/histories", admin_push.ListHistories)
pushGroup.POST("/test", admin_push.TestPush)
channels := pushGroup.Group("/channels")
{
channels.GET("/definitions", admin_push.ListChannelDefinitions)
channels.GET("", admin_push.ListChannels)
channels.POST("", admin_push.CreateChannel)
channels.PUT("/:id", admin_push.UpdateChannel)
channels.DELETE("/:id", admin_push.DeleteChannel)
channels.POST("/test", admin_push.TestChannel)
schedules.GET("", ListSchedules)
schedules.POST("", CreateSchedule)
schedules.PUT("/:id", UpdateSchedule)
schedules.DELETE("/:id", DeleteSchedule)
}
}
}
+10
View File
@@ -0,0 +1,10 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package cap 提供人机验证中间件
package cap
const (
errCapTokenMissing = "验证码验证失败,缺少验证码凭证" //nolint:gosec // false positive: this is an error message, not hardcoded credentials
errCapTokenInvalidOrExpired = "验证码校验失败或已过期,请重试" //nolint:gosec // false positive: this is an error message, not hardcoded credentials
)
+101
View File
@@ -0,0 +1,101 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cap
import (
"net/http"
"github.com/Rain-kl/Wavelet/internal/shared/response"
pkgcap "github.com/Rain-kl/Wavelet/pkg/cap"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
)
// ChallengeResponse is a local type alias for the pkg/cap.ChallengeResponse struct
type ChallengeResponse = pkgcap.ChallengeResponse
type challengeRequest struct {
Scope string `json:"scope" form:"scope"`
}
type redeemRequest struct {
Token string `json:"token" binding:"required"`
Solutions []int `json:"solutions" binding:"required"`
Scope string `json:"scope" form:"scope"`
}
// Challenge 生成 PoW 人机验证难题
// @Summary 生成人机验证难题
// @Description 客户端获取 PoW 难题和签名的 JWT Token,并在后台计算。
// @Tags cap
// @Accept json
// @Produce json
// @Param request body challengeRequest false "可选范围限制参数"
// @Success 200 {object} response.Any{data=cap.ChallengeResponse} "成功返回 PoW 难题"
// @Failure 500 {object} response.Any "内部服务错误"
// @Router /api/cap/challenge [post]
func Challenge(c *gin.Context) {
var req challengeRequest
_ = c.ShouldBind(&req) // 允许不传 body,默认使用 login scope
if req.Scope == "" {
req.Scope = "login"
}
mgr := GetDefaultManager()
if mgr == nil {
response.AbortInternal(c, "captcha is not configured")
return
}
resp, err := mgr.Generate(c.Request.Context(), req.Scope)
if err != nil {
logger.ErrorF(c.Request.Context(), "Generate cap challenge failed: %v", err)
response.AbortInternal(c, "生成验证难题失败,请稍后再试")
return
}
c.JSON(http.StatusOK, response.OK(resp))
}
// Redeem 提交 PoW 解答并兑换一次性凭证 Token
// @Summary 校验人机验证解答
// @Description 提交 PoW 解答进行核销,成功后返回一次性 X-Cap-Token 凭证
// @Tags cap
// @Accept json
// @Produce json
// @Param request body redeemRequest true "难题 Token 与解答 solutions 数组"
// @Success 200 {object} response.Any{data=cap.RedeemResponse} "核销成功,返回 X-Cap-Token"
// @Failure 400 {object} response.Any "参数错误或核销失败"
// @Failure 500 {object} response.Any "内部服务错误"
// @Router /api/cap/redeem [post]
func Redeem(c *gin.Context) {
var req redeemRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, "无效的参数")
return
}
if req.Scope == "" {
req.Scope = "login"
}
mgr := GetDefaultManager()
if mgr == nil {
response.AbortInternal(c, "captcha is not configured")
return
}
resp, err := mgr.Redeem(c.Request.Context(), req.Token, req.Solutions, req.Scope)
if err != nil {
logger.ErrorF(c.Request.Context(), "Redeem cap solutions failed: %v", err)
response.AbortInternal(c, "校验验证解答失败,请稍后再试")
return
}
if !resp.Success {
response.AbortBadRequest(c, resp.Error)
return
}
c.JSON(http.StatusOK, response.OK(resp))
}
+199
View File
@@ -0,0 +1,199 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package cap provides CAPTCHA and proof-of-work (PoW) verification services.
package cap
import (
"context"
"crypto/sha256"
"encoding/hex"
"strconv"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
pkgcap "github.com/Rain-kl/Wavelet/pkg/cap"
)
const (
redeemTokenIDLength = 8 // 兑换 Token ID 字节长度
redeemVerTokenLength = 15 // 兑换验证 Token 字节长度
tokenPartsCount = 2 // 兑换 Token 由两部分组成
valuePartsCount = 2 // 存储值由 scope 和过期时间组成
)
// Manager orchestrates challenge generation and solution validation.
type Manager struct {
secret []byte
store pkgcap.Store
}
// NewManager creates a new CAPTCHA Manager.
func NewManager(secret []byte, store pkgcap.Store) *Manager {
return &Manager{
secret: secret,
store: store,
}
}
// Generate creates a challenge response.
func (m *Manager) Generate(ctx context.Context, scope string) (*pkgcap.ChallengeResponse, error) {
settings, err := CurrentSettings(ctx)
if err != nil {
return nil, err
}
challengeConfig := pkgcap.ChallengeConfig{
Count: settings.ChallengeCount,
Size: settings.ChallengeSize,
Difficulty: settings.ChallengeDifficulty,
Expires: settings.ChallengeTTL,
}
return pkgcap.GenerateChallenge(m.secret, challengeConfig, scope)
}
// RedeemResponse is returned to the client on redeem.
type RedeemResponse struct {
Success bool `json:"success"`
Token string `json:"token,omitempty"`
Expires int64 `json:"expires,omitempty"`
Error string `json:"error,omitempty"`
}
// Redeem verifies PoW solutions and returns a one-time redeem token.
func (m *Manager) Redeem(ctx context.Context, token string, solutions []int, scope string) (*RedeemResponse, error) {
sigHex := pkgcap.JwtSigHex(token)
if sigHex == "" {
return &RedeemResponse{Success: false, Error: "invalid_token"}, nil
}
nonceKey := "cap:nonce:" + sigHex
payload, err := pkgcap.VerifyChallengeSolutions(token, solutions, m.secret, scope)
if err != nil {
return &RedeemResponse{Success: false, Error: err.Error()}, nil //nolint:nilerr // validation errors are returned as response, not system errors
}
now := time.Now().UnixNano() / int64(time.Millisecond)
nonceTTL := time.Duration(payload.Expires-now) * time.Millisecond
if nonceTTL < time.Second {
nonceTTL = time.Second
}
set, err := m.store.SetNX(ctx, nonceKey, "1", nonceTTL)
if err != nil {
return &RedeemResponse{Success: false, Error: "nonce_store_error"}, err
}
if !set {
return &RedeemResponse{Success: false, Error: "already_redeemed"}, nil
}
settings, err := CurrentSettings(ctx)
if err != nil {
return &RedeemResponse{Success: false, Error: "settings_load_error"}, err
}
id := pkgcap.RandomHex(redeemTokenIDLength)
verToken := pkgcap.RandomHex(redeemVerTokenLength)
verHashBytes := sha256.Sum256([]byte(verToken))
verHashHex := hex.EncodeToString(verHashBytes[:])
tokenKey := "cap:token:" + id + ":" + verHashHex
tokenExpires := time.Now().Add(settings.TokenTTL)
storeVal := strconv.FormatInt(tokenExpires.UnixNano(), 10) + "|" + scope
if err := m.store.Set(ctx, tokenKey, storeVal, settings.TokenTTL); err != nil {
return &RedeemResponse{Success: false, Error: "token_store_error"}, err
}
return &RedeemResponse{
Success: true,
Token: id + ":" + verToken,
Expires: tokenExpires.UnixNano() / int64(time.Millisecond),
}, nil
}
// VerifyToken validates and consumes the redeem token (single-use).
func (m *Manager) VerifyToken(ctx context.Context, token string, expectedScope string) (bool, error) {
if token == "" {
return false, nil
}
parts := strings.Split(token, ":")
if len(parts) != tokenPartsCount {
return false, nil
}
id := parts[0]
verToken := parts[1]
verHashBytes := sha256.Sum256([]byte(verToken))
verHashHex := hex.EncodeToString(verHashBytes[:])
tokenKey := "cap:token:" + id + ":" + verHashHex
val, exists, err := sGetAndDelete(ctx, m.store, tokenKey)
if err != nil {
return false, err
}
if !exists {
return false, nil
}
valParts := strings.Split(val, "|")
if len(valParts) != valuePartsCount {
return false, nil
}
expNano, err := strconv.ParseInt(valParts[0], 10, 64)
if err != nil {
return false, nil //nolint:nilerr // invalid format is treated as validation failure
}
tokenScope := valParts[1]
if expectedScope != "" && tokenScope != expectedScope {
return false, nil
}
if time.Now().UnixNano() > expNano {
return false, nil
}
return true, nil
}
func sGetAndDelete(ctx context.Context, store pkgcap.Store, key string) (string, bool, error) {
if store == nil {
return "", false, nil
}
return store.GetAndDelete(ctx, key)
}
var (
defaultManager *Manager
once sync.Once
)
// GetDefaultManager yields the global singleton CAPTCHA manager.
func GetDefaultManager() *Manager {
once.Do(func() {
var secret []byte
if config.Config != nil && strings.TrimSpace(config.Config.App.SessionSecret) != "" {
secret = []byte(config.Config.App.SessionSecret)
}
if len(secret) == 0 {
return
}
var store pkgcap.Store
if config.Config != nil && config.Config.Redis.Enabled && db.Redis != nil {
store = pkgcap.NewRedisStore(db.Redis)
} else {
store = pkgcap.NewMemoryStore(1 * time.Minute)
}
defaultManager = NewManager(secret, store)
})
return defaultManager
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cap
import (
"github.com/gin-gonic/gin"
"github.com/Rain-kl/Wavelet/internal/shared/response"
)
// VerifyMiddleware returns a Gin middleware that checks and consumes the X-Cap-Token header.
func VerifyMiddleware(mgr *Manager, scope string) gin.HandlerFunc {
return func(c *gin.Context) {
if !ProtectionEnabled(c.Request.Context()) {
c.Next()
return
}
if mgr == nil {
response.AbortUnauthorized(c, errCapTokenInvalidOrExpired)
return
}
token := c.GetHeader("X-Cap-Token")
if token == "" {
response.AbortUnauthorized(c, errCapTokenMissing)
return
}
valid, err := mgr.VerifyToken(c.Request.Context(), token, scope)
if err != nil || !valid {
response.AbortUnauthorized(c, errCapTokenInvalidOrExpired)
return
}
c.Next()
}
}
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package cap provides the proof-of-work (PoW) CAPTCHA verification domain plugin for Cordis.
package cap
import (
"github.com/Rain-kl/Wavelet/core"
"github.com/Rain-kl/Wavelet/core/extpoints"
)
// Plugin implements core.Plugin to provide CAPTCHA generation, validation, and route protection.
type Plugin struct{}
// New creates a new cap domain plugin.
func New() *Plugin {
return &Plugin{}
}
// Name returns the unique identifier for the cap domain plugin.
func (p *Plugin) Name() string {
return "cap"
}
// Manifest returns the plugin metadata.
func (p *Plugin) Manifest() core.Manifest {
return core.Manifest{
Name: "cap",
Version: "1.0.0",
Description: "Proof-of-work CAPTCHA challenge and verification domain plugin",
Author: "Wavelet Team",
}
}
// Apply registers the cap routes and settings into the Context.
func (p *Plugin) Apply(ctx *core.Context) error {
// Register HTTP Routes
capGroup := ctx.Router().Group("/api/v1/cap")
{
capGroup.GET("/challenge", Challenge)
capGroup.POST("/challenge", Challenge)
capGroup.POST("/redeem", Redeem)
}
// Register Settings Schemas
ctx.Settings().Register(extpoints.SettingSchema{
Key: "cap.login_enabled",
Default: false,
Description: "Whether to require CAPTCHA verification for user login",
Type: "boolean",
Category: "security",
})
ctx.Settings().Register(extpoints.SettingSchema{
Key: "cap.challenge_count",
Default: 1,
Description: "Number of PoW puzzle challenges to solve",
Type: "integer",
Category: "security",
})
return nil
}
+213
View File
@@ -0,0 +1,213 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cap
import (
"context"
"encoding/json"
"errors"
"strconv"
"sync"
"sync/atomic"
"time"
"golang.org/x/sync/singleflight"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/pkg/util"
)
const (
defaultChallengeCount = 1
defaultChallengeSize = 32
defaultChallengeDifficulty = 4
defaultChallengeTTL = 10 * time.Minute
defaultTokenTTL = 20 * time.Minute
)
// RuntimeSettings is the parsed CAPTCHA runtime configuration loaded from system_configs.
type RuntimeSettings struct {
LoginEnabled bool
ChallengeCount int
ChallengeSize int
ChallengeDifficulty int
ChallengeTTL time.Duration
TokenTTL time.Duration
}
var runtimeConfigKeys = []string{
model.ConfigKeyCapLoginEnabled,
model.ConfigKeyCapChallengeCount,
model.ConfigKeyCapChallengeSize,
model.ConfigKeyCapChallengeDifficulty,
model.ConfigKeyCapChallengeTTL,
model.ConfigKeyCapTokenTTL,
}
var runtimeConfigKeySet = func() map[string]struct{} {
set := make(map[string]struct{}, len(runtimeConfigKeys))
for _, key := range runtimeConfigKeys {
set[key] = struct{}{}
}
return set
}()
type runtimeSettingsStore struct {
snapshot atomic.Pointer[RuntimeSettings]
loadGroup singleflight.Group
listenerOnce sync.Once
}
var settingsStore = &runtimeSettingsStore{}
// IsRuntimeConfigKey reports whether a system config key affects CAPTCHA runtime settings.
func IsRuntimeConfigKey(key string) bool {
_, ok := runtimeConfigKeySet[key]
return ok
}
// CurrentSettings returns the cached CAPTCHA runtime settings snapshot.
func CurrentSettings(ctx context.Context) (RuntimeSettings, error) {
return settingsStore.current(ctx)
}
// ProtectionEnabled reports whether CAPTCHA verification is required for protected routes.
func ProtectionEnabled(ctx context.Context) bool {
settings, err := CurrentSettings(ctx)
if err != nil {
return false
}
return settings.LoginEnabled
}
// InvalidateRuntimeSettings drops the in-process CAPTCHA settings snapshot.
func InvalidateRuntimeSettings() {
settingsStore.snapshot.Store(nil)
}
// ResetRuntimeSettingsForTest clears the CAPTCHA runtime snapshot.
func ResetRuntimeSettingsForTest() {
InvalidateRuntimeSettings()
}
// InstallTestRuntimeSettings installs a fixed snapshot for unit tests.
func InstallTestRuntimeSettings(settings RuntimeSettings) func() {
snapshot := settings
settingsStore.snapshot.Store(&snapshot)
return InvalidateRuntimeSettings
}
func (s *runtimeSettingsStore) current(ctx context.Context) (RuntimeSettings, error) {
s.ensureInvalidationListener()
if snapshot := s.snapshot.Load(); snapshot != nil {
return *snapshot, nil
}
loaded, err, _ := s.loadGroup.Do("cap-runtime-settings", func() (any, error) {
if snapshot := s.snapshot.Load(); snapshot != nil {
return *snapshot, nil
}
settings, loadErr := loadRuntimeSettings(ctx)
if loadErr != nil {
return RuntimeSettings{}, loadErr
}
s.snapshot.Store(&settings)
return settings, nil
})
if err != nil {
return RuntimeSettings{}, err
}
settings, ok := loaded.(RuntimeSettings)
if !ok {
return RuntimeSettings{}, errors.New("cap runtime settings loader returned unexpected type")
}
return settings, nil
}
func loadRuntimeSettings(ctx context.Context) (RuntimeSettings, error) {
configs, err := repository.ListSystemConfigsByKeys(ctx, runtimeConfigKeys)
if err != nil {
return RuntimeSettings{}, err
}
return parseRuntimeSettings(configs), nil
}
func parseRuntimeSettings(configs map[string]model.SystemConfig) RuntimeSettings {
settings := RuntimeSettings{
ChallengeCount: defaultChallengeCount,
ChallengeSize: defaultChallengeSize,
ChallengeDifficulty: defaultChallengeDifficulty,
ChallengeTTL: defaultChallengeTTL,
TokenTTL: defaultTokenTTL,
}
if sc, ok := configs[model.ConfigKeyCapLoginEnabled]; ok {
if enabled, err := strconv.ParseBool(sc.Value); err == nil {
settings.LoginEnabled = enabled
}
}
if sc, ok := configs[model.ConfigKeyCapChallengeCount]; ok {
if count, err := strconv.Atoi(sc.Value); err == nil && count > 0 {
settings.ChallengeCount = count
}
}
if sc, ok := configs[model.ConfigKeyCapChallengeSize]; ok {
if size, err := strconv.Atoi(sc.Value); err == nil && size > 0 {
settings.ChallengeSize = size
}
}
if sc, ok := configs[model.ConfigKeyCapChallengeDifficulty]; ok {
if difficulty, err := strconv.Atoi(sc.Value); err == nil && difficulty > 0 {
settings.ChallengeDifficulty = difficulty
}
}
if sc, ok := configs[model.ConfigKeyCapChallengeTTL]; ok {
if ttlSeconds, err := strconv.Atoi(sc.Value); err == nil && ttlSeconds > 0 {
settings.ChallengeTTL = time.Duration(ttlSeconds) * time.Second
}
}
if sc, ok := configs[model.ConfigKeyCapTokenTTL]; ok {
if ttlSeconds, err := strconv.Atoi(sc.Value); err == nil && ttlSeconds > 0 {
settings.TokenTTL = time.Duration(ttlSeconds) * time.Second
}
}
return settings
}
func (s *runtimeSettingsStore) ensureInvalidationListener() {
s.listenerOnce.Do(startRuntimeSettingsInvalidationListener)
}
func startRuntimeSettingsInvalidationListener() {
if db.Redis == nil {
return
}
util.Go(func() {
pubsub := db.Redis.Subscribe(context.Background(), repository.SystemConfigInvalidationChannel)
defer func() {
_ = pubsub.Close()
}()
for msg := range pubsub.Channel() {
var payload struct {
Key string `json:"key"`
}
if err := json.Unmarshal([]byte(msg.Payload), &payload); err != nil {
InvalidateRuntimeSettings()
continue
}
if payload.Key == "" || payload.Key == "*" || IsRuntimeConfigKey(payload.Key) {
InvalidateRuntimeSettings()
}
}
})
}
+5 -5
View File
@@ -64,7 +64,7 @@ func (m *mockOAuthProvider) Name() string {
}
func (m *mockOAuthProvider) GetAuthURL(state string) string {
return "https://oauth.example.com/auth?state=" + state
return "https://auth.example.com/auth?state=" + state
}
func (m *mockOAuthProvider) ExchangeCode(ctx context.Context, code string) (*contracts.OAuthUserInfoDTO, error) {
@@ -349,7 +349,7 @@ func TestAdminPlugin(t *testing.T) {
// 1. Admin Routes
routes := ctx.Router().Routes()
var hasStatus, hasDBOverview, hasUsers, hasTasks, hasPushEvents bool
var hasStatus, hasDBOverview, hasUsers, hasTasks, hasConfigs bool
for _, r := range routes {
if r.Path == "/api/v1/admin/status" {
hasStatus = true
@@ -363,15 +363,15 @@ func TestAdminPlugin(t *testing.T) {
if r.Path == "/api/v1/admin/tasks/types" {
hasTasks = true
}
if r.Path == "/api/v1/admin/push/events" {
hasPushEvents = true
if r.Path == "/api/v1/admin/system-configs" {
hasConfigs = true
}
}
assert.True(t, hasStatus)
assert.True(t, hasDBOverview)
assert.True(t, hasUsers)
assert.True(t, hasTasks)
assert.True(t, hasPushEvents)
assert.True(t, hasConfigs)
// 2. Task & Schedule
_, ok := ctx.Tasks().Get("admin:system_cleanup")
+3 -3
View File
@@ -8,14 +8,14 @@ import (
"net/http"
"strconv"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/gin-gonic/gin"
)
func currentUser(c *gin.Context) (*model.User, bool) {
return oauth.GetFromContext[*model.User](c, oauth.UserObjKey)
return auth.GetFromContext[*model.User](c, auth.UserObjKey)
}
// ListChannels lists enabled channels a user can bind.
@@ -137,7 +137,7 @@ func UnbindBinding(c *gin.Context) {
// RegisterUserRoutes mounts user-facing message gateway endpoints.
func RegisterUserRoutes(r *gin.RouterGroup) {
mg := r.Group("/message-gateway", oauth.LoginRequired())
mg := r.Group("/message-gateway", auth.LoginRequired())
{
mg.GET("/channels", ListChannels)
mg.GET("/bindings", ListBindings)
+5 -5
View File
@@ -10,8 +10,8 @@ import (
"github.com/Rain-kl/Wavelet/core"
"github.com/Rain-kl/Wavelet/core/extpoints"
"github.com/Rain-kl/Wavelet/internal/apps/admin"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/plugins/domain/admin"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/hibiken/asynq"
)
@@ -75,7 +75,7 @@ func (p *Plugin) Apply(ctx *core.Context) error {
ctx.Migrations().Register("message_gateway", mgMigrations)
// 2. Register User HTTP Routes
mgGroup := ctx.Router().Group("/api/v1/message-gateway", oauth.LoginRequired())
mgGroup := ctx.Router().Group("/api/v1/message-gateway", auth.LoginRequired())
{
mgGroup.GET("/channels", ListChannels)
mgGroup.GET("/bindings", ListBindings)
@@ -84,7 +84,7 @@ func (p *Plugin) Apply(ctx *core.Context) error {
}
// 3. Register Admin Message Gateway HTTP Routes
adminMgGroup := ctx.Router().Group("/api/v1/admin/message-gateway", oauth.LoginRequired(), admin.LoginAdminRequired())
adminMgGroup := ctx.Router().Group("/api/v1/admin/message-gateway", auth.LoginRequired(), admin.LoginAdminRequired())
{
adminMgGroup.GET("/channels/definitions", ListAdminChannelDefinitions)
adminMgGroup.GET("/channels", ListAdminChannels)
@@ -95,7 +95,7 @@ func (p *Plugin) Apply(ctx *core.Context) error {
}
// 4. Register Admin Push HTTP Routes
adminPushGroup := ctx.Router().Group("/api/v1/admin/push", oauth.LoginRequired(), admin.LoginAdminRequired())
adminPushGroup := ctx.Router().Group("/api/v1/admin/push", auth.LoginRequired(), admin.LoginAdminRequired())
{
events := adminPushGroup.Group("/events")
{
+5 -2
View File
@@ -9,15 +9,18 @@ import (
"net/http"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/gin-gonic/gin"
)
// Middleware is an alias for RiskControlMiddleware.
var Middleware = RiskControlMiddleware
// RiskControlMiddleware 全局日志采集中间件
func RiskControlMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
@@ -39,7 +42,7 @@ func RiskControlMiddleware() gin.HandlerFunc {
c.Next()
// 3. 后置身份检查:仅记录通过认证的请求
userObj, exists := oauth.GetFromContext[*model.User](c, oauth.UserObjKey)
userObj, exists := auth.GetFromContext[*model.User](c, auth.UserObjKey)
if !exists || userObj == nil {
return
}
@@ -12,12 +12,12 @@ import (
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/risk_control"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
@@ -99,7 +99,7 @@ func TestRiskControlMiddleware(t *testing.T) {
r := gin.New()
r.Use(func(c *gin.Context) {
user := &model.User{ID: 12345}
oauth.SetToContext(c, oauth.UserObjKey, user)
auth.SetToContext(c, auth.UserObjKey, user)
c.Next()
})
r.Use(risk_control.RiskControlMiddleware())
+71
View File
@@ -0,0 +1,71 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package system provides core system health probes, public config endpoints, and static frontend assets dispatch plugin for Cordis.
package system
import (
"net/http"
"github.com/Rain-kl/Wavelet/core"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/gin-gonic/gin"
)
// Plugin implements core.Plugin to provide system-level basic routes.
type Plugin struct{}
// New creates a new system domain plugin.
func New() *Plugin {
return &Plugin{}
}
// Name returns the unique identifier for the system domain plugin.
func (p *Plugin) Name() string {
return "system"
}
// Manifest returns the plugin metadata.
func (p *Plugin) Manifest() core.Manifest {
return core.Manifest{
Name: "system",
Version: "1.0.0",
Description: "System health check, public config, and assets domain plugin",
Author: "Wavelet Team",
}
}
// Apply registers system routes.
func (p *Plugin) Apply(ctx *core.Context) error {
// 1. Health check
ctx.Router().GET("/healthz", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"status": "ok"})
})
ctx.Router().GET("/api/healthz", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"status": "ok"})
})
// 2. Public config
ctx.Router().GET("/api/v1/config/public", func(c *gin.Context) {
configs, err := repository.ListVisibleSystemConfigs(c.Request.Context())
if err != nil {
response.AbortInternal(c, "获取公开配置失败")
return
}
c.JSON(http.StatusOK, response.OK(gin.H{
"configs": configs,
"app": gin.H{
"name": config.Config.App.AppName,
},
}))
})
// 3. Custom injection
ctx.Router().GET("/custom", func(c *gin.Context) {
c.JSON(http.StatusOK, response.OK(gin.H{"custom": true}))
})
return nil
}
+144
View File
@@ -0,0 +1,144 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package cache provides in-process upload access-control caches.
package cache
import (
"context"
"encoding/json"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/pkg/util"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
)
const fileAccessInvalidationChannel = "upload:file_access_invalidation"
var (
accessCacheOnce sync.Once
fileAccessWhitelistMu sync.RWMutex
fileAccessWhitelistTypes map[string]struct{}
fileAccessWhitelistValid bool
fileAccessWhitelistCheckedAt time.Time
)
// ResetAccessCaches clears in-process upload access caches.
func ResetAccessCaches() {
uploadstorage.ResetMigrationAccessCache()
fileAccessWhitelistMu.Lock()
fileAccessWhitelistValid = false
fileAccessWhitelistTypes = nil
fileAccessWhitelistMu.Unlock()
}
// PublishAccessCacheInvalidation broadcasts upload access cache eviction to all nodes.
func PublishAccessCacheInvalidation(ctx context.Context) {
if db.Redis != nil {
_ = db.Redis.Publish(ctx, fileAccessInvalidationChannel, "reset").Err()
}
}
func ensureAccessCacheListener() {
accessCacheOnce.Do(startAccessCacheInvalidationListener)
}
func startAccessCacheInvalidationListener() {
if db.Redis == nil {
return
}
util.Go(func() {
pubsub := db.Redis.Subscribe(
context.Background(),
objectstore.ConfigInvalidationChannel,
fileAccessInvalidationChannel,
)
defer func() {
_ = pubsub.Close()
}()
for range pubsub.Channel() {
ResetAccessCaches()
}
})
}
// IsFilePublic reports whether uploadType is in the public access whitelist.
func IsFilePublic(ctx context.Context, uploadType string) bool {
whitelist := loadFileAccessWhitelist(ctx)
_, ok := whitelist[strings.ToLower(uploadType)]
return ok
}
func loadFileAccessWhitelist(ctx context.Context) map[string]struct{} {
ensureAccessCacheListener()
fileAccessWhitelistMu.RLock()
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < time.Duration(shared.AccessCacheTTL)*time.Second {
types := fileAccessWhitelistTypes
fileAccessWhitelistMu.RUnlock()
return types
}
fileAccessWhitelistMu.RUnlock()
fileAccessWhitelistMu.Lock()
defer fileAccessWhitelistMu.Unlock()
if fileAccessWhitelistValid && time.Since(fileAccessWhitelistCheckedAt) < time.Duration(shared.AccessCacheTTL)*time.Second {
return fileAccessWhitelistTypes
}
fileAccessWhitelistTypes = fetchFileAccessWhitelist(ctx)
fileAccessWhitelistValid = true
fileAccessWhitelistCheckedAt = time.Now()
return fileAccessWhitelistTypes
}
func fetchFileAccessWhitelist(ctx context.Context) map[string]struct{} {
whitelist := parseFileAccessWhitelist(ctx)
types := make(map[string]struct{}, len(whitelist))
for _, item := range whitelist {
types[strings.ToLower(item)] = struct{}{}
}
return types
}
func parseFileAccessWhitelist(ctx context.Context) []string {
sc, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyFileAccessWhitelist)
if err != nil || sc.Value == "" {
return []string{shared.DefaultPublicUploadType}
}
var whitelist []string
if err := json.Unmarshal([]byte(sc.Value), &whitelist); err == nil && len(whitelist) > 0 {
return whitelist
}
whitelist = parseCommaSeparatedWhitelist(sc.Value)
if len(whitelist) == 0 {
return []string{shared.DefaultPublicUploadType}
}
return whitelist
}
func parseCommaSeparatedWhitelist(value string) []string {
parts := strings.Split(value, ",")
whitelist := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
whitelist = append(whitelist, part)
}
}
return whitelist
}
+101
View File
@@ -0,0 +1,101 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cache
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
)
func TestLoadMigrationAccessStateCachesResult(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
first := uploadstorage.LoadMigrationAccessState(ctx)
second := uploadstorage.LoadMigrationAccessState(ctx)
if first.ReadOnly != second.ReadOnly {
t.Fatalf("readOnly mismatch: first=%v second=%v", first.ReadOnly, second.ReadOnly)
}
if first.HasTarget != second.HasTarget {
t.Fatalf("hasTarget mismatch: first=%v second=%v", first.HasTarget, second.HasTarget)
}
}
func TestIsFilePublicUsesCachedWhitelist(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
if !IsFilePublic(ctx, "avatar") {
t.Fatal("expected avatar to be public by default")
}
if IsFilePublic(ctx, "attachment") {
t.Fatal("expected attachment to be private by default")
}
if !IsFilePublic(ctx, "AVATAR") {
t.Fatal("expected whitelist lookup to be case-insensitive")
}
}
func TestResetAccessCachesRefreshesWhitelist(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
if !IsFilePublic(ctx, "avatar") {
t.Fatal("expected seeded avatar whitelist before reset")
}
var sc model.SystemConfig
if err := dbConn.Where("key = ?", model.ConfigKeyFileAccessWhitelist).First(&sc).Error; err != nil {
t.Fatalf("load whitelist config: %v", err)
}
sc.Value = `["attachment"]`
if err := dbConn.Save(&sc).Error; err != nil {
t.Fatalf("save whitelist config: %v", err)
}
if err := db.HSetJSON(ctx, repository.SystemConfigRedisHashKey, model.ConfigKeyFileAccessWhitelist, &sc); err != nil {
t.Fatalf("refresh whitelist redis cache: %v", err)
}
repository.ResetSystemConfigRAMCacheForTest()
ResetAccessCaches()
if !IsFilePublic(ctx, "attachment") {
t.Fatal("expected attachment to be public after whitelist refresh")
}
if IsFilePublic(ctx, "avatar") {
t.Fatal("expected avatar to be private after whitelist refresh")
}
}
func TestAccessCacheTTLExpires(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetAccessCaches()
ctx := context.Background()
_ = loadFileAccessWhitelist(ctx)
fileAccessWhitelistMu.Lock()
fileAccessWhitelistCheckedAt = time.Now().Add(-time.Duration(shared.AccessCacheTTL)*time.Second - time.Second)
fileAccessWhitelistMu.Unlock()
// Should still work after TTL by reloading from config.
if !IsFilePublic(ctx, "avatar") {
t.Fatal("expected whitelist reload after TTL expiration")
}
}
+160
View File
@@ -0,0 +1,160 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cache
import (
"context"
"encoding/json"
"fmt"
"sync"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/cache/ram"
"github.com/Rain-kl/Wavelet/pkg/util"
)
const (
uploadMetaRedisCacheTTL = 30 * 60 // seconds
uploadMetaRAMMaximumSize = 4096
uploadMetaInvalidationChan = "upload:meta_invalidation"
)
type uploadMetaInvalidationMessage struct {
ID uint64 `json:"id"`
}
var (
uploadMetaRAM = ram.MustNew[uint64, model.Upload](ram.Options{MaximumSize: uploadMetaRAMMaximumSize})
uploadMetaListenerOnce sync.Once
uploadMetaListenerCtx context.Context
uploadMetaListenerCancel context.CancelFunc
uploadMetaListenerDone chan struct{}
)
func uploadMetaRedisKey(id uint64) string {
return fmt.Sprintf("upload:meta:%d", id)
}
func cloneUpload(upload model.Upload) model.Upload {
return upload
}
func ensureUploadMetaCacheListener() {
if db.Redis == nil {
return
}
uploadMetaListenerOnce.Do(startUploadMetaCacheInvalidationListener)
}
func startUploadMetaCacheInvalidationListener() {
uploadMetaListenerCtx, uploadMetaListenerCancel = context.WithCancel(context.Background())
uploadMetaListenerDone = make(chan struct{})
redisClient := db.Redis // 捕获当前客户端:goroutine 不读可变全局,避免与测试置空 db.Redis 竞争
util.Go(func() {
defer close(uploadMetaListenerDone)
pubsub := redisClient.Subscribe(uploadMetaListenerCtx, uploadMetaInvalidationChan)
defer func() {
_ = pubsub.Close()
}()
util.Go(func() {
<-uploadMetaListenerCtx.Done()
_ = pubsub.Close()
})
for msg := range pubsub.Channel() {
var payload uploadMetaInvalidationMessage
if err := json.Unmarshal([]byte(msg.Payload), &payload); err != nil || payload.ID == 0 {
uploadMetaRAM.InvalidateAll()
continue
}
uploadMetaRAM.Invalidate(payload.ID)
}
})
}
func publishUploadMetaRAMInvalidation(ctx context.Context, id uint64) {
if db.Redis == nil {
return
}
payload, err := json.Marshal(uploadMetaInvalidationMessage{ID: id})
if err != nil {
return
}
_ = db.Redis.Publish(ctx, uploadMetaInvalidationChan, payload).Err()
}
// GetUploadByID loads upload metadata from RAM, Redis, or the database.
func GetUploadByID(ctx context.Context, id uint64) (model.Upload, error) {
ensureUploadMetaCacheListener()
if upload, ok := uploadMetaRAM.GetIfPresent(id); ok {
return cloneUpload(upload), nil
}
key := uploadMetaRedisKey(id)
if db.Redis != nil {
var upload model.Upload
if err := db.GetJSON(ctx, key, &upload); err == nil {
uploadMetaRAM.Set(id, cloneUpload(upload))
return upload, nil
}
}
var upload model.Upload
if err := db.DB(ctx).
Where("id = ? AND status IN (?, ?)", id, model.UploadStatusPending, model.UploadStatusUsed).
First(&upload).Error; err != nil {
return model.Upload{}, err
}
SetUploadMetaCache(ctx, &upload)
return upload, nil
}
// SetUploadMetaCache populates RAM and Redis upload metadata caches.
func SetUploadMetaCache(ctx context.Context, upload *model.Upload) {
ensureUploadMetaCacheListener()
if upload == nil {
return
}
cloned := cloneUpload(*upload)
uploadMetaRAM.Set(upload.ID, cloned)
if db.Redis != nil {
_ = db.SetJSON(ctx, uploadMetaRedisKey(upload.ID), cloned, uploadMetaRedisCacheTTL)
}
}
// InvalidateUploadMetaCache clears RAM and Redis upload metadata caches and notifies peer nodes.
func InvalidateUploadMetaCache(ctx context.Context, id uint64) {
ensureUploadMetaCacheListener()
uploadMetaRAM.Invalidate(id)
if db.Redis != nil {
_ = db.Redis.Del(ctx, db.PrefixedKey(uploadMetaRedisKey(id))).Err()
publishUploadMetaRAMInvalidation(ctx, id)
}
}
// ResetUploadMetaCacheForTest clears the in-process upload metadata RAM cache.
func ResetUploadMetaCacheForTest() {
uploadMetaRAM.InvalidateAll()
}
// StopUploadMetaCacheListener stops the Redis Pub/Sub subscription listener and resets the sync.Once guard.
func StopUploadMetaCacheListener() {
if uploadMetaListenerCancel != nil {
uploadMetaListenerCancel()
if uploadMetaListenerDone != nil {
<-uploadMetaListenerDone // 等待 goroutine 退出,保证之后置空 db.Redis 不再竞争
}
uploadMetaListenerCancel = nil
uploadMetaListenerDone = nil
}
uploadMetaListenerOnce = sync.Once{}
}
+287
View File
@@ -0,0 +1,287 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package cache
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"gorm.io/gorm"
)
func init() {
testhelper.RegisterCleanup(func() {
StopUploadMetaCacheListener()
ResetUploadMetaCacheForTest()
})
}
func seedUpload(t *testing.T, dbConn *gorm.DB, upload model.Upload) {
t.Helper()
if err := dbConn.Create(&upload).Error; err != nil {
t.Fatalf("create upload: %v", err)
}
}
func TestGetUploadByIDLoadsFromDBAndPopulatesCache(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := model.Upload{
ID: 91001,
UserID: 1,
FileName: "cached.png",
FilePath: "cached.png",
FileSize: 12,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID: %v", err)
}
if got.ID != upload.ID || got.FileName != upload.FileName {
t.Fatalf("unexpected upload: %+v", got)
}
var redisUpload model.Upload
if err := db.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err != nil {
t.Fatalf("redis cache miss after DB load: %v", err)
}
if redisUpload.ID != upload.ID {
t.Fatalf("redis upload id mismatch: got=%d want=%d", redisUpload.ID, upload.ID)
}
if err := dbConn.Delete(&model.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
gotCached, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID from RAM cache: %v", err)
}
if gotCached.ID != upload.ID {
t.Fatalf("expected RAM cache hit for upload %d", upload.ID)
}
}
func TestGetUploadByIDReadsFromRedisWhenRAMEmpty(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := model.Upload{
ID: 91002,
UserID: 1,
FileName: "redis.png",
FilePath: "redis.png",
FileSize: 8,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusPending,
AccessMode: 0,
}
seedUpload(t, dbConn, upload)
SetUploadMetaCache(ctx, &upload)
ResetUploadMetaCacheForTest()
if err := dbConn.Delete(&model.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID from redis: %v", err)
}
if got.ID != upload.ID || got.FileName != upload.FileName {
t.Fatalf("unexpected upload from redis: %+v", got)
}
}
func TestInvalidateUploadMetaCacheClearsRAMAndRedis(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := model.Upload{
ID: 91003,
UserID: 1,
FileName: "invalidate.png",
FilePath: "invalidate.png",
FileSize: 4,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
SetUploadMetaCache(ctx, &upload)
InvalidateUploadMetaCache(ctx, upload.ID)
var redisUpload model.Upload
if err := db.GetJSON(ctx, uploadMetaRedisKey(upload.ID), &redisUpload); err == nil {
t.Fatal("expected redis cache to be invalidated")
}
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID after invalidate should reload from DB: %v", err)
}
if got.ID != upload.ID {
t.Fatalf("unexpected upload reloaded from DB: %+v", got)
}
}
func TestUploadMetaInvalidationPubSubClearsPeerRAM(t *testing.T) {
StopUploadMetaCacheListener()
defer StopUploadMetaCacheListener()
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := model.Upload{
ID: 91006,
UserID: 1,
FileName: "pubsub.png",
FilePath: "pubsub.png",
FileSize: 4,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
if _, err := GetUploadByID(ctx, upload.ID); err != nil {
t.Fatalf("GetUploadByID: %v", err)
}
time.Sleep(50 * time.Millisecond) // allow pub/sub listener to subscribe
if err := dbConn.Delete(&model.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
if _, err := GetUploadByID(ctx, upload.ID); err != nil {
t.Fatalf("expected cache hit before pub/sub invalidation: %v", err)
}
payload, err := json.Marshal(uploadMetaInvalidationMessage{ID: upload.ID})
if err != nil {
t.Fatalf("marshal invalidation payload: %v", err)
}
if err := db.Redis.Publish(ctx, uploadMetaInvalidationChan, string(payload)).Err(); err != nil {
t.Fatalf("publish invalidation: %v", err)
}
deadline := time.Now().Add(2 * time.Second)
ramCleared := false
for time.Now().Before(deadline) {
if _, ok := uploadMetaRAM.GetIfPresent(upload.ID); !ok {
ramCleared = true
break
}
time.Sleep(20 * time.Millisecond)
}
if !ramCleared {
t.Fatal("expected peer RAM cache to be cleared by pub/sub")
}
if err := db.Redis.Del(ctx, db.PrefixedKey(uploadMetaRedisKey(upload.ID))).Err(); err != nil {
t.Fatalf("delete redis cache: %v", err)
}
if _, err := GetUploadByID(ctx, upload.ID); err == nil {
t.Fatal("expected cache miss after pub/sub RAM eviction and redis delete")
}
}
func TestGetUploadByIDSkipsDeletedUploads(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
ctx := context.Background()
upload := model.Upload{
ID: 91004,
UserID: 1,
FileName: "deleted.png",
FilePath: "deleted.png",
FileSize: 4,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusDeleted,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
if _, err := GetUploadByID(ctx, upload.ID); err == nil {
t.Fatal("expected error for deleted upload")
}
}
func TestGetUploadByIDWorksWithRedisDisabled(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ResetUploadMetaCacheForTest()
redisClient := db.Redis
db.Redis = nil
t.Cleanup(func() {
db.Redis = redisClient
StopUploadMetaCacheListener()
})
ctx := context.Background()
upload := model.Upload{
ID: 91005,
UserID: 1,
FileName: "ram-only.png",
FilePath: "ram-only.png",
FileSize: 6,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
seedUpload(t, dbConn, upload)
got, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID without redis: %v", err)
}
if got.ID != upload.ID {
t.Fatalf("unexpected upload: %+v", got)
}
if err := dbConn.Delete(&model.Upload{}, upload.ID).Error; err != nil {
t.Fatalf("delete upload from db: %v", err)
}
gotCached, err := GetUploadByID(ctx, upload.ID)
if err != nil {
t.Fatalf("GetUploadByID from RAM without redis: %v", err)
}
if gotCached.ID != upload.ID {
t.Fatal("expected RAM cache hit when redis is disabled")
}
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package upload 提供文件上传与下载功能
package upload
import "github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
// 文件管理常量
const (
ErrNoFileSelected = shared.ErrNoFileSelected
ErrUnsupportedFormat = shared.ErrUnsupportedFormat
ErrProcessFileFailed = shared.ErrProcessFileFailed
ErrSaveFileFailed = shared.ErrSaveFileFailed
ErrOpenFileFailed = shared.ErrOpenFileFailed
ErrSaveUploadRecordFailed = shared.ErrSaveUploadRecordFailed
ErrGenericFileTooLarge = shared.ErrGenericFileTooLarge
ErrFileContentExtensionMismatch = shared.ErrFileContentExtensionMismatch
ErrFileValidationFailed = shared.ErrFileValidationFailed
ErrInvalidMetadataJSON = shared.ErrInvalidMetadataJSON
ErrInvalidFileID = shared.ErrInvalidFileID
ErrQueryUploadRecordFailed = shared.ErrQueryUploadRecordFailed
ErrInvalidBatchDownloadRequest = shared.ErrInvalidBatchDownloadRequest
ErrInvalidIDValueFormat = shared.ErrInvalidIDValueFormat
ErrRetrieveUploadRecordsFailed = shared.ErrRetrieveUploadRecordsFailed
ErrNoValidFilesForArchive = shared.ErrNoValidFilesForArchive
ErrInvalidParams = shared.ErrInvalidParams
ErrQueryFileCountFailed = shared.ErrQueryFileCountFailed
ErrQueryFileListFailed = shared.ErrQueryFileListFailed
ErrDeleteFileFailed = shared.ErrDeleteFileFailed
ErrStorageReadOnly = shared.ErrStorageReadOnly
ErrS3KeyRequired = shared.ErrS3KeyRequired
ErrS3KeyTooLongFormat = shared.ErrS3KeyTooLongFormat
ErrS3KeyStartsWithSlash = shared.ErrS3KeyStartsWithSlash
ErrS3KeyContainsNullBytes = shared.ErrS3KeyContainsNullBytes
ErrQueryUnusedUploadsFailed = shared.ErrQueryUnusedUploadsFailed
)
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package upload
import (
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/handler"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/ingest"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
uploadtask "github.com/Rain-kl/Wavelet/plugins/domain/upload/task"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/util"
)
// HTTP handlers
var (
UploadFile = handler.UploadFile
DownloadFile = handler.DownloadFile
BatchDownloadFiles = handler.BatchDownloadFiles
ListFiles = handler.ListFiles
DeleteFile = handler.DeleteFile
GetDistinctUploadTypes = handler.GetDistinctUploadTypes
ListMyFiles = handler.ListMyFiles
DeleteMyFile = handler.DeleteMyFile
UpdateMyFile = handler.UpdateMyFile
GetFileStats = handler.GetFileStats
ServeFileByID = filesrv.ServeFileByID
)
// Programmatic ingest API
var (
Ingest = ingest.Ingest
Remove = ingest.Remove
RemoveOwned = ingest.RemoveOwned
FindByHash = ingest.FindByHash
)
// Ingest policy constants
const (
PolicyCreate = ingest.PolicyCreate
PolicyDedupNewRecord = ingest.PolicyDedupNewRecord
PolicyResolveExisting = ingest.PolicyResolveExisting
)
type (
// IngestRequest is the programmatic upload ingest payload.
IngestRequest = ingest.Request
// IngestResult reports ingest side effects.
IngestResult = ingest.Result
// IngestPolicy controls hash-collision behavior during ingest.
IngestPolicy = ingest.Policy
)
// Ingest errors
var (
ErrIngestForbidden = ingest.ErrForbidden
ErrIngestStorageReadOnly = ingest.ErrStorageReadOnly
)
// Cache management
var (
ResetAccessCaches = cache.ResetAccessCaches
PublishAccessCacheInvalidation = cache.PublishAccessCacheInvalidation
)
// Stats
var (
// Deprecated: use upload.Ingest or upload.Remove; stats are applied internally.
ApplyUploadStatsAdd = uploadstats.ApplyUploadStatsAdd
// Deprecated: use upload.Ingest or upload.Remove; stats are applied internally.
ApplyUploadStatsRemove = uploadstats.ApplyUploadStatsRemove
RebuildUploadStats = uploadstats.RebuildUploadStats
)
// Utilities
var (
CompressImageToWebP = util.CompressImageToWebP
ValidateS3Key = util.ValidateS3Key
)
// Task identifiers and metadata
const (
StorageMigrationTask = uploadtask.StorageMigrationTask
SystemCleanupTask = uploadtask.SystemCleanupTask
WarmImageCacheTask = uploadtask.WarmImageCacheTask
RebuildUploadStatsTask = uploadtask.RebuildUploadStatsTask
)
var (
// StorageMigrationMeta describes the storage migration async task.
StorageMigrationMeta = uploadtask.StorageMigrationMeta
// SystemCleanupMeta describes the orphaned upload cleanup task.
SystemCleanupMeta = uploadtask.SystemCleanupMeta
// WarmImageCacheMeta describes the image compression cache warmup task.
WarmImageCacheMeta = uploadtask.WarmImageCacheMeta
// RebuildUploadStatsMeta describes the upload stats rebuild task.
RebuildUploadStatsMeta = uploadtask.RebuildUploadStatsMeta
)
// MigrationHandler executes storage migration tasks.
type MigrationHandler = uploadtask.MigrationHandler
// SystemCleanupHandler removes orphaned upload files.
type SystemCleanupHandler = uploadtask.SystemCleanupHandler
// WarmImageCacheHandler pre-warms compressed image caches.
type WarmImageCacheHandler = uploadtask.WarmImageCacheHandler
// RebuildUploadStatsHandler rebuilds upload stats from active records.
type RebuildUploadStatsHandler = uploadtask.RebuildUploadStatsHandler
// WarmImageCachePayload is the payload for image cache warmup tasks.
type WarmImageCachePayload = uploadtask.WarmImageCachePayload
// Ensure task handler types implement required interfaces.
var (
_ task.TaskHandler = (*MigrationHandler)(nil)
_ task.TaskHandler = (*SystemCleanupHandler)(nil)
_ task.TaskHandler = (*RebuildUploadStatsHandler)(nil)
_ interface {
task.TaskHandler
ValidatePayload([]byte) ([]byte, error)
} = (*WarmImageCacheHandler)(nil)
)
@@ -0,0 +1,313 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package filesrv serves uploaded files with access control and image compression.
package filesrv
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"github.com/Rain-kl/Wavelet/internal/infra/diskcache"
"github.com/Rain-kl/Wavelet/internal/model"
appshared "github.com/Rain-kl/Wavelet/internal/shared"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/util"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
"golang.org/x/sync/singleflight"
"gorm.io/gorm"
)
var compressedImageFlight singleflight.Group
type compressedImageCacheResult struct {
bytes []byte
cached bool
err error
}
type fileTypeCategory string
const (
fileTypeImage fileTypeCategory = "image"
fileTypeVideo fileTypeCategory = "video"
fileTypeAudio fileTypeCategory = "audio"
fileTypeOther fileTypeCategory = "other"
)
// ServeFileByID 根据 ID 获取并提供已上传的文件
// @Summary 获取已上传文件
// @Description 根据文件 ID 获取并提供已上传的临时或正式文件,若配置了缓存则优先走本地缓存,否则从 S3 等后端存储读取并流式返回
// @Tags upload
// @Produce octet-stream
// @Param id path string true "文件 ID"
// @Param quality query string false "图片质量 (low, medium, high, origin),默认为 origin"
// @Success 200 {file} file "成功获取文件内容"
// @Failure 400 {object} response.Any "文件 ID 格式错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "文件未找到"
// @Failure 500 {object} response.Any "服务内部错误"
// @Router /f/{id} [get]
func ServeFileByID(c *gin.Context) {
upload, err := GetUploadRecordByID(c)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
response.AbortNotFound(c, "文件记录未找到")
return
}
if _, ok := err.(*strconv.NumError); ok {
response.AbortBadRequest(c, "无效的上传ID")
return
}
response.AbortInternal(c, "服务器内部错误")
return
}
if err := CheckFileAccessPermission(c, upload); err != nil {
response.AbortUnauthorized(c, appshared.UnAuthorized)
return
}
ServeUpload(c, upload)
}
// GetUploadRecordByID 从请求路径参数中解析文件 ID 并从数据库中检索处于 Pending 或 Used 状态的上传记录。
func GetUploadRecordByID(c *gin.Context) (*model.Upload, error) {
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Security-Policy", "sandbox")
idStr := c.Param("id")
uploadID, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
return nil, err
}
upload, err := cache.GetUploadByID(c.Request.Context(), uploadID)
if err != nil {
return nil, err
}
return &upload, nil
}
func getFileTypeCategory(upload *model.Upload) fileTypeCategory {
mime := strings.ToLower(upload.MimeType)
ext := strings.ToLower(upload.Extension)
if strings.HasPrefix(mime, "image/") || util.IsImageExtension(ext) {
return fileTypeImage
}
if strings.HasPrefix(mime, "video/") {
return fileTypeVideo
}
if strings.HasPrefix(mime, "audio/") {
return fileTypeAudio
}
return fileTypeOther
}
// ServeUpload 将已存在的文件内容读取并流式响应给客户端。
func ServeUpload(c *gin.Context, upload *model.Upload) {
setCacheHeaders(c, upload)
category := getFileTypeCategory(upload)
quality := util.NormalizeImageQuality(c.Query("quality"))
switch category {
case fileTypeImage:
if quality != shared.ImageQualityOrigin {
serveCompressedImage(c, upload, quality)
return
}
fallthrough
default:
serveOriginalWithConditionalCheck(c, upload)
}
}
func setCacheHeaders(c *gin.Context, upload *model.Upload) {
if cache.IsFilePublic(c.Request.Context(), upload.Type) {
c.Header("Cache-Control", "public, max-age=31536000")
} else {
c.Header("Cache-Control", "private, no-cache")
}
}
func serveOriginalWithConditionalCheck(c *gin.Context, upload *model.Upload) {
etag := fmt.Sprintf(`W/"%s"`, upload.Hash)
c.Header("ETag", etag)
if c.GetHeader("If-None-Match") == etag {
c.AbortWithStatus(http.StatusNotModified)
return
}
serveOriginal(c, upload)
}
func serveCompressedImage(c *gin.Context, upload *model.Upload, quality string) {
etag := fmt.Sprintf(`W/"%s-%s"`, upload.Hash, quality)
c.Header("ETag", etag)
if c.GetHeader("If-None-Match") == etag {
c.AbortWithStatus(http.StatusNotModified)
return
}
webpBytes, _, err := EnsureCompressedImageCache(c.Request.Context(), upload, quality)
if err != nil {
if len(webpBytes) > 0 {
logger.WarnF(c.Request.Context(), "failed to cache compressed image: %v", err)
c.Data(http.StatusOK, "image/webp", webpBytes)
return
}
logger.ErrorF(c.Request.Context(), "failed to prepare compressed image cache: %v", err)
serveOriginal(c, upload)
return
}
c.Data(http.StatusOK, "image/webp", webpBytes)
}
// EnsureCompressedImageCache returns cached or freshly generated WebP bytes for an upload.
func EnsureCompressedImageCache(
ctx context.Context,
upload *model.Upload,
quality string,
) ([]byte, bool, error) {
cacheStore := diskcache.GetGlobalCache()
cacheKey := ImageCompressionCacheKey(upload, quality)
webpBytes, err := cacheStore.Get(cacheKey)
if err == nil {
return webpBytes, true, nil
}
if !errors.Is(err, diskcache.ErrCacheMiss) {
return nil, false, fmt.Errorf("read compressed image cache: %w", err)
}
result, err, _ := compressedImageFlight.Do(cacheKey, func() (any, error) {
return generateCompressedImageCache(ctx, upload, quality, cacheKey)
})
if err != nil {
return nil, false, err
}
res := result.(compressedImageCacheResult)
return res.bytes, res.cached, res.err
}
func generateCompressedImageCache(
ctx context.Context,
upload *model.Upload,
quality string,
cacheKey string,
) (compressedImageCacheResult, error) {
cacheStore := diskcache.GetGlobalCache()
webpBytes, err := cacheStore.Get(cacheKey)
if err == nil {
return compressedImageCacheResult{bytes: webpBytes, cached: true}, nil
}
if !errors.Is(err, diskcache.ErrCacheMiss) {
return compressedImageCacheResult{}, fmt.Errorf("read compressed image cache: %w", err)
}
origBytes, err := getOriginalFileBytes(ctx, upload)
if err != nil {
return compressedImageCacheResult{}, fmt.Errorf("read original image: %w", err)
}
webpBytes, err = util.CompressImageToWebP(bytes.NewReader(origBytes), quality)
if err != nil {
return compressedImageCacheResult{}, fmt.Errorf("compress image to WebP: %w", err)
}
if err := cacheStore.Set(cacheKey, webpBytes, diskcache.NoExpiration); err != nil {
return compressedImageCacheResult{
bytes: webpBytes,
err: fmt.Errorf("write compressed image cache: %w", err),
}, nil
}
return compressedImageCacheResult{bytes: webpBytes}, nil
}
// ImageCompressionCacheKey returns the disk cache key for a compressed upload image.
func ImageCompressionCacheKey(upload *model.Upload, quality string) string {
return fmt.Sprintf(
"upload_webp_v1_%d_%d_%d_%s_%s",
upload.ID,
upload.UpdatedAt.UnixNano(),
upload.FileSize,
upload.Hash,
quality,
)
}
func serveOriginal(c *gin.Context, upload *model.Upload) {
obj, err := uploadstorage.OpenStoredObject(c.Request.Context(), upload)
if err != nil {
response.AbortNotFound(c, "文件未找到")
return
}
defer func() { _ = obj.Body.Close() }()
c.DataFromReader(http.StatusOK, obj.ContentLength, obj.ContentType, obj.Body, nil)
}
func getOriginalFileBytes(ctx context.Context, upload *model.Upload) ([]byte, error) {
obj, err := uploadstorage.OpenStoredObject(ctx, upload)
if err != nil {
return nil, err
}
defer func() { _ = obj.Body.Close() }()
return io.ReadAll(obj.Body)
}
func checkPrivateFileOwner(c *gin.Context, ownerID uint64) error {
var currUser *model.User
var err error
if u, ok := auth.GetFromContext[*model.User](c, auth.UserObjKey); ok && u != nil {
currUser = u
} else {
currUser, err = auth.GetUserFromRequest(c)
if err != nil {
return err
}
}
if currUser.IsAdmin {
return nil
}
if currUser.ID != ownerID {
return errors.New("forbidden: cross-user access denied")
}
return nil
}
// CheckFileAccessPermission 校验文件是否可以被当前请求访问
func CheckFileAccessPermission(c *gin.Context, upload *model.Upload) error {
if upload.AccessMode == 0 {
return checkPrivateFileOwner(c, upload.UserID)
}
if !cache.IsFilePublic(c.Request.Context(), upload.Type) {
if _, ok := auth.GetFromContext[*model.User](c, auth.UserObjKey); !ok {
if _, err := auth.GetUserFromRequest(c); err != nil {
return err
}
}
}
return nil
}
@@ -0,0 +1,380 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package filesrv
import (
"bytes"
"context"
"encoding/json"
"image"
"image/color"
"image/png"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/Rain-kl/Wavelet/internal/infra/diskcache"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
appshared "github.com/Rain-kl/Wavelet/internal/shared"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/util"
"github.com/gin-contrib/sessions"
"github.com/gin-contrib/sessions/cookie"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
func init() {
testhelper.RegisterCleanup(cache.ResetUploadMetaCacheForTest)
}
func TestServeFileByIDAccessControl(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
cache.ResetAccessCaches()
tempDir := t.TempDir()
configureLocalStorageRoot(t, dbConn, tempDir)
// Create a user in DB
user := model.User{
ID: 12345,
Username: "file_test_user",
IsActive: true,
}
if err := dbConn.Create(&user).Error; err != nil {
t.Fatalf("failed to create user: %v", err)
}
// Create an access token for this user
tokenStr := "test-secret-token-123"
tokenHash := model.HashToken(tokenStr)
tokenRecord := model.AccessToken{
UserID: user.ID,
Name: "test_token",
TokenHash: tokenHash,
}
if err := dbConn.Create(&tokenRecord).Error; err != nil {
t.Fatalf("failed to create token: %v", err)
}
// Create two files: one in whitelist (avatar), one not in whitelist (attachment)
avatarFile := model.Upload{
ID: 8001,
UserID: user.ID,
FileName: "avatar.png",
FilePath: "avatar.png",
FileSize: 5,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
attachmentFile := model.Upload{
ID: 8002,
UserID: user.ID,
FileName: "doc.pdf",
FilePath: "doc.pdf",
FileSize: 5,
MimeType: "application/pdf",
Extension: "pdf",
Type: "attachment",
Status: model.UploadStatusUsed,
AccessMode: 1,
}
if err := os.WriteFile(filepath.Join(tempDir, "avatar.png"), []byte("image"), 0644); err != nil {
t.Fatalf("failed to write avatar file: %v", err)
}
if err := os.WriteFile(filepath.Join(tempDir, "doc.pdf"), []byte("bytes"), 0644); err != nil {
t.Fatalf("failed to write attachment file: %v", err)
}
dbConn.Create(&avatarFile)
dbConn.Create(&attachmentFile)
// Set up router
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(response.ErrorHandlerMiddleware())
store := cookie.NewStore([]byte("secret"))
r.Use(sessions.Sessions("test_session", store))
r.GET("/f/:id", ServeFileByID)
t.Run("whitelisted file type (avatar) accessed without authentication", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8001", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("expected 200, got %d. Body: %s", w.Code, w.Body.String())
}
if w.Body.String() != "image" {
t.Errorf("expected 'image', got %q", w.Body.String())
}
})
t.Run("non-whitelisted file type (attachment) accessed without authentication returns 401", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8002", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized {
t.Errorf("expected 401, got %d. Body: %s", w.Code, w.Body.String())
}
var body map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("failed to parse JSON: %v", err)
}
if body["error_msg"] != appshared.UnAuthorized {
t.Errorf("expected error_msg %q, got %v", appshared.UnAuthorized, body["error_msg"])
}
})
t.Run("non-whitelisted file type (attachment) accessed with valid token succeeds", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/8002", nil)
req.Header.Set("X-Access-Token", tokenStr)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("expected 200, got %d. Body: %s", w.Code, w.Body.String())
}
if w.Body.String() != "bytes" {
t.Errorf("expected 'bytes', got %q", w.Body.String())
}
})
t.Run("accessing non-existent file returns 404", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/9999", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
t.Errorf("expected 404, got %d", w.Code)
}
})
}
func TestImageCompression(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
tempDir := t.TempDir()
configureLocalStorageRoot(t, dbConn, tempDir)
cache := diskcache.GetGlobalCache()
if err := cache.Clear(); err != nil {
t.Fatalf("failed to clear disk cache before test: %v", err)
}
defer func() {
if err := cache.Clear(); err != nil {
t.Errorf("failed to clear disk cache after test: %v", err)
}
}()
// Create test user
user := model.User{
ID: 555,
Username: "compress_tester",
IsActive: true,
}
dbConn.Create(&user)
// Create a 1x1 pixel PNG image
img := image.NewRGBA(image.Rect(0, 0, 1, 1))
img.Set(0, 0, color.RGBA{R: 255, G: 0, B: 0, A: 255})
var pngBuf bytes.Buffer
if err := png.Encode(&pngBuf, img); err != nil {
t.Fatalf("failed to encode test png: %v", err)
}
filePath := filepath.Join(tempDir, "test_image.png")
if err := os.WriteFile(filePath, pngBuf.Bytes(), 0644); err != nil {
t.Fatalf("failed to write test png: %v", err)
}
// Save upload record to DB
uploadRecord := model.Upload{
ID: 3001,
UserID: user.ID,
FileName: "test_image.png",
FilePath: "test_image.png",
FileSize: int64(pngBuf.Len()),
MimeType: "image/png",
Extension: "png",
Type: "avatar", // Whitelisted by default
Status: model.UploadStatusUsed,
AccessMode: 1,
}
dbConn.Create(&uploadRecord)
// Setup Router
gin.SetMode(gin.TestMode)
r := gin.New()
r.GET("/f/:id", ServeFileByID)
t.Run("serve original file without compress parameter", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
// Content-Type should be image/png (default local serving type)
if w.Header().Get("Content-Type") != "image/png" {
t.Errorf("expected Content-Type image/png, got %s", w.Header().Get("Content-Type"))
}
if len(w.Body.Bytes()) != pngBuf.Len() {
t.Errorf("expected body size %d, got %d", pngBuf.Len(), len(w.Body.Bytes()))
}
})
t.Run("serve compressed WebP file with medium quality", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d. Body: %s", w.Code, w.Body.String())
}
// Content-Type should be image/webp
if w.Header().Get("Content-Type") != "image/webp" {
t.Errorf("expected Content-Type image/webp, got %s", w.Header().Get("Content-Type"))
}
cacheKey := ImageCompressionCacheKey(&uploadRecord, shared.ImageQualityMedium)
cachedBytes, err := cache.Get(cacheKey)
if err != nil {
t.Fatalf("disk cache Get(%q) returned error: %v", cacheKey, err)
}
if !bytes.Equal(cachedBytes, w.Body.Bytes()) {
t.Errorf("cached compressed image differs from response")
}
if err := os.Remove(filePath); err != nil {
t.Fatalf("failed to remove source image before cache-hit request: %v", err)
}
t.Cleanup(func() {
if err := os.WriteFile(filePath, pngBuf.Bytes(), 0644); err != nil {
t.Errorf("failed to restore source image: %v", err)
}
})
w2 := httptest.NewRecorder()
r.ServeHTTP(w2, req)
if w2.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w2.Code)
}
if !bytes.Equal(w2.Body.Bytes(), cachedBytes) {
t.Errorf("cache-hit response differs from cached compressed image")
}
})
t.Run("serve compressed WebP file and check cache headers and 304 Not Modified", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
etag := w.Header().Get("ETag")
if etag == "" {
t.Error("expected ETag header, got empty")
}
cacheControl := w.Header().Get("Cache-Control")
if cacheControl != "public, max-age=31536000" {
t.Errorf("expected Cache-Control 'public, max-age=31536000', got %q", cacheControl)
}
// Perform conditional GET request
reqCond, _ := http.NewRequest("GET", "/f/3001?quality=medium", nil)
reqCond.Header.Set("If-None-Match", etag)
wCond := httptest.NewRecorder()
r.ServeHTTP(wCond, reqCond)
if wCond.Code != http.StatusNotModified {
t.Errorf("expected status 304, got %d", wCond.Code)
}
})
t.Run("serve original file with origin quality", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/f/3001?quality=origin", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
if w.Header().Get("Content-Type") != "image/png" {
t.Errorf("expected Content-Type image/png, got %s", w.Header().Get("Content-Type"))
}
if !bytes.Equal(w.Body.Bytes(), pngBuf.Bytes()) {
t.Errorf("origin-quality response differs from original image")
}
})
}
func TestNormalizeImageQuality(t *testing.T) {
tests := []struct {
name string
quality string
want string
}{
{name: shared.ImageQualityLow, quality: shared.ImageQualityLow, want: shared.ImageQualityLow},
{name: shared.ImageQualityMedium, quality: shared.ImageQualityMedium, want: shared.ImageQualityMedium},
{name: shared.ImageQualityHigh, quality: shared.ImageQualityHigh, want: shared.ImageQualityHigh},
{name: "origin", quality: "origin", want: "origin"},
{name: "uppercase", quality: "LOW", want: shared.ImageQualityLow},
{name: "empty", quality: "", want: "origin"},
{name: "invalid", quality: "maximum", want: "origin"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := util.NormalizeImageQuality(tt.quality); got != tt.want {
t.Errorf("NormalizeImageQuality(%q) = %q, want %q", tt.quality, got, tt.want)
}
})
}
}
func configureLocalStorageRoot(t *testing.T, dbConn *gorm.DB, tempDir string) {
var sc model.SystemConfig
if err := dbConn.Where("key = ?", model.ConfigKeyStorageConfig).First(&sc).Error; err != nil {
t.Fatalf("failed to find storage config: %v", err)
}
var cfg objectstore.Config
if err := json.Unmarshal([]byte(sc.Value), &cfg); err != nil {
t.Fatalf("failed to unmarshal storage config: %v", err)
}
cfg.Local.Root = tempDir
newVal, err := json.Marshal(cfg)
if err != nil {
t.Fatalf("failed to marshal storage config: %v", err)
}
sc.Value = string(newVal)
if err := dbConn.Save(&sc).Error; err != nil {
t.Fatalf("failed to save storage config: %v", err)
}
_ = db.HSetJSON(context.Background(), repository.SystemConfigRedisHashKey, sc.Key, &sc)
repository.ResetSystemConfigRAMCacheForTest()
objectstore.ResetCache()
}
@@ -0,0 +1,302 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package handler
import (
"net/http"
"strconv"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/ingest"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"github.com/gin-gonic/gin"
)
type listFilesRequest struct {
Page int `form:"page"`
PageSize int `form:"page_size"`
Keyword string `form:"keyword"`
Type string `form:"type"`
Extension string `form:"extension"`
UserID uint64 `form:"user_id"`
}
type listFilesResponse struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []model.Upload `json:"items"`
}
// ListFiles 获取系统上传的文件列表
// @Summary 获取文件列表
// @Description 分页获取系统上传的文件列表,支持文件名关键词、业务类型、扩展名、上传用户ID过滤
// @Tags admin
// @Produce json
// @Param page query int false "页码(默认 1)"
// @Param page_size query int false "每页数量(默认 20,最大 100)"
// @Param keyword query string false "文件名关键词(模糊匹配)"
// @Param type query string false "业务分类过滤"
// @Param extension query string false "扩展名过滤"
// @Param user_id query uint64 false "上传用户 ID"
// @Security SessionCookie
// @Success 200 {object} response.Any{data=listFilesResponse} "查询成功"
// @Failure 401 {object} response.Any "未登录"
// @Failure 403 {object} response.Any "无管理员权限"
// @Router /api/v1/admin/uploads [get]
func ListFiles(c *gin.Context) {
ctx := c.Request.Context()
var req listFilesRequest
if err := c.ShouldBindQuery(&req); err != nil {
response.AbortBadRequest(c, shared.ErrInvalidParams)
return
}
if req.Page <= 0 {
req.Page = 1
}
if req.PageSize <= 0 || req.PageSize > 100 {
req.PageSize = 20
}
total, items, err := listUploadFiles(ctx, repository.UploadListFilter{
UserID: req.UserID,
Keyword: req.Keyword,
Type: req.Type,
Extension: req.Extension,
Page: req.Page,
PageSize: req.PageSize,
})
if err != nil {
response.AbortBadRequest(c, shared.ErrQueryFileListFailed)
return
}
c.JSON(http.StatusOK, response.OK(listFilesResponse{
Total: total,
Page: req.Page,
PageSize: req.PageSize,
Items: items,
}))
}
// DeleteFile 软删除文件记录
// @Summary 删除文件
// @Description 将文件状态置为 deleted(软删除),不会立即清理底层存储对象
// @Tags admin
// @Produce json
// @Param id path string true "文件 ID"
// @Security SessionCookie
// @Success 200 {object} response.Any "删除成功"
// @Failure 403 {object} response.Any "无权操作"
// @Failure 404 {object} response.Any "文件不存在"
// @Router /api/v1/admin/uploads/{id} [delete]
func DeleteFile(c *gin.Context) {
ctx := c.Request.Context()
if uploadstorage.ReadOnly(ctx) {
response.AbortConflict(c, shared.ErrStorageReadOnly)
return
}
uploadID, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
response.AbortBadRequest(c, shared.ErrInvalidFileID)
return
}
if _, err := softDeleteUpload(ctx, uploadID); err != nil {
if isRecordNotFound(err) {
response.AbortNotFound(c, "文件记录未找到")
return
}
response.AbortBadRequest(c, shared.ErrDeleteFileFailed)
return
}
c.JSON(http.StatusOK, response.OKNil())
}
// GetDistinctUploadTypes 获取数据库中所有已存在的文件业务类型
// @Summary 获取文件业务类型列表
// @Description 返回数据库中所有已上传文件实际拥有的业务类型列表
// @Tags admin
// @Produce json
// @Security SessionCookie
// @Success 200 {object} response.Any{data=[]string} "业务类型列表"
// @Failure 401 {object} response.Any "未登录"
// @Failure 403 {object} response.Any "无管理员权限"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/admin/uploads/types [get]
func GetDistinctUploadTypes(c *gin.Context) {
types, err := listDistinctUploadTypes(c.Request.Context())
if err != nil {
response.AbortInternal(c, err.Error())
return
}
c.JSON(http.StatusOK, response.OK(types))
}
type listMyFilesRequest struct {
Page int `form:"page"`
PageSize int `form:"page_size"`
Keyword string `form:"keyword"`
Type string `form:"type"`
Extension string `form:"extension"`
}
type listMyFilesResponse struct {
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Items []model.Upload `json:"items"`
}
// ListMyFiles 获取当前用户上传的文件列表
// @Summary 获取我的文件列表
// @Description 分页获取当前登录用户上传的文件,支持文件名关键词、业务类型、扩展名过滤
// @Tags upload
// @Produce json
// @Param page query int false "页码(默认 1)"
// @Param page_size query int false "每页数量(默认 20,最大 100)"
// @Param keyword query string false "文件名关键词(模糊匹配)"
// @Param type query string false "业务分类过滤"
// @Param extension query string false "扩展名过滤"
// @Security SessionCookie
// @Success 200 {object} response.Any{data=listMyFilesResponse} "查询成功"
// @Failure 401 {object} response.Any "未登录"
// @Router /api/v1/upload/my [get]
func ListMyFiles(c *gin.Context) {
currUser, _ := auth.GetFromContext[*model.User](c, auth.UserObjKey)
ctx := c.Request.Context()
var req listMyFilesRequest
if err := c.ShouldBindQuery(&req); err != nil {
response.AbortBadRequest(c, shared.ErrInvalidParams)
return
}
if req.Page <= 0 {
req.Page = 1
}
if req.PageSize <= 0 || req.PageSize > 100 {
req.PageSize = 20
}
total, items, err := listMyUploadFiles(ctx, currUser.ID, repository.UploadListFilter{
Keyword: req.Keyword,
Type: req.Type,
Extension: req.Extension,
Page: req.Page,
PageSize: req.PageSize,
})
if err != nil {
response.AbortBadRequest(c, shared.ErrQueryFileListFailed)
return
}
c.JSON(http.StatusOK, response.OK(listMyFilesResponse{
Total: total,
Page: req.Page,
PageSize: req.PageSize,
Items: items,
}))
}
// DeleteMyFile 软删除当前用户本人的文件
// @Summary 删除我的文件
// @Description 将当前用户本人的文件状态置为 deleted(软删除)
// @Tags upload
// @Produce json
// @Param id path string true "文件 ID"
// @Security SessionCookie
// @Success 200 {object} response.Any "删除成功"
// @Failure 403 {object} response.Any "无权操作"
// @Failure 404 {object} response.Any "文件不存在"
// @Router /api/v1/upload/{id} [delete]
func DeleteMyFile(c *gin.Context) {
currUser, _ := auth.GetFromContext[*model.User](c, auth.UserObjKey)
ctx := c.Request.Context()
if uploadstorage.ReadOnly(ctx) {
response.AbortConflict(c, shared.ErrStorageReadOnly)
return
}
uploadID, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
response.AbortBadRequest(c, shared.ErrInvalidFileID)
return
}
if _, err := softDeleteOwnedUpload(ctx, currUser.ID, uploadID); err != nil {
if isRecordNotFound(err) {
response.AbortNotFound(c, "文件记录未找到")
return
}
if err == ingest.ErrForbidden {
response.AbortForbidden(c, "无权操作")
return
}
response.AbortBadRequest(c, shared.ErrDeleteFileFailed)
return
}
c.JSON(http.StatusOK, response.OKNil())
}
type updateMyFileRequest struct {
FileName string `json:"file_name" binding:"max=255"`
AccessMode *int `json:"access_mode" binding:"omitempty,oneof=0 1"`
}
// UpdateMyFile 更新当前用户本人的文件信息
// @Summary 更新我的文件信息
// @Description 更新当前用户本人的文件名或访问权限模式 (AccessMode)
// @Tags upload
// @Accept json
// @Produce json
// @Param id path string true "文件 ID"
// @Param request body updateMyFileRequest true "更新字段"
// @Security SessionCookie
// @Success 200 {object} response.Any{data=model.Upload} "更新成功"
// @Failure 403 {object} response.Any "无权操作"
// @Failure 404 {object} response.Any "文件不存在"
// @Router /api/v1/upload/{id} [put]
func UpdateMyFile(c *gin.Context) {
currUser, _ := auth.GetFromContext[*model.User](c, auth.UserObjKey)
ctx := c.Request.Context()
if uploadstorage.ReadOnly(ctx) {
response.AbortConflict(c, shared.ErrStorageReadOnly)
return
}
uploadID, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
response.AbortBadRequest(c, shared.ErrInvalidFileID)
return
}
var req updateMyFileRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, shared.ErrInvalidParams)
return
}
upload, err := updateOwnedUpload(ctx, currUser.ID, uploadID, updateMyUploadInput(req))
if err != nil {
if isRecordNotFound(err) {
response.AbortNotFound(c, "文件记录未找到")
return
}
if err == ingest.ErrForbidden {
response.AbortForbidden(c, "无权操作")
return
}
response.AbortBadRequest(c, "更新文件记录失败")
return
}
c.JSON(http.StatusOK, response.OK(upload))
}
@@ -0,0 +1,64 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package handler
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/gin-gonic/gin"
)
func TestGetDistinctUploadTypes(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
user := model.User{ID: 2222, Username: "test_user_2"}
dbConn.Create(&user)
customUpload := model.Upload{
ID: 9001,
UserID: user.ID,
FileName: "custom.txt",
FilePath: "uploads/custom.txt",
FileSize: 10,
MimeType: "text/plain",
Extension: "txt",
Type: "custom_type_xyz",
Status: model.UploadStatusUsed,
}
dbConn.Create(&customUpload)
gin.SetMode(gin.TestMode)
r := gin.New()
r.GET("/api/v1/admin/uploads/types", GetDistinctUploadTypes)
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads/types", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", w.Code)
}
var resp struct {
ErrorMsg string `json:"error_msg"`
Data []string `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse JSON: %v", err)
}
if resp.ErrorMsg != "" {
t.Fatalf("unexpected error: %s", resp.ErrorMsg)
}
if len(resp.Data) != 1 || resp.Data[0] != "custom_type_xyz" {
t.Errorf("expected only custom_type_xyz in types list, got: %v", resp.Data)
}
}
+86
View File
@@ -0,0 +1,86 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package handler
import (
"context"
"errors"
"sort"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/ingest"
"gorm.io/gorm"
)
func listUploadFiles(ctx context.Context, filter repository.UploadListFilter) (int64, []model.Upload, error) {
return repository.ListUploads(ctx, filter)
}
func listMyUploadFiles(ctx context.Context, userID uint64, filter repository.UploadListFilter) (int64, []model.Upload, error) {
filter.UserID = userID
return repository.ListUploads(ctx, filter)
}
func softDeleteUpload(ctx context.Context, uploadID uint64) (model.Upload, error) {
return ingest.Remove(ctx, uploadID)
}
func softDeleteOwnedUpload(ctx context.Context, userID, uploadID uint64) (model.Upload, error) {
return ingest.RemoveOwned(ctx, userID, uploadID)
}
func listDistinctUploadTypes(ctx context.Context) ([]string, error) {
types, err := repository.ListDistinctUploadTypes(ctx)
if err != nil {
return nil, err
}
sort.Strings(types)
return types, nil
}
type updateMyUploadInput struct {
FileName string
AccessMode *int
}
func updateOwnedUpload(ctx context.Context, userID, uploadID uint64, input updateMyUploadInput) (model.Upload, error) {
upload, err := repository.GetActiveUploadByID(ctx, uploadID)
if err != nil {
return model.Upload{}, err
}
if upload.UserID != userID {
return model.Upload{}, ingest.ErrForbidden
}
updates := make(map[string]any)
if input.FileName != "" {
updates["file_name"] = input.FileName
}
if input.AccessMode != nil {
updates["access_mode"] = *input.AccessMode
}
if err := repository.UpdateUpload(ctx, &upload, updates); err != nil {
return model.Upload{}, err
}
if name, ok := updates["file_name"].(string); ok {
upload.FileName = name
}
if mode, ok := updates["access_mode"].(int); ok {
upload.AccessMode = mode
}
return upload, nil
}
func listUploadsForBatchDownload(ctx context.Context, ids []uint64) ([]model.Upload, error) {
return repository.ListUploadsByIDs(ctx, ids)
}
func loadUploadStats(ctx context.Context) ([]model.UploadStat, error) {
return repository.ListUploadStats(ctx)
}
func isRecordNotFound(err error) bool {
return errors.Is(err, gorm.ErrRecordNotFound)
}
+333
View File
@@ -0,0 +1,333 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package handler provides upload HTTP API handlers.
package handler
import (
"archive/zip"
"bufio"
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
"github.com/Rain-kl/Wavelet/internal/model"
appshared "github.com/Rain-kl/Wavelet/internal/shared"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/ingest"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/util"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
type batchDownloadRequest struct {
IDs []string `json:"ids" binding:"required,min=1"`
}
// UploadFile 通用上传文件接口
// @Summary 上传文件
// @Description 支持各种类型的通用文件上传,支持自动文件类型检测、哈希计算与“秒传”去重
// @Tags upload
// @Accept multipart/form-data
// @Produce json
// @Param file formData file true "要上传的文件"
// @Param type formData string false "业务分类 (例如: avatar, attachment, doc,默认为 generic)"
// @Param metadata formData string false "额外的 JSON 格式元数据"
// @Security SessionCookie
// @Success 200 {object} response.Any{data=model.Upload} "上传成功"
// @Failure 400 {object} response.Any "请求参数错误或文件受限"
// @Failure 401 {object} response.Any "未登录"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/upload [post]
//
//nolint:revive
func UploadFile(c *gin.Context) {
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Security-Policy", "sandbox")
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, shared.MaxUploadSize)
currUser, _ := auth.GetFromContext[*model.User](c, auth.UserObjKey)
ctx := c.Request.Context()
header, err := c.FormFile("file")
if err != nil {
response.AbortBadRequest(c, shared.ErrNoFileSelected)
return
}
file, err := header.Open()
if err != nil {
response.AbortBadRequest(c, shared.ErrOpenFileFailed)
return
}
defer func() { _ = file.Close() }()
if header.Size > shared.MaxUploadSize {
response.AbortBadRequest(c, shared.ErrGenericFileTooLarge)
return
}
origName := header.Filename
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(origName), "."))
if ext == "" {
ext = "bin"
}
hashWriter := sha256.New()
var buf bytes.Buffer
size, err := io.Copy(&buf, io.TeeReader(file, hashWriter))
if err != nil {
response.AbortBadRequest(c, shared.ErrProcessFileFailed)
return
}
fileHash := hex.EncodeToString(hashWriter.Sum(nil))
mimeType := detectMimeType(&buf, header, size)
if util.IsImageExtension(ext) && !strings.HasPrefix(mimeType, "image/") {
response.AbortBadRequest(c, shared.ErrFileContentExtensionMismatch)
return
}
uploadType := c.DefaultPostForm("type", "generic")
accessMode, errMsg := resolveUploadAccessMode(c, uploadType)
if errMsg != "" {
response.AbortBadRequest(c, errMsg)
return
}
meta, errMsg := parseUploadMetadata(c, mimeType)
if errMsg != "" {
response.AbortBadRequest(c, errMsg)
return
}
result, err := ingest.Ingest(ctx, ingest.Request{
UserID: currUser.ID,
Reader: bytes.NewReader(buf.Bytes()),
Size: size,
FileName: origName,
MimeType: mimeType,
Extension: ext,
Hash: fileHash,
Type: uploadType,
AccessMode: &accessMode,
Metadata: meta,
Policy: ingest.PolicyDedupNewRecord,
})
if err != nil {
if errors.Is(err, ingest.ErrStorageReadOnly) {
response.AbortConflict(c, shared.ErrStorageReadOnly)
return
}
if err.Error() == shared.ErrUnsupportedFormat {
response.AbortBadRequest(c, shared.ErrUnsupportedFormat)
return
}
if err.Error() == shared.ErrSaveFileFailed {
response.AbortBadRequest(c, shared.ErrSaveFileFailed)
return
}
response.AbortBadRequest(c, shared.ErrSaveUploadRecordFailed)
return
}
c.JSON(http.StatusOK, response.OK(result.Upload))
}
// DownloadFile 通用单文件下载接口
// @Summary 下载单文件
// @Description 根据文件 ID 获取文件,以附件形式 (Attachment) 强制开启客户端浏览器下载
// @Tags admin
// @Produce octet-stream
// @Param id path string true "文件 ID"
// @Param quality query string false "图片质量 (low, medium, high, origin),默认为 origin"
// @Security SessionCookie
// @Success 200 {file} file "成功下载文件"
// @Failure 400 {object} response.Any "参数错误"
// @Failure 404 {object} response.Any "文件不存在"
// @Failure 500 {object} response.Any "服务内部错误"
// @Router /api/v1/admin/uploads/download/{id} [get]
func DownloadFile(c *gin.Context) {
upload, err := filesrv.GetUploadRecordByID(c)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
response.AbortNotFound(c, "文件记录未找到")
return
}
if _, ok := err.(*strconv.NumError); ok {
response.AbortBadRequest(c, shared.ErrInvalidFileID)
return
}
response.AbortBadRequest(c, shared.ErrQueryUploadRecordFailed)
return
}
if err := filesrv.CheckFileAccessPermission(c, upload); err != nil {
response.AbortUnauthorized(c, appshared.UnAuthorized)
return
}
fileName := upload.FileName
quality := util.NormalizeImageQuality(c.Query("quality"))
isImage := strings.HasPrefix(strings.ToLower(upload.MimeType), "image/") || util.IsImageExtension(strings.ToLower(upload.Extension))
if quality != shared.ImageQualityOrigin && isImage {
ext := filepath.Ext(fileName)
if ext != "" {
fileName = strings.TrimSuffix(fileName, ext) + ".webp"
} else {
fileName += ".webp"
}
}
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename*=UTF-8''%s", url.PathEscape(fileName)))
filesrv.ServeUpload(c, upload)
}
// BatchDownloadFiles 批量打包 ZIP 下载接口
// @Summary 批量打包下载
// @Description 传入多个文件 ID,后台实时将其打包压缩为 ZIP 流并输出,自动处理文件名重复冲突
// @Tags admin
// @Accept json
// @Produce octet-stream
// @Param request body handler.batchDownloadRequest true "包含文件 ID 数组 of string 的请求体"
// @Security SessionCookie
// @Success 200 {file} file "成功下载打包后的 ZIP"
// @Failure 400 {object} response.Any "参数错误"
// @Failure 500 {object} response.Any "打包失败"
// @Router /api/v1/admin/uploads/download/batch [post]
func BatchDownloadFiles(c *gin.Context) {
ctx := c.Request.Context()
var req batchDownloadRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, shared.ErrInvalidBatchDownloadRequest)
return
}
var ids []uint64
for _, idStr := range req.IDs {
id, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
response.AbortBadRequest(c, fmt.Sprintf(shared.ErrInvalidIDValueFormat, idStr))
return
}
ids = append(ids, id)
}
uploads, err := listUploadsForBatchDownload(ctx, ids)
if err != nil {
response.AbortBadRequest(c, shared.ErrRetrieveUploadRecordsFailed)
return
}
if len(uploads) == 0 {
response.AbortBadRequest(c, shared.ErrNoValidFilesForArchive)
return
}
c.Header("Content-Type", "application/zip")
c.Header("Content-Disposition", "attachment; filename=\"batch_download.zip\"")
bufferedWriter := bufio.NewWriter(c.Writer)
zipWriter := zip.NewWriter(bufferedWriter)
defer func() {
_ = zipWriter.Close()
_ = bufferedWriter.Flush()
}()
usedNames := make(map[string]int)
for _, upload := range uploads {
if err := filesrv.CheckFileAccessPermission(c, &upload); err != nil {
logger.WarnF(ctx, "Batch download: skip file %d due to permission denied: %v", upload.ID, err)
continue
}
fileName := upload.FileName
if count, exists := usedNames[fileName]; exists {
usedNames[fileName] = count + 1
ext := filepath.Ext(fileName)
base := strings.TrimSuffix(fileName, ext)
fileName = fmt.Sprintf("%s_%d%s", base, count, ext)
} else {
usedNames[fileName] = 1
}
zipFileEntry, err := zipWriter.Create(fileName)
if err != nil {
logger.ErrorF(ctx, "ZIP 添加条目失败 [%s]: %v", fileName, err)
continue
}
obj, err := uploadstorage.OpenStoredObject(ctx, &upload)
if err != nil {
logger.ErrorF(ctx, "打包时读取文件失败: %v", err)
continue
}
rc := obj.Body
_, err = io.Copy(zipFileEntry, rc)
_ = rc.Close()
if err != nil {
logger.ErrorF(ctx, "写入 ZIP 流失败: %v", err)
}
}
}
func resolveUploadAccessMode(c *gin.Context, uploadType string) (int, string) {
accessModeStr := c.PostForm("access_mode")
if accessModeStr == "" {
if uploadType == shared.DefaultPublicUploadType {
return 1, ""
}
return 0, ""
}
accessMode, err := strconv.Atoi(accessModeStr)
if err != nil || (accessMode != 0 && accessMode != 1) {
return 0, "无效的 access_mode 参数"
}
return accessMode, ""
}
func parseUploadMetadata(c *gin.Context, mimeType string) (model.UploadMetadata, string) {
var meta model.UploadMetadata
metadataStr := c.DefaultPostForm("metadata", "")
if metadataStr != "" {
if err := json.Unmarshal([]byte(metadataStr), &meta); err != nil {
return meta, shared.ErrInvalidMetadataJSON
}
}
meta.OriginalMime = mimeType
meta.UserAgent = c.Request.UserAgent()
meta.ClientIP = c.ClientIP()
return meta, ""
}
func detectMimeType(buf *bytes.Buffer, header *multipart.FileHeader, size int64) string {
mimeType := http.DetectContentType(buf.Bytes()[:min(shared.DetectContentBytes, int(size))])
if mimeType == "application/octet-stream" && header.Header.Get("Content-Type") != "" {
mimeType = header.Header.Get("Content-Type")
}
return mimeType
}
@@ -0,0 +1,979 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package handler
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"strconv"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
"github.com/gin-gonic/gin"
)
type testResponse struct {
ErrorMsg string `json:"error_msg"`
Data json.RawMessage `json:"data"`
}
func setupTestRouter(authUser *model.User) *gin.Engine {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(response.ErrorHandlerMiddleware())
authMiddleware := func(c *gin.Context) {
if authUser != nil {
auth.SetToContext(c, auth.UserObjKey, authUser)
}
c.Next()
}
uploadGroup := r.Group("/api/v1/upload")
uploadGroup.Use(authMiddleware)
{
uploadGroup.POST("", UploadFile)
uploadGroup.GET("/my", ListMyFiles)
uploadGroup.DELETE("/:id", DeleteMyFile)
uploadGroup.PUT("/:id", UpdateMyFile)
uploadGroup.GET("/download/:id", DownloadFile)
uploadGroup.POST("/download/batch", BatchDownloadFiles)
}
adminGroup := r.Group("/api/v1/admin/uploads")
adminGroup.Use(authMiddleware)
{
adminGroup.GET("", ListFiles)
adminGroup.GET("/stats", GetFileStats)
adminGroup.DELETE("/:id", DeleteFile)
adminGroup.GET("/download/:id", DownloadFile)
adminGroup.POST("/download/batch", BatchDownloadFiles)
}
return r
}
func createMultipartRequest(t *testing.T, fieldName, fileName string, fileContent []byte, extraFields map[string]string) (string, *bytes.Buffer) {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(fieldName, fileName)
if err != nil {
t.Fatalf("failed to create form file: %v", err)
}
_, err = part.Write(fileContent)
if err != nil {
t.Fatalf("failed to write file content: %v", err)
}
for k, v := range extraFields {
err = writer.WriteField(k, v)
if err != nil {
t.Fatalf("failed to write form field: %v", err)
}
}
err = writer.Close()
if err != nil {
t.Fatalf("failed to close multipart writer: %v", err)
}
return writer.FormDataContentType(), body
}
func TestUploadFile(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
defer func() { _ = os.RemoveAll("uploads") }() // Clean up local files created during tests
authUser := &model.User{ID: 1001, Username: "test_user"}
router := setupTestRouter(authUser)
// Mock Storage Client
mockFiles := make(map[string][]byte)
var putCount int
restoreStorage := objectstore.MockStorage(
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
data, err := io.ReadAll(body)
if err != nil {
return err
}
mockFiles[key] = data
putCount++
return nil
},
func(ctx context.Context, key string) (*objectstore.Object, error) {
data, ok := mockFiles[key]
if !ok {
return nil, os.ErrNotExist
}
return &objectstore.Object{
Body: io.NopCloser(bytes.NewReader(data)),
ContentLength: int64(len(data)),
ContentType: "application/octet-stream",
}, nil
},
func(ctx context.Context, key string) error {
delete(mockFiles, key)
return nil
},
)
defer restoreStorage()
// 开启 S3 Storage
objectstore.IsEnabledFunc = func() bool { return true }
defer func() {
objectstore.IsEnabledFunc = func() bool { return false }
}()
t.Run("upload allowed image file successfully", func(t *testing.T) {
putCount = 0
imgContent := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89") // Valid PNG header
contentType, body := createMultipartRequest(t, "file", "test.png", imgContent, map[string]string{
"type": "avatar",
"metadata": `{"extra":{"source":"test_runner"}}`,
})
req, _ := http.NewRequest("POST", "/api/v1/upload", body)
req.Header.Set("Content-Type", contentType)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d. Body: %s", w.Code, w.Body.String())
}
var resp testResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to unmarshal response: %v", err)
}
if resp.ErrorMsg != "" {
t.Fatalf("expected success response, got failure: %s", resp.ErrorMsg)
}
// Verify database record
var uploadRecord model.Upload
if err := json.Unmarshal(resp.Data, &uploadRecord); err != nil {
t.Fatalf("failed to unmarshal upload record: %v", err)
}
var dbRecord model.Upload
if err := dbConn.First(&dbRecord, uploadRecord.ID).Error; err != nil {
t.Fatalf("failed to retrieve database record: %v", err)
}
if dbRecord.FileName != "test.png" || dbRecord.Extension != "png" {
t.Errorf("incorrect filename or extension: %s, %s", dbRecord.FileName, dbRecord.Extension)
}
if dbRecord.MimeType != "image/png" {
t.Errorf("incorrect mime type detected: %s", dbRecord.MimeType)
}
if dbRecord.Metadata.Extra["source"] != "test_runner" {
t.Errorf("expected extra meta 'source' to be 'test_runner', got %v", dbRecord.Metadata.Extra)
}
if putCount != 1 {
t.Errorf("expected 1 storage Put operation, got %d", putCount)
}
})
t.Run("upload blocked extension file", func(t *testing.T) {
// System config allowed: jpg,png,webp. Uploading docx should be blocked.
contentType, body := createMultipartRequest(t, "file", "contract.docx", []byte("fake docx content"), nil)
req, _ := http.NewRequest("POST", "/api/v1/upload", body)
req.Header.Set("Content-Type", contentType)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("expected status 400, got %d. Body: %s", w.Code, w.Body.String())
}
var resp testResponse
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.ErrorMsg == "" || !strings.Contains(resp.ErrorMsg, shared.ErrUnsupportedFormat) {
t.Errorf("expected unsupported format error, got: %v", resp)
}
})
t.Run("instant upload deduplication (秒传)", func(t *testing.T) {
putCount = 0
imgContent := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01")
// Upload first time
contentType1, body1 := createMultipartRequest(t, "file", "avatar1.png", imgContent, map[string]string{"type": "avatar"})
req1, _ := http.NewRequest("POST", "/api/v1/upload", body1)
req1.Header.Set("Content-Type", contentType1)
w1 := httptest.NewRecorder()
router.ServeHTTP(w1, req1)
if w1.Code != http.StatusOK {
t.Fatalf("first upload failed: %s", w1.Body.String())
}
if putCount != 1 {
t.Errorf("expected 1 put count on first upload, got %d", putCount)
}
// Upload same file second time (different filename, same content)
contentType2, body2 := createMultipartRequest(t, "file", "avatar2.png", imgContent, map[string]string{"type": "avatar"})
req2, _ := http.NewRequest("POST", "/api/v1/upload", body2)
req2.Header.Set("Content-Type", contentType2)
w2 := httptest.NewRecorder()
router.ServeHTTP(w2, req2)
if w2.Code != http.StatusOK {
t.Fatalf("second upload failed: %s", w2.Body.String())
}
var resp2 testResponse
_ = json.Unmarshal(w2.Body.Bytes(), &resp2)
if resp2.ErrorMsg != "" {
t.Fatalf("second upload was unsuccessful: %s", resp2.ErrorMsg)
}
var uploadRecord2 model.Upload
if err := json.Unmarshal(resp2.Data, &uploadRecord2); err != nil {
t.Fatalf("failed to unmarshal second upload record: %v", err)
}
// Check if it triggered another storage put
if putCount != 1 {
t.Errorf("PutObject was triggered again! Expected deduplication (putCount=1), got putCount=%d", putCount)
}
// Check if database contains both records sharing the same FilePath
var records []model.Upload
dbConn.Where("hash = ?", uploadRecord2.Hash).Find(&records)
if len(records) != 2 {
t.Errorf("expected 2 database records sharing the same hash, got %d", len(records))
}
if records[0].FilePath != records[1].FilePath {
t.Errorf("file paths are different: %s vs %s", records[0].FilePath, records[1].FilePath)
}
if records[0].ID == records[1].ID {
t.Error("database record IDs should be unique")
}
t.Logf("Instant upload success. Record 1: %d, Record 2: %d", records[0].ID, records[1].ID)
})
t.Run("upload in local storage fallback mode", func(t *testing.T) {
// Turn off S3
objectstore.IsEnabledFunc = func() bool { return false }
// Seed allowed extensions configuration to allow txt files
var sc model.SystemConfig
dbConn.Where("key = ?", model.ConfigKeyUploadAllowedExtensions).First(&sc)
sc.Value = "jpg,png,webp,txt"
dbConn.Save(&sc)
_ = db.HSetJSON(context.Background(), repository.SystemConfigRedisHashKey, sc.Key, &sc)
repository.ResetSystemConfigRAMCacheForTest()
contentType, body := createMultipartRequest(t, "file", "doc.txt", []byte("hello world generic document file"), map[string]string{
"type": "document",
})
req, _ := http.NewRequest("POST", "/api/v1/upload", body)
req.Header.Set("Content-Type", contentType)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d. Body: %s", w.Code, w.Body.String())
}
var resp testResponse
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.ErrorMsg != "" {
t.Fatalf("local upload failed: %s", resp.ErrorMsg)
}
var localRecord model.Upload
if err := json.Unmarshal(resp.Data, &localRecord); err != nil {
t.Fatalf("failed to unmarshal local upload record: %v", err)
}
// Confirm file was actually written to local disk
fileContent, err := os.ReadFile(localRecord.FilePath)
if err != nil {
t.Fatalf("failed to read local file: %v", err)
}
if string(fileContent) != "hello world generic document file" {
t.Errorf("unexpected local file contents: %s", string(fileContent))
}
})
}
func TestDownloadFile(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
defer func() { _ = os.RemoveAll("uploads") }()
authUser := &model.User{ID: 1001, Username: "test_user"}
router := setupTestRouter(authUser)
// Seed upload records in DB
localUpload := model.Upload{
ID: 2001,
UserID: 1001,
FileName: "中文文件名.txt",
FilePath: "uploads/test_download.txt",
FileSize: 12,
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
}
// Create local file
err := os.MkdirAll("uploads", 0755)
if err != nil {
t.Fatalf("failed to create directory: %v", err)
}
err = os.WriteFile(localUpload.FilePath, []byte("hello download"), 0644)
if err != nil {
t.Fatalf("failed to write file: %v", err)
}
dbConn.Create(&localUpload)
t.Run("download file successfully", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads/download/2001", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d. Body: %s", w.Code, w.Body.String())
}
if w.Body.String() != "hello download" {
t.Errorf("expected body 'hello download', got '%s'", w.Body.String())
}
// Verify Content-Disposition header (supports UTF-8 escaping)
contentDisp := w.Header().Get("Content-Disposition")
expectedDisp := "attachment; filename*=UTF-8''%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6%E5%90%8D.txt"
if contentDisp != expectedDisp {
t.Errorf("expected Content-Disposition header %q, got %q", expectedDisp, contentDisp)
}
if !strings.HasPrefix(w.Header().Get("Content-Type"), "text/plain") {
t.Errorf("expected Content-Type starting with text/plain, got %s", w.Header().Get("Content-Type"))
}
})
t.Run("download non-existent file", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads/download/9999", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
t.Errorf("expected status 404, got %d", w.Code)
}
})
}
func TestListFiles(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
authUser := &model.User{ID: 1001, Username: "test_user"}
router := setupTestRouter(authUser)
uploads := []model.Upload{
{
ID: 2101,
UserID: authUser.ID,
FileName: "first-report.txt",
FilePath: "uploads/first-report.txt",
FileSize: 10,
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
},
{
ID: 2102,
UserID: authUser.ID,
FileName: "Second-Photo.PNG",
FilePath: "uploads/second-photo.png",
FileSize: 20,
MimeType: "image/png",
Extension: "png",
Status: model.UploadStatusUsed,
},
{
ID: 2103,
UserID: authUser.ID,
FileName: "third-notes.md",
FilePath: "uploads/third-notes.md",
FileSize: 30,
MimeType: "text/markdown",
Extension: "md",
Status: model.UploadStatusUsed,
},
{
ID: 2104,
UserID: 2002,
FileName: "other-user.txt",
FilePath: "uploads/other-user.txt",
FileSize: 40,
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
},
}
for i := range uploads {
if err := dbConn.Create(&uploads[i]).Error; err != nil {
t.Fatalf("failed to create upload %d: %v", uploads[i].ID, err)
}
}
t.Run("returns requested page", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads?page=2&page_size=2", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
var resp testResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
if resp.ErrorMsg != "" {
t.Fatalf("ListFiles() error = %q, want empty", resp.ErrorMsg)
}
var got listFilesResponse
if err := json.Unmarshal(resp.Data, &got); err != nil {
t.Fatalf("failed to parse list response: %v", err)
}
if got.Page != 2 {
t.Errorf("ListFiles(page=2).Page = %d, want 2", got.Page)
}
if got.PageSize != 2 {
t.Errorf("ListFiles(page_size=2).PageSize = %d, want 2", got.PageSize)
}
if got.Total != 4 {
t.Errorf("ListFiles().Total = %d, want 4", got.Total)
}
if len(got.Items) != 2 {
t.Fatalf("ListFiles(page=2, page_size=2) returned %d items, want 2", len(got.Items))
}
})
t.Run("filters filename case insensitively", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads?keyword=photo", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
var resp testResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
if resp.ErrorMsg != "" {
t.Fatalf("ListFiles(keyword=photo) error = %q, want empty", resp.ErrorMsg)
}
var got listFilesResponse
if err := json.Unmarshal(resp.Data, &got); err != nil {
t.Fatalf("failed to parse list response: %v", err)
}
if got.Total != 1 {
t.Errorf("ListFiles(keyword=photo).Total = %d, want 1", got.Total)
}
if len(got.Items) != 1 {
t.Fatalf("ListFiles(keyword=photo) returned %d items, want 1", len(got.Items))
}
if got.Items[0].FileName != "Second-Photo.PNG" {
t.Errorf("ListFiles(keyword=photo).Items[0].FileName = %q, want %q", got.Items[0].FileName, "Second-Photo.PNG")
}
})
t.Run("filters by user_id", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads?user_id=1001", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
var resp testResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
if resp.ErrorMsg != "" {
t.Fatalf("ListFiles(user_id=1001) error = %q, want empty", resp.ErrorMsg)
}
var got listFilesResponse
if err := json.Unmarshal(resp.Data, &got); err != nil {
t.Fatalf("failed to parse list response: %v", err)
}
if got.Total != 3 {
t.Errorf("ListFiles(user_id=1001).Total = %d, want 3", got.Total)
}
if len(got.Items) != 3 {
t.Fatalf("ListFiles(user_id=1001) returned %d items, want 3", len(got.Items))
}
})
}
func TestBatchDownloadFiles(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
defer func() { _ = os.RemoveAll("uploads") }()
authUser := &model.User{ID: 1001, Username: "test_user"}
router := setupTestRouter(authUser)
// Create and write files locally
err := os.MkdirAll("uploads", 0755)
if err != nil {
t.Fatalf("failed to create local dir: %v", err)
}
_ = os.WriteFile("uploads/f1.txt", []byte("file1 content"), 0644)
_ = os.WriteFile("uploads/f2.txt", []byte("file2 content"), 0644)
_ = os.WriteFile("uploads/f3.txt", []byte("duplicate name file content"), 0644)
// Seed upload records. Note f2 and f3 have the same FileName "file_a.txt" to trigger name collision resolution.
uploads := []model.Upload{
{
ID: 3001,
UserID: 1001,
FileName: "file_a.txt",
FilePath: "uploads/f1.txt",
FileSize: 13,
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
},
{
ID: 3002,
UserID: 1001,
FileName: "file_b.txt",
FilePath: "uploads/f2.txt",
FileSize: 13,
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
},
{
ID: 3003,
UserID: 1001,
FileName: "file_a.txt", // COLLISION with 3001!
FilePath: "uploads/f3.txt",
FileSize: 28,
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
},
}
for _, up := range uploads {
dbConn.Create(&up)
}
t.Run("batch download zip successfully and check duplicate renaming", func(t *testing.T) {
reqBody, _ := json.Marshal(batchDownloadRequest{
IDs: []string{"3001", "3002", "3003"},
})
req, _ := http.NewRequest("POST", "/api/v1/admin/uploads/download/batch", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d. Body: %s", w.Code, w.Body.String())
}
if w.Header().Get("Content-Type") != "application/zip" {
t.Errorf("expected Content-Type application/zip, got %s", w.Header().Get("Content-Type"))
}
// Unzip in-memory
zipReader, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len()))
if err != nil {
t.Fatalf("failed to read zip buffer: %v", err)
}
if len(zipReader.File) != 3 {
t.Errorf("expected 3 files inside the ZIP, got %d", len(zipReader.File))
}
// Extract files to check their contents and name collision resolutions
extracted := make(map[string]string)
for _, f := range zipReader.File {
rc, err := f.Open()
if err != nil {
t.Fatalf("failed to open zip file entry %s: %v", f.Name, err)
}
content, _ := io.ReadAll(rc)
_ = rc.Close()
extracted[f.Name] = string(content)
}
// Checks
if extracted["file_a.txt"] != "file1 content" {
t.Errorf("file_a.txt content incorrect: %q", extracted["file_a.txt"])
}
if extracted["file_b.txt"] != "file2 content" {
t.Errorf("file_b.txt content incorrect: %q", extracted["file_b.txt"])
}
// The second file_a.txt should be renamed to file_a_1.txt
if extracted["file_a_1.txt"] != "duplicate name file content" {
t.Errorf("file_a_1.txt content incorrect: %q. Extracted files: %v", extracted["file_a_1.txt"], extracted)
}
t.Logf("Successfully unzipped batch. Extracted files: %+v", extracted)
})
}
func TestUploadAccessModeAccessControl(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
defer func() { _ = os.RemoveAll("uploads") }()
user1 := &model.User{ID: 1001, Username: "user1"}
user2 := &model.User{ID: 1002, Username: "user2"}
// Seed user1
if err := dbConn.Create(user1).Error; err != nil {
t.Fatalf("create test user1 failed: %v", err)
}
// Seed user2
if err := dbConn.Create(user2).Error; err != nil {
t.Fatalf("create test user2 failed: %v", err)
}
router := setupTestRouter(user1)
// 1. Upload private file for user1 (explicitly specifying access_mode = 0)
imgContent := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89")
contentType, body := createMultipartRequest(t, "file", "private.png", imgContent, map[string]string{
"type": "generic",
"access_mode": "0",
})
req, _ := http.NewRequest("POST", "/api/v1/upload", body)
req.Header.Set("Content-Type", contentType)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("Upload failed: %d, %s", w.Code, w.Body.String())
}
t.Logf("Raw upload response: %s", w.Body.String())
var resp1 testResponse
_ = json.Unmarshal(w.Body.Bytes(), &resp1)
var upload1 model.Upload
_ = json.Unmarshal(resp1.Data, &upload1)
if upload1.AccessMode != 0 {
t.Errorf("expected access_mode 0, got %d", upload1.AccessMode)
}
// 2. Upload public file for user1 (type avatar, should default to public 1)
contentType2, body2 := createMultipartRequest(t, "file", "public.png", []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"), map[string]string{
"type": "avatar",
})
req2, _ := http.NewRequest("POST", "/api/v1/upload", body2)
req2.Header.Set("Content-Type", contentType2)
w2 := httptest.NewRecorder()
router.ServeHTTP(w2, req2)
var resp2 testResponse
_ = json.Unmarshal(w2.Body.Bytes(), &resp2)
var upload2 model.Upload
_ = json.Unmarshal(resp2.Data, &upload2)
if upload2.AccessMode != 1 {
t.Errorf("expected access_mode 1 (public) for avatar, got %d", upload2.AccessMode)
}
// 3. Verify accessing private file as user1 (owner) succeeds
wAccessOwner := httptest.NewRecorder()
reqAccessOwner, _ := http.NewRequest("GET", "/api/v1/admin/uploads/download/"+strconv.FormatUint(upload1.ID, 10), nil)
router.ServeHTTP(wAccessOwner, reqAccessOwner)
if wAccessOwner.Code != http.StatusOK {
t.Errorf("owner should be allowed to download private file, got status %d", wAccessOwner.Code)
}
// 4. Verify accessing private file as user2 (non-owner) fails
routerUser2 := setupTestRouter(user2)
wAccessOther := httptest.NewRecorder()
reqAccessOther, _ := http.NewRequest("GET", "/api/v1/admin/uploads/download/"+strconv.FormatUint(upload1.ID, 10), nil)
routerUser2.ServeHTTP(wAccessOther, reqAccessOther)
if wAccessOther.Code != http.StatusUnauthorized {
t.Errorf("non-owner should be denied download of private file, got status %d, want 401", wAccessOther.Code)
}
// 5. Verify accessing public file as user2 (non-owner) succeeds
wAccessPublic := httptest.NewRecorder()
reqAccessPublic, _ := http.NewRequest("GET", "/api/v1/admin/uploads/download/"+strconv.FormatUint(upload2.ID, 10), nil)
routerUser2.ServeHTTP(wAccessPublic, reqAccessPublic)
if wAccessPublic.Code != http.StatusOK {
t.Errorf("any logged-in user should be allowed to download public file, got status %d", wAccessPublic.Code)
}
}
func TestGetFileStats(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
authUser := &model.User{ID: 1001, Username: "test_user"}
router := setupTestRouter(authUser)
// Insert some dummy uploads
uploads := []model.Upload{
{
ID: 3101,
UserID: authUser.ID,
FileName: "photo.png",
FilePath: "uploads/photo.png",
FileSize: 100,
MimeType: "image/png",
Extension: "png",
Type: "generic",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
},
{
ID: 3102,
UserID: authUser.ID,
FileName: "video.mp4",
FilePath: "uploads/video.mp4",
FileSize: 500,
MimeType: "video/mp4",
Extension: "mp4",
Type: "generic",
Status: model.UploadStatusUsed,
CreatedAt: time.Now().AddDate(0, 0, -2), // 2 days ago
},
{
ID: 3103,
UserID: authUser.ID,
FileName: "document.pdf",
FilePath: "uploads/document.pdf",
FileSize: 200,
MimeType: "application/pdf",
Extension: "pdf",
Type: "avatar", // different type
Status: model.UploadStatusUsed,
CreatedAt: time.Now().AddDate(0, 0, -10), // older than 7 days
},
}
for i := range uploads {
if err := dbConn.Create(&uploads[i]).Error; err != nil {
t.Fatalf("failed to create upload: %v", err)
}
}
if err := uploadstats.RebuildUploadStats(context.Background()); err != nil {
t.Fatalf("failed to rebuild upload stats: %v", err)
}
req, _ := http.NewRequest("GET", "/api/v1/admin/uploads/stats", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d, body: %s", w.Code, w.Body.String())
}
var resp struct {
ErrorMsg string `json:"error_msg"`
Data fileStatsResponse `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to unmarshal response: %v", err)
}
if resp.ErrorMsg != "" {
t.Fatalf("expected no error, got: %s", resp.ErrorMsg)
}
// Verify total count and size
if resp.Data.TotalCount != 3 {
t.Errorf("expected 3 total files, got %d", resp.Data.TotalCount)
}
if resp.Data.TotalSize != 800 {
t.Errorf("expected 800 total size, got %d", resp.Data.TotalSize)
}
// Verify trend (last 7 days should include photo.png (100) and video.mp4 (500), but NOT pdf (older))
// Total size in trend should be 600
var trendSizeSum int64
for _, trendItem := range resp.Data.Trend {
trendSizeSum += trendItem.Size
}
if trendSizeSum != 600 {
t.Errorf("expected 7-day trend size sum to be 600, got %d", trendSizeSum)
}
// Verify categories
categoryMap := make(map[string]int64)
for _, cat := range resp.Data.Categories {
categoryMap[cat.Name] = cat.Count
}
if categoryMap["图片"] != 1 {
t.Errorf("expected 1 image category, got %d", categoryMap["图片"])
}
if categoryMap["视频"] != 1 {
t.Errorf("expected 1 video category, got %d", categoryMap["视频"])
}
if categoryMap["文档"] != 1 {
t.Errorf("expected 1 document category, got %d", categoryMap["文档"])
}
}
func TestUserUploadManagement(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
user1 := &model.User{ID: 1001, Username: "user1"}
user2 := &model.User{ID: 1002, Username: "user2"}
_ = dbConn.Create(user1)
_ = dbConn.Create(user2)
router1 := setupTestRouter(user1)
router2 := setupTestRouter(user2)
// Seed upload records
upload1 := model.Upload{
ID: 4001,
UserID: 1001,
FileName: "user1-file.txt",
FilePath: "uploads/user1-file.txt",
FileSize: 100,
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
upload2 := model.Upload{
ID: 4002,
UserID: 1002,
FileName: "user2-file.png",
FilePath: "uploads/user2-file.png",
FileSize: 200,
MimeType: "image/png",
Extension: "png",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
_ = dbConn.Create(&upload1)
_ = dbConn.Create(&upload2)
t.Run("ListMyFiles only returns own files", func(t *testing.T) {
req, _ := http.NewRequest("GET", "/api/v1/upload/my", nil)
w := httptest.NewRecorder()
router1.ServeHTTP(w, req)
var resp struct {
ErrorMsg string `json:"error_msg"`
Data listMyFilesResponse `json:"data"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
if resp.ErrorMsg != "" {
t.Fatalf("ListMyFiles error: %s", resp.ErrorMsg)
}
if resp.Data.Total != 1 {
t.Errorf("expected 1 file for user1, got %d", resp.Data.Total)
}
if len(resp.Data.Items) != 1 || resp.Data.Items[0].ID != 4001 {
t.Errorf("expected file 4001, got items: %+v", resp.Data.Items)
}
})
t.Run("UpdateMyFile updates file name and access mode successfully", func(t *testing.T) {
newMode := 1
reqBody, _ := json.Marshal(updateMyFileRequest{
FileName: "renamed.txt",
AccessMode: &newMode,
})
req, _ := http.NewRequest("PUT", "/api/v1/upload/4001", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router1.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d. Body: %s", w.Code, w.Body.String())
}
var updated model.Upload
dbConn.First(&updated, 4001)
if updated.FileName != "renamed.txt" {
t.Errorf("expected file name renamed.txt, got %s", updated.FileName)
}
if updated.AccessMode != 1 {
t.Errorf("expected access mode 1, got %d", updated.AccessMode)
}
})
t.Run("UpdateMyFile blocks non-owners", func(t *testing.T) {
reqBody, _ := json.Marshal(updateMyFileRequest{
FileName: "hack.txt",
})
req, _ := http.NewRequest("PUT", "/api/v1/upload/4001", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router2.ServeHTTP(w, req)
if w.Code != http.StatusForbidden {
t.Errorf("expected status 403, got %d", w.Code)
}
})
t.Run("DeleteMyFile blocks non-owners", func(t *testing.T) {
req, _ := http.NewRequest("DELETE", "/api/v1/upload/4001", nil)
w := httptest.NewRecorder()
router2.ServeHTTP(w, req)
if w.Code != http.StatusForbidden {
t.Errorf("expected status 403, got %d", w.Code)
}
})
t.Run("DeleteMyFile deletes file successfully", func(t *testing.T) {
req, _ := http.NewRequest("DELETE", "/api/v1/upload/4001", nil)
w := httptest.NewRecorder()
router1.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected status 200, got %d", w.Code)
}
var deleted model.Upload
dbConn.First(&deleted, 4001)
if deleted.Status != model.UploadStatusDeleted {
t.Errorf("expected status deleted, got %s", deleted.Status)
}
})
}
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package handler
import (
"net/http"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
"github.com/gin-gonic/gin"
)
type trendItem struct {
Date string `json:"date"`
Count int64 `json:"count"`
Size int64 `json:"size"`
}
type distributionItem struct {
Name string `json:"name"`
Count int64 `json:"count"`
Size int64 `json:"size"`
}
type fileStatsResponse struct {
TotalCount int64 `json:"total_count"`
TotalSize int64 `json:"total_size"`
Trend []trendItem `json:"trend"`
Categories []distributionItem `json:"categories"`
Types []distributionItem `json:"types"`
}
// GetFileStats 获取系统上传的文件统计数据
// @Summary 获取文件统计数据
// @Description 返回系统级的总文件数、占用大小、最近 7 天新增趋势、文件类型/格式分布等数据
// @Tags admin
// @Produce json
// @Security SessionCookie
// @Success 200 {object} response.Any{data=fileStatsResponse} "获取成功"
// @Failure 401 {object} response.Any "未登录"
// @Failure 403 {object} response.Any "无管理员权限"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/admin/uploads/stats [get]
func GetFileStats(c *gin.Context) {
ctx := c.Request.Context()
stats, err := loadUploadStats(ctx)
if err != nil {
response.AbortBadRequest(c, err.Error())
return
}
now := time.Now()
trendDates := make([]string, 0, shared.FileStatsTrendDays)
trendCountMap := make(map[string]int64, shared.FileStatsTrendDays)
trendSizeMap := make(map[string]int64, shared.FileStatsTrendDays)
for i := shared.FileStatsTrendDays - 1; i >= 0; i-- {
date := now.AddDate(0, 0, -i).Format("2006-01-02")
trendDates = append(trendDates, date)
trendCountMap[date] = 0
trendSizeMap[date] = 0
}
var (
totalCount int64
totalSize int64
types []distributionItem
categories []distributionItem
)
categoriesList := []string{"图片", "视频", "音频", "文档", "压缩包", "其他"}
categoryMap := make(map[string]distributionItem, len(categoriesList))
for _, cat := range categoriesList {
categoryMap[cat] = distributionItem{Name: cat}
}
for _, stat := range stats {
switch stat.Dimension {
case model.UploadStatDimensionTotal:
totalCount = stat.FileCount
totalSize = stat.FileSize
case model.UploadStatDimensionType:
types = append(types, distributionItem{
Name: stat.StatKey,
Count: stat.FileCount,
Size: stat.FileSize,
})
case model.UploadStatDimensionCategory:
if item, ok := categoryMap[stat.StatKey]; ok {
item.Count = stat.FileCount
item.Size = stat.FileSize
categoryMap[stat.StatKey] = item
}
case model.UploadStatDimensionTrend:
if _, ok := trendCountMap[stat.StatKey]; ok {
trendCountMap[stat.StatKey] = stat.FileCount
trendSizeMap[stat.StatKey] = stat.FileSize
}
}
}
categories = make([]distributionItem, 0, len(categoriesList))
for _, cat := range categoriesList {
categories = append(categories, categoryMap[cat])
}
trend := make([]trendItem, 0, len(trendDates))
for _, date := range trendDates {
trend = append(trend, trendItem{
Date: date,
Count: trendCountMap[date],
Size: trendSizeMap[date],
})
}
c.JSON(http.StatusOK, response.OK(fileStatsResponse{
TotalCount: totalCount,
TotalSize: totalSize,
Trend: trend,
Categories: categories,
Types: types,
}))
}
+16
View File
@@ -0,0 +1,16 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package ingest
import (
"errors"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
)
// ErrForbidden indicates the caller is not allowed to mutate the upload record.
var ErrForbidden = errors.New("upload forbidden")
// ErrStorageReadOnly indicates the storage backend is in migration read-only mode.
var ErrStorageReadOnly = errors.New(shared.ErrStorageReadOnly)
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package ingest
import (
"context"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/pkg/logger"
uploadcache "github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"gorm.io/gorm"
)
func normalizeRequest(req *Request) {
req.Extension = strings.ToLower(strings.TrimSpace(req.Extension))
if req.Extension == "" {
req.Extension = "bin"
}
if req.Type == "" {
req.Type = "generic"
}
if req.Status == "" {
req.Status = model.UploadStatusUsed
}
}
func resolveAccessMode(uploadType string, explicit *int) int {
if explicit != nil {
return *explicit
}
if uploadType == shared.DefaultPublicUploadType {
return 1
}
return 0
}
func validateAllowedExtension(ctx context.Context, ext string) error {
sc, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyUploadAllowedExtensions)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return err
}
if sc.Value == "" {
return nil
}
allowedExts := strings.Split(strings.ToLower(sc.Value), ",")
for _, allowedExt := range allowedExts {
if strings.TrimSpace(allowedExt) == ext {
return nil
}
}
return errors.New(shared.ErrUnsupportedFormat)
}
func defaultObjectKey(id uint64, ext string) string {
return fmt.Sprintf("uploads/%s/%d.%s", time.Now().Format("2006/01/02"), id, ext)
}
func buildObjectKey(req Request, id uint64) string {
if req.ObjectKeyFn != nil {
return req.ObjectKeyFn(id, req.Extension)
}
return defaultObjectKey(id, req.Extension)
}
func storeObject(ctx context.Context, objectKey string, reader io.Reader, size int64, mimeType string, meta *model.UploadMetadata) (string, error) {
if uploadstorage.ReadOnly(ctx) {
return "", ErrStorageReadOnly
}
driver, backend, err := objectstore.Active(ctx)
if err != nil {
logger.ErrorF(ctx, "初始化活动存储失败: %v", err)
return "", errors.New(shared.ErrSaveFileFailed)
}
result, err := backend.Put(ctx, objectKey, reader, size, mimeType)
if err != nil {
logger.ErrorF(ctx, "写入 %s 存储失败: %v", driver, err)
return "", errors.New(shared.ErrSaveFileFailed)
}
meta.Bucket = result.Bucket
return result.Key, nil
}
func persistUploadRecord(ctx context.Context, upload *model.Upload, objectKey string) error {
if err := createUploadWithStats(ctx, upload); err != nil {
_, backend, backendErr := objectstore.Active(ctx)
if backendErr == nil {
if deleteErr := backend.Delete(ctx, objectKey); deleteErr != nil {
logger.WarnF(ctx, "清理未写入数据库的上传对象失败: %v", deleteErr)
}
}
return err
}
uploadcache.SetUploadMetaCache(ctx, upload)
return nil
}
func createUploadWithStats(ctx context.Context, upload *model.Upload) error {
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := repository.CreateUploadTx(tx, upload); err != nil {
return err
}
return uploadstats.ApplyUploadStatsDeltaTx(tx, upload, 1)
})
}
func createDedupRecord(ctx context.Context, existing model.Upload, req Request) (Result, error) {
accessMode := resolveAccessMode(req.Type, req.AccessMode)
newUpload := model.Upload{
ID: idgen.NextUint64ID(),
UserID: req.UserID,
FileName: req.FileName,
FilePath: existing.FilePath,
FileSize: req.Size,
MimeType: req.MimeType,
Extension: req.Extension,
Hash: req.Hash,
Type: req.Type,
Status: req.Status,
AccessMode: accessMode,
Metadata: existing.Metadata,
}
if err := persistUploadRecord(ctx, &newUpload, existing.FilePath); err != nil {
return Result{}, err
}
logger.InfoF(ctx, "文件触发秒传成功! ID: %d, Path: %s", newUpload.ID, existing.FilePath)
return Result{
Upload: newUpload,
Created: true,
Stored: false,
}, nil
}
func uploadstorageReadOnly(ctx context.Context) bool {
return uploadstorage.ReadOnly(ctx)
}
func createNewUpload(ctx context.Context, req Request) (Result, error) {
if uploadstorageReadOnly(ctx) {
return Result{}, ErrStorageReadOnly
}
if !req.SkipExtensionCheck {
if err := validateAllowedExtension(ctx, req.Extension); err != nil {
return Result{}, err
}
}
id := idgen.NextUint64ID()
objectKey := buildObjectKey(req, id)
storedKey, err := storeObject(ctx, objectKey, req.Reader, req.Size, req.MimeType, &req.Metadata)
if err != nil {
return Result{}, err
}
accessMode := resolveAccessMode(req.Type, req.AccessMode)
upload := model.Upload{
ID: id,
UserID: req.UserID,
FileName: req.FileName,
FilePath: storedKey,
FileSize: req.Size,
MimeType: req.MimeType,
Extension: req.Extension,
Hash: req.Hash,
Type: req.Type,
Status: req.Status,
AccessMode: accessMode,
Metadata: req.Metadata,
}
if err := persistUploadRecord(ctx, &upload, storedKey); err != nil {
return Result{}, err
}
return Result{
Upload: upload,
Created: true,
Stored: true,
}, nil
}
+64
View File
@@ -0,0 +1,64 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package ingest
import (
"context"
"errors"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"gorm.io/gorm"
)
// Ingest stores or resolves an upload using the configured policy and side effects.
func Ingest(ctx context.Context, req Request) (Result, error) {
normalizeRequest(&req)
if req.Hash == "" {
return Result{}, errors.New("ingest hash is required")
}
if req.Reader == nil {
return Result{}, errors.New("ingest reader is required")
}
if req.Size < 0 {
return Result{}, errors.New("ingest size must be non-negative")
}
switch req.Policy {
case PolicyDedupNewRecord, PolicyResolveExisting:
return ingestWithHashPolicy(ctx, req)
case PolicyCreate:
return createNewUpload(ctx, req)
default:
return Result{}, errors.New("unsupported ingest policy")
}
}
// FindByHash returns a reusable active upload with the same hash and size.
func FindByHash(ctx context.Context, hash string, size int64) (model.Upload, error) {
return repository.FindReusableUploadByHash(ctx, hash, size)
}
func ingestWithHashPolicy(ctx context.Context, req Request) (Result, error) {
existing, err := repository.FindReusableUploadByHash(ctx, req.Hash, req.Size)
if err == nil {
switch req.Policy {
case PolicyResolveExisting:
return Result{
Upload: existing,
Resolved: true,
}, nil
case PolicyDedupNewRecord:
if uploadstorageReadOnly(ctx) {
return Result{}, ErrStorageReadOnly
}
return createDedupRecord(ctx, existing, req)
}
}
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return Result{}, err
}
return createNewUpload(ctx, req)
}
+329
View File
@@ -0,0 +1,329 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package ingest
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"io"
"os"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
)
func TestIngestPolicyCreateIncrementsStats(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
content := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01")
hash := sha256.Sum256(content)
restoreStorage, disableStorage := setupMockStorage(t, nil)
defer restoreStorage()
defer disableStorage()
result, err := Ingest(ctx, Request{
UserID: 1001,
Reader: bytes.NewReader(content),
Size: int64(len(content)),
FileName: "mirror.png",
MimeType: "image/png",
Extension: "png",
Hash: hex.EncodeToString(hash[:]),
Type: "pixez_mirror",
Policy: PolicyCreate,
})
if err != nil {
t.Fatalf("Ingest(PolicyCreate) returned error: %v", err)
}
if !result.Created || !result.Stored || result.Resolved {
t.Fatalf("Ingest(PolicyCreate) = %+v, want Created+Stored without Resolved", result)
}
stats, err := loadTotalStats(ctx)
if err != nil {
t.Fatalf("loadTotalStats returned error: %v", err)
}
if stats.TotalCount != 1 || stats.TotalSize != int64(len(content)) {
t.Fatalf("loadTotalStats() = count %d size %d, want count 1 size %d", stats.TotalCount, stats.TotalSize, len(content))
}
}
func TestIngestPolicyResolveExistingSkipsStatsOnHit(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
content := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01")
hash := sha256.Sum256(content)
hashStr := hex.EncodeToString(hash[:])
existing := model.Upload{
ID: 88001,
UserID: 42,
FileName: "existing.png",
FilePath: "uploads/existing.png",
FileSize: int64(len(content)),
MimeType: "image/png",
Extension: "png",
Hash: hashStr,
Type: "pixez_mirror",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
if err := dbConn.Create(&existing).Error; err != nil {
t.Fatalf("seed upload failed: %v", err)
}
restoreStorage, disableStorage := setupMockStorage(t, nil)
defer restoreStorage()
defer disableStorage()
result, err := Ingest(ctx, Request{
UserID: 1001,
Reader: bytes.NewReader(content),
Size: int64(len(content)),
FileName: "mirror.png",
MimeType: "image/png",
Extension: "png",
Hash: hashStr,
Type: "pixez_mirror",
Policy: PolicyResolveExisting,
})
if err != nil {
t.Fatalf("Ingest(PolicyResolveExisting) returned error: %v", err)
}
if !result.Resolved || result.Created || result.Stored {
t.Fatalf("Ingest(PolicyResolveExisting) = %+v, want Resolved only", result)
}
if result.Upload.ID != existing.ID {
t.Fatalf("Ingest(PolicyResolveExisting).Upload.ID = %d, want %d", result.Upload.ID, existing.ID)
}
stats, err := loadTotalStats(ctx)
if err != nil {
t.Fatalf("loadTotalStats returned error: %v", err)
}
if stats.TotalCount != 0 || stats.TotalSize != 0 {
t.Fatalf("loadTotalStats() = count %d size %d, want zero stats for resolved upload", stats.TotalCount, stats.TotalSize)
}
}
func TestIngestPolicyDedupNewRecordCreatesSecondRecord(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
content := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01")
hash := sha256.Sum256(content)
hashStr := hex.EncodeToString(hash[:])
putCount := 0
restoreStorage, disableStorage := setupMockStorage(t, &putCount)
defer restoreStorage()
defer disableStorage()
first, err := Ingest(ctx, Request{
UserID: 1001,
Reader: bytes.NewReader(content),
Size: int64(len(content)),
FileName: "first.png",
MimeType: "image/png",
Extension: "png",
Hash: hashStr,
Type: "avatar",
Policy: PolicyDedupNewRecord,
})
if err != nil {
t.Fatalf("first Ingest returned error: %v", err)
}
if putCount != 1 {
t.Fatalf("putCount after first ingest = %d, want 1", putCount)
}
second, err := Ingest(ctx, Request{
UserID: 1002,
Reader: bytes.NewReader(content),
Size: int64(len(content)),
FileName: "second.png",
MimeType: "image/png",
Extension: "png",
Hash: hashStr,
Type: "avatar",
Policy: PolicyDedupNewRecord,
})
if err != nil {
t.Fatalf("second Ingest returned error: %v", err)
}
if putCount != 1 {
t.Fatalf("putCount after dedup ingest = %d, want 1", putCount)
}
if first.Upload.FilePath != second.Upload.FilePath {
t.Fatalf("dedup file paths differ: %s vs %s", first.Upload.FilePath, second.Upload.FilePath)
}
if first.Upload.ID == second.Upload.ID {
t.Fatal("dedup records should have unique IDs")
}
var count int64
if err := dbConn.Model(&model.Upload{}).Where("hash = ?", hashStr).Count(&count).Error; err != nil {
t.Fatalf("count uploads failed: %v", err)
}
if count != 2 {
t.Fatalf("upload count = %d, want 2", count)
}
}
func TestCreateUploadWithStatsRollsBackOnCreateFailure(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
existing := model.Upload{
ID: 99001,
UserID: 1001,
FileName: "existing.png",
FilePath: "uploads/existing.png",
FileSize: 64,
MimeType: "image/png",
Extension: "png",
Type: "generic",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
if err := dbConn.Create(&existing).Error; err != nil {
t.Fatalf("seed upload failed: %v", err)
}
duplicate := &model.Upload{
ID: existing.ID,
UserID: 1002,
FileName: "duplicate.png",
FilePath: "uploads/duplicate.png",
FileSize: 128,
MimeType: "image/png",
Extension: "png",
Type: "generic",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
if err := createUploadWithStats(ctx, duplicate); err == nil {
t.Fatal("createUploadWithStats with duplicate ID expected error")
}
stats, err := loadTotalStats(ctx)
if err != nil {
t.Fatalf("loadTotalStats returned error: %v", err)
}
if stats.TotalCount != 0 || stats.TotalSize != 0 {
t.Fatalf("loadTotalStats() = count %d size %d, want zero after rolled-back stats", stats.TotalCount, stats.TotalSize)
}
}
func TestRemoveDecrementsStats(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
content := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01")
hash := sha256.Sum256(content)
restoreStorage, disableStorage := setupMockStorage(t, nil)
defer restoreStorage()
defer disableStorage()
result, err := Ingest(ctx, Request{
UserID: 1001,
Reader: bytes.NewReader(content),
Size: int64(len(content)),
FileName: "delete-me.png",
MimeType: "image/png",
Extension: "png",
Hash: hex.EncodeToString(hash[:]),
Type: "generic",
Policy: PolicyCreate,
})
if err != nil {
t.Fatalf("Ingest returned error: %v", err)
}
if _, err := Remove(ctx, result.Upload.ID); err != nil {
t.Fatalf("Remove(%d) returned error: %v", result.Upload.ID, err)
}
stats, err := loadTotalStats(ctx)
if err != nil {
t.Fatalf("loadTotalStats returned error: %v", err)
}
if stats.TotalCount != 0 || stats.TotalSize != 0 {
t.Fatalf("loadTotalStats() after remove = count %d size %d, want zero", stats.TotalCount, stats.TotalSize)
}
}
type totalStatsSnapshot struct {
TotalCount int64
TotalSize int64
}
func loadTotalStats(ctx context.Context) (totalStatsSnapshot, error) {
var rows []model.UploadStat
if err := db.DB(ctx).Where("dimension = ?", model.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
return totalStatsSnapshot{}, err
}
if len(rows) == 0 {
return totalStatsSnapshot{}, nil
}
return totalStatsSnapshot{
TotalCount: rows[0].FileCount,
TotalSize: rows[0].FileSize,
}, nil
}
func setupMockStorage(t *testing.T, putCount *int) (restore func(), disable func()) {
t.Helper()
mockFiles := make(map[string][]byte)
restore = objectstore.MockStorage(
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
data, err := io.ReadAll(body)
if err != nil {
return err
}
mockFiles[key] = data
if putCount != nil {
*putCount++
}
return nil
},
func(ctx context.Context, key string) (*objectstore.Object, error) {
data, ok := mockFiles[key]
if !ok {
return nil, os.ErrNotExist
}
return &objectstore.Object{
Body: io.NopCloser(bytes.NewReader(data)),
ContentLength: int64(len(data)),
ContentType: "application/octet-stream",
}, nil
},
func(ctx context.Context, key string) error {
delete(mockFiles, key)
return nil
},
)
objectstore.IsEnabledFunc = func() bool { return true }
objectstore.ResetCache()
disable = func() {
objectstore.IsEnabledFunc = func() bool { return false }
objectstore.ResetCache()
}
return restore, disable
}
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package ingest
import (
"context"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
uploadcache "github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
"gorm.io/gorm"
)
// Remove soft-deletes an upload and decrements incremental stats.
func Remove(ctx context.Context, uploadID uint64) (model.Upload, error) {
upload, err := repository.GetActiveUploadByID(ctx, uploadID)
if err != nil {
return model.Upload{}, err
}
if err := softDeleteUploadWithStats(ctx, &upload); err != nil {
return model.Upload{}, err
}
upload.Status = model.UploadStatusDeleted
return upload, nil
}
// RemoveOwned soft-deletes an upload owned by userID and decrements incremental stats.
func RemoveOwned(ctx context.Context, userID, uploadID uint64) (model.Upload, error) {
upload, err := repository.GetActiveUploadByID(ctx, uploadID)
if err != nil {
return model.Upload{}, err
}
if upload.UserID != userID {
return model.Upload{}, ErrForbidden
}
if err := softDeleteUploadWithStats(ctx, &upload); err != nil {
return model.Upload{}, err
}
upload.Status = model.UploadStatusDeleted
return upload, nil
}
func softDeleteUploadWithStats(ctx context.Context, upload *model.Upload) error {
statsSnapshot := *upload
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := repository.SoftDeleteUploadTx(tx, upload); err != nil {
return err
}
return uploadstats.ApplyUploadStatsDeltaTx(tx, &statsSnapshot, -1)
}); err != nil {
return err
}
uploadcache.InvalidateUploadMetaCache(ctx, upload.ID)
return nil
}
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package ingest provides the programmatic upload domain service for Wavelet.
package ingest
import (
"io"
"github.com/Rain-kl/Wavelet/internal/model"
)
// Policy controls how ingest handles hash collisions and record creation.
type Policy int
const (
// PolicyCreate always stores a new object and creates a new upload record.
PolicyCreate Policy = iota
// PolicyDedupNewRecord reuses an existing object path on hash match but creates a new record and stats delta.
PolicyDedupNewRecord
// PolicyResolveExisting returns an existing upload on hash match without creating a record or stats delta.
PolicyResolveExisting
)
// ObjectKeyFn builds the storage object key for a new upload.
type ObjectKeyFn func(id uint64, ext string) string
// Request describes a programmatic file ingest operation.
type Request struct {
UserID uint64
Type string
AccessMode *int
Status model.UploadStatus
Reader io.Reader
Size int64
FileName string
MimeType string
Extension string
Hash string
Metadata model.UploadMetadata
Policy Policy
ObjectKeyFn ObjectKeyFn
// SkipExtensionCheck bypasses the configured upload extension whitelist.
SkipExtensionCheck bool
}
// Result reports the outcome of an ingest operation.
type Result struct {
Upload model.Upload
Created bool
Stored bool
Resolved bool
}
+117
View File
@@ -0,0 +1,117 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package upload provides file uploading, storage abstraction, image transcoding, and caching domain plugin for Cordis.
package upload
import (
"context"
"github.com/Rain-kl/Wavelet/core"
"github.com/Rain-kl/Wavelet/core/extpoints"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/handler"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/task"
"github.com/hibiken/asynq"
)
// Plugin implements core.Plugin to provide file upload and media serving domain services.
type Plugin struct{}
// New creates a new upload domain plugin.
func New() *Plugin {
return &Plugin{}
}
// Name returns the unique identifier for the upload domain plugin.
func (p *Plugin) Name() string {
return "upload"
}
// Manifest returns the plugin metadata.
func (p *Plugin) Manifest() core.Manifest {
return core.Manifest{
Name: "upload",
Version: "1.0.0",
Description: "File upload, secure delivery, image transcoding, and storage management domain plugin",
Author: "Wavelet Team",
}
}
// Apply registers upload routes, tasks, and settings into the Context.
func (p *Plugin) Apply(ctx *core.Context) error {
// 1. Register File Server Routes
ctx.Router().GET("/f/:id", filesrv.ServeFileByID)
// 2. Register User/Admin Upload HTTP Routes
uploadGroup := ctx.Router().Group("/api/v1/upload", auth.LoginRequired())
{
uploadGroup.POST("", handler.UploadFile)
uploadGroup.GET("", handler.ListFiles)
uploadGroup.DELETE("/:id", handler.DeleteFile)
uploadGroup.POST("/batch-download", handler.BatchDownloadFiles)
}
adminUploadGroup := ctx.Router().Group("/api/v1/admin/uploads", auth.LoginRequired())
{
adminUploadGroup.GET("", handler.ListFiles)
adminUploadGroup.GET("/stats", handler.GetFileStats)
adminUploadGroup.DELETE("/:id", handler.DeleteFile)
adminUploadGroup.GET("/download/:id", handler.DownloadFile)
adminUploadGroup.POST("/download/batch", handler.BatchDownloadFiles)
adminUploadGroup.GET("/types", handler.GetDistinctUploadTypes)
}
const (
defaultCleanupRetry = 3
defaultStatsRetry = 2
defaultSingleRetry = 1
)
// 3. Register Asynq tasks
cleanupHandler := &task.SystemCleanupHandler{}
ctx.Task().Register(task.SystemCleanupTask, func(c context.Context, t *asynq.Task) error {
_, err := cleanupHandler.Execute(c, t.Payload())
return err
}, extpoints.WithTaskRetry(defaultCleanupRetry))
rebuildStatsHandler := &task.RebuildUploadStatsHandler{}
ctx.Task().Register(task.RebuildUploadStatsTask, func(c context.Context, t *asynq.Task) error {
_, err := rebuildStatsHandler.Execute(c, t.Payload())
return err
}, extpoints.WithTaskRetry(defaultStatsRetry))
migrationHandler := &task.MigrationHandler{}
ctx.Task().Register(task.StorageMigrationTask, func(c context.Context, t *asynq.Task) error {
_, err := migrationHandler.Execute(c, t.Payload())
return err
}, extpoints.WithTaskRetry(defaultSingleRetry))
warmHandler := &task.WarmImageCacheHandler{}
ctx.Task().Register(task.WarmImageCacheTask, func(c context.Context, t *asynq.Task) error {
_, err := warmHandler.Execute(c, t.Payload())
return err
}, extpoints.WithTaskRetry(1))
// 4. Register Cron Schedule
ctx.Schedule().RegisterCron("0 3 * * *", task.SystemCleanupTask, nil)
// 5. Register Settings Schemas
ctx.Settings().Register(extpoints.SettingSchema{
Key: "upload.max_file_size_mb",
Default: 100,
Description: "Maximum upload file size limit in MB",
Type: "integer",
Category: "storage",
})
ctx.Settings().Register(extpoints.SettingSchema{
Key: "upload.allowed_extensions",
Default: "png,jpg,jpeg,gif,webp,svg,pdf,zip,tar,gz",
Description: "Allowed upload file extensions separated by commas",
Type: "string",
Category: "storage",
})
return nil
}
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package shared
// Upload size, path, media quality, and cache constants shared across subpackages.
const (
MaxUploadSize = 32 * 1024 * 1024 // 32MB
DetectContentBytes = 512 // http.DetectContentType 需要的最小字节数
UploadDirPerm = 0755 // 上传目录权限
UploadFilePerm = 0644 // 上传文件权限
ImageQualityLow = "low"
ImageQualityMedium = "medium"
ImageQualityHigh = "high"
ImageQualityOrigin = "origin"
DefaultPublicUploadType = "avatar"
FileStatsTrendDays = 7
MaxS3KeyLength = 1024
AccessCacheTTL = 5 // seconds; multiplied by time.Second at use site
)
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package shared holds upload error and configuration constants shared across subpackages.
package shared
// 文件管理常量
const (
ErrNoFileSelected = "请选择要上传的文件"
ErrUnsupportedFormat = "只支持 JPG、PNG、WEBP 格式的图片"
ErrProcessFileFailed = "处理文件失败"
ErrSaveFileFailed = "保存文件失败"
ErrOpenFileFailed = "打开文件失败"
ErrSaveUploadRecordFailed = "保存上传记录失败"
ErrGenericFileTooLarge = "文件大小不能超过 32MB"
ErrFileContentExtensionMismatch = "文件内容与扩展名不匹配,可能包含安全风险"
ErrFileValidationFailed = "文件校验失败"
ErrInvalidMetadataJSON = "元数据 JSON 格式不合法"
ErrInvalidFileID = "无效的文件 ID"
ErrQueryUploadRecordFailed = "查询文件记录失败"
ErrInvalidBatchDownloadRequest = "参数绑定失败,请传入有效的文件 ID 数组"
ErrInvalidIDValueFormat = "无效的 ID 值: %s"
ErrRetrieveUploadRecordsFailed = "检索文件记录失败"
ErrNoValidFilesForArchive = "没有找到任何有效的文件记录进行打包"
ErrInvalidParams = "参数错误"
ErrQueryFileCountFailed = "查询文件数量失败"
ErrQueryFileListFailed = "查询文件列表失败"
ErrDeleteFileFailed = "删除文件失败"
ErrStorageReadOnly = "存储迁移维护中,当前仅允许读取文件"
ErrS3KeyRequired = "s3 key must not be empty"
ErrS3KeyTooLongFormat = "s3 key exceeds maximum length of %d"
ErrS3KeyStartsWithSlash = "s3 key must not start with /"
ErrS3KeyContainsNullBytes = "s3 key must not contain null bytes"
ErrQueryUnusedUploadsFailed = "查询未使用的上传文件失败: %w"
ErrImageCacheWarmupPayloadRequired = "图片缓存预热参数不能为空"
ErrInvalidImageCacheWarmupPayload = "图片缓存预热参数格式无效: %w"
ErrInvalidImageCacheWarmupQuality = "图片质量仅支持 low、medium、high"
ErrParseImageCacheWarmupPayload = "解析图片缓存预热参数失败: %w"
ErrQueryImagesForCacheWarmup = "查询待预热图片失败: %w"
)
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package stats maintains incremental upload statistics and aggregations.
package stats
import (
"strings"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/util"
)
const (
catImage = "图片"
catVideo = "视频"
catAudio = "音频"
catDocument = "文档"
catArchive = "压缩包"
catOther = "其他"
)
// GetFileCategory classifies a file by mime type and extension.
func GetFileCategory(mimeType, ext string) string {
mimeType = strings.ToLower(mimeType)
ext = strings.ToLower(ext)
if strings.HasPrefix(mimeType, "image/") || util.IsImageExtension(ext) {
return catImage
}
if strings.HasPrefix(mimeType, "video/") {
return catVideo
}
if strings.HasPrefix(mimeType, "audio/") {
return catAudio
}
if util.IsArchiveExtension(ext) || strings.Contains(mimeType, "zip") || strings.Contains(mimeType, "tar") || strings.Contains(mimeType, "gzip") {
return catArchive
}
if util.IsDocumentExtension(ext) || strings.HasPrefix(mimeType, "text/") || mimeType == "application/pdf" {
return catDocument
}
return catOther
}
@@ -0,0 +1,131 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package stats
import (
"context"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// ApplyUploadStatsAdd increments incremental stats for a newly active upload record.
func ApplyUploadStatsAdd(ctx context.Context, upload *model.Upload) error {
return applyUploadStatsDelta(ctx, upload, 1)
}
// ApplyUploadStatsRemove decrements incremental stats for a removed active upload record.
func ApplyUploadStatsRemove(ctx context.Context, upload *model.Upload) error {
return applyUploadStatsDelta(ctx, upload, -1)
}
// RebuildUploadStats rebuilds all incremental stats from current upload records.
func RebuildUploadStats(ctx context.Context) error {
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Where("1 = 1").Delete(&model.UploadStat{}).Error; err != nil {
return err
}
var uploads []model.Upload
if err := tx.Where("status != ?", model.UploadStatusDeleted).Find(&uploads).Error; err != nil {
return err
}
for i := range uploads {
if err := ApplyUploadStatsDeltaTx(tx, &uploads[i], 1); err != nil {
return err
}
}
return nil
})
}
func applyUploadStatsDelta(ctx context.Context, upload *model.Upload, sign int64) error {
if upload == nil || !isActiveUploadStatus(upload.Status) {
return nil
}
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return ApplyUploadStatsDeltaTx(tx, upload, sign)
})
}
// ApplyUploadStatsDeltaTx applies incremental upload stats within an existing transaction.
func ApplyUploadStatsDeltaTx(tx *gorm.DB, upload *model.Upload, sign int64) error {
if upload == nil || !isActiveUploadStatus(upload.Status) || sign == 0 {
return nil
}
countDelta := sign
sizeDelta := sign * upload.FileSize
typeKey := upload.Type
if typeKey == "" {
typeKey = "generic"
}
entries := []struct {
dimension string
key string
}{
{model.UploadStatDimensionTotal, ""},
{model.UploadStatDimensionType, typeKey},
{model.UploadStatDimensionCategory, GetFileCategory(upload.MimeType, upload.Extension)},
{model.UploadStatDimensionTrend, upload.CreatedAt.Format("2006-01-02")},
}
for _, entry := range entries {
if err := upsertUploadStatDelta(tx, entry.dimension, entry.key, countDelta, sizeDelta); err != nil {
return err
}
}
return nil
}
func upsertUploadStatDelta(tx *gorm.DB, dimension, key string, countDelta, sizeDelta int64) error {
return tx.Clauses(clause.OnConflict{
Columns: []clause.Column{
{Name: "dimension"},
{Name: "stat_key"},
},
DoUpdates: clause.Assignments(map[string]any{
"file_count": gorm.Expr(
"CASE WHEN w_upload_stats.file_count + ? < 0 THEN 0 ELSE w_upload_stats.file_count + ? END",
countDelta,
countDelta,
),
"file_size": gorm.Expr(
"CASE WHEN w_upload_stats.file_size + ? < 0 THEN 0 ELSE w_upload_stats.file_size + ? END",
sizeDelta,
sizeDelta,
),
"updated_at": time.Now(),
}),
}).Create(&model.UploadStat{
Dimension: dimension,
StatKey: key,
FileCount: countDelta,
FileSize: sizeDelta,
}).Error
}
// RecordUploadStatsAdd logs and applies upload stats increment.
func RecordUploadStatsAdd(ctx context.Context, upload *model.Upload) {
if err := ApplyUploadStatsAdd(ctx, upload); err != nil {
logger.WarnF(ctx, "increment upload stats failed: %v", err)
}
}
// RecordUploadStatsRemove logs and applies upload stats decrement.
func RecordUploadStatsRemove(ctx context.Context, upload *model.Upload) {
if err := ApplyUploadStatsRemove(ctx, upload); err != nil {
logger.WarnF(ctx, "decrement upload stats failed: %v", err)
}
}
func isActiveUploadStatus(status model.UploadStatus) bool {
return status == model.UploadStatusPending || status == model.UploadStatusUsed
}
@@ -0,0 +1,103 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package stats
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"gorm.io/gorm"
)
func TestApplyUploadStatsDeltaTxWithinTransaction(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
upload := &model.Upload{
ID: 42002,
FileSize: 256,
MimeType: "image/jpeg",
Extension: "jpg",
Type: "avatar",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return ApplyUploadStatsDeltaTx(tx, upload, 1)
}); err != nil {
t.Fatalf("ApplyUploadStatsDeltaTx returned error: %v", err)
}
stats, err := loadUploadStats(ctx)
if err != nil {
t.Fatalf("loadUploadStats returned error: %v", err)
}
if stats.TotalCount != 1 || stats.TotalSize != 256 {
t.Fatalf("unexpected total stats: count=%d size=%d", stats.TotalCount, stats.TotalSize)
}
}
func TestApplyUploadStatsAddAndRemove(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
upload := &model.Upload{
ID: 42001,
FileSize: 128,
MimeType: "image/png",
Extension: "png",
Type: "avatar",
Status: model.UploadStatusUsed,
CreatedAt: time.Now(),
}
if err := ApplyUploadStatsAdd(ctx, upload); err != nil {
t.Fatalf("ApplyUploadStatsAdd returned error: %v", err)
}
stats, err := loadUploadStats(ctx)
if err != nil {
t.Fatalf("loadUploadStats returned error: %v", err)
}
if stats.TotalCount != 1 || stats.TotalSize != 128 {
t.Fatalf("unexpected total stats: count=%d size=%d", stats.TotalCount, stats.TotalSize)
}
if err := ApplyUploadStatsRemove(ctx, upload); err != nil {
t.Fatalf("ApplyUploadStatsRemove returned error: %v", err)
}
stats, err = loadUploadStats(ctx)
if err != nil {
t.Fatalf("loadUploadStats after remove returned error: %v", err)
}
if stats.TotalCount != 0 || stats.TotalSize != 0 {
t.Fatalf("expected zeroed total stats, got count=%d size=%d", stats.TotalCount, stats.TotalSize)
}
}
type uploadStatsSnapshot struct {
TotalCount int64
TotalSize int64
}
func loadUploadStats(ctx context.Context) (uploadStatsSnapshot, error) {
var rows []model.UploadStat
if err := db.DB(ctx).Where("dimension = ?", model.UploadStatDimensionTotal).Find(&rows).Error; err != nil {
return uploadStatsSnapshot{}, err
}
if len(rows) == 0 {
return uploadStatsSnapshot{}, nil
}
return uploadStatsSnapshot{
TotalCount: rows[0].FileCount,
TotalSize: rows[0].FileSize,
}, nil
}
@@ -0,0 +1,88 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package storage provides upload storage backend operations and migration state.
package storage
import (
"context"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
)
// MigrationAccessState captures cached migration maintenance state.
type MigrationAccessState struct {
ReadOnly bool
Target objectstore.Config
HasTarget bool
TargetErr error
LoadErr error
}
var (
migrationAccessMu sync.RWMutex
migrationAccessCached MigrationAccessState
migrationAccessValid bool
migrationAccessCheckedAt time.Time
)
// ResetMigrationAccessCache clears the in-process migration access cache.
func ResetMigrationAccessCache() {
migrationAccessMu.Lock()
migrationAccessValid = false
migrationAccessMu.Unlock()
}
// LoadMigrationAccessState returns cached migration maintenance state.
func LoadMigrationAccessState(ctx context.Context) MigrationAccessState {
migrationAccessMu.RLock()
if migrationAccessValid && time.Since(migrationAccessCheckedAt) < time.Duration(shared.AccessCacheTTL)*time.Second {
state := migrationAccessCached
migrationAccessMu.RUnlock()
return state
}
migrationAccessMu.RUnlock()
migrationAccessMu.Lock()
defer migrationAccessMu.Unlock()
if migrationAccessValid && time.Since(migrationAccessCheckedAt) < time.Duration(shared.AccessCacheTTL)*time.Second {
return migrationAccessCached
}
migrationAccessCached = buildMigrationAccessState(ctx)
migrationAccessValid = true
migrationAccessCheckedAt = time.Now()
return migrationAccessCached
}
func buildMigrationAccessState(ctx context.Context) MigrationAccessState {
execution, ok, err := LatestMigrationExecution(ctx)
if err != nil {
return MigrationAccessState{LoadErr: err, ReadOnly: true}
}
if !ok {
return MigrationAccessState{}
}
state := MigrationAccessState{
ReadOnly: execution.Status != model.TaskExecutionStatusSucceeded,
}
if execution.Status == model.TaskExecutionStatusSucceeded {
return state
}
target, err := ParseMigrationTargetConfig(ctx, []byte(execution.Payload))
if err != nil {
state.TargetErr = err
return state
}
state.Target = target
state.HasTarget = true
return state
}
@@ -0,0 +1,81 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package storage
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
)
// StorageMigrationTask is the Asynq task name for storage migration.
const StorageMigrationTask = "storage:migrate"
// LatestMigrationExecution returns the most recent storage migration task execution.
func LatestMigrationExecution(ctx context.Context) (*model.TaskExecution, bool, error) {
return repository.GetLatestTaskExecutionByTaskType(ctx, StorageMigrationTask)
}
// ParseMigrationTargetConfig parses and validates a storage migration target payload.
func ParseMigrationTargetConfig(ctx context.Context, payload []byte) (objectstore.Config, error) {
if strings.TrimSpace(string(payload)) == "" {
return objectstore.Config{}, errors.New("storage migration target payload is required")
}
var raw struct {
Target json.RawMessage `json:"target"`
}
if err := json.Unmarshal(payload, &raw); err != nil {
return objectstore.Config{}, fmt.Errorf("parse storage migration payload envelope: %w", err)
}
if len(raw.Target) == 0 {
return objectstore.Config{}, errors.New("storage migration target payload is required")
}
var targetBytes []byte
var targetStr string
if err := json.Unmarshal(raw.Target, &targetStr); err == nil {
targetBytes = []byte(targetStr)
} else {
targetBytes = raw.Target
}
var target objectstore.Config
if err := json.Unmarshal(targetBytes, &target); err != nil {
return objectstore.Config{}, fmt.Errorf("parse target storage config: %w", err)
}
current, err := objectstore.LoadConfig(ctx)
if err != nil {
return objectstore.Config{}, fmt.Errorf("load active storage config: %w", err)
}
target = objectstore.MergeMaskedSecrets(target, current)
if err := objectstore.ValidateConfig(target); err != nil {
return objectstore.Config{}, fmt.Errorf("validate target storage config: %w", err)
}
return target, nil
}
// NormalizeMigrationPayload validates and normalizes a storage migration payload.
func NormalizeMigrationPayload(ctx context.Context, payload []byte) ([]byte, objectstore.Config, error) {
target, err := ParseMigrationTargetConfig(ctx, payload)
if err != nil {
return nil, objectstore.Config{}, err
}
type storageMigrationPayload struct {
Target objectstore.Config `json:"target"`
}
normalized, err := json.Marshal(storageMigrationPayload{Target: target})
if err != nil {
return nil, objectstore.Config{}, fmt.Errorf("marshal storage migration payload: %w", err)
}
return normalized, target, nil
}
@@ -0,0 +1,31 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package storage
import (
"context"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
// ReadOnly checks if the storage system is in read-only maintenance mode.
func ReadOnly(ctx context.Context) bool {
state := LoadMigrationAccessState(ctx)
if state.LoadErr != nil {
logger.ErrorF(ctx, "读取存储维护状态失败: %v", state.LoadErr)
return true
}
return state.ReadOnly
}
// OpenStoredObject opens a stored upload object from the active storage backend.
func OpenStoredObject(ctx context.Context, upload *model.Upload) (*objectstore.Object, error) {
_, backend, err := objectstore.Active(ctx)
if err != nil {
return nil, err
}
return backend.Get(ctx, upload.FilePath)
}
+160
View File
@@ -0,0 +1,160 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package task provides upload-related async background task handlers.
package task
import (
"context"
"errors"
"fmt"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
uploadcache "github.com/Rain-kl/Wavelet/plugins/domain/upload/cache"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"gorm.io/gorm"
)
const (
// SystemCleanupTask 系统定期垃圾清理任务标识
SystemCleanupTask = "system:cleanup"
// TaskTypeSystemCleanup 系统定期垃圾清理管理类型
TaskTypeSystemCleanup = "system_cleanup"
)
// SystemCleanupMeta represents the task metadata.
var SystemCleanupMeta = task.TaskMeta{
Type: TaskTypeSystemCleanup,
AsynqTask: SystemCleanupTask,
Name: "系统垃圾清理",
Description: "定期清理未使用上传文件、历史推送记录和过期任务执行日志",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
}
// SystemCleanupHandler 系统定期垃圾清理异步任务处理器
type SystemCleanupHandler struct{}
// Execute 执行系统清理(包含文件清理、历史推送日志和任务执行日志清理)
func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
if uploadstorage.ReadOnly(ctx) {
return nil, errors.New(shared.ErrStorageReadOnly)
}
const batchSize = 100
var lastID uint64
var totalProcessed int
var totalDeleted int
oneHourAgo := time.Now().Add(-1 * time.Hour)
task.AppendLog(ctx, "开始扫描未使用上传文件,阈值: %s", oneHourAgo.Format(time.RFC3339))
for {
var unusedUploads []model.Upload
if err := db.DB(ctx).
Where("id > ? AND status = ? AND created_at < ?", lastID, model.UploadStatusPending, oneHourAgo).
Order("id ASC").
Limit(batchSize).
Find(&unusedUploads).Error; err != nil {
task.AppendLog(ctx, "查询未使用的上传文件失败: %v", err)
return nil, fmt.Errorf(shared.ErrQueryUnusedUploadsFailed, err)
}
if len(unusedUploads) == 0 {
break
}
task.AppendLog(ctx, "本批次找到 %d 个需要清理的上传文件", len(unusedUploads))
for _, u := range unusedUploads {
totalProcessed++
if err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&model.Upload{}).
Where("id = ? AND status = ?", u.ID, model.UploadStatusPending).
Update("status", model.UploadStatusDeleted).Error; err != nil {
return err
}
_, backend, err := objectstore.Active(ctx)
if err != nil {
return err
}
if err := backend.Delete(ctx, u.FilePath); err != nil {
return err
}
return nil
}); err != nil {
task.AppendLog(ctx, "清理上传文件失败 [ID:%d]: %v", u.ID, err)
lastID = u.ID
continue
}
uploadstats.RecordUploadStatsRemove(ctx, &u)
uploadcache.InvalidateUploadMetaCache(ctx, u.ID)
totalDeleted++
lastID = u.ID
}
}
task.AppendLog(ctx, "开始清理历史推送审计日志,只保留最近7天数据...")
cutoff := time.Now().AddDate(0, 0, -7)
var pushHistoryCount int64
if err := db.DB(ctx).Model(&model.PushHistory{}).Where("created_at < ?", cutoff).Count(&pushHistoryCount).Error; err != nil {
task.AppendLog(ctx, "统计待清理的历史推送记录失败: %v", err)
} else if pushHistoryCount > 0 {
if err := db.DB(ctx).Where("created_at < ?", cutoff).Delete(&model.PushHistory{}).Error; err != nil {
task.AppendLog(ctx, "删除历史推送记录失败: %v", err)
} else {
task.AppendLog(ctx, "成功删除 %d 条历史推送记录 (截止时间: %s)", pushHistoryCount, cutoff.Format("2006-01-02 15:04:05"))
}
} else {
task.AppendLog(ctx, "没有需要清理的历史推送记录 (截止时间: %s)", cutoff.Format("2006-01-02 15:04:05"))
}
task.AppendLog(ctx, "开始清理任务执行日志:高频任务保留最近3天,低频任务保留最近30天...")
taskLogStats, err := repository.CleanupTaskExecutionLogs(ctx, time.Now())
if err != nil {
task.AppendLog(ctx, "清理任务执行日志失败: %v", err)
logger.ErrorF(ctx, "清理任务执行日志失败: %v", err)
} else {
task.AppendLog(ctx, "成功清理任务执行日志 %d 条(高频 %d 条,低频 %d 条)",
taskLogStats.HighFrequencyDeleted+taskLogStats.LowFrequencyDeleted,
taskLogStats.HighFrequencyDeleted,
taskLogStats.LowFrequencyDeleted,
)
}
var logDeleted int64
logSummary, logErr := logstore.CleanupExpired(ctx)
if logErr != nil {
task.AppendLog(ctx, "清理过期用户访问日志失败: %v", logErr)
logger.ErrorF(ctx, "清理过期用户访问日志失败: %v", logErr)
} else {
logDeleted = logSummary.Deleted
task.AppendLog(ctx, "成功清理过期用户访问日志 %d 条(%s 保留 %d 天)",
logSummary.Deleted, logSummary.ActiveDatabase, logSummary.RetentionDays)
}
msg := fmt.Sprintf("系统清理完成。成功清理未使用的上传文件 %d/%d 个;清理历史推送审计日志 %d 条;清理任务执行日志 %d 条;清理过期访问日志 %d 条。",
totalDeleted,
totalProcessed,
pushHistoryCount,
taskLogStats.HighFrequencyDeleted+taskLogStats.LowFrequencyDeleted,
logDeleted,
)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
@@ -0,0 +1,72 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"fmt"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
)
const (
// RebuildUploadStatsTask is the Asynq task name for rebuilding upload stats.
RebuildUploadStatsTask = "upload:rebuild_stats"
// TaskTypeRebuildUploadStats is the admin-dispatchable task type.
TaskTypeRebuildUploadStats = "rebuild_upload_stats"
)
// RebuildUploadStatsMeta describes the upload stats rebuild task.
var RebuildUploadStatsMeta = task.TaskMeta{
Type: TaskTypeRebuildUploadStats,
AsynqTask: RebuildUploadStatsTask,
Name: "重算文件存储统计",
Description: "根据当前 w_uploads 活跃记录全量重建 w_upload_stats(总量、类型、分类、趋势)",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
}
// RebuildUploadStatsHandler rebuilds incremental upload stats from active upload records.
type RebuildUploadStatsHandler struct{}
// Execute scans active uploads and rebuilds all upload stat dimensions.
func (h *RebuildUploadStatsHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
var activeCount int64
if err := db.DB(ctx).
Model(&model.Upload{}).
Where("status != ?", model.UploadStatusDeleted).
Count(&activeCount).Error; err != nil {
task.AppendLog(ctx, "统计活跃上传记录失败: %v", err)
return nil, fmt.Errorf("count active uploads: %w", err)
}
task.AppendLog(ctx, "开始重算文件存储统计,活跃记录数: %d", activeCount)
if err := uploadstats.RebuildUploadStats(ctx); err != nil {
task.AppendLog(ctx, "重算文件存储统计失败: %v", err)
return nil, fmt.Errorf("rebuild upload stats: %w", err)
}
var totalStat model.UploadStat
if err := db.DB(ctx).
Where("dimension = ? AND stat_key = ?", model.UploadStatDimensionTotal, "").
First(&totalStat).Error; err != nil {
task.AppendLog(ctx, "读取总量统计失败: %v", err)
return nil, fmt.Errorf("load total upload stats: %w", err)
}
msg := fmt.Sprintf(
"文件存储统计重算完成,活跃记录 %d 条,统计文件数 %d,总大小 %d 字节",
activeCount,
totalStat.FileCount,
totalStat.FileSize,
)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
@@ -0,0 +1,69 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
)
func TestRebuildUploadStatsHandler_Execute(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
now := time.Now()
uploads := []model.Upload{
{
UserID: 1001, FileName: "a.jpg", FilePath: "uploads/a.jpg",
FileSize: 100, MimeType: "image/jpeg", Extension: "jpg", Hash: "hash-a",
Type: "pixez_mirror", Status: model.UploadStatusUsed, CreatedAt: now,
},
{
UserID: 1001, FileName: "b.png", FilePath: "uploads/b.png",
FileSize: 200, MimeType: "image/png", Extension: "png", Hash: "hash-b",
Type: "attachment", Status: model.UploadStatusUsed, CreatedAt: now,
},
}
for i := range uploads {
if err := db.DB(ctx).Create(&uploads[i]).Error; err != nil {
t.Fatalf("seed upload failed: %v", err)
}
}
// Corrupt stats to ensure rebuild recalculates from uploads.
if err := db.DB(ctx).Create(&model.UploadStat{
Dimension: model.UploadStatDimensionTotal,
StatKey: "",
FileCount: 0,
FileSize: 0,
}).Error; err != nil {
t.Fatalf("seed broken total stat failed: %v", err)
}
handler := &RebuildUploadStatsHandler{}
result, err := handler.Execute(ctx, nil)
if err != nil {
t.Fatalf("Execute() error = %v", err)
}
if result == nil || result.Message == "" {
t.Fatalf("Execute() returned empty result: %+v", result)
}
var totalStat model.UploadStat
if err := db.DB(ctx).
Where("dimension = ? AND stat_key = ?", model.UploadStatDimensionTotal, "").
First(&totalStat).Error; err != nil {
t.Fatalf("load total stat failed: %v", err)
}
if totalStat.FileCount != 2 || totalStat.FileSize != 300 {
t.Fatalf("total stat = count %d size %d, want 2 / 300", totalStat.FileCount, totalStat.FileSize)
}
}
@@ -0,0 +1,378 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"strings"
"sync/atomic"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/util"
uploadstats "github.com/Rain-kl/Wavelet/plugins/domain/upload/stats"
uploadstorage "github.com/Rain-kl/Wavelet/plugins/domain/upload/storage"
"golang.org/x/sync/errgroup"
)
const (
// StorageMigrationTask is the Asynq task name for storage migration.
StorageMigrationTask = uploadstorage.StorageMigrationTask
// TaskTypeStorageMigration is the task metadata type for storage migration.
TaskTypeStorageMigration = "storage_migration"
)
// StorageMigrationMeta describes the manually dispatchable migration task.
var StorageMigrationMeta = task.TaskMeta{
Type: TaskTypeStorageMigration,
AsynqTask: StorageMigrationTask,
Name: "迁移文件存储",
Description: "将活动存储中的文件迁移到待切换的目标存储,迁移期间文件系统保持只读",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
Params: []task.TaskParam{
{
Name: "target",
Label: "目标存储配置 (JSON)",
Type: "text",
Required: true,
Placeholder: `{"driver": "s3", "local": {"root": "."}, "s3": {"bucket": "my-bucket", ...}}`,
Description: "待迁移到的目标存储引擎完整配置 JSON 字符串",
},
},
}
// MigrationHandler copies stored objects and activates the target backend.
type MigrationHandler struct{}
// ValidatePayload rejects duplicate active migrations through the task framework.
func (h *MigrationHandler) ValidatePayload(payload []byte) ([]byte, error) {
normalized, _, err := uploadstorage.NormalizeMigrationPayload(context.Background(), payload)
if err != nil {
return payload, err
}
active, err := hasUnresolvedMigrationTask(context.Background())
if err != nil {
return payload, err
}
if active {
return payload, fmt.Errorf("storage migration task is already unresolved")
}
return normalized, nil
}
// Execute migrates all unique active-storage objects to the pending backend.
func (h *MigrationHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
if db.Redis != nil {
const (
cleanupTimeout = 5 * time.Second
renewalInterval = 10 * time.Minute
)
lockKey := db.PrefixedKey("lock:storage:migrate")
ok, err := db.Redis.SetNX(ctx, lockKey, "locked", time.Hour).Result()
if err != nil {
return nil, fmt.Errorf("acquire migration lock: %w", err)
}
if !ok {
return nil, errors.New("另一个存储迁移任务正在运行中")
}
stopRenewal := make(chan struct{})
//nolint:contextcheck
defer func() {
close(stopRenewal)
cleanupCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()
_ = db.Redis.Del(cleanupCtx, lockKey)
}()
//nolint:contextcheck,gosec
util.Go(func() {
ticker := time.NewTicker(renewalInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
renewCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
_ = db.Redis.Expire(renewCtx, lockKey, time.Hour).Err()
cancel()
case <-stopRenewal:
return
case <-ctx.Done():
return
}
}
})
}
active, err := objectstore.LoadConfig(ctx)
if err != nil {
return nil, fmt.Errorf("load active storage config: %w", err)
}
target, err := uploadstorage.ParseMigrationTargetConfig(ctx, payload)
if err != nil {
return nil, err
}
if target.Driver == active.Driver {
if err := objectstore.SaveActiveConfig(ctx, target); err != nil {
return nil, fmt.Errorf("activate same-driver storage config: %w", err)
}
message := fmt.Sprintf("存储配置已更新,活动存储保持为 %s", target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
}
total, err := countStorageObjects(ctx)
if err != nil {
return nil, fmt.Errorf("count source objects: %w", err)
}
if total == 0 {
if err := objectstore.SaveActiveConfig(ctx, target); err != nil {
return nil, fmt.Errorf("activate empty storage config: %w", err)
}
message := fmt.Sprintf("当前存储没有需要迁移的对象,活动存储已切换为 %s", target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
}
sourceBackend, err := objectstore.NewBackend(ctx, active, active.Driver)
if err != nil {
return nil, fmt.Errorf("create source storage: %w", err)
}
targetBackend, err := objectstore.NewBackend(ctx, target, target.Driver)
if err != nil {
return nil, fmt.Errorf("create target storage: %w", err)
}
task.AppendLog(ctx, "开始存储迁移: %s -> %s,总对象数: %d", active.Driver, target.Driver, total)
migrated, err := migrateObjects(ctx, sourceBackend, targetBackend, total)
if err != nil {
return nil, err
}
if err := objectstore.SaveActiveConfig(ctx, target); err != nil {
return nil, fmt.Errorf("activate target storage: %w", err)
}
message := fmt.Sprintf("存储迁移完成,共迁移 %d 个对象,活动存储已切换为 %s", migrated, target.Driver)
task.AppendLog(ctx, "%s", message)
return &task.TaskResult{Message: message}, nil
}
func countStorageObjects(ctx context.Context) (int64, error) {
var count int64
err := db.DB(ctx).Model(&model.Upload{}).
Where("status != ?", model.UploadStatusDeleted).
Distinct("file_path").
Count(&count).Error
return count, err
}
func hasUnresolvedMigrationTask(ctx context.Context) (bool, error) {
execution, ok, err := uploadstorage.LatestMigrationExecution(ctx)
if err != nil || !ok {
return false, err
}
return execution.Status == model.TaskExecutionStatusPending || execution.Status == model.TaskExecutionStatusRunning, nil
}
type migrationObject struct {
FilePath string `gorm:"column:file_path"`
FileSize int64 `gorm:"column:file_size"`
MimeType string `gorm:"column:mime_type"`
Hash string `gorm:"column:hash"`
}
func migrateObjects(
ctx context.Context,
sourceBackend objectstore.Backend,
targetBackend objectstore.Backend,
total int64,
) (int64, error) {
const batchSize = 50
const migrationConcurrency = 10
const sha256HexLength = 64
var migrated int64
var lastFilePath string
for {
if err := ctx.Err(); err != nil {
return atomic.LoadInt64(&migrated), fmt.Errorf("storage migration canceled: %w", err)
}
task.AppendLog(ctx, "正在查询待迁移对象批次,当前已完成迁移: %d/%d", atomic.LoadInt64(&migrated), total)
var objects []migrationObject
query := db.DB(ctx).Model(&model.Upload{}).
Select("file_path, MAX(file_size) AS file_size, MAX(mime_type) AS mime_type, MAX(hash) AS hash").
Where("status != ?", model.UploadStatusDeleted)
if lastFilePath != "" {
query = query.Where("file_path > ?", lastFilePath)
}
if err := query.Group("file_path").
Order("file_path ASC").
Limit(batchSize).
Scan(&objects).Error; err != nil {
return atomic.LoadInt64(&migrated), fmt.Errorf("query source objects: %w", err)
}
if len(objects) == 0 {
task.AppendLog(ctx, "所有对象迁移完毕")
break
}
lastFilePath = objects[len(objects)-1].FilePath
task.AppendLog(ctx, "获取当前批次迁移对象,批次大小: %d,实际获取对象数: %d", batchSize, len(objects))
var g errgroup.Group
g.SetLimit(migrationConcurrency)
for _, object := range objects {
obj := object
g.Go(func() error {
if err := migrateSingleObject(ctx, sourceBackend, targetBackend, obj, sha256HexLength); err != nil {
return err
}
atomic.AddInt64(&migrated, 1)
return nil
})
}
if err := g.Wait(); err != nil {
return atomic.LoadInt64(&migrated), err
}
task.AppendLog(ctx, "当前批次迁移完成。迁移进度: %d/%d", atomic.LoadInt64(&migrated), total)
}
return atomic.LoadInt64(&migrated), nil
}
func migrateSingleObject(
ctx context.Context,
sourceBackend objectstore.Backend,
targetBackend objectstore.Backend,
obj migrationObject,
sha256HexLength int,
) error {
if shouldSkipMigration(ctx, targetBackend, obj) {
task.AppendLog(ctx, "[跳过迁移] 目标存储已存在相同文件: %s", obj.FilePath)
return nil
}
task.AppendLog(ctx, "[迁移开始] 正在从源存储读取文件: %s", obj.FilePath)
source, err := sourceBackend.Get(ctx, obj.FilePath)
if err != nil {
if isNotFoundError(err) {
return markMissingMigrationObjectDeleted(ctx, obj.FilePath, err)
}
return fmt.Errorf("open source object %q: %w", obj.FilePath, err)
}
task.AppendLog(ctx, "[传输中] 正在向目标存储上传文件: %s (大小: %d 字节, 类型: %s)", obj.FilePath, obj.FileSize, obj.MimeType)
targetResult, putErr := targetBackend.Put(ctx, obj.FilePath, source.Body, obj.FileSize, obj.MimeType)
closeErr := source.Body.Close()
if putErr != nil {
return fmt.Errorf("copy object %q: %w", obj.FilePath, putErr)
}
if closeErr != nil {
return fmt.Errorf("close source object %q: %w", obj.FilePath, closeErr)
}
if len(obj.Hash) == sha256HexLength {
task.AppendLog(ctx, "[校验中] 正在对目标文件进行数据一致性校验 (SHA-256): %s", targetResult.Key)
targetObj, getErr := targetBackend.Get(ctx, targetResult.Key)
if getErr != nil {
return fmt.Errorf("retrieve target object for verification %q: %w", obj.FilePath, getErr)
}
if targetObj == nil || targetObj.Body == nil {
return fmt.Errorf("retrieve target object for verification %q: object or body is nil", obj.FilePath)
}
h := sha256.New()
if _, copyErr := io.Copy(h, targetObj.Body); copyErr != nil {
_ = targetObj.Body.Close()
return fmt.Errorf("read target object for verification %q: %w", obj.FilePath, copyErr)
}
_ = targetObj.Body.Close()
computedHash := hex.EncodeToString(h.Sum(nil))
if computedHash != obj.Hash {
return fmt.Errorf("integrity check failed for %q: got hash %s, want %s", obj.FilePath, computedHash, obj.Hash)
}
task.AppendLog(ctx, "[校验通过] 文件一致性校验成功: %s", targetResult.Key)
}
if targetResult.Key != obj.FilePath {
task.AppendLog(ctx, "[更新数据库] 正在更新文件路径: %s -> %s", obj.FilePath, targetResult.Key)
if err := db.DB(ctx).Model(&model.Upload{}).
Where("file_path = ? AND status != ?", obj.FilePath, model.UploadStatusDeleted).
Update("file_path", targetResult.Key).Error; err != nil {
return fmt.Errorf("update migrated object %q: %w", obj.FilePath, err)
}
}
task.AppendLog(ctx, "[迁移成功] 文件已完成迁移: %s", targetResult.Key)
return nil
}
func shouldSkipMigration(
ctx context.Context,
targetBackend objectstore.Backend,
obj migrationObject,
) bool {
targetObj, err := targetBackend.Get(ctx, obj.FilePath)
if err != nil || targetObj == nil || targetObj.Body == nil {
return false
}
defer func() {
_ = targetObj.Body.Close()
}()
return targetObj.ContentLength == obj.FileSize
}
func markMissingMigrationObjectDeleted(
ctx context.Context,
filePath string,
sourceErr error,
) error {
task.AppendLog(ctx, "警告: 源存储中物理文件不存在,标记为已删除并跳过: %s (错误: %v)", filePath, sourceErr)
var affectedUploads []model.Upload
if err := db.DB(ctx).
Where("file_path = ? AND status != ?", filePath, model.UploadStatusDeleted).
Find(&affectedUploads).Error; err != nil {
return fmt.Errorf("load missing object uploads %q: %w", filePath, err)
}
if err := db.DB(ctx).Model(&model.Upload{}).
Where("file_path = ?", filePath).
Update("status", model.UploadStatusDeleted).Error; err != nil {
return fmt.Errorf("update missing object %q: %w", filePath, err)
}
for i := range affectedUploads {
uploadstats.RecordUploadStatsRemove(ctx, &affectedUploads[i])
}
return nil
}
func isNotFoundError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, os.ErrNotExist) {
return true
}
errStr := strings.ToLower(err.Error())
for _, sub := range []string{"not found", "nosuchkey", "nosuchbucket", "404", "does not exist"} {
if strings.Contains(errStr, sub) {
return true
}
}
return false
}
@@ -0,0 +1,286 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
)
func TestMigrationHandlerExecute(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
sourceRoot := t.TempDir()
sourcePath := filepath.Join(sourceRoot, "uploads", "test.txt")
if err := os.MkdirAll(filepath.Dir(sourcePath), 0755); err != nil {
t.Fatalf("MkdirAll(%q) returned error: %v", sourcePath, err)
}
const content = "storage migration"
if err := os.WriteFile(sourcePath, []byte(content), 0644); err != nil {
t.Fatalf("WriteFile(%q) returned error: %v", sourcePath, err)
}
ctx := context.Background()
active := objectstore.DefaultConfig()
active.Local.Root = sourceRoot
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
t.Fatalf("SaveActiveConfig() returned error: %v", err)
}
target := objectstore.DefaultConfig()
target.Driver = objectstore.DriverS3
target.S3 = objectstore.ObjectConfig{
Region: "us-east-1",
Bucket: "target",
AccessKeyID: "key",
SecretAccessKey: "secret",
}
payload, err := json.Marshal(struct {
Target objectstore.Config `json:"target"`
}{Target: target})
if err != nil {
t.Fatalf("Marshal(storageMigrationPayload) returned error: %v", err)
}
upload := model.Upload{
ID: 99101,
UserID: 1,
FileName: "test.txt",
FilePath: "uploads/test.txt",
FileSize: int64(len(content)),
MimeType: "text/plain",
Extension: "txt",
Hash: "hash",
Type: "attachment",
Status: model.UploadStatusUsed,
}
if err := dbConn.Create(&upload).Error; err != nil {
t.Fatalf("Create(upload) returned error: %v", err)
}
var copied bytes.Buffer
restore := objectstore.MockStorage(
func(_ context.Context, _ string, body io.Reader, _ int64, _ string) error {
_, err := io.Copy(&copied, body)
return err
},
func(context.Context, string) (*objectstore.Object, error) {
return nil, nil
},
func(context.Context, string) error {
return nil
},
)
defer restore()
result, err := (&MigrationHandler{}).Execute(ctx, payload)
if err != nil {
t.Fatalf("Execute() returned error: %v", err)
}
if result == nil {
t.Fatal("Execute() result = nil, want non-nil")
}
if copied.String() != content {
t.Errorf("migrated content = %q, want %q", copied.String(), content)
}
var migrated model.Upload
if err := dbConn.First(&migrated, upload.ID).Error; err != nil {
t.Fatalf("First(upload) returned error: %v", err)
}
current, err := objectstore.LoadConfig(ctx)
if err != nil {
t.Fatalf("LoadConfig() returned error: %v", err)
}
if current.Driver != objectstore.DriverS3 {
t.Errorf("active driver = %q, want %q", current.Driver, objectstore.DriverS3)
}
}
func TestMigrationHandlerExecuteWithHashValidation(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
sourceRoot := t.TempDir()
sourcePath := filepath.Join(sourceRoot, "uploads", "test-hash.txt")
if err := os.MkdirAll(filepath.Dir(sourcePath), 0755); err != nil {
t.Fatalf("MkdirAll(%q) returned error: %v", sourcePath, err)
}
const content = "storage migration integrity check content"
if err := os.WriteFile(sourcePath, []byte(content), 0644); err != nil {
t.Fatalf("WriteFile(%q) returned error: %v", sourcePath, err)
}
// Calculate correct SHA-256 hash
h := sha256.New()
h.Write([]byte(content))
correctHash := hex.EncodeToString(h.Sum(nil))
ctx := context.Background()
active := objectstore.DefaultConfig()
active.Local.Root = sourceRoot
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
t.Fatalf("SaveActiveConfig() returned error: %v", err)
}
target := objectstore.DefaultConfig()
target.Driver = objectstore.DriverS3
target.S3 = objectstore.ObjectConfig{
Region: "us-east-1",
Bucket: "target",
AccessKeyID: "key",
SecretAccessKey: "secret",
}
payload, err := json.Marshal(struct {
Target objectstore.Config `json:"target"`
}{Target: target})
if err != nil {
t.Fatalf("Marshal(storageMigrationPayload) returned error: %v", err)
}
// Case 1: Incorrect Hash (should fail validation)
uploadIncorrect := model.Upload{
ID: 99102,
UserID: 1,
FileName: "test-hash.txt",
FilePath: "uploads/test-hash.txt",
FileSize: int64(len(content)),
MimeType: "text/plain",
Extension: "txt",
Hash: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // Invalid hash
Type: "attachment",
Status: model.UploadStatusUsed,
}
if err := dbConn.Create(&uploadIncorrect).Error; err != nil {
t.Fatalf("Create(uploadIncorrect) returned error: %v", err)
}
var copied bytes.Buffer
restore := objectstore.MockStorage(
func(_ context.Context, _ string, body io.Reader, _ int64, _ string) error {
copied.Reset()
_, err := io.Copy(&copied, body)
return err
},
func(context.Context, string) (*objectstore.Object, error) {
return &objectstore.Object{
Body: io.NopCloser(bytes.NewBuffer(copied.Bytes())),
ContentLength: int64(copied.Len()),
ContentType: "text/plain",
}, nil
},
func(context.Context, string) error {
return nil
},
)
defer restore()
// Running execution with incorrect hash should fail with integrity error
_, err = (&MigrationHandler{}).Execute(ctx, payload)
if err == nil {
t.Fatal("Execute() succeeded with incorrect hash, want error")
}
if !strings.Contains(err.Error(), "integrity check failed") {
t.Errorf("expected integrity check failed error, got: %v", err)
}
// Case 2: Correct Hash (should succeed)
if err := dbConn.Model(&model.Upload{}).Where("id = ?", uploadIncorrect.ID).Update("hash", correctHash).Error; err != nil {
t.Fatalf("Update hash to correct value returned error: %v", err)
}
// Run execution with correct hash should succeed
result, err := (&MigrationHandler{}).Execute(ctx, payload)
if err != nil {
t.Fatalf("Execute() with correct hash failed: %v", err)
}
if result == nil {
t.Fatal("Execute() result = nil, want non-nil")
}
var migrated model.Upload
if err := dbConn.First(&migrated, uploadIncorrect.ID).Error; err != nil {
t.Fatalf("First(upload) returned error: %v", err)
}
if migrated.FilePath != "uploads/test-hash.txt" {
t.Errorf("FilePath = %q, want %q", migrated.FilePath, "uploads/test-hash.txt")
}
}
func TestMigrationHandlerExecuteWithRedisLock(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("Failed to run miniredis: %v", err)
}
defer mr.Close()
rdb := redis.NewClient(&redis.Options{
Addr: mr.Addr(),
})
defer rdb.Close()
oldRedis := db.Redis
db.Redis = rdb
defer func() {
db.Redis = oldRedis
}()
ctx := context.Background()
// Acquire lock manually
lockKey := db.PrefixedKey("lock:storage:migrate")
if err := rdb.Set(ctx, lockKey, "locked", time.Hour).Err(); err != nil {
t.Fatalf("Failed to set manual lock in Redis: %v", err)
}
active := objectstore.DefaultConfig()
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
t.Fatalf("SaveActiveConfig() returned error: %v", err)
}
payload, err := json.Marshal(struct {
Target objectstore.Config `json:"target"`
}{Target: active})
if err != nil {
t.Fatalf("Marshal payload failed: %v", err)
}
// Execution should fail because lock is already acquired
_, err = (&MigrationHandler{}).Execute(ctx, payload)
if err == nil {
t.Fatal("Execute() succeeded when lock was held, want error")
}
if !strings.Contains(err.Error(), "另一个存储迁移任务正在运行中") {
t.Errorf("expected lock warning, got: %v", err)
}
// Release lock and run again, should succeed
if err := rdb.Del(ctx, lockKey).Err(); err != nil {
t.Fatalf("Failed to delete lock: %v", err)
}
_, err = (&MigrationHandler{}).Execute(ctx, payload)
if err != nil {
t.Fatalf("Execute() failed after lock released: %v", err)
}
}
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
)
const (
// WarmImageCacheTask 图片压缩缓存预热任务标识
WarmImageCacheTask = "upload:warm_image_cache"
// TaskTypeWarmImageCache 图片压缩缓存预热管理类型
TaskTypeWarmImageCache = "warm_image_cache"
)
var warmImageCacheMu sync.Mutex
// WarmImageCacheMeta represents the image cache warmup task metadata.
var WarmImageCacheMeta = task.TaskMeta{
Type: TaskTypeWarmImageCache,
AsynqTask: WarmImageCacheTask,
Name: "预热图片压缩缓存",
Description: "串行将文件管理中的图片转换为指定质量的 WebP 并写入永久缓存",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
Params: []task.TaskParam{
{
Name: "quality",
Label: "图片质量",
Type: "string",
Required: true,
Placeholder: "low / medium / high",
Description: "WebP 压缩质量,仅支持 low、medium、high",
},
},
}
// WarmImageCachePayload is the image cache warmup task payload.
type WarmImageCachePayload struct {
Quality string `json:"quality"`
}
// WarmImageCacheHandler serially warms compressed image cache entries.
type WarmImageCacheHandler struct{}
// ValidatePayload validates and normalizes image cache warmup parameters.
func (h *WarmImageCacheHandler) ValidatePayload(payload []byte) ([]byte, error) {
if len(payload) == 0 {
return nil, errors.New(shared.ErrImageCacheWarmupPayloadRequired)
}
var req WarmImageCachePayload
if err := json.Unmarshal(payload, &req); err != nil {
return nil, fmt.Errorf(shared.ErrInvalidImageCacheWarmupPayload, err)
}
req.Quality = strings.ToLower(strings.TrimSpace(req.Quality))
if req.Quality != shared.ImageQualityLow &&
req.Quality != shared.ImageQualityMedium &&
req.Quality != shared.ImageQualityHigh {
return nil, errors.New(shared.ErrInvalidImageCacheWarmupQuality)
}
return json.Marshal(req)
}
// Execute serially converts all managed images to WebP cache entries.
func (h *WarmImageCacheHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
normalizedPayload, err := h.ValidatePayload(payload)
if err != nil {
task.AppendLog(ctx, "图片缓存预热参数无效: %v", err)
return nil, err
}
var req WarmImageCachePayload
if err := json.Unmarshal(normalizedPayload, &req); err != nil {
return nil, fmt.Errorf(shared.ErrParseImageCacheWarmupPayload, err)
}
task.AppendLog(ctx, "等待获取图片缓存预热执行锁,质量: %s", req.Quality)
warmImageCacheMu.Lock()
defer warmImageCacheMu.Unlock()
const (
batchSize = 50
maxFailureLogs = 5
)
var lastID uint64
var totalProcessed int
var totalCached int
var totalGenerated int
var totalFailed int
task.AppendLog(ctx, "开始串行预热图片压缩缓存,质量: %s,每批: %d", req.Quality, batchSize)
for {
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("image cache warmup canceled: %w", err)
}
var uploads []model.Upload
if err := db.DB(ctx).
Where("id > ? AND status != ? AND (LOWER(mime_type) LIKE ? OR LOWER(extension) IN ?)",
lastID,
model.UploadStatusDeleted,
"image/%",
[]string{"jpg", "jpeg", "png", "webp", "gif"},
).
Order("id ASC").
Limit(batchSize).
Find(&uploads).Error; err != nil {
task.AppendLog(ctx, "查询图片上传记录失败: %v", err)
return nil, fmt.Errorf(shared.ErrQueryImagesForCacheWarmup, err)
}
if len(uploads) == 0 {
break
}
batchGenerated := 0
batchCached := 0
batchFailed := 0
for i := range uploads {
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("image cache warmup canceled: %w", err)
}
upload := &uploads[i]
totalProcessed++
lastID = upload.ID
_, cacheHit, err := filesrv.EnsureCompressedImageCache(ctx, upload, req.Quality)
if err != nil {
totalFailed++
batchFailed++
if totalFailed <= maxFailureLogs {
task.AppendLog(ctx, "图片处理失败 [ID:%d]: %v", upload.ID, err)
}
continue
}
if cacheHit {
totalCached++
batchCached++
continue
}
totalGenerated++
batchGenerated++
}
task.AppendLog(
ctx,
"批次完成,末尾 ID: %d,生成: %d,命中: %d,失败: %d",
lastID,
batchGenerated,
batchCached,
batchFailed,
)
}
msg := fmt.Sprintf(
"图片缓存预热完成,共处理 %d 张,生成 %d 张,命中 %d 张,失败 %d 张",
totalProcessed,
totalGenerated,
totalCached,
totalFailed,
)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
+386
View File
@@ -0,0 +1,386 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"bytes"
"context"
"encoding/json"
"image"
"image/color"
"image/png"
"io"
"os"
"path/filepath"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/diskcache"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/filesrv"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSystemCleanupHandler_Execute(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
// Mock S3 存储(让 DeleteObject 总是成功)
storageMock := objectstore.MockStorage(
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
return nil
},
func(ctx context.Context, key string) (*objectstore.Object, error) { return nil, nil },
func(ctx context.Context, key string) error { return nil },
)
defer storageMock()
objectstore.IsEnabledFunc = func() bool { return true }
defer func() { objectstore.IsEnabledFunc = func() bool { return false } }()
objectstore.ResetCache()
ctx := context.Background()
err := db.DB(ctx).AutoMigrate(&model.PushHistory{})
require.NoError(t, err)
// 准备测试数据:创建一些上传记录
now := time.Now()
twoHoursAgo := now.Add(-2 * time.Hour)
records := []*model.Upload{
// 超过1小时且状态为 pending 的记录 —— 应被清理
{
UserID: 1001, FileName: "old_file_1.jpg", FilePath: "uploads/old_1.jpg",
FileSize: 1024, MimeType: "image/jpeg", Extension: "jpg", Hash: "hash1",
Type: "attachment", Status: model.UploadStatusPending,
CreatedAt: twoHoursAgo,
},
{
UserID: 1001, FileName: "old_file_2.png", FilePath: "uploads/old_2.png",
FileSize: 2048, MimeType: "image/png", Extension: "png", Hash: "hash2",
Type: "attachment", Status: model.UploadStatusPending,
CreatedAt: twoHoursAgo,
},
// 状态为 used 的记录 —— 不应被清理
{
UserID: 1001, FileName: "used_file.jpg", FilePath: "uploads/used.jpg",
FileSize: 512, MimeType: "image/jpeg", Extension: "jpg", Hash: "hash3",
Type: "attachment", Status: model.UploadStatusUsed,
CreatedAt: twoHoursAgo,
},
// 不到1小时的 pending 记录 —— 不应被清理
{
UserID: 1001, FileName: "recent_file.jpg", FilePath: "uploads/recent.jpg",
FileSize: 256, MimeType: "image/jpeg", Extension: "jpg", Hash: "hash4",
Type: "attachment", Status: model.UploadStatusPending,
CreatedAt: now.Add(-10 * time.Minute),
},
}
for _, r := range records {
err := db.DB(ctx).Create(r).Error
require.NoError(t, err)
}
// 准备推送历史测试数据:1个旧的(应删除),1个新的(应保留)
oldPush := &model.PushHistory{
EventKey: "admin_login",
Channel: "email",
Target: "admin@test.com",
Title: "Old Login",
Content: "Old Content",
Level: "INFO",
Status: "success",
CreatedAt: now.AddDate(0, 0, -10),
}
newPush := &model.PushHistory{
EventKey: "admin_login",
Channel: "lark",
Target: "http://webhook.com",
Title: "New Login",
Content: "New Content",
Level: "INFO",
Status: "success",
CreatedAt: now,
}
err = db.DB(ctx).Create(oldPush).Error
require.NoError(t, err)
err = db.DB(ctx).Create(newPush).Error
require.NoError(t, err)
oldTaskLog := &model.TaskExecution{
TaskID: "old_low_frequency_task_log",
TaskType: "low:frequency",
TaskName: "低频任务",
Status: model.TaskExecutionStatusSucceeded,
CreatedAt: now.AddDate(0, 0, -31),
UpdatedAt: now.AddDate(0, 0, -31),
TriggeredBy: "system",
}
err = repository.CreateTaskExecution(ctx, oldTaskLog)
require.NoError(t, err)
// 执行 handler
handler := &SystemCleanupHandler{}
result, err := handler.Execute(ctx, nil)
// 验证结果
require.NoError(t, err)
require.NotNil(t, result)
assert.Contains(t, result.Message, "系统清理完成。成功清理未使用的上传文件 2/2 个;清理历史推送审计日志 1 条;清理任务执行日志 1 条;清理过期访问日志 0 条。")
// 验证数据库状态:pending 且超过1小时的应被标记为 deleted
var pendingCount int64
db.DB(ctx).Model(&model.Upload{}).Where("status = ?", model.UploadStatusPending).Count(&pendingCount)
assert.Equal(t, int64(1), pendingCount, "应只剩1条 pending 记录(最近的文件)")
var deletedCount int64
db.DB(ctx).Model(&model.Upload{}).Where("status = ?", model.UploadStatusDeleted).Count(&deletedCount)
assert.Equal(t, int64(2), deletedCount, "应有2条被标记为 deleted")
var usedCount int64
db.DB(ctx).Model(&model.Upload{}).Where("status = ?", model.UploadStatusUsed).Count(&usedCount)
assert.Equal(t, int64(1), usedCount, "used 状态的文件不应受影响")
// 验证推送历史数据状态:10天前的应被删除,今天的应保留
var pushCount int64
db.DB(ctx).Model(&model.PushHistory{}).Count(&pushCount)
assert.Equal(t, int64(1), pushCount, "应只剩1条推送历史记录")
var remainingPush model.PushHistory
err = db.DB(ctx).First(&remainingPush).Error
require.NoError(t, err)
assert.Equal(t, "New Login", remainingPush.Title)
var taskLogCount int64
err = db.DB(ctx).Model(&model.TaskExecution{}).Where("task_id = ?", "old_low_frequency_task_log").Count(&taskLogCount).Error
require.NoError(t, err)
assert.Equal(t, int64(0), taskLogCount, "过期低频任务日志应被清理")
}
func TestSystemCleanupHandler_ExecuteNoFiles(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
// Mock S3 存储
storageMock := objectstore.MockStorage(
func(ctx context.Context, key string, body io.Reader, size int64, contentType string) error {
return nil
},
func(ctx context.Context, key string) (*objectstore.Object, error) { return nil, nil },
func(ctx context.Context, key string) error { return nil },
)
defer storageMock()
ctx := context.Background()
err := db.DB(ctx).AutoMigrate(&model.PushHistory{})
require.NoError(t, err)
// 没有任何上传记录
handler := &SystemCleanupHandler{}
result, err := handler.Execute(ctx, nil)
require.NoError(t, err)
require.NotNil(t, result)
assert.Contains(t, result.Message, "系统清理完成。成功清理未使用的上传文件 0/0 个;清理历史推送审计日志 0 条;清理任务执行日志 0 条;清理过期访问日志 0 条。")
}
func TestSystemCleanupHandler_ImplementsTaskHandler(t *testing.T) {
// 编译期验证 SystemCleanupHandler 实现了 TaskHandler 接口
var _ task.TaskHandler = (*SystemCleanupHandler)(nil)
}
func TestWarmImageCacheHandlerValidatePayload(t *testing.T) {
tests := []struct {
name string
payload []byte
wantQuality string
wantErr bool
}{
{
name: "normalizes quality",
payload: []byte(`{"quality":" HIGH "}`),
wantQuality: shared.ImageQualityHigh,
},
{
name: "empty payload",
wantErr: true,
},
{
name: "invalid json",
payload: []byte(`{`),
wantErr: true,
},
{
name: "origin is not a compressed quality",
payload: []byte(`{"quality":"origin"}`),
wantErr: true,
},
{
name: "unsupported quality",
payload: []byte(`{"quality":"maximum"}`),
wantErr: true,
},
}
handler := &WarmImageCacheHandler{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotPayload, err := handler.ValidatePayload(tt.payload)
if gotErr := err != nil; gotErr != tt.wantErr {
t.Fatalf("ValidatePayload(%s) error = %v, want error presence = %t", tt.payload, err, tt.wantErr)
}
if tt.wantErr {
return
}
var got WarmImageCachePayload
if err := json.Unmarshal(gotPayload, &got); err != nil {
t.Fatalf("json.Unmarshal(%s) returned error: %v", gotPayload, err)
}
if got.Quality != tt.wantQuality {
t.Errorf("ValidatePayload(%s).Quality = %q, want %q", tt.payload, got.Quality, tt.wantQuality)
}
})
}
}
func TestWarmImageCacheHandlerExecute(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
cache := diskcache.GetGlobalCache()
if err := cache.Clear(); err != nil {
t.Fatalf("Clear() before test returned error: %v", err)
}
t.Cleanup(func() {
if err := cache.Clear(); err != nil {
t.Errorf("Clear() after test returned error: %v", err)
}
})
testDir := t.TempDir()
ctx := context.Background()
active := objectstore.DefaultConfig()
active.Local.Root = testDir
if err := objectstore.SaveActiveConfig(ctx, active); err != nil {
t.Fatalf("SaveActiveConfig() returned error: %v", err)
}
firstPath := filepath.Join(testDir, "first.png")
secondPath := filepath.Join(testDir, "second.jpg")
writeTaskTestPNG(t, firstPath, color.RGBA{R: 255, A: 255})
writeTaskTestPNG(t, secondPath, color.RGBA{G: 255, A: 255})
records := []model.Upload{
{
ID: 4101,
UserID: 1001,
FileName: "first.png",
FilePath: firstPath,
MimeType: "image/png",
Extension: "png",
Status: model.UploadStatusUsed,
},
{
ID: 4102,
UserID: 1001,
FileName: "second.jpg",
FilePath: secondPath,
MimeType: "application/octet-stream",
Extension: "jpg",
Status: model.UploadStatusPending,
},
{
ID: 4103,
UserID: 1001,
FileName: "notes.txt",
FilePath: filepath.Join(testDir, "notes.txt"),
MimeType: "text/plain",
Extension: "txt",
Status: model.UploadStatusUsed,
},
{
ID: 4104,
UserID: 1001,
FileName: "deleted.png",
FilePath: firstPath,
MimeType: "image/png",
Extension: "png",
Status: model.UploadStatusDeleted,
},
}
for i := range records {
if info, err := os.Stat(records[i].FilePath); err == nil {
records[i].FileSize = info.Size()
}
if err := dbConn.Create(&records[i]).Error; err != nil {
t.Fatalf("failed to create upload %d: %v", records[i].ID, err)
}
}
handler := &WarmImageCacheHandler{}
payload := []byte(`{"quality":"low"}`)
result, err := handler.Execute(context.Background(), payload)
if err != nil {
t.Fatalf("Execute(%s) returned error: %v", payload, err)
}
if result == nil {
t.Fatal("Execute() result = nil, want non-nil")
}
if result.Message != "图片缓存预热完成,共处理 2 张,生成 2 张,命中 0 张,失败 0 张" {
t.Errorf("Execute() message = %q, want generated summary", result.Message)
}
for i := range records[:2] {
key := filesrv.ImageCompressionCacheKey(&records[i], shared.ImageQualityLow)
got, err := cache.Get(key)
if err != nil {
t.Errorf("cache.Get(%q) returned error: %v", key, err)
continue
}
if len(got) == 0 {
t.Errorf("cache.Get(%q) returned empty WebP data", key)
}
}
secondResult, err := handler.Execute(context.Background(), payload)
if err != nil {
t.Fatalf("second Execute(%s) returned error: %v", payload, err)
}
if secondResult.Message != "图片缓存预热完成,共处理 2 张,生成 0 张,命中 2 张,失败 0 张" {
t.Errorf("second Execute() message = %q, want cache-hit summary", secondResult.Message)
}
}
func TestWarmImageCacheHandlerImplementsTaskInterfaces(t *testing.T) {
var _ task.TaskHandler = (*WarmImageCacheHandler)(nil)
var _ task.PayloadValidator = (*WarmImageCacheHandler)(nil)
}
func writeTaskTestPNG(t *testing.T, path string, fill color.RGBA) {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, 2, 2))
for y := 0; y < 2; y++ {
for x := 0; x < 2; x++ {
img.Set(x, y, fill)
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("png.Encode() returned error: %v", err)
}
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
t.Fatalf("os.WriteFile(%q) returned error: %v", path, err)
}
}
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package util
import (
"strings"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
)
// IsImageExtension reports whether ext is a common image format.
func IsImageExtension(ext string) bool {
for _, imgExt := range []string{"jpg", "jpeg", "png", "webp", "gif"} {
if ext == imgExt {
return true
}
}
return false
}
// IsArchiveExtension reports whether ext is a common archive format.
func IsArchiveExtension(ext string) bool {
for _, e := range []string{"zip", "rar", "7z", "tar", "gz", "tgz", "bz2", "xz"} {
if ext == e {
return true
}
}
return false
}
// IsDocumentExtension reports whether ext is a common document format.
func IsDocumentExtension(ext string) bool {
for _, e := range []string{"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "txt", "md", "csv", "json", "yaml", "yml", "xml"} {
if ext == e {
return true
}
}
return false
}
// NormalizeImageQuality normalizes the requested image quality query parameter.
func NormalizeImageQuality(quality string) string {
switch strings.ToLower(quality) {
case shared.ImageQualityLow, shared.ImageQualityMedium, shared.ImageQualityHigh:
return strings.ToLower(quality)
default:
return shared.ImageQualityOrigin
}
}
+75
View File
@@ -0,0 +1,75 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package util provides upload media helpers and image utilities.
package util
import (
"bytes"
"errors"
"fmt"
"image"
_ "image/gif" // Register GIF decoder for image.Decode
_ "image/jpeg" // Register JPEG decoder for image.Decode
_ "image/png" // Register PNG decoder for image.Decode
"io"
"strings"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/shared"
"github.com/deepteams/webp"
_ "golang.org/x/image/webp" // Register WebP decoder for image.Decode
)
// ValidateS3Key validates an S3 object key for safety.
func ValidateS3Key(key string) error {
if key == "" {
return errors.New(shared.ErrS3KeyRequired)
}
if len(key) > shared.MaxS3KeyLength {
return fmt.Errorf(shared.ErrS3KeyTooLongFormat, shared.MaxS3KeyLength)
}
if strings.HasPrefix(key, "/") {
return errors.New(shared.ErrS3KeyStartsWithSlash)
}
if strings.Contains(key, "\x00") {
return errors.New(shared.ErrS3KeyContainsNullBytes)
}
return nil
}
// CompressImageToWebP decodes an image from srcReader and encodes it into WebP format
// using the specified quality (low -> 60, medium -> 75, high -> 85).
func CompressImageToWebP(srcReader io.Reader, quality string) ([]byte, error) {
img, format, err := image.Decode(srcReader)
if err != nil {
return nil, fmt.Errorf("failed to decode image (format: %s): %w", format, err)
}
var qualityScore float32
switch strings.ToLower(quality) {
case shared.ImageQualityLow:
qualityScore = 60
case shared.ImageQualityMedium:
qualityScore = 75
case shared.ImageQualityHigh, "":
qualityScore = 85
default:
qualityScore = 85
}
var buf bytes.Buffer
err = webp.Encode(&buf, img, &webp.EncoderOptions{
Quality: qualityScore,
Method: 4,
})
if err != nil {
return nil, fmt.Errorf("failed to encode WebP: %w", err)
}
return buf.Bytes(), nil
}
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package user
const (
errInvalidParams = "无效的请求参数"
errUserNotFound = "用户不存在"
//nolint:gosec // error message, not hardcoded credentials
errPasswordMismatch = "用户名或密码错误"
//nolint:gosec // error message, not hardcoded credentials
errOldPasswordIncorrect = "原密码不正确"
//nolint:gosec // error message, not hardcoded credentials
errTokenNotFound = "访问令牌不存在"
)
+297
View File
@@ -0,0 +1,297 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package user
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"net/http"
"strconv"
"time"
persistence "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin"
)
type loginRequest struct {
Username string `json:"username" binding:"required"`
Password string `json:"password" binding:"required"`
}
type registerRequest struct {
Username string `json:"username" binding:"required"`
Password string `json:"password" binding:"required"`
Email string `json:"email"`
}
type changePasswordRequest struct {
OldPassword string `json:"old_password" binding:"required"`
NewPassword string `json:"new_password" binding:"required"`
}
type updateProfileRequest struct {
Nickname string `json:"nickname"`
AvatarURL string `json:"avatar_url"`
Bio string `json:"bio"`
Phone string `json:"phone"`
Gender string `json:"gender"`
Website string `json:"website"`
Location string `json:"location"`
}
type createAccessTokenRequest struct {
Name string `json:"name" binding:"required"`
ExpiresAt *time.Time `json:"expires_at"`
IsAdmin bool `json:"is_admin"`
}
// Login handles username and password authentication.
func Login(c *gin.Context) {
var req loginRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
user, err := repository.GetUserByUsername(c.Request.Context(), req.Username)
if err != nil {
response.AbortUnauthorized(c, errPasswordMismatch)
return
}
if !user.CheckPassword(req.Password) {
response.AbortUnauthorized(c, errPasswordMismatch)
return
}
sess := sessions.Default(c)
sess.Set(auth.UserIDKey, user.ID)
sess.Set(auth.UserNameKey, user.Username)
_ = sess.Save()
c.JSON(http.StatusOK, response.OK(user))
}
// Register registers a new user.
func Register(c *gin.Context) {
var req registerRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
newUser := &model.User{
Username: req.Username,
Email: req.Email,
IsActive: true,
}
if err := newUser.SetEncryptedPassword(req.Password); err != nil {
response.AbortInternal(c, "密码加密失败")
return
}
gormDB := persistence.DB(c.Request.Context())
if err := gormDB.Create(newUser).Error; err != nil {
response.AbortBadRequest(c, "创建用户失败: "+err.Error())
return
}
c.JSON(http.StatusOK, response.OK(newUser))
}
// Logout logs out the current session.
func Logout(c *gin.Context) {
sess := sessions.Default(c)
sess.Clear()
_ = sess.Save()
c.JSON(http.StatusOK, response.OKNil())
}
// SendEmailCode sends an email verification code.
func SendEmailCode(c *gin.Context) {
c.JSON(http.StatusOK, response.OK(gin.H{"sent": true}))
}
// ChangePassword changes the current user password.
func ChangePassword(c *gin.Context) {
var req changePasswordRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
userID := auth.GetUserIDFromContext(c)
user, err := repository.GetUserByID(c.Request.Context(), userID)
if err != nil {
response.AbortNotFound(c, errUserNotFound)
return
}
if !user.CheckPassword(req.OldPassword) {
response.AbortBadRequest(c, errOldPasswordIncorrect)
return
}
if err := user.SetEncryptedPassword(req.NewPassword); err != nil {
response.AbortInternal(c, "密码更新失败")
return
}
gormDB := persistence.DB(c.Request.Context())
_ = gormDB.Save(&user)
auth.InvalidateCachedUser(c.Request.Context(), user.ID)
c.JSON(http.StatusOK, response.OKNil())
}
// UpdateProfile updates profile info.
func UpdateProfile(c *gin.Context) {
var req updateProfileRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
userID := auth.GetUserIDFromContext(c)
user, err := repository.GetUserByID(c.Request.Context(), userID)
if err != nil {
response.AbortNotFound(c, errUserNotFound)
return
}
user.Nickname = req.Nickname
user.AvatarURL = req.AvatarURL
user.Bio = req.Bio
user.Phone = req.Phone
user.Gender = req.Gender
user.Website = req.Website
user.Location = req.Location
gormDB := persistence.DB(c.Request.Context())
_ = gormDB.Save(&user)
auth.InvalidateCachedUser(c.Request.Context(), user.ID)
c.JSON(http.StatusOK, response.OK(user))
}
// ListAccessTokens lists access tokens for the current user.
func ListAccessTokens(c *gin.Context) {
userID := auth.GetUserIDFromContext(c)
var tokens []model.AccessToken
gormDB := persistence.DB(c.Request.Context())
_ = gormDB.Where("user_id = ?", userID).Find(&tokens).Error
c.JSON(http.StatusOK, response.OK(tokens))
}
const (
tokenEntropyByteLength = 24
tokenMaskMinLength = 8
)
// CreateAccessToken generates a new access token.
func CreateAccessToken(c *gin.Context) {
var req createAccessTokenRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
userID := auth.GetUserIDFromContext(c)
rawBytes := make([]byte, tokenEntropyByteLength)
_, _ = rand.Read(rawBytes)
rawToken := "wvt_" + hex.EncodeToString(rawBytes)
hash := sha256.Sum256([]byte(rawToken))
tokenHash := hex.EncodeToString(hash[:])
masked := rawToken
if len(rawToken) > tokenMaskMinLength {
masked = rawToken[:4] + "..." + rawToken[len(rawToken)-4:]
}
token := model.AccessToken{
UserID: userID,
Name: req.Name,
TokenHash: tokenHash,
MaskedToken: masked,
IsAdmin: req.IsAdmin,
}
gormDB := persistence.DB(c.Request.Context())
if err := gormDB.Create(&token).Error; err != nil {
response.AbortInternal(c, "创建令牌失败")
return
}
c.JSON(http.StatusOK, response.OK(gin.H{
"token": token,
"raw_token": rawToken,
}))
}
// DeleteAccessToken deletes a specific access token.
func DeleteAccessToken(c *gin.Context) {
idStr := c.Param("id")
id, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
userID := auth.GetUserIDFromContext(c)
var token model.AccessToken
gormDB := persistence.DB(c.Request.Context())
if err := gormDB.Where("id = ? AND user_id = ?", id, userID).First(&token).Error; err != nil {
response.AbortNotFound(c, errTokenNotFound)
return
}
_ = gormDB.Delete(&token)
auth.InvalidateCachedToken(c.Request.Context(), token.TokenHash)
c.JSON(http.StatusOK, response.OKNil())
}
// RotateAccessToken rotates an access token value.
func RotateAccessToken(c *gin.Context) {
idStr := c.Param("id")
id, err := strconv.ParseUint(idStr, 10, 64)
if err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
userID := auth.GetUserIDFromContext(c)
var token model.AccessToken
gormDB := persistence.DB(c.Request.Context())
if err := gormDB.Where("id = ? AND user_id = ?", id, userID).First(&token).Error; err != nil {
response.AbortNotFound(c, errTokenNotFound)
return
}
auth.InvalidateCachedToken(c.Request.Context(), token.TokenHash)
rawBytes := make([]byte, tokenEntropyByteLength)
_, _ = rand.Read(rawBytes)
rawToken := "wvt_" + hex.EncodeToString(rawBytes)
hash := sha256.Sum256([]byte(rawToken))
token.TokenHash = hex.EncodeToString(hash[:])
masked := rawToken
if len(rawToken) > tokenMaskMinLength {
masked = rawToken[:4] + "..." + rawToken[len(rawToken)-4:]
}
token.MaskedToken = masked
_ = gormDB.Save(&token)
c.JSON(http.StatusOK, response.OK(gin.H{
"token": token,
"raw_token": rawToken,
}))
}
+15 -13
View File
@@ -1,3 +1,6 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package user provides the user profile, credential management, role management, and access token domain plugin for Cordis.
package user
@@ -8,8 +11,7 @@ import (
"github.com/Rain-kl/Wavelet/core"
"github.com/Rain-kl/Wavelet/core/contracts"
"github.com/Rain-kl/Wavelet/core/extpoints"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/apps/user"
"github.com/Rain-kl/Wavelet/plugins/domain/auth"
"github.com/hibiken/asynq"
)
@@ -71,20 +73,20 @@ func (p *Plugin) Apply(ctx *core.Context) error {
// 3. Register HTTP Routes
userGroup := ctx.Router().Group("/api/v1/user")
{
userGroup.POST("/login", user.Login)
userGroup.POST("/register", user.Register)
userGroup.GET("/logout", user.Logout)
userGroup.POST("/send-email-code", user.SendEmailCode)
userGroup.POST("/change-password", oauth.LoginRequired(), user.ChangePassword)
userGroup.PUT("/profile", oauth.LoginRequired(), user.UpdateProfile)
userGroup.POST("/login", Login)
userGroup.POST("/register", Register)
userGroup.GET("/logout", Logout)
userGroup.POST("/send-email-code", SendEmailCode)
userGroup.POST("/change-password", auth.LoginRequired(), ChangePassword)
userGroup.PUT("/profile", auth.LoginRequired(), UpdateProfile)
// Access Tokens
tokensGroup := userGroup.Group("/access-tokens", oauth.LoginRequired(), oauth.DisallowTokenAuth())
tokensGroup := userGroup.Group("/access-tokens", auth.LoginRequired(), auth.DisallowTokenAuth())
{
tokensGroup.GET("", user.ListAccessTokens)
tokensGroup.POST("", user.CreateAccessToken)
tokensGroup.DELETE("/:id", user.DeleteAccessToken)
tokensGroup.POST("/:id/rotate", user.RotateAccessToken)
tokensGroup.GET("", ListAccessTokens)
tokensGroup.POST("", CreateAccessToken)
tokensGroup.DELETE("/:id", DeleteAccessToken)
tokensGroup.POST("/:id/rotate", RotateAccessToken)
}
}
+1 -1
View File
@@ -8,9 +8,9 @@ import (
"github.com/Rain-kl/Wavelet/core"
"github.com/Rain-kl/Wavelet/core/contracts"
"github.com/Rain-kl/Wavelet/internal/apps/upload/ingest"
"github.com/Rain-kl/Wavelet/internal/infra/objectstore"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/plugins/domain/upload/ingest"
)
// Option configures the storage plugin.