refactor(structure): group platform, infra, and shared packages

Move process wiring, technical adapters, and cross-cutting contracts out of flat internal/ packages so new code has a clear home without changing business layout.
This commit is contained in:
ryan
2026-07-24 15:28:39 +08:00
parent 290427d5fa
commit fbbb75095f
235 changed files with 434 additions and 429 deletions
+254
View File
@@ -0,0 +1,254 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package config 负责应用配置的加载、解析与环境变量覆盖。
package config
import (
"encoding/json"
"flag"
"log"
"os"
"strconv"
"strings"
"github.com/spf13/viper"
)
// 默认队列优先级
const (
webhookQueuePriority = 10
whitelistQueuePriority = 5
defaultQueuePriority = 3
)
// Config 全局配置单例,初始化后不可变
var Config *configModel
// findConfigPath searches upward for the config file to handle tests running in subdirectories.
func findConfigPath(configPath string) string {
if _, err := os.Stat(configPath); err == nil {
return configPath
}
dir := "."
for i := 0; i < 5; i++ {
dir += "/.."
path := dir + "/" + configPath
if _, err := os.Stat(path); err == nil {
return path
}
}
return configPath
}
// isTest checks if the current execution context is within 'go test'.
func isTest() bool {
if flag.Lookup("test.v") != nil {
return true
}
for _, arg := range os.Args {
if strings.HasPrefix(arg, "-test.") || strings.HasSuffix(arg, ".test") {
return true
}
}
return false
}
func init() {
// 加载配置文件路径
configPath := os.Getenv("CONFIG_PATH")
if configPath == "" {
configPath = findConfigPath("config.yaml")
}
// 设置配置文件
viper.SetConfigFile(configPath)
viper.AutomaticEnv()
// 读取配置文件(可选:找不到文件时使用空默认值 + 环境变量)
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
// 文件存在但读取/解析失败
if _, statErr := os.Stat(configPath); statErr == nil { //nolint:gosec // configPath is loaded from CONFIG_PATH environment variable
log.Fatalf("[Config] read config failed: %v\n", err)
}
}
log.Println("[Config] no config file found, using environment variables only")
viper.SetConfigType("yaml")
if err := viper.ReadConfig(strings.NewReader("")); err != nil {
log.Fatalf("[Config] failed to init empty config: %v\n", err)
}
}
// 解析配置到结构体
var c configModel
if err := viper.Unmarshal(&c); err != nil {
log.Fatalf("[Config] parse config failed: %v\n", err)
}
applyDefaults(&c)
// 环境变量覆盖(优先级高于 config.yaml)
applyEnvOverrides(&c)
applyDefaults(&c)
// Disable standard DB/Redis initializations during tests to prevent connection attempts.
if isTest() {
c.Database.Enabled = false
c.Database.SQLitePath = ":memory:"
c.Redis.Enabled = false
c.ClickHouse.Enabled = false
}
// 设置全局配置
Config = &c
// 打印配置
printConfig(&c)
}
func applyDefaults(c *configModel) {
if c.App.SessionAge <= 0 {
c.App.SessionAge = 86400
}
if c.Otel.TracerName == "" {
c.Otel.TracerName = "github.com/Rain-kl/Wavelet"
}
}
// ─── 环境变量覆盖层 ────────────────────────────────────────────────────────────
// 环境变量优先级高于 config.yaml,未设置则保留 yaml 中的值。
func envStr(key, fallback string) string {
if v, ok := os.LookupEnv(key); ok {
return v
}
return fallback
}
func envInt(key string, fallback int) int {
if v, ok := os.LookupEnv(key); ok {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return fallback
}
func envInt64(key string, fallback int64) int64 {
if v, ok := os.LookupEnv(key); ok {
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
return n
}
}
return fallback
}
func envFloat64(key string, fallback float64) float64 {
if v, ok := os.LookupEnv(key); ok {
if n, err := strconv.ParseFloat(v, 64); err == nil {
return n
}
}
return fallback
}
func envBool(key string, fallback bool) bool {
if v, ok := os.LookupEnv(key); ok {
if b, err := strconv.ParseBool(v); err == nil {
return b
}
}
return fallback
}
// applyEnvOverrides 将环境变量值覆盖到配置结构体上(仅当环境变量已设置时生效)
func applyEnvOverrides(c *configModel) {
// ─── App ───
c.App.AppName = envStr("APP_NAME", c.App.AppName)
c.App.Env = envStr("APP_ENV", c.App.Env)
c.App.Addr = envStr("APP_ADDR", c.App.Addr)
c.App.NodeID = envInt64("APP_NODE_ID", c.App.NodeID)
c.App.APIPrefix = envStr("APP_API_PREFIX", c.App.APIPrefix)
c.App.GracefulShutdownTimeout = envInt("APP_GRACEFUL_SHUTDOWN_TIMEOUT", c.App.GracefulShutdownTimeout)
c.App.SessionCookieName = envStr("APP_SESSION_COOKIE_NAME", c.App.SessionCookieName)
c.App.SessionSecret = envStr("APP_SESSION_SECRET", c.App.SessionSecret)
c.App.SessionDomain = envStr("APP_SESSION_DOMAIN", c.App.SessionDomain)
c.App.SessionAge = envInt("APP_SESSION_AGE", c.App.SessionAge)
c.App.SessionHTTPOnly = envBool("APP_SESSION_HTTP_ONLY", c.App.SessionHTTPOnly)
c.App.SessionSecure = envBool("APP_SESSION_SECURE", c.App.SessionSecure)
// ─── Database ───
c.Database.Host = envStr("DB_HOST", c.Database.Host)
c.Database.Port = envInt("DB_PORT", c.Database.Port)
c.Database.Username = envStr("DB_USERNAME", c.Database.Username)
c.Database.Password = envStr("DB_PASSWORD", c.Database.Password)
c.Database.Database = envStr("DB_NAME", c.Database.Database)
c.Database.SSLMode = envStr("DB_SSL_MODE", c.Database.SSLMode)
c.Database.TimeZone = envStr("DB_TIMEZONE", c.Database.TimeZone)
c.Database.LogLevel = envStr("DB_LOG_LEVEL", c.Database.LogLevel)
c.Database.MaxIdleConn = envInt("DB_MAX_IDLE_CONN", c.Database.MaxIdleConn)
c.Database.MaxOpenConn = envInt("DB_MAX_OPEN_CONN", c.Database.MaxOpenConn)
// 当 DB_HOST 环境变量已设置时自动启用数据库
if _, ok := os.LookupEnv("DB_HOST"); ok {
c.Database.Enabled = true
}
c.Database.Enabled = envBool("DB_ENABLED", c.Database.Enabled)
c.Database.SQLitePath = envStr("SQLITE_PATH", c.Database.SQLitePath)
// ─── Redis ───
if v, ok := os.LookupEnv("REDIS_ADDR"); ok {
c.Redis.Addrs = []string{v}
c.Redis.Enabled = true // 当 REDIS_ADDR 已设置时自动启用
}
c.Redis.Enabled = envBool("REDIS_ENABLED", c.Redis.Enabled)
c.Redis.Username = envStr("REDIS_USERNAME", c.Redis.Username)
c.Redis.Password = envStr("REDIS_PASSWORD", c.Redis.Password)
c.Redis.DB = envInt("REDIS_DB", c.Redis.DB)
c.Redis.KeyPrefix = envStr("REDIS_KEY_PREFIX", c.Redis.KeyPrefix)
c.Redis.PoolSize = envInt("REDIS_POOL_SIZE", c.Redis.PoolSize)
c.Redis.MaintNotifications = envBool("REDIS_MAINT_NOTIFICATIONS", c.Redis.MaintNotifications)
// ─── ClickHouse ───
if v, ok := os.LookupEnv("CLICKHOUSE_HOST"); ok {
c.ClickHouse.Hosts = []string{v}
c.ClickHouse.Enabled = true
}
c.ClickHouse.Enabled = envBool("CLICKHOUSE_ENABLED", c.ClickHouse.Enabled)
c.ClickHouse.Username = envStr("CLICKHOUSE_USERNAME", c.ClickHouse.Username)
c.ClickHouse.Password = envStr("CLICKHOUSE_PASSWORD", c.ClickHouse.Password)
c.ClickHouse.Database = envStr("CLICKHOUSE_NAME", c.ClickHouse.Database)
// ─── Log ───
c.Log.Level = envStr("LOG_LEVEL", c.Log.Level)
c.Log.Format = envStr("LOG_FORMAT", c.Log.Format)
c.Log.Output = envStr("LOG_OUTPUT", c.Log.Output)
// ─── OTel ───
c.Otel.SamplingRate = envFloat64("OTEL_SAMPLING_RATE", c.Otel.SamplingRate)
c.Otel.TracerName = envStr("OTEL_TRACER_NAME", c.Otel.TracerName)
// ─── Worker ───
c.Worker.Concurrency = envInt("WORKER_CONCURRENCY", c.Worker.Concurrency)
c.Worker.StrictPriority = envBool("WORKER_STRICT_PRIORITY", c.Worker.StrictPriority)
// 无 yaml 且无环境变量时,使用代码级默认队列
if len(c.Worker.Queues) == 0 {
c.Worker.Queues = []QueueConfig{
{Name: "webhook", Priority: webhookQueuePriority},
{Name: "whitelist_only", Priority: whitelistQueuePriority},
{Name: "default", Priority: defaultQueuePriority},
}
}
}
// printConfig 打印配置内容
func printConfig(c *configModel) {
configJSON, err := json.MarshalIndent(c, "", " ")
if err != nil {
log.Printf("[Config] failed to marshal config: %v\n", err)
return
}
log.Printf("[Config] loaded configuration:\n%s\n", string(configJSON))
}
+14
View File
@@ -0,0 +1,14 @@
package config
import "testing"
func TestApplyEnvOverridesRedisMaintNotifications(t *testing.T) {
t.Setenv("REDIS_MAINT_NOTIFICATIONS", "true")
cfg := &configModel{}
applyEnvOverrides(cfg)
if !cfg.Redis.MaintNotifications {
t.Fatal("REDIS_MAINT_NOTIFICATIONS=true was not applied")
}
}
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package config
import "time"
type configModel struct {
App appConfig `mapstructure:"app"`
Database databaseConfig `mapstructure:"database"`
Redis redisConfig `mapstructure:"redis"`
Log logConfig `mapstructure:"log"`
Scheduler schedulerConfig `mapstructure:"scheduler"`
Worker workerConfig `mapstructure:"worker"`
ClickHouse clickHouseConfig `mapstructure:"clickhouse"`
Otel otelConfig `mapstructure:"otel"`
}
// appConfig 应用基本配置
type appConfig struct {
AppName string `mapstructure:"app_name"`
Env string `mapstructure:"env"`
Addr string `mapstructure:"addr"`
NodeID int64 `mapstructure:"node_id"`
APIPrefix string `mapstructure:"api_prefix"`
GracefulShutdownTimeout int `mapstructure:"graceful_shutdown_timeout"`
SessionCookieName string `mapstructure:"session_cookie_name"`
SessionSecret string `mapstructure:"session_secret"`
SessionDomain string `mapstructure:"session_domain"`
SessionAge int `mapstructure:"session_age"`
SessionHTTPOnly bool `mapstructure:"session_http_only"`
SessionSecure bool `mapstructure:"session_secure"`
}
// IsProduction 检查当前环境是否为生产环境
func (a *appConfig) IsProduction() bool {
return a.Env == "production"
}
// databaseConfig 数据库配置
type databaseConfig struct {
Enabled bool `mapstructure:"enabled"`
SQLitePath string `mapstructure:"sqlite_path"` // PostgreSQL 禁用时的 SQLite 文件路径
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Database string `mapstructure:"database"`
MaxIdleConn int `mapstructure:"max_idle_conn"`
MaxOpenConn int `mapstructure:"max_open_conn"`
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
LogLevel string `mapstructure:"log_level"`
SSLMode string `mapstructure:"ssl_mode"`
TimeZone string `mapstructure:"time_zone"`
ApplicationName string `mapstructure:"application_name"`
SearchPath string `mapstructure:"search_path"`
PreferSimpleProtocol bool `mapstructure:"prefer_simple_protocol"`
StatementCacheCapacity int `mapstructure:"statement_cache_capacity"`
DefaultQueryExecMode string `mapstructure:"default_query_exec_mode"`
Replicas []databaseReplicaConfig `mapstructure:"replicas"`
SlowThreshold time.Duration `mapstructure:"slow_threshold"`
}
// databaseReplicaConfig 只读副本配置
type databaseReplicaConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
}
// clickhouse 配置
type clickHouseConfig struct {
Enabled bool `mapstructure:"enabled"`
Hosts []string `mapstructure:"hosts"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Database string `mapstructure:"database"`
MaxIdleConn int `mapstructure:"max_idle_conn"`
MaxOpenConn int `mapstructure:"max_open_conn"`
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
DialTimeout int `mapstructure:"dial_timeout"`
BlockBufferSize uint8 `mapstructure:"block_buffer_size"`
}
// redisConfig Redis配置
type redisConfig struct {
Enabled bool `mapstructure:"enabled"`
Addrs []string `mapstructure:"addrs"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
DB int `mapstructure:"db"`
ClusterMode bool `mapstructure:"cluster_mode"`
MasterName string `mapstructure:"master_name"`
KeyPrefix string `mapstructure:"key_prefix"`
PoolSize int `mapstructure:"pool_size"`
MinIdleConn int `mapstructure:"min_idle_conn"`
DialTimeout int `mapstructure:"dial_timeout"`
ReadTimeout int `mapstructure:"read_timeout"`
WriteTimeout int `mapstructure:"write_timeout"`
MaxRetries int `mapstructure:"max_retries"`
PoolTimeout int `mapstructure:"pool_timeout"`
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
MaintNotifications bool `mapstructure:"maint_notifications"`
}
// logConfig 日志配置
type logConfig struct {
Level string `mapstructure:"level"`
Format string `mapstructure:"format"`
Output string `mapstructure:"output"`
FilePath string `mapstructure:"file_path"`
MaxSize int `mapstructure:"max_size"`
MaxAge int `mapstructure:"max_age"`
MaxBackups int `mapstructure:"max_backups"`
Compress bool `mapstructure:"compress"`
}
// schedulerConfig 定时任务配置
type schedulerConfig struct {
}
// workerConfig 工作配置
type workerConfig struct {
Concurrency int `mapstructure:"concurrency"`
StrictPriority bool `mapstructure:"strict_priority"`
Queues []QueueConfig `mapstructure:"queues"`
}
// QueueConfig 队列配置
type QueueConfig struct {
Name string `mapstructure:"name"`
Priority int `mapstructure:"priority"`
}
// otelConfig OpenTelemetry 配置
type otelConfig struct {
SamplingRate float64 `mapstructure:"sampling_rate"`
TracerName string `mapstructure:"tracer_name"`
}
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package diskcache wraps the generic pkg/cache/disk to provide database configuration integration.
package diskcache
import (
"context"
"strconv"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
pkgcache "github.com/Rain-kl/Wavelet/pkg/cache/disk"
)
// Status represents the runtime cache statistics.
type Status = pkgcache.Status
const (
defaultCacheDir = "uploads/diskcache"
defaultMaxSizeMB = 100
defaultTTLMinutes = 60
defaultCleanupInterval = 10
// DefaultExpiration applies the cache-wide default TTL.
DefaultExpiration = pkgcache.DefaultExpiration
// NoExpiration stores the item without a TTL. Size limits and LRU eviction still apply.
NoExpiration = pkgcache.NoExpiration
)
// ErrCacheMiss represents a cache miss.
var ErrCacheMiss = pkgcache.ErrCacheMiss
// DiskCache is a wrapper around the generic pkg/diskcache that integrates with the DB for configs.
type DiskCache struct {
*pkgcache.Cache
}
var (
globalCache *DiskCache
globalCacheOnce sync.Once
)
// GetGlobalCache returns the global singleton DiskCache instance.
func GetGlobalCache() *DiskCache {
globalCacheOnce.Do(func() {
pureCache := pkgcache.New(defaultCacheDir)
globalCache = &DiskCache{pureCache}
// Load initial configs from database
globalCache.ReloadConfig(context.Background())
// Start background routine to clean expired items every 10 minutes
go globalCache.StartCleanupWorker(defaultCleanupInterval * time.Minute)
})
return globalCache
}
// New creates a new DiskCache wrapper.
func New(basePath string) *DiskCache {
return &DiskCache{pkgcache.New(basePath)}
}
// ReloadConfig reloads policies from database configs dynamically.
func (c *DiskCache) ReloadConfig(ctx context.Context) {
// Ensure DB is initialized before querying
if db.DB(ctx) == nil {
return
}
// 1. Max Size
maxSizeMB := int64(defaultMaxSizeMB)
if scMaxSize, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyDiskCacheMaxSizeMB); err == nil && scMaxSize.Value != "" {
if val, err := strconv.ParseInt(scMaxSize.Value, 10, 64); err == nil && val > 0 {
maxSizeMB = val
}
}
// 2. Default TTL
ttlMinutes := int64(defaultTTLMinutes)
if scTTL, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyDiskCacheTTLMinutes); err == nil && scTTL.Value != "" {
if val, err := strconv.ParseInt(scTTL.Value, 10, 64); err == nil && val >= 0 {
ttlMinutes = val
}
}
// 3. LRU Enabled
lruEnabled := true
if scLRU, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyDiskCacheLRUEnabled); err == nil && scLRU.Value != "" {
if val, err := strconv.ParseBool(scLRU.Value); err == nil {
lruEnabled = val
}
}
c.UpdatePolicy(maxSizeMB, ttlMinutes, lruEnabled)
}
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package diskcache
import (
"context"
"os"
"testing"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
)
func TestDiskCacheReloadConfig(t *testing.T) {
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
testDir := "uploads/test_diskcache_reload"
defer func() { _ = os.RemoveAll(testDir) }()
_ = os.RemoveAll(testDir)
c := New(testDir)
defer func() { _ = c.Clear() }()
// Update DB config values
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheMaxSizeMB).Update("value", "250")
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheTTLMinutes).Update("value", "120")
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheLRUEnabled).Update("value", "false")
// Invalidate Redis config cache to force DB reload
if db.Redis != nil {
db.Redis.Del(context.Background(), db.PrefixedKey(repository.SystemConfigRedisHashKey))
}
// Reload config
c.ReloadConfig(context.Background())
status := c.Status()
if status.MaxSizeMB != 250 {
t.Errorf("expected MaxSizeMB to be 250, got %d", status.MaxSizeMB)
}
if status.TTLMinutes != 120 {
t.Errorf("expected TTLMinutes to be 120, got %d", status.TTLMinutes)
}
if status.LRUEnabled != false {
t.Errorf("expected LRUEnabled to be false, got %t", status.LRUEnabled)
}
}
+253
View File
@@ -0,0 +1,253 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package objectstore provides dynamically configured file storage backends.
package objectstore
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"gorm.io/gorm"
)
// Driver identifies a supported storage backend.
type Driver string
const (
// DriverLocal stores files on the local filesystem.
DriverLocal Driver = "local"
// DriverS3 stores files in an S3-compatible object store.
DriverS3 Driver = "s3"
// DriverR2 stores files in Cloudflare R2.
DriverR2 Driver = "r2"
// DriverMinIO stores files in MinIO.
DriverMinIO Driver = "minio"
// DriverOSS stores files in Aliyun OSS.
DriverOSS Driver = "oss"
// DriverWebDAV stores files through WebDAV.
DriverWebDAV Driver = "webdav"
// ConfigMask replaces secrets returned to the frontend.
ConfigMask = "******"
)
// LocalConfig configures local filesystem storage.
type LocalConfig struct {
Root string `json:"root"`
}
// ObjectConfig configures S3-compatible or OSS object storage.
type ObjectConfig struct {
Endpoint string `json:"endpoint"`
Region string `json:"region"`
Bucket string `json:"bucket"`
AccessKeyID string `json:"access_key_id"`
SecretAccessKey string `json:"secret_access_key"`
AccountID string `json:"account_id,omitempty"`
PathStyle bool `json:"path_style"`
KeyPrefix string `json:"key_prefix"`
CDNURL string `json:"cdn_url"`
}
// WebDAVConfig configures WebDAV storage.
type WebDAVConfig struct {
Endpoint string `json:"endpoint"`
Username string `json:"username"`
Password string `json:"password"`
BasePath string `json:"base_path"`
}
// Config contains all storage backends and the currently active driver.
type Config struct {
Driver Driver `json:"driver"`
Local LocalConfig `json:"local"`
S3 ObjectConfig `json:"s3"`
R2 ObjectConfig `json:"r2"`
MinIO ObjectConfig `json:"minio"`
OSS ObjectConfig `json:"oss"`
WebDAV WebDAVConfig `json:"webdav"`
}
// DefaultConfig returns the local-storage default configuration.
func DefaultConfig() Config {
return Config{
Driver: DriverLocal,
Local: LocalConfig{Root: "."},
S3: ObjectConfig{Region: "us-east-1"},
R2: ObjectConfig{Region: "auto"},
MinIO: ObjectConfig{Region: "us-east-1", PathStyle: true},
}
}
// LoadConfig loads the active storage configuration.
func LoadConfig(ctx context.Context) (Config, error) {
pubSubOnce.Do(startPubSubListener)
cacheMutex.RLock()
isCacheValid := time.Since(lastChecked) < 5*time.Second && activeConfigJSON != ""
configJSON := activeConfigJSON
cacheMutex.RUnlock()
if isCacheValid {
cfg := DefaultConfig()
if strings.TrimSpace(configJSON) != "" {
if err := json.Unmarshal([]byte(configJSON), &cfg); err != nil {
return Config{}, fmt.Errorf("parse storage config from cache: %w", err)
}
}
return cfg, nil
}
return loadConfigByKey(ctx, model.ConfigKeyStorageConfig, DefaultConfig())
}
func loadConfigByKey(ctx context.Context, key string, fallback Config) (Config, error) {
sc, err := repository.GetSystemConfigByKey(ctx, key)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return fallback, nil
}
return Config{}, err
}
if strings.TrimSpace(sc.Value) == "" {
return fallback, nil
}
if err := json.Unmarshal([]byte(sc.Value), &fallback); err != nil {
return Config{}, fmt.Errorf("parse %s: %w", key, err)
}
return fallback, nil
}
// ValidateConfig validates the selected backend configuration.
func ValidateConfig(cfg Config) error {
switch cfg.Driver {
case DriverLocal:
if strings.TrimSpace(cfg.Local.Root) == "" {
return errors.New("local root is required")
}
case DriverS3:
return validateObjectConfig(cfg.S3, false)
case DriverR2:
if strings.TrimSpace(cfg.R2.AccountID) == "" && strings.TrimSpace(cfg.R2.Endpoint) == "" {
return errors.New("R2 account ID or endpoint is required")
}
return validateObjectConfig(cfg.R2, false)
case DriverMinIO:
if strings.TrimSpace(cfg.MinIO.Endpoint) == "" {
return errors.New("MinIO endpoint is required")
}
return validateObjectConfig(cfg.MinIO, true)
case DriverOSS:
if strings.TrimSpace(cfg.OSS.Endpoint) == "" {
return errors.New("OSS endpoint is required")
}
return validateObjectConfig(cfg.OSS, true)
case DriverWebDAV:
if strings.TrimSpace(cfg.WebDAV.Endpoint) == "" {
return errors.New("WebDAV endpoint is required")
}
default:
return fmt.Errorf("unsupported storage driver %q", cfg.Driver)
}
return nil
}
func validateObjectConfig(cfg ObjectConfig, endpointRequired bool) error {
if endpointRequired && strings.TrimSpace(cfg.Endpoint) == "" {
return errors.New("endpoint is required")
}
if strings.TrimSpace(cfg.Region) == "" {
return errors.New("region is required")
}
if strings.TrimSpace(cfg.Bucket) == "" {
return errors.New("bucket is required")
}
if strings.TrimSpace(cfg.AccessKeyID) == "" || strings.TrimSpace(cfg.SecretAccessKey) == "" {
return errors.New("access key ID and secret access key are required")
}
return nil
}
// SaveActiveConfig persists the active storage configuration.
func SaveActiveConfig(ctx context.Context, cfg Config) error {
return saveSystemConfig(ctx, model.ConfigKeyStorageConfig, cfg, "文件存储驱动及连接配置(JSON)")
}
func saveSystemConfig(ctx context.Context, key string, value any, description string) error {
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
return upsertSystemConfig(ctx, tx, key, value, description)
})
if err == nil && key == model.ConfigKeyStorageConfig {
ResetCache()
PublishCacheInvalidation(ctx)
}
return err
}
func upsertSystemConfig(ctx context.Context, tx *gorm.DB, key string, value any, description string) error {
data, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("marshal %s: %w", key, err)
}
sc := model.SystemConfig{
Key: key,
Value: string(data),
Type: "system",
Visibility: model.ConfigVisibilityHidden,
Description: description,
}
if err := tx.Where("key = ?", key).
Assign(map[string]any{"value": sc.Value, "description": description, "visibility": model.ConfigVisibilityHidden}).
FirstOrCreate(&sc).Error; err != nil {
return err
}
return repository.InvalidateSystemConfigCache(ctx, key)
}
// MergeMaskedSecrets restores unchanged secrets from the current configuration.
func MergeMaskedSecrets(next, current Config) Config {
mergeObjectSecret := func(dst *ObjectConfig, src ObjectConfig) {
if dst.AccessKeyID == ConfigMask {
dst.AccessKeyID = src.AccessKeyID
}
if dst.SecretAccessKey == ConfigMask {
dst.SecretAccessKey = src.SecretAccessKey
}
}
mergeObjectSecret(&next.S3, current.S3)
mergeObjectSecret(&next.R2, current.R2)
mergeObjectSecret(&next.MinIO, current.MinIO)
mergeObjectSecret(&next.OSS, current.OSS)
if next.WebDAV.Password == ConfigMask {
next.WebDAV.Password = current.WebDAV.Password
}
return next
}
// MaskSecrets replaces stored credentials with placeholders for API responses.
func MaskSecrets(cfg Config) Config {
maskObject := func(value *ObjectConfig) {
if value.AccessKeyID != "" {
value.AccessKeyID = ConfigMask
}
if value.SecretAccessKey != "" {
value.SecretAccessKey = ConfigMask
}
}
maskObject(&cfg.S3)
maskObject(&cfg.R2)
maskObject(&cfg.MinIO)
maskObject(&cfg.OSS)
if cfg.WebDAV.Password != "" {
cfg.WebDAV.Password = ConfigMask
}
return cfg
}
+44
View File
@@ -0,0 +1,44 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"context"
"fmt"
"net/http"
"net/url"
"time"
"github.com/Rain-kl/Wavelet/pkg/httppool"
)
func getHTTPObject(ctx context.Context, baseURL, key string) (*Object, error) {
objectURL, err := url.JoinPath(baseURL, key)
if err != nil {
return nil, fmt.Errorf("build CDN object URL: %w", err)
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, objectURL, nil)
if err != nil {
return nil, fmt.Errorf("create CDN request: %w", err)
}
const cdnRequestTimeout = 30 * time.Second
client := httppool.NewClient(cdnRequestTimeout)
response, err := client.Do(request)
if err != nil {
return nil, fmt.Errorf("get CDN object: %w", err)
}
if response.StatusCode != http.StatusOK {
_ = response.Body.Close()
return nil, fmt.Errorf("get CDN object: unexpected status %d", response.StatusCode)
}
contentType := response.Header.Get("Content-Type")
if contentType == "" {
contentType = defaultContentType
}
return &Object{
Body: response.Body,
ContentLength: response.ContentLength,
ContentType: contentType,
}, nil
}
+120
View File
@@ -0,0 +1,120 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"context"
"fmt"
"io"
"mime"
"os"
"path/filepath"
"strings"
)
type localBackend struct {
root string
}
func newLocalBackend(cfg LocalConfig) (*localBackend, error) {
root := filepath.Clean(cfg.Root)
if root == "" {
return nil, fmt.Errorf("local root is required")
}
return &localBackend{root: root}, nil
}
func (b *localBackend) Put(_ context.Context, key string, body io.Reader, _ int64, _ string) (PutResult, error) {
path, err := b.path(key)
if err != nil {
return PutResult{}, err
}
if err := os.MkdirAll(filepath.Dir(path), storageDirPerm); err != nil {
return PutResult{}, err
}
file, err := os.OpenFile( //nolint:gosec // path is constrained to the configured storage root.
path,
os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
storageFilePerm,
)
if err != nil {
return PutResult{}, err
}
if _, err := io.Copy(file, body); err != nil {
_ = file.Close()
_ = os.Remove(path)
return PutResult{}, err
}
if err := file.Close(); err != nil {
_ = os.Remove(path)
return PutResult{}, err
}
return PutResult{Key: filepath.ToSlash(key)}, nil
}
func (b *localBackend) Get(_ context.Context, key string) (*Object, error) {
path, err := b.path(key)
if err != nil {
return nil, err
}
file, err := os.Open(path) //nolint:gosec // path is constrained to the configured storage root.
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
_ = file.Close()
return nil, err
}
contentType := mime.TypeByExtension(filepath.Ext(path))
if contentType == "" {
contentType = defaultContentType
}
return &Object{Body: file, ContentLength: info.Size(), ContentType: contentType}, nil
}
func (b *localBackend) Delete(_ context.Context, key string) error {
path, err := b.path(key)
if err != nil {
return err
}
err = os.Remove(path)
if os.IsNotExist(err) {
return nil
}
return err
}
func (b *localBackend) Test(_ context.Context) error {
return os.MkdirAll(b.root, storageDirPerm)
}
func (b *localBackend) path(key string) (string, error) {
if filepath.IsAbs(key) {
cleanPath := filepath.Clean(key)
absRoot, err := filepath.Abs(b.root)
if err != nil {
return "", err
}
absPath, err := filepath.Abs(cleanPath)
if err != nil {
return "", err
}
rel, err := filepath.Rel(absRoot, absPath)
if err != nil || strings.HasPrefix(rel, "..") {
return "", fmt.Errorf("storage key escapes local root")
}
return cleanPath, nil
}
cleanKey := filepath.Clean(filepath.FromSlash(strings.TrimPrefix(key, "/")))
if cleanKey == "." || cleanKey == "" || strings.HasPrefix(cleanKey, "..") {
return "", fmt.Errorf("invalid local storage key %q", key)
}
path := filepath.Join(b.root, cleanKey)
rel, err := filepath.Rel(b.root, path)
if err != nil || strings.HasPrefix(rel, "..") {
return "", fmt.Errorf("storage key escapes local root")
}
return path, nil
}
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"bytes"
"context"
"io"
"testing"
)
func TestLocalBackendRoundTrip(t *testing.T) {
backend, err := newLocalBackend(LocalConfig{Root: t.TempDir()})
if err != nil {
t.Fatalf("newLocalBackend() returned error: %v", err)
}
ctx := context.Background()
const key = "uploads/2026/06/13/test.txt"
const content = "wavelet storage"
storedResult, err := backend.Put(ctx, key, bytes.NewBufferString(content), int64(len(content)), "text/plain")
if err != nil {
t.Fatalf("Put(%q) returned error: %v", key, err)
}
if storedResult.Key != key {
t.Errorf("Put(%q) key = %q, want %q", key, storedResult.Key, key)
}
object, err := backend.Get(ctx, key)
if err != nil {
t.Fatalf("Get(%q) returned error: %v", key, err)
}
got, err := io.ReadAll(object.Body)
if err != nil {
t.Fatalf("ReadAll(Get(%q)) returned error: %v", key, err)
}
if err := object.Body.Close(); err != nil {
t.Fatalf("Close(Get(%q)) returned error: %v", key, err)
}
if string(got) != content {
t.Errorf("Get(%q) content = %q, want %q", key, got, content)
}
if err := backend.Delete(ctx, key); err != nil {
t.Fatalf("Delete(%q) returned error: %v", key, err)
}
if _, err := backend.Get(ctx, key); err == nil {
t.Errorf("Get(%q) after Delete() returned nil error", key)
}
}
+98
View File
@@ -0,0 +1,98 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"context"
"fmt"
"io"
"strings"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)
type ossBackend struct {
client *oss.Client
bucket string
keyPrefix string
cdnURL string
}
func newOSSBackend(cfg ObjectConfig) (*ossBackend, error) {
options := oss.LoadDefaultConfig().
WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey)).
WithRegion(cfg.Region)
if cfg.Endpoint != "" {
options.WithEndpoint(cfg.Endpoint)
}
return &ossBackend{
client: oss.NewClient(options),
bucket: cfg.Bucket,
keyPrefix: strings.Trim(cfg.KeyPrefix, "/"),
cdnURL: strings.TrimRight(cfg.CDNURL, "/"),
}, nil
}
func (b *ossBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) (PutResult, error) {
key = b.key(key)
_, err := b.client.PutObject(ctx, &oss.PutObjectRequest{
Bucket: oss.Ptr(b.bucket),
Key: oss.Ptr(key),
Body: body,
})
if err != nil {
return PutResult{}, fmt.Errorf("put OSS object: %w", err)
}
return PutResult{Key: key, Bucket: b.bucket}, nil
}
func (b *ossBackend) Get(ctx context.Context, key string) (*Object, error) {
key = b.key(key)
if b.cdnURL != "" {
return getHTTPObject(ctx, b.cdnURL, key)
}
output, err := b.client.GetObject(ctx, &oss.GetObjectRequest{
Bucket: oss.Ptr(b.bucket),
Key: oss.Ptr(key),
})
if err != nil {
return nil, fmt.Errorf("get OSS object: %w", err)
}
contentType := defaultContentType
if output.ContentType != nil {
contentType = *output.ContentType
}
return &Object{Body: output.Body, ContentLength: output.ContentLength, ContentType: contentType}, nil
}
func (b *ossBackend) Delete(ctx context.Context, key string) error {
_, err := b.client.DeleteObject(ctx, &oss.DeleteObjectRequest{
Bucket: oss.Ptr(b.bucket),
Key: oss.Ptr(b.key(key)),
})
if err != nil {
return fmt.Errorf("delete OSS object: %w", err)
}
return nil
}
func (b *ossBackend) Test(ctx context.Context) error {
ok, err := b.client.IsBucketExist(ctx, b.bucket)
if err != nil {
return fmt.Errorf("access OSS bucket: %w", err)
}
if !ok {
return fmt.Errorf("OSS bucket %q does not exist", b.bucket)
}
return nil
}
func (b *ossBackend) key(key string) string {
key = strings.TrimLeft(key, "/")
if b.keyPrefix == "" || strings.HasPrefix(key, b.keyPrefix+"/") {
return key
}
return b.keyPrefix + "/" + key
}
+122
View File
@@ -0,0 +1,122 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"context"
"fmt"
"io"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type s3Backend struct {
client *s3.Client
bucket string
keyPrefix string
cdnURL string
}
func newS3Backend(ctx context.Context, cfg ObjectConfig) (*s3Backend, error) {
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(cfg.Region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
cfg.AccessKeyID,
cfg.SecretAccessKey,
"",
)),
)
if err != nil {
return nil, fmt.Errorf("load S3 config: %w", err)
}
client := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
if cfg.Endpoint != "" {
options.BaseEndpoint = aws.String(strings.TrimRight(cfg.Endpoint, "/"))
}
options.UsePathStyle = cfg.PathStyle
})
return &s3Backend{
client: client,
bucket: cfg.Bucket,
keyPrefix: strings.Trim(cfg.KeyPrefix, "/"),
cdnURL: strings.TrimRight(cfg.CDNURL, "/"),
}, nil
}
func newR2Backend(ctx context.Context, cfg ObjectConfig) (*s3Backend, error) {
if cfg.Endpoint == "" {
cfg.Endpoint = fmt.Sprintf("https://%s.r2.cloudflarestorage.com", cfg.AccountID)
}
cfg.Region = "auto"
return newS3Backend(ctx, cfg)
}
func (b *s3Backend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error) {
key = b.key(key)
_, err := b.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
Body: body,
ContentLength: aws.Int64(size),
ContentType: aws.String(contentType),
})
if err != nil {
return PutResult{}, fmt.Errorf("put S3 object: %w", err)
}
return PutResult{Key: key, Bucket: b.bucket}, nil
}
func (b *s3Backend) Get(ctx context.Context, key string) (*Object, error) {
key = b.key(key)
if b.cdnURL != "" {
return getHTTPObject(ctx, b.cdnURL, key)
}
output, err := b.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
})
if err != nil {
return nil, fmt.Errorf("get S3 object: %w", err)
}
contentType := defaultContentType
if output.ContentType != nil {
contentType = *output.ContentType
}
var size int64
if output.ContentLength != nil {
size = *output.ContentLength
}
return &Object{Body: output.Body, ContentLength: size, ContentType: contentType}, nil
}
func (b *s3Backend) Delete(ctx context.Context, key string) error {
_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(b.key(key)),
})
if err != nil {
return fmt.Errorf("delete S3 object: %w", err)
}
return nil
}
func (b *s3Backend) Test(ctx context.Context) error {
_, err := b.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(b.bucket)})
if err != nil {
return fmt.Errorf("access S3 bucket: %w", err)
}
return nil
}
func (b *s3Backend) key(key string) string {
key = strings.TrimLeft(key, "/")
if b.keyPrefix == "" || strings.HasPrefix(key, b.keyPrefix+"/") {
return key
}
return b.keyPrefix + "/" + key
}
+217
View File
@@ -0,0 +1,217 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"gorm.io/gorm"
)
const (
defaultContentType = "application/octet-stream"
storageDirPerm = 0o750
storageFilePerm = 0o600
)
// Object describes a readable stored object.
type Object struct {
CachePath string
Body io.ReadCloser
ContentLength int64
ContentType string
}
// PutResult describes the result of a successful Put operation.
type PutResult struct {
Key string
Bucket string
}
// Backend defines storage operations used by the upload domain.
type Backend interface {
Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error)
Get(ctx context.Context, key string) (*Object, error)
Delete(ctx context.Context, key string) error
Test(ctx context.Context) error
}
var (
// IsEnabledFunc preserves the legacy S3 test hook while tests migrate to backend injection.
IsEnabledFunc = func() bool { return false }
mockBackend Backend
activeBackend Backend
activeDriver Driver
activeConfigJSON string
lastChecked time.Time
cacheMutex sync.RWMutex
)
// ConfigInvalidationChannel is the Redis pub/sub channel used to evict storage caches cluster-wide.
const ConfigInvalidationChannel = "storage:config_invalidation"
var pubSubOnce sync.Once
// ResetCache clears the local cache for storage configuration and client singletons.
func ResetCache() {
cacheMutex.Lock()
defer cacheMutex.Unlock()
activeBackend = nil
activeDriver = ""
activeConfigJSON = ""
lastChecked = time.Time{}
}
// PublishCacheInvalidation broadcasts cache eviction to all nodes in the cluster via Redis.
func PublishCacheInvalidation(ctx context.Context) {
if db.Redis != nil {
_ = db.Redis.Publish(ctx, ConfigInvalidationChannel, "reset").Err()
}
}
// startPubSubListener starts the background subscriber for cache invalidations.
func startPubSubListener() {
if db.Redis == nil {
return
}
go func() {
pubsub := db.Redis.Subscribe(context.Background(), ConfigInvalidationChannel)
defer func() {
_ = pubsub.Close()
}()
ch := pubsub.Channel()
for range ch {
ResetCache()
}
}()
}
// Active returns the configured active driver and backend, using an in-memory cache with 5s TTL.
func Active(ctx context.Context) (Driver, Backend, error) {
if IsEnabledFunc() && mockBackend != nil {
return DriverS3, mockBackend, nil
}
pubSubOnce.Do(startPubSubListener)
cacheMutex.RLock()
isCacheValid := time.Since(lastChecked) < 5*time.Second && activeBackend != nil
if isCacheValid {
d, b := activeDriver, activeBackend
cacheMutex.RUnlock()
return d, b, nil
}
cacheMutex.RUnlock()
cacheMutex.Lock()
defer cacheMutex.Unlock()
// Double-check under write lock
if time.Since(lastChecked) < 5*time.Second && activeBackend != nil {
return activeDriver, activeBackend, nil
}
sc, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyStorageConfig)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return "", nil, err
}
lastChecked = time.Now()
// Reuse existing backend client singleton if configuration JSON matches
if sc.Value == activeConfigJSON && activeBackend != nil {
return activeDriver, activeBackend, nil
}
cfg := DefaultConfig()
if strings.TrimSpace(sc.Value) != "" {
if err := json.Unmarshal([]byte(sc.Value), &cfg); err != nil {
return "", nil, fmt.Errorf("parse storage config: %w", err)
}
}
backend, err := NewBackend(ctx, cfg, cfg.Driver)
if err != nil {
return "", nil, err
}
activeDriver = cfg.Driver
activeBackend = backend
activeConfigJSON = sc.Value
return activeDriver, activeBackend, nil
}
type functionBackend struct {
put func(context.Context, string, io.Reader, int64, string) error
get func(context.Context, string) (*Object, error)
delete func(context.Context, string) error
}
func (b *functionBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error) {
if err := b.put(ctx, key, body, size, contentType); err != nil {
return PutResult{}, err
}
return PutResult{Key: key}, nil
}
func (b *functionBackend) Get(ctx context.Context, key string) (*Object, error) {
return b.get(ctx, key)
}
func (b *functionBackend) Delete(ctx context.Context, key string) error {
return b.delete(ctx, key)
}
func (b *functionBackend) Test(context.Context) error {
return nil
}
// MockStorage replaces object operations for package tests and returns a restore function.
func MockStorage(
put func(context.Context, string, io.Reader, int64, string) error,
get func(context.Context, string) (*Object, error),
deleteObject func(context.Context, string) error,
) func() {
previous := mockBackend
mockBackend = &functionBackend{put: put, get: get, delete: deleteObject}
return func() {
mockBackend = previous
}
}
// NewBackend constructs a concrete backend from configuration.
func NewBackend(ctx context.Context, cfg Config, driver Driver) (Backend, error) {
if driver == DriverS3 && mockBackend != nil {
return mockBackend, nil
}
switch driver {
case DriverLocal:
return newLocalBackend(cfg.Local)
case DriverS3:
return newS3Backend(ctx, cfg.S3)
case DriverR2:
return newR2Backend(ctx, cfg.R2)
case DriverMinIO:
return newS3Backend(ctx, cfg.MinIO)
case DriverOSS:
return newOSSBackend(cfg.OSS)
case DriverWebDAV:
return newWebDAVBackend(cfg.WebDAV)
default:
return nil, fmt.Errorf("unsupported storage driver %q", driver)
}
}
+143
View File
@@ -0,0 +1,143 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"context"
"encoding/json"
"io"
"sync"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
)
func TestStorageCache(t *testing.T) {
// 1. Reset cache
ResetCache()
if activeConfigJSON != "" || activeDriver != "" || activeBackend != nil || !lastChecked.IsZero() {
t.Fatal("ResetCache did not clear cache variables")
}
// 2. Set up cache manually
expectedConfig := Config{
Driver: DriverLocal,
Local: LocalConfig{Root: "/tmp/wavelet-test"},
}
cfgJSON, err := json.Marshal(expectedConfig)
if err != nil {
t.Fatalf("Marshal config failed: %v", err)
}
cacheMutex.Lock()
activeConfigJSON = string(cfgJSON)
lastChecked = time.Now()
cacheMutex.Unlock()
// 3. Call LoadConfig and verify it loads from cache (doesn't hit database, which would fail/panic because DB is not initialized)
ctx := context.Background()
loadedCfg, err := LoadConfig(ctx)
if err != nil {
t.Fatalf("LoadConfig failed: %v", err)
}
if loadedCfg.Driver != expectedConfig.Driver || loadedCfg.Local.Root != expectedConfig.Local.Root {
t.Errorf("Loaded config %+v, expected %+v", loadedCfg, expectedConfig)
}
// 4. Test Active() returns cached driver and backend
mockBnd := &functionBackend{
put: func(context.Context, string, io.Reader, int64, string) error { return nil },
get: func(context.Context, string) (*Object, error) { return nil, nil },
delete: func(context.Context, string) error { return nil },
}
cacheMutex.Lock()
activeBackend = mockBnd
activeDriver = DriverLocal
cacheMutex.Unlock()
drv, bnd, err := Active(ctx)
if err != nil {
t.Fatalf("Active failed: %v", err)
}
if drv != DriverLocal || bnd != mockBnd {
t.Errorf("Active returned driver %v, backend %v; expected %v, %v", drv, bnd, DriverLocal, mockBnd)
}
// 5. Test ResetCache again
ResetCache()
if activeConfigJSON != "" || activeDriver != "" || activeBackend != nil || !lastChecked.IsZero() {
t.Fatal("ResetCache did not clear cache variables after setting them")
}
}
func TestStorageCachePubSub(t *testing.T) {
// 1. Start miniredis
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("Failed to run miniredis: %v", err)
}
defer mr.Close()
// 2. Initialize Redis client
rdb := redis.NewClient(&redis.Options{
Addr: mr.Addr(),
})
defer rdb.Close()
// 3. Set db.Redis to our client
oldRedis := db.Redis
db.Redis = rdb
defer func() {
db.Redis = oldRedis
}()
// Reset cache and set some cached config
ResetCache()
cacheMutex.Lock()
activeConfigJSON = "some_config"
lastChecked = time.Now()
cacheMutex.Unlock()
// 4. Force trigger lazy initialization of subscription
// Reset the once guard so it runs the listener
pubSubOnce = sync.Once{}
ctx := context.Background()
// Create mock backend for Active call
mockBnd := &functionBackend{
put: func(context.Context, string, io.Reader, int64, string) error { return nil },
get: func(context.Context, string) (*Object, error) { return nil, nil },
delete: func(context.Context, string) error { return nil },
}
cacheMutex.Lock()
activeBackend = mockBnd
activeDriver = DriverLocal
cacheMutex.Unlock()
_, _, _ = Active(ctx) // This calls startPubSubListener()
// Allow some time for subscriber connection
time.Sleep(100 * time.Millisecond)
// 5. Publish cache invalidation
PublishCacheInvalidation(ctx)
// Allow message propagation
time.Sleep(100 * time.Millisecond)
// 6. Verify cache was cleared
cacheMutex.RLock()
configJSON := activeConfigJSON
cacheMutex.RUnlock()
if configJSON != "" {
t.Error("Memory cache was not cleared after Redis Pub/Sub broadcast")
}
}
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package objectstore
import (
"context"
"fmt"
"io"
"net/http"
"path"
"strings"
"github.com/Rain-kl/Wavelet/pkg/httppool"
"github.com/studio-b12/gowebdav"
)
type contextTransport struct {
ctx context.Context
parent http.RoundTripper
}
func (t *contextTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return t.parent.RoundTrip(req.WithContext(t.ctx))
}
type webDAVBackend struct {
endpoint string
username string
password string
basePath string
}
func newWebDAVBackend(cfg WebDAVConfig) (*webDAVBackend, error) {
return &webDAVBackend{
endpoint: strings.TrimRight(cfg.Endpoint, "/"),
username: cfg.Username,
password: cfg.Password,
basePath: strings.Trim(cfg.BasePath, "/"),
}, nil
}
func (b *webDAVBackend) newClient(ctx context.Context) *gowebdav.Client {
client := gowebdav.NewClient(b.endpoint, b.username, b.password)
client.SetTransport(&contextTransport{
ctx: ctx,
parent: httppool.DefaultTransport(),
})
return client
}
func (b *webDAVBackend) Put(ctx context.Context, key string, body io.Reader, size int64, _ string) (PutResult, error) {
key = b.key(key)
client := b.newClient(ctx)
if dir := path.Dir(key); dir != "." && dir != "/" {
if err := client.MkdirAll(dir, storageDirPerm); err != nil {
return PutResult{}, fmt.Errorf("create WebDAV directory: %w", err)
}
}
if err := client.WriteStreamWithLength(key, body, size, storageFilePerm); err != nil {
return PutResult{}, fmt.Errorf("put WebDAV object: %w", err)
}
return PutResult{Key: key}, nil
}
func (b *webDAVBackend) Get(ctx context.Context, key string) (*Object, error) {
key = b.key(key)
client := b.newClient(ctx)
info, err := client.Stat(key)
if err != nil {
return nil, fmt.Errorf("stat WebDAV object: %w", err)
}
body, err := client.ReadStream(key)
if err != nil {
return nil, fmt.Errorf("get WebDAV object: %w", err)
}
contentType := defaultContentType
if typed, ok := info.(interface{ ContentType() string }); ok && typed.ContentType() != "" {
contentType = typed.ContentType()
}
return &Object{Body: body, ContentLength: info.Size(), ContentType: contentType}, nil
}
func (b *webDAVBackend) Delete(ctx context.Context, key string) error {
client := b.newClient(ctx)
if err := client.Remove(b.key(key)); err != nil {
return fmt.Errorf("delete WebDAV object: %w", err)
}
return nil
}
func (b *webDAVBackend) Test(ctx context.Context) error {
client := b.newClient(ctx)
if err := client.Connect(); err != nil {
return fmt.Errorf("connect WebDAV: %w", err)
}
return nil
}
func (b *webDAVBackend) key(key string) string {
return "/" + path.Join(b.basePath, strings.TrimLeft(key, "/"))
}
@@ -0,0 +1,52 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package batchwriter
import (
"fmt"
"time"
)
const (
defaultQueueSize = 10_000
defaultMaxBatchSize = 1_000
defaultFlushEvery = time.Second
)
// Config controls queue capacity and flush thresholds for a Writer instance.
type Config struct {
// Name identifies the writer in logs and diagnostics. Optional.
Name string
// QueueSize is the buffered channel capacity.
QueueSize int
// MaxBatchSize triggers a flush when the in-memory batch reaches this count.
MaxBatchSize int
// FlushInterval triggers a time-based flush even when the batch is smaller.
FlushInterval time.Duration
}
// DefaultConfig returns production-friendly defaults aligned with audit log batching.
func DefaultConfig() Config {
return Config{
QueueSize: defaultQueueSize,
MaxBatchSize: defaultMaxBatchSize,
FlushInterval: defaultFlushEvery,
}
}
func (c Config) validate() error {
if c.QueueSize <= 0 {
return fmt.Errorf("batchwriter: queue size must be positive")
}
if c.MaxBatchSize <= 0 {
return fmt.Errorf("batchwriter: max batch size must be positive")
}
if c.FlushInterval <= 0 {
return fmt.Errorf("batchwriter: flush interval must be positive")
}
return nil
}
@@ -0,0 +1,8 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package batchwriter
import "errors"
var errNilFlushFunc = errors.New("batchwriter: flush func is required")
@@ -0,0 +1,215 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package batchwriter provides a reusable buffered batch writer for high-throughput
// append-only sinks such as ClickHouse. Each business domain should own an independent
// Writer instance with its own queue, flush callback, and tuning parameters.
package batchwriter
import (
"context"
"sync"
"time"
)
// FlushFunc persists a batch of queued items. It is invoked from the worker goroutine.
type FlushFunc[T any] func(ctx context.Context, items []T) error
// FlushErrorHandler is called when FlushFunc returns an error. The batch is discarded
// after the handler returns; the worker continues processing.
type FlushErrorHandler func(ctx context.Context, batchSize int, err error)
// Writer buffers items and flushes them by size or interval.
type Writer[T any] struct {
cfg Config
flush FlushFunc[T]
onFlushError FlushErrorHandler
onDrop func(T)
startOnce sync.Once
stopOnce sync.Once
mu sync.RWMutex
ch chan T
workerCtx context.Context
done chan struct{}
}
// Option configures optional Writer callbacks.
type Option[T any] func(*Writer[T])
// WithFlushErrorHandler registers a callback for flush failures.
func WithFlushErrorHandler[T any](handler FlushErrorHandler) Option[T] {
return func(w *Writer[T]) {
w.onFlushError = handler
}
}
// WithDropHandler registers a callback when TryEnqueue cannot accept an item.
func WithDropHandler[T any](handler func(T)) Option[T] {
return func(w *Writer[T]) {
w.onDrop = handler
}
}
// New creates a Writer. Call Start before enqueueing items.
func New[T any](cfg Config, flush FlushFunc[T], opts ...Option[T]) (*Writer[T], error) {
if flush == nil {
return nil, errNilFlushFunc
}
if err := cfg.validate(); err != nil {
return nil, err
}
w := &Writer[T]{
cfg: cfg,
flush: flush,
done: make(chan struct{}),
}
for _, opt := range opts {
opt(w)
}
return w, nil
}
// Start launches the background worker. It is safe to call at most once.
func (w *Writer[T]) Start(parent context.Context) {
w.startOnce.Do(func() {
w.mu.Lock()
defer w.mu.Unlock()
w.ch = make(chan T, w.cfg.QueueSize)
w.workerCtx = context.WithoutCancel(parent)
go w.run()
})
}
// Stop closes the queue and waits until the worker drains pending items and exits.
func (w *Writer[T]) Stop(ctx context.Context) error {
w.mu.RLock()
ch := w.ch
done := w.done
w.mu.RUnlock()
if ch == nil {
return nil
}
w.stopOnce.Do(func() {
close(ch)
})
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// Running reports whether Start has been called and Stop has not completed.
func (w *Writer[T]) Running() bool {
w.mu.RLock()
defer w.mu.RUnlock()
if w.ch == nil {
return false
}
select {
case <-w.done:
return false
default:
return true
}
}
// TryEnqueue adds one item without blocking. It returns false when the writer is not
// running or the queue is full.
func (w *Writer[T]) TryEnqueue(item T) bool {
w.mu.RLock()
ch := w.ch
w.mu.RUnlock()
if ch == nil {
w.notifyDrop(item)
return false
}
select {
case ch <- item:
return true
default:
w.notifyDrop(item)
return false
}
}
// IsFull reports whether the queue has no remaining capacity.
func (w *Writer[T]) IsFull() bool {
w.mu.RLock()
defer w.mu.RUnlock()
if w.ch == nil {
return false
}
return len(w.ch) >= cap(w.ch)
}
// Len returns the current queue depth.
func (w *Writer[T]) Len() int {
w.mu.RLock()
defer w.mu.RUnlock()
if w.ch == nil {
return 0
}
return len(w.ch)
}
// Cap returns the queue capacity.
func (w *Writer[T]) Cap() int {
return w.cfg.QueueSize
}
func (w *Writer[T]) run() {
ticker := time.NewTicker(w.cfg.FlushInterval)
defer ticker.Stop()
batch := make([]T, 0, w.cfg.MaxBatchSize)
flush := func() {
if len(batch) == 0 {
return
}
items := append([]T(nil), batch...)
if err := w.flush(w.workerCtx, items); err != nil {
if w.onFlushError != nil {
w.onFlushError(w.workerCtx, len(items), err)
}
}
batch = batch[:0]
}
defer func() {
flush()
close(w.done)
}()
for {
select {
case item, ok := <-w.ch:
if !ok {
return
}
batch = append(batch, item)
if len(batch) >= w.cfg.MaxBatchSize {
flush()
}
case <-ticker.C:
flush()
}
}
}
func (w *Writer[T]) notifyDrop(item T) {
if w.onDrop == nil {
return
}
w.onDrop(item)
}
@@ -0,0 +1,284 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package batchwriter
import (
"context"
"errors"
"sync"
"testing"
"time"
"github.com/google/go-cmp/cmp"
)
func TestNewRejectsInvalidConfig(t *testing.T) {
t.Parallel()
_, err := New[int](Config{}, func(context.Context, []int) error { return nil })
if err == nil {
t.Fatal("New() = nil, want validation error")
}
}
func TestNewRejectsNilFlushFunc(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
_, err := New[int](cfg, nil)
if !errors.Is(err, errNilFlushFunc) {
t.Fatalf("New() error = %v, want %v", err, errNilFlushFunc)
}
}
func TestWriterFlushesOnMaxBatchSize(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
batches [][]int
)
cfg := DefaultConfig()
cfg.MaxBatchSize = 3
cfg.FlushInterval = time.Hour
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
mu.Lock()
defer mu.Unlock()
batches = append(batches, append([]int(nil), items...))
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
})
for i := range 3 {
if !writer.TryEnqueue(i + 1) {
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
}
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := len(batches) == 1
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(10 * time.Millisecond)
}
mu.Lock()
got := batches
mu.Unlock()
want := [][]int{{1, 2, 3}}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("flush batches mismatch (-want +got):\n%s", diff)
}
}
func TestWriterFlushesOnInterval(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
batch []int
)
cfg := DefaultConfig()
cfg.MaxBatchSize = 100
cfg.FlushInterval = 20 * time.Millisecond
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
mu.Lock()
defer mu.Unlock()
batch = append([]int(nil), items...)
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
})
if !writer.TryEnqueue(42) {
t.Fatal("TryEnqueue() = false, want true")
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := len(batch) == 1
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(5 * time.Millisecond)
}
mu.Lock()
got := batch
mu.Unlock()
want := []int{42}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("interval flush mismatch (-want +got):\n%s", diff)
}
}
func TestWriterTryEnqueueDropsWhenFull(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
cfg.QueueSize = 1
cfg.MaxBatchSize = 10
cfg.FlushInterval = time.Hour
var dropped int
writer, err := New[int](cfg, func(context.Context, []int) error { return nil }, WithDropHandler[int](func(int) {
dropped++
}))
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = writer.Stop(stopCtx)
})
if !writer.TryEnqueue(1) {
t.Fatal("TryEnqueue(1) = false, want true")
}
if writer.TryEnqueue(2) {
t.Fatal("TryEnqueue(2) = true, want false")
}
if !writer.IsFull() {
t.Fatal("IsFull() = false, want true")
}
if dropped != 1 {
t.Fatalf("dropped = %d, want 1", dropped)
}
}
func TestWriterStopDrainsQueuedItems(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
cfg.MaxBatchSize = 10
cfg.FlushInterval = time.Hour
var flushed []int
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
flushed = append(flushed, items...)
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
for i := range 2 {
if !writer.TryEnqueue(i + 1) {
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
}
}
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
want := []int{1, 2}
if diff := cmp.Diff(want, flushed); diff != "" {
t.Fatalf("Stop() drain mismatch (-want +got):\n%s", diff)
}
if writer.Running() {
t.Fatal("Running() = true after Stop(), want false")
}
}
func TestWriterInvokesFlushErrorHandler(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
cfg.MaxBatchSize = 1
cfg.FlushInterval = time.Hour
flushErr := errors.New("flush failed")
var (
mu sync.Mutex
errCount int
batchSize int
)
writer, err := New[int](cfg, func(context.Context, []int) error {
return flushErr
}, WithFlushErrorHandler[int](func(_ context.Context, size int, err error) {
mu.Lock()
defer mu.Unlock()
errCount++
batchSize = size
if !errors.Is(err, flushErr) {
t.Errorf("flush error = %v, want %v", err, flushErr)
}
}))
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = writer.Stop(stopCtx)
})
if !writer.TryEnqueue(7) {
t.Fatal("TryEnqueue() = false, want true")
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := errCount == 1
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(5 * time.Millisecond)
}
mu.Lock()
gotCount := errCount
gotSize := batchSize
mu.Unlock()
if gotCount != 1 {
t.Fatalf("flush error handler count = %d, want 1", gotCount)
}
if gotSize != 1 {
t.Fatalf("flush error handler batch size = %d, want 1", gotSize)
}
}
+155
View File
@@ -0,0 +1,155 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package db 提供数据库连接与基础设施
package db
import (
"context"
"fmt"
"log"
"net/url"
"strconv"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"go.opentelemetry.io/otel/attribute"
clickhouseDriver "gorm.io/driver/clickhouse"
"gorm.io/gorm"
"gorm.io/plugin/opentelemetry/tracing"
)
const (
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
)
var (
// ChConn ClickHouse 原生连接实例,用于批量写入
ChConn driver.Conn
chDB *gorm.DB
)
func init() {
if !config.Config.ClickHouse.Enabled {
return
}
cfg := config.Config.ClickHouse
if cfg.Database == "" {
log.Fatalf("[ClickHouse] database name is required (expected: wavelet)\n")
}
opts := buildClickHouseOptions()
var err error
ChConn, err = clickhouse.Open(opts)
if err != nil {
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
}
if err = ChConn.Ping(context.Background()); err != nil {
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
}
chDB, err = gorm.Open(clickhouseDriver.New(clickhouseDriver.Config{
DSN: buildClickHouseDSN(),
}), &gorm.Config{
SkipDefaultTransaction: true,
})
if err != nil {
log.Fatalf("[ClickHouse] init gorm connection failed: %v\n", err)
}
if err = chDB.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", cfg.Database),
attribute.String("db.system", "ClickHouse"),
),
),
); err != nil {
log.Fatalf("[ClickHouse] init trace failed: %v\n", err)
}
sqlDB, err := chDB.DB()
if err != nil {
log.Fatalf("[ClickHouse] load sql db failed: %v\n", err)
}
sqlDB.SetMaxIdleConns(cfg.MaxIdleConn)
sqlDB.SetMaxOpenConns(cfg.MaxOpenConn)
sqlDB.SetConnMaxLifetime(time.Duration(cfg.ConnMaxLifetime) * time.Second)
log.Println("[ClickHouse] connection established successfully")
}
func buildClickHouseOptions() *clickhouse.Options {
cfg := config.Config.ClickHouse
return &clickhouse.Options{
Addr: cfg.Hosts,
Auth: clickhouse.Auth{
Database: cfg.Database,
Username: cfg.Username,
Password: cfg.Password,
},
Settings: clickhouse.Settings{
"max_execution_time": clickhouseMaxExecTime,
},
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
MaxOpenConns: cfg.MaxOpenConn,
MaxIdleConns: cfg.MaxIdleConn,
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
BlockBufferSize: cfg.BlockBufferSize,
}
}
func buildClickHouseDSN() string {
cfg := config.Config.ClickHouse
chURL := &url.URL{
Scheme: "clickhouse",
Host: strings.Join(cfg.Hosts, ","),
Path: "/" + cfg.Database,
}
if cfg.Username != "" || cfg.Password != "" {
chURL.User = url.UserPassword(cfg.Username, cfg.Password)
}
query := chURL.Query()
query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout))
query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor))
query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime))
chURL.RawQuery = query.Encode()
return chURL.String()
}
// ChDB returns a context-aware GORM ClickHouse instance.
func ChDB(ctx context.Context) *gorm.DB {
if chDB == nil {
return nil
}
return chDB.WithContext(ctx)
}
// SetChDBForTest sets the package-level ClickHouse GORM instance for testing.
func SetChDBForTest(d *gorm.DB) {
chDB = d
}
// SetChConnForTest sets the package-level native ClickHouse connection for testing.
func SetChConnForTest(c driver.Conn) {
ChConn = c
}
+12
View File
@@ -0,0 +1,12 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
const (
errRedisHashSetFailed = "failed to set redis hash: %w"
errRedisHashDeleteFailed = "failed to delete redis hash field: %w"
errUnmarshalDataFailed = "failed to unmarshal data: %w"
errMarshalDataFailed = "failed to marshal data: %w"
errRedisKeySetFailed = "failed to set redis key: %w"
)
@@ -0,0 +1,46 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package idgen 提供分布式 ID 生成器
package idgen
import (
"fmt"
"log"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/bwmarrin/snowflake"
)
// 2025-12-01 00:00:00 UTC 的毫秒时间戳
const epoch int64 = 1764547200000
const maxNegativeIDRetries = 3
var node *snowflake.Node
func init() {
snowflake.Epoch = epoch
nodeID := config.Config.App.NodeID
var err error
node, err = snowflake.NewNode(nodeID)
if err != nil {
log.Fatalf("[Snowflake] init failed: %v\n", err)
}
log.Printf("[Snowflake] initialized with node ID: %d, epoch: 2025-12-01\n", nodeID)
}
// NextUint64ID 生成下一个分布式唯一 ID。
// 理论上不应出现负值;若出现则最多重试 maxNegativeIDRetries 次,仍失败则 panic。
func NextUint64ID() uint64 {
for attempt := 1; attempt <= maxNegativeIDRetries; attempt++ {
id := node.Generate().Int64()
if id >= 0 {
return uint64(id)
}
log.Printf("[Snowflake] generated negative ID: %d (attempt %d/%d)", id, attempt, maxNegativeIDRetries)
}
panic(fmt.Sprintf("[Snowflake] generated negative ID after %d attempts", maxNegativeIDRetries))
}
@@ -0,0 +1,15 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package idgen
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNextUint64ID(t *testing.T) {
id := NextUint64ID()
assert.NotZero(t, id)
}
@@ -0,0 +1,107 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"context"
"database/sql"
"embed"
"io/fs"
"log"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/pressly/goose/v3"
)
const (
clickhouseMigrationDir = "goose/clickhouse"
clickhouseGooseVersionTable = "goose_clickhouse_version"
clickhouseMaxExecTime = 60
clickhouseReadTimeoutFactor = 2
)
// clickhouseMigrationFS contains SQL migrations under goose/clickhouse.
//
//go:embed goose/clickhouse/*.sql
var clickhouseMigrationFS embed.FS
// MigrateClickHouse runs goose migrations against ClickHouse when enabled.
func MigrateClickHouse() Report {
if !config.Config.ClickHouse.Enabled {
return Report{Backend: "ClickHouse"}
}
cfg := config.Config.ClickHouse
sqlDB := clickhouse.OpenDB(&clickhouse.Options{
Addr: cfg.Hosts,
Auth: clickhouse.Auth{
Database: cfg.Database,
Username: cfg.Username,
Password: cfg.Password,
},
Settings: clickhouse.Settings{
"max_execution_time": clickhouseMaxExecTime,
},
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
MaxOpenConns: cfg.MaxOpenConn,
MaxIdleConns: cfg.MaxIdleConn,
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
BlockBufferSize: cfg.BlockBufferSize,
})
subFS, err := fs.Sub(clickhouseMigrationFS, "goose/clickhouse")
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] get sub fs failed: %v\n", err)
}
provider, err := goose.NewProvider(
"clickhouse",
sqlDB,
subFS,
goose.WithTableName(clickhouseGooseVersionTable),
goose.WithDisableGlobalRegistry(true),
)
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] create goose provider failed: %v\n", err)
}
previousVersion, err := provider.GetDBVersion(context.Background())
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] get goose version failed: %v\n", err)
}
if _, err := provider.Up(context.Background()); err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] goose migrate failed: %v\n", err)
}
currentVersion, err := provider.GetDBVersion(context.Background())
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] get migrated goose version failed: %v\n", err)
}
closeClickHouseDB(sqlDB)
log.Println("[ClickHouse] goose migrate success")
return Report{
Backend: "ClickHouse",
Enabled: true,
Version: currentVersion,
Applied: currentVersion != previousVersion,
}
}
func closeClickHouseDB(sqlDB *sql.DB) {
if err := sqlDB.Close(); err != nil {
log.Printf("[ClickHouse] close sql db failed: %v\n", err)
}
}
@@ -0,0 +1,51 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"testing"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/pressly/goose/v3"
)
func TestClickHouseMigrationFilesEmbedded(t *testing.T) {
entries, err := clickhouseMigrationFS.ReadDir(clickhouseMigrationDir)
if err != nil {
t.Fatalf("ReadDir(%q) error = %v", clickhouseMigrationDir, err)
}
if len(entries) == 0 {
t.Fatal("expected embedded ClickHouse migrations, got none")
}
found := false
for _, entry := range entries {
if entry.IsDir() {
continue
}
if entry.Name() == "202606190001_create_user_access_logs.sql" {
found = true
break
}
}
if !found {
t.Fatal("expected 202606190001_create_user_access_logs.sql in embedded migrations")
}
}
func TestClickHouseGooseDialect(t *testing.T) {
if err := goose.SetDialect("clickhouse"); err != nil {
t.Fatalf("SetDialect(clickhouse) error = %v", err)
}
}
func TestMigrateClickHouseSkipsWhenDisabled(t *testing.T) {
previousEnabled := config.Config.ClickHouse.Enabled
config.Config.ClickHouse.Enabled = false
t.Cleanup(func() {
config.Config.ClickHouse.Enabled = previousEnabled
})
MigrateClickHouse()
}
@@ -0,0 +1,21 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_user_access_logs
(
id UInt64,
user_id UInt64,
path String,
method String,
ip String,
user_agent String,
headers String,
status Int32,
latency Int64,
created_at DateTime
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (created_at, ip, user_id)
SETTINGS index_granularity = 8192;
-- +goose Down
DROP TABLE IF EXISTS w_user_access_logs;
@@ -0,0 +1,184 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS users (
id BIGINT PRIMARY KEY,
username VARCHAR(64) UNIQUE,
password VARCHAR(255),
nickname VARCHAR(255),
email VARCHAR(255),
avatar_url VARCHAR(255),
is_active BOOLEAN DEFAULT TRUE,
is_admin BOOLEAN DEFAULT FALSE,
bio VARCHAR(500),
phone VARCHAR(32),
gender VARCHAR(16),
website VARCHAR(255),
location VARCHAR(255),
last_login_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
CREATE TABLE IF NOT EXISTS auth_sources (
id BIGSERIAL PRIMARY KEY,
name VARCHAR(80) NOT NULL UNIQUE,
type VARCHAR(20) NOT NULL,
display_name VARCHAR(100),
is_active BOOLEAN NOT NULL DEFAULT FALSE,
client_id VARCHAR(255),
client_secret VARCHAR(1024),
openid_discovery_url VARCHAR(1024),
scopes VARCHAR(255),
icon_url VARCHAR(1024),
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
CREATE TABLE IF NOT EXISTS external_accounts (
id BIGSERIAL PRIMARY KEY,
auth_source_id BIGINT,
user_id BIGINT NOT NULL,
external_id VARCHAR(255) NOT NULL,
external_username VARCHAR(255),
email VARCHAR(255),
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
CREATE TABLE IF NOT EXISTS system_configs (
key VARCHAR(64) PRIMARY KEY,
value TEXT NOT NULL,
type VARCHAR(32) NOT NULL DEFAULT 'system',
visibility INTEGER NOT NULL DEFAULT 0,
description VARCHAR(255),
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS uploads (
id BIGINT PRIMARY KEY,
user_id BIGINT NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_path VARCHAR(500) NOT NULL,
file_size BIGINT NOT NULL,
mime_type VARCHAR(100) NOT NULL,
extension VARCHAR(50) NOT NULL,
hash VARCHAR(64),
storage_driver VARCHAR(50) NOT NULL,
type VARCHAR(50) NOT NULL,
status VARCHAR(20) NOT NULL,
metadata JSONB,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
CREATE TABLE IF NOT EXISTS access_tokens (
id BIGSERIAL PRIMARY KEY,
user_id BIGINT NOT NULL,
name VARCHAR(128) NOT NULL,
token_hash VARCHAR(64) NOT NULL UNIQUE,
masked_token VARCHAR(64) NOT NULL,
last_used_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
CREATE TABLE IF NOT EXISTS task_executions (
id BIGINT PRIMARY KEY,
task_id VARCHAR(128) NOT NULL UNIQUE,
task_type VARCHAR(64) NOT NULL,
task_name VARCHAR(128),
status VARCHAR(32) NOT NULL,
retryable BOOLEAN NOT NULL DEFAULT FALSE,
max_retry INTEGER NOT NULL DEFAULT 0,
retry_count INTEGER NOT NULL DEFAULT 0,
log TEXT,
error_message TEXT,
result TEXT,
started_at TIMESTAMPTZ,
finished_at TIMESTAMPTZ,
duration BIGINT,
payload TEXT,
triggered_by VARCHAR(32) NOT NULL DEFAULT 'system',
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
CREATE TABLE IF NOT EXISTS templates (
id BIGSERIAL PRIMARY KEY,
key VARCHAR(80) NOT NULL UNIQUE,
name VARCHAR(100) NOT NULL,
type VARCHAR(20) NOT NULL DEFAULT 'email',
subject VARCHAR(255),
content TEXT NOT NULL,
description VARCHAR(255),
is_system BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_difficulty', '4', 'system', 0, '人机验证 PoW 难度(目标前缀长度)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_ttl_seconds', '600', 'system', 0, '人机验证难题有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_token_ttl_seconds', '1200', 'system', 0, '人机验证兑换凭证有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('server_address', '', 'system', 0, '服务器地址(用于跨域源控制,不设定则允许任意源)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_host', '', 'system', 0, 'SMTP 服务器地址(例如 smtp.example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_port', '587', 'system', 0, 'SMTP 端口(例如 587 或 465)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_username', '', 'system', 0, 'SMTP 账户(如 sender@example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_password', '', 'system', 0, 'SMTP 访问凭证(授权码/密码)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('upload_allowed_extensions', 'jpg,png,webp', 'system', 1, '允许上传的图片扩展名(逗号分隔)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('site_name', 'Wavelet', 'system', 1, '系统平台的展示名称', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_login_enabled', 'true', 'system', 1, '是否允许使用账号密码登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('registration_enabled', 'true', 'system', 1, '控制普通用户是否可以自主注册(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_register_enabled', 'true', 'system', 1, '是否允许通过密码创建本地账号', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('oidc_login_enabled', 'true', 'system', 1, '是否允许使用第三方 OIDC 认证源登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('max_api_keys_per_user', '5', 'business', 1, '限制每个普通用户可以创建的 API Key 最大数量', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_login_verification_enabled', 'false', 'system', 1, '是否开启邮箱登录验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_register_verification_enabled', 'false', 'system', 1, '是否开启邮箱注册验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('menu_display_config', '{}', 'system', 1, '目录显示配置(JSON 字符串,格式为 {url: enabled})', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('search_engine_indexing_enabled', 'false', 'system', 1, '是否允许搜索引擎爬取/检索该站点(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('storage_config', '{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}', 'system', 0, '文件存储驱动及连接配置(JSON)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
INSERT INTO users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (1, 'admin', '12345678', 'Administrator', '', TRUE, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
INSERT INTO templates (key, name, type, subject, content, description, is_system, created_at, updated_at) VALUES
('login_email', '登录验证码邮件', 'email', 'Wavelet 登录验证码', '<h3>Wavelet 登录验证</h3><p>您的登录验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿将验证码泄露给他人。</p>', '用户密码登录时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('register_email', '注册验证码邮件', 'email', 'Wavelet 注册验证码', '<h3>Wavelet 注册验证</h3><p>您的注册验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿泄露给他人。</p>', '用户注册时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS templates;
DROP TABLE IF EXISTS task_executions;
DROP TABLE IF EXISTS access_tokens;
DROP TABLE IF EXISTS uploads;
DROP TABLE IF EXISTS system_configs;
DROP TABLE IF EXISTS external_accounts;
DROP TABLE IF EXISTS auth_sources;
DROP TABLE IF EXISTS users;
@@ -0,0 +1,20 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS schedules (
id BIGINT PRIMARY KEY,
name VARCHAR(128) NOT NULL,
task_type VARCHAR(64) NOT NULL,
cron VARCHAR(64) NOT NULL,
payload TEXT,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
-- Seed initial cleanup task
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
VALUES (1, '清理未使用上传', 'cleanup_unused_uploads', '0 */2 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (id) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS schedules;
@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE access_tokens ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE;
-- +goose Down
ALTER TABLE access_tokens DROP COLUMN IF EXISTS is_admin;
@@ -0,0 +1,9 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE access_tokens DROP COLUMN IF EXISTS last_used_at;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE access_tokens ADD COLUMN last_used_at TIMESTAMPTZ;
-- +goose StatementEnd
@@ -0,0 +1,9 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE schedules ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (START WITH 100);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE schedules ALTER COLUMN id DROP IDENTITY IF EXISTS;
-- +goose StatementEnd
@@ -0,0 +1,66 @@
-- +goose Up
ALTER TABLE users RENAME TO w_users;
ALTER TABLE auth_sources RENAME TO w_auth_sources;
ALTER TABLE external_accounts RENAME TO w_external_accounts;
ALTER TABLE system_configs RENAME TO w_system_configs;
ALTER TABLE uploads RENAME TO w_uploads;
ALTER TABLE access_tokens RENAME TO w_access_tokens;
ALTER TABLE task_executions RENAME TO w_task_executions;
ALTER TABLE templates RENAME TO w_templates;
ALTER TABLE schedules RENAME TO w_schedules;
-- Rename indexes for consistency
ALTER INDEX IF EXISTS idx_users_email RENAME TO idx_w_users_email;
ALTER INDEX IF EXISTS idx_users_is_active RENAME TO idx_w_users_is_active;
ALTER INDEX IF EXISTS idx_users_last_login_at RENAME TO idx_w_users_last_login_at;
ALTER INDEX IF EXISTS idx_users_created_at RENAME TO idx_w_users_created_at;
ALTER INDEX IF EXISTS idx_auth_sources_is_active RENAME TO idx_w_auth_sources_is_active;
ALTER INDEX IF EXISTS idx_external_accounts_auth_source_id RENAME TO idx_w_external_accounts_auth_source_id;
ALTER INDEX IF EXISTS idx_external_accounts_user_id RENAME TO idx_w_external_accounts_user_id;
ALTER INDEX IF EXISTS idx_external_accounts_source_external RENAME TO idx_w_external_accounts_source_external;
ALTER INDEX IF EXISTS idx_uploads_user_id RENAME TO idx_w_uploads_user_id;
ALTER INDEX IF EXISTS idx_uploads_file_path RENAME TO idx_w_uploads_file_path;
ALTER INDEX IF EXISTS idx_uploads_hash RENAME TO idx_w_uploads_hash;
ALTER INDEX IF EXISTS idx_uploads_type RENAME TO idx_w_uploads_type;
ALTER INDEX IF EXISTS idx_access_tokens_user_id RENAME TO idx_w_access_tokens_user_id;
ALTER INDEX IF EXISTS idx_task_executions_task_type RENAME TO idx_w_task_executions_task_type;
ALTER INDEX IF EXISTS idx_task_executions_status RENAME TO idx_w_task_executions_status;
ALTER INDEX IF EXISTS idx_task_executions_started_at RENAME TO idx_w_task_executions_started_at;
ALTER INDEX IF EXISTS idx_task_executions_created_at RENAME TO idx_w_task_executions_created_at;
ALTER INDEX IF EXISTS idx_templates_is_system RENAME TO idx_w_templates_is_system;
ALTER INDEX IF EXISTS idx_templates_created_at RENAME TO idx_w_templates_created_at;
ALTER INDEX IF EXISTS idx_templates_updated_at RENAME TO idx_w_templates_updated_at;
ALTER INDEX IF EXISTS idx_schedules_is_active RENAME TO idx_w_schedules_is_active;
-- +goose Down
ALTER INDEX IF EXISTS idx_w_schedules_is_active RENAME TO idx_schedules_is_active;
ALTER INDEX IF EXISTS idx_w_templates_updated_at RENAME TO idx_templates_updated_at;
ALTER INDEX IF EXISTS idx_w_templates_created_at RENAME TO idx_templates_created_at;
ALTER INDEX IF EXISTS idx_w_templates_is_system RENAME TO idx_templates_is_system;
ALTER INDEX IF EXISTS idx_w_task_executions_created_at RENAME TO idx_task_executions_created_at;
ALTER INDEX IF EXISTS idx_w_task_executions_started_at RENAME TO idx_task_executions_started_at;
ALTER INDEX IF EXISTS idx_w_task_executions_status RENAME TO idx_task_executions_status;
ALTER INDEX IF EXISTS idx_w_task_executions_task_type RENAME TO idx_task_executions_task_type;
ALTER INDEX IF EXISTS idx_w_access_tokens_user_id RENAME TO idx_access_tokens_user_id;
ALTER INDEX IF EXISTS idx_w_uploads_type RENAME TO idx_uploads_type;
ALTER INDEX IF EXISTS idx_w_uploads_hash RENAME TO idx_uploads_hash;
ALTER INDEX IF EXISTS idx_w_uploads_file_path RENAME TO idx_uploads_file_path;
ALTER INDEX IF EXISTS idx_w_uploads_user_id RENAME TO idx_uploads_user_id;
ALTER INDEX IF EXISTS idx_w_external_accounts_source_external RENAME TO idx_external_accounts_source_external;
ALTER INDEX IF EXISTS idx_w_external_accounts_user_id RENAME TO idx_external_accounts_user_id;
ALTER INDEX IF EXISTS idx_w_external_accounts_auth_source_id RENAME TO idx_external_accounts_auth_source_id;
ALTER INDEX IF EXISTS idx_w_auth_sources_is_active RENAME TO idx_auth_sources_is_active;
ALTER INDEX IF EXISTS idx_w_users_created_at RENAME TO idx_users_created_at;
ALTER INDEX IF EXISTS idx_w_users_last_login_at RENAME TO idx_users_last_login_at;
ALTER INDEX IF EXISTS idx_w_users_is_active RENAME TO idx_users_is_active;
ALTER INDEX IF EXISTS idx_w_users_email RENAME TO idx_users_email;
ALTER TABLE w_schedules RENAME TO schedules;
ALTER TABLE w_templates RENAME TO templates;
ALTER TABLE w_task_executions RENAME TO task_executions;
ALTER TABLE w_access_tokens RENAME TO access_tokens;
ALTER TABLE w_uploads RENAME TO uploads;
ALTER TABLE w_system_configs RENAME TO system_configs;
ALTER TABLE w_external_accounts RENAME TO external_accounts;
ALTER TABLE w_auth_sources RENAME TO auth_sources;
ALTER TABLE w_users RENAME TO users;
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('file_access_whitelist', '["avatar"]', 'system', 1, '免登录访问的文件业务类型白名单 (JSON 数组格式)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'file_access_whitelist';
@@ -0,0 +1,10 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
@@ -0,0 +1,8 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('login_session_ttl_hours', '0', 'system', 0, '登录会话过期时间 (小时,0表示浏览器关闭后自动退出,-1表示永不过期)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'login_session_ttl_hours';
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'update_upstream_repository';
@@ -0,0 +1,6 @@
-- +goose Up
ALTER TABLE w_uploads ADD COLUMN access_mode INTEGER NOT NULL DEFAULT 0;
UPDATE w_uploads SET access_mode = 1 WHERE type = 'avatar';
-- +goose Down
ALTER TABLE w_uploads DROP COLUMN access_mode;
@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE w_system_configs ALTER COLUMN value TYPE TEXT;
-- +goose Down
ALTER TABLE w_system_configs ALTER COLUMN value TYPE VARCHAR(255);
@@ -0,0 +1,15 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES (
'storage_config',
'{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}',
'system',
0,
'文件存储驱动及连接配置(JSON)',
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'storage_config';
@@ -0,0 +1,34 @@
-- +goose Up
CREATE TABLE w_push_events (
id BIGSERIAL PRIMARY KEY,
event_key VARCHAR(80) NOT NULL UNIQUE,
name VARCHAR(100) NOT NULL,
channels TEXT NOT NULL,
targets TEXT NOT NULL,
template TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX idx_w_push_events_enabled ON w_push_events(enabled);
CREATE TABLE w_push_histories (
id BIGSERIAL PRIMARY KEY,
event_key VARCHAR(80) NOT NULL,
channel VARCHAR(50) NOT NULL,
target VARCHAR(255) NOT NULL,
title VARCHAR(255) NOT NULL,
content TEXT NOT NULL,
level VARCHAR(20) NOT NULL,
status VARCHAR(20) NOT NULL,
error_msg TEXT,
created_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX idx_w_push_histories_event ON w_push_histories(event_key);
CREATE INDEX idx_w_push_histories_created ON w_push_histories(created_at);
-- +goose Down
DROP TABLE IF EXISTS w_push_histories;
DROP TABLE IF EXISTS w_push_events;
@@ -0,0 +1,8 @@
-- +goose Up
INSERT INTO w_users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (999, 'system', '*', '系统', '', TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
SELECT setval(pg_get_serial_sequence('w_users', 'id'), COALESCE((SELECT MAX(id) FROM w_users), 1));
-- +goose Down
DELETE FROM w_users WHERE username = 'system';
@@ -0,0 +1,19 @@
-- +goose Up
CREATE TABLE w_push_channels (
id BIGSERIAL PRIMARY KEY,
name VARCHAR(80) NOT NULL UNIQUE,
description VARCHAR(255),
type VARCHAR(50) NOT NULL DEFAULT 'custom',
token VARCHAR(100),
url TEXT NOT NULL,
other TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX idx_w_push_channels_name ON w_push_channels(name);
CREATE INDEX idx_w_push_channels_enabled ON w_push_channels(enabled);
-- +goose Down
DROP TABLE IF EXISTS w_push_channels;
@@ -0,0 +1,11 @@
-- +goose Up
UPDATE w_schedules
SET name = '系统定期垃圾清理',
task_type = 'system_cleanup'
WHERE id = 1;
-- +goose Down
UPDATE w_schedules
SET name = '清理未使用上传',
task_type = 'cleanup_unused_uploads'
WHERE id = 1;
@@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE w_push_events ADD COLUMN task_type VARCHAR(100) NOT NULL DEFAULT '';
CREATE INDEX idx_w_push_events_task_type ON w_push_events(task_type);
-- +goose Down
DROP INDEX IF EXISTS idx_w_push_events_task_type;
ALTER TABLE w_push_events DROP COLUMN task_type;
@@ -0,0 +1,6 @@
-- +goose Up
DELETE FROM w_system_configs WHERE key = 'push_config';
DELETE FROM w_push_events WHERE event_key = 'admin_login';
DELETE FROM w_system_configs WHERE key = 'push_global_token';
-- +goose Down
@@ -0,0 +1,9 @@
-- +goose Up
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
-- +goose Down
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
@@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_upload_stats (
dimension VARCHAR(32) NOT NULL,
stat_key VARCHAR(64) NOT NULL DEFAULT '',
file_count BIGINT NOT NULL DEFAULT 0,
file_size BIGINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dimension, stat_key)
);
-- +goose Down
DROP TABLE IF EXISTS w_upload_stats;
@@ -0,0 +1,67 @@
-- +goose Up
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'type',
COALESCE(NULLIF(type, ''), 'generic'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'category',
CASE
WHEN LOWER(mime_type) LIKE 'image/%'
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
OR LOWER(mime_type) LIKE '%zip%'
OR LOWER(mime_type) LIKE '%tar%'
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
OR LOWER(mime_type) LIKE 'text/%'
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
ELSE '其他'
END,
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY 2
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'trend',
TO_CHAR(created_at, 'YYYY-MM-DD'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY TO_CHAR(created_at, 'YYYY-MM-DD')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
-- +goose Down
DELETE FROM w_upload_stats;
@@ -0,0 +1,7 @@
-- +goose Up
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
ALTER TABLE w_uploads DROP COLUMN IF EXISTS storage_driver;
-- +goose Down
ALTER TABLE w_uploads ADD COLUMN IF NOT EXISTS storage_driver VARCHAR(50) NOT NULL DEFAULT 'local';
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
@@ -0,0 +1,184 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS users (
id BIGINT PRIMARY KEY,
username VARCHAR(64) UNIQUE,
password VARCHAR(255),
nickname VARCHAR(255),
email VARCHAR(255),
avatar_url VARCHAR(255),
is_active BOOLEAN DEFAULT TRUE,
is_admin BOOLEAN DEFAULT FALSE,
bio VARCHAR(500),
phone VARCHAR(32),
gender VARCHAR(16),
website VARCHAR(255),
location VARCHAR(255),
last_login_at DATETIME,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
CREATE TABLE IF NOT EXISTS auth_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name VARCHAR(80) NOT NULL UNIQUE,
type VARCHAR(20) NOT NULL,
display_name VARCHAR(100),
is_active BOOLEAN NOT NULL DEFAULT FALSE,
client_id VARCHAR(255),
client_secret VARCHAR(1024),
openid_discovery_url VARCHAR(1024),
scopes VARCHAR(255),
icon_url VARCHAR(1024),
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
CREATE TABLE IF NOT EXISTS external_accounts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
auth_source_id BIGINT,
user_id BIGINT NOT NULL,
external_id VARCHAR(255) NOT NULL,
external_username VARCHAR(255),
email VARCHAR(255),
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
CREATE TABLE IF NOT EXISTS system_configs (
key VARCHAR(64) PRIMARY KEY,
value TEXT NOT NULL,
type VARCHAR(32) NOT NULL DEFAULT 'system',
visibility INTEGER NOT NULL DEFAULT 0,
description VARCHAR(255),
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS uploads (
id BIGINT PRIMARY KEY,
user_id BIGINT NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_path VARCHAR(500) NOT NULL,
file_size BIGINT NOT NULL,
mime_type VARCHAR(100) NOT NULL,
extension VARCHAR(50) NOT NULL,
hash VARCHAR(64),
storage_driver VARCHAR(50) NOT NULL,
type VARCHAR(50) NOT NULL,
status VARCHAR(20) NOT NULL,
metadata JSON,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
CREATE TABLE IF NOT EXISTS access_tokens (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id BIGINT NOT NULL,
name VARCHAR(128) NOT NULL,
token_hash VARCHAR(64) NOT NULL UNIQUE,
masked_token VARCHAR(64) NOT NULL,
last_used_at DATETIME,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
CREATE TABLE IF NOT EXISTS task_executions (
id BIGINT PRIMARY KEY,
task_id VARCHAR(128) NOT NULL UNIQUE,
task_type VARCHAR(64) NOT NULL,
task_name VARCHAR(128),
status VARCHAR(32) NOT NULL,
retryable BOOLEAN NOT NULL DEFAULT FALSE,
max_retry INTEGER NOT NULL DEFAULT 0,
retry_count INTEGER NOT NULL DEFAULT 0,
log TEXT,
error_message TEXT,
result TEXT,
started_at DATETIME,
finished_at DATETIME,
duration BIGINT,
payload TEXT,
triggered_by VARCHAR(32) NOT NULL DEFAULT 'system',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
CREATE TABLE IF NOT EXISTS templates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
key VARCHAR(80) NOT NULL UNIQUE,
name VARCHAR(100) NOT NULL,
type VARCHAR(20) NOT NULL DEFAULT 'email',
subject VARCHAR(255),
content TEXT NOT NULL,
description VARCHAR(255),
is_system BOOLEAN NOT NULL DEFAULT FALSE,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_difficulty', '4', 'system', 0, '人机验证 PoW 难度(目标前缀长度)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_ttl_seconds', '600', 'system', 0, '人机验证难题有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_token_ttl_seconds', '1200', 'system', 0, '人机验证兑换凭证有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('server_address', '', 'system', 0, '服务器地址(用于跨域源控制,不设定则允许任意源)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_host', '', 'system', 0, 'SMTP 服务器地址(例如 smtp.example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_port', '587', 'system', 0, 'SMTP 端口(例如 587 或 465)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_username', '', 'system', 0, 'SMTP 账户(如 sender@example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_password', '', 'system', 0, 'SMTP 访问凭证(授权码/密码)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('upload_allowed_extensions', 'jpg,png,webp', 'system', 1, '允许上传的图片扩展名(逗号分隔)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('site_name', 'Wavelet', 'system', 1, '系统平台的展示名称', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_login_enabled', 'true', 'system', 1, '是否允许使用账号密码登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('registration_enabled', 'true', 'system', 1, '控制普通用户是否可以自主注册(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_register_enabled', 'true', 'system', 1, '是否允许通过密码创建本地账号', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('oidc_login_enabled', 'true', 'system', 1, '是否允许使用第三方 OIDC 认证源登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('max_api_keys_per_user', '5', 'business', 1, '限制每个普通用户可以创建的 API Key 最大数量', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_login_verification_enabled', 'false', 'system', 1, '是否开启邮箱登录验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_register_verification_enabled', 'false', 'system', 1, '是否开启邮箱注册验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('menu_display_config', '{}', 'system', 1, '目录显示配置(JSON 字符串,格式为 {url: enabled})', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('search_engine_indexing_enabled', 'false', 'system', 1, '是否允许搜索引擎爬取/检索该站点(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('storage_config', '{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}', 'system', 0, '文件存储驱动及连接配置(JSON)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
INSERT INTO users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (1, 'admin', '12345678', 'Administrator', '', TRUE, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
INSERT INTO templates (key, name, type, subject, content, description, is_system, created_at, updated_at) VALUES
('login_email', '登录验证码邮件', 'email', 'Wavelet 登录验证码', '<h3>Wavelet 登录验证</h3><p>您的登录验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿将验证码泄露给他人。</p>', '用户密码登录时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('register_email', '注册验证码邮件', 'email', 'Wavelet 注册验证码', '<h3>Wavelet 注册验证</h3><p>您的注册验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿泄露给他人。</p>', '用户注册时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS templates;
DROP TABLE IF EXISTS task_executions;
DROP TABLE IF EXISTS access_tokens;
DROP TABLE IF EXISTS uploads;
DROP TABLE IF EXISTS system_configs;
DROP TABLE IF EXISTS external_accounts;
DROP TABLE IF EXISTS auth_sources;
DROP TABLE IF EXISTS users;
@@ -0,0 +1,20 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS schedules (
id BIGINT PRIMARY KEY,
name VARCHAR(128) NOT NULL,
task_type VARCHAR(64) NOT NULL,
cron VARCHAR(64) NOT NULL,
payload TEXT,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
-- Seed initial cleanup task
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
VALUES (1, '清理未使用上传', 'cleanup_unused_uploads', '0 */2 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (id) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS schedules;
@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE access_tokens ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE access_tokens DROP COLUMN is_admin;
@@ -0,0 +1,9 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE access_tokens DROP COLUMN last_used_at;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE access_tokens ADD COLUMN last_used_at DATETIME;
-- +goose StatementEnd
@@ -0,0 +1,32 @@
-- +goose Up
-- +goose StatementBegin
-- 1. Rename existing schedules table
ALTER TABLE schedules RENAME TO schedules_old;
-- 2. Create new schedules table with AUTOINCREMENT
CREATE TABLE schedules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name VARCHAR(128) NOT NULL,
task_type VARCHAR(64) NOT NULL,
cron VARCHAR(64) NOT NULL,
payload TEXT,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- 3. Copy existing data
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at FROM schedules_old;
-- 4. Drop the old table
DROP TABLE schedules_old;
-- 5. Recreate index
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- Re-creating table with AUTOINCREMENT cannot be undone simply without recreating table again.
-- +goose StatementEnd
@@ -0,0 +1,123 @@
-- +goose Up
ALTER TABLE users RENAME TO w_users;
ALTER TABLE auth_sources RENAME TO w_auth_sources;
ALTER TABLE external_accounts RENAME TO w_external_accounts;
ALTER TABLE system_configs RENAME TO w_system_configs;
ALTER TABLE uploads RENAME TO w_uploads;
ALTER TABLE access_tokens RENAME TO w_access_tokens;
ALTER TABLE task_executions RENAME TO w_task_executions;
ALTER TABLE templates RENAME TO w_templates;
ALTER TABLE schedules RENAME TO w_schedules;
-- Drop and Recreate SQLite indexes
DROP INDEX IF EXISTS idx_users_email;
CREATE INDEX IF NOT EXISTS idx_w_users_email ON w_users (email);
DROP INDEX IF EXISTS idx_users_is_active;
CREATE INDEX IF NOT EXISTS idx_w_users_is_active ON w_users (is_active);
DROP INDEX IF EXISTS idx_users_last_login_at;
CREATE INDEX IF NOT EXISTS idx_w_users_last_login_at ON w_users (last_login_at);
DROP INDEX IF EXISTS idx_users_created_at;
CREATE INDEX IF NOT EXISTS idx_w_users_created_at ON w_users (created_at);
DROP INDEX IF EXISTS idx_auth_sources_is_active;
CREATE INDEX IF NOT EXISTS idx_w_auth_sources_is_active ON w_auth_sources (is_active);
DROP INDEX IF EXISTS idx_external_accounts_auth_source_id;
CREATE INDEX IF NOT EXISTS idx_w_external_accounts_auth_source_id ON w_external_accounts (auth_source_id);
DROP INDEX IF EXISTS idx_external_accounts_user_id;
CREATE INDEX IF NOT EXISTS idx_w_external_accounts_user_id ON w_external_accounts (user_id);
DROP INDEX IF EXISTS idx_external_accounts_source_external;
CREATE UNIQUE INDEX IF NOT EXISTS idx_w_external_accounts_source_external ON w_external_accounts (auth_source_id, external_id);
DROP INDEX IF EXISTS idx_uploads_user_id;
CREATE INDEX IF NOT EXISTS idx_w_uploads_user_id ON w_uploads (user_id);
DROP INDEX IF EXISTS idx_uploads_file_path;
CREATE INDEX IF NOT EXISTS idx_w_uploads_file_path ON w_uploads (file_path);
DROP INDEX IF EXISTS idx_uploads_hash;
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash ON w_uploads (hash);
DROP INDEX IF EXISTS idx_uploads_type;
CREATE INDEX IF NOT EXISTS idx_w_uploads_type ON w_uploads (type);
DROP INDEX IF EXISTS idx_access_tokens_user_id;
CREATE INDEX IF NOT EXISTS idx_w_access_tokens_user_id ON w_access_tokens (user_id);
DROP INDEX IF EXISTS idx_task_executions_task_type;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_task_type ON w_task_executions (task_type);
DROP INDEX IF EXISTS idx_task_executions_status;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_status ON w_task_executions (status);
DROP INDEX IF EXISTS idx_task_executions_started_at;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_started_at ON w_task_executions (started_at);
DROP INDEX IF EXISTS idx_task_executions_created_at;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_created_at ON w_task_executions (created_at);
DROP INDEX IF EXISTS idx_templates_is_system;
CREATE INDEX IF NOT EXISTS idx_w_templates_is_system ON w_templates (is_system);
DROP INDEX IF EXISTS idx_templates_created_at;
CREATE INDEX IF NOT EXISTS idx_w_templates_created_at ON w_templates (created_at);
DROP INDEX IF EXISTS idx_templates_updated_at;
CREATE INDEX IF NOT EXISTS idx_w_templates_updated_at ON w_templates (updated_at);
DROP INDEX IF EXISTS idx_schedules_is_active;
CREATE INDEX IF NOT EXISTS idx_w_schedules_is_active ON w_schedules (is_active);
-- +goose Down
ALTER TABLE w_schedules RENAME TO schedules;
ALTER TABLE w_templates RENAME TO templates;
ALTER TABLE w_task_executions RENAME TO task_executions;
ALTER TABLE w_access_tokens RENAME TO access_tokens;
ALTER TABLE w_uploads RENAME TO uploads;
ALTER TABLE w_system_configs RENAME TO system_configs;
ALTER TABLE w_external_accounts RENAME TO external_accounts;
ALTER TABLE w_auth_sources RENAME TO auth_sources;
ALTER TABLE w_users RENAME TO users;
-- Revert indexes
DROP INDEX IF EXISTS idx_w_users_email;
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
DROP INDEX IF EXISTS idx_w_users_is_active;
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
DROP INDEX IF EXISTS idx_w_users_last_login_at;
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
DROP INDEX IF EXISTS idx_w_users_created_at;
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
DROP INDEX IF EXISTS idx_w_auth_sources_is_active;
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
DROP INDEX IF EXISTS idx_w_external_accounts_auth_source_id;
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
DROP INDEX IF EXISTS idx_w_external_accounts_user_id;
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
DROP INDEX IF EXISTS idx_w_external_accounts_source_external;
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
DROP INDEX IF EXISTS idx_w_uploads_user_id;
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
DROP INDEX IF EXISTS idx_w_uploads_file_path;
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
DROP INDEX IF EXISTS idx_w_uploads_hash;
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
DROP INDEX IF EXISTS idx_w_uploads_type;
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
DROP INDEX IF EXISTS idx_w_access_tokens_user_id;
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
DROP INDEX IF EXISTS idx_w_task_executions_task_type;
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
DROP INDEX IF EXISTS idx_w_task_executions_status;
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
DROP INDEX IF EXISTS idx_w_task_executions_started_at;
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
DROP INDEX IF EXISTS idx_w_task_executions_created_at;
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
DROP INDEX IF EXISTS idx_w_templates_is_system;
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
DROP INDEX IF EXISTS idx_w_templates_created_at;
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
DROP INDEX IF EXISTS idx_w_templates_updated_at;
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
DROP INDEX IF EXISTS idx_w_schedules_is_active;
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('file_access_whitelist', '["avatar"]', 'system', 1, '免登录访问的文件业务类型白名单 (JSON 数组格式)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'file_access_whitelist';
@@ -0,0 +1,10 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
@@ -0,0 +1,8 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('login_session_ttl_hours', '0', 'system', 0, '登录会话过期时间 (小时,0表示浏览器关闭后自动退出,-1表示永不过期)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'login_session_ttl_hours';
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'update_upstream_repository';
@@ -0,0 +1,6 @@
-- +goose Up
ALTER TABLE w_uploads ADD COLUMN access_mode INTEGER NOT NULL DEFAULT 0;
UPDATE w_uploads SET access_mode = 1 WHERE type = 'avatar';
-- +goose Down
ALTER TABLE w_uploads DROP COLUMN access_mode;
@@ -0,0 +1,6 @@
-- +goose Up
-- SQLite stores VARCHAR and TEXT with the same TEXT affinity.
SELECT 1;
-- +goose Down
SELECT 1;
@@ -0,0 +1,15 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES (
'storage_config',
'{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}',
'system',
0,
'文件存储驱动及连接配置(JSON)',
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'storage_config';
@@ -0,0 +1,34 @@
-- +goose Up
CREATE TABLE w_push_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event_key TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
channels TEXT NOT NULL,
targets TEXT NOT NULL,
template TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
);
CREATE INDEX idx_w_push_events_enabled ON w_push_events(enabled);
CREATE TABLE w_push_histories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event_key TEXT NOT NULL,
channel TEXT NOT NULL,
target TEXT NOT NULL,
title TEXT NOT NULL,
content TEXT NOT NULL,
level TEXT NOT NULL,
status TEXT NOT NULL,
error_msg TEXT,
created_at DATETIME NOT NULL
);
CREATE INDEX idx_w_push_histories_event ON w_push_histories(event_key);
CREATE INDEX idx_w_push_histories_created ON w_push_histories(created_at);
-- +goose Down
DROP TABLE IF EXISTS w_push_histories;
DROP TABLE IF EXISTS w_push_events;
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (999, 'system', '*', '系统', '', TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
-- +goose Down
DELETE FROM w_users WHERE username = 'system';
@@ -0,0 +1,19 @@
-- +goose Up
CREATE TABLE w_push_channels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
description TEXT,
type TEXT NOT NULL DEFAULT 'custom',
token TEXT,
url TEXT NOT NULL,
other TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
);
CREATE INDEX idx_w_push_channels_name ON w_push_channels(name);
CREATE INDEX idx_w_push_channels_enabled ON w_push_channels(enabled);
-- +goose Down
DROP TABLE IF EXISTS w_push_channels;
@@ -0,0 +1,11 @@
-- +goose Up
UPDATE w_schedules
SET name = '系统定期垃圾清理',
task_type = 'system_cleanup'
WHERE id = 1;
-- +goose Down
UPDATE w_schedules
SET name = '清理未使用上传',
task_type = 'cleanup_unused_uploads'
WHERE id = 1;
@@ -0,0 +1,9 @@
-- +goose Up
ALTER TABLE w_push_events ADD COLUMN task_type VARCHAR(100) NOT NULL DEFAULT '';
CREATE INDEX idx_w_push_events_task_type ON w_push_events(task_type);
-- +goose Down
DROP INDEX IF EXISTS idx_w_push_events_task_type;
-- SQLite does not support DROP COLUMN in older versions easily, but standard ALTER TABLE DROP COLUMN works in SQLite 3.35.0+.
-- We can write standard DROP COLUMN.
ALTER TABLE w_push_events DROP COLUMN task_type;
@@ -0,0 +1,6 @@
-- +goose Up
DELETE FROM w_system_configs WHERE key = 'push_config';
DELETE FROM w_push_events WHERE event_key = 'admin_login';
DELETE FROM w_system_configs WHERE key = 'push_global_token';
-- +goose Down
@@ -0,0 +1,9 @@
-- +goose Up
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
-- +goose Down
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
@@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_upload_stats (
dimension VARCHAR(32) NOT NULL,
stat_key VARCHAR(64) NOT NULL DEFAULT '',
file_count BIGINT NOT NULL DEFAULT 0,
file_size BIGINT NOT NULL DEFAULT 0,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dimension, stat_key)
);
-- +goose Down
DROP TABLE IF EXISTS w_upload_stats;
@@ -0,0 +1,67 @@
-- +goose Up
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'type',
COALESCE(NULLIF(type, ''), 'generic'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'category',
CASE
WHEN LOWER(mime_type) LIKE 'image/%'
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
OR LOWER(mime_type) LIKE '%zip%'
OR LOWER(mime_type) LIKE '%tar%'
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
OR LOWER(mime_type) LIKE 'text/%'
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
ELSE '其他'
END,
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY 2
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'trend',
STRFTIME('%Y-%m-%d', created_at),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY STRFTIME('%Y-%m-%d', created_at)
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
-- +goose Down
DELETE FROM w_upload_stats;
@@ -0,0 +1,8 @@
-- +goose Up
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
-- SQLite lacks DROP COLUMN IF EXISTS; goose runs this only after initial schema created the column.
ALTER TABLE w_uploads DROP COLUMN storage_driver;
-- +goose Down
ALTER TABLE w_uploads ADD COLUMN storage_driver VARCHAR(50) NOT NULL DEFAULT 'local';
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
@@ -0,0 +1,103 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package migrator 提供数据库迁移功能
package migrator
import (
"context"
"embed"
"log"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/pressly/goose/v3"
)
// migrationFS contains SQL migrations under goose/<dialect>.
//
//go:embed goose/postgres/*.sql goose/sqlite/*.sql
var migrationFS embed.FS
// dbType 返回当前数据库类型名称(用于日志输出)
func dbType() string {
if !config.Config.Database.Enabled {
return "SQLite"
}
return "PostgreSQL"
}
const (
dialectSqlite = "sqlite3"
dialectPostgres = "postgres"
cascadeSuffix = " CASCADE"
)
// Report describes the database migration state observed during startup.
type Report struct {
Backend string
Enabled bool
Version int64
Applied bool
}
func gooseDialect() string {
if !config.Config.Database.Enabled {
return dialectSqlite
}
return dialectPostgres
}
func migrationDir() string {
if !config.Config.Database.Enabled {
return "goose/sqlite"
}
return "goose/postgres"
}
// Migrate 执行数据库迁移
func Migrate() Report {
gormDB := db.DB(context.Background())
if gormDB == nil {
log.Fatalf("[%s] database not initialized\n", dbType())
}
sqlDB, err := gormDB.DB()
if err != nil {
log.Fatalf("[%s] load sql db failed: %v\n", dbType(), err)
}
goose.SetBaseFS(migrationFS)
if err := goose.SetDialect(gooseDialect()); err != nil {
log.Fatalf("[%s] set goose dialect failed: %v\n", dbType(), err)
}
previousVersion, err := goose.GetDBVersion(sqlDB)
if err != nil {
log.Fatalf("[%s] get goose version failed: %v\n", dbType(), err)
}
if err := goose.Up(sqlDB, migrationDir()); err != nil {
log.Fatalf("[%s] goose migrate failed: %v\n", dbType(), err)
}
clearSystemConfigCache()
currentVersion, err := goose.GetDBVersion(sqlDB)
if err != nil {
log.Fatalf("[%s] get migrated goose version failed: %v\n", dbType(), err)
}
log.Printf("[%s] goose migrate success\n", dbType())
return Report{
Backend: dbType(),
Enabled: true,
Version: currentVersion,
Applied: currentVersion != previousVersion,
}
}
func clearSystemConfigCache() {
if err := repository.InvalidateAllSystemConfigCaches(context.Background()); err != nil {
log.Printf("[%s] clear system config cache failed: %v\n", dbType(), err)
}
}
@@ -0,0 +1,132 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"context"
"testing"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/alicebob/miniredis/v2"
"github.com/glebarez/sqlite"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
"gorm.io/gorm"
)
const expectedMigratedSystemConfigCount = 30
func TestMigrateInitializesSQLiteDatabase(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
if err != nil {
t.Fatalf("gorm.Open(sqlite) error = %v", err)
}
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("miniredis.Run() error = %v", err)
}
redisClient := redis.NewClient(&redis.Options{
Addr: mr.Addr(),
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeDisabled,
},
})
previousDBEnabled := config.Config.Database.Enabled
config.Config.Database.Enabled = false
db.SetDB(sqliteDB)
t.Cleanup(func() {
config.Config.Database.Enabled = previousDBEnabled
db.SetDB(nil)
_ = redisClient.Close()
mr.Close()
})
Migrate()
var systemConfigCount int64
if err := sqliteDB.Table("w_system_configs").Count(&systemConfigCount).Error; err != nil {
t.Fatalf("Migrate() count w_system_configs error = %v", err)
}
if systemConfigCount != expectedMigratedSystemConfigCount {
t.Errorf("Migrate() w_system_configs count = %d, want %d", systemConfigCount, expectedMigratedSystemConfigCount)
}
var adminCount int64
if err := sqliteDB.Table("w_users").Where("username = ?", "admin").Count(&adminCount).Error; err != nil {
t.Fatalf("Migrate() count admin user error = %v", err)
}
if adminCount != 1 {
t.Errorf("Migrate() admin user count = %d, want %d", adminCount, 1)
}
var templateCount int64
if err := sqliteDB.Table("w_templates").Count(&templateCount).Error; err != nil {
t.Fatalf("Migrate() count templates error = %v", err)
}
if templateCount != 2 {
t.Errorf("Migrate() templates count = %d, want %d", templateCount, 2)
}
}
func TestMigrateClearsStaleSystemConfigCache(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
if err != nil {
t.Fatalf("gorm.Open(sqlite) error = %v", err)
}
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("miniredis.Run() error = %v", err)
}
redisClient := redis.NewClient(&redis.Options{Addr: mr.Addr()})
previousDBEnabled := config.Config.Database.Enabled
previousRedis := db.Redis
config.Config.Database.Enabled = false
db.SetDB(sqliteDB)
db.Redis = redisClient
t.Cleanup(func() {
config.Config.Database.Enabled = previousDBEnabled
db.SetDB(nil)
db.Redis = previousRedis
_ = redisClient.Close()
mr.Close()
})
staleConfig := model.SystemConfig{
Key: model.ConfigKeyCapLoginEnabled,
Value: "true",
Type: "system",
}
if err := db.HSetJSON(context.Background(), repository.SystemConfigRedisHashKey, model.ConfigKeyCapLoginEnabled, &staleConfig); err != nil {
t.Fatalf("HSetJSON() error = %v", err)
}
Migrate()
exists, err := db.Redis.Exists(context.Background(), db.PrefixedKey(repository.SystemConfigRedisHashKey)).Result()
if err != nil {
t.Fatalf("Redis.Exists() error = %v", err)
}
if exists != 0 {
t.Fatalf("system config cache exists = %d, want 0", exists)
}
enabled, err := repository.GetBoolByKey(context.Background(), model.ConfigKeyCapLoginEnabled)
if err != nil {
t.Fatalf("GetBoolByKey(%s) error = %v", model.ConfigKeyCapLoginEnabled, err)
}
if enabled {
t.Fatalf("GetBoolByKey(%s) = true, want false", model.ConfigKeyCapLoginEnabled)
}
}
+217
View File
@@ -0,0 +1,217 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"log"
"net"
"net/url"
"strconv"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/glebarez/sqlite"
"go.opentelemetry.io/otel/attribute"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/plugin/dbresolver"
"gorm.io/plugin/opentelemetry/tracing"
)
var (
db *gorm.DB
)
func init() {
if !config.Config.Database.Enabled {
// PostgreSQL 禁用,使用 SQLite
initSQLite()
return
}
initPostgres()
}
// initSQLite 初始化 SQLite 数据库(PostgreSQL 禁用时的后备方案)
func initSQLite() {
sqlitePath := config.Config.Database.SQLitePath
if sqlitePath == "" {
sqlitePath = "./data/wavelet.db"
}
var err error
db, err = gorm.Open(sqlite.Open(sqlitePath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: &gormZapLogger{
logLevel: parseLogLevel(config.Config.Database.LogLevel),
slowThreshold: config.Config.Database.SlowThreshold,
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
},
})
if err != nil {
log.Fatalf("[SQLite] init connection failed: %v\n", err)
}
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", sqlitePath),
attribute.String("db.system", "SQLite"),
),
),
); err != nil {
log.Fatalf("[SQLite] init trace failed: %v\n", err)
}
log.Printf("[SQLite] initialized (path: %s)\n", sqlitePath)
}
// initPostgres 初始化 PostgreSQL 数据库
func initPostgres() {
var err error
dbConfig := config.Config.Database
// 构建主库 DSN 并连接
primaryDSN := buildDSN(dbConfig.Host, dbConfig.Port, dbConfig.Username, dbConfig.Password)
pgConfig := postgres.Config{
DSN: primaryDSN,
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
}
db, err = gorm.Open(postgres.New(pgConfig), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: &gormZapLogger{
logLevel: parseLogLevel(config.Config.Database.LogLevel),
slowThreshold: config.Config.Database.SlowThreshold,
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
},
})
if err != nil {
log.Fatalf("[PostgreSQL] init connection failed: %v\n", err)
}
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", dbConfig.Database),
attribute.String("db.ip", dbConfig.Host),
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
attribute.String("db.system", "PostgreSQL"),
),
),
); err != nil {
log.Fatalf("[PostgreSQL] init trace failed: %v\n", err)
}
if len(dbConfig.Replicas) > 0 {
var replicaDialectors []gorm.Dialector
for _, replica := range dbConfig.Replicas {
username := replica.Username
if username == "" {
username = dbConfig.Username
}
password := replica.Password
if password == "" {
password = dbConfig.Password
}
replicaDSN := buildDSN(replica.Host, replica.Port, username, password)
replicaDialectors = append(replicaDialectors, postgres.New(postgres.Config{
DSN: replicaDSN,
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
}))
}
resolver := dbresolver.Register(dbresolver.Config{
Replicas: replicaDialectors,
Policy: dbresolver.RandomPolicy{},
})
resolver.SetMaxIdleConns(dbConfig.MaxIdleConn).
SetMaxOpenConns(dbConfig.MaxOpenConn).
SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second).
SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
if err = db.Use(resolver); err != nil {
log.Fatalf("[PostgreSQL] init dbresolver failed: %v\n", err)
}
log.Printf("[PostgreSQL] initialized in Primary-Replica mode (%d replicas)\n", len(dbConfig.Replicas))
} else {
log.Println("[PostgreSQL] initialized in Standalone mode")
}
// 获取通用数据库对象设置连接池
sqlDB, err := db.DB()
if err != nil {
log.Fatalf("[PostgreSQL] load sql db failed: %v\n", err)
}
sqlDB.SetMaxIdleConns(dbConfig.MaxIdleConn)
sqlDB.SetMaxOpenConns(dbConfig.MaxOpenConn)
sqlDB.SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second)
sqlDB.SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
}
// buildDSN 构建 PostgreSQL DSN
func buildDSN(host string, port int, username, password string) string {
cfg := config.Config.Database
pqURL := &url.URL{
Scheme: "postgres",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
Path: cfg.Database,
}
if username != "" {
pqURL.User = url.UserPassword(username, password)
}
query := pqURL.Query()
sslMode := cfg.SSLMode
if sslMode == "" {
sslMode = "disable"
}
query.Set("sslmode", sslMode)
if cfg.ApplicationName != "" {
query.Set("application_name", cfg.ApplicationName)
}
if cfg.SearchPath != "" {
query.Set("search_path", cfg.SearchPath)
}
if cfg.DefaultQueryExecMode != "" {
query.Set("default_query_exec_mode", cfg.DefaultQueryExecMode)
}
if cfg.StatementCacheCapacity > 0 {
query.Set("statement_cache_capacity", strconv.Itoa(cfg.StatementCacheCapacity))
}
rawQuery := query.Encode()
if cfg.TimeZone != "" {
if rawQuery != "" {
rawQuery += "&"
}
rawQuery += "TimeZone=" + cfg.TimeZone
}
pqURL.RawQuery = rawQuery
return pqURL.String()
}
// DB 返回带上下文追踪的 GORM 数据库实例
func DB(ctx context.Context) *gorm.DB {
if db == nil {
return nil
}
return db.WithContext(ctx)
}
// SetDB sets the package-level database instance for testing.
func SetDB(d *gorm.DB) {
db = d
}
@@ -0,0 +1,91 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
)
// nanoToMilli 纳秒转毫秒的除数
const nanoToMilli = 1e6
type gormZapLogger struct {
logLevel gormLogger.LogLevel
ignoreRecordNotFoundError bool
slowThreshold time.Duration
}
func (l *gormZapLogger) LogMode(level gormLogger.LogLevel) gormLogger.Interface {
clone := *l
clone.logLevel = level
return &clone
}
func (l *gormZapLogger) Info(ctx context.Context, fmt string, args ...interface{}) {
if l.logLevel >= gormLogger.Info {
logger.InfoF(ctx, fmt, args...)
}
}
func (l *gormZapLogger) Warn(ctx context.Context, fmt string, args ...interface{}) {
if l.logLevel >= gormLogger.Warn {
logger.WarnF(ctx, fmt, args...)
}
}
func (l *gormZapLogger) Error(ctx context.Context, fmt string, args ...interface{}) {
if l.logLevel >= gormLogger.Error {
logger.ErrorF(ctx, fmt, args...)
}
}
func (l *gormZapLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
elapsed := time.Since(begin)
switch {
case err != nil && l.logLevel >= gormLogger.Error && (!errors.Is(err, gorm.ErrRecordNotFound) || !l.ignoreRecordNotFoundError):
_, rows := fc()
logger.ErrorF(ctx, "database query failed: %s [%.3fms] [rows:%v]", err, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
case elapsed > l.slowThreshold && l.slowThreshold != 0 && l.logLevel >= gormLogger.Warn:
_, rows := fc()
slowLog := fmt.Sprintf("SLOW SQL >= %v", l.slowThreshold)
logger.WarnF(ctx, "%s [%.3fms] [rows:%v]", slowLog, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
case l.logLevel == gormLogger.Info:
sql, rows := fc()
logger.DebugF(ctx, "[%.3fms] [rows:%v] %s", float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows), sql)
}
}
func formatRows(rows int64) interface{} {
if rows == -1 {
return "-"
}
return rows
}
func parseLogLevel(level string) gormLogger.LogLevel {
level = strings.ToLower(level)
switch level {
case "silent":
return gormLogger.Silent
case "error":
return gormLogger.Error
case "warn":
return gormLogger.Warn
case "info":
return gormLogger.Info
case "debug":
return gormLogger.Info
default:
return gormLogger.Info
}
}
@@ -0,0 +1,39 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"testing"
gormLogger "gorm.io/gorm/logger"
)
func TestParseLogLevel(t *testing.T) {
t.Parallel()
tests := []struct {
name string
configuredLevel string
want gormLogger.LogLevel
}{
{
name: "debug enables SQL trace processing",
configuredLevel: "debug",
want: gormLogger.Info,
},
{
name: "development preserves configured level",
configuredLevel: "warn",
want: gormLogger.Warn,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := parseLogLevel(tt.configuredLevel); got != tt.want {
t.Fatalf("parseLogLevel() = %v, want %v", got, tt.want)
}
})
}
}
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/redis/go-redis/extra/redisotel/v9"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
"go.opentelemetry.io/otel/attribute"
)
var (
// Redis 全局 Redis 客户端实例
Redis redis.UniversalClient
)
func init() {
cfg := config.Config.Redis
if !cfg.Enabled {
log.Println("[Redis] is disabled, skipping Redis initialization")
return
}
if cfg.ClusterMode {
// Cluster 模式
Redis = redis.NewClusterClient(&redis.ClusterOptions{
Addrs: cfg.Addrs,
Username: cfg.Username,
Password: cfg.Password,
PoolSize: cfg.PoolSize,
MinIdleConns: cfg.MinIdleConn,
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
MaxRetries: cfg.MaxRetries,
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
})
log.Println("[Redis] initialized in Cluster mode")
} else {
// Standalone 或 Sentinel 模式
options := &redis.UniversalOptions{
Addrs: cfg.Addrs,
MasterName: cfg.MasterName, // 非空时启用 Sentinel
Username: cfg.Username,
Password: cfg.Password,
DB: cfg.DB,
PoolSize: cfg.PoolSize,
MinIdleConns: cfg.MinIdleConn,
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
MaxRetries: cfg.MaxRetries,
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
}
if cfg.MasterName != "" {
client := redis.NewFailoverClient(options.Failover())
// FailoverOptions 暂不暴露该配置,在首次建连前写入客户端选项。
client.Options().MaintNotificationsConfig = redisMaintNotificationsConfig(cfg.MaintNotifications)
Redis = client
log.Println("[Redis] initialized in Sentinel mode")
} else {
Redis = redis.NewUniversalClient(options)
log.Println("[Redis] initialized in Standalone mode")
}
}
// OpenTelemetry 追踪(UniversalClient 兼容)
if err := redisotel.InstrumentTracing(
Redis,
redisotel.WithAttributes(
attribute.String("db.instance", fmt.Sprintf("%v", cfg.DB)),
attribute.String("db.ip", strings.Join(cfg.Addrs, ",")),
attribute.String("db.system", "Redis"),
),
); err != nil {
log.Fatalf("[Redis] failed to init trace: %v\n", err)
}
// 测试连接
_, err := Redis.Ping(context.Background()).Result()
if err != nil {
log.Fatalf("[Redis] failed to connect to redis: %v\n", err)
}
}
func redisMaintNotificationsConfig(enabled bool) *maintnotifications.Config {
mode := maintnotifications.ModeDisabled
if enabled {
mode = maintnotifications.ModeAuto
}
return &maintnotifications.Config{Mode: mode}
}
// PrefixedKey 返回带前缀的 Key
func PrefixedKey(key string) string {
prefix := config.Config.Redis.KeyPrefix
if prefix == "" {
return key
}
return prefix + key
}
// HSetJSON 将泛型数据序列化为 JSON 并设置到 Redis Hash
// ctx: 上下文
// hashKey: Redis Hash key
// fieldKey: Hash field key
// data: 要存储的数据(泛型)
func HSetJSON[T any](ctx context.Context, hashKey, fieldKey string, data T) error {
jsonData, err := json.Marshal(data)
if err != nil {
return err
}
if err := Redis.HSet(ctx, PrefixedKey(hashKey), fieldKey, jsonData).Err(); err != nil {
return fmt.Errorf(errRedisHashSetFailed, err)
}
return nil
}
// HDel removes one or more fields from a Redis Hash.
func HDel(ctx context.Context, hashKey string, fieldKeys ...string) error {
if Redis == nil || len(fieldKeys) == 0 {
return nil
}
if err := Redis.HDel(ctx, PrefixedKey(hashKey), fieldKeys...).Err(); err != nil {
return fmt.Errorf(errRedisHashDeleteFailed, err)
}
return nil
}
// HGetJSON 从 Redis Hash 获取数据并反序列化为泛型类型
// ctx: 上下文
// hashKey: Redis Hash key
// fieldKey: Hash field key
// data: 用于接收数据的指针(泛型)
func HGetJSON[T any](ctx context.Context, hashKey, fieldKey string, data *T) error {
val, err := Redis.HGet(ctx, PrefixedKey(hashKey), fieldKey).Result()
if err != nil {
return err
}
if err := json.Unmarshal([]byte(val), data); err != nil {
return fmt.Errorf(errUnmarshalDataFailed, err)
}
return nil
}
// GetJSON 从Redis获取数据并反序列化为泛型类型
// ctx: 上下文
// key: Redis key
// data: 用于接收数据的指针(泛型)
func GetJSON[T any](ctx context.Context, key string, data *T) error {
val, err := Redis.Get(ctx, PrefixedKey(key)).Bytes()
if err != nil {
return err
}
if err := json.Unmarshal(val, data); err != nil {
return fmt.Errorf(errUnmarshalDataFailed, err)
}
return nil
}
// SetJSON 将泛型数据序列化为JSON并设置到Redis
// ctx: 上下文
// key: Redis key
// data: 要存储的数据(泛型)
// expiration: 过期时间
func SetJSON[T any](ctx context.Context, key string, data T, expiration time.Duration) error {
jsonData, err := json.Marshal(data)
if err != nil {
return fmt.Errorf(errMarshalDataFailed, err)
}
if err := Redis.Set(ctx, PrefixedKey(key), jsonData, expiration).Err(); err != nil {
return fmt.Errorf(errRedisKeySetFailed, err)
}
return nil
}
+25
View File
@@ -0,0 +1,25 @@
package db
import (
"testing"
"github.com/redis/go-redis/v9/maintnotifications"
)
func TestRedisMaintNotificationsConfig(t *testing.T) {
for _, test := range []struct {
name string
enabled bool
want maintnotifications.Mode
}{
{name: "disabled by default", enabled: false, want: maintnotifications.ModeDisabled},
{name: "auto when enabled", enabled: true, want: maintnotifications.ModeAuto},
} {
t.Run(test.name, func(t *testing.T) {
cfg := redisMaintNotificationsConfig(test.enabled)
if cfg.Mode != test.want {
t.Fatalf("maintenance notifications mode = %v, want %v", cfg.Mode, test.want)
}
})
}
}
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package task 定义异步任务类型与调度常量
package task
// 任务队列名称
const (
QueueDefault = "default"
)
// DefaultMaxRetry 任务默认最大重试次数
const DefaultMaxRetry = 3
+17
View File
@@ -0,0 +1,17 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
const (
errUnknownTaskType = "未知的任务类型: %s"
errCreateTaskExecutionFailed = "创建任务执行记录失败: %w"
errTaskEnqueueFailed = "任务入队失败: %w"
errTaskExecutionNotFound = "任务执行记录不存在: %w"
errRetryOnlyFailedTask = "只有失败的任务才能重试,当前状态: %s"
errTaskNotRetryable = "该任务不支持重试"
errTaskMaxRetryExceeded = "已达到最大重试次数 %d"
errCreateRetryExecutionFailed = "创建重试任务执行记录失败: %w"
errRetryTaskEnqueueFailed = "重试任务入队失败: %w"
errUnregisteredTaskHandler = "未注册的任务处理器: %s"
)
+479
View File
@@ -0,0 +1,479 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
otel_trace "github.com/Rain-kl/Wavelet/pkg/trace"
"github.com/hibiken/asynq"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
// handlerRegistry 已注册的任务处理器
var handlerRegistry = make(map[string]TaskHandler)
// CompletedHandler is called when a task execution completes.
type CompletedHandler func(ctx context.Context, execution *model.TaskExecution, result *TaskResult, execErr error)
var taskCompletedHandlers []CompletedHandler
// OnTaskCompleted registers a handler for task completion events.
// Handlers must be registered during application bootstrap before processing tasks.
func OnTaskCompleted(handler CompletedHandler) {
taskCompletedHandlers = append(taskCompletedHandlers, handler)
}
// RegisterHandler 注册任务处理器
// 传入任务类型标识(对应 constants.go 中的 AsynqTask 常量)和 TaskHandler 实现
func RegisterHandler(asynqTaskType string, handler TaskHandler) {
handlerRegistry[asynqTaskType] = handler
}
// getHandler 获取已注册的处理器
func getHandler(asynqTaskType string) (TaskHandler, bool) {
h, ok := handlerRegistry[asynqTaskType]
return h, ok
}
// ValidateAndNormalizePayload 校验并标准化任务参数。
// 如果 Handler 实现了 PayloadValidator,调用其 ValidatePayload 方法;
// 否则直接返回原始 payload。
func ValidateAndNormalizePayload(asynqTaskType string, payload []byte) ([]byte, error) {
handler, ok := getHandler(asynqTaskType)
if !ok {
return payload, nil
}
if validator, ok := handler.(PayloadValidator); ok {
return validator.ValidatePayload(payload)
}
return payload, nil
}
// contextKey 用于 context 存取 taskID
type contextKey string
const taskIDKey contextKey = "task_execution_task_id"
const traceEnvelopeVersion = 1
type traceEnvelope struct {
WaveletTraceEnvelope bool `json:"_wavelet_trace_envelope"`
Version int `json:"version"`
TraceContext map[string]string `json:"trace_context,omitempty"`
Payload []byte `json:"payload"`
}
// withTaskID 将 taskID 注入 context
func withTaskID(ctx context.Context, taskID string) context.Context {
return context.WithValue(ctx, taskIDKey, taskID)
}
// GetTaskID 从 context 中获取 taskID
func GetTaskID(ctx context.Context) string {
if v, ok := ctx.Value(taskIDKey).(string); ok {
return v
}
return ""
}
// IsFinalAttempt 判断当前任务执行是否为最后一次重试尝试(若再次失败即为最终失败)
func IsFinalAttempt(ctx context.Context) bool {
retryCount, hasRetryCount := asynq.GetRetryCount(ctx)
maxRetry, hasMaxRetry := asynq.GetMaxRetry(ctx)
if !hasRetryCount || !hasMaxRetry {
return true
}
return retryCount >= maxRetry
}
// AppendLog 追加日志到任务执行记录
// 在 TaskHandler.Execute 中调用,日志会自动追加到 TaskExecution.Log 字段
func AppendLog(ctx context.Context, format string, args ...interface{}) {
taskID := GetTaskID(ctx)
if taskID == "" {
// 上下文中没有 taskID,降级到普通日志
logger.InfoF(ctx, format, args...)
return
}
logLine := fmt.Sprintf(format, args...)
if err := model.AppendTaskExecutionLog(ctx, taskID, logLine); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 追加任务日志失败 taskID=%s: %v", taskID, err)
}
}
// DispatchTask 下发任务(创建 TaskExecution 记录 → 入队 Asynq)
func DispatchTask(ctx context.Context, taskType string, payload []byte, triggeredBy string) (string, error) {
meta := GetTaskMeta(taskType)
if meta == nil {
return "", fmt.Errorf(errUnknownTaskType, taskType)
}
// 生成唯一的 TaskID
taskID := generateTaskID(taskType, triggeredBy)
// 创建任务执行记录
execution := &model.TaskExecution{
TaskID: taskID,
TaskType: meta.AsynqTask,
TaskName: meta.Name,
Status: model.TaskExecutionStatusPending,
Retryable: meta.Retryable,
MaxRetry: meta.MaxRetry,
RetryCount: 0,
Payload: string(payload),
TriggeredBy: triggeredBy,
}
if err := model.CreateTaskExecution(ctx, execution); err != nil {
return "", fmt.Errorf(errCreateTaskExecutionFailed, err)
}
// 入队 Asynq
taskInfo := asynq.NewTask(meta.AsynqTask, injectTaskTraceContext(ctx, payload))
if _, err := AsynqClient.Enqueue(
taskInfo,
asynq.TaskID(taskID),
asynq.MaxRetry(meta.MaxRetry),
asynq.Queue(meta.Queue),
); err != nil {
// 入队失败,更新执行记录状态
execution.Status = model.TaskExecutionStatusFailed
execution.ErrorMessage = fmt.Sprintf("入队失败: %v", err)
now := time.Now()
execution.StartedAt = &now
execution.FinishedAt = &now
_ = model.UpdateTaskExecution(ctx, execution)
return "", fmt.Errorf(errTaskEnqueueFailed, err)
}
if err := model.AppendTaskExecutionLog(ctx, taskID, fmt.Sprintf("[系统] 任务已成功入队,等待调度执行 (队列: %s, 最大重试次数: %d)", meta.Queue, meta.MaxRetry)); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 追加入队日志失败 taskID=%s: %v", taskID, err)
}
return taskID, nil
}
// RetryTask 重试失败的任务
func RetryTask(ctx context.Context, id uint64) (string, error) {
execution, err := model.GetTaskExecutionByID(ctx, id)
if err != nil {
return "", fmt.Errorf(errTaskExecutionNotFound, err)
}
if execution.Status != model.TaskExecutionStatusFailed {
return "", fmt.Errorf(errRetryOnlyFailedTask, execution.Status)
}
if !execution.Retryable {
return "", errors.New(errTaskNotRetryable)
}
// 生成新的 TaskID
newTaskID := generateRetryTaskID(execution.TaskID, execution.RetryCount+1)
// 创建新的执行记录
newExecution := &model.TaskExecution{
TaskID: newTaskID,
TaskType: execution.TaskType,
TaskName: execution.TaskName,
Status: model.TaskExecutionStatusPending,
Retryable: execution.Retryable,
MaxRetry: execution.MaxRetry,
RetryCount: execution.RetryCount + 1,
Payload: execution.Payload,
TriggeredBy: "retry",
}
if err := model.CreateTaskExecution(ctx, newExecution); err != nil {
return "", fmt.Errorf(errCreateRetryExecutionFailed, err)
}
meta := GetTaskMeta(execution.TaskType)
queueName := QueueDefault
if meta != nil {
queueName = meta.Queue
}
// 入队 Asynq
taskInfo := asynq.NewTask(execution.TaskType, injectTaskTraceContext(ctx, []byte(execution.Payload)))
if _, err := AsynqClient.Enqueue(
taskInfo,
asynq.TaskID(newTaskID),
asynq.MaxRetry(execution.MaxRetry),
asynq.Queue(queueName),
); err != nil {
newExecution.Status = model.TaskExecutionStatusFailed
newExecution.ErrorMessage = fmt.Sprintf("重试入队失败: %v", err)
now := time.Now()
newExecution.StartedAt = &now
newExecution.FinishedAt = &now
_ = model.UpdateTaskExecution(ctx, newExecution)
return "", fmt.Errorf(errRetryTaskEnqueueFailed, err)
}
if err := model.AppendTaskExecutionLog(ctx, newTaskID, fmt.Sprintf("[系统] 手动触发重试,已重新创建任务并入队 (原任务ID: %s, 重试次数: %d/%d)", execution.TaskID, execution.RetryCount+1, execution.MaxRetry)); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 追加重试日志失败 taskID=%s: %v", newTaskID, err)
}
return newTaskID, nil
}
// ProcessTask Asynq 实际调用的统一处理函数
// Worker 注册时统一使用此函数,内部自动分发到对应的 TaskHandler
func ProcessTask(ctx context.Context, t *asynq.Task) error {
taskPayload := t.Payload()
ctx, taskPayload, hasRemoteTraceContext := extractTaskTraceContext(ctx, taskPayload)
// 初始化 Trace
ctx, span := otel_trace.Start(ctx, "TaskProcess_"+t.Type(), trace.WithSpanKind(trace.SpanKindConsumer))
defer span.End()
// 添加任务信息到 Span
span.SetAttributes(
attribute.String("task.type", t.Type()),
attribute.Int("task.payload_size", len(taskPayload)),
attribute.Bool("task.trace_context_propagated", hasRemoteTraceContext),
attribute.String("task.id", t.ResultWriter().TaskID()),
)
taskID := t.ResultWriter().TaskID()
// 注入 taskID 到 context
ctx = withTaskID(ctx, taskID)
// 查找处理器
handler, ok := getHandler(t.Type())
if !ok {
err := fmt.Errorf(errUnregisteredTaskHandler, t.Type())
logger.ErrorF(ctx, "[TaskExecutor] %v", err)
span.SetStatus(codes.Error, err.Error())
return err
}
// 加载或动态创建执行记录
now := time.Now()
execution, err := getOrCreateTaskExecution(ctx, taskID, t, taskPayload, now)
if err == nil {
updateExecutionOnStart(ctx, execution, now)
}
if execution != nil {
AppendLog(ctx, "[系统] 开始执行异步任务 [名称: %s, 类型: %s],重试次数: %d/%d",
execution.TaskName, t.Type(), execution.RetryCount, execution.MaxRetry)
} else {
AppendLog(ctx, "[系统] 开始执行异步任务 [类型: %s]", t.Type())
}
// 开始计时
start := time.Now()
// 执行业务逻辑
result, execErr := handler.Execute(ctx, taskPayload)
// 计算耗时并归档记录
duration := time.Since(start)
finishTime := time.Now()
completeTaskExecution(ctx, execution, t, duration, finishTime, result, execErr, span)
if execution == nil && execErr != nil {
span.SetStatus(codes.Error, execErr.Error())
return execErr
}
return execErr
}
func injectTaskTraceContext(ctx context.Context, payload []byte) []byte {
carrier := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(ctx, carrier)
if len(carrier) == 0 {
return payload
}
envelope := traceEnvelope{
WaveletTraceEnvelope: true,
Version: traceEnvelopeVersion,
TraceContext: map[string]string(carrier),
Payload: payload,
}
data, err := json.Marshal(envelope)
if err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 序列化任务 Trace 上下文失败: %v", err)
return payload
}
return data
}
func extractTaskTraceContext(ctx context.Context, payload []byte) (context.Context, []byte, bool) {
var envelope traceEnvelope
if err := json.Unmarshal(payload, &envelope); err != nil {
return ctx, payload, false
}
if !envelope.WaveletTraceEnvelope || envelope.Version != traceEnvelopeVersion {
return ctx, payload, false
}
if len(envelope.TraceContext) == 0 {
return ctx, envelope.Payload, false
}
extractedCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(envelope.TraceContext))
return extractedCtx, envelope.Payload, true
}
func updateExecutionOnStart(ctx context.Context, execution *model.TaskExecution, now time.Time) {
if execution == nil {
return
}
dirty := false
if retryCount, hasRetry := asynq.GetRetryCount(ctx); hasRetry && execution.RetryCount != retryCount {
execution.RetryCount = retryCount
dirty = true
}
if execution.Status != model.TaskExecutionStatusRunning {
execution.Status = model.TaskExecutionStatusRunning
execution.StartedAt = &now
dirty = true
}
if dirty {
if updateErr := model.UpdateTaskExecution(ctx, execution); updateErr != nil {
logger.ErrorF(ctx, "[TaskExecutor] 更新执行状态失败 taskID=%s: %v", execution.TaskID, updateErr)
}
}
}
// getOrCreateTaskExecution 获取已有的任务执行记录,如果不存在则针对已知任务类型动态创建记录
func getOrCreateTaskExecution(ctx context.Context, taskID string, t *asynq.Task, payload []byte, now time.Time) (*model.TaskExecution, error) {
execution, err := model.GetTaskExecutionByTaskID(ctx, taskID)
if err == nil {
return execution, nil
}
meta := GetTaskMetaByAsynqTask(t.Type())
if meta == nil {
return nil, err
}
execution = &model.TaskExecution{
TaskID: taskID,
TaskType: meta.AsynqTask,
TaskName: meta.Name,
Status: model.TaskExecutionStatusRunning,
Retryable: meta.Retryable,
MaxRetry: meta.MaxRetry,
RetryCount: 0,
Payload: string(payload),
TriggeredBy: "schedule",
StartedAt: &now,
}
if createErr := model.CreateTaskExecution(ctx, execution); createErr != nil {
logger.ErrorF(ctx, "[TaskExecutor] 动态创建执行记录失败 taskID=%s: %v", taskID, createErr)
return nil, createErr
}
return execution, nil
}
// completeTaskExecution 完成并更新任务执行记录的状态和执行结果
func completeTaskExecution(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, finishTime time.Time, result *TaskResult, execErr error, span trace.Span) {
if execution == nil {
return
}
execution.Duration = duration.Milliseconds()
execution.FinishedAt = &finishTime
if execErr != nil {
handleFailedTask(ctx, execution, t, duration, execErr, span)
} else {
handleSuccessfulTask(ctx, execution, t, duration, result)
}
if err := model.UpdateTaskExecution(ctx, execution); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 更新执行记录失败 taskID=%s: %v", execution.TaskID, err)
}
if shouldFlushTaskExecutionLog(ctx, execErr) {
if err := model.FlushTaskExecutionLog(ctx, execution.TaskID); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 持久化任务日志失败 taskID=%s: %v", execution.TaskID, err)
}
}
notifyTaskCompleted(ctx, execution, result, execErr)
}
func notifyTaskCompleted(ctx context.Context, execution *model.TaskExecution, result *TaskResult, execErr error) {
if len(taskCompletedHandlers) == 0 {
return
}
asyncCtx := context.WithoutCancel(ctx)
for _, handler := range taskCompletedHandlers {
go handler(asyncCtx, execution, result, execErr)
}
}
func shouldFlushTaskExecutionLog(ctx context.Context, execErr error) bool {
if execErr == nil {
return true
}
retryCount, hasRetryCount := asynq.GetRetryCount(ctx)
maxRetry, hasMaxRetry := asynq.GetMaxRetry(ctx)
if !hasRetryCount || !hasMaxRetry {
return true
}
return retryCount >= maxRetry
}
func handleFailedTask(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, execErr error, span trace.Span) {
execution.Status = model.TaskExecutionStatusFailed
execution.ErrorMessage = execErr.Error()
logger.ErrorF(ctx, "[TaskExecutor] 任务处理失败 Type: %s TaskID: %s Duration: %d ms Error: %v", t.Type(), execution.TaskID, duration.Milliseconds(), execErr)
span.SetStatus(codes.Error, execErr.Error())
span.RecordError(execErr)
AppendLog(ctx, "[系统] 任务执行失败,耗时: %d ms,错误原因: %v", duration.Milliseconds(), execErr)
}
func handleSuccessfulTask(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, result *TaskResult) {
execution.Status = model.TaskExecutionStatusSucceeded
execution.ErrorMessage = "" // 清除历史重试失败遗留的错误信息
if result != nil {
execution.Result = result.Message
if result.Detail != "" {
execution.Result = fmt.Sprintf("%s\n%s", result.Message, result.Detail)
}
}
logger.InfoF(ctx, "[TaskExecutor] 任务处理完成 Type: %s TaskID: %s Duration: %d ms", t.Type(), execution.TaskID, duration.Milliseconds())
resultMsg := "成功"
if result != nil {
resultMsg = result.Message
}
AppendLog(ctx, "[系统] 任务执行成功,耗时: %d ms,执行结果: %s", duration.Milliseconds(), resultMsg)
}
// generateTaskID 生成任务 ID
func generateTaskID(taskType string, triggeredBy string) string {
return fmt.Sprintf("%s_%s_%d", triggeredBy, taskType, idgen.NextUint64ID())
}
// generateRetryTaskID 生成重试任务 ID
func generateRetryTaskID(originalTaskID string, retryCount int) string {
return fmt.Sprintf("retry_%d_%s", retryCount, originalTaskID)
}
+401
View File
@@ -0,0 +1,401 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"fmt"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
// mockHandler 用于测试的模拟任务处理器
type mockHandler struct {
executeFunc func(ctx context.Context, payload []byte) (*TaskResult, error)
}
func (h *mockHandler) Execute(ctx context.Context, payload []byte) (*TaskResult, error) {
if h.executeFunc != nil {
return h.executeFunc(ctx, payload)
}
return &TaskResult{Message: "mock success"}, nil
}
// successHandler 返回成功的处理器
func successHandler() *mockHandler {
return &mockHandler{
executeFunc: func(ctx context.Context, payload []byte) (*TaskResult, error) {
AppendLog(ctx, "执行成功,处理了 %d 条数据", 100)
return &TaskResult{Message: "处理完成,共 100 条"}, nil
},
}
}
// failHandler 返回失败的处理器
func failHandler() *mockHandler {
return &mockHandler{
executeFunc: func(ctx context.Context, payload []byte) (*TaskResult, error) {
AppendLog(ctx, "开始执行任务")
return nil, fmt.Errorf("模拟执行失败: 数据库连接超时")
},
}
}
const testTaskType = "test:mock_task"
func setupTest(t *testing.T) func() {
_, mr, cleanup := testhelper.SetupTestEnvironment(t)
AsynqClient = asynq.NewClient(asynq.RedisClientOpt{
Addr: mr.Addr(),
})
// 注册测试用 handler
RegisterHandler(testTaskType, successHandler())
return func() {
if AsynqClient != nil {
_ = AsynqClient.Close()
AsynqClient = nil
}
cleanup()
}
}
func TestRegisterAndGetHandler(t *testing.T) {
_ = testTaskType
cleanup := setupTest(t)
defer cleanup()
// 验证 handler 已注册
h, ok := getHandler(testTaskType)
assert.True(t, ok, "handler should be registered")
assert.NotNil(t, h)
// 未注册的 handler
_, ok = getHandler("nonexistent")
assert.False(t, ok, "non-existent handler should return false")
}
func TestGetTaskIDFromContext(t *testing.T) {
ctx := context.Background()
// 空 context
taskID := GetTaskID(ctx)
assert.Equal(t, "", taskID)
// 注入 taskID
ctx = withTaskID(ctx, "test_task_123")
taskID = GetTaskID(ctx)
assert.Equal(t, "test_task_123", taskID)
}
func TestAppendLogWithoutTaskID(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 没有 taskID 的 context,应降级到普通日志,不报错
AppendLog(ctx, "这条日志应该降级处理,不会报错")
}
func TestAppendLogWithTaskID(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 先创建一条执行记录
execution := &model.TaskExecution{
TaskID: "log_test_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusRunning,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 注入 taskID 并追加日志
ctx = withTaskID(ctx, "log_test_001")
AppendLog(ctx, "第一条日志")
AppendLog(ctx, "处理了 %d 条数据", 50)
// 验证日志
found, err := model.GetTaskExecutionByTaskID(ctx, "log_test_001")
require.NoError(t, err)
assert.Contains(t, found.Log, "第一条日志")
assert.Contains(t, found.Log, "处理了 50 条数据")
}
func TestTaskTraceContextEnvelope(t *testing.T) {
payload := []byte(`{"hello":"wavelet"}`)
traceID := "4bf92f3577b34da6a3ce929d0e0e4736"
parentCtx := otel.GetTextMapPropagator().Extract(
context.Background(),
propagation.MapCarrier{
"traceparent": "00-" + traceID + "-00f067aa0ba902b7-01",
},
)
wrappedPayload := injectTaskTraceContext(parentCtx, payload)
require.NotEqual(t, string(payload), string(wrappedPayload))
gotCtx, gotPayload, ok := extractTaskTraceContext(context.Background(), wrappedPayload)
require.True(t, ok)
assert.Equal(t, payload, gotPayload)
assert.Equal(t, traceID, trace.SpanContextFromContext(gotCtx).TraceID().String())
}
func TestTaskTraceContextEnvelopeKeepsLegacyPayload(t *testing.T) {
payload := []byte(`{"legacy":true}`)
gotCtx, gotPayload, ok := extractTaskTraceContext(context.Background(), payload)
require.False(t, ok)
assert.Equal(t, context.Background(), gotCtx)
assert.Equal(t, payload, gotPayload)
}
func TestTaskTraceContextEnvelopeSkipsEmptyContext(t *testing.T) {
payload := []byte(`{"background":true}`)
wrappedPayload := injectTaskTraceContext(context.Background(), payload)
assert.Equal(t, payload, wrappedPayload)
}
func TestProcessTaskSuccess(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 注册成功 handler
RegisterHandler(testTaskType, successHandler())
// 创建执行记录
execution := &model.TaskExecution{
TaskID: "process_success_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusPending,
Retryable: true,
MaxRetry: 3,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 通过 asynq 的 Task 不能直接设置 taskID,ProcessTask 通过 t.ResultWriter().TaskID() 获取
// 但 asynq.Task 在没有经过 asynq server 的情况下 ResultWriter 可能为 nil
// 我们需要在 ProcessTask 内部改用 taskID 注入的方式测试
// 为了测试 ProcessTask,我们直接模拟调用 handler
// 直接通过 handler 测试
handler, ok := getHandler(testTaskType)
require.True(t, ok)
ctx = withTaskID(ctx, "process_success_001")
result, err := handler.Execute(ctx, nil)
require.NoError(t, err)
assert.Equal(t, "处理完成,共 100 条", result.Message)
// 验证日志被追加
found, err := model.GetTaskExecutionByTaskID(ctx, "process_success_001")
require.NoError(t, err)
assert.Contains(t, found.Log, "执行成功,处理了 100 条数据")
}
func TestProcessTaskFailure(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 注册失败 handler
RegisterHandler(testTaskType, failHandler())
// 创建执行记录
execution := &model.TaskExecution{
TaskID: "process_fail_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusPending,
Retryable: true,
MaxRetry: 3,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 直接调用 handler
handler, ok := getHandler(testTaskType)
require.True(t, ok)
ctx = withTaskID(ctx, "process_fail_001")
_, err = handler.Execute(ctx, nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "模拟执行失败")
// 验证日志
found, err := model.GetTaskExecutionByTaskID(ctx, "process_fail_001")
require.NoError(t, err)
assert.Contains(t, found.Log, "开始执行任务")
}
func TestCompleteTaskExecutionFlushesLog(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
execution := &model.TaskExecution{
TaskID: "complete_flush_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusRunning,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
ctx = withTaskID(ctx, execution.TaskID)
AppendLog(ctx, "任务执行中的日志")
finishTime := time.Now()
completeTaskExecution(
ctx,
execution,
asynq.NewTask(testTaskType, nil),
100*time.Millisecond,
finishTime,
&TaskResult{Message: "处理完成"},
nil,
trace.SpanFromContext(ctx),
)
found, err := model.GetTaskExecutionByTaskID(ctx, execution.TaskID)
require.NoError(t, err)
assert.Equal(t, model.TaskExecutionStatusSucceeded, found.Status)
assert.Contains(t, found.Log, "任务执行中的日志")
assert.Contains(t, found.Log, "任务执行成功")
}
func TestRetryTask(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 创建一条失败的执行记录(可重试)
now := time.Now()
execution := &model.TaskExecution{
TaskID: "retry_test_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusFailed,
Retryable: true,
MaxRetry: 3,
RetryCount: 0,
ErrorMessage: "首次执行失败",
StartedAt: &now,
FinishedAt: &now,
Duration: 100,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 重试
newTaskID, err := RetryTask(ctx, execution.ID)
require.NoError(t, err)
assert.NotEmpty(t, newTaskID)
assert.Contains(t, newTaskID, "retry_1_")
// 验证新记录
newExecution, err := model.GetTaskExecutionByTaskID(ctx, newTaskID)
require.NoError(t, err)
assert.Equal(t, model.TaskExecutionStatusPending, newExecution.Status)
assert.Equal(t, 1, newExecution.RetryCount)
assert.Equal(t, "retry", newExecution.TriggeredBy)
assert.Equal(t, execution.TaskType, newExecution.TaskType)
assert.True(t, newExecution.Retryable)
// 原记录不变
original, err := model.GetTaskExecutionByID(ctx, execution.ID)
require.NoError(t, err)
assert.Equal(t, model.TaskExecutionStatusFailed, original.Status)
assert.Equal(t, 0, original.RetryCount)
}
func TestRetryTaskNotFailed(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 创建一条成功的记录
execution := &model.TaskExecution{
TaskID: "retry_not_failed_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusSucceeded,
Retryable: true,
MaxRetry: 3,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 尝试重试成功的任务
_, err = RetryTask(ctx, execution.ID)
assert.Error(t, err)
assert.Contains(t, err.Error(), "只有失败的任务才能重试")
}
func TestRetryTaskNotRetryable(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
execution := &model.TaskExecution{
TaskID: "retry_not_allowed_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: model.TaskExecutionStatusFailed,
Retryable: false,
MaxRetry: 0,
TriggeredBy: "manual",
}
err := model.CreateTaskExecution(ctx, execution)
require.NoError(t, err)
_, err = RetryTask(ctx, execution.ID)
assert.Error(t, err)
assert.Contains(t, err.Error(), "不支持重试")
}
func TestRetryTaskNonExistent(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
_, err := RetryTask(ctx, 99999999)
assert.Error(t, err)
assert.Contains(t, err.Error(), "不存在")
}
func TestGenerateTaskID(t *testing.T) {
id1 := generateTaskID("test_type", "manual")
id2 := generateTaskID("test_type", "manual")
// 两个 ID 应不同(包含 Snowflake ID)
assert.NotEqual(t, id1, id2)
assert.Contains(t, id1, "manual_test_type_")
}
func TestGenerateRetryTaskID(t *testing.T) {
id := generateRetryTaskID("original_task_123", 2)
assert.Equal(t, "retry_2_original_task_123", id)
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import "context"
// TaskResult 任务执行结果
//
//nolint:revive // TaskResult 保留完整名称以避免与通用 Result 混淆
type TaskResult struct {
Message string // 结果摘要,如 "共清理 120 个文件,耗时 3.2s"
Detail string // 可选的详细结果 JSON
}
// PayloadValidator 可选接口,带参数的任务 Handler 应实现此接口。
// 框架在 Admin 下发时自动调用,完成参数校验和标准化(如 Trim 空白)。
// 无参数的任务无需实现,框架会直接透传 payload。
type PayloadValidator interface {
ValidatePayload(payload []byte) ([]byte, error)
}
// TaskHandler 异步任务处理器接口
// 所有异步任务必须实现此接口,框架将自动管理任务执行记录的创建、状态流转和日志写入。
//
// 开发者只需实现 Execute 方法编写业务逻辑,在方法内通过 task.AppendLog(ctx, ...) 追加执行日志。
// 任务的创建、状态更新、错误记录、重试计数全部由框架透明处理。
//
//nolint:revive // TaskHandler 保留完整名称以避免与通用 Handler 混淆
type TaskHandler interface {
// Execute 执行任务业务逻辑
// - ctx: 已注入 Trace Span 和 taskID 的上下文
// - payload: 调度时传入的原始参数(可为 nil)
// - 返回 TaskResult 描述执行结果,或 error 表示执行失败
Execute(ctx context.Context, payload []byte) (*TaskResult, error)
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package handlers 注册异步任务处理器
package handlers
import (
"github.com/Rain-kl/Wavelet/internal/apps/admin/push"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/apps/user"
"github.com/Rain-kl/Wavelet/internal/infra/task"
)
// Register registers all built-in task handlers and their metadata.
func Register() {
task.RegisterHandler(upload.StorageMigrationTask, &upload.MigrationHandler{})
task.RegisterTaskMeta(upload.StorageMigrationMeta)
// system cleanup
task.RegisterHandler(upload.SystemCleanupTask, &upload.SystemCleanupHandler{})
task.RegisterTaskMeta(upload.SystemCleanupMeta)
// upload
task.RegisterHandler(upload.WarmImageCacheTask, &upload.WarmImageCacheHandler{})
task.RegisterTaskMeta(upload.WarmImageCacheMeta)
task.RegisterHandler(upload.RebuildUploadStatsTask, &upload.RebuildUploadStatsHandler{})
task.RegisterTaskMeta(upload.RebuildUploadStatsMeta)
// user
task.RegisterHandler(user.SendEmailTask, &user.SendEmailHandler{})
task.RegisterTaskMeta(user.SendEmailMeta)
// push
task.RegisterHandler(push.SendNotificationTask, &push.PushHandler{})
task.RegisterTaskMeta(push.SendNotificationMeta)
}
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"sync"
)
// TaskParam 任务参数定义
//
//nolint:revive // TaskParam 保留完整名称以避免与通用 Param 混淆
type TaskParam struct {
Name string `json:"name"` // 参数键名
Label string `json:"label"` // 显示名称
Type string `json:"type"` // 类型:string, text, number, boolean
Required bool `json:"required"` // 是否必填
Placeholder string `json:"placeholder"` // 占位符
Description string `json:"description"` // 描述
}
// TaskMeta 任务元数据
//
//nolint:revive // TaskMeta 保留完整名称以避免与通用 Meta 混淆
type TaskMeta struct {
Type string `json:"type"`
AsynqTask string `json:"asynq_task"`
Name string `json:"name"`
Description string `json:"description"`
SupportsTime bool `json:"supports_time"`
MaxRetry int `json:"max_retry"`
Queue string `json:"queue"`
Retryable bool `json:"retryable"` // 是否支持手动重试
Params []TaskParam `json:"params,omitempty"`
}
var (
dispatchableTasksMutex sync.RWMutex
dispatchableTasks []TaskMeta
)
// RegisterTaskMeta 注册任务元数据到全局列表
func RegisterTaskMeta(meta TaskMeta) {
dispatchableTasksMutex.Lock()
defer dispatchableTasksMutex.Unlock()
for _, t := range dispatchableTasks {
if t.Type == meta.Type {
return
}
}
dispatchableTasks = append(dispatchableTasks, meta)
}
// GetDispatchableTasks 获取所有已注册的元数据列表(返回副本以避免并发并发读写冲突)
func GetDispatchableTasks() []TaskMeta {
dispatchableTasksMutex.RLock()
defer dispatchableTasksMutex.RUnlock()
metas := make([]TaskMeta, len(dispatchableTasks))
copy(metas, dispatchableTasks)
return metas
}
// GetTaskMeta 根据任务类型获取元数据
func GetTaskMeta(taskType string) *TaskMeta {
dispatchableTasksMutex.RLock()
defer dispatchableTasksMutex.RUnlock()
for _, t := range dispatchableTasks {
if t.Type == taskType {
copied := t
return &copied
}
}
return nil
}
// GetTaskMetaByAsynqTask 根据 Asynq 任务名称获取元数据
func GetTaskMetaByAsynqTask(asynqTask string) *TaskMeta {
dispatchableTasksMutex.RLock()
defer dispatchableTasksMutex.RUnlock()
for _, t := range dispatchableTasks {
if t.AsynqTask == asynqTask {
copied := t
return &copied
}
}
return nil
}
// GetRegisteredAsynqTasks 返回所有已注册的 Asynq 任务名称,以便动态注册路由
func GetRegisteredAsynqTasks() []string {
keys := make([]string, 0, len(handlerRegistry))
for k := range handlerRegistry {
keys = append(keys, k)
}
return keys
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task_test
import (
"testing"
"github.com/Rain-kl/Wavelet/internal/infra/task"
taskhandlers "github.com/Rain-kl/Wavelet/internal/infra/task/handlers"
)
func TestDuplicateTaskMeta(t *testing.T) {
// Call Register twice to simulate being imported by multiple packages (routers, worker, etc.)
taskhandlers.Register()
taskhandlers.Register()
metas := task.GetDispatchableTasks()
// Check if we have duplicates by checking if a Type appears more than once
seen := make(map[string]int)
for _, m := range metas {
seen[m.Type]++
}
for taskType, count := range seen {
if count > 1 {
t.Errorf("Task type %q registered %d times, expected at most 1", taskType, count)
}
}
}
+9
View File
@@ -0,0 +1,9 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package scheduler 提供定时任务调度功能
package scheduler
const (
errLoadLocationFailed = "failed to load location: %w"
)
+144
View File
@@ -0,0 +1,144 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package scheduler
import (
"context"
"fmt"
"os/signal"
"sync"
"syscall"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/platform/bootstrap"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/hibiken/asynq"
)
var (
activeScheduler *asynq.Scheduler
schedulerMutex sync.Mutex
quitChan chan struct{}
schedulerOnce sync.Once
)
// GetAsynqClient 获取全局 AsynqClient
func GetAsynqClient() *asynq.Client {
return task.AsynqClient
}
// StartScheduler 启动调度器 (该函数阻塞,直到调度器退出)
func StartScheduler() error {
bootstrap.RegisterScheduler()
var err error
schedulerOnce.Do(func() {
quitChan = make(chan struct{})
done := quitChan
// 初始化并运行首次调度
if err = ReloadScheduler(); err != nil {
err = fmt.Errorf("initial reload failed: %w", err)
return
}
signalCtx, stopSignals := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stopSignals()
if waitForStop(done, signalCtx.Done()) {
StopScheduler()
}
})
return err
}
// StopScheduler 停止调度服务并解除 StartScheduler 阻塞
func StopScheduler() {
schedulerMutex.Lock()
defer schedulerMutex.Unlock()
if activeScheduler != nil {
activeScheduler.Shutdown()
activeScheduler = nil
}
if quitChan != nil {
close(quitChan)
quitChan = nil
}
}
// ReloadScheduler 重载调度器配置 (线程安全)
func ReloadScheduler() error {
schedulerMutex.Lock()
defer schedulerMutex.Unlock()
// 1. 如果有运行中的调度器,先关闭它
if activeScheduler != nil {
activeScheduler.Shutdown()
activeScheduler = nil
}
// 2. 从数据库载入启用的定时任务配置
schedules, err := model.ListActiveSchedules(context.Background())
if err != nil {
return fmt.Errorf("load schedules from db failed: %w", err)
}
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
return fmt.Errorf(errLoadLocationFailed, err)
}
// 3. 实例化新的调度器
newScheduler := asynq.NewScheduler(
task.RedisOpt,
&asynq.SchedulerOpts{
Location: location,
},
)
// 4. 遍历并注册任务
for _, s := range schedules {
meta := task.GetTaskMeta(s.TaskType)
if meta == nil {
continue // 忽略排程配置中无效的任务类型
}
// 构造 Asynq 载荷。定时任务使用对应 Meta 中的 Asynq 标识,同时将数据库中保存的 json 作为参数
t := asynq.NewTask(meta.AsynqTask, []byte(s.Payload))
if _, err := newScheduler.Register(
s.Cron,
t,
asynq.MaxRetry(meta.MaxRetry),
asynq.Queue(meta.Queue),
); err != nil {
// 定时任务配置可能有误(如 Cron 格式不被 Asynq 识别),记录日志并跳过
logger.ErrorF(context.Background(), "[Scheduler] 注册定时任务失败 id=%d name=%s: %v", s.ID, s.Name, err)
continue
}
}
// 5. 启动并替换全局调度器。进程信号由 StartScheduler 统一处理。
if err := newScheduler.Start(); err != nil {
return fmt.Errorf("start scheduler failed: %w", err)
}
activeScheduler = newScheduler
logger.InfoF(context.Background(), "[Scheduler] 成功重新加载定时任务,共注册 %d 个活动任务", len(schedules))
return nil
}
func waitForStop(done <-chan struct{}, signals <-chan struct{}) bool {
select {
case <-done:
return false
case <-signals:
return true
}
}
@@ -0,0 +1,42 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package scheduler
import "testing"
func TestWaitForStop(t *testing.T) {
tests := []struct {
name string
closeDone bool
closeSignal bool
wantSignal bool
}{
{
name: "explicit stop",
closeDone: true,
},
{
name: "process signal",
closeSignal: true,
wantSignal: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
done := make(chan struct{})
signals := make(chan struct{})
if tt.closeDone {
close(done)
}
if tt.closeSignal {
close(signals)
}
if got := waitForStop(done, signals); got != tt.wantSignal {
t.Errorf("waitForStop() = %t, want %t", got, tt.wantSignal)
}
})
}
}
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
)
type redisClientConnOpt struct {
options redis.Options
}
func (opt redisClientConnOpt) MakeRedisClient() interface{} {
return redis.NewClient(&opt.options)
}
type redisClusterConnOpt struct {
options redis.ClusterOptions
}
func (opt redisClusterConnOpt) MakeRedisClient() interface{} {
return redis.NewClusterClient(&opt.options)
}
type redisFailoverConnOpt struct {
options redis.FailoverOptions
maintNotificationsEnabled bool
}
func (opt redisFailoverConnOpt) MakeRedisClient() interface{} {
client := redis.NewFailoverClient(&opt.options)
// go-redis v9.16 does not expose maintenance notification settings on
// FailoverOptions, so apply the configured mode before the client is used.
client.Options().MaintNotificationsConfig = maintNotificationsConfig(opt.maintNotificationsEnabled)
return client
}
func maintNotificationsConfig(enabled bool) *maintnotifications.Config {
mode := maintnotifications.ModeDisabled
if enabled {
mode = maintnotifications.ModeAuto
}
return &maintnotifications.Config{Mode: mode}
}
// RedisOpt asynq Redis 连接配置(兼容 Standalone/Sentinel/Cluster)
var RedisOpt asynq.RedisConnOpt
// AsynqClient asynq 客户端,用于任务入队
var AsynqClient *asynq.Client
func init() {
RedisOpt = NewRedisConnOpt()
AsynqClient = asynq.NewClient(RedisOpt)
}
// NewRedisConnOpt 根据配置返回对应的 asynq Redis 连接选项
func NewRedisConnOpt() asynq.RedisConnOpt {
cfg := config.Config.Redis
addrs := cfg.Addrs
if cfg.ClusterMode {
return redisClusterConnOpt{
options: redis.ClusterOptions{
Addrs: addrs,
Username: cfg.Username,
Password: cfg.Password,
MaintNotificationsConfig: maintNotificationsConfig(cfg.MaintNotifications),
},
}
}
if cfg.MasterName != "" {
return redisFailoverConnOpt{
maintNotificationsEnabled: cfg.MaintNotifications,
options: redis.FailoverOptions{
MasterName: cfg.MasterName,
SentinelAddrs: addrs,
Username: cfg.Username,
Password: cfg.Password,
DB: cfg.DB,
},
}
}
addr := "localhost:6379"
if len(addrs) > 0 {
addr = addrs[0]
}
return redisClientConnOpt{
options: redis.Options{
Addr: addr,
Username: cfg.Username,
Password: cfg.Password,
DB: cfg.DB,
PoolSize: cfg.PoolSize,
MaintNotificationsConfig: maintNotificationsConfig(cfg.MaintNotifications),
},
}
}
// PrefixedQueue 返回带前缀的队列名,用于 Cluster 模式隔离
func PrefixedQueue(queue string) string {
prefix := config.Config.Redis.KeyPrefix
if prefix == "" {
return queue
}
return prefix + queue
}
+73
View File
@@ -0,0 +1,73 @@
package task
import (
"testing"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
)
func TestNewRedisConnOptConfiguresMaintenanceNotifications(t *testing.T) {
previous := config.Config.Redis
t.Cleanup(func() { config.Config.Redis = previous })
for _, test := range []struct {
name string
enabled bool
want maintnotifications.Mode
}{
{name: "disabled by default", enabled: false, want: maintnotifications.ModeDisabled},
{name: "auto when enabled", enabled: true, want: maintnotifications.ModeAuto},
} {
t.Run(test.name, func(t *testing.T) {
config.Config.Redis.MaintNotifications = test.enabled
t.Run("standalone", func(t *testing.T) {
config.Config.Redis.ClusterMode = false
config.Config.Redis.MasterName = ""
config.Config.Redis.Addrs = []string{"127.0.0.1:6379"}
client, ok := NewRedisConnOpt().MakeRedisClient().(*redis.Client)
if !ok {
t.Fatal("standalone option did not create *redis.Client")
}
defer func() { _ = client.Close() }()
assertMaintenanceNotificationsMode(t, client.Options().MaintNotificationsConfig, test.want)
})
t.Run("cluster", func(t *testing.T) {
config.Config.Redis.ClusterMode = true
config.Config.Redis.MasterName = ""
config.Config.Redis.Addrs = []string{"127.0.0.1:6379"}
client, ok := NewRedisConnOpt().MakeRedisClient().(*redis.ClusterClient)
if !ok {
t.Fatal("cluster option did not create *redis.ClusterClient")
}
defer func() { _ = client.Close() }()
assertMaintenanceNotificationsMode(t, client.Options().MaintNotificationsConfig, test.want)
})
t.Run("sentinel", func(t *testing.T) {
config.Config.Redis.ClusterMode = false
config.Config.Redis.MasterName = "openflare"
config.Config.Redis.Addrs = []string{"127.0.0.1:26379"}
client, ok := NewRedisConnOpt().MakeRedisClient().(*redis.Client)
if !ok {
t.Fatal("sentinel option did not create *redis.Client")
}
defer func() { _ = client.Close() }()
assertMaintenanceNotificationsMode(t, client.Options().MaintNotificationsConfig, test.want)
})
})
}
}
func assertMaintenanceNotificationsMode(t *testing.T, cfg *maintnotifications.Config, want maintnotifications.Mode) {
t.Helper()
if cfg == nil || cfg.Mode != want {
t.Fatalf("maintenance notifications mode = %v, want %v", cfg, want)
}
}
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package worker 提供 Asynq 任务处理服务器与中间件
package worker
import (
"context"
"github.com/hibiken/asynq"
)
// taskLoggingMiddleware 任务处理中间件
// 注意:OTel Span 创建、日志记录、TaskExecution 状态管理
// 已由 task.ProcessTask 统一处理,此中间件保留用于未来扩展(如限流、监控等)
func taskLoggingMiddleware(h asynq.Handler) asynq.Handler {
return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error {
return h.ProcessTask(ctx, t)
})
}
+68
View File
@@ -0,0 +1,68 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package worker
import (
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/platform/bootstrap"
"github.com/hibiken/asynq"
)
// workerShutdownTimeout Worker 优雅关闭超时时间
const workerShutdownTimeout = 3 * time.Minute
// StartWorker 启动任务处理服务器
func StartWorker() error {
bootstrap.RegisterWorker()
asynqServer := asynq.NewServer(
task.RedisOpt,
asynq.Config{
Concurrency: config.Config.Worker.Concurrency,
ShutdownTimeout: workerShutdownTimeout,
Queues: buildQueuesFromConfig(),
StrictPriority: config.Config.Worker.StrictPriority,
},
)
// 注册 Asynq 任务路由
mux := asynq.NewServeMux()
mux.Use(taskLoggingMiddleware)
// 统一使用 task.ProcessTask 处理所有任务类型
// 框架内部自动分发到对应的 TaskHandler 实现
// 动态注册所有已注册的任务处理器路由,框架内部自动分发到对应的 TaskHandler 实现
for _, taskName := range task.GetRegisteredAsynqTasks() {
mux.HandleFunc(taskName, task.ProcessTask)
}
// 启动服务器
return asynqServer.Run(mux)
}
// buildQueuesFromConfig 从配置构建队列映射
func buildQueuesFromConfig() map[string]int {
queues := make(map[string]int)
// 从配置读取队列
if len(config.Config.Worker.Queues) > 0 {
for _, q := range config.Config.Worker.Queues {
if q.Name != "" && q.Priority > 0 {
queues[q.Name] = q.Priority
}
}
}
// 如果配置为空,使用默认队列
if len(queues) == 0 {
queues = map[string]int{
task.QueueDefault: 1,
}
}
return queues
}