mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-03 15:06:36 +08:00
refactor(structure): group platform, infra, and shared packages
Move process wiring, technical adapters, and cross-cutting contracts out of flat internal/ packages so new code has a clear home without changing business layout.
This commit is contained in:
@@ -0,0 +1,254 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package config 负责应用配置的加载、解析与环境变量覆盖。
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// 默认队列优先级
|
||||
const (
|
||||
webhookQueuePriority = 10
|
||||
whitelistQueuePriority = 5
|
||||
defaultQueuePriority = 3
|
||||
)
|
||||
|
||||
// Config 全局配置单例,初始化后不可变
|
||||
var Config *configModel
|
||||
|
||||
// findConfigPath searches upward for the config file to handle tests running in subdirectories.
|
||||
func findConfigPath(configPath string) string {
|
||||
if _, err := os.Stat(configPath); err == nil {
|
||||
return configPath
|
||||
}
|
||||
dir := "."
|
||||
for i := 0; i < 5; i++ {
|
||||
dir += "/.."
|
||||
path := dir + "/" + configPath
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path
|
||||
}
|
||||
}
|
||||
return configPath
|
||||
}
|
||||
|
||||
// isTest checks if the current execution context is within 'go test'.
|
||||
func isTest() bool {
|
||||
if flag.Lookup("test.v") != nil {
|
||||
return true
|
||||
}
|
||||
for _, arg := range os.Args {
|
||||
if strings.HasPrefix(arg, "-test.") || strings.HasSuffix(arg, ".test") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func init() {
|
||||
// 加载配置文件路径
|
||||
configPath := os.Getenv("CONFIG_PATH")
|
||||
if configPath == "" {
|
||||
configPath = findConfigPath("config.yaml")
|
||||
}
|
||||
|
||||
// 设置配置文件
|
||||
viper.SetConfigFile(configPath)
|
||||
viper.AutomaticEnv()
|
||||
|
||||
// 读取配置文件(可选:找不到文件时使用空默认值 + 环境变量)
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
|
||||
// 文件存在但读取/解析失败
|
||||
if _, statErr := os.Stat(configPath); statErr == nil { //nolint:gosec // configPath is loaded from CONFIG_PATH environment variable
|
||||
log.Fatalf("[Config] read config failed: %v\n", err)
|
||||
}
|
||||
}
|
||||
log.Println("[Config] no config file found, using environment variables only")
|
||||
viper.SetConfigType("yaml")
|
||||
if err := viper.ReadConfig(strings.NewReader("")); err != nil {
|
||||
log.Fatalf("[Config] failed to init empty config: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 解析配置到结构体
|
||||
var c configModel
|
||||
if err := viper.Unmarshal(&c); err != nil {
|
||||
log.Fatalf("[Config] parse config failed: %v\n", err)
|
||||
}
|
||||
|
||||
applyDefaults(&c)
|
||||
|
||||
// 环境变量覆盖(优先级高于 config.yaml)
|
||||
applyEnvOverrides(&c)
|
||||
applyDefaults(&c)
|
||||
|
||||
// Disable standard DB/Redis initializations during tests to prevent connection attempts.
|
||||
if isTest() {
|
||||
c.Database.Enabled = false
|
||||
c.Database.SQLitePath = ":memory:"
|
||||
c.Redis.Enabled = false
|
||||
c.ClickHouse.Enabled = false
|
||||
}
|
||||
|
||||
// 设置全局配置
|
||||
Config = &c
|
||||
|
||||
// 打印配置
|
||||
printConfig(&c)
|
||||
}
|
||||
|
||||
func applyDefaults(c *configModel) {
|
||||
if c.App.SessionAge <= 0 {
|
||||
c.App.SessionAge = 86400
|
||||
}
|
||||
if c.Otel.TracerName == "" {
|
||||
c.Otel.TracerName = "github.com/Rain-kl/Wavelet"
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 环境变量覆盖层 ────────────────────────────────────────────────────────────
|
||||
// 环境变量优先级高于 config.yaml,未设置则保留 yaml 中的值。
|
||||
|
||||
func envStr(key, fallback string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envInt64(key string, fallback int64) int64 {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envFloat64(key string, fallback float64) float64 {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
if n, err := strconv.ParseFloat(v, 64); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envBool(key string, fallback bool) bool {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
return b
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// applyEnvOverrides 将环境变量值覆盖到配置结构体上(仅当环境变量已设置时生效)
|
||||
func applyEnvOverrides(c *configModel) {
|
||||
// ─── App ───
|
||||
c.App.AppName = envStr("APP_NAME", c.App.AppName)
|
||||
c.App.Env = envStr("APP_ENV", c.App.Env)
|
||||
c.App.Addr = envStr("APP_ADDR", c.App.Addr)
|
||||
c.App.NodeID = envInt64("APP_NODE_ID", c.App.NodeID)
|
||||
c.App.APIPrefix = envStr("APP_API_PREFIX", c.App.APIPrefix)
|
||||
c.App.GracefulShutdownTimeout = envInt("APP_GRACEFUL_SHUTDOWN_TIMEOUT", c.App.GracefulShutdownTimeout)
|
||||
c.App.SessionCookieName = envStr("APP_SESSION_COOKIE_NAME", c.App.SessionCookieName)
|
||||
c.App.SessionSecret = envStr("APP_SESSION_SECRET", c.App.SessionSecret)
|
||||
c.App.SessionDomain = envStr("APP_SESSION_DOMAIN", c.App.SessionDomain)
|
||||
c.App.SessionAge = envInt("APP_SESSION_AGE", c.App.SessionAge)
|
||||
c.App.SessionHTTPOnly = envBool("APP_SESSION_HTTP_ONLY", c.App.SessionHTTPOnly)
|
||||
c.App.SessionSecure = envBool("APP_SESSION_SECURE", c.App.SessionSecure)
|
||||
|
||||
// ─── Database ───
|
||||
c.Database.Host = envStr("DB_HOST", c.Database.Host)
|
||||
c.Database.Port = envInt("DB_PORT", c.Database.Port)
|
||||
c.Database.Username = envStr("DB_USERNAME", c.Database.Username)
|
||||
c.Database.Password = envStr("DB_PASSWORD", c.Database.Password)
|
||||
c.Database.Database = envStr("DB_NAME", c.Database.Database)
|
||||
c.Database.SSLMode = envStr("DB_SSL_MODE", c.Database.SSLMode)
|
||||
c.Database.TimeZone = envStr("DB_TIMEZONE", c.Database.TimeZone)
|
||||
c.Database.LogLevel = envStr("DB_LOG_LEVEL", c.Database.LogLevel)
|
||||
c.Database.MaxIdleConn = envInt("DB_MAX_IDLE_CONN", c.Database.MaxIdleConn)
|
||||
c.Database.MaxOpenConn = envInt("DB_MAX_OPEN_CONN", c.Database.MaxOpenConn)
|
||||
// 当 DB_HOST 环境变量已设置时自动启用数据库
|
||||
if _, ok := os.LookupEnv("DB_HOST"); ok {
|
||||
c.Database.Enabled = true
|
||||
}
|
||||
c.Database.Enabled = envBool("DB_ENABLED", c.Database.Enabled)
|
||||
c.Database.SQLitePath = envStr("SQLITE_PATH", c.Database.SQLitePath)
|
||||
|
||||
// ─── Redis ───
|
||||
if v, ok := os.LookupEnv("REDIS_ADDR"); ok {
|
||||
c.Redis.Addrs = []string{v}
|
||||
c.Redis.Enabled = true // 当 REDIS_ADDR 已设置时自动启用
|
||||
}
|
||||
c.Redis.Enabled = envBool("REDIS_ENABLED", c.Redis.Enabled)
|
||||
c.Redis.Username = envStr("REDIS_USERNAME", c.Redis.Username)
|
||||
c.Redis.Password = envStr("REDIS_PASSWORD", c.Redis.Password)
|
||||
c.Redis.DB = envInt("REDIS_DB", c.Redis.DB)
|
||||
c.Redis.KeyPrefix = envStr("REDIS_KEY_PREFIX", c.Redis.KeyPrefix)
|
||||
c.Redis.PoolSize = envInt("REDIS_POOL_SIZE", c.Redis.PoolSize)
|
||||
c.Redis.MaintNotifications = envBool("REDIS_MAINT_NOTIFICATIONS", c.Redis.MaintNotifications)
|
||||
|
||||
// ─── ClickHouse ───
|
||||
if v, ok := os.LookupEnv("CLICKHOUSE_HOST"); ok {
|
||||
c.ClickHouse.Hosts = []string{v}
|
||||
c.ClickHouse.Enabled = true
|
||||
}
|
||||
c.ClickHouse.Enabled = envBool("CLICKHOUSE_ENABLED", c.ClickHouse.Enabled)
|
||||
c.ClickHouse.Username = envStr("CLICKHOUSE_USERNAME", c.ClickHouse.Username)
|
||||
c.ClickHouse.Password = envStr("CLICKHOUSE_PASSWORD", c.ClickHouse.Password)
|
||||
c.ClickHouse.Database = envStr("CLICKHOUSE_NAME", c.ClickHouse.Database)
|
||||
|
||||
// ─── Log ───
|
||||
c.Log.Level = envStr("LOG_LEVEL", c.Log.Level)
|
||||
c.Log.Format = envStr("LOG_FORMAT", c.Log.Format)
|
||||
c.Log.Output = envStr("LOG_OUTPUT", c.Log.Output)
|
||||
|
||||
// ─── OTel ───
|
||||
c.Otel.SamplingRate = envFloat64("OTEL_SAMPLING_RATE", c.Otel.SamplingRate)
|
||||
c.Otel.TracerName = envStr("OTEL_TRACER_NAME", c.Otel.TracerName)
|
||||
|
||||
// ─── Worker ───
|
||||
c.Worker.Concurrency = envInt("WORKER_CONCURRENCY", c.Worker.Concurrency)
|
||||
c.Worker.StrictPriority = envBool("WORKER_STRICT_PRIORITY", c.Worker.StrictPriority)
|
||||
|
||||
// 无 yaml 且无环境变量时,使用代码级默认队列
|
||||
if len(c.Worker.Queues) == 0 {
|
||||
c.Worker.Queues = []QueueConfig{
|
||||
{Name: "webhook", Priority: webhookQueuePriority},
|
||||
{Name: "whitelist_only", Priority: whitelistQueuePriority},
|
||||
{Name: "default", Priority: defaultQueuePriority},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// printConfig 打印配置内容
|
||||
func printConfig(c *configModel) {
|
||||
configJSON, err := json.MarshalIndent(c, "", " ")
|
||||
if err != nil {
|
||||
log.Printf("[Config] failed to marshal config: %v\n", err)
|
||||
return
|
||||
}
|
||||
log.Printf("[Config] loaded configuration:\n%s\n", string(configJSON))
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestApplyEnvOverridesRedisMaintNotifications(t *testing.T) {
|
||||
t.Setenv("REDIS_MAINT_NOTIFICATIONS", "true")
|
||||
|
||||
cfg := &configModel{}
|
||||
applyEnvOverrides(cfg)
|
||||
|
||||
if !cfg.Redis.MaintNotifications {
|
||||
t.Fatal("REDIS_MAINT_NOTIFICATIONS=true was not applied")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import "time"
|
||||
|
||||
type configModel struct {
|
||||
App appConfig `mapstructure:"app"`
|
||||
Database databaseConfig `mapstructure:"database"`
|
||||
Redis redisConfig `mapstructure:"redis"`
|
||||
Log logConfig `mapstructure:"log"`
|
||||
Scheduler schedulerConfig `mapstructure:"scheduler"`
|
||||
Worker workerConfig `mapstructure:"worker"`
|
||||
ClickHouse clickHouseConfig `mapstructure:"clickhouse"`
|
||||
Otel otelConfig `mapstructure:"otel"`
|
||||
}
|
||||
|
||||
// appConfig 应用基本配置
|
||||
type appConfig struct {
|
||||
AppName string `mapstructure:"app_name"`
|
||||
Env string `mapstructure:"env"`
|
||||
Addr string `mapstructure:"addr"`
|
||||
NodeID int64 `mapstructure:"node_id"`
|
||||
APIPrefix string `mapstructure:"api_prefix"`
|
||||
GracefulShutdownTimeout int `mapstructure:"graceful_shutdown_timeout"`
|
||||
SessionCookieName string `mapstructure:"session_cookie_name"`
|
||||
SessionSecret string `mapstructure:"session_secret"`
|
||||
SessionDomain string `mapstructure:"session_domain"`
|
||||
SessionAge int `mapstructure:"session_age"`
|
||||
SessionHTTPOnly bool `mapstructure:"session_http_only"`
|
||||
SessionSecure bool `mapstructure:"session_secure"`
|
||||
}
|
||||
|
||||
// IsProduction 检查当前环境是否为生产环境
|
||||
func (a *appConfig) IsProduction() bool {
|
||||
return a.Env == "production"
|
||||
}
|
||||
|
||||
// databaseConfig 数据库配置
|
||||
type databaseConfig struct {
|
||||
Enabled bool `mapstructure:"enabled"`
|
||||
SQLitePath string `mapstructure:"sqlite_path"` // PostgreSQL 禁用时的 SQLite 文件路径
|
||||
Host string `mapstructure:"host"`
|
||||
Port int `mapstructure:"port"`
|
||||
Username string `mapstructure:"username"`
|
||||
Password string `mapstructure:"password"`
|
||||
Database string `mapstructure:"database"`
|
||||
MaxIdleConn int `mapstructure:"max_idle_conn"`
|
||||
MaxOpenConn int `mapstructure:"max_open_conn"`
|
||||
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
|
||||
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
|
||||
LogLevel string `mapstructure:"log_level"`
|
||||
SSLMode string `mapstructure:"ssl_mode"`
|
||||
TimeZone string `mapstructure:"time_zone"`
|
||||
ApplicationName string `mapstructure:"application_name"`
|
||||
SearchPath string `mapstructure:"search_path"`
|
||||
PreferSimpleProtocol bool `mapstructure:"prefer_simple_protocol"`
|
||||
StatementCacheCapacity int `mapstructure:"statement_cache_capacity"`
|
||||
DefaultQueryExecMode string `mapstructure:"default_query_exec_mode"`
|
||||
Replicas []databaseReplicaConfig `mapstructure:"replicas"`
|
||||
SlowThreshold time.Duration `mapstructure:"slow_threshold"`
|
||||
}
|
||||
|
||||
// databaseReplicaConfig 只读副本配置
|
||||
type databaseReplicaConfig struct {
|
||||
Host string `mapstructure:"host"`
|
||||
Port int `mapstructure:"port"`
|
||||
Username string `mapstructure:"username"`
|
||||
Password string `mapstructure:"password"`
|
||||
}
|
||||
|
||||
// clickhouse 配置
|
||||
type clickHouseConfig struct {
|
||||
Enabled bool `mapstructure:"enabled"`
|
||||
Hosts []string `mapstructure:"hosts"`
|
||||
Username string `mapstructure:"username"`
|
||||
Password string `mapstructure:"password"`
|
||||
Database string `mapstructure:"database"`
|
||||
MaxIdleConn int `mapstructure:"max_idle_conn"`
|
||||
MaxOpenConn int `mapstructure:"max_open_conn"`
|
||||
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
|
||||
DialTimeout int `mapstructure:"dial_timeout"`
|
||||
BlockBufferSize uint8 `mapstructure:"block_buffer_size"`
|
||||
}
|
||||
|
||||
// redisConfig Redis配置
|
||||
type redisConfig struct {
|
||||
Enabled bool `mapstructure:"enabled"`
|
||||
Addrs []string `mapstructure:"addrs"`
|
||||
Username string `mapstructure:"username"`
|
||||
Password string `mapstructure:"password"`
|
||||
DB int `mapstructure:"db"`
|
||||
ClusterMode bool `mapstructure:"cluster_mode"`
|
||||
MasterName string `mapstructure:"master_name"`
|
||||
KeyPrefix string `mapstructure:"key_prefix"`
|
||||
PoolSize int `mapstructure:"pool_size"`
|
||||
MinIdleConn int `mapstructure:"min_idle_conn"`
|
||||
DialTimeout int `mapstructure:"dial_timeout"`
|
||||
ReadTimeout int `mapstructure:"read_timeout"`
|
||||
WriteTimeout int `mapstructure:"write_timeout"`
|
||||
MaxRetries int `mapstructure:"max_retries"`
|
||||
PoolTimeout int `mapstructure:"pool_timeout"`
|
||||
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
|
||||
MaintNotifications bool `mapstructure:"maint_notifications"`
|
||||
}
|
||||
|
||||
// logConfig 日志配置
|
||||
type logConfig struct {
|
||||
Level string `mapstructure:"level"`
|
||||
Format string `mapstructure:"format"`
|
||||
Output string `mapstructure:"output"`
|
||||
FilePath string `mapstructure:"file_path"`
|
||||
MaxSize int `mapstructure:"max_size"`
|
||||
MaxAge int `mapstructure:"max_age"`
|
||||
MaxBackups int `mapstructure:"max_backups"`
|
||||
Compress bool `mapstructure:"compress"`
|
||||
}
|
||||
|
||||
// schedulerConfig 定时任务配置
|
||||
type schedulerConfig struct {
|
||||
}
|
||||
|
||||
// workerConfig 工作配置
|
||||
type workerConfig struct {
|
||||
Concurrency int `mapstructure:"concurrency"`
|
||||
StrictPriority bool `mapstructure:"strict_priority"`
|
||||
Queues []QueueConfig `mapstructure:"queues"`
|
||||
}
|
||||
|
||||
// QueueConfig 队列配置
|
||||
type QueueConfig struct {
|
||||
Name string `mapstructure:"name"`
|
||||
Priority int `mapstructure:"priority"`
|
||||
}
|
||||
|
||||
// otelConfig OpenTelemetry 配置
|
||||
type otelConfig struct {
|
||||
SamplingRate float64 `mapstructure:"sampling_rate"`
|
||||
TracerName string `mapstructure:"tracer_name"`
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package diskcache wraps the generic pkg/cache/disk to provide database configuration integration.
|
||||
package diskcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
pkgcache "github.com/Rain-kl/Wavelet/pkg/cache/disk"
|
||||
)
|
||||
|
||||
// Status represents the runtime cache statistics.
|
||||
type Status = pkgcache.Status
|
||||
|
||||
const (
|
||||
defaultCacheDir = "uploads/diskcache"
|
||||
defaultMaxSizeMB = 100
|
||||
defaultTTLMinutes = 60
|
||||
defaultCleanupInterval = 10
|
||||
|
||||
// DefaultExpiration applies the cache-wide default TTL.
|
||||
DefaultExpiration = pkgcache.DefaultExpiration
|
||||
// NoExpiration stores the item without a TTL. Size limits and LRU eviction still apply.
|
||||
NoExpiration = pkgcache.NoExpiration
|
||||
)
|
||||
|
||||
// ErrCacheMiss represents a cache miss.
|
||||
var ErrCacheMiss = pkgcache.ErrCacheMiss
|
||||
|
||||
// DiskCache is a wrapper around the generic pkg/diskcache that integrates with the DB for configs.
|
||||
type DiskCache struct {
|
||||
*pkgcache.Cache
|
||||
}
|
||||
|
||||
var (
|
||||
globalCache *DiskCache
|
||||
globalCacheOnce sync.Once
|
||||
)
|
||||
|
||||
// GetGlobalCache returns the global singleton DiskCache instance.
|
||||
func GetGlobalCache() *DiskCache {
|
||||
globalCacheOnce.Do(func() {
|
||||
pureCache := pkgcache.New(defaultCacheDir)
|
||||
globalCache = &DiskCache{pureCache}
|
||||
// Load initial configs from database
|
||||
globalCache.ReloadConfig(context.Background())
|
||||
// Start background routine to clean expired items every 10 minutes
|
||||
go globalCache.StartCleanupWorker(defaultCleanupInterval * time.Minute)
|
||||
})
|
||||
return globalCache
|
||||
}
|
||||
|
||||
// New creates a new DiskCache wrapper.
|
||||
func New(basePath string) *DiskCache {
|
||||
return &DiskCache{pkgcache.New(basePath)}
|
||||
}
|
||||
|
||||
// ReloadConfig reloads policies from database configs dynamically.
|
||||
func (c *DiskCache) ReloadConfig(ctx context.Context) {
|
||||
// Ensure DB is initialized before querying
|
||||
if db.DB(ctx) == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Max Size
|
||||
maxSizeMB := int64(defaultMaxSizeMB)
|
||||
if scMaxSize, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyDiskCacheMaxSizeMB); err == nil && scMaxSize.Value != "" {
|
||||
if val, err := strconv.ParseInt(scMaxSize.Value, 10, 64); err == nil && val > 0 {
|
||||
maxSizeMB = val
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Default TTL
|
||||
ttlMinutes := int64(defaultTTLMinutes)
|
||||
if scTTL, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyDiskCacheTTLMinutes); err == nil && scTTL.Value != "" {
|
||||
if val, err := strconv.ParseInt(scTTL.Value, 10, 64); err == nil && val >= 0 {
|
||||
ttlMinutes = val
|
||||
}
|
||||
}
|
||||
|
||||
// 3. LRU Enabled
|
||||
lruEnabled := true
|
||||
if scLRU, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyDiskCacheLRUEnabled); err == nil && scLRU.Value != "" {
|
||||
if val, err := strconv.ParseBool(scLRU.Value); err == nil {
|
||||
lruEnabled = val
|
||||
}
|
||||
}
|
||||
|
||||
c.UpdatePolicy(maxSizeMB, ttlMinutes, lruEnabled)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package diskcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
)
|
||||
|
||||
func TestDiskCacheReloadConfig(t *testing.T) {
|
||||
dbConn, _, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
testDir := "uploads/test_diskcache_reload"
|
||||
defer func() { _ = os.RemoveAll(testDir) }()
|
||||
_ = os.RemoveAll(testDir)
|
||||
|
||||
c := New(testDir)
|
||||
defer func() { _ = c.Clear() }()
|
||||
|
||||
// Update DB config values
|
||||
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheMaxSizeMB).Update("value", "250")
|
||||
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheTTLMinutes).Update("value", "120")
|
||||
dbConn.Model(&model.SystemConfig{}).Where("key = ?", model.ConfigKeyDiskCacheLRUEnabled).Update("value", "false")
|
||||
|
||||
// Invalidate Redis config cache to force DB reload
|
||||
if db.Redis != nil {
|
||||
db.Redis.Del(context.Background(), db.PrefixedKey(repository.SystemConfigRedisHashKey))
|
||||
}
|
||||
|
||||
// Reload config
|
||||
c.ReloadConfig(context.Background())
|
||||
|
||||
status := c.Status()
|
||||
if status.MaxSizeMB != 250 {
|
||||
t.Errorf("expected MaxSizeMB to be 250, got %d", status.MaxSizeMB)
|
||||
}
|
||||
if status.TTLMinutes != 120 {
|
||||
t.Errorf("expected TTLMinutes to be 120, got %d", status.TTLMinutes)
|
||||
}
|
||||
if status.LRUEnabled != false {
|
||||
t.Errorf("expected LRUEnabled to be false, got %t", status.LRUEnabled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package objectstore provides dynamically configured file storage backends.
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Driver identifies a supported storage backend.
|
||||
type Driver string
|
||||
|
||||
const (
|
||||
// DriverLocal stores files on the local filesystem.
|
||||
DriverLocal Driver = "local"
|
||||
// DriverS3 stores files in an S3-compatible object store.
|
||||
DriverS3 Driver = "s3"
|
||||
// DriverR2 stores files in Cloudflare R2.
|
||||
DriverR2 Driver = "r2"
|
||||
// DriverMinIO stores files in MinIO.
|
||||
DriverMinIO Driver = "minio"
|
||||
// DriverOSS stores files in Aliyun OSS.
|
||||
DriverOSS Driver = "oss"
|
||||
// DriverWebDAV stores files through WebDAV.
|
||||
DriverWebDAV Driver = "webdav"
|
||||
|
||||
// ConfigMask replaces secrets returned to the frontend.
|
||||
ConfigMask = "******"
|
||||
)
|
||||
|
||||
// LocalConfig configures local filesystem storage.
|
||||
type LocalConfig struct {
|
||||
Root string `json:"root"`
|
||||
}
|
||||
|
||||
// ObjectConfig configures S3-compatible or OSS object storage.
|
||||
type ObjectConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
AccountID string `json:"account_id,omitempty"`
|
||||
PathStyle bool `json:"path_style"`
|
||||
KeyPrefix string `json:"key_prefix"`
|
||||
CDNURL string `json:"cdn_url"`
|
||||
}
|
||||
|
||||
// WebDAVConfig configures WebDAV storage.
|
||||
type WebDAVConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
BasePath string `json:"base_path"`
|
||||
}
|
||||
|
||||
// Config contains all storage backends and the currently active driver.
|
||||
type Config struct {
|
||||
Driver Driver `json:"driver"`
|
||||
Local LocalConfig `json:"local"`
|
||||
S3 ObjectConfig `json:"s3"`
|
||||
R2 ObjectConfig `json:"r2"`
|
||||
MinIO ObjectConfig `json:"minio"`
|
||||
OSS ObjectConfig `json:"oss"`
|
||||
WebDAV WebDAVConfig `json:"webdav"`
|
||||
}
|
||||
|
||||
// DefaultConfig returns the local-storage default configuration.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Driver: DriverLocal,
|
||||
Local: LocalConfig{Root: "."},
|
||||
S3: ObjectConfig{Region: "us-east-1"},
|
||||
R2: ObjectConfig{Region: "auto"},
|
||||
MinIO: ObjectConfig{Region: "us-east-1", PathStyle: true},
|
||||
}
|
||||
}
|
||||
|
||||
// LoadConfig loads the active storage configuration.
|
||||
func LoadConfig(ctx context.Context) (Config, error) {
|
||||
pubSubOnce.Do(startPubSubListener)
|
||||
|
||||
cacheMutex.RLock()
|
||||
isCacheValid := time.Since(lastChecked) < 5*time.Second && activeConfigJSON != ""
|
||||
configJSON := activeConfigJSON
|
||||
cacheMutex.RUnlock()
|
||||
|
||||
if isCacheValid {
|
||||
cfg := DefaultConfig()
|
||||
if strings.TrimSpace(configJSON) != "" {
|
||||
if err := json.Unmarshal([]byte(configJSON), &cfg); err != nil {
|
||||
return Config{}, fmt.Errorf("parse storage config from cache: %w", err)
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
return loadConfigByKey(ctx, model.ConfigKeyStorageConfig, DefaultConfig())
|
||||
}
|
||||
|
||||
func loadConfigByKey(ctx context.Context, key string, fallback Config) (Config, error) {
|
||||
sc, err := repository.GetSystemConfigByKey(ctx, key)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return fallback, nil
|
||||
}
|
||||
return Config{}, err
|
||||
}
|
||||
if strings.TrimSpace(sc.Value) == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
if err := json.Unmarshal([]byte(sc.Value), &fallback); err != nil {
|
||||
return Config{}, fmt.Errorf("parse %s: %w", key, err)
|
||||
}
|
||||
return fallback, nil
|
||||
}
|
||||
|
||||
// ValidateConfig validates the selected backend configuration.
|
||||
func ValidateConfig(cfg Config) error {
|
||||
switch cfg.Driver {
|
||||
case DriverLocal:
|
||||
if strings.TrimSpace(cfg.Local.Root) == "" {
|
||||
return errors.New("local root is required")
|
||||
}
|
||||
case DriverS3:
|
||||
return validateObjectConfig(cfg.S3, false)
|
||||
case DriverR2:
|
||||
if strings.TrimSpace(cfg.R2.AccountID) == "" && strings.TrimSpace(cfg.R2.Endpoint) == "" {
|
||||
return errors.New("R2 account ID or endpoint is required")
|
||||
}
|
||||
return validateObjectConfig(cfg.R2, false)
|
||||
case DriverMinIO:
|
||||
if strings.TrimSpace(cfg.MinIO.Endpoint) == "" {
|
||||
return errors.New("MinIO endpoint is required")
|
||||
}
|
||||
return validateObjectConfig(cfg.MinIO, true)
|
||||
case DriverOSS:
|
||||
if strings.TrimSpace(cfg.OSS.Endpoint) == "" {
|
||||
return errors.New("OSS endpoint is required")
|
||||
}
|
||||
return validateObjectConfig(cfg.OSS, true)
|
||||
case DriverWebDAV:
|
||||
if strings.TrimSpace(cfg.WebDAV.Endpoint) == "" {
|
||||
return errors.New("WebDAV endpoint is required")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported storage driver %q", cfg.Driver)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateObjectConfig(cfg ObjectConfig, endpointRequired bool) error {
|
||||
if endpointRequired && strings.TrimSpace(cfg.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Region) == "" {
|
||||
return errors.New("region is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.Bucket) == "" {
|
||||
return errors.New("bucket is required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.AccessKeyID) == "" || strings.TrimSpace(cfg.SecretAccessKey) == "" {
|
||||
return errors.New("access key ID and secret access key are required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveActiveConfig persists the active storage configuration.
|
||||
func SaveActiveConfig(ctx context.Context, cfg Config) error {
|
||||
return saveSystemConfig(ctx, model.ConfigKeyStorageConfig, cfg, "文件存储驱动及连接配置(JSON)")
|
||||
}
|
||||
|
||||
func saveSystemConfig(ctx context.Context, key string, value any, description string) error {
|
||||
err := db.DB(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
return upsertSystemConfig(ctx, tx, key, value, description)
|
||||
})
|
||||
if err == nil && key == model.ConfigKeyStorageConfig {
|
||||
ResetCache()
|
||||
PublishCacheInvalidation(ctx)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func upsertSystemConfig(ctx context.Context, tx *gorm.DB, key string, value any, description string) error {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal %s: %w", key, err)
|
||||
}
|
||||
sc := model.SystemConfig{
|
||||
Key: key,
|
||||
Value: string(data),
|
||||
Type: "system",
|
||||
Visibility: model.ConfigVisibilityHidden,
|
||||
Description: description,
|
||||
}
|
||||
if err := tx.Where("key = ?", key).
|
||||
Assign(map[string]any{"value": sc.Value, "description": description, "visibility": model.ConfigVisibilityHidden}).
|
||||
FirstOrCreate(&sc).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return repository.InvalidateSystemConfigCache(ctx, key)
|
||||
}
|
||||
|
||||
// MergeMaskedSecrets restores unchanged secrets from the current configuration.
|
||||
func MergeMaskedSecrets(next, current Config) Config {
|
||||
mergeObjectSecret := func(dst *ObjectConfig, src ObjectConfig) {
|
||||
if dst.AccessKeyID == ConfigMask {
|
||||
dst.AccessKeyID = src.AccessKeyID
|
||||
}
|
||||
if dst.SecretAccessKey == ConfigMask {
|
||||
dst.SecretAccessKey = src.SecretAccessKey
|
||||
}
|
||||
}
|
||||
mergeObjectSecret(&next.S3, current.S3)
|
||||
mergeObjectSecret(&next.R2, current.R2)
|
||||
mergeObjectSecret(&next.MinIO, current.MinIO)
|
||||
mergeObjectSecret(&next.OSS, current.OSS)
|
||||
if next.WebDAV.Password == ConfigMask {
|
||||
next.WebDAV.Password = current.WebDAV.Password
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// MaskSecrets replaces stored credentials with placeholders for API responses.
|
||||
func MaskSecrets(cfg Config) Config {
|
||||
maskObject := func(value *ObjectConfig) {
|
||||
if value.AccessKeyID != "" {
|
||||
value.AccessKeyID = ConfigMask
|
||||
}
|
||||
if value.SecretAccessKey != "" {
|
||||
value.SecretAccessKey = ConfigMask
|
||||
}
|
||||
}
|
||||
maskObject(&cfg.S3)
|
||||
maskObject(&cfg.R2)
|
||||
maskObject(&cfg.MinIO)
|
||||
maskObject(&cfg.OSS)
|
||||
if cfg.WebDAV.Password != "" {
|
||||
cfg.WebDAV.Password = ConfigMask
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/pkg/httppool"
|
||||
)
|
||||
|
||||
func getHTTPObject(ctx context.Context, baseURL, key string) (*Object, error) {
|
||||
objectURL, err := url.JoinPath(baseURL, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build CDN object URL: %w", err)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, objectURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create CDN request: %w", err)
|
||||
}
|
||||
const cdnRequestTimeout = 30 * time.Second
|
||||
client := httppool.NewClient(cdnRequestTimeout)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get CDN object: %w", err)
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
_ = response.Body.Close()
|
||||
return nil, fmt.Errorf("get CDN object: unexpected status %d", response.StatusCode)
|
||||
}
|
||||
contentType := response.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = defaultContentType
|
||||
}
|
||||
return &Object{
|
||||
Body: response.Body,
|
||||
ContentLength: response.ContentLength,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type localBackend struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func newLocalBackend(cfg LocalConfig) (*localBackend, error) {
|
||||
root := filepath.Clean(cfg.Root)
|
||||
if root == "" {
|
||||
return nil, fmt.Errorf("local root is required")
|
||||
}
|
||||
return &localBackend{root: root}, nil
|
||||
}
|
||||
|
||||
func (b *localBackend) Put(_ context.Context, key string, body io.Reader, _ int64, _ string) (PutResult, error) {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), storageDirPerm); err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
file, err := os.OpenFile( //nolint:gosec // path is constrained to the configured storage root.
|
||||
path,
|
||||
os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
|
||||
storageFilePerm,
|
||||
)
|
||||
if err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
if _, err := io.Copy(file, body); err != nil {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(path)
|
||||
return PutResult{}, err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
_ = os.Remove(path)
|
||||
return PutResult{}, err
|
||||
}
|
||||
return PutResult{Key: filepath.ToSlash(key)}, nil
|
||||
}
|
||||
|
||||
func (b *localBackend) Get(_ context.Context, key string) (*Object, error) {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file, err := os.Open(path) //nolint:gosec // path is constrained to the configured storage root.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, err
|
||||
}
|
||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||
if contentType == "" {
|
||||
contentType = defaultContentType
|
||||
}
|
||||
return &Object{Body: file, ContentLength: info.Size(), ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *localBackend) Delete(_ context.Context, key string) error {
|
||||
path, err := b.path(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Remove(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *localBackend) Test(_ context.Context) error {
|
||||
return os.MkdirAll(b.root, storageDirPerm)
|
||||
}
|
||||
|
||||
func (b *localBackend) path(key string) (string, error) {
|
||||
if filepath.IsAbs(key) {
|
||||
cleanPath := filepath.Clean(key)
|
||||
absRoot, err := filepath.Abs(b.root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
absPath, err := filepath.Abs(cleanPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, err := filepath.Rel(absRoot, absPath)
|
||||
if err != nil || strings.HasPrefix(rel, "..") {
|
||||
return "", fmt.Errorf("storage key escapes local root")
|
||||
}
|
||||
return cleanPath, nil
|
||||
}
|
||||
cleanKey := filepath.Clean(filepath.FromSlash(strings.TrimPrefix(key, "/")))
|
||||
if cleanKey == "." || cleanKey == "" || strings.HasPrefix(cleanKey, "..") {
|
||||
return "", fmt.Errorf("invalid local storage key %q", key)
|
||||
}
|
||||
path := filepath.Join(b.root, cleanKey)
|
||||
rel, err := filepath.Rel(b.root, path)
|
||||
if err != nil || strings.HasPrefix(rel, "..") {
|
||||
return "", fmt.Errorf("storage key escapes local root")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocalBackendRoundTrip(t *testing.T) {
|
||||
backend, err := newLocalBackend(LocalConfig{Root: t.TempDir()})
|
||||
if err != nil {
|
||||
t.Fatalf("newLocalBackend() returned error: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
const key = "uploads/2026/06/13/test.txt"
|
||||
const content = "wavelet storage"
|
||||
|
||||
storedResult, err := backend.Put(ctx, key, bytes.NewBufferString(content), int64(len(content)), "text/plain")
|
||||
if err != nil {
|
||||
t.Fatalf("Put(%q) returned error: %v", key, err)
|
||||
}
|
||||
if storedResult.Key != key {
|
||||
t.Errorf("Put(%q) key = %q, want %q", key, storedResult.Key, key)
|
||||
}
|
||||
|
||||
object, err := backend.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%q) returned error: %v", key, err)
|
||||
}
|
||||
got, err := io.ReadAll(object.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll(Get(%q)) returned error: %v", key, err)
|
||||
}
|
||||
if err := object.Body.Close(); err != nil {
|
||||
t.Fatalf("Close(Get(%q)) returned error: %v", key, err)
|
||||
}
|
||||
if string(got) != content {
|
||||
t.Errorf("Get(%q) content = %q, want %q", key, got, content)
|
||||
}
|
||||
|
||||
if err := backend.Delete(ctx, key); err != nil {
|
||||
t.Fatalf("Delete(%q) returned error: %v", key, err)
|
||||
}
|
||||
if _, err := backend.Get(ctx, key); err == nil {
|
||||
t.Errorf("Get(%q) after Delete() returned nil error", key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
|
||||
)
|
||||
|
||||
type ossBackend struct {
|
||||
client *oss.Client
|
||||
bucket string
|
||||
keyPrefix string
|
||||
cdnURL string
|
||||
}
|
||||
|
||||
func newOSSBackend(cfg ObjectConfig) (*ossBackend, error) {
|
||||
options := oss.LoadDefaultConfig().
|
||||
WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey)).
|
||||
WithRegion(cfg.Region)
|
||||
if cfg.Endpoint != "" {
|
||||
options.WithEndpoint(cfg.Endpoint)
|
||||
}
|
||||
return &ossBackend{
|
||||
client: oss.NewClient(options),
|
||||
bucket: cfg.Bucket,
|
||||
keyPrefix: strings.Trim(cfg.KeyPrefix, "/"),
|
||||
cdnURL: strings.TrimRight(cfg.CDNURL, "/"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Put(ctx context.Context, key string, body io.Reader, _ int64, _ string) (PutResult, error) {
|
||||
key = b.key(key)
|
||||
_, err := b.client.PutObject(ctx, &oss.PutObjectRequest{
|
||||
Bucket: oss.Ptr(b.bucket),
|
||||
Key: oss.Ptr(key),
|
||||
Body: body,
|
||||
})
|
||||
if err != nil {
|
||||
return PutResult{}, fmt.Errorf("put OSS object: %w", err)
|
||||
}
|
||||
return PutResult{Key: key, Bucket: b.bucket}, nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
key = b.key(key)
|
||||
if b.cdnURL != "" {
|
||||
return getHTTPObject(ctx, b.cdnURL, key)
|
||||
}
|
||||
output, err := b.client.GetObject(ctx, &oss.GetObjectRequest{
|
||||
Bucket: oss.Ptr(b.bucket),
|
||||
Key: oss.Ptr(key),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get OSS object: %w", err)
|
||||
}
|
||||
contentType := defaultContentType
|
||||
if output.ContentType != nil {
|
||||
contentType = *output.ContentType
|
||||
}
|
||||
return &Object{Body: output.Body, ContentLength: output.ContentLength, ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Delete(ctx context.Context, key string) error {
|
||||
_, err := b.client.DeleteObject(ctx, &oss.DeleteObjectRequest{
|
||||
Bucket: oss.Ptr(b.bucket),
|
||||
Key: oss.Ptr(b.key(key)),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete OSS object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) Test(ctx context.Context) error {
|
||||
ok, err := b.client.IsBucketExist(ctx, b.bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("access OSS bucket: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("OSS bucket %q does not exist", b.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *ossBackend) key(key string) string {
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if b.keyPrefix == "" || strings.HasPrefix(key, b.keyPrefix+"/") {
|
||||
return key
|
||||
}
|
||||
return b.keyPrefix + "/" + key
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type s3Backend struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
keyPrefix string
|
||||
cdnURL string
|
||||
}
|
||||
|
||||
func newS3Backend(ctx context.Context, cfg ObjectConfig) (*s3Backend, error) {
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(ctx,
|
||||
awsconfig.WithRegion(cfg.Region),
|
||||
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
cfg.AccessKeyID,
|
||||
cfg.SecretAccessKey,
|
||||
"",
|
||||
)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load S3 config: %w", err)
|
||||
}
|
||||
client := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
|
||||
if cfg.Endpoint != "" {
|
||||
options.BaseEndpoint = aws.String(strings.TrimRight(cfg.Endpoint, "/"))
|
||||
}
|
||||
options.UsePathStyle = cfg.PathStyle
|
||||
})
|
||||
return &s3Backend{
|
||||
client: client,
|
||||
bucket: cfg.Bucket,
|
||||
keyPrefix: strings.Trim(cfg.KeyPrefix, "/"),
|
||||
cdnURL: strings.TrimRight(cfg.CDNURL, "/"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newR2Backend(ctx context.Context, cfg ObjectConfig) (*s3Backend, error) {
|
||||
if cfg.Endpoint == "" {
|
||||
cfg.Endpoint = fmt.Sprintf("https://%s.r2.cloudflarestorage.com", cfg.AccountID)
|
||||
}
|
||||
cfg.Region = "auto"
|
||||
return newS3Backend(ctx, cfg)
|
||||
}
|
||||
|
||||
func (b *s3Backend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error) {
|
||||
key = b.key(key)
|
||||
_, err := b.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: body,
|
||||
ContentLength: aws.Int64(size),
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return PutResult{}, fmt.Errorf("put S3 object: %w", err)
|
||||
}
|
||||
return PutResult{Key: key, Bucket: b.bucket}, nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
key = b.key(key)
|
||||
if b.cdnURL != "" {
|
||||
return getHTTPObject(ctx, b.cdnURL, key)
|
||||
}
|
||||
output, err := b.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get S3 object: %w", err)
|
||||
}
|
||||
contentType := defaultContentType
|
||||
if output.ContentType != nil {
|
||||
contentType = *output.ContentType
|
||||
}
|
||||
var size int64
|
||||
if output.ContentLength != nil {
|
||||
size = *output.ContentLength
|
||||
}
|
||||
return &Object{Body: output.Body, ContentLength: size, ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) Delete(ctx context.Context, key string) error {
|
||||
_, err := b.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(b.bucket),
|
||||
Key: aws.String(b.key(key)),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete S3 object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) Test(ctx context.Context) error {
|
||||
_, err := b.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(b.bucket)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("access S3 bucket: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *s3Backend) key(key string) string {
|
||||
key = strings.TrimLeft(key, "/")
|
||||
if b.keyPrefix == "" || strings.HasPrefix(key, b.keyPrefix+"/") {
|
||||
return key
|
||||
}
|
||||
return b.keyPrefix + "/" + key
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultContentType = "application/octet-stream"
|
||||
storageDirPerm = 0o750
|
||||
storageFilePerm = 0o600
|
||||
)
|
||||
|
||||
// Object describes a readable stored object.
|
||||
type Object struct {
|
||||
CachePath string
|
||||
Body io.ReadCloser
|
||||
ContentLength int64
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// PutResult describes the result of a successful Put operation.
|
||||
type PutResult struct {
|
||||
Key string
|
||||
Bucket string
|
||||
}
|
||||
|
||||
// Backend defines storage operations used by the upload domain.
|
||||
type Backend interface {
|
||||
Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error)
|
||||
Get(ctx context.Context, key string) (*Object, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
Test(ctx context.Context) error
|
||||
}
|
||||
|
||||
var (
|
||||
// IsEnabledFunc preserves the legacy S3 test hook while tests migrate to backend injection.
|
||||
IsEnabledFunc = func() bool { return false }
|
||||
mockBackend Backend
|
||||
|
||||
activeBackend Backend
|
||||
activeDriver Driver
|
||||
activeConfigJSON string
|
||||
lastChecked time.Time
|
||||
cacheMutex sync.RWMutex
|
||||
)
|
||||
|
||||
// ConfigInvalidationChannel is the Redis pub/sub channel used to evict storage caches cluster-wide.
|
||||
const ConfigInvalidationChannel = "storage:config_invalidation"
|
||||
|
||||
var pubSubOnce sync.Once
|
||||
|
||||
// ResetCache clears the local cache for storage configuration and client singletons.
|
||||
func ResetCache() {
|
||||
cacheMutex.Lock()
|
||||
defer cacheMutex.Unlock()
|
||||
activeBackend = nil
|
||||
activeDriver = ""
|
||||
activeConfigJSON = ""
|
||||
lastChecked = time.Time{}
|
||||
}
|
||||
|
||||
// PublishCacheInvalidation broadcasts cache eviction to all nodes in the cluster via Redis.
|
||||
func PublishCacheInvalidation(ctx context.Context) {
|
||||
if db.Redis != nil {
|
||||
_ = db.Redis.Publish(ctx, ConfigInvalidationChannel, "reset").Err()
|
||||
}
|
||||
}
|
||||
|
||||
// startPubSubListener starts the background subscriber for cache invalidations.
|
||||
func startPubSubListener() {
|
||||
if db.Redis == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
pubsub := db.Redis.Subscribe(context.Background(), ConfigInvalidationChannel)
|
||||
defer func() {
|
||||
_ = pubsub.Close()
|
||||
}()
|
||||
|
||||
ch := pubsub.Channel()
|
||||
for range ch {
|
||||
ResetCache()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Active returns the configured active driver and backend, using an in-memory cache with 5s TTL.
|
||||
func Active(ctx context.Context) (Driver, Backend, error) {
|
||||
if IsEnabledFunc() && mockBackend != nil {
|
||||
return DriverS3, mockBackend, nil
|
||||
}
|
||||
|
||||
pubSubOnce.Do(startPubSubListener)
|
||||
|
||||
cacheMutex.RLock()
|
||||
isCacheValid := time.Since(lastChecked) < 5*time.Second && activeBackend != nil
|
||||
if isCacheValid {
|
||||
d, b := activeDriver, activeBackend
|
||||
cacheMutex.RUnlock()
|
||||
return d, b, nil
|
||||
}
|
||||
cacheMutex.RUnlock()
|
||||
|
||||
cacheMutex.Lock()
|
||||
defer cacheMutex.Unlock()
|
||||
|
||||
// Double-check under write lock
|
||||
if time.Since(lastChecked) < 5*time.Second && activeBackend != nil {
|
||||
return activeDriver, activeBackend, nil
|
||||
}
|
||||
|
||||
sc, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyStorageConfig)
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
lastChecked = time.Now()
|
||||
|
||||
// Reuse existing backend client singleton if configuration JSON matches
|
||||
if sc.Value == activeConfigJSON && activeBackend != nil {
|
||||
return activeDriver, activeBackend, nil
|
||||
}
|
||||
|
||||
cfg := DefaultConfig()
|
||||
if strings.TrimSpace(sc.Value) != "" {
|
||||
if err := json.Unmarshal([]byte(sc.Value), &cfg); err != nil {
|
||||
return "", nil, fmt.Errorf("parse storage config: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
backend, err := NewBackend(ctx, cfg, cfg.Driver)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
activeDriver = cfg.Driver
|
||||
activeBackend = backend
|
||||
activeConfigJSON = sc.Value
|
||||
|
||||
return activeDriver, activeBackend, nil
|
||||
}
|
||||
|
||||
type functionBackend struct {
|
||||
put func(context.Context, string, io.Reader, int64, string) error
|
||||
get func(context.Context, string) (*Object, error)
|
||||
delete func(context.Context, string) error
|
||||
}
|
||||
|
||||
func (b *functionBackend) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) (PutResult, error) {
|
||||
if err := b.put(ctx, key, body, size, contentType); err != nil {
|
||||
return PutResult{}, err
|
||||
}
|
||||
return PutResult{Key: key}, nil
|
||||
}
|
||||
|
||||
func (b *functionBackend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
return b.get(ctx, key)
|
||||
}
|
||||
|
||||
func (b *functionBackend) Delete(ctx context.Context, key string) error {
|
||||
return b.delete(ctx, key)
|
||||
}
|
||||
|
||||
func (b *functionBackend) Test(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// MockStorage replaces object operations for package tests and returns a restore function.
|
||||
func MockStorage(
|
||||
put func(context.Context, string, io.Reader, int64, string) error,
|
||||
get func(context.Context, string) (*Object, error),
|
||||
deleteObject func(context.Context, string) error,
|
||||
) func() {
|
||||
previous := mockBackend
|
||||
mockBackend = &functionBackend{put: put, get: get, delete: deleteObject}
|
||||
return func() {
|
||||
mockBackend = previous
|
||||
}
|
||||
}
|
||||
|
||||
// NewBackend constructs a concrete backend from configuration.
|
||||
func NewBackend(ctx context.Context, cfg Config, driver Driver) (Backend, error) {
|
||||
if driver == DriverS3 && mockBackend != nil {
|
||||
return mockBackend, nil
|
||||
}
|
||||
switch driver {
|
||||
case DriverLocal:
|
||||
return newLocalBackend(cfg.Local)
|
||||
case DriverS3:
|
||||
return newS3Backend(ctx, cfg.S3)
|
||||
case DriverR2:
|
||||
return newR2Backend(ctx, cfg.R2)
|
||||
case DriverMinIO:
|
||||
return newS3Backend(ctx, cfg.MinIO)
|
||||
case DriverOSS:
|
||||
return newOSSBackend(cfg.OSS)
|
||||
case DriverWebDAV:
|
||||
return newWebDAVBackend(cfg.WebDAV)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage driver %q", driver)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func TestStorageCache(t *testing.T) {
|
||||
// 1. Reset cache
|
||||
ResetCache()
|
||||
|
||||
if activeConfigJSON != "" || activeDriver != "" || activeBackend != nil || !lastChecked.IsZero() {
|
||||
t.Fatal("ResetCache did not clear cache variables")
|
||||
}
|
||||
|
||||
// 2. Set up cache manually
|
||||
expectedConfig := Config{
|
||||
Driver: DriverLocal,
|
||||
Local: LocalConfig{Root: "/tmp/wavelet-test"},
|
||||
}
|
||||
cfgJSON, err := json.Marshal(expectedConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal config failed: %v", err)
|
||||
}
|
||||
|
||||
cacheMutex.Lock()
|
||||
activeConfigJSON = string(cfgJSON)
|
||||
lastChecked = time.Now()
|
||||
cacheMutex.Unlock()
|
||||
|
||||
// 3. Call LoadConfig and verify it loads from cache (doesn't hit database, which would fail/panic because DB is not initialized)
|
||||
ctx := context.Background()
|
||||
loadedCfg, err := LoadConfig(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig failed: %v", err)
|
||||
}
|
||||
|
||||
if loadedCfg.Driver != expectedConfig.Driver || loadedCfg.Local.Root != expectedConfig.Local.Root {
|
||||
t.Errorf("Loaded config %+v, expected %+v", loadedCfg, expectedConfig)
|
||||
}
|
||||
|
||||
// 4. Test Active() returns cached driver and backend
|
||||
mockBnd := &functionBackend{
|
||||
put: func(context.Context, string, io.Reader, int64, string) error { return nil },
|
||||
get: func(context.Context, string) (*Object, error) { return nil, nil },
|
||||
delete: func(context.Context, string) error { return nil },
|
||||
}
|
||||
|
||||
cacheMutex.Lock()
|
||||
activeBackend = mockBnd
|
||||
activeDriver = DriverLocal
|
||||
cacheMutex.Unlock()
|
||||
|
||||
drv, bnd, err := Active(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Active failed: %v", err)
|
||||
}
|
||||
if drv != DriverLocal || bnd != mockBnd {
|
||||
t.Errorf("Active returned driver %v, backend %v; expected %v, %v", drv, bnd, DriverLocal, mockBnd)
|
||||
}
|
||||
|
||||
// 5. Test ResetCache again
|
||||
ResetCache()
|
||||
if activeConfigJSON != "" || activeDriver != "" || activeBackend != nil || !lastChecked.IsZero() {
|
||||
t.Fatal("ResetCache did not clear cache variables after setting them")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageCachePubSub(t *testing.T) {
|
||||
// 1. Start miniredis
|
||||
mr, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to run miniredis: %v", err)
|
||||
}
|
||||
defer mr.Close()
|
||||
|
||||
// 2. Initialize Redis client
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: mr.Addr(),
|
||||
})
|
||||
defer rdb.Close()
|
||||
|
||||
// 3. Set db.Redis to our client
|
||||
oldRedis := db.Redis
|
||||
db.Redis = rdb
|
||||
defer func() {
|
||||
db.Redis = oldRedis
|
||||
}()
|
||||
|
||||
// Reset cache and set some cached config
|
||||
ResetCache()
|
||||
cacheMutex.Lock()
|
||||
activeConfigJSON = "some_config"
|
||||
lastChecked = time.Now()
|
||||
cacheMutex.Unlock()
|
||||
|
||||
// 4. Force trigger lazy initialization of subscription
|
||||
// Reset the once guard so it runs the listener
|
||||
pubSubOnce = sync.Once{}
|
||||
ctx := context.Background()
|
||||
|
||||
// Create mock backend for Active call
|
||||
mockBnd := &functionBackend{
|
||||
put: func(context.Context, string, io.Reader, int64, string) error { return nil },
|
||||
get: func(context.Context, string) (*Object, error) { return nil, nil },
|
||||
delete: func(context.Context, string) error { return nil },
|
||||
}
|
||||
cacheMutex.Lock()
|
||||
activeBackend = mockBnd
|
||||
activeDriver = DriverLocal
|
||||
cacheMutex.Unlock()
|
||||
|
||||
_, _, _ = Active(ctx) // This calls startPubSubListener()
|
||||
|
||||
// Allow some time for subscriber connection
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// 5. Publish cache invalidation
|
||||
PublishCacheInvalidation(ctx)
|
||||
|
||||
// Allow message propagation
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// 6. Verify cache was cleared
|
||||
cacheMutex.RLock()
|
||||
configJSON := activeConfigJSON
|
||||
cacheMutex.RUnlock()
|
||||
|
||||
if configJSON != "" {
|
||||
t.Error("Memory cache was not cleared after Redis Pub/Sub broadcast")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/pkg/httppool"
|
||||
"github.com/studio-b12/gowebdav"
|
||||
)
|
||||
|
||||
type contextTransport struct {
|
||||
ctx context.Context
|
||||
parent http.RoundTripper
|
||||
}
|
||||
|
||||
func (t *contextTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return t.parent.RoundTrip(req.WithContext(t.ctx))
|
||||
}
|
||||
|
||||
type webDAVBackend struct {
|
||||
endpoint string
|
||||
username string
|
||||
password string
|
||||
basePath string
|
||||
}
|
||||
|
||||
func newWebDAVBackend(cfg WebDAVConfig) (*webDAVBackend, error) {
|
||||
return &webDAVBackend{
|
||||
endpoint: strings.TrimRight(cfg.Endpoint, "/"),
|
||||
username: cfg.Username,
|
||||
password: cfg.Password,
|
||||
basePath: strings.Trim(cfg.BasePath, "/"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) newClient(ctx context.Context) *gowebdav.Client {
|
||||
client := gowebdav.NewClient(b.endpoint, b.username, b.password)
|
||||
client.SetTransport(&contextTransport{
|
||||
ctx: ctx,
|
||||
parent: httppool.DefaultTransport(),
|
||||
})
|
||||
return client
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Put(ctx context.Context, key string, body io.Reader, size int64, _ string) (PutResult, error) {
|
||||
key = b.key(key)
|
||||
client := b.newClient(ctx)
|
||||
if dir := path.Dir(key); dir != "." && dir != "/" {
|
||||
if err := client.MkdirAll(dir, storageDirPerm); err != nil {
|
||||
return PutResult{}, fmt.Errorf("create WebDAV directory: %w", err)
|
||||
}
|
||||
}
|
||||
if err := client.WriteStreamWithLength(key, body, size, storageFilePerm); err != nil {
|
||||
return PutResult{}, fmt.Errorf("put WebDAV object: %w", err)
|
||||
}
|
||||
return PutResult{Key: key}, nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Get(ctx context.Context, key string) (*Object, error) {
|
||||
key = b.key(key)
|
||||
client := b.newClient(ctx)
|
||||
info, err := client.Stat(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat WebDAV object: %w", err)
|
||||
}
|
||||
body, err := client.ReadStream(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get WebDAV object: %w", err)
|
||||
}
|
||||
contentType := defaultContentType
|
||||
if typed, ok := info.(interface{ ContentType() string }); ok && typed.ContentType() != "" {
|
||||
contentType = typed.ContentType()
|
||||
}
|
||||
return &Object{Body: body, ContentLength: info.Size(), ContentType: contentType}, nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Delete(ctx context.Context, key string) error {
|
||||
client := b.newClient(ctx)
|
||||
if err := client.Remove(b.key(key)); err != nil {
|
||||
return fmt.Errorf("delete WebDAV object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) Test(ctx context.Context) error {
|
||||
client := b.newClient(ctx)
|
||||
if err := client.Connect(); err != nil {
|
||||
return fmt.Errorf("connect WebDAV: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *webDAVBackend) key(key string) string {
|
||||
return "/" + path.Join(b.basePath, strings.TrimLeft(key, "/"))
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package batchwriter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultQueueSize = 10_000
|
||||
defaultMaxBatchSize = 1_000
|
||||
defaultFlushEvery = time.Second
|
||||
)
|
||||
|
||||
// Config controls queue capacity and flush thresholds for a Writer instance.
|
||||
type Config struct {
|
||||
// Name identifies the writer in logs and diagnostics. Optional.
|
||||
Name string
|
||||
|
||||
// QueueSize is the buffered channel capacity.
|
||||
QueueSize int
|
||||
|
||||
// MaxBatchSize triggers a flush when the in-memory batch reaches this count.
|
||||
MaxBatchSize int
|
||||
|
||||
// FlushInterval triggers a time-based flush even when the batch is smaller.
|
||||
FlushInterval time.Duration
|
||||
}
|
||||
|
||||
// DefaultConfig returns production-friendly defaults aligned with audit log batching.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
QueueSize: defaultQueueSize,
|
||||
MaxBatchSize: defaultMaxBatchSize,
|
||||
FlushInterval: defaultFlushEvery,
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
if c.QueueSize <= 0 {
|
||||
return fmt.Errorf("batchwriter: queue size must be positive")
|
||||
}
|
||||
if c.MaxBatchSize <= 0 {
|
||||
return fmt.Errorf("batchwriter: max batch size must be positive")
|
||||
}
|
||||
if c.FlushInterval <= 0 {
|
||||
return fmt.Errorf("batchwriter: flush interval must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package batchwriter
|
||||
|
||||
import "errors"
|
||||
|
||||
var errNilFlushFunc = errors.New("batchwriter: flush func is required")
|
||||
@@ -0,0 +1,215 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package batchwriter provides a reusable buffered batch writer for high-throughput
|
||||
// append-only sinks such as ClickHouse. Each business domain should own an independent
|
||||
// Writer instance with its own queue, flush callback, and tuning parameters.
|
||||
package batchwriter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FlushFunc persists a batch of queued items. It is invoked from the worker goroutine.
|
||||
type FlushFunc[T any] func(ctx context.Context, items []T) error
|
||||
|
||||
// FlushErrorHandler is called when FlushFunc returns an error. The batch is discarded
|
||||
// after the handler returns; the worker continues processing.
|
||||
type FlushErrorHandler func(ctx context.Context, batchSize int, err error)
|
||||
|
||||
// Writer buffers items and flushes them by size or interval.
|
||||
type Writer[T any] struct {
|
||||
cfg Config
|
||||
flush FlushFunc[T]
|
||||
|
||||
onFlushError FlushErrorHandler
|
||||
onDrop func(T)
|
||||
|
||||
startOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
|
||||
mu sync.RWMutex
|
||||
ch chan T
|
||||
workerCtx context.Context
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// Option configures optional Writer callbacks.
|
||||
type Option[T any] func(*Writer[T])
|
||||
|
||||
// WithFlushErrorHandler registers a callback for flush failures.
|
||||
func WithFlushErrorHandler[T any](handler FlushErrorHandler) Option[T] {
|
||||
return func(w *Writer[T]) {
|
||||
w.onFlushError = handler
|
||||
}
|
||||
}
|
||||
|
||||
// WithDropHandler registers a callback when TryEnqueue cannot accept an item.
|
||||
func WithDropHandler[T any](handler func(T)) Option[T] {
|
||||
return func(w *Writer[T]) {
|
||||
w.onDrop = handler
|
||||
}
|
||||
}
|
||||
|
||||
// New creates a Writer. Call Start before enqueueing items.
|
||||
func New[T any](cfg Config, flush FlushFunc[T], opts ...Option[T]) (*Writer[T], error) {
|
||||
if flush == nil {
|
||||
return nil, errNilFlushFunc
|
||||
}
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
w := &Writer[T]{
|
||||
cfg: cfg,
|
||||
flush: flush,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(w)
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Start launches the background worker. It is safe to call at most once.
|
||||
func (w *Writer[T]) Start(parent context.Context) {
|
||||
w.startOnce.Do(func() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
w.ch = make(chan T, w.cfg.QueueSize)
|
||||
w.workerCtx = context.WithoutCancel(parent)
|
||||
go w.run()
|
||||
})
|
||||
}
|
||||
|
||||
// Stop closes the queue and waits until the worker drains pending items and exits.
|
||||
func (w *Writer[T]) Stop(ctx context.Context) error {
|
||||
w.mu.RLock()
|
||||
ch := w.ch
|
||||
done := w.done
|
||||
w.mu.RUnlock()
|
||||
|
||||
if ch == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
w.stopOnce.Do(func() {
|
||||
close(ch)
|
||||
})
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Running reports whether Start has been called and Stop has not completed.
|
||||
func (w *Writer[T]) Running() bool {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
if w.ch == nil {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-w.done:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// TryEnqueue adds one item without blocking. It returns false when the writer is not
|
||||
// running or the queue is full.
|
||||
func (w *Writer[T]) TryEnqueue(item T) bool {
|
||||
w.mu.RLock()
|
||||
ch := w.ch
|
||||
w.mu.RUnlock()
|
||||
if ch == nil {
|
||||
w.notifyDrop(item)
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- item:
|
||||
return true
|
||||
default:
|
||||
w.notifyDrop(item)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// IsFull reports whether the queue has no remaining capacity.
|
||||
func (w *Writer[T]) IsFull() bool {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
if w.ch == nil {
|
||||
return false
|
||||
}
|
||||
return len(w.ch) >= cap(w.ch)
|
||||
}
|
||||
|
||||
// Len returns the current queue depth.
|
||||
func (w *Writer[T]) Len() int {
|
||||
w.mu.RLock()
|
||||
defer w.mu.RUnlock()
|
||||
if w.ch == nil {
|
||||
return 0
|
||||
}
|
||||
return len(w.ch)
|
||||
}
|
||||
|
||||
// Cap returns the queue capacity.
|
||||
func (w *Writer[T]) Cap() int {
|
||||
return w.cfg.QueueSize
|
||||
}
|
||||
|
||||
func (w *Writer[T]) run() {
|
||||
ticker := time.NewTicker(w.cfg.FlushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
batch := make([]T, 0, w.cfg.MaxBatchSize)
|
||||
flush := func() {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
items := append([]T(nil), batch...)
|
||||
if err := w.flush(w.workerCtx, items); err != nil {
|
||||
if w.onFlushError != nil {
|
||||
w.onFlushError(w.workerCtx, len(items), err)
|
||||
}
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
|
||||
defer func() {
|
||||
flush()
|
||||
close(w.done)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case item, ok := <-w.ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
batch = append(batch, item)
|
||||
if len(batch) >= w.cfg.MaxBatchSize {
|
||||
flush()
|
||||
}
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Writer[T]) notifyDrop(item T) {
|
||||
if w.onDrop == nil {
|
||||
return
|
||||
}
|
||||
w.onDrop(item)
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package batchwriter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
)
|
||||
|
||||
func TestNewRejectsInvalidConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := New[int](Config{}, func(context.Context, []int) error { return nil })
|
||||
if err == nil {
|
||||
t.Fatal("New() = nil, want validation error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRejectsNilFlushFunc(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
_, err := New[int](cfg, nil)
|
||||
if !errors.Is(err, errNilFlushFunc) {
|
||||
t.Fatalf("New() error = %v, want %v", err, errNilFlushFunc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterFlushesOnMaxBatchSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
batches [][]int
|
||||
)
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxBatchSize = 3
|
||||
cfg.FlushInterval = time.Hour
|
||||
|
||||
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
batches = append(batches, append([]int(nil), items...))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
writer.Start(context.Background())
|
||||
t.Cleanup(func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := writer.Stop(stopCtx); err != nil {
|
||||
t.Fatalf("Stop() error = %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for i := range 3 {
|
||||
if !writer.TryEnqueue(i + 1) {
|
||||
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
ready := len(batches) == 1
|
||||
mu.Unlock()
|
||||
if ready || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
got := batches
|
||||
mu.Unlock()
|
||||
|
||||
want := [][]int{{1, 2, 3}}
|
||||
if diff := cmp.Diff(want, got); diff != "" {
|
||||
t.Fatalf("flush batches mismatch (-want +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterFlushesOnInterval(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
batch []int
|
||||
)
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxBatchSize = 100
|
||||
cfg.FlushInterval = 20 * time.Millisecond
|
||||
|
||||
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
batch = append([]int(nil), items...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
writer.Start(context.Background())
|
||||
t.Cleanup(func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := writer.Stop(stopCtx); err != nil {
|
||||
t.Fatalf("Stop() error = %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if !writer.TryEnqueue(42) {
|
||||
t.Fatal("TryEnqueue() = false, want true")
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
ready := len(batch) == 1
|
||||
mu.Unlock()
|
||||
if ready || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
got := batch
|
||||
mu.Unlock()
|
||||
|
||||
want := []int{42}
|
||||
if diff := cmp.Diff(want, got); diff != "" {
|
||||
t.Fatalf("interval flush mismatch (-want +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterTryEnqueueDropsWhenFull(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.QueueSize = 1
|
||||
cfg.MaxBatchSize = 10
|
||||
cfg.FlushInterval = time.Hour
|
||||
|
||||
var dropped int
|
||||
writer, err := New[int](cfg, func(context.Context, []int) error { return nil }, WithDropHandler[int](func(int) {
|
||||
dropped++
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
writer.Start(context.Background())
|
||||
t.Cleanup(func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_ = writer.Stop(stopCtx)
|
||||
})
|
||||
|
||||
if !writer.TryEnqueue(1) {
|
||||
t.Fatal("TryEnqueue(1) = false, want true")
|
||||
}
|
||||
if writer.TryEnqueue(2) {
|
||||
t.Fatal("TryEnqueue(2) = true, want false")
|
||||
}
|
||||
if !writer.IsFull() {
|
||||
t.Fatal("IsFull() = false, want true")
|
||||
}
|
||||
if dropped != 1 {
|
||||
t.Fatalf("dropped = %d, want 1", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterStopDrainsQueuedItems(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxBatchSize = 10
|
||||
cfg.FlushInterval = time.Hour
|
||||
|
||||
var flushed []int
|
||||
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
|
||||
flushed = append(flushed, items...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
writer.Start(context.Background())
|
||||
for i := range 2 {
|
||||
if !writer.TryEnqueue(i + 1) {
|
||||
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := writer.Stop(stopCtx); err != nil {
|
||||
t.Fatalf("Stop() error = %v", err)
|
||||
}
|
||||
|
||||
want := []int{1, 2}
|
||||
if diff := cmp.Diff(want, flushed); diff != "" {
|
||||
t.Fatalf("Stop() drain mismatch (-want +got):\n%s", diff)
|
||||
}
|
||||
if writer.Running() {
|
||||
t.Fatal("Running() = true after Stop(), want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterInvokesFlushErrorHandler(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxBatchSize = 1
|
||||
cfg.FlushInterval = time.Hour
|
||||
|
||||
flushErr := errors.New("flush failed")
|
||||
var (
|
||||
mu sync.Mutex
|
||||
errCount int
|
||||
batchSize int
|
||||
)
|
||||
|
||||
writer, err := New[int](cfg, func(context.Context, []int) error {
|
||||
return flushErr
|
||||
}, WithFlushErrorHandler[int](func(_ context.Context, size int, err error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
errCount++
|
||||
batchSize = size
|
||||
if !errors.Is(err, flushErr) {
|
||||
t.Errorf("flush error = %v, want %v", err, flushErr)
|
||||
}
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
writer.Start(context.Background())
|
||||
t.Cleanup(func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_ = writer.Stop(stopCtx)
|
||||
})
|
||||
|
||||
if !writer.TryEnqueue(7) {
|
||||
t.Fatal("TryEnqueue() = false, want true")
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
ready := errCount == 1
|
||||
mu.Unlock()
|
||||
if ready || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
gotCount := errCount
|
||||
gotSize := batchSize
|
||||
mu.Unlock()
|
||||
|
||||
if gotCount != 1 {
|
||||
t.Fatalf("flush error handler count = %d, want 1", gotCount)
|
||||
}
|
||||
if gotSize != 1 {
|
||||
t.Fatalf("flush error handler batch size = %d, want 1", gotSize)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package db 提供数据库连接与基础设施
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
clickhouseDriver "gorm.io/driver/clickhouse"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/plugin/opentelemetry/tracing"
|
||||
)
|
||||
|
||||
const (
|
||||
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
|
||||
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
|
||||
)
|
||||
|
||||
var (
|
||||
// ChConn ClickHouse 原生连接实例,用于批量写入
|
||||
ChConn driver.Conn
|
||||
|
||||
chDB *gorm.DB
|
||||
)
|
||||
|
||||
func init() {
|
||||
if !config.Config.ClickHouse.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
cfg := config.Config.ClickHouse
|
||||
if cfg.Database == "" {
|
||||
log.Fatalf("[ClickHouse] database name is required (expected: wavelet)\n")
|
||||
}
|
||||
|
||||
opts := buildClickHouseOptions()
|
||||
|
||||
var err error
|
||||
ChConn, err = clickhouse.Open(opts)
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = ChConn.Ping(context.Background()); err != nil {
|
||||
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
|
||||
}
|
||||
|
||||
chDB, err = gorm.Open(clickhouseDriver.New(clickhouseDriver.Config{
|
||||
DSN: buildClickHouseDSN(),
|
||||
}), &gorm.Config{
|
||||
SkipDefaultTransaction: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init gorm connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = chDB.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", cfg.Database),
|
||||
attribute.String("db.system", "ClickHouse"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[ClickHouse] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB, err := chDB.DB()
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] load sql db failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB.SetMaxIdleConns(cfg.MaxIdleConn)
|
||||
sqlDB.SetMaxOpenConns(cfg.MaxOpenConn)
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(cfg.ConnMaxLifetime) * time.Second)
|
||||
|
||||
log.Println("[ClickHouse] connection established successfully")
|
||||
}
|
||||
|
||||
func buildClickHouseOptions() *clickhouse.Options {
|
||||
cfg := config.Config.ClickHouse
|
||||
|
||||
return &clickhouse.Options{
|
||||
Addr: cfg.Hosts,
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
Settings: clickhouse.Settings{
|
||||
"max_execution_time": clickhouseMaxExecTime,
|
||||
},
|
||||
Compression: &clickhouse.Compression{
|
||||
Method: clickhouse.CompressionLZ4,
|
||||
},
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
MaxOpenConns: cfg.MaxOpenConn,
|
||||
MaxIdleConns: cfg.MaxIdleConn,
|
||||
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
|
||||
BlockBufferSize: cfg.BlockBufferSize,
|
||||
}
|
||||
}
|
||||
|
||||
func buildClickHouseDSN() string {
|
||||
cfg := config.Config.ClickHouse
|
||||
|
||||
chURL := &url.URL{
|
||||
Scheme: "clickhouse",
|
||||
Host: strings.Join(cfg.Hosts, ","),
|
||||
Path: "/" + cfg.Database,
|
||||
}
|
||||
if cfg.Username != "" || cfg.Password != "" {
|
||||
chURL.User = url.UserPassword(cfg.Username, cfg.Password)
|
||||
}
|
||||
|
||||
query := chURL.Query()
|
||||
query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout))
|
||||
query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor))
|
||||
query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime))
|
||||
chURL.RawQuery = query.Encode()
|
||||
|
||||
return chURL.String()
|
||||
}
|
||||
|
||||
// ChDB returns a context-aware GORM ClickHouse instance.
|
||||
func ChDB(ctx context.Context) *gorm.DB {
|
||||
if chDB == nil {
|
||||
return nil
|
||||
}
|
||||
return chDB.WithContext(ctx)
|
||||
}
|
||||
|
||||
// SetChDBForTest sets the package-level ClickHouse GORM instance for testing.
|
||||
func SetChDBForTest(d *gorm.DB) {
|
||||
chDB = d
|
||||
}
|
||||
|
||||
// SetChConnForTest sets the package-level native ClickHouse connection for testing.
|
||||
func SetChConnForTest(c driver.Conn) {
|
||||
ChConn = c
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package db
|
||||
|
||||
const (
|
||||
errRedisHashSetFailed = "failed to set redis hash: %w"
|
||||
errRedisHashDeleteFailed = "failed to delete redis hash field: %w"
|
||||
errUnmarshalDataFailed = "failed to unmarshal data: %w"
|
||||
errMarshalDataFailed = "failed to marshal data: %w"
|
||||
errRedisKeySetFailed = "failed to set redis key: %w"
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package idgen 提供分布式 ID 生成器
|
||||
package idgen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/bwmarrin/snowflake"
|
||||
)
|
||||
|
||||
// 2025-12-01 00:00:00 UTC 的毫秒时间戳
|
||||
const epoch int64 = 1764547200000
|
||||
|
||||
const maxNegativeIDRetries = 3
|
||||
|
||||
var node *snowflake.Node
|
||||
|
||||
func init() {
|
||||
snowflake.Epoch = epoch
|
||||
|
||||
nodeID := config.Config.App.NodeID
|
||||
var err error
|
||||
node, err = snowflake.NewNode(nodeID)
|
||||
if err != nil {
|
||||
log.Fatalf("[Snowflake] init failed: %v\n", err)
|
||||
}
|
||||
log.Printf("[Snowflake] initialized with node ID: %d, epoch: 2025-12-01\n", nodeID)
|
||||
}
|
||||
|
||||
// NextUint64ID 生成下一个分布式唯一 ID。
|
||||
// 理论上不应出现负值;若出现则最多重试 maxNegativeIDRetries 次,仍失败则 panic。
|
||||
func NextUint64ID() uint64 {
|
||||
for attempt := 1; attempt <= maxNegativeIDRetries; attempt++ {
|
||||
id := node.Generate().Int64()
|
||||
if id >= 0 {
|
||||
return uint64(id)
|
||||
}
|
||||
log.Printf("[Snowflake] generated negative ID: %d (attempt %d/%d)", id, attempt, maxNegativeIDRetries)
|
||||
}
|
||||
panic(fmt.Sprintf("[Snowflake] generated negative ID after %d attempts", maxNegativeIDRetries))
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package idgen
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNextUint64ID(t *testing.T) {
|
||||
id := NextUint64ID()
|
||||
assert.NotZero(t, id)
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"io/fs"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
clickhouseMigrationDir = "goose/clickhouse"
|
||||
clickhouseGooseVersionTable = "goose_clickhouse_version"
|
||||
clickhouseMaxExecTime = 60
|
||||
clickhouseReadTimeoutFactor = 2
|
||||
)
|
||||
|
||||
// clickhouseMigrationFS contains SQL migrations under goose/clickhouse.
|
||||
//
|
||||
//go:embed goose/clickhouse/*.sql
|
||||
var clickhouseMigrationFS embed.FS
|
||||
|
||||
// MigrateClickHouse runs goose migrations against ClickHouse when enabled.
|
||||
func MigrateClickHouse() Report {
|
||||
if !config.Config.ClickHouse.Enabled {
|
||||
return Report{Backend: "ClickHouse"}
|
||||
}
|
||||
|
||||
cfg := config.Config.ClickHouse
|
||||
sqlDB := clickhouse.OpenDB(&clickhouse.Options{
|
||||
Addr: cfg.Hosts,
|
||||
Auth: clickhouse.Auth{
|
||||
Database: cfg.Database,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
Settings: clickhouse.Settings{
|
||||
"max_execution_time": clickhouseMaxExecTime,
|
||||
},
|
||||
Compression: &clickhouse.Compression{
|
||||
Method: clickhouse.CompressionLZ4,
|
||||
},
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
MaxOpenConns: cfg.MaxOpenConn,
|
||||
MaxIdleConns: cfg.MaxIdleConn,
|
||||
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
|
||||
BlockBufferSize: cfg.BlockBufferSize,
|
||||
})
|
||||
|
||||
subFS, err := fs.Sub(clickhouseMigrationFS, "goose/clickhouse")
|
||||
if err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] get sub fs failed: %v\n", err)
|
||||
}
|
||||
|
||||
provider, err := goose.NewProvider(
|
||||
"clickhouse",
|
||||
sqlDB,
|
||||
subFS,
|
||||
goose.WithTableName(clickhouseGooseVersionTable),
|
||||
goose.WithDisableGlobalRegistry(true),
|
||||
)
|
||||
if err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] create goose provider failed: %v\n", err)
|
||||
}
|
||||
previousVersion, err := provider.GetDBVersion(context.Background())
|
||||
if err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] get goose version failed: %v\n", err)
|
||||
}
|
||||
|
||||
if _, err := provider.Up(context.Background()); err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] goose migrate failed: %v\n", err)
|
||||
}
|
||||
currentVersion, err := provider.GetDBVersion(context.Background())
|
||||
if err != nil {
|
||||
closeClickHouseDB(sqlDB)
|
||||
log.Fatalf("[ClickHouse] get migrated goose version failed: %v\n", err)
|
||||
}
|
||||
closeClickHouseDB(sqlDB)
|
||||
|
||||
log.Println("[ClickHouse] goose migrate success")
|
||||
return Report{
|
||||
Backend: "ClickHouse",
|
||||
Enabled: true,
|
||||
Version: currentVersion,
|
||||
Applied: currentVersion != previousVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func closeClickHouseDB(sqlDB *sql.DB) {
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
log.Printf("[ClickHouse] close sql db failed: %v\n", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func TestClickHouseMigrationFilesEmbedded(t *testing.T) {
|
||||
entries, err := clickhouseMigrationFS.ReadDir(clickhouseMigrationDir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir(%q) error = %v", clickhouseMigrationDir, err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatal("expected embedded ClickHouse migrations, got none")
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if entry.Name() == "202606190001_create_user_access_logs.sql" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("expected 202606190001_create_user_access_logs.sql in embedded migrations")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseGooseDialect(t *testing.T) {
|
||||
if err := goose.SetDialect("clickhouse"); err != nil {
|
||||
t.Fatalf("SetDialect(clickhouse) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateClickHouseSkipsWhenDisabled(t *testing.T) {
|
||||
previousEnabled := config.Config.ClickHouse.Enabled
|
||||
config.Config.ClickHouse.Enabled = false
|
||||
t.Cleanup(func() {
|
||||
config.Config.ClickHouse.Enabled = previousEnabled
|
||||
})
|
||||
|
||||
MigrateClickHouse()
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS w_user_access_logs
|
||||
(
|
||||
id UInt64,
|
||||
user_id UInt64,
|
||||
path String,
|
||||
method String,
|
||||
ip String,
|
||||
user_agent String,
|
||||
headers String,
|
||||
status Int32,
|
||||
latency Int64,
|
||||
created_at DateTime
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (created_at, ip, user_id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_user_access_logs;
|
||||
@@ -0,0 +1,184 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id BIGINT PRIMARY KEY,
|
||||
username VARCHAR(64) UNIQUE,
|
||||
password VARCHAR(255),
|
||||
nickname VARCHAR(255),
|
||||
email VARCHAR(255),
|
||||
avatar_url VARCHAR(255),
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
is_admin BOOLEAN DEFAULT FALSE,
|
||||
bio VARCHAR(500),
|
||||
phone VARCHAR(32),
|
||||
gender VARCHAR(16),
|
||||
website VARCHAR(255),
|
||||
location VARCHAR(255),
|
||||
last_login_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_sources (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(80) NOT NULL UNIQUE,
|
||||
type VARCHAR(20) NOT NULL,
|
||||
display_name VARCHAR(100),
|
||||
is_active BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
client_id VARCHAR(255),
|
||||
client_secret VARCHAR(1024),
|
||||
openid_discovery_url VARCHAR(1024),
|
||||
scopes VARCHAR(255),
|
||||
icon_url VARCHAR(1024),
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS external_accounts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
auth_source_id BIGINT,
|
||||
user_id BIGINT NOT NULL,
|
||||
external_id VARCHAR(255) NOT NULL,
|
||||
external_username VARCHAR(255),
|
||||
email VARCHAR(255),
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS system_configs (
|
||||
key VARCHAR(64) PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
type VARCHAR(32) NOT NULL DEFAULT 'system',
|
||||
visibility INTEGER NOT NULL DEFAULT 0,
|
||||
description VARCHAR(255),
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS uploads (
|
||||
id BIGINT PRIMARY KEY,
|
||||
user_id BIGINT NOT NULL,
|
||||
file_name VARCHAR(255) NOT NULL,
|
||||
file_path VARCHAR(500) NOT NULL,
|
||||
file_size BIGINT NOT NULL,
|
||||
mime_type VARCHAR(100) NOT NULL,
|
||||
extension VARCHAR(50) NOT NULL,
|
||||
hash VARCHAR(64),
|
||||
storage_driver VARCHAR(50) NOT NULL,
|
||||
type VARCHAR(50) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL,
|
||||
metadata JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS access_tokens (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
user_id BIGINT NOT NULL,
|
||||
name VARCHAR(128) NOT NULL,
|
||||
token_hash VARCHAR(64) NOT NULL UNIQUE,
|
||||
masked_token VARCHAR(64) NOT NULL,
|
||||
last_used_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS task_executions (
|
||||
id BIGINT PRIMARY KEY,
|
||||
task_id VARCHAR(128) NOT NULL UNIQUE,
|
||||
task_type VARCHAR(64) NOT NULL,
|
||||
task_name VARCHAR(128),
|
||||
status VARCHAR(32) NOT NULL,
|
||||
retryable BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
max_retry INTEGER NOT NULL DEFAULT 0,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
log TEXT,
|
||||
error_message TEXT,
|
||||
result TEXT,
|
||||
started_at TIMESTAMPTZ,
|
||||
finished_at TIMESTAMPTZ,
|
||||
duration BIGINT,
|
||||
payload TEXT,
|
||||
triggered_by VARCHAR(32) NOT NULL DEFAULT 'system',
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS templates (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
key VARCHAR(80) NOT NULL UNIQUE,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
type VARCHAR(20) NOT NULL DEFAULT 'email',
|
||||
subject VARCHAR(255),
|
||||
content TEXT NOT NULL,
|
||||
description VARCHAR(255),
|
||||
is_system BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
|
||||
|
||||
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
|
||||
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_difficulty', '4', 'system', 0, '人机验证 PoW 难度(目标前缀长度)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_ttl_seconds', '600', 'system', 0, '人机验证难题有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_token_ttl_seconds', '1200', 'system', 0, '人机验证兑换凭证有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('server_address', '', 'system', 0, '服务器地址(用于跨域源控制,不设定则允许任意源)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_host', '', 'system', 0, 'SMTP 服务器地址(例如 smtp.example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_port', '587', 'system', 0, 'SMTP 端口(例如 587 或 465)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_username', '', 'system', 0, 'SMTP 账户(如 sender@example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_password', '', 'system', 0, 'SMTP 访问凭证(授权码/密码)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('upload_allowed_extensions', 'jpg,png,webp', 'system', 1, '允许上传的图片扩展名(逗号分隔)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('site_name', 'Wavelet', 'system', 1, '系统平台的展示名称', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('password_login_enabled', 'true', 'system', 1, '是否允许使用账号密码登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('registration_enabled', 'true', 'system', 1, '控制普通用户是否可以自主注册(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('password_register_enabled', 'true', 'system', 1, '是否允许通过密码创建本地账号', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('oidc_login_enabled', 'true', 'system', 1, '是否允许使用第三方 OIDC 认证源登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('max_api_keys_per_user', '5', 'business', 1, '限制每个普通用户可以创建的 API Key 最大数量', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('email_login_verification_enabled', 'false', 'system', 1, '是否开启邮箱登录验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('email_register_verification_enabled', 'false', 'system', 1, '是否开启邮箱注册验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('menu_display_config', '{}', 'system', 1, '目录显示配置(JSON 字符串,格式为 {url: enabled})', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('search_engine_indexing_enabled', 'false', 'system', 1, '是否允许搜索引擎爬取/检索该站点(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('storage_config', '{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}', 'system', 0, '文件存储驱动及连接配置(JSON)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
INSERT INTO users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
|
||||
VALUES (1, 'admin', '12345678', 'Administrator', '', TRUE, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (username) DO NOTHING;
|
||||
|
||||
INSERT INTO templates (key, name, type, subject, content, description, is_system, created_at, updated_at) VALUES
|
||||
('login_email', '登录验证码邮件', 'email', 'Wavelet 登录验证码', '<h3>Wavelet 登录验证</h3><p>您的登录验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿将验证码泄露给他人。</p>', '用户密码登录时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('register_email', '注册验证码邮件', 'email', 'Wavelet 注册验证码', '<h3>Wavelet 注册验证</h3><p>您的注册验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿泄露给他人。</p>', '用户注册时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS templates;
|
||||
DROP TABLE IF EXISTS task_executions;
|
||||
DROP TABLE IF EXISTS access_tokens;
|
||||
DROP TABLE IF EXISTS uploads;
|
||||
DROP TABLE IF EXISTS system_configs;
|
||||
DROP TABLE IF EXISTS external_accounts;
|
||||
DROP TABLE IF EXISTS auth_sources;
|
||||
DROP TABLE IF EXISTS users;
|
||||
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS schedules (
|
||||
id BIGINT PRIMARY KEY,
|
||||
name VARCHAR(128) NOT NULL,
|
||||
task_type VARCHAR(64) NOT NULL,
|
||||
cron VARCHAR(64) NOT NULL,
|
||||
payload TEXT,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
|
||||
|
||||
-- Seed initial cleanup task
|
||||
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
||||
VALUES (1, '清理未使用上传', 'cleanup_unused_uploads', '0 */2 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS schedules;
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE access_tokens ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE access_tokens DROP COLUMN IF EXISTS is_admin;
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE access_tokens DROP COLUMN IF EXISTS last_used_at;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE access_tokens ADD COLUMN last_used_at TIMESTAMPTZ;
|
||||
-- +goose StatementEnd
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE schedules ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (START WITH 100);
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE schedules ALTER COLUMN id DROP IDENTITY IF EXISTS;
|
||||
-- +goose StatementEnd
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE users RENAME TO w_users;
|
||||
ALTER TABLE auth_sources RENAME TO w_auth_sources;
|
||||
ALTER TABLE external_accounts RENAME TO w_external_accounts;
|
||||
ALTER TABLE system_configs RENAME TO w_system_configs;
|
||||
ALTER TABLE uploads RENAME TO w_uploads;
|
||||
ALTER TABLE access_tokens RENAME TO w_access_tokens;
|
||||
ALTER TABLE task_executions RENAME TO w_task_executions;
|
||||
ALTER TABLE templates RENAME TO w_templates;
|
||||
ALTER TABLE schedules RENAME TO w_schedules;
|
||||
|
||||
-- Rename indexes for consistency
|
||||
ALTER INDEX IF EXISTS idx_users_email RENAME TO idx_w_users_email;
|
||||
ALTER INDEX IF EXISTS idx_users_is_active RENAME TO idx_w_users_is_active;
|
||||
ALTER INDEX IF EXISTS idx_users_last_login_at RENAME TO idx_w_users_last_login_at;
|
||||
ALTER INDEX IF EXISTS idx_users_created_at RENAME TO idx_w_users_created_at;
|
||||
ALTER INDEX IF EXISTS idx_auth_sources_is_active RENAME TO idx_w_auth_sources_is_active;
|
||||
ALTER INDEX IF EXISTS idx_external_accounts_auth_source_id RENAME TO idx_w_external_accounts_auth_source_id;
|
||||
ALTER INDEX IF EXISTS idx_external_accounts_user_id RENAME TO idx_w_external_accounts_user_id;
|
||||
ALTER INDEX IF EXISTS idx_external_accounts_source_external RENAME TO idx_w_external_accounts_source_external;
|
||||
ALTER INDEX IF EXISTS idx_uploads_user_id RENAME TO idx_w_uploads_user_id;
|
||||
ALTER INDEX IF EXISTS idx_uploads_file_path RENAME TO idx_w_uploads_file_path;
|
||||
ALTER INDEX IF EXISTS idx_uploads_hash RENAME TO idx_w_uploads_hash;
|
||||
ALTER INDEX IF EXISTS idx_uploads_type RENAME TO idx_w_uploads_type;
|
||||
ALTER INDEX IF EXISTS idx_access_tokens_user_id RENAME TO idx_w_access_tokens_user_id;
|
||||
ALTER INDEX IF EXISTS idx_task_executions_task_type RENAME TO idx_w_task_executions_task_type;
|
||||
ALTER INDEX IF EXISTS idx_task_executions_status RENAME TO idx_w_task_executions_status;
|
||||
ALTER INDEX IF EXISTS idx_task_executions_started_at RENAME TO idx_w_task_executions_started_at;
|
||||
ALTER INDEX IF EXISTS idx_task_executions_created_at RENAME TO idx_w_task_executions_created_at;
|
||||
ALTER INDEX IF EXISTS idx_templates_is_system RENAME TO idx_w_templates_is_system;
|
||||
ALTER INDEX IF EXISTS idx_templates_created_at RENAME TO idx_w_templates_created_at;
|
||||
ALTER INDEX IF EXISTS idx_templates_updated_at RENAME TO idx_w_templates_updated_at;
|
||||
ALTER INDEX IF EXISTS idx_schedules_is_active RENAME TO idx_w_schedules_is_active;
|
||||
|
||||
-- +goose Down
|
||||
ALTER INDEX IF EXISTS idx_w_schedules_is_active RENAME TO idx_schedules_is_active;
|
||||
ALTER INDEX IF EXISTS idx_w_templates_updated_at RENAME TO idx_templates_updated_at;
|
||||
ALTER INDEX IF EXISTS idx_w_templates_created_at RENAME TO idx_templates_created_at;
|
||||
ALTER INDEX IF EXISTS idx_w_templates_is_system RENAME TO idx_templates_is_system;
|
||||
ALTER INDEX IF EXISTS idx_w_task_executions_created_at RENAME TO idx_task_executions_created_at;
|
||||
ALTER INDEX IF EXISTS idx_w_task_executions_started_at RENAME TO idx_task_executions_started_at;
|
||||
ALTER INDEX IF EXISTS idx_w_task_executions_status RENAME TO idx_task_executions_status;
|
||||
ALTER INDEX IF EXISTS idx_w_task_executions_task_type RENAME TO idx_task_executions_task_type;
|
||||
ALTER INDEX IF EXISTS idx_w_access_tokens_user_id RENAME TO idx_access_tokens_user_id;
|
||||
ALTER INDEX IF EXISTS idx_w_uploads_type RENAME TO idx_uploads_type;
|
||||
ALTER INDEX IF EXISTS idx_w_uploads_hash RENAME TO idx_uploads_hash;
|
||||
ALTER INDEX IF EXISTS idx_w_uploads_file_path RENAME TO idx_uploads_file_path;
|
||||
ALTER INDEX IF EXISTS idx_w_uploads_user_id RENAME TO idx_uploads_user_id;
|
||||
ALTER INDEX IF EXISTS idx_w_external_accounts_source_external RENAME TO idx_external_accounts_source_external;
|
||||
ALTER INDEX IF EXISTS idx_w_external_accounts_user_id RENAME TO idx_external_accounts_user_id;
|
||||
ALTER INDEX IF EXISTS idx_w_external_accounts_auth_source_id RENAME TO idx_external_accounts_auth_source_id;
|
||||
ALTER INDEX IF EXISTS idx_w_auth_sources_is_active RENAME TO idx_auth_sources_is_active;
|
||||
ALTER INDEX IF EXISTS idx_w_users_created_at RENAME TO idx_users_created_at;
|
||||
ALTER INDEX IF EXISTS idx_w_users_last_login_at RENAME TO idx_users_last_login_at;
|
||||
ALTER INDEX IF EXISTS idx_w_users_is_active RENAME TO idx_users_is_active;
|
||||
ALTER INDEX IF EXISTS idx_w_users_email RENAME TO idx_users_email;
|
||||
|
||||
ALTER TABLE w_schedules RENAME TO schedules;
|
||||
ALTER TABLE w_templates RENAME TO templates;
|
||||
ALTER TABLE w_task_executions RENAME TO task_executions;
|
||||
ALTER TABLE w_access_tokens RENAME TO access_tokens;
|
||||
ALTER TABLE w_uploads RENAME TO uploads;
|
||||
ALTER TABLE w_system_configs RENAME TO system_configs;
|
||||
ALTER TABLE w_external_accounts RENAME TO external_accounts;
|
||||
ALTER TABLE w_auth_sources RENAME TO auth_sources;
|
||||
ALTER TABLE w_users RENAME TO users;
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES ('file_access_whitelist', '["avatar"]', 'system', 1, '免登录访问的文件业务类型白名单 (JSON 数组格式)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'file_access_whitelist';
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES
|
||||
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES
|
||||
('login_session_ttl_hours', '0', 'system', 0, '登录会话过期时间 (小时,0表示浏览器关闭后自动退出,-1表示永不过期)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'login_session_ttl_hours';
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES ('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'update_upstream_repository';
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE w_uploads ADD COLUMN access_mode INTEGER NOT NULL DEFAULT 0;
|
||||
UPDATE w_uploads SET access_mode = 1 WHERE type = 'avatar';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE w_uploads DROP COLUMN access_mode;
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE w_system_configs ALTER COLUMN value TYPE TEXT;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE w_system_configs ALTER COLUMN value TYPE VARCHAR(255);
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES (
|
||||
'storage_config',
|
||||
'{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}',
|
||||
'system',
|
||||
0,
|
||||
'文件存储驱动及连接配置(JSON)',
|
||||
CURRENT_TIMESTAMP,
|
||||
CURRENT_TIMESTAMP
|
||||
)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'storage_config';
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE w_push_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
event_key VARCHAR(80) NOT NULL UNIQUE,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
channels TEXT NOT NULL,
|
||||
targets TEXT NOT NULL,
|
||||
template TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_w_push_events_enabled ON w_push_events(enabled);
|
||||
|
||||
CREATE TABLE w_push_histories (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
event_key VARCHAR(80) NOT NULL,
|
||||
channel VARCHAR(50) NOT NULL,
|
||||
target VARCHAR(255) NOT NULL,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
level VARCHAR(20) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL,
|
||||
error_msg TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_w_push_histories_event ON w_push_histories(event_key);
|
||||
CREATE INDEX idx_w_push_histories_created ON w_push_histories(created_at);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_push_histories;
|
||||
DROP TABLE IF EXISTS w_push_events;
|
||||
@@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
|
||||
VALUES (999, 'system', '*', '系统', '', TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (username) DO NOTHING;
|
||||
SELECT setval(pg_get_serial_sequence('w_users', 'id'), COALESCE((SELECT MAX(id) FROM w_users), 1));
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_users WHERE username = 'system';
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE w_push_channels (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(80) NOT NULL UNIQUE,
|
||||
description VARCHAR(255),
|
||||
type VARCHAR(50) NOT NULL DEFAULT 'custom',
|
||||
token VARCHAR(100),
|
||||
url TEXT NOT NULL,
|
||||
other TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_w_push_channels_name ON w_push_channels(name);
|
||||
CREATE INDEX idx_w_push_channels_enabled ON w_push_channels(enabled);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_push_channels;
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
UPDATE w_schedules
|
||||
SET name = '系统定期垃圾清理',
|
||||
task_type = 'system_cleanup'
|
||||
WHERE id = 1;
|
||||
|
||||
-- +goose Down
|
||||
UPDATE w_schedules
|
||||
SET name = '清理未使用上传',
|
||||
task_type = 'cleanup_unused_uploads'
|
||||
WHERE id = 1;
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE w_push_events ADD COLUMN task_type VARCHAR(100) NOT NULL DEFAULT '';
|
||||
CREATE INDEX idx_w_push_events_task_type ON w_push_events(task_type);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS idx_w_push_events_task_type;
|
||||
ALTER TABLE w_push_events DROP COLUMN task_type;
|
||||
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
DELETE FROM w_system_configs WHERE key = 'push_config';
|
||||
DELETE FROM w_push_events WHERE event_key = 'admin_login';
|
||||
DELETE FROM w_system_configs WHERE key = 'push_global_token';
|
||||
|
||||
-- +goose Down
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS w_upload_stats (
|
||||
dimension VARCHAR(32) NOT NULL,
|
||||
stat_key VARCHAR(64) NOT NULL DEFAULT '',
|
||||
file_count BIGINT NOT NULL DEFAULT 0,
|
||||
file_size BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (dimension, stat_key)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_upload_stats;
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'type',
|
||||
COALESCE(NULLIF(type, ''), 'generic'),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'category',
|
||||
CASE
|
||||
WHEN LOWER(mime_type) LIKE 'image/%'
|
||||
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
|
||||
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
|
||||
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
|
||||
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
|
||||
OR LOWER(mime_type) LIKE '%zip%'
|
||||
OR LOWER(mime_type) LIKE '%tar%'
|
||||
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
|
||||
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
|
||||
OR LOWER(mime_type) LIKE 'text/%'
|
||||
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
|
||||
ELSE '其他'
|
||||
END,
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY 2
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'trend',
|
||||
TO_CHAR(created_at, 'YYYY-MM-DD'),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY TO_CHAR(created_at, 'YYYY-MM-DD')
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = EXCLUDED.file_count,
|
||||
file_size = EXCLUDED.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_upload_stats;
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
|
||||
ALTER TABLE w_uploads DROP COLUMN IF EXISTS storage_driver;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE w_uploads ADD COLUMN IF NOT EXISTS storage_driver VARCHAR(50) NOT NULL DEFAULT 'local';
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
|
||||
@@ -0,0 +1,184 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id BIGINT PRIMARY KEY,
|
||||
username VARCHAR(64) UNIQUE,
|
||||
password VARCHAR(255),
|
||||
nickname VARCHAR(255),
|
||||
email VARCHAR(255),
|
||||
avatar_url VARCHAR(255),
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
is_admin BOOLEAN DEFAULT FALSE,
|
||||
bio VARCHAR(500),
|
||||
phone VARCHAR(32),
|
||||
gender VARCHAR(16),
|
||||
website VARCHAR(255),
|
||||
location VARCHAR(255),
|
||||
last_login_at DATETIME,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name VARCHAR(80) NOT NULL UNIQUE,
|
||||
type VARCHAR(20) NOT NULL,
|
||||
display_name VARCHAR(100),
|
||||
is_active BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
client_id VARCHAR(255),
|
||||
client_secret VARCHAR(1024),
|
||||
openid_discovery_url VARCHAR(1024),
|
||||
scopes VARCHAR(255),
|
||||
icon_url VARCHAR(1024),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS external_accounts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
auth_source_id BIGINT,
|
||||
user_id BIGINT NOT NULL,
|
||||
external_id VARCHAR(255) NOT NULL,
|
||||
external_username VARCHAR(255),
|
||||
email VARCHAR(255),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS system_configs (
|
||||
key VARCHAR(64) PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
type VARCHAR(32) NOT NULL DEFAULT 'system',
|
||||
visibility INTEGER NOT NULL DEFAULT 0,
|
||||
description VARCHAR(255),
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS uploads (
|
||||
id BIGINT PRIMARY KEY,
|
||||
user_id BIGINT NOT NULL,
|
||||
file_name VARCHAR(255) NOT NULL,
|
||||
file_path VARCHAR(500) NOT NULL,
|
||||
file_size BIGINT NOT NULL,
|
||||
mime_type VARCHAR(100) NOT NULL,
|
||||
extension VARCHAR(50) NOT NULL,
|
||||
hash VARCHAR(64),
|
||||
storage_driver VARCHAR(50) NOT NULL,
|
||||
type VARCHAR(50) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL,
|
||||
metadata JSON,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS access_tokens (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id BIGINT NOT NULL,
|
||||
name VARCHAR(128) NOT NULL,
|
||||
token_hash VARCHAR(64) NOT NULL UNIQUE,
|
||||
masked_token VARCHAR(64) NOT NULL,
|
||||
last_used_at DATETIME,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS task_executions (
|
||||
id BIGINT PRIMARY KEY,
|
||||
task_id VARCHAR(128) NOT NULL UNIQUE,
|
||||
task_type VARCHAR(64) NOT NULL,
|
||||
task_name VARCHAR(128),
|
||||
status VARCHAR(32) NOT NULL,
|
||||
retryable BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
max_retry INTEGER NOT NULL DEFAULT 0,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
log TEXT,
|
||||
error_message TEXT,
|
||||
result TEXT,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
duration BIGINT,
|
||||
payload TEXT,
|
||||
triggered_by VARCHAR(32) NOT NULL DEFAULT 'system',
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS templates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
key VARCHAR(80) NOT NULL UNIQUE,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
type VARCHAR(20) NOT NULL DEFAULT 'email',
|
||||
subject VARCHAR(255),
|
||||
content TEXT NOT NULL,
|
||||
description VARCHAR(255),
|
||||
is_system BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
|
||||
|
||||
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
|
||||
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_difficulty', '4', 'system', 0, '人机验证 PoW 难度(目标前缀长度)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_ttl_seconds', '600', 'system', 0, '人机验证难题有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_token_ttl_seconds', '1200', 'system', 0, '人机验证兑换凭证有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('server_address', '', 'system', 0, '服务器地址(用于跨域源控制,不设定则允许任意源)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_host', '', 'system', 0, 'SMTP 服务器地址(例如 smtp.example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_port', '587', 'system', 0, 'SMTP 端口(例如 587 或 465)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_username', '', 'system', 0, 'SMTP 账户(如 sender@example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('smtp_password', '', 'system', 0, 'SMTP 访问凭证(授权码/密码)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('upload_allowed_extensions', 'jpg,png,webp', 'system', 1, '允许上传的图片扩展名(逗号分隔)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('site_name', 'Wavelet', 'system', 1, '系统平台的展示名称', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('password_login_enabled', 'true', 'system', 1, '是否允许使用账号密码登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('registration_enabled', 'true', 'system', 1, '控制普通用户是否可以自主注册(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('password_register_enabled', 'true', 'system', 1, '是否允许通过密码创建本地账号', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('oidc_login_enabled', 'true', 'system', 1, '是否允许使用第三方 OIDC 认证源登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('max_api_keys_per_user', '5', 'business', 1, '限制每个普通用户可以创建的 API Key 最大数量', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('email_login_verification_enabled', 'false', 'system', 1, '是否开启邮箱登录验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('email_register_verification_enabled', 'false', 'system', 1, '是否开启邮箱注册验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('menu_display_config', '{}', 'system', 1, '目录显示配置(JSON 字符串,格式为 {url: enabled})', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('search_engine_indexing_enabled', 'false', 'system', 1, '是否允许搜索引擎爬取/检索该站点(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('storage_config', '{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}', 'system', 0, '文件存储驱动及连接配置(JSON)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
INSERT INTO users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
|
||||
VALUES (1, 'admin', '12345678', 'Administrator', '', TRUE, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (username) DO NOTHING;
|
||||
|
||||
INSERT INTO templates (key, name, type, subject, content, description, is_system, created_at, updated_at) VALUES
|
||||
('login_email', '登录验证码邮件', 'email', 'Wavelet 登录验证码', '<h3>Wavelet 登录验证</h3><p>您的登录验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿将验证码泄露给他人。</p>', '用户密码登录时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('register_email', '注册验证码邮件', 'email', 'Wavelet 注册验证码', '<h3>Wavelet 注册验证</h3><p>您的注册验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿泄露给他人。</p>', '用户注册时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS templates;
|
||||
DROP TABLE IF EXISTS task_executions;
|
||||
DROP TABLE IF EXISTS access_tokens;
|
||||
DROP TABLE IF EXISTS uploads;
|
||||
DROP TABLE IF EXISTS system_configs;
|
||||
DROP TABLE IF EXISTS external_accounts;
|
||||
DROP TABLE IF EXISTS auth_sources;
|
||||
DROP TABLE IF EXISTS users;
|
||||
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS schedules (
|
||||
id BIGINT PRIMARY KEY,
|
||||
name VARCHAR(128) NOT NULL,
|
||||
task_type VARCHAR(64) NOT NULL,
|
||||
cron VARCHAR(64) NOT NULL,
|
||||
payload TEXT,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
|
||||
|
||||
-- Seed initial cleanup task
|
||||
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
||||
VALUES (1, '清理未使用上传', 'cleanup_unused_uploads', '0 */2 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS schedules;
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE access_tokens ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE access_tokens DROP COLUMN is_admin;
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE access_tokens DROP COLUMN last_used_at;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE access_tokens ADD COLUMN last_used_at DATETIME;
|
||||
-- +goose StatementEnd
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- 1. Rename existing schedules table
|
||||
ALTER TABLE schedules RENAME TO schedules_old;
|
||||
|
||||
-- 2. Create new schedules table with AUTOINCREMENT
|
||||
CREATE TABLE schedules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name VARCHAR(128) NOT NULL,
|
||||
task_type VARCHAR(64) NOT NULL,
|
||||
cron VARCHAR(64) NOT NULL,
|
||||
payload TEXT,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- 3. Copy existing data
|
||||
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
|
||||
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at FROM schedules_old;
|
||||
|
||||
-- 4. Drop the old table
|
||||
DROP TABLE schedules_old;
|
||||
|
||||
-- 5. Recreate index
|
||||
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- Re-creating table with AUTOINCREMENT cannot be undone simply without recreating table again.
|
||||
-- +goose StatementEnd
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE users RENAME TO w_users;
|
||||
ALTER TABLE auth_sources RENAME TO w_auth_sources;
|
||||
ALTER TABLE external_accounts RENAME TO w_external_accounts;
|
||||
ALTER TABLE system_configs RENAME TO w_system_configs;
|
||||
ALTER TABLE uploads RENAME TO w_uploads;
|
||||
ALTER TABLE access_tokens RENAME TO w_access_tokens;
|
||||
ALTER TABLE task_executions RENAME TO w_task_executions;
|
||||
ALTER TABLE templates RENAME TO w_templates;
|
||||
ALTER TABLE schedules RENAME TO w_schedules;
|
||||
|
||||
-- Drop and Recreate SQLite indexes
|
||||
DROP INDEX IF EXISTS idx_users_email;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_users_email ON w_users (email);
|
||||
DROP INDEX IF EXISTS idx_users_is_active;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_users_is_active ON w_users (is_active);
|
||||
DROP INDEX IF EXISTS idx_users_last_login_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_users_last_login_at ON w_users (last_login_at);
|
||||
DROP INDEX IF EXISTS idx_users_created_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_users_created_at ON w_users (created_at);
|
||||
|
||||
DROP INDEX IF EXISTS idx_auth_sources_is_active;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_auth_sources_is_active ON w_auth_sources (is_active);
|
||||
|
||||
DROP INDEX IF EXISTS idx_external_accounts_auth_source_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_external_accounts_auth_source_id ON w_external_accounts (auth_source_id);
|
||||
DROP INDEX IF EXISTS idx_external_accounts_user_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_external_accounts_user_id ON w_external_accounts (user_id);
|
||||
DROP INDEX IF EXISTS idx_external_accounts_source_external;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_w_external_accounts_source_external ON w_external_accounts (auth_source_id, external_id);
|
||||
|
||||
DROP INDEX IF EXISTS idx_uploads_user_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_user_id ON w_uploads (user_id);
|
||||
DROP INDEX IF EXISTS idx_uploads_file_path;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_file_path ON w_uploads (file_path);
|
||||
DROP INDEX IF EXISTS idx_uploads_hash;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash ON w_uploads (hash);
|
||||
DROP INDEX IF EXISTS idx_uploads_type;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_type ON w_uploads (type);
|
||||
|
||||
DROP INDEX IF EXISTS idx_access_tokens_user_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_access_tokens_user_id ON w_access_tokens (user_id);
|
||||
|
||||
DROP INDEX IF EXISTS idx_task_executions_task_type;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_task_executions_task_type ON w_task_executions (task_type);
|
||||
DROP INDEX IF EXISTS idx_task_executions_status;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_task_executions_status ON w_task_executions (status);
|
||||
DROP INDEX IF EXISTS idx_task_executions_started_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_task_executions_started_at ON w_task_executions (started_at);
|
||||
DROP INDEX IF EXISTS idx_task_executions_created_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_task_executions_created_at ON w_task_executions (created_at);
|
||||
|
||||
DROP INDEX IF EXISTS idx_templates_is_system;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_templates_is_system ON w_templates (is_system);
|
||||
DROP INDEX IF EXISTS idx_templates_created_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_templates_created_at ON w_templates (created_at);
|
||||
DROP INDEX IF EXISTS idx_templates_updated_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_templates_updated_at ON w_templates (updated_at);
|
||||
|
||||
DROP INDEX IF EXISTS idx_schedules_is_active;
|
||||
CREATE INDEX IF NOT EXISTS idx_w_schedules_is_active ON w_schedules (is_active);
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE w_schedules RENAME TO schedules;
|
||||
ALTER TABLE w_templates RENAME TO templates;
|
||||
ALTER TABLE w_task_executions RENAME TO task_executions;
|
||||
ALTER TABLE w_access_tokens RENAME TO access_tokens;
|
||||
ALTER TABLE w_uploads RENAME TO uploads;
|
||||
ALTER TABLE w_system_configs RENAME TO system_configs;
|
||||
ALTER TABLE w_external_accounts RENAME TO external_accounts;
|
||||
ALTER TABLE w_auth_sources RENAME TO auth_sources;
|
||||
ALTER TABLE w_users RENAME TO users;
|
||||
|
||||
-- Revert indexes
|
||||
DROP INDEX IF EXISTS idx_w_users_email;
|
||||
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
|
||||
DROP INDEX IF EXISTS idx_w_users_is_active;
|
||||
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
|
||||
DROP INDEX IF EXISTS idx_w_users_last_login_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
|
||||
DROP INDEX IF EXISTS idx_w_users_created_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
|
||||
|
||||
DROP INDEX IF EXISTS idx_w_auth_sources_is_active;
|
||||
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
|
||||
|
||||
DROP INDEX IF EXISTS idx_w_external_accounts_auth_source_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
|
||||
DROP INDEX IF EXISTS idx_w_external_accounts_user_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
|
||||
DROP INDEX IF EXISTS idx_w_external_accounts_source_external;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
|
||||
|
||||
DROP INDEX IF EXISTS idx_w_uploads_user_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
|
||||
DROP INDEX IF EXISTS idx_w_uploads_file_path;
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
|
||||
DROP INDEX IF EXISTS idx_w_uploads_hash;
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
|
||||
DROP INDEX IF EXISTS idx_w_uploads_type;
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
|
||||
|
||||
DROP INDEX IF EXISTS idx_w_access_tokens_user_id;
|
||||
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
|
||||
|
||||
DROP INDEX IF EXISTS idx_w_task_executions_task_type;
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
|
||||
DROP INDEX IF EXISTS idx_w_task_executions_status;
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
|
||||
DROP INDEX IF EXISTS idx_w_task_executions_started_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
|
||||
DROP INDEX IF EXISTS idx_w_task_executions_created_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
|
||||
|
||||
DROP INDEX IF EXISTS idx_w_templates_is_system;
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
|
||||
DROP INDEX IF EXISTS idx_w_templates_created_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
|
||||
DROP INDEX IF EXISTS idx_w_templates_updated_at;
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
|
||||
|
||||
DROP INDEX IF EXISTS idx_w_schedules_is_active;
|
||||
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES ('file_access_whitelist', '["avatar"]', 'system', 1, '免登录访问的文件业务类型白名单 (JSON 数组格式)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'file_access_whitelist';
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES
|
||||
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES
|
||||
('login_session_ttl_hours', '0', 'system', 0, '登录会话过期时间 (小时,0表示浏览器关闭后自动退出,-1表示永不过期)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'login_session_ttl_hours';
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES ('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'update_upstream_repository';
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE w_uploads ADD COLUMN access_mode INTEGER NOT NULL DEFAULT 0;
|
||||
UPDATE w_uploads SET access_mode = 1 WHERE type = 'avatar';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE w_uploads DROP COLUMN access_mode;
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
-- SQLite stores VARCHAR and TEXT with the same TEXT affinity.
|
||||
SELECT 1;
|
||||
|
||||
-- +goose Down
|
||||
SELECT 1;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
|
||||
VALUES (
|
||||
'storage_config',
|
||||
'{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}',
|
||||
'system',
|
||||
0,
|
||||
'文件存储驱动及连接配置(JSON)',
|
||||
CURRENT_TIMESTAMP,
|
||||
CURRENT_TIMESTAMP
|
||||
)
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_system_configs WHERE key = 'storage_config';
|
||||
@@ -0,0 +1,34 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE w_push_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
event_key TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
channels TEXT NOT NULL,
|
||||
targets TEXT NOT NULL,
|
||||
template TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_w_push_events_enabled ON w_push_events(enabled);
|
||||
|
||||
CREATE TABLE w_push_histories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
event_key TEXT NOT NULL,
|
||||
channel TEXT NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
level TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
error_msg TEXT,
|
||||
created_at DATETIME NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_w_push_histories_event ON w_push_histories(event_key);
|
||||
CREATE INDEX idx_w_push_histories_created ON w_push_histories(created_at);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_push_histories;
|
||||
DROP TABLE IF EXISTS w_push_events;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
|
||||
VALUES (999, 'system', '*', '系统', '', TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (username) DO NOTHING;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_users WHERE username = 'system';
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE w_push_channels (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
type TEXT NOT NULL DEFAULT 'custom',
|
||||
token TEXT,
|
||||
url TEXT NOT NULL,
|
||||
other TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_w_push_channels_name ON w_push_channels(name);
|
||||
CREATE INDEX idx_w_push_channels_enabled ON w_push_channels(enabled);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_push_channels;
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
UPDATE w_schedules
|
||||
SET name = '系统定期垃圾清理',
|
||||
task_type = 'system_cleanup'
|
||||
WHERE id = 1;
|
||||
|
||||
-- +goose Down
|
||||
UPDATE w_schedules
|
||||
SET name = '清理未使用上传',
|
||||
task_type = 'cleanup_unused_uploads'
|
||||
WHERE id = 1;
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE w_push_events ADD COLUMN task_type VARCHAR(100) NOT NULL DEFAULT '';
|
||||
CREATE INDEX idx_w_push_events_task_type ON w_push_events(task_type);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS idx_w_push_events_task_type;
|
||||
-- SQLite does not support DROP COLUMN in older versions easily, but standard ALTER TABLE DROP COLUMN works in SQLite 3.35.0+.
|
||||
-- We can write standard DROP COLUMN.
|
||||
ALTER TABLE w_push_events DROP COLUMN task_type;
|
||||
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
DELETE FROM w_system_configs WHERE key = 'push_config';
|
||||
DELETE FROM w_push_events WHERE event_key = 'admin_login';
|
||||
DELETE FROM w_system_configs WHERE key = 'push_global_token';
|
||||
|
||||
-- +goose Down
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
|
||||
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS w_upload_stats (
|
||||
dimension VARCHAR(32) NOT NULL,
|
||||
stat_key VARCHAR(64) NOT NULL DEFAULT '',
|
||||
file_count BIGINT NOT NULL DEFAULT 0,
|
||||
file_size BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (dimension, stat_key)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS w_upload_stats;
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
-- +goose Up
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'type',
|
||||
COALESCE(NULLIF(type, ''), 'generic'),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'category',
|
||||
CASE
|
||||
WHEN LOWER(mime_type) LIKE 'image/%'
|
||||
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
|
||||
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
|
||||
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
|
||||
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
|
||||
OR LOWER(mime_type) LIKE '%zip%'
|
||||
OR LOWER(mime_type) LIKE '%tar%'
|
||||
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
|
||||
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
|
||||
OR LOWER(mime_type) LIKE 'text/%'
|
||||
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
|
||||
ELSE '其他'
|
||||
END,
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY 2
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
|
||||
SELECT
|
||||
'trend',
|
||||
STRFTIME('%Y-%m-%d', created_at),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(file_size), 0)
|
||||
FROM w_uploads
|
||||
WHERE status != 'deleted'
|
||||
GROUP BY STRFTIME('%Y-%m-%d', created_at)
|
||||
ON CONFLICT (dimension, stat_key) DO UPDATE SET
|
||||
file_count = excluded.file_count,
|
||||
file_size = excluded.file_size,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
-- +goose Down
|
||||
DELETE FROM w_upload_stats;
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
|
||||
-- SQLite lacks DROP COLUMN IF EXISTS; goose runs this only after initial schema created the column.
|
||||
ALTER TABLE w_uploads DROP COLUMN storage_driver;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE w_uploads ADD COLUMN storage_driver VARCHAR(50) NOT NULL DEFAULT 'local';
|
||||
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package migrator 提供数据库迁移功能
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"log"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
// migrationFS contains SQL migrations under goose/<dialect>.
|
||||
//
|
||||
//go:embed goose/postgres/*.sql goose/sqlite/*.sql
|
||||
var migrationFS embed.FS
|
||||
|
||||
// dbType 返回当前数据库类型名称(用于日志输出)
|
||||
func dbType() string {
|
||||
if !config.Config.Database.Enabled {
|
||||
return "SQLite"
|
||||
}
|
||||
return "PostgreSQL"
|
||||
}
|
||||
|
||||
const (
|
||||
dialectSqlite = "sqlite3"
|
||||
dialectPostgres = "postgres"
|
||||
cascadeSuffix = " CASCADE"
|
||||
)
|
||||
|
||||
// Report describes the database migration state observed during startup.
|
||||
type Report struct {
|
||||
Backend string
|
||||
Enabled bool
|
||||
Version int64
|
||||
Applied bool
|
||||
}
|
||||
|
||||
func gooseDialect() string {
|
||||
if !config.Config.Database.Enabled {
|
||||
return dialectSqlite
|
||||
}
|
||||
return dialectPostgres
|
||||
}
|
||||
|
||||
func migrationDir() string {
|
||||
if !config.Config.Database.Enabled {
|
||||
return "goose/sqlite"
|
||||
}
|
||||
return "goose/postgres"
|
||||
}
|
||||
|
||||
// Migrate 执行数据库迁移
|
||||
func Migrate() Report {
|
||||
gormDB := db.DB(context.Background())
|
||||
if gormDB == nil {
|
||||
log.Fatalf("[%s] database not initialized\n", dbType())
|
||||
}
|
||||
|
||||
sqlDB, err := gormDB.DB()
|
||||
if err != nil {
|
||||
log.Fatalf("[%s] load sql db failed: %v\n", dbType(), err)
|
||||
}
|
||||
|
||||
goose.SetBaseFS(migrationFS)
|
||||
if err := goose.SetDialect(gooseDialect()); err != nil {
|
||||
log.Fatalf("[%s] set goose dialect failed: %v\n", dbType(), err)
|
||||
}
|
||||
previousVersion, err := goose.GetDBVersion(sqlDB)
|
||||
if err != nil {
|
||||
log.Fatalf("[%s] get goose version failed: %v\n", dbType(), err)
|
||||
}
|
||||
if err := goose.Up(sqlDB, migrationDir()); err != nil {
|
||||
log.Fatalf("[%s] goose migrate failed: %v\n", dbType(), err)
|
||||
}
|
||||
|
||||
clearSystemConfigCache()
|
||||
currentVersion, err := goose.GetDBVersion(sqlDB)
|
||||
if err != nil {
|
||||
log.Fatalf("[%s] get migrated goose version failed: %v\n", dbType(), err)
|
||||
}
|
||||
|
||||
log.Printf("[%s] goose migrate success\n", dbType())
|
||||
return Report{
|
||||
Backend: dbType(),
|
||||
Enabled: true,
|
||||
Version: currentVersion,
|
||||
Applied: currentVersion != previousVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func clearSystemConfigCache() {
|
||||
if err := repository.InvalidateAllSystemConfigCaches(context.Background()); err != nil {
|
||||
log.Printf("[%s] clear system config cache failed: %v\n", dbType(), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const expectedMigratedSystemConfigCount = 30
|
||||
|
||||
func TestMigrateInitializesSQLiteDatabase(t *testing.T) {
|
||||
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("gorm.Open(sqlite) error = %v", err)
|
||||
}
|
||||
|
||||
mr, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("miniredis.Run() error = %v", err)
|
||||
}
|
||||
redisClient := redis.NewClient(&redis.Options{
|
||||
Addr: mr.Addr(),
|
||||
MaintNotificationsConfig: &maintnotifications.Config{
|
||||
Mode: maintnotifications.ModeDisabled,
|
||||
},
|
||||
})
|
||||
|
||||
previousDBEnabled := config.Config.Database.Enabled
|
||||
config.Config.Database.Enabled = false
|
||||
db.SetDB(sqliteDB)
|
||||
t.Cleanup(func() {
|
||||
config.Config.Database.Enabled = previousDBEnabled
|
||||
db.SetDB(nil)
|
||||
_ = redisClient.Close()
|
||||
mr.Close()
|
||||
})
|
||||
|
||||
Migrate()
|
||||
|
||||
var systemConfigCount int64
|
||||
if err := sqliteDB.Table("w_system_configs").Count(&systemConfigCount).Error; err != nil {
|
||||
t.Fatalf("Migrate() count w_system_configs error = %v", err)
|
||||
}
|
||||
if systemConfigCount != expectedMigratedSystemConfigCount {
|
||||
t.Errorf("Migrate() w_system_configs count = %d, want %d", systemConfigCount, expectedMigratedSystemConfigCount)
|
||||
}
|
||||
|
||||
var adminCount int64
|
||||
if err := sqliteDB.Table("w_users").Where("username = ?", "admin").Count(&adminCount).Error; err != nil {
|
||||
t.Fatalf("Migrate() count admin user error = %v", err)
|
||||
}
|
||||
if adminCount != 1 {
|
||||
t.Errorf("Migrate() admin user count = %d, want %d", adminCount, 1)
|
||||
}
|
||||
|
||||
var templateCount int64
|
||||
if err := sqliteDB.Table("w_templates").Count(&templateCount).Error; err != nil {
|
||||
t.Fatalf("Migrate() count templates error = %v", err)
|
||||
}
|
||||
if templateCount != 2 {
|
||||
t.Errorf("Migrate() templates count = %d, want %d", templateCount, 2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateClearsStaleSystemConfigCache(t *testing.T) {
|
||||
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("gorm.Open(sqlite) error = %v", err)
|
||||
}
|
||||
|
||||
mr, err := miniredis.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("miniredis.Run() error = %v", err)
|
||||
}
|
||||
redisClient := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
|
||||
previousDBEnabled := config.Config.Database.Enabled
|
||||
previousRedis := db.Redis
|
||||
config.Config.Database.Enabled = false
|
||||
db.SetDB(sqliteDB)
|
||||
db.Redis = redisClient
|
||||
t.Cleanup(func() {
|
||||
config.Config.Database.Enabled = previousDBEnabled
|
||||
db.SetDB(nil)
|
||||
db.Redis = previousRedis
|
||||
_ = redisClient.Close()
|
||||
mr.Close()
|
||||
})
|
||||
|
||||
staleConfig := model.SystemConfig{
|
||||
Key: model.ConfigKeyCapLoginEnabled,
|
||||
Value: "true",
|
||||
Type: "system",
|
||||
}
|
||||
if err := db.HSetJSON(context.Background(), repository.SystemConfigRedisHashKey, model.ConfigKeyCapLoginEnabled, &staleConfig); err != nil {
|
||||
t.Fatalf("HSetJSON() error = %v", err)
|
||||
}
|
||||
|
||||
Migrate()
|
||||
|
||||
exists, err := db.Redis.Exists(context.Background(), db.PrefixedKey(repository.SystemConfigRedisHashKey)).Result()
|
||||
if err != nil {
|
||||
t.Fatalf("Redis.Exists() error = %v", err)
|
||||
}
|
||||
if exists != 0 {
|
||||
t.Fatalf("system config cache exists = %d, want 0", exists)
|
||||
}
|
||||
|
||||
enabled, err := repository.GetBoolByKey(context.Background(), model.ConfigKeyCapLoginEnabled)
|
||||
if err != nil {
|
||||
t.Fatalf("GetBoolByKey(%s) error = %v", model.ConfigKeyCapLoginEnabled, err)
|
||||
}
|
||||
if enabled {
|
||||
t.Fatalf("GetBoolByKey(%s) = true, want false", model.ConfigKeyCapLoginEnabled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/plugin/dbresolver"
|
||||
"gorm.io/plugin/opentelemetry/tracing"
|
||||
)
|
||||
|
||||
var (
|
||||
db *gorm.DB
|
||||
)
|
||||
|
||||
func init() {
|
||||
if !config.Config.Database.Enabled {
|
||||
// PostgreSQL 禁用,使用 SQLite
|
||||
initSQLite()
|
||||
return
|
||||
}
|
||||
|
||||
initPostgres()
|
||||
}
|
||||
|
||||
// initSQLite 初始化 SQLite 数据库(PostgreSQL 禁用时的后备方案)
|
||||
func initSQLite() {
|
||||
sqlitePath := config.Config.Database.SQLitePath
|
||||
if sqlitePath == "" {
|
||||
sqlitePath = "./data/wavelet.db"
|
||||
}
|
||||
|
||||
var err error
|
||||
db, err = gorm.Open(sqlite.Open(sqlitePath), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: &gormZapLogger{
|
||||
logLevel: parseLogLevel(config.Config.Database.LogLevel),
|
||||
slowThreshold: config.Config.Database.SlowThreshold,
|
||||
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[SQLite] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
// Trace 注入
|
||||
if err = db.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", sqlitePath),
|
||||
attribute.String("db.system", "SQLite"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[SQLite] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
log.Printf("[SQLite] initialized (path: %s)\n", sqlitePath)
|
||||
}
|
||||
|
||||
// initPostgres 初始化 PostgreSQL 数据库
|
||||
func initPostgres() {
|
||||
var err error
|
||||
dbConfig := config.Config.Database
|
||||
|
||||
// 构建主库 DSN 并连接
|
||||
primaryDSN := buildDSN(dbConfig.Host, dbConfig.Port, dbConfig.Username, dbConfig.Password)
|
||||
|
||||
pgConfig := postgres.Config{
|
||||
DSN: primaryDSN,
|
||||
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
|
||||
}
|
||||
|
||||
db, err = gorm.Open(postgres.New(pgConfig), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: &gormZapLogger{
|
||||
logLevel: parseLogLevel(config.Config.Database.LogLevel),
|
||||
slowThreshold: config.Config.Database.SlowThreshold,
|
||||
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[PostgreSQL] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
// Trace 注入
|
||||
if err = db.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", dbConfig.Database),
|
||||
attribute.String("db.ip", dbConfig.Host),
|
||||
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
|
||||
attribute.String("db.system", "PostgreSQL"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[PostgreSQL] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
if len(dbConfig.Replicas) > 0 {
|
||||
var replicaDialectors []gorm.Dialector
|
||||
for _, replica := range dbConfig.Replicas {
|
||||
username := replica.Username
|
||||
if username == "" {
|
||||
username = dbConfig.Username
|
||||
}
|
||||
password := replica.Password
|
||||
if password == "" {
|
||||
password = dbConfig.Password
|
||||
}
|
||||
replicaDSN := buildDSN(replica.Host, replica.Port, username, password)
|
||||
replicaDialectors = append(replicaDialectors, postgres.New(postgres.Config{
|
||||
DSN: replicaDSN,
|
||||
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
|
||||
}))
|
||||
}
|
||||
|
||||
resolver := dbresolver.Register(dbresolver.Config{
|
||||
Replicas: replicaDialectors,
|
||||
Policy: dbresolver.RandomPolicy{},
|
||||
})
|
||||
|
||||
resolver.SetMaxIdleConns(dbConfig.MaxIdleConn).
|
||||
SetMaxOpenConns(dbConfig.MaxOpenConn).
|
||||
SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second).
|
||||
SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
|
||||
|
||||
if err = db.Use(resolver); err != nil {
|
||||
log.Fatalf("[PostgreSQL] init dbresolver failed: %v\n", err)
|
||||
}
|
||||
log.Printf("[PostgreSQL] initialized in Primary-Replica mode (%d replicas)\n", len(dbConfig.Replicas))
|
||||
} else {
|
||||
log.Println("[PostgreSQL] initialized in Standalone mode")
|
||||
}
|
||||
|
||||
// 获取通用数据库对象设置连接池
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
log.Fatalf("[PostgreSQL] load sql db failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB.SetMaxIdleConns(dbConfig.MaxIdleConn)
|
||||
sqlDB.SetMaxOpenConns(dbConfig.MaxOpenConn)
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second)
|
||||
sqlDB.SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
|
||||
|
||||
}
|
||||
|
||||
// buildDSN 构建 PostgreSQL DSN
|
||||
func buildDSN(host string, port int, username, password string) string {
|
||||
cfg := config.Config.Database
|
||||
pqURL := &url.URL{
|
||||
Scheme: "postgres",
|
||||
Host: net.JoinHostPort(host, strconv.Itoa(port)),
|
||||
Path: cfg.Database,
|
||||
}
|
||||
if username != "" {
|
||||
pqURL.User = url.UserPassword(username, password)
|
||||
}
|
||||
|
||||
query := pqURL.Query()
|
||||
sslMode := cfg.SSLMode
|
||||
if sslMode == "" {
|
||||
sslMode = "disable"
|
||||
}
|
||||
query.Set("sslmode", sslMode)
|
||||
if cfg.ApplicationName != "" {
|
||||
query.Set("application_name", cfg.ApplicationName)
|
||||
}
|
||||
if cfg.SearchPath != "" {
|
||||
query.Set("search_path", cfg.SearchPath)
|
||||
}
|
||||
if cfg.DefaultQueryExecMode != "" {
|
||||
query.Set("default_query_exec_mode", cfg.DefaultQueryExecMode)
|
||||
}
|
||||
if cfg.StatementCacheCapacity > 0 {
|
||||
query.Set("statement_cache_capacity", strconv.Itoa(cfg.StatementCacheCapacity))
|
||||
}
|
||||
|
||||
rawQuery := query.Encode()
|
||||
if cfg.TimeZone != "" {
|
||||
if rawQuery != "" {
|
||||
rawQuery += "&"
|
||||
}
|
||||
rawQuery += "TimeZone=" + cfg.TimeZone
|
||||
}
|
||||
pqURL.RawQuery = rawQuery
|
||||
|
||||
return pqURL.String()
|
||||
}
|
||||
|
||||
// DB 返回带上下文追踪的 GORM 数据库实例
|
||||
func DB(ctx context.Context) *gorm.DB {
|
||||
if db == nil {
|
||||
return nil
|
||||
}
|
||||
return db.WithContext(ctx)
|
||||
}
|
||||
|
||||
// SetDB sets the package-level database instance for testing.
|
||||
func SetDB(d *gorm.DB) {
|
||||
db = d
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
"gorm.io/gorm"
|
||||
gormLogger "gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// nanoToMilli 纳秒转毫秒的除数
|
||||
const nanoToMilli = 1e6
|
||||
|
||||
type gormZapLogger struct {
|
||||
logLevel gormLogger.LogLevel
|
||||
ignoreRecordNotFoundError bool
|
||||
slowThreshold time.Duration
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) LogMode(level gormLogger.LogLevel) gormLogger.Interface {
|
||||
clone := *l
|
||||
clone.logLevel = level
|
||||
return &clone
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Info(ctx context.Context, fmt string, args ...interface{}) {
|
||||
if l.logLevel >= gormLogger.Info {
|
||||
logger.InfoF(ctx, fmt, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Warn(ctx context.Context, fmt string, args ...interface{}) {
|
||||
if l.logLevel >= gormLogger.Warn {
|
||||
logger.WarnF(ctx, fmt, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Error(ctx context.Context, fmt string, args ...interface{}) {
|
||||
if l.logLevel >= gormLogger.Error {
|
||||
logger.ErrorF(ctx, fmt, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *gormZapLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
|
||||
elapsed := time.Since(begin)
|
||||
switch {
|
||||
case err != nil && l.logLevel >= gormLogger.Error && (!errors.Is(err, gorm.ErrRecordNotFound) || !l.ignoreRecordNotFoundError):
|
||||
_, rows := fc()
|
||||
logger.ErrorF(ctx, "database query failed: %s [%.3fms] [rows:%v]", err, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
|
||||
case elapsed > l.slowThreshold && l.slowThreshold != 0 && l.logLevel >= gormLogger.Warn:
|
||||
_, rows := fc()
|
||||
slowLog := fmt.Sprintf("SLOW SQL >= %v", l.slowThreshold)
|
||||
logger.WarnF(ctx, "%s [%.3fms] [rows:%v]", slowLog, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
|
||||
case l.logLevel == gormLogger.Info:
|
||||
sql, rows := fc()
|
||||
logger.DebugF(ctx, "[%.3fms] [rows:%v] %s", float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows), sql)
|
||||
}
|
||||
}
|
||||
|
||||
func formatRows(rows int64) interface{} {
|
||||
if rows == -1 {
|
||||
return "-"
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func parseLogLevel(level string) gormLogger.LogLevel {
|
||||
level = strings.ToLower(level)
|
||||
switch level {
|
||||
case "silent":
|
||||
return gormLogger.Silent
|
||||
case "error":
|
||||
return gormLogger.Error
|
||||
case "warn":
|
||||
return gormLogger.Warn
|
||||
case "info":
|
||||
return gormLogger.Info
|
||||
case "debug":
|
||||
return gormLogger.Info
|
||||
default:
|
||||
return gormLogger.Info
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
gormLogger "gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestParseLogLevel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
configuredLevel string
|
||||
want gormLogger.LogLevel
|
||||
}{
|
||||
{
|
||||
name: "debug enables SQL trace processing",
|
||||
configuredLevel: "debug",
|
||||
want: gormLogger.Info,
|
||||
},
|
||||
{
|
||||
name: "development preserves configured level",
|
||||
configuredLevel: "warn",
|
||||
want: gormLogger.Warn,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := parseLogLevel(tt.configuredLevel); got != tt.want {
|
||||
t.Fatalf("parseLogLevel() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/redis/go-redis/extra/redisotel/v9"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
)
|
||||
|
||||
var (
|
||||
// Redis 全局 Redis 客户端实例
|
||||
Redis redis.UniversalClient
|
||||
)
|
||||
|
||||
func init() {
|
||||
cfg := config.Config.Redis
|
||||
|
||||
if !cfg.Enabled {
|
||||
log.Println("[Redis] is disabled, skipping Redis initialization")
|
||||
return
|
||||
}
|
||||
|
||||
if cfg.ClusterMode {
|
||||
// Cluster 模式
|
||||
Redis = redis.NewClusterClient(&redis.ClusterOptions{
|
||||
Addrs: cfg.Addrs,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
PoolSize: cfg.PoolSize,
|
||||
MinIdleConns: cfg.MinIdleConn,
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
|
||||
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
|
||||
MaxRetries: cfg.MaxRetries,
|
||||
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
|
||||
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
|
||||
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
|
||||
})
|
||||
log.Println("[Redis] initialized in Cluster mode")
|
||||
} else {
|
||||
// Standalone 或 Sentinel 模式
|
||||
options := &redis.UniversalOptions{
|
||||
Addrs: cfg.Addrs,
|
||||
MasterName: cfg.MasterName, // 非空时启用 Sentinel
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
DB: cfg.DB,
|
||||
PoolSize: cfg.PoolSize,
|
||||
MinIdleConns: cfg.MinIdleConn,
|
||||
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
|
||||
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
|
||||
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
|
||||
MaxRetries: cfg.MaxRetries,
|
||||
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
|
||||
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
|
||||
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
|
||||
}
|
||||
if cfg.MasterName != "" {
|
||||
client := redis.NewFailoverClient(options.Failover())
|
||||
// FailoverOptions 暂不暴露该配置,在首次建连前写入客户端选项。
|
||||
client.Options().MaintNotificationsConfig = redisMaintNotificationsConfig(cfg.MaintNotifications)
|
||||
Redis = client
|
||||
log.Println("[Redis] initialized in Sentinel mode")
|
||||
} else {
|
||||
Redis = redis.NewUniversalClient(options)
|
||||
log.Println("[Redis] initialized in Standalone mode")
|
||||
}
|
||||
}
|
||||
|
||||
// OpenTelemetry 追踪(UniversalClient 兼容)
|
||||
if err := redisotel.InstrumentTracing(
|
||||
Redis,
|
||||
redisotel.WithAttributes(
|
||||
attribute.String("db.instance", fmt.Sprintf("%v", cfg.DB)),
|
||||
attribute.String("db.ip", strings.Join(cfg.Addrs, ",")),
|
||||
attribute.String("db.system", "Redis"),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[Redis] failed to init trace: %v\n", err)
|
||||
}
|
||||
|
||||
// 测试连接
|
||||
_, err := Redis.Ping(context.Background()).Result()
|
||||
if err != nil {
|
||||
log.Fatalf("[Redis] failed to connect to redis: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func redisMaintNotificationsConfig(enabled bool) *maintnotifications.Config {
|
||||
mode := maintnotifications.ModeDisabled
|
||||
if enabled {
|
||||
mode = maintnotifications.ModeAuto
|
||||
}
|
||||
return &maintnotifications.Config{Mode: mode}
|
||||
}
|
||||
|
||||
// PrefixedKey 返回带前缀的 Key
|
||||
func PrefixedKey(key string) string {
|
||||
prefix := config.Config.Redis.KeyPrefix
|
||||
if prefix == "" {
|
||||
return key
|
||||
}
|
||||
return prefix + key
|
||||
}
|
||||
|
||||
// HSetJSON 将泛型数据序列化为 JSON 并设置到 Redis Hash
|
||||
// ctx: 上下文
|
||||
// hashKey: Redis Hash key
|
||||
// fieldKey: Hash field key
|
||||
// data: 要存储的数据(泛型)
|
||||
func HSetJSON[T any](ctx context.Context, hashKey, fieldKey string, data T) error {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := Redis.HSet(ctx, PrefixedKey(hashKey), fieldKey, jsonData).Err(); err != nil {
|
||||
return fmt.Errorf(errRedisHashSetFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HDel removes one or more fields from a Redis Hash.
|
||||
func HDel(ctx context.Context, hashKey string, fieldKeys ...string) error {
|
||||
if Redis == nil || len(fieldKeys) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := Redis.HDel(ctx, PrefixedKey(hashKey), fieldKeys...).Err(); err != nil {
|
||||
return fmt.Errorf(errRedisHashDeleteFailed, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HGetJSON 从 Redis Hash 获取数据并反序列化为泛型类型
|
||||
// ctx: 上下文
|
||||
// hashKey: Redis Hash key
|
||||
// fieldKey: Hash field key
|
||||
// data: 用于接收数据的指针(泛型)
|
||||
func HGetJSON[T any](ctx context.Context, hashKey, fieldKey string, data *T) error {
|
||||
val, err := Redis.HGet(ctx, PrefixedKey(hashKey), fieldKey).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(val), data); err != nil {
|
||||
return fmt.Errorf(errUnmarshalDataFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetJSON 从Redis获取数据并反序列化为泛型类型
|
||||
// ctx: 上下文
|
||||
// key: Redis key
|
||||
// data: 用于接收数据的指针(泛型)
|
||||
func GetJSON[T any](ctx context.Context, key string, data *T) error {
|
||||
val, err := Redis.Get(ctx, PrefixedKey(key)).Bytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(val, data); err != nil {
|
||||
return fmt.Errorf(errUnmarshalDataFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetJSON 将泛型数据序列化为JSON并设置到Redis
|
||||
// ctx: 上下文
|
||||
// key: Redis key
|
||||
// data: 要存储的数据(泛型)
|
||||
// expiration: 过期时间
|
||||
func SetJSON[T any](ctx context.Context, key string, data T, expiration time.Duration) error {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf(errMarshalDataFailed, err)
|
||||
}
|
||||
|
||||
if err := Redis.Set(ctx, PrefixedKey(key), jsonData, expiration).Err(); err != nil {
|
||||
return fmt.Errorf(errRedisKeySetFailed, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
)
|
||||
|
||||
func TestRedisMaintNotificationsConfig(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
enabled bool
|
||||
want maintnotifications.Mode
|
||||
}{
|
||||
{name: "disabled by default", enabled: false, want: maintnotifications.ModeDisabled},
|
||||
{name: "auto when enabled", enabled: true, want: maintnotifications.ModeAuto},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := redisMaintNotificationsConfig(test.enabled)
|
||||
if cfg.Mode != test.want {
|
||||
t.Fatalf("maintenance notifications mode = %v, want %v", cfg.Mode, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package task 定义异步任务类型与调度常量
|
||||
package task
|
||||
|
||||
// 任务队列名称
|
||||
const (
|
||||
QueueDefault = "default"
|
||||
)
|
||||
|
||||
// DefaultMaxRetry 任务默认最大重试次数
|
||||
const DefaultMaxRetry = 3
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package task
|
||||
|
||||
const (
|
||||
errUnknownTaskType = "未知的任务类型: %s"
|
||||
errCreateTaskExecutionFailed = "创建任务执行记录失败: %w"
|
||||
errTaskEnqueueFailed = "任务入队失败: %w"
|
||||
errTaskExecutionNotFound = "任务执行记录不存在: %w"
|
||||
errRetryOnlyFailedTask = "只有失败的任务才能重试,当前状态: %s"
|
||||
errTaskNotRetryable = "该任务不支持重试"
|
||||
errTaskMaxRetryExceeded = "已达到最大重试次数 %d"
|
||||
errCreateRetryExecutionFailed = "创建重试任务执行记录失败: %w"
|
||||
errRetryTaskEnqueueFailed = "重试任务入队失败: %w"
|
||||
errUnregisteredTaskHandler = "未注册的任务处理器: %s"
|
||||
)
|
||||
@@ -0,0 +1,479 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
otel_trace "github.com/Rain-kl/Wavelet/pkg/trace"
|
||||
"github.com/hibiken/asynq"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// handlerRegistry 已注册的任务处理器
|
||||
var handlerRegistry = make(map[string]TaskHandler)
|
||||
|
||||
// CompletedHandler is called when a task execution completes.
|
||||
type CompletedHandler func(ctx context.Context, execution *model.TaskExecution, result *TaskResult, execErr error)
|
||||
|
||||
var taskCompletedHandlers []CompletedHandler
|
||||
|
||||
// OnTaskCompleted registers a handler for task completion events.
|
||||
// Handlers must be registered during application bootstrap before processing tasks.
|
||||
func OnTaskCompleted(handler CompletedHandler) {
|
||||
taskCompletedHandlers = append(taskCompletedHandlers, handler)
|
||||
}
|
||||
|
||||
// RegisterHandler 注册任务处理器
|
||||
// 传入任务类型标识(对应 constants.go 中的 AsynqTask 常量)和 TaskHandler 实现
|
||||
func RegisterHandler(asynqTaskType string, handler TaskHandler) {
|
||||
handlerRegistry[asynqTaskType] = handler
|
||||
}
|
||||
|
||||
// getHandler 获取已注册的处理器
|
||||
func getHandler(asynqTaskType string) (TaskHandler, bool) {
|
||||
h, ok := handlerRegistry[asynqTaskType]
|
||||
return h, ok
|
||||
}
|
||||
|
||||
// ValidateAndNormalizePayload 校验并标准化任务参数。
|
||||
// 如果 Handler 实现了 PayloadValidator,调用其 ValidatePayload 方法;
|
||||
// 否则直接返回原始 payload。
|
||||
func ValidateAndNormalizePayload(asynqTaskType string, payload []byte) ([]byte, error) {
|
||||
handler, ok := getHandler(asynqTaskType)
|
||||
if !ok {
|
||||
return payload, nil
|
||||
}
|
||||
if validator, ok := handler.(PayloadValidator); ok {
|
||||
return validator.ValidatePayload(payload)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
// contextKey 用于 context 存取 taskID
|
||||
type contextKey string
|
||||
|
||||
const taskIDKey contextKey = "task_execution_task_id"
|
||||
const traceEnvelopeVersion = 1
|
||||
|
||||
type traceEnvelope struct {
|
||||
WaveletTraceEnvelope bool `json:"_wavelet_trace_envelope"`
|
||||
Version int `json:"version"`
|
||||
TraceContext map[string]string `json:"trace_context,omitempty"`
|
||||
Payload []byte `json:"payload"`
|
||||
}
|
||||
|
||||
// withTaskID 将 taskID 注入 context
|
||||
func withTaskID(ctx context.Context, taskID string) context.Context {
|
||||
return context.WithValue(ctx, taskIDKey, taskID)
|
||||
}
|
||||
|
||||
// GetTaskID 从 context 中获取 taskID
|
||||
func GetTaskID(ctx context.Context) string {
|
||||
if v, ok := ctx.Value(taskIDKey).(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// IsFinalAttempt 判断当前任务执行是否为最后一次重试尝试(若再次失败即为最终失败)
|
||||
func IsFinalAttempt(ctx context.Context) bool {
|
||||
retryCount, hasRetryCount := asynq.GetRetryCount(ctx)
|
||||
maxRetry, hasMaxRetry := asynq.GetMaxRetry(ctx)
|
||||
if !hasRetryCount || !hasMaxRetry {
|
||||
return true
|
||||
}
|
||||
return retryCount >= maxRetry
|
||||
}
|
||||
|
||||
// AppendLog 追加日志到任务执行记录
|
||||
// 在 TaskHandler.Execute 中调用,日志会自动追加到 TaskExecution.Log 字段
|
||||
func AppendLog(ctx context.Context, format string, args ...interface{}) {
|
||||
taskID := GetTaskID(ctx)
|
||||
if taskID == "" {
|
||||
// 上下文中没有 taskID,降级到普通日志
|
||||
logger.InfoF(ctx, format, args...)
|
||||
return
|
||||
}
|
||||
|
||||
logLine := fmt.Sprintf(format, args...)
|
||||
if err := model.AppendTaskExecutionLog(ctx, taskID, logLine); err != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 追加任务日志失败 taskID=%s: %v", taskID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// DispatchTask 下发任务(创建 TaskExecution 记录 → 入队 Asynq)
|
||||
func DispatchTask(ctx context.Context, taskType string, payload []byte, triggeredBy string) (string, error) {
|
||||
meta := GetTaskMeta(taskType)
|
||||
if meta == nil {
|
||||
return "", fmt.Errorf(errUnknownTaskType, taskType)
|
||||
}
|
||||
|
||||
// 生成唯一的 TaskID
|
||||
taskID := generateTaskID(taskType, triggeredBy)
|
||||
|
||||
// 创建任务执行记录
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: taskID,
|
||||
TaskType: meta.AsynqTask,
|
||||
TaskName: meta.Name,
|
||||
Status: model.TaskExecutionStatusPending,
|
||||
Retryable: meta.Retryable,
|
||||
MaxRetry: meta.MaxRetry,
|
||||
RetryCount: 0,
|
||||
Payload: string(payload),
|
||||
TriggeredBy: triggeredBy,
|
||||
}
|
||||
|
||||
if err := model.CreateTaskExecution(ctx, execution); err != nil {
|
||||
return "", fmt.Errorf(errCreateTaskExecutionFailed, err)
|
||||
}
|
||||
|
||||
// 入队 Asynq
|
||||
taskInfo := asynq.NewTask(meta.AsynqTask, injectTaskTraceContext(ctx, payload))
|
||||
if _, err := AsynqClient.Enqueue(
|
||||
taskInfo,
|
||||
asynq.TaskID(taskID),
|
||||
asynq.MaxRetry(meta.MaxRetry),
|
||||
asynq.Queue(meta.Queue),
|
||||
); err != nil {
|
||||
// 入队失败,更新执行记录状态
|
||||
execution.Status = model.TaskExecutionStatusFailed
|
||||
execution.ErrorMessage = fmt.Sprintf("入队失败: %v", err)
|
||||
now := time.Now()
|
||||
execution.StartedAt = &now
|
||||
execution.FinishedAt = &now
|
||||
_ = model.UpdateTaskExecution(ctx, execution)
|
||||
return "", fmt.Errorf(errTaskEnqueueFailed, err)
|
||||
}
|
||||
|
||||
if err := model.AppendTaskExecutionLog(ctx, taskID, fmt.Sprintf("[系统] 任务已成功入队,等待调度执行 (队列: %s, 最大重试次数: %d)", meta.Queue, meta.MaxRetry)); err != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 追加入队日志失败 taskID=%s: %v", taskID, err)
|
||||
}
|
||||
|
||||
return taskID, nil
|
||||
}
|
||||
|
||||
// RetryTask 重试失败的任务
|
||||
func RetryTask(ctx context.Context, id uint64) (string, error) {
|
||||
execution, err := model.GetTaskExecutionByID(ctx, id)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf(errTaskExecutionNotFound, err)
|
||||
}
|
||||
|
||||
if execution.Status != model.TaskExecutionStatusFailed {
|
||||
return "", fmt.Errorf(errRetryOnlyFailedTask, execution.Status)
|
||||
}
|
||||
|
||||
if !execution.Retryable {
|
||||
return "", errors.New(errTaskNotRetryable)
|
||||
}
|
||||
|
||||
// 生成新的 TaskID
|
||||
newTaskID := generateRetryTaskID(execution.TaskID, execution.RetryCount+1)
|
||||
|
||||
// 创建新的执行记录
|
||||
newExecution := &model.TaskExecution{
|
||||
TaskID: newTaskID,
|
||||
TaskType: execution.TaskType,
|
||||
TaskName: execution.TaskName,
|
||||
Status: model.TaskExecutionStatusPending,
|
||||
Retryable: execution.Retryable,
|
||||
MaxRetry: execution.MaxRetry,
|
||||
RetryCount: execution.RetryCount + 1,
|
||||
Payload: execution.Payload,
|
||||
TriggeredBy: "retry",
|
||||
}
|
||||
|
||||
if err := model.CreateTaskExecution(ctx, newExecution); err != nil {
|
||||
return "", fmt.Errorf(errCreateRetryExecutionFailed, err)
|
||||
}
|
||||
|
||||
meta := GetTaskMeta(execution.TaskType)
|
||||
queueName := QueueDefault
|
||||
if meta != nil {
|
||||
queueName = meta.Queue
|
||||
}
|
||||
|
||||
// 入队 Asynq
|
||||
taskInfo := asynq.NewTask(execution.TaskType, injectTaskTraceContext(ctx, []byte(execution.Payload)))
|
||||
if _, err := AsynqClient.Enqueue(
|
||||
taskInfo,
|
||||
asynq.TaskID(newTaskID),
|
||||
asynq.MaxRetry(execution.MaxRetry),
|
||||
asynq.Queue(queueName),
|
||||
); err != nil {
|
||||
newExecution.Status = model.TaskExecutionStatusFailed
|
||||
newExecution.ErrorMessage = fmt.Sprintf("重试入队失败: %v", err)
|
||||
now := time.Now()
|
||||
newExecution.StartedAt = &now
|
||||
newExecution.FinishedAt = &now
|
||||
_ = model.UpdateTaskExecution(ctx, newExecution)
|
||||
return "", fmt.Errorf(errRetryTaskEnqueueFailed, err)
|
||||
}
|
||||
|
||||
if err := model.AppendTaskExecutionLog(ctx, newTaskID, fmt.Sprintf("[系统] 手动触发重试,已重新创建任务并入队 (原任务ID: %s, 重试次数: %d/%d)", execution.TaskID, execution.RetryCount+1, execution.MaxRetry)); err != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 追加重试日志失败 taskID=%s: %v", newTaskID, err)
|
||||
}
|
||||
|
||||
return newTaskID, nil
|
||||
}
|
||||
|
||||
// ProcessTask Asynq 实际调用的统一处理函数
|
||||
// Worker 注册时统一使用此函数,内部自动分发到对应的 TaskHandler
|
||||
func ProcessTask(ctx context.Context, t *asynq.Task) error {
|
||||
taskPayload := t.Payload()
|
||||
ctx, taskPayload, hasRemoteTraceContext := extractTaskTraceContext(ctx, taskPayload)
|
||||
|
||||
// 初始化 Trace
|
||||
ctx, span := otel_trace.Start(ctx, "TaskProcess_"+t.Type(), trace.WithSpanKind(trace.SpanKindConsumer))
|
||||
defer span.End()
|
||||
|
||||
// 添加任务信息到 Span
|
||||
span.SetAttributes(
|
||||
attribute.String("task.type", t.Type()),
|
||||
attribute.Int("task.payload_size", len(taskPayload)),
|
||||
attribute.Bool("task.trace_context_propagated", hasRemoteTraceContext),
|
||||
attribute.String("task.id", t.ResultWriter().TaskID()),
|
||||
)
|
||||
|
||||
taskID := t.ResultWriter().TaskID()
|
||||
|
||||
// 注入 taskID 到 context
|
||||
ctx = withTaskID(ctx, taskID)
|
||||
|
||||
// 查找处理器
|
||||
handler, ok := getHandler(t.Type())
|
||||
if !ok {
|
||||
err := fmt.Errorf(errUnregisteredTaskHandler, t.Type())
|
||||
logger.ErrorF(ctx, "[TaskExecutor] %v", err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// 加载或动态创建执行记录
|
||||
now := time.Now()
|
||||
execution, err := getOrCreateTaskExecution(ctx, taskID, t, taskPayload, now)
|
||||
if err == nil {
|
||||
updateExecutionOnStart(ctx, execution, now)
|
||||
}
|
||||
|
||||
if execution != nil {
|
||||
AppendLog(ctx, "[系统] 开始执行异步任务 [名称: %s, 类型: %s],重试次数: %d/%d",
|
||||
execution.TaskName, t.Type(), execution.RetryCount, execution.MaxRetry)
|
||||
} else {
|
||||
AppendLog(ctx, "[系统] 开始执行异步任务 [类型: %s]", t.Type())
|
||||
}
|
||||
|
||||
// 开始计时
|
||||
start := time.Now()
|
||||
|
||||
// 执行业务逻辑
|
||||
result, execErr := handler.Execute(ctx, taskPayload)
|
||||
|
||||
// 计算耗时并归档记录
|
||||
duration := time.Since(start)
|
||||
finishTime := time.Now()
|
||||
|
||||
completeTaskExecution(ctx, execution, t, duration, finishTime, result, execErr, span)
|
||||
|
||||
if execution == nil && execErr != nil {
|
||||
span.SetStatus(codes.Error, execErr.Error())
|
||||
return execErr
|
||||
}
|
||||
|
||||
return execErr
|
||||
}
|
||||
|
||||
func injectTaskTraceContext(ctx context.Context, payload []byte) []byte {
|
||||
carrier := propagation.MapCarrier{}
|
||||
otel.GetTextMapPropagator().Inject(ctx, carrier)
|
||||
if len(carrier) == 0 {
|
||||
return payload
|
||||
}
|
||||
|
||||
envelope := traceEnvelope{
|
||||
WaveletTraceEnvelope: true,
|
||||
Version: traceEnvelopeVersion,
|
||||
TraceContext: map[string]string(carrier),
|
||||
Payload: payload,
|
||||
}
|
||||
data, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 序列化任务 Trace 上下文失败: %v", err)
|
||||
return payload
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func extractTaskTraceContext(ctx context.Context, payload []byte) (context.Context, []byte, bool) {
|
||||
var envelope traceEnvelope
|
||||
if err := json.Unmarshal(payload, &envelope); err != nil {
|
||||
return ctx, payload, false
|
||||
}
|
||||
if !envelope.WaveletTraceEnvelope || envelope.Version != traceEnvelopeVersion {
|
||||
return ctx, payload, false
|
||||
}
|
||||
if len(envelope.TraceContext) == 0 {
|
||||
return ctx, envelope.Payload, false
|
||||
}
|
||||
|
||||
extractedCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(envelope.TraceContext))
|
||||
return extractedCtx, envelope.Payload, true
|
||||
}
|
||||
|
||||
func updateExecutionOnStart(ctx context.Context, execution *model.TaskExecution, now time.Time) {
|
||||
if execution == nil {
|
||||
return
|
||||
}
|
||||
dirty := false
|
||||
if retryCount, hasRetry := asynq.GetRetryCount(ctx); hasRetry && execution.RetryCount != retryCount {
|
||||
execution.RetryCount = retryCount
|
||||
dirty = true
|
||||
}
|
||||
if execution.Status != model.TaskExecutionStatusRunning {
|
||||
execution.Status = model.TaskExecutionStatusRunning
|
||||
execution.StartedAt = &now
|
||||
dirty = true
|
||||
}
|
||||
if dirty {
|
||||
if updateErr := model.UpdateTaskExecution(ctx, execution); updateErr != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 更新执行状态失败 taskID=%s: %v", execution.TaskID, updateErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getOrCreateTaskExecution 获取已有的任务执行记录,如果不存在则针对已知任务类型动态创建记录
|
||||
func getOrCreateTaskExecution(ctx context.Context, taskID string, t *asynq.Task, payload []byte, now time.Time) (*model.TaskExecution, error) {
|
||||
execution, err := model.GetTaskExecutionByTaskID(ctx, taskID)
|
||||
if err == nil {
|
||||
return execution, nil
|
||||
}
|
||||
|
||||
meta := GetTaskMetaByAsynqTask(t.Type())
|
||||
if meta == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
execution = &model.TaskExecution{
|
||||
TaskID: taskID,
|
||||
TaskType: meta.AsynqTask,
|
||||
TaskName: meta.Name,
|
||||
Status: model.TaskExecutionStatusRunning,
|
||||
Retryable: meta.Retryable,
|
||||
MaxRetry: meta.MaxRetry,
|
||||
RetryCount: 0,
|
||||
Payload: string(payload),
|
||||
TriggeredBy: "schedule",
|
||||
StartedAt: &now,
|
||||
}
|
||||
|
||||
if createErr := model.CreateTaskExecution(ctx, execution); createErr != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 动态创建执行记录失败 taskID=%s: %v", taskID, createErr)
|
||||
return nil, createErr
|
||||
}
|
||||
|
||||
return execution, nil
|
||||
}
|
||||
|
||||
// completeTaskExecution 完成并更新任务执行记录的状态和执行结果
|
||||
func completeTaskExecution(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, finishTime time.Time, result *TaskResult, execErr error, span trace.Span) {
|
||||
if execution == nil {
|
||||
return
|
||||
}
|
||||
|
||||
execution.Duration = duration.Milliseconds()
|
||||
execution.FinishedAt = &finishTime
|
||||
|
||||
if execErr != nil {
|
||||
handleFailedTask(ctx, execution, t, duration, execErr, span)
|
||||
} else {
|
||||
handleSuccessfulTask(ctx, execution, t, duration, result)
|
||||
}
|
||||
|
||||
if err := model.UpdateTaskExecution(ctx, execution); err != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 更新执行记录失败 taskID=%s: %v", execution.TaskID, err)
|
||||
}
|
||||
if shouldFlushTaskExecutionLog(ctx, execErr) {
|
||||
if err := model.FlushTaskExecutionLog(ctx, execution.TaskID); err != nil {
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 持久化任务日志失败 taskID=%s: %v", execution.TaskID, err)
|
||||
}
|
||||
}
|
||||
|
||||
notifyTaskCompleted(ctx, execution, result, execErr)
|
||||
}
|
||||
|
||||
func notifyTaskCompleted(ctx context.Context, execution *model.TaskExecution, result *TaskResult, execErr error) {
|
||||
if len(taskCompletedHandlers) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
asyncCtx := context.WithoutCancel(ctx)
|
||||
for _, handler := range taskCompletedHandlers {
|
||||
go handler(asyncCtx, execution, result, execErr)
|
||||
}
|
||||
}
|
||||
|
||||
func shouldFlushTaskExecutionLog(ctx context.Context, execErr error) bool {
|
||||
if execErr == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
retryCount, hasRetryCount := asynq.GetRetryCount(ctx)
|
||||
maxRetry, hasMaxRetry := asynq.GetMaxRetry(ctx)
|
||||
if !hasRetryCount || !hasMaxRetry {
|
||||
return true
|
||||
}
|
||||
return retryCount >= maxRetry
|
||||
}
|
||||
|
||||
func handleFailedTask(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, execErr error, span trace.Span) {
|
||||
execution.Status = model.TaskExecutionStatusFailed
|
||||
execution.ErrorMessage = execErr.Error()
|
||||
logger.ErrorF(ctx, "[TaskExecutor] 任务处理失败 Type: %s TaskID: %s Duration: %d ms Error: %v", t.Type(), execution.TaskID, duration.Milliseconds(), execErr)
|
||||
span.SetStatus(codes.Error, execErr.Error())
|
||||
span.RecordError(execErr)
|
||||
|
||||
AppendLog(ctx, "[系统] 任务执行失败,耗时: %d ms,错误原因: %v", duration.Milliseconds(), execErr)
|
||||
}
|
||||
|
||||
func handleSuccessfulTask(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, result *TaskResult) {
|
||||
execution.Status = model.TaskExecutionStatusSucceeded
|
||||
execution.ErrorMessage = "" // 清除历史重试失败遗留的错误信息
|
||||
if result != nil {
|
||||
execution.Result = result.Message
|
||||
if result.Detail != "" {
|
||||
execution.Result = fmt.Sprintf("%s\n%s", result.Message, result.Detail)
|
||||
}
|
||||
}
|
||||
logger.InfoF(ctx, "[TaskExecutor] 任务处理完成 Type: %s TaskID: %s Duration: %d ms", t.Type(), execution.TaskID, duration.Milliseconds())
|
||||
|
||||
resultMsg := "成功"
|
||||
if result != nil {
|
||||
resultMsg = result.Message
|
||||
}
|
||||
AppendLog(ctx, "[系统] 任务执行成功,耗时: %d ms,执行结果: %s", duration.Milliseconds(), resultMsg)
|
||||
}
|
||||
|
||||
// generateTaskID 生成任务 ID
|
||||
func generateTaskID(taskType string, triggeredBy string) string {
|
||||
return fmt.Sprintf("%s_%s_%d", triggeredBy, taskType, idgen.NextUint64ID())
|
||||
}
|
||||
|
||||
// generateRetryTaskID 生成重试任务 ID
|
||||
func generateRetryTaskID(originalTaskID string, retryCount int) string {
|
||||
return fmt.Sprintf("retry_%d_%s", retryCount, originalTaskID)
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/testhelper"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// mockHandler 用于测试的模拟任务处理器
|
||||
type mockHandler struct {
|
||||
executeFunc func(ctx context.Context, payload []byte) (*TaskResult, error)
|
||||
}
|
||||
|
||||
func (h *mockHandler) Execute(ctx context.Context, payload []byte) (*TaskResult, error) {
|
||||
if h.executeFunc != nil {
|
||||
return h.executeFunc(ctx, payload)
|
||||
}
|
||||
return &TaskResult{Message: "mock success"}, nil
|
||||
}
|
||||
|
||||
// successHandler 返回成功的处理器
|
||||
func successHandler() *mockHandler {
|
||||
return &mockHandler{
|
||||
executeFunc: func(ctx context.Context, payload []byte) (*TaskResult, error) {
|
||||
AppendLog(ctx, "执行成功,处理了 %d 条数据", 100)
|
||||
return &TaskResult{Message: "处理完成,共 100 条"}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// failHandler 返回失败的处理器
|
||||
func failHandler() *mockHandler {
|
||||
return &mockHandler{
|
||||
executeFunc: func(ctx context.Context, payload []byte) (*TaskResult, error) {
|
||||
AppendLog(ctx, "开始执行任务")
|
||||
return nil, fmt.Errorf("模拟执行失败: 数据库连接超时")
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const testTaskType = "test:mock_task"
|
||||
|
||||
func setupTest(t *testing.T) func() {
|
||||
_, mr, cleanup := testhelper.SetupTestEnvironment(t)
|
||||
AsynqClient = asynq.NewClient(asynq.RedisClientOpt{
|
||||
Addr: mr.Addr(),
|
||||
})
|
||||
// 注册测试用 handler
|
||||
RegisterHandler(testTaskType, successHandler())
|
||||
return func() {
|
||||
if AsynqClient != nil {
|
||||
_ = AsynqClient.Close()
|
||||
AsynqClient = nil
|
||||
}
|
||||
cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterAndGetHandler(t *testing.T) {
|
||||
_ = testTaskType
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
|
||||
// 验证 handler 已注册
|
||||
h, ok := getHandler(testTaskType)
|
||||
assert.True(t, ok, "handler should be registered")
|
||||
assert.NotNil(t, h)
|
||||
|
||||
// 未注册的 handler
|
||||
_, ok = getHandler("nonexistent")
|
||||
assert.False(t, ok, "non-existent handler should return false")
|
||||
}
|
||||
|
||||
func TestGetTaskIDFromContext(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// 空 context
|
||||
taskID := GetTaskID(ctx)
|
||||
assert.Equal(t, "", taskID)
|
||||
|
||||
// 注入 taskID
|
||||
ctx = withTaskID(ctx, "test_task_123")
|
||||
taskID = GetTaskID(ctx)
|
||||
assert.Equal(t, "test_task_123", taskID)
|
||||
}
|
||||
|
||||
func TestAppendLogWithoutTaskID(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// 没有 taskID 的 context,应降级到普通日志,不报错
|
||||
AppendLog(ctx, "这条日志应该降级处理,不会报错")
|
||||
}
|
||||
|
||||
func TestAppendLogWithTaskID(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// 先创建一条执行记录
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: "log_test_001",
|
||||
TaskType: testTaskType,
|
||||
TaskName: "测试任务",
|
||||
Status: model.TaskExecutionStatusRunning,
|
||||
TriggeredBy: "manual",
|
||||
}
|
||||
err := model.CreateTaskExecution(ctx, execution)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 注入 taskID 并追加日志
|
||||
ctx = withTaskID(ctx, "log_test_001")
|
||||
AppendLog(ctx, "第一条日志")
|
||||
AppendLog(ctx, "处理了 %d 条数据", 50)
|
||||
|
||||
// 验证日志
|
||||
found, err := model.GetTaskExecutionByTaskID(ctx, "log_test_001")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, found.Log, "第一条日志")
|
||||
assert.Contains(t, found.Log, "处理了 50 条数据")
|
||||
}
|
||||
|
||||
func TestTaskTraceContextEnvelope(t *testing.T) {
|
||||
payload := []byte(`{"hello":"wavelet"}`)
|
||||
traceID := "4bf92f3577b34da6a3ce929d0e0e4736"
|
||||
parentCtx := otel.GetTextMapPropagator().Extract(
|
||||
context.Background(),
|
||||
propagation.MapCarrier{
|
||||
"traceparent": "00-" + traceID + "-00f067aa0ba902b7-01",
|
||||
},
|
||||
)
|
||||
|
||||
wrappedPayload := injectTaskTraceContext(parentCtx, payload)
|
||||
require.NotEqual(t, string(payload), string(wrappedPayload))
|
||||
|
||||
gotCtx, gotPayload, ok := extractTaskTraceContext(context.Background(), wrappedPayload)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, payload, gotPayload)
|
||||
assert.Equal(t, traceID, trace.SpanContextFromContext(gotCtx).TraceID().String())
|
||||
}
|
||||
|
||||
func TestTaskTraceContextEnvelopeKeepsLegacyPayload(t *testing.T) {
|
||||
payload := []byte(`{"legacy":true}`)
|
||||
|
||||
gotCtx, gotPayload, ok := extractTaskTraceContext(context.Background(), payload)
|
||||
require.False(t, ok)
|
||||
assert.Equal(t, context.Background(), gotCtx)
|
||||
assert.Equal(t, payload, gotPayload)
|
||||
}
|
||||
|
||||
func TestTaskTraceContextEnvelopeSkipsEmptyContext(t *testing.T) {
|
||||
payload := []byte(`{"background":true}`)
|
||||
|
||||
wrappedPayload := injectTaskTraceContext(context.Background(), payload)
|
||||
assert.Equal(t, payload, wrappedPayload)
|
||||
}
|
||||
|
||||
func TestProcessTaskSuccess(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// 注册成功 handler
|
||||
RegisterHandler(testTaskType, successHandler())
|
||||
|
||||
// 创建执行记录
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: "process_success_001",
|
||||
TaskType: testTaskType,
|
||||
TaskName: "测试任务",
|
||||
Status: model.TaskExecutionStatusPending,
|
||||
Retryable: true,
|
||||
MaxRetry: 3,
|
||||
TriggeredBy: "manual",
|
||||
}
|
||||
err := model.CreateTaskExecution(ctx, execution)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 通过 asynq 的 Task 不能直接设置 taskID,ProcessTask 通过 t.ResultWriter().TaskID() 获取
|
||||
// 但 asynq.Task 在没有经过 asynq server 的情况下 ResultWriter 可能为 nil
|
||||
// 我们需要在 ProcessTask 内部改用 taskID 注入的方式测试
|
||||
// 为了测试 ProcessTask,我们直接模拟调用 handler
|
||||
|
||||
// 直接通过 handler 测试
|
||||
handler, ok := getHandler(testTaskType)
|
||||
require.True(t, ok)
|
||||
|
||||
ctx = withTaskID(ctx, "process_success_001")
|
||||
result, err := handler.Execute(ctx, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "处理完成,共 100 条", result.Message)
|
||||
|
||||
// 验证日志被追加
|
||||
found, err := model.GetTaskExecutionByTaskID(ctx, "process_success_001")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, found.Log, "执行成功,处理了 100 条数据")
|
||||
}
|
||||
|
||||
func TestProcessTaskFailure(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// 注册失败 handler
|
||||
RegisterHandler(testTaskType, failHandler())
|
||||
|
||||
// 创建执行记录
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: "process_fail_001",
|
||||
TaskType: testTaskType,
|
||||
TaskName: "测试任务",
|
||||
Status: model.TaskExecutionStatusPending,
|
||||
Retryable: true,
|
||||
MaxRetry: 3,
|
||||
TriggeredBy: "manual",
|
||||
}
|
||||
err := model.CreateTaskExecution(ctx, execution)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 直接调用 handler
|
||||
handler, ok := getHandler(testTaskType)
|
||||
require.True(t, ok)
|
||||
|
||||
ctx = withTaskID(ctx, "process_fail_001")
|
||||
_, err = handler.Execute(ctx, nil)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "模拟执行失败")
|
||||
|
||||
// 验证日志
|
||||
found, err := model.GetTaskExecutionByTaskID(ctx, "process_fail_001")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, found.Log, "开始执行任务")
|
||||
}
|
||||
|
||||
func TestCompleteTaskExecutionFlushesLog(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: "complete_flush_001",
|
||||
TaskType: testTaskType,
|
||||
TaskName: "测试任务",
|
||||
Status: model.TaskExecutionStatusRunning,
|
||||
TriggeredBy: "manual",
|
||||
}
|
||||
err := model.CreateTaskExecution(ctx, execution)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx = withTaskID(ctx, execution.TaskID)
|
||||
AppendLog(ctx, "任务执行中的日志")
|
||||
|
||||
finishTime := time.Now()
|
||||
completeTaskExecution(
|
||||
ctx,
|
||||
execution,
|
||||
asynq.NewTask(testTaskType, nil),
|
||||
100*time.Millisecond,
|
||||
finishTime,
|
||||
&TaskResult{Message: "处理完成"},
|
||||
nil,
|
||||
trace.SpanFromContext(ctx),
|
||||
)
|
||||
|
||||
found, err := model.GetTaskExecutionByTaskID(ctx, execution.TaskID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, model.TaskExecutionStatusSucceeded, found.Status)
|
||||
assert.Contains(t, found.Log, "任务执行中的日志")
|
||||
assert.Contains(t, found.Log, "任务执行成功")
|
||||
}
|
||||
|
||||
func TestRetryTask(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// 创建一条失败的执行记录(可重试)
|
||||
now := time.Now()
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: "retry_test_001",
|
||||
TaskType: testTaskType,
|
||||
TaskName: "测试任务",
|
||||
Status: model.TaskExecutionStatusFailed,
|
||||
Retryable: true,
|
||||
MaxRetry: 3,
|
||||
RetryCount: 0,
|
||||
ErrorMessage: "首次执行失败",
|
||||
StartedAt: &now,
|
||||
FinishedAt: &now,
|
||||
Duration: 100,
|
||||
TriggeredBy: "manual",
|
||||
}
|
||||
err := model.CreateTaskExecution(ctx, execution)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 重试
|
||||
newTaskID, err := RetryTask(ctx, execution.ID)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, newTaskID)
|
||||
assert.Contains(t, newTaskID, "retry_1_")
|
||||
|
||||
// 验证新记录
|
||||
newExecution, err := model.GetTaskExecutionByTaskID(ctx, newTaskID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, model.TaskExecutionStatusPending, newExecution.Status)
|
||||
assert.Equal(t, 1, newExecution.RetryCount)
|
||||
assert.Equal(t, "retry", newExecution.TriggeredBy)
|
||||
assert.Equal(t, execution.TaskType, newExecution.TaskType)
|
||||
assert.True(t, newExecution.Retryable)
|
||||
|
||||
// 原记录不变
|
||||
original, err := model.GetTaskExecutionByID(ctx, execution.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, model.TaskExecutionStatusFailed, original.Status)
|
||||
assert.Equal(t, 0, original.RetryCount)
|
||||
}
|
||||
|
||||
func TestRetryTaskNotFailed(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
// 创建一条成功的记录
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: "retry_not_failed_001",
|
||||
TaskType: testTaskType,
|
||||
TaskName: "测试任务",
|
||||
Status: model.TaskExecutionStatusSucceeded,
|
||||
Retryable: true,
|
||||
MaxRetry: 3,
|
||||
TriggeredBy: "manual",
|
||||
}
|
||||
err := model.CreateTaskExecution(ctx, execution)
|
||||
require.NoError(t, err)
|
||||
|
||||
// 尝试重试成功的任务
|
||||
_, err = RetryTask(ctx, execution.ID)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "只有失败的任务才能重试")
|
||||
}
|
||||
|
||||
func TestRetryTaskNotRetryable(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
execution := &model.TaskExecution{
|
||||
TaskID: "retry_not_allowed_001",
|
||||
TaskType: testTaskType,
|
||||
TaskName: "测试任务",
|
||||
Status: model.TaskExecutionStatusFailed,
|
||||
Retryable: false,
|
||||
MaxRetry: 0,
|
||||
TriggeredBy: "manual",
|
||||
}
|
||||
err := model.CreateTaskExecution(ctx, execution)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = RetryTask(ctx, execution.ID)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "不支持重试")
|
||||
}
|
||||
|
||||
func TestRetryTaskNonExistent(t *testing.T) {
|
||||
cleanup := setupTest(t)
|
||||
defer cleanup()
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := RetryTask(ctx, 99999999)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "不存在")
|
||||
}
|
||||
|
||||
func TestGenerateTaskID(t *testing.T) {
|
||||
id1 := generateTaskID("test_type", "manual")
|
||||
id2 := generateTaskID("test_type", "manual")
|
||||
|
||||
// 两个 ID 应不同(包含 Snowflake ID)
|
||||
assert.NotEqual(t, id1, id2)
|
||||
assert.Contains(t, id1, "manual_test_type_")
|
||||
}
|
||||
|
||||
func TestGenerateRetryTaskID(t *testing.T) {
|
||||
id := generateRetryTaskID("original_task_123", 2)
|
||||
assert.Equal(t, "retry_2_original_task_123", id)
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package task
|
||||
|
||||
import "context"
|
||||
|
||||
// TaskResult 任务执行结果
|
||||
//
|
||||
//nolint:revive // TaskResult 保留完整名称以避免与通用 Result 混淆
|
||||
type TaskResult struct {
|
||||
Message string // 结果摘要,如 "共清理 120 个文件,耗时 3.2s"
|
||||
Detail string // 可选的详细结果 JSON
|
||||
}
|
||||
|
||||
// PayloadValidator 可选接口,带参数的任务 Handler 应实现此接口。
|
||||
// 框架在 Admin 下发时自动调用,完成参数校验和标准化(如 Trim 空白)。
|
||||
// 无参数的任务无需实现,框架会直接透传 payload。
|
||||
type PayloadValidator interface {
|
||||
ValidatePayload(payload []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// TaskHandler 异步任务处理器接口
|
||||
// 所有异步任务必须实现此接口,框架将自动管理任务执行记录的创建、状态流转和日志写入。
|
||||
//
|
||||
// 开发者只需实现 Execute 方法编写业务逻辑,在方法内通过 task.AppendLog(ctx, ...) 追加执行日志。
|
||||
// 任务的创建、状态更新、错误记录、重试计数全部由框架透明处理。
|
||||
//
|
||||
//nolint:revive // TaskHandler 保留完整名称以避免与通用 Handler 混淆
|
||||
type TaskHandler interface {
|
||||
// Execute 执行任务业务逻辑
|
||||
// - ctx: 已注入 Trace Span 和 taskID 的上下文
|
||||
// - payload: 调度时传入的原始参数(可为 nil)
|
||||
// - 返回 TaskResult 描述执行结果,或 error 表示执行失败
|
||||
Execute(ctx context.Context, payload []byte) (*TaskResult, error)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package handlers 注册异步任务处理器
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/admin/push"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/upload"
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/user"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/task"
|
||||
)
|
||||
|
||||
// Register registers all built-in task handlers and their metadata.
|
||||
func Register() {
|
||||
task.RegisterHandler(upload.StorageMigrationTask, &upload.MigrationHandler{})
|
||||
task.RegisterTaskMeta(upload.StorageMigrationMeta)
|
||||
|
||||
// system cleanup
|
||||
task.RegisterHandler(upload.SystemCleanupTask, &upload.SystemCleanupHandler{})
|
||||
task.RegisterTaskMeta(upload.SystemCleanupMeta)
|
||||
|
||||
// upload
|
||||
task.RegisterHandler(upload.WarmImageCacheTask, &upload.WarmImageCacheHandler{})
|
||||
task.RegisterTaskMeta(upload.WarmImageCacheMeta)
|
||||
|
||||
task.RegisterHandler(upload.RebuildUploadStatsTask, &upload.RebuildUploadStatsHandler{})
|
||||
task.RegisterTaskMeta(upload.RebuildUploadStatsMeta)
|
||||
|
||||
// user
|
||||
task.RegisterHandler(user.SendEmailTask, &user.SendEmailHandler{})
|
||||
task.RegisterTaskMeta(user.SendEmailMeta)
|
||||
|
||||
// push
|
||||
task.RegisterHandler(push.SendNotificationTask, &push.PushHandler{})
|
||||
task.RegisterTaskMeta(push.SendNotificationMeta)
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package task
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// TaskParam 任务参数定义
|
||||
//
|
||||
//nolint:revive // TaskParam 保留完整名称以避免与通用 Param 混淆
|
||||
type TaskParam struct {
|
||||
Name string `json:"name"` // 参数键名
|
||||
Label string `json:"label"` // 显示名称
|
||||
Type string `json:"type"` // 类型:string, text, number, boolean
|
||||
Required bool `json:"required"` // 是否必填
|
||||
Placeholder string `json:"placeholder"` // 占位符
|
||||
Description string `json:"description"` // 描述
|
||||
}
|
||||
|
||||
// TaskMeta 任务元数据
|
||||
//
|
||||
//nolint:revive // TaskMeta 保留完整名称以避免与通用 Meta 混淆
|
||||
type TaskMeta struct {
|
||||
Type string `json:"type"`
|
||||
AsynqTask string `json:"asynq_task"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
SupportsTime bool `json:"supports_time"`
|
||||
MaxRetry int `json:"max_retry"`
|
||||
Queue string `json:"queue"`
|
||||
Retryable bool `json:"retryable"` // 是否支持手动重试
|
||||
Params []TaskParam `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
dispatchableTasksMutex sync.RWMutex
|
||||
dispatchableTasks []TaskMeta
|
||||
)
|
||||
|
||||
// RegisterTaskMeta 注册任务元数据到全局列表
|
||||
func RegisterTaskMeta(meta TaskMeta) {
|
||||
dispatchableTasksMutex.Lock()
|
||||
defer dispatchableTasksMutex.Unlock()
|
||||
for _, t := range dispatchableTasks {
|
||||
if t.Type == meta.Type {
|
||||
return
|
||||
}
|
||||
}
|
||||
dispatchableTasks = append(dispatchableTasks, meta)
|
||||
}
|
||||
|
||||
// GetDispatchableTasks 获取所有已注册的元数据列表(返回副本以避免并发并发读写冲突)
|
||||
func GetDispatchableTasks() []TaskMeta {
|
||||
dispatchableTasksMutex.RLock()
|
||||
defer dispatchableTasksMutex.RUnlock()
|
||||
|
||||
metas := make([]TaskMeta, len(dispatchableTasks))
|
||||
copy(metas, dispatchableTasks)
|
||||
return metas
|
||||
}
|
||||
|
||||
// GetTaskMeta 根据任务类型获取元数据
|
||||
func GetTaskMeta(taskType string) *TaskMeta {
|
||||
dispatchableTasksMutex.RLock()
|
||||
defer dispatchableTasksMutex.RUnlock()
|
||||
for _, t := range dispatchableTasks {
|
||||
if t.Type == taskType {
|
||||
copied := t
|
||||
return &copied
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTaskMetaByAsynqTask 根据 Asynq 任务名称获取元数据
|
||||
func GetTaskMetaByAsynqTask(asynqTask string) *TaskMeta {
|
||||
dispatchableTasksMutex.RLock()
|
||||
defer dispatchableTasksMutex.RUnlock()
|
||||
for _, t := range dispatchableTasks {
|
||||
if t.AsynqTask == asynqTask {
|
||||
copied := t
|
||||
return &copied
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRegisteredAsynqTasks 返回所有已注册的 Asynq 任务名称,以便动态注册路由
|
||||
func GetRegisteredAsynqTasks() []string {
|
||||
keys := make([]string, 0, len(handlerRegistry))
|
||||
for k := range handlerRegistry {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package task_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/task"
|
||||
taskhandlers "github.com/Rain-kl/Wavelet/internal/infra/task/handlers"
|
||||
)
|
||||
|
||||
func TestDuplicateTaskMeta(t *testing.T) {
|
||||
// Call Register twice to simulate being imported by multiple packages (routers, worker, etc.)
|
||||
taskhandlers.Register()
|
||||
taskhandlers.Register()
|
||||
|
||||
metas := task.GetDispatchableTasks()
|
||||
|
||||
// Check if we have duplicates by checking if a Type appears more than once
|
||||
seen := make(map[string]int)
|
||||
for _, m := range metas {
|
||||
seen[m.Type]++
|
||||
}
|
||||
|
||||
for taskType, count := range seen {
|
||||
if count > 1 {
|
||||
t.Errorf("Task type %q registered %d times, expected at most 1", taskType, count)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package scheduler 提供定时任务调度功能
|
||||
package scheduler
|
||||
|
||||
const (
|
||||
errLoadLocationFailed = "failed to load location: %w"
|
||||
)
|
||||
@@ -0,0 +1,144 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/task"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/platform/bootstrap"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
var (
|
||||
activeScheduler *asynq.Scheduler
|
||||
schedulerMutex sync.Mutex
|
||||
quitChan chan struct{}
|
||||
schedulerOnce sync.Once
|
||||
)
|
||||
|
||||
// GetAsynqClient 获取全局 AsynqClient
|
||||
func GetAsynqClient() *asynq.Client {
|
||||
return task.AsynqClient
|
||||
}
|
||||
|
||||
// StartScheduler 启动调度器 (该函数阻塞,直到调度器退出)
|
||||
func StartScheduler() error {
|
||||
bootstrap.RegisterScheduler()
|
||||
|
||||
var err error
|
||||
schedulerOnce.Do(func() {
|
||||
quitChan = make(chan struct{})
|
||||
done := quitChan
|
||||
|
||||
// 初始化并运行首次调度
|
||||
if err = ReloadScheduler(); err != nil {
|
||||
err = fmt.Errorf("initial reload failed: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
signalCtx, stopSignals := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stopSignals()
|
||||
|
||||
if waitForStop(done, signalCtx.Done()) {
|
||||
StopScheduler()
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// StopScheduler 停止调度服务并解除 StartScheduler 阻塞
|
||||
func StopScheduler() {
|
||||
schedulerMutex.Lock()
|
||||
defer schedulerMutex.Unlock()
|
||||
|
||||
if activeScheduler != nil {
|
||||
activeScheduler.Shutdown()
|
||||
activeScheduler = nil
|
||||
}
|
||||
|
||||
if quitChan != nil {
|
||||
close(quitChan)
|
||||
quitChan = nil
|
||||
}
|
||||
}
|
||||
|
||||
// ReloadScheduler 重载调度器配置 (线程安全)
|
||||
func ReloadScheduler() error {
|
||||
schedulerMutex.Lock()
|
||||
defer schedulerMutex.Unlock()
|
||||
|
||||
// 1. 如果有运行中的调度器,先关闭它
|
||||
if activeScheduler != nil {
|
||||
activeScheduler.Shutdown()
|
||||
activeScheduler = nil
|
||||
}
|
||||
|
||||
// 2. 从数据库载入启用的定时任务配置
|
||||
schedules, err := model.ListActiveSchedules(context.Background())
|
||||
if err != nil {
|
||||
return fmt.Errorf("load schedules from db failed: %w", err)
|
||||
}
|
||||
|
||||
location, err := time.LoadLocation("Asia/Shanghai")
|
||||
if err != nil {
|
||||
return fmt.Errorf(errLoadLocationFailed, err)
|
||||
}
|
||||
|
||||
// 3. 实例化新的调度器
|
||||
newScheduler := asynq.NewScheduler(
|
||||
task.RedisOpt,
|
||||
&asynq.SchedulerOpts{
|
||||
Location: location,
|
||||
},
|
||||
)
|
||||
|
||||
// 4. 遍历并注册任务
|
||||
for _, s := range schedules {
|
||||
meta := task.GetTaskMeta(s.TaskType)
|
||||
if meta == nil {
|
||||
continue // 忽略排程配置中无效的任务类型
|
||||
}
|
||||
|
||||
// 构造 Asynq 载荷。定时任务使用对应 Meta 中的 Asynq 标识,同时将数据库中保存的 json 作为参数
|
||||
t := asynq.NewTask(meta.AsynqTask, []byte(s.Payload))
|
||||
|
||||
if _, err := newScheduler.Register(
|
||||
s.Cron,
|
||||
t,
|
||||
asynq.MaxRetry(meta.MaxRetry),
|
||||
asynq.Queue(meta.Queue),
|
||||
); err != nil {
|
||||
// 定时任务配置可能有误(如 Cron 格式不被 Asynq 识别),记录日志并跳过
|
||||
logger.ErrorF(context.Background(), "[Scheduler] 注册定时任务失败 id=%d name=%s: %v", s.ID, s.Name, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 启动并替换全局调度器。进程信号由 StartScheduler 统一处理。
|
||||
if err := newScheduler.Start(); err != nil {
|
||||
return fmt.Errorf("start scheduler failed: %w", err)
|
||||
}
|
||||
activeScheduler = newScheduler
|
||||
|
||||
logger.InfoF(context.Background(), "[Scheduler] 成功重新加载定时任务,共注册 %d 个活动任务", len(schedules))
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForStop(done <-chan struct{}, signals <-chan struct{}) bool {
|
||||
select {
|
||||
case <-done:
|
||||
return false
|
||||
case <-signals:
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package scheduler
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestWaitForStop(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
closeDone bool
|
||||
closeSignal bool
|
||||
wantSignal bool
|
||||
}{
|
||||
{
|
||||
name: "explicit stop",
|
||||
closeDone: true,
|
||||
},
|
||||
{
|
||||
name: "process signal",
|
||||
closeSignal: true,
|
||||
wantSignal: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
signals := make(chan struct{})
|
||||
if tt.closeDone {
|
||||
close(done)
|
||||
}
|
||||
if tt.closeSignal {
|
||||
close(signals)
|
||||
}
|
||||
|
||||
if got := waitForStop(done, signals); got != tt.wantSignal {
|
||||
t.Errorf("waitForStop() = %t, want %t", got, tt.wantSignal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package task
|
||||
|
||||
import (
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
)
|
||||
|
||||
type redisClientConnOpt struct {
|
||||
options redis.Options
|
||||
}
|
||||
|
||||
func (opt redisClientConnOpt) MakeRedisClient() interface{} {
|
||||
return redis.NewClient(&opt.options)
|
||||
}
|
||||
|
||||
type redisClusterConnOpt struct {
|
||||
options redis.ClusterOptions
|
||||
}
|
||||
|
||||
func (opt redisClusterConnOpt) MakeRedisClient() interface{} {
|
||||
return redis.NewClusterClient(&opt.options)
|
||||
}
|
||||
|
||||
type redisFailoverConnOpt struct {
|
||||
options redis.FailoverOptions
|
||||
maintNotificationsEnabled bool
|
||||
}
|
||||
|
||||
func (opt redisFailoverConnOpt) MakeRedisClient() interface{} {
|
||||
client := redis.NewFailoverClient(&opt.options)
|
||||
// go-redis v9.16 does not expose maintenance notification settings on
|
||||
// FailoverOptions, so apply the configured mode before the client is used.
|
||||
client.Options().MaintNotificationsConfig = maintNotificationsConfig(opt.maintNotificationsEnabled)
|
||||
return client
|
||||
}
|
||||
|
||||
func maintNotificationsConfig(enabled bool) *maintnotifications.Config {
|
||||
mode := maintnotifications.ModeDisabled
|
||||
if enabled {
|
||||
mode = maintnotifications.ModeAuto
|
||||
}
|
||||
return &maintnotifications.Config{Mode: mode}
|
||||
}
|
||||
|
||||
// RedisOpt asynq Redis 连接配置(兼容 Standalone/Sentinel/Cluster)
|
||||
var RedisOpt asynq.RedisConnOpt
|
||||
|
||||
// AsynqClient asynq 客户端,用于任务入队
|
||||
var AsynqClient *asynq.Client
|
||||
|
||||
func init() {
|
||||
RedisOpt = NewRedisConnOpt()
|
||||
AsynqClient = asynq.NewClient(RedisOpt)
|
||||
}
|
||||
|
||||
// NewRedisConnOpt 根据配置返回对应的 asynq Redis 连接选项
|
||||
func NewRedisConnOpt() asynq.RedisConnOpt {
|
||||
cfg := config.Config.Redis
|
||||
addrs := cfg.Addrs
|
||||
|
||||
if cfg.ClusterMode {
|
||||
return redisClusterConnOpt{
|
||||
options: redis.ClusterOptions{
|
||||
Addrs: addrs,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
MaintNotificationsConfig: maintNotificationsConfig(cfg.MaintNotifications),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.MasterName != "" {
|
||||
return redisFailoverConnOpt{
|
||||
maintNotificationsEnabled: cfg.MaintNotifications,
|
||||
options: redis.FailoverOptions{
|
||||
MasterName: cfg.MasterName,
|
||||
SentinelAddrs: addrs,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
DB: cfg.DB,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
addr := "localhost:6379"
|
||||
if len(addrs) > 0 {
|
||||
addr = addrs[0]
|
||||
}
|
||||
return redisClientConnOpt{
|
||||
options: redis.Options{
|
||||
Addr: addr,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
DB: cfg.DB,
|
||||
PoolSize: cfg.PoolSize,
|
||||
MaintNotificationsConfig: maintNotificationsConfig(cfg.MaintNotifications),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// PrefixedQueue 返回带前缀的队列名,用于 Cluster 模式隔离
|
||||
func PrefixedQueue(queue string) string {
|
||||
prefix := config.Config.Redis.KeyPrefix
|
||||
if prefix == "" {
|
||||
return queue
|
||||
}
|
||||
return prefix + queue
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
)
|
||||
|
||||
func TestNewRedisConnOptConfiguresMaintenanceNotifications(t *testing.T) {
|
||||
previous := config.Config.Redis
|
||||
t.Cleanup(func() { config.Config.Redis = previous })
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
enabled bool
|
||||
want maintnotifications.Mode
|
||||
}{
|
||||
{name: "disabled by default", enabled: false, want: maintnotifications.ModeDisabled},
|
||||
{name: "auto when enabled", enabled: true, want: maintnotifications.ModeAuto},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
config.Config.Redis.MaintNotifications = test.enabled
|
||||
|
||||
t.Run("standalone", func(t *testing.T) {
|
||||
config.Config.Redis.ClusterMode = false
|
||||
config.Config.Redis.MasterName = ""
|
||||
config.Config.Redis.Addrs = []string{"127.0.0.1:6379"}
|
||||
|
||||
client, ok := NewRedisConnOpt().MakeRedisClient().(*redis.Client)
|
||||
if !ok {
|
||||
t.Fatal("standalone option did not create *redis.Client")
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
assertMaintenanceNotificationsMode(t, client.Options().MaintNotificationsConfig, test.want)
|
||||
})
|
||||
|
||||
t.Run("cluster", func(t *testing.T) {
|
||||
config.Config.Redis.ClusterMode = true
|
||||
config.Config.Redis.MasterName = ""
|
||||
config.Config.Redis.Addrs = []string{"127.0.0.1:6379"}
|
||||
|
||||
client, ok := NewRedisConnOpt().MakeRedisClient().(*redis.ClusterClient)
|
||||
if !ok {
|
||||
t.Fatal("cluster option did not create *redis.ClusterClient")
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
assertMaintenanceNotificationsMode(t, client.Options().MaintNotificationsConfig, test.want)
|
||||
})
|
||||
|
||||
t.Run("sentinel", func(t *testing.T) {
|
||||
config.Config.Redis.ClusterMode = false
|
||||
config.Config.Redis.MasterName = "openflare"
|
||||
config.Config.Redis.Addrs = []string{"127.0.0.1:26379"}
|
||||
|
||||
client, ok := NewRedisConnOpt().MakeRedisClient().(*redis.Client)
|
||||
if !ok {
|
||||
t.Fatal("sentinel option did not create *redis.Client")
|
||||
}
|
||||
defer func() { _ = client.Close() }()
|
||||
assertMaintenanceNotificationsMode(t, client.Options().MaintNotificationsConfig, test.want)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertMaintenanceNotificationsMode(t *testing.T, cfg *maintnotifications.Config, want maintnotifications.Mode) {
|
||||
t.Helper()
|
||||
if cfg == nil || cfg.Mode != want {
|
||||
t.Fatalf("maintenance notifications mode = %v, want %v", cfg, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package worker 提供 Asynq 任务处理服务器与中间件
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// taskLoggingMiddleware 任务处理中间件
|
||||
// 注意:OTel Span 创建、日志记录、TaskExecution 状态管理
|
||||
// 已由 task.ProcessTask 统一处理,此中间件保留用于未来扩展(如限流、监控等)
|
||||
func taskLoggingMiddleware(h asynq.Handler) asynq.Handler {
|
||||
return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error {
|
||||
return h.ProcessTask(ctx, t)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2025 linux.do
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package worker
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/task"
|
||||
"github.com/Rain-kl/Wavelet/internal/platform/bootstrap"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// workerShutdownTimeout Worker 优雅关闭超时时间
|
||||
const workerShutdownTimeout = 3 * time.Minute
|
||||
|
||||
// StartWorker 启动任务处理服务器
|
||||
func StartWorker() error {
|
||||
bootstrap.RegisterWorker()
|
||||
asynqServer := asynq.NewServer(
|
||||
task.RedisOpt,
|
||||
asynq.Config{
|
||||
Concurrency: config.Config.Worker.Concurrency,
|
||||
ShutdownTimeout: workerShutdownTimeout,
|
||||
Queues: buildQueuesFromConfig(),
|
||||
StrictPriority: config.Config.Worker.StrictPriority,
|
||||
},
|
||||
)
|
||||
|
||||
// 注册 Asynq 任务路由
|
||||
mux := asynq.NewServeMux()
|
||||
mux.Use(taskLoggingMiddleware)
|
||||
|
||||
// 统一使用 task.ProcessTask 处理所有任务类型
|
||||
// 框架内部自动分发到对应的 TaskHandler 实现
|
||||
// 动态注册所有已注册的任务处理器路由,框架内部自动分发到对应的 TaskHandler 实现
|
||||
for _, taskName := range task.GetRegisteredAsynqTasks() {
|
||||
mux.HandleFunc(taskName, task.ProcessTask)
|
||||
}
|
||||
|
||||
// 启动服务器
|
||||
return asynqServer.Run(mux)
|
||||
}
|
||||
|
||||
// buildQueuesFromConfig 从配置构建队列映射
|
||||
func buildQueuesFromConfig() map[string]int {
|
||||
queues := make(map[string]int)
|
||||
|
||||
// 从配置读取队列
|
||||
if len(config.Config.Worker.Queues) > 0 {
|
||||
for _, q := range config.Config.Worker.Queues {
|
||||
if q.Name != "" && q.Priority > 0 {
|
||||
queues[q.Name] = q.Priority
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果配置为空,使用默认队列
|
||||
if len(queues) == 0 {
|
||||
queues = map[string]int{
|
||||
task.QueueDefault: 1,
|
||||
}
|
||||
}
|
||||
|
||||
return queues
|
||||
}
|
||||
Reference in New Issue
Block a user