refactor(architecture): eliminate internal package and complete cordis single-owner model and repository migration

- Physically purged all legacy internal/ packages, centralized pkg/model/ and pkg/repository/
- Migrated domain models and database repositories into self-contained owner plugins (user, auth, message_gateway, admin, upload, risk_control)
- Decoupled cross-plugin interactions via pure core/contracts and typed EventBus
- Ensured 100% test coverage pass, zero data races (-race clean), and 0 lint issues in make code-check
This commit is contained in:
ryan
2026-08-28 08:40:43 +08:00
parent 1f348fd425
commit fb6a3edb89
323 changed files with 8222 additions and 17693 deletions
+12
View File
@@ -0,0 +1,12 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package buildinfo exposes metadata injected by the release workflow.
package buildinfo
var (
// Version is the application version.
Version = "dev"
// BuildTime is the UTC release build timestamp.
BuildTime = ""
)
+254
View File
@@ -0,0 +1,254 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package config 负责应用配置的加载、解析与环境变量覆盖。
package config
import (
"encoding/json"
"flag"
"log"
"os"
"strconv"
"strings"
"github.com/spf13/viper"
)
// 默认队列优先级
const (
webhookQueuePriority = 10
whitelistQueuePriority = 5
defaultQueuePriority = 3
)
// Config 全局配置单例,初始化后不可变
var Config *configModel
// findConfigPath searches upward for the config file to handle tests running in subdirectories.
func findConfigPath(configPath string) string {
if _, err := os.Stat(configPath); err == nil {
return configPath
}
dir := "."
for i := 0; i < 5; i++ {
dir += "/.."
path := dir + "/" + configPath
if _, err := os.Stat(path); err == nil {
return path
}
}
return configPath
}
// isTest checks if the current execution context is within 'go test'.
func isTest() bool {
if flag.Lookup("test.v") != nil {
return true
}
for _, arg := range os.Args {
if strings.HasPrefix(arg, "-test.") || strings.HasSuffix(arg, ".test") {
return true
}
}
return false
}
func init() {
// 加载配置文件路径
configPath := os.Getenv("CONFIG_PATH")
if configPath == "" {
configPath = findConfigPath("config.yaml")
}
// 设置配置文件
viper.SetConfigFile(configPath)
viper.AutomaticEnv()
// 读取配置文件(可选:找不到文件时使用空默认值 + 环境变量)
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
// 文件存在但读取/解析失败
if _, statErr := os.Stat(configPath); statErr == nil { //nolint:gosec // configPath is loaded from CONFIG_PATH environment variable
log.Fatalf("[Config] read config failed: %v\n", err)
}
}
log.Println("[Config] no config file found, using environment variables only")
viper.SetConfigType("yaml")
if err := viper.ReadConfig(strings.NewReader("")); err != nil {
log.Fatalf("[Config] failed to init empty config: %v\n", err)
}
}
// 解析配置到结构体
var c configModel
if err := viper.Unmarshal(&c); err != nil {
log.Fatalf("[Config] parse config failed: %v\n", err)
}
applyDefaults(&c)
// 环境变量覆盖(优先级高于 config.yaml)
applyEnvOverrides(&c)
applyDefaults(&c)
// Disable standard DB/Redis initializations during tests to prevent connection attempts.
if isTest() {
c.Database.Enabled = false
c.Database.SQLitePath = ":memory:"
c.Redis.Enabled = false
c.ClickHouse.Enabled = false
}
// 设置全局配置
Config = &c
// 打印配置
printConfig(&c)
}
func applyDefaults(c *configModel) {
if c.App.SessionAge <= 0 {
c.App.SessionAge = 86400
}
if c.Otel.TracerName == "" {
c.Otel.TracerName = "github.com/Rain-kl/Wavelet"
}
}
// ─── 环境变量覆盖层 ────────────────────────────────────────────────────────────
// 环境变量优先级高于 config.yaml,未设置则保留 yaml 中的值。
func envStr(key, fallback string) string {
if v, ok := os.LookupEnv(key); ok {
return v
}
return fallback
}
func envInt(key string, fallback int) int {
if v, ok := os.LookupEnv(key); ok {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return fallback
}
func envInt64(key string, fallback int64) int64 {
if v, ok := os.LookupEnv(key); ok {
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
return n
}
}
return fallback
}
func envFloat64(key string, fallback float64) float64 {
if v, ok := os.LookupEnv(key); ok {
if n, err := strconv.ParseFloat(v, 64); err == nil {
return n
}
}
return fallback
}
func envBool(key string, fallback bool) bool {
if v, ok := os.LookupEnv(key); ok {
if b, err := strconv.ParseBool(v); err == nil {
return b
}
}
return fallback
}
// applyEnvOverrides 将环境变量值覆盖到配置结构体上(仅当环境变量已设置时生效)
func applyEnvOverrides(c *configModel) {
// ─── App ───
c.App.AppName = envStr("APP_NAME", c.App.AppName)
c.App.Env = envStr("APP_ENV", c.App.Env)
c.App.Addr = envStr("APP_ADDR", c.App.Addr)
c.App.NodeID = envInt64("APP_NODE_ID", c.App.NodeID)
c.App.APIPrefix = envStr("APP_API_PREFIX", c.App.APIPrefix)
c.App.GracefulShutdownTimeout = envInt("APP_GRACEFUL_SHUTDOWN_TIMEOUT", c.App.GracefulShutdownTimeout)
c.App.SessionCookieName = envStr("APP_SESSION_COOKIE_NAME", c.App.SessionCookieName)
c.App.SessionSecret = envStr("APP_SESSION_SECRET", c.App.SessionSecret)
c.App.SessionDomain = envStr("APP_SESSION_DOMAIN", c.App.SessionDomain)
c.App.SessionAge = envInt("APP_SESSION_AGE", c.App.SessionAge)
c.App.SessionHTTPOnly = envBool("APP_SESSION_HTTP_ONLY", c.App.SessionHTTPOnly)
c.App.SessionSecure = envBool("APP_SESSION_SECURE", c.App.SessionSecure)
// ─── Database ───
c.Database.Host = envStr("DB_HOST", c.Database.Host)
c.Database.Port = envInt("DB_PORT", c.Database.Port)
c.Database.Username = envStr("DB_USERNAME", c.Database.Username)
c.Database.Password = envStr("DB_PASSWORD", c.Database.Password)
c.Database.Database = envStr("DB_NAME", c.Database.Database)
c.Database.SSLMode = envStr("DB_SSL_MODE", c.Database.SSLMode)
c.Database.TimeZone = envStr("DB_TIMEZONE", c.Database.TimeZone)
c.Database.LogLevel = envStr("DB_LOG_LEVEL", c.Database.LogLevel)
c.Database.MaxIdleConn = envInt("DB_MAX_IDLE_CONN", c.Database.MaxIdleConn)
c.Database.MaxOpenConn = envInt("DB_MAX_OPEN_CONN", c.Database.MaxOpenConn)
// 当 DB_HOST 环境变量已设置时自动启用数据库
if _, ok := os.LookupEnv("DB_HOST"); ok {
c.Database.Enabled = true
}
c.Database.Enabled = envBool("DB_ENABLED", c.Database.Enabled)
c.Database.SQLitePath = envStr("SQLITE_PATH", c.Database.SQLitePath)
// ─── Redis ───
if v, ok := os.LookupEnv("REDIS_ADDR"); ok {
c.Redis.Addrs = []string{v}
c.Redis.Enabled = true // 当 REDIS_ADDR 已设置时自动启用
}
c.Redis.Enabled = envBool("REDIS_ENABLED", c.Redis.Enabled)
c.Redis.Username = envStr("REDIS_USERNAME", c.Redis.Username)
c.Redis.Password = envStr("REDIS_PASSWORD", c.Redis.Password)
c.Redis.DB = envInt("REDIS_DB", c.Redis.DB)
c.Redis.KeyPrefix = envStr("REDIS_KEY_PREFIX", c.Redis.KeyPrefix)
c.Redis.PoolSize = envInt("REDIS_POOL_SIZE", c.Redis.PoolSize)
c.Redis.MaintNotifications = envBool("REDIS_MAINT_NOTIFICATIONS", c.Redis.MaintNotifications)
// ─── ClickHouse ───
if v, ok := os.LookupEnv("CLICKHOUSE_HOST"); ok {
c.ClickHouse.Hosts = []string{v}
c.ClickHouse.Enabled = true
}
c.ClickHouse.Enabled = envBool("CLICKHOUSE_ENABLED", c.ClickHouse.Enabled)
c.ClickHouse.Username = envStr("CLICKHOUSE_USERNAME", c.ClickHouse.Username)
c.ClickHouse.Password = envStr("CLICKHOUSE_PASSWORD", c.ClickHouse.Password)
c.ClickHouse.Database = envStr("CLICKHOUSE_NAME", c.ClickHouse.Database)
// ─── Log ───
c.Log.Level = envStr("LOG_LEVEL", c.Log.Level)
c.Log.Format = envStr("LOG_FORMAT", c.Log.Format)
c.Log.Output = envStr("LOG_OUTPUT", c.Log.Output)
// ─── OTel ───
c.Otel.SamplingRate = envFloat64("OTEL_SAMPLING_RATE", c.Otel.SamplingRate)
c.Otel.TracerName = envStr("OTEL_TRACER_NAME", c.Otel.TracerName)
// ─── Worker ───
c.Worker.Concurrency = envInt("WORKER_CONCURRENCY", c.Worker.Concurrency)
c.Worker.StrictPriority = envBool("WORKER_STRICT_PRIORITY", c.Worker.StrictPriority)
// 无 yaml 且无环境变量时,使用代码级默认队列
if len(c.Worker.Queues) == 0 {
c.Worker.Queues = []QueueConfig{
{Name: "webhook", Priority: webhookQueuePriority},
{Name: "whitelist_only", Priority: whitelistQueuePriority},
{Name: "default", Priority: defaultQueuePriority},
}
}
}
// printConfig 打印配置内容
func printConfig(c *configModel) {
configJSON, err := json.MarshalIndent(c, "", " ")
if err != nil {
log.Printf("[Config] failed to marshal config: %v\n", err)
return
}
log.Printf("[Config] loaded configuration:\n%s\n", string(configJSON))
}
+17
View File
@@ -0,0 +1,17 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package config
import "testing"
func TestApplyEnvOverridesRedisMaintNotifications(t *testing.T) {
t.Setenv("REDIS_MAINT_NOTIFICATIONS", "true")
cfg := &configModel{}
applyEnvOverrides(cfg)
if !cfg.Redis.MaintNotifications {
t.Fatal("REDIS_MAINT_NOTIFICATIONS=true was not applied")
}
}
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package config
import "time"
type configModel struct {
App appConfig `mapstructure:"app"`
Database databaseConfig `mapstructure:"database"`
Redis redisConfig `mapstructure:"redis"`
Log logConfig `mapstructure:"log"`
Scheduler schedulerConfig `mapstructure:"scheduler"`
Worker workerConfig `mapstructure:"worker"`
ClickHouse clickHouseConfig `mapstructure:"clickhouse"`
Otel otelConfig `mapstructure:"otel"`
}
// appConfig 应用基本配置
type appConfig struct {
AppName string `mapstructure:"app_name"`
Env string `mapstructure:"env"`
Addr string `mapstructure:"addr"`
NodeID int64 `mapstructure:"node_id"`
APIPrefix string `mapstructure:"api_prefix"`
GracefulShutdownTimeout int `mapstructure:"graceful_shutdown_timeout"`
SessionCookieName string `mapstructure:"session_cookie_name"`
SessionSecret string `mapstructure:"session_secret"`
SessionDomain string `mapstructure:"session_domain"`
SessionAge int `mapstructure:"session_age"`
SessionHTTPOnly bool `mapstructure:"session_http_only"`
SessionSecure bool `mapstructure:"session_secure"`
}
// IsProduction 检查当前环境是否为生产环境
func (a *appConfig) IsProduction() bool {
return a.Env == "production"
}
// databaseConfig 数据库配置
type databaseConfig struct {
Enabled bool `mapstructure:"enabled"`
SQLitePath string `mapstructure:"sqlite_path"` // PostgreSQL 禁用时的 SQLite 文件路径
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Database string `mapstructure:"database"`
MaxIdleConn int `mapstructure:"max_idle_conn"`
MaxOpenConn int `mapstructure:"max_open_conn"`
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
LogLevel string `mapstructure:"log_level"`
SSLMode string `mapstructure:"ssl_mode"`
TimeZone string `mapstructure:"time_zone"`
ApplicationName string `mapstructure:"application_name"`
SearchPath string `mapstructure:"search_path"`
PreferSimpleProtocol bool `mapstructure:"prefer_simple_protocol"`
StatementCacheCapacity int `mapstructure:"statement_cache_capacity"`
DefaultQueryExecMode string `mapstructure:"default_query_exec_mode"`
Replicas []databaseReplicaConfig `mapstructure:"replicas"`
SlowThreshold time.Duration `mapstructure:"slow_threshold"`
}
// databaseReplicaConfig 只读副本配置
type databaseReplicaConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
}
// clickhouse 配置
type clickHouseConfig struct {
Enabled bool `mapstructure:"enabled"`
Hosts []string `mapstructure:"hosts"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Database string `mapstructure:"database"`
MaxIdleConn int `mapstructure:"max_idle_conn"`
MaxOpenConn int `mapstructure:"max_open_conn"`
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
DialTimeout int `mapstructure:"dial_timeout"`
BlockBufferSize uint8 `mapstructure:"block_buffer_size"`
}
// redisConfig Redis配置
type redisConfig struct {
Enabled bool `mapstructure:"enabled"`
Addrs []string `mapstructure:"addrs"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
DB int `mapstructure:"db"`
ClusterMode bool `mapstructure:"cluster_mode"`
MasterName string `mapstructure:"master_name"`
KeyPrefix string `mapstructure:"key_prefix"`
PoolSize int `mapstructure:"pool_size"`
MinIdleConn int `mapstructure:"min_idle_conn"`
DialTimeout int `mapstructure:"dial_timeout"`
ReadTimeout int `mapstructure:"read_timeout"`
WriteTimeout int `mapstructure:"write_timeout"`
MaxRetries int `mapstructure:"max_retries"`
PoolTimeout int `mapstructure:"pool_timeout"`
ConnMaxIdleTime int `mapstructure:"conn_max_idle_time"`
MaintNotifications bool `mapstructure:"maint_notifications"`
}
// logConfig 日志配置
type logConfig struct {
Level string `mapstructure:"level"`
Format string `mapstructure:"format"`
Output string `mapstructure:"output"`
FilePath string `mapstructure:"file_path"`
MaxSize int `mapstructure:"max_size"`
MaxAge int `mapstructure:"max_age"`
MaxBackups int `mapstructure:"max_backups"`
Compress bool `mapstructure:"compress"`
}
// schedulerConfig 定时任务配置
type schedulerConfig struct {
}
// workerConfig 工作配置
type workerConfig struct {
Concurrency int `mapstructure:"concurrency"`
StrictPriority bool `mapstructure:"strict_priority"`
Queues []QueueConfig `mapstructure:"queues"`
}
// QueueConfig 队列配置
type QueueConfig struct {
Name string `mapstructure:"name"`
Priority int `mapstructure:"priority"`
}
// otelConfig OpenTelemetry 配置
type otelConfig struct {
SamplingRate float64 `mapstructure:"sampling_rate"`
TracerName string `mapstructure:"tracer_name"`
}
+120
View File
@@ -0,0 +1,120 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics provides ClickHouse data access for analytics tables.
package analytics
import (
"context"
"fmt"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/util"
"gorm.io/gorm"
)
// CountAccessLogs returns the number of access logs matching filter.
func CountAccessLogs(ctx context.Context, filter AccessLogFilter) (uint64, error) {
ch := db.ChDB(ctx)
if ch == nil {
return 0, fmt.Errorf("clickhouse gorm connection is not initialized")
}
var count int64
query := applyFilter(ch.Model(&UserAccessLog{}), filter)
if err := query.Count(&count).Error; err != nil {
return 0, fmt.Errorf("count access logs: %w", err)
}
return safeUint64Count(count), nil
}
// ListAccessLogs returns paginated access logs and the total match count.
func ListAccessLogs(ctx context.Context, filter AccessLogFilter, page, pageSize int) ([]UserAccessLog, uint64, error) {
ch := db.ChDB(ctx)
if ch == nil {
return nil, 0, fmt.Errorf("clickhouse gorm connection is not initialized")
}
if filter.UserIDs != nil && len(filter.UserIDs) == 0 {
return []UserAccessLog{}, 0, nil
}
var total int64
baseQuery := applyFilter(ch.Model(&UserAccessLog{}), filter)
if err := baseQuery.Count(&total).Error; err != nil {
return nil, 0, fmt.Errorf("count access logs: %w", err)
}
if total == 0 {
return []UserAccessLog{}, 0, nil
}
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = 20
}
offset := (page - 1) * pageSize
var logs []UserAccessLog
err := applyFilter(ch.Model(&UserAccessLog{}), filter).
Order("created_at DESC, id DESC").
Limit(pageSize).
Offset(offset).
Find(&logs).Error
if err != nil {
return nil, 0, fmt.Errorf("list access logs: %w", err)
}
return logs, safeUint64Count(total), nil
}
// DeleteAllUserAccessLogs hard-deletes all user access logs via TRUNCATE.
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
if db.ChConn == nil {
return 0, fmt.Errorf("clickhouse connection is not initialized")
}
if err := db.ChConn.Exec(ctx, "TRUNCATE TABLE "+UserAccessLog{}.TableName()); err != nil {
return 0, fmt.Errorf("truncate user access logs: %w", err)
}
return 0, nil
}
// DeleteUserAccessLogsBefore deletes user access logs older than cutoff.
func DeleteUserAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if db.ChConn == nil {
return 0, fmt.Errorf("clickhouse connection is not initialized")
}
if err := db.ChConn.Exec(ctx, "ALTER TABLE "+UserAccessLog{}.TableName()+" DELETE WHERE created_at < ?", cutoff); err != nil {
return 0, fmt.Errorf("delete expired user access logs: %w", err)
}
return 0, nil
}
func safeUint64Count(count int64) uint64 {
if count < 0 {
return 0
}
return uint64(count)
}
func applyFilter(query *gorm.DB, filter AccessLogFilter) *gorm.DB {
if filter.UserIDs != nil {
if len(filter.UserIDs) == 0 {
return query.Where("1 = 0")
}
query = query.Where("user_id IN ?", filter.UserIDs)
}
if filter.Path != "" {
query = query.Where("path LIKE ?", "%"+util.EscapeLike(filter.Path)+"%")
}
if filter.StartTime != nil {
query = query.Where("created_at >= ?", *filter.StartTime)
}
if filter.EndTime != nil {
query = query.Where("created_at <= ?", *filter.EndTime)
}
return query
}
@@ -0,0 +1,17 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import "time"
// AccessLogFilter scopes ClickHouse user access log queries.
type AccessLogFilter struct {
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
UserIDs []uint64
Path string
// StartTime filters created_at >= StartTime when non-nil.
StartTime *time.Time
// EndTime filters created_at <= EndTime when non-nil.
EndTime *time.Time
}
@@ -0,0 +1,158 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"fmt"
"sort"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence"
)
const hoursInDay = 24
// DailyTrend is a single day's access count.
type DailyTrend struct {
Date string
Count uint64
}
// BrowserShare is a browser group's share of access logs.
type BrowserShare struct {
Browser string
Count uint64
}
// TopUser is an active user ranked by access count.
type TopUser struct {
UserID uint64
Count uint64
}
// GetDailyTrend returns per-day access counts for the last days days (inclusive of today).
func GetDailyTrend(ctx context.Context, days int) ([]DailyTrend, error) {
if days < 1 {
days = 7
}
ch := db.ChDB(ctx)
if ch == nil {
return nil, fmt.Errorf("clickhouse gorm connection is not initialized")
}
startTime := time.Now().AddDate(0, 0, -(days - 1)).Truncate(hoursInDay * time.Hour)
tableName := UserAccessLog{}.TableName()
query := fmt.Sprintf(`
SELECT toDate(created_at) AS date, count() AS count
FROM %s
WHERE created_at >= ?
GROUP BY date
ORDER BY date ASC
`, tableName)
type trendRow struct {
Date time.Time
Count uint64
}
var rows []trendRow
if err := ch.Raw(query, startTime).Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("get daily trend: %w", err)
}
trendMap := make(map[string]uint64, days)
for i := 0; i < days; i++ {
dateStr := time.Now().AddDate(0, 0, -i).Format("2006-01-02")
trendMap[dateStr] = 0
}
for _, row := range rows {
dateStr := row.Date.Format("2006-01-02")
trendMap[dateStr] = row.Count
}
result := make([]DailyTrend, 0, days)
for i := days - 1; i >= 0; i-- {
dateStr := time.Now().AddDate(0, 0, -i).Format("2006-01-02")
result = append(result, DailyTrend{
Date: dateStr,
Count: trendMap[dateStr],
})
}
return result, nil
}
// GetBrowserDistribution returns browser-grouped access counts since startTime.
func GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]BrowserShare, error) {
ch := db.ChDB(ctx)
if ch == nil {
return nil, fmt.Errorf("clickhouse gorm connection is not initialized")
}
tableName := UserAccessLog{}.TableName()
query := fmt.Sprintf(`
SELECT user_agent, count() AS count
FROM %s
WHERE created_at >= ?
GROUP BY user_agent
`, tableName)
type uaRow struct {
UserAgent string
Count uint64
}
var rows []uaRow
if err := ch.Raw(query, startTime).Scan(&rows).Error; err != nil {
return nil, fmt.Errorf("get browser distribution: %w", err)
}
browserCounts := make(map[string]uint64)
for _, row := range rows {
browser := ParseBrowserName(row.UserAgent)
browserCounts[browser] += row.Count
}
result := make([]BrowserShare, 0, len(browserCounts))
for browser, count := range browserCounts {
result = append(result, BrowserShare{
Browser: browser,
Count: count,
})
}
sort.Slice(result, func(i, j int) bool {
return result[i].Count > result[j].Count
})
return result, nil
}
// GetTopActiveUsers returns the most active users since startTime.
func GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]TopUser, error) {
if limit < 1 {
limit = 10
}
ch := db.ChDB(ctx)
if ch == nil {
return nil, fmt.Errorf("clickhouse gorm connection is not initialized")
}
tableName := UserAccessLog{}.TableName()
query := fmt.Sprintf(`
SELECT user_id, count() AS count
FROM %s
WHERE created_at >= ? AND user_id > 0
GROUP BY user_id
ORDER BY count DESC
LIMIT ?
`, tableName)
var users []TopUser
if err := ch.Raw(query, startTime, limit).Scan(&users).Error; err != nil {
return nil, fmt.Errorf("get top active users: %w", err)
}
return users, nil
}
@@ -0,0 +1,214 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"io"
"testing"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/column"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func setupChGormDB(t *testing.T) *gorm.DB {
t.Helper()
gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, gormDB.AutoMigrate(&UserAccessLog{}))
db.SetChDBForTest(gormDB)
return gormDB
}
func TestParseBrowserName(t *testing.T) {
tests := []struct {
name string
ua string
want string
}{
{name: "chrome", ua: "Mozilla/5.0 Chrome/120.0.0.0", want: "Chrome"},
{name: "firefox", ua: "Mozilla/5.0 Firefox/121.0", want: "Firefox"},
{name: "safari", ua: "Mozilla/5.0 Safari/605.1.15", want: "Safari"},
{name: "edge", ua: "Mozilla/5.0 Edg/120.0.0.0", want: "Edge"},
{name: "wechat", ua: "MicroMessenger/8.0", want: "WeChat"},
{name: "postman", ua: "PostmanRuntime/7.36.0", want: "Postman"},
{name: "other", ua: "curl/8.0", want: "Other"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, ParseBrowserName(tt.ua))
})
}
}
func TestCountAccessLogs_EmptyUserIDs(t *testing.T) {
setupChGormDB(t)
t.Cleanup(func() { db.SetChDBForTest(nil) })
count, err := CountAccessLogs(context.Background(), AccessLogFilter{UserIDs: []uint64{}})
require.NoError(t, err)
assert.Equal(t, uint64(0), count)
}
func TestListAccessLogs_EmptyUserIDs(t *testing.T) {
setupChGormDB(t)
t.Cleanup(func() { db.SetChDBForTest(nil) })
logs, total, err := ListAccessLogs(context.Background(), AccessLogFilter{UserIDs: []uint64{}}, 1, 20)
require.NoError(t, err)
assert.Equal(t, uint64(0), total)
assert.Empty(t, logs)
}
func TestListAccessLogs_WithFilters(t *testing.T) {
gormDB := setupChGormDB(t)
t.Cleanup(func() { db.SetChDBForTest(nil) })
now := time.Now().UTC().Truncate(time.Second)
logs := []UserAccessLog{
{ID: 1, UserID: 10, Path: "/api/v1/users", Method: "GET", Status: 200, CreatedAt: now},
{ID: 2, UserID: 20, Path: "/api/v1/admin/logs", Method: "GET", Status: 200, CreatedAt: now},
{ID: 3, UserID: 10, Path: "/api/v1/other", Method: "POST", Status: 201, CreatedAt: now},
}
require.NoError(t, gormDB.Create(&logs).Error)
start := now.Add(-time.Hour)
filter := AccessLogFilter{
UserIDs: []uint64{10},
Path: "users",
StartTime: &start,
}
count, err := CountAccessLogs(context.Background(), filter)
require.NoError(t, err)
assert.Equal(t, uint64(1), count)
result, total, err := ListAccessLogs(context.Background(), filter, 1, 10)
require.NoError(t, err)
assert.Equal(t, uint64(1), total)
require.Len(t, result, 1)
assert.Equal(t, uint64(1), result[0].ID)
assert.Equal(t, "/api/v1/users", result[0].Path)
}
func TestBatchInsert_Empty(t *testing.T) {
err := BatchInsert(context.Background(), nil)
require.NoError(t, err)
}
func TestBatchInsert_UsesModelBatchSQL(t *testing.T) {
ctx := context.Background()
mockBatch := &mockBatch{}
mockConn := &mockConn{
batch: mockBatch,
batchQuery: UserAccessLog{}.BatchInsertSQL(),
}
db.SetChConnForTest(mockConn)
t.Cleanup(func() { db.SetChConnForTest(nil) })
createdAt := time.Now().UTC()
err := BatchInsert(ctx, []UserAccessLog{
{
ID: 1,
UserID: 42,
Path: "/api/v1/test",
Method: "GET",
IP: "127.0.0.1",
UserAgent: "test-agent",
Headers: "{}",
Status: 200,
Latency: 12,
CreatedAt: createdAt,
},
})
require.NoError(t, err)
assert.True(t, mockConn.prepareCalled)
assert.Equal(t, UserAccessLog{}.BatchInsertSQL(), mockConn.preparedQuery)
assert.True(t, mockBatch.sendCalled)
require.Len(t, mockBatch.rows, 1)
assert.Equal(t, uint64(42), mockBatch.rows[0][1])
}
type mockConn struct {
batch driver.Batch
batchQuery string
prepareCalled bool
preparedQuery string
}
func (m *mockConn) Contributors() []string { return nil }
func (m *mockConn) ServerVersion() (*driver.ServerVersion, error) { return nil, nil }
func (m *mockConn) Select(_ context.Context, _ any, _ string, _ ...any) error { return nil }
func (m *mockConn) Query(_ context.Context, _ string, _ ...any) (driver.Rows, error) {
return nil, nil
}
func (m *mockConn) QueryRow(_ context.Context, _ string, _ ...any) driver.Row { return nil }
func (m *mockConn) PrepareBatch(_ context.Context, query string, _ ...driver.PrepareBatchOption) (driver.Batch, error) {
m.prepareCalled = true
m.preparedQuery = query
return m.batch, nil
}
func (m *mockConn) Exec(_ context.Context, _ string, _ ...any) error { return nil }
func (m *mockConn) AsyncInsert(_ context.Context, _ string, _ bool, _ ...any) error { return nil }
func (m *mockConn) InsertFormat(_ context.Context, _ string, _ string, _ io.Reader) error { return nil }
func (m *mockConn) QueryFormat(_ context.Context, _ string, _ string, _ ...any) (io.ReadCloser, error) {
return nil, nil
}
func (m *mockConn) Ping(_ context.Context) error { return nil }
func (m *mockConn) Stats() driver.Stats { return driver.Stats{} }
func (m *mockConn) Close() error { return nil }
type mockBatch struct {
rows [][]any
sendCalled bool
}
func (m *mockBatch) Abort() error { return nil }
func (m *mockBatch) Append(v ...any) error {
m.rows = append(m.rows, v)
return nil
}
func (m *mockBatch) AppendStruct(_ any) error { return nil }
func (m *mockBatch) Column(_ int) driver.BatchColumn { return nil }
func (m *mockBatch) Flush() error { return nil }
func (m *mockBatch) Send() error {
m.sendCalled = true
return nil
}
func (m *mockBatch) IsSent() bool { return m.sendCalled }
func (m *mockBatch) Rows() int { return len(m.rows) }
func (m *mockBatch) Columns() []column.Interface { return nil }
func (m *mockBatch) Close() error { return nil }
@@ -0,0 +1,48 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"fmt"
"github.com/Rain-kl/Wavelet/pkg/persistence"
)
// BatchInsert writes access logs to ClickHouse using the native batch API.
func BatchInsert(ctx context.Context, logs []UserAccessLog) error {
if len(logs) == 0 {
return nil
}
if db.ChConn == nil {
return fmt.Errorf("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, UserAccessLog{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
for _, logItem := range logs {
if err := batch.Append(
logItem.ID,
logItem.UserID,
logItem.Path,
logItem.Method,
logItem.IP,
logItem.UserAgent,
logItem.Headers,
logItem.Status,
logItem.Latency,
logItem.CreatedAt,
); err != nil {
return fmt.Errorf("append access log to batch: %w", err)
}
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send clickhouse batch: %w", err)
}
return nil
}
+30
View File
@@ -0,0 +1,30 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import "strings"
// ParseBrowserName performs lightweight User-Agent browser identification.
func ParseBrowserName(ua string) string {
uaLower := strings.ToLower(ua)
if strings.Contains(uaLower, "micromessenger") {
return "WeChat"
}
if strings.Contains(uaLower, "postman") {
return "Postman"
}
if strings.Contains(uaLower, "edg/") || strings.Contains(uaLower, "edge") {
return "Edge"
}
if strings.Contains(uaLower, "firefox") {
return "Firefox"
}
if strings.Contains(uaLower, "chrome") {
return "Chrome"
}
if strings.Contains(uaLower, "safari") {
return "Safari"
}
return "Other"
}
+44
View File
@@ -0,0 +1,44 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics defines ClickHouse analytics domain models.
package analytics
import (
"fmt"
"time"
)
const (
userAccessLogTableName = "w_user_access_logs"
userAccessLogInsertColumns = "id, user_id, path, method, ip, user_agent, headers, status, latency, created_at"
)
// UserAccessLog stores HTTP access records in ClickHouse.
type UserAccessLog struct {
ID uint64 `gorm:"column:id"`
UserID uint64 `gorm:"column:user_id"`
Path string `gorm:"column:path"`
Method string `gorm:"column:method"`
IP string `gorm:"column:ip"`
UserAgent string `gorm:"column:user_agent"`
Headers string `gorm:"column:headers"`
Status int32 `gorm:"column:status"`
Latency int64 `gorm:"column:latency"`
CreatedAt time.Time `gorm:"column:created_at"`
}
// TableName returns the ClickHouse table name.
func (UserAccessLog) TableName() string {
return userAccessLogTableName
}
// InsertColumns returns comma-separated column names for batch insert.
func (UserAccessLog) InsertColumns() string {
return userAccessLogInsertColumns
}
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
func (UserAccessLog) BatchInsertSQL() string {
return fmt.Sprintf("INSERT INTO %s (%s)", userAccessLogTableName, userAccessLogInsertColumns)
}
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package batchwriter
import (
"fmt"
"time"
)
const (
defaultQueueSize = 10_000
defaultMaxBatchSize = 1_000
defaultMinBatchSize = 50
defaultFlushEvery = time.Second
)
// Config controls queue capacity and flush thresholds for a Writer instance.
type Config struct {
// Name identifies the writer in logs and diagnostics. Optional.
Name string
// QueueSize is the buffered channel capacity.
QueueSize int
// MaxBatchSize triggers a flush when the in-memory batch reaches this count.
MaxBatchSize int
// MinBatchSize is the minimum in-memory batch size for time-based flushes.
// Zero disables the threshold and preserves legacy interval flush behavior.
// When set, interval flushes below this size are skipped unless MaxFlushWait elapses.
MinBatchSize int
// FlushInterval is how often the worker checks whether a time-based flush should run.
FlushInterval time.Duration
// MaxFlushWait forces a flush of any non-empty batch once the oldest item has waited
// this long, even if MinBatchSize has not been reached. Zero disables the force path.
MaxFlushWait time.Duration
}
// DefaultConfig returns production-friendly defaults aligned with audit log batching.
func DefaultConfig() Config {
return Config{
QueueSize: defaultQueueSize,
MaxBatchSize: defaultMaxBatchSize,
MinBatchSize: defaultMinBatchSize,
FlushInterval: defaultFlushEvery,
}
}
func (c Config) validate() error {
if c.QueueSize <= 0 {
return fmt.Errorf("batchwriter: queue size must be positive")
}
if c.MaxBatchSize <= 0 {
return fmt.Errorf("batchwriter: max batch size must be positive")
}
if c.MinBatchSize < 0 {
return fmt.Errorf("batchwriter: min batch size must be non-negative")
}
if c.FlushInterval <= 0 {
return fmt.Errorf("batchwriter: flush interval must be positive")
}
if c.MaxFlushWait < 0 {
return fmt.Errorf("batchwriter: max flush wait must be non-negative")
}
return nil
}
+8
View File
@@ -0,0 +1,8 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package batchwriter
import "errors"
var errNilFlushFunc = errors.New("batchwriter: flush func is required")
+264
View File
@@ -0,0 +1,264 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package batchwriter provides a reusable buffered batch writer for high-throughput
// append-only sinks such as ClickHouse. Each business domain should own an independent
// Writer instance with its own queue, flush callback, and tuning parameters.
package batchwriter
import (
"context"
"sync"
"sync/atomic"
"time"
)
// FlushFunc persists a batch of queued items. It is invoked from the worker goroutine.
type FlushFunc[T any] func(ctx context.Context, items []T) error
// FlushErrorHandler is called when FlushFunc returns an error after optional retries.
// The batch is discarded after the handler returns; the worker continues processing.
// Handlers receive the failed items so callers can release dedup keys or re-queue.
type FlushErrorHandler[T any] func(ctx context.Context, items []T, err error)
// Stats is a point-in-time snapshot of Writer queue and failure counters.
type Stats struct {
Name string
Depth int
Cap int
Drops int64
FlushErrors int64
Running bool
}
// Writer buffers items and flushes them by size or interval.
type Writer[T any] struct {
cfg Config
flush FlushFunc[T]
onFlushError FlushErrorHandler[T]
onDrop func(T)
startOnce sync.Once
stopOnce sync.Once
mu sync.RWMutex
ch chan T
workerCtx context.Context
done chan struct{}
drops atomic.Int64
flushErrors atomic.Int64
}
// Option configures optional Writer callbacks.
type Option[T any] func(*Writer[T])
// WithFlushErrorHandler registers a callback for flush failures.
func WithFlushErrorHandler[T any](handler FlushErrorHandler[T]) Option[T] {
return func(w *Writer[T]) {
w.onFlushError = handler
}
}
// WithDropHandler registers a callback when TryEnqueue cannot accept an item.
func WithDropHandler[T any](handler func(T)) Option[T] {
return func(w *Writer[T]) {
w.onDrop = handler
}
}
// New creates a Writer. Call Start before enqueueing items.
func New[T any](cfg Config, flush FlushFunc[T], opts ...Option[T]) (*Writer[T], error) {
if flush == nil {
return nil, errNilFlushFunc
}
if err := cfg.validate(); err != nil {
return nil, err
}
w := &Writer[T]{
cfg: cfg,
flush: flush,
done: make(chan struct{}),
}
for _, opt := range opts {
opt(w)
}
return w, nil
}
// Start launches the background worker. It is safe to call at most once.
func (w *Writer[T]) Start(parent context.Context) {
w.startOnce.Do(func() {
w.mu.Lock()
defer w.mu.Unlock()
w.ch = make(chan T, w.cfg.QueueSize)
w.workerCtx = context.WithoutCancel(parent)
go w.run()
})
}
// Stop closes the queue and waits until the worker drains pending items and exits.
func (w *Writer[T]) Stop(ctx context.Context) error {
w.mu.RLock()
ch := w.ch
done := w.done
w.mu.RUnlock()
if ch == nil {
return nil
}
w.stopOnce.Do(func() {
close(ch)
})
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// Running reports whether Start has been called and Stop has not completed.
func (w *Writer[T]) Running() bool {
w.mu.RLock()
defer w.mu.RUnlock()
if w.ch == nil {
return false
}
select {
case <-w.done:
return false
default:
return true
}
}
// TryEnqueue adds one item without blocking. It returns false when the writer is not
// running or the queue is full.
func (w *Writer[T]) TryEnqueue(item T) bool {
w.mu.RLock()
ch := w.ch
w.mu.RUnlock()
if ch == nil {
w.notifyDrop(item)
return false
}
select {
case ch <- item:
return true
default:
w.notifyDrop(item)
return false
}
}
// IsFull reports whether the queue has no remaining capacity.
func (w *Writer[T]) IsFull() bool {
w.mu.RLock()
defer w.mu.RUnlock()
if w.ch == nil {
return false
}
return len(w.ch) >= cap(w.ch)
}
// Len returns the current queue depth.
func (w *Writer[T]) Len() int {
w.mu.RLock()
defer w.mu.RUnlock()
if w.ch == nil {
return 0
}
return len(w.ch)
}
// Cap returns the queue capacity.
func (w *Writer[T]) Cap() int {
return w.cfg.QueueSize
}
// Stats returns a point-in-time snapshot of queue depth and failure counters.
func (w *Writer[T]) Stats() Stats {
return Stats{
Name: w.cfg.Name,
Depth: w.Len(),
Cap: w.Cap(),
Drops: w.drops.Load(),
FlushErrors: w.flushErrors.Load(),
Running: w.Running(),
}
}
func (w *Writer[T]) run() {
ticker := time.NewTicker(w.cfg.FlushInterval)
defer ticker.Stop()
batch := make([]T, 0, w.cfg.MaxBatchSize)
var batchStartedAt time.Time
flush := func() {
if len(batch) == 0 {
return
}
items := append([]T(nil), batch...)
if err := w.flush(w.workerCtx, items); err != nil {
w.flushErrors.Add(1)
if w.onFlushError != nil {
w.onFlushError(w.workerCtx, items, err)
}
}
batch = batch[:0]
batchStartedAt = time.Time{}
}
defer func() {
flush()
close(w.done)
}()
for {
select {
case item, ok := <-w.ch:
if !ok {
return
}
if len(batch) == 0 {
batchStartedAt = time.Now()
}
batch = append(batch, item)
if len(batch) >= w.cfg.MaxBatchSize {
flush()
}
case <-ticker.C:
if w.shouldFlushOnInterval(len(batch), batchStartedAt, time.Now()) {
flush()
}
}
}
}
func (w *Writer[T]) shouldFlushOnInterval(batchLen int, batchStartedAt time.Time, now time.Time) bool {
if batchLen == 0 {
return false
}
if w.cfg.MinBatchSize == 0 || batchLen >= w.cfg.MinBatchSize {
return true
}
if w.cfg.MaxFlushWait <= 0 || batchStartedAt.IsZero() {
return false
}
return !now.Before(batchStartedAt.Add(w.cfg.MaxFlushWait))
}
func (w *Writer[T]) notifyDrop(item T) {
w.drops.Add(1)
if w.onDrop == nil {
return
}
w.onDrop(item)
}
+491
View File
@@ -0,0 +1,491 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package batchwriter
import (
"context"
"errors"
"sync"
"testing"
"time"
"github.com/google/go-cmp/cmp"
)
func TestNewRejectsInvalidConfig(t *testing.T) {
t.Parallel()
_, err := New[int](Config{}, func(context.Context, []int) error { return nil })
if err == nil {
t.Fatal("New() = nil, want validation error")
}
}
func TestNewRejectsNilFlushFunc(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
_, err := New[int](cfg, nil)
if !errors.Is(err, errNilFlushFunc) {
t.Fatalf("New() error = %v, want %v", err, errNilFlushFunc)
}
}
func TestWriterFlushesOnMaxBatchSize(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
batches [][]int
)
cfg := DefaultConfig()
cfg.MaxBatchSize = 3
cfg.FlushInterval = time.Hour
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
mu.Lock()
defer mu.Unlock()
batches = append(batches, append([]int(nil), items...))
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
})
for i := range 3 {
if !writer.TryEnqueue(i + 1) {
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
}
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := len(batches) == 1
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(10 * time.Millisecond)
}
mu.Lock()
got := batches
mu.Unlock()
want := [][]int{{1, 2, 3}}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("flush batches mismatch (-want +got):\n%s", diff)
}
}
func TestWriterFlushesOnInterval(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
batch []int
)
cfg := DefaultConfig()
cfg.MaxBatchSize = 100
cfg.MinBatchSize = 0
cfg.FlushInterval = 20 * time.Millisecond
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
mu.Lock()
defer mu.Unlock()
batch = append([]int(nil), items...)
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
})
if !writer.TryEnqueue(42) {
t.Fatal("TryEnqueue() = false, want true")
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := len(batch) == 1
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(5 * time.Millisecond)
}
mu.Lock()
got := batch
mu.Unlock()
want := []int{42}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("interval flush mismatch (-want +got):\n%s", diff)
}
}
func TestWriterTryEnqueueDropsWhenFull(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
cfg.QueueSize = 1
cfg.MaxBatchSize = 10
cfg.FlushInterval = time.Hour
var dropped int
writer, err := New[int](cfg, func(context.Context, []int) error { return nil }, WithDropHandler[int](func(int) {
dropped++
}))
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = writer.Stop(stopCtx)
})
if !writer.TryEnqueue(1) {
t.Fatal("TryEnqueue(1) = false, want true")
}
if writer.TryEnqueue(2) {
t.Fatal("TryEnqueue(2) = true, want false")
}
if !writer.IsFull() {
t.Fatal("IsFull() = false, want true")
}
if dropped != 1 {
t.Fatalf("dropped = %d, want 1", dropped)
}
}
func TestWriterStopDrainsQueuedItems(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
cfg.MaxBatchSize = 10
cfg.FlushInterval = time.Hour
var flushed []int
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
flushed = append(flushed, items...)
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
for i := range 2 {
if !writer.TryEnqueue(i + 1) {
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
}
}
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
want := []int{1, 2}
if diff := cmp.Diff(want, flushed); diff != "" {
t.Fatalf("Stop() drain mismatch (-want +got):\n%s", diff)
}
if writer.Running() {
t.Fatal("Running() = true after Stop(), want false")
}
}
func TestWriterInvokesFlushErrorHandler(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
cfg.MaxBatchSize = 1
cfg.FlushInterval = time.Hour
flushErr := errors.New("flush failed")
var (
mu sync.Mutex
errCount int
gotItems []int
)
writer, err := New[int](cfg, func(context.Context, []int) error {
return flushErr
}, WithFlushErrorHandler[int](func(_ context.Context, items []int, err error) {
mu.Lock()
defer mu.Unlock()
errCount++
gotItems = append([]int(nil), items...)
if !errors.Is(err, flushErr) {
t.Errorf("flush error = %v, want %v", err, flushErr)
}
}))
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = writer.Stop(stopCtx)
})
if !writer.TryEnqueue(7) {
t.Fatal("TryEnqueue() = false, want true")
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := errCount == 1
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(5 * time.Millisecond)
}
mu.Lock()
gotCount := errCount
items := gotItems
mu.Unlock()
if gotCount != 1 {
t.Fatalf("flush error handler count = %d, want 1", gotCount)
}
if diff := cmp.Diff([]int{7}, items); diff != "" {
t.Fatalf("flush error handler items mismatch (-want +got):\n%s", diff)
}
stats := writer.Stats()
if stats.FlushErrors != 1 {
t.Fatalf("Stats().FlushErrors = %d, want 1", stats.FlushErrors)
}
}
func TestWriterSkipsIntervalFlushBelowMinBatchSize(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
batch []int
)
cfg := DefaultConfig()
cfg.MaxBatchSize = 100
cfg.MinBatchSize = 5
cfg.FlushInterval = 20 * time.Millisecond
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
mu.Lock()
defer mu.Unlock()
batch = append([]int(nil), items...)
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
})
for i := range 3 {
if !writer.TryEnqueue(i + 1) {
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
}
}
time.Sleep(100 * time.Millisecond)
mu.Lock()
got := batch
mu.Unlock()
if len(got) != 0 {
t.Fatalf("interval flush with below-min batch = %v, want no flush", got)
}
}
func TestWriterFlushesOnIntervalWhenMinBatchSizeReached(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
batch []int
)
cfg := DefaultConfig()
cfg.MaxBatchSize = 100
cfg.MinBatchSize = 3
cfg.FlushInterval = 20 * time.Millisecond
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
mu.Lock()
defer mu.Unlock()
batch = append([]int(nil), items...)
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
})
for i := range 3 {
if !writer.TryEnqueue(i + 1) {
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
}
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := len(batch) == 3
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(5 * time.Millisecond)
}
mu.Lock()
got := batch
mu.Unlock()
want := []int{1, 2, 3}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("interval flush at min batch size mismatch (-want +got):\n%s", diff)
}
}
func TestWriterForcesFlushAfterMaxFlushWait(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
batch []int
)
cfg := DefaultConfig()
cfg.MaxBatchSize = 100
cfg.MinBatchSize = 50
cfg.FlushInterval = 20 * time.Millisecond
cfg.MaxFlushWait = 80 * time.Millisecond
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
mu.Lock()
defer mu.Unlock()
batch = append([]int(nil), items...)
return nil
})
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if err := writer.Stop(stopCtx); err != nil {
t.Fatalf("Stop() error = %v", err)
}
})
if !writer.TryEnqueue(1) {
t.Fatal("TryEnqueue(1) = false, want true")
}
deadline := time.Now().Add(time.Second)
for {
mu.Lock()
ready := len(batch) == 1
mu.Unlock()
if ready || time.Now().After(deadline) {
break
}
time.Sleep(5 * time.Millisecond)
}
mu.Lock()
got := batch
mu.Unlock()
if diff := cmp.Diff([]int{1}, got); diff != "" {
t.Fatalf("max flush wait mismatch (-want +got):\n%s", diff)
}
}
func TestWriterStatsTracksDrops(t *testing.T) {
t.Parallel()
cfg := DefaultConfig()
cfg.Name = "test-drops"
cfg.QueueSize = 1
cfg.MaxBatchSize = 10
cfg.FlushInterval = time.Hour
writer, err := New[int](cfg, func(context.Context, []int) error { return nil })
if err != nil {
t.Fatalf("New() error = %v", err)
}
writer.Start(context.Background())
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = writer.Stop(stopCtx)
})
if !writer.TryEnqueue(1) {
t.Fatal("TryEnqueue(1) = false, want true")
}
if writer.TryEnqueue(2) {
t.Fatal("TryEnqueue(2) = true, want false")
}
stats := writer.Stats()
if stats.Drops != 1 {
t.Fatalf("Stats().Drops = %d, want 1", stats.Drops)
}
if stats.Cap != 1 {
t.Fatalf("Stats().Cap = %d, want 1", stats.Cap)
}
if !stats.Running {
t.Fatal("Stats().Running = false, want true")
}
}
+155
View File
@@ -0,0 +1,155 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package db 提供数据库连接与基础设施
package db
import (
"context"
"fmt"
"log"
"net/url"
"strconv"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/Rain-kl/Wavelet/pkg/config"
"go.opentelemetry.io/otel/attribute"
clickhouseDriver "gorm.io/driver/clickhouse"
"gorm.io/gorm"
"gorm.io/plugin/opentelemetry/tracing"
)
const (
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
)
var (
// ChConn ClickHouse 原生连接实例,用于批量写入
ChConn driver.Conn
chDB *gorm.DB
)
func init() {
if !config.Config.ClickHouse.Enabled {
return
}
cfg := config.Config.ClickHouse
if cfg.Database == "" {
log.Fatalf("[ClickHouse] database name is required (expected: wavelet)\n")
}
opts := buildClickHouseOptions()
var err error
ChConn, err = clickhouse.Open(opts)
if err != nil {
log.Fatalf("[ClickHouse] init connection failed: %v\n", err)
}
if err = ChConn.Ping(context.Background()); err != nil {
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
}
chDB, err = gorm.Open(clickhouseDriver.New(clickhouseDriver.Config{
DSN: buildClickHouseDSN(),
}), &gorm.Config{
SkipDefaultTransaction: true,
})
if err != nil {
log.Fatalf("[ClickHouse] init gorm connection failed: %v\n", err)
}
if err = chDB.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", cfg.Database),
attribute.String("db.system", "ClickHouse"),
),
),
); err != nil {
log.Fatalf("[ClickHouse] init trace failed: %v\n", err)
}
sqlDB, err := chDB.DB()
if err != nil {
log.Fatalf("[ClickHouse] load sql db failed: %v\n", err)
}
sqlDB.SetMaxIdleConns(cfg.MaxIdleConn)
sqlDB.SetMaxOpenConns(cfg.MaxOpenConn)
sqlDB.SetConnMaxLifetime(time.Duration(cfg.ConnMaxLifetime) * time.Second)
log.Println("[ClickHouse] connection established successfully")
}
func buildClickHouseOptions() *clickhouse.Options {
cfg := config.Config.ClickHouse
return &clickhouse.Options{
Addr: cfg.Hosts,
Auth: clickhouse.Auth{
Database: cfg.Database,
Username: cfg.Username,
Password: cfg.Password,
},
Settings: clickhouse.Settings{
"max_execution_time": clickhouseMaxExecTime,
},
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
MaxOpenConns: cfg.MaxOpenConn,
MaxIdleConns: cfg.MaxIdleConn,
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
BlockBufferSize: cfg.BlockBufferSize,
}
}
func buildClickHouseDSN() string {
cfg := config.Config.ClickHouse
chURL := &url.URL{
Scheme: "clickhouse",
Host: strings.Join(cfg.Hosts, ","),
Path: "/" + cfg.Database,
}
if cfg.Username != "" || cfg.Password != "" {
chURL.User = url.UserPassword(cfg.Username, cfg.Password)
}
query := chURL.Query()
query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout))
query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor))
query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime))
chURL.RawQuery = query.Encode()
return chURL.String()
}
// ChDB returns a context-aware GORM ClickHouse instance.
func ChDB(ctx context.Context) *gorm.DB {
if chDB == nil {
return nil
}
return chDB.WithContext(ctx)
}
// SetChDBForTest sets the package-level ClickHouse GORM instance for testing.
func SetChDBForTest(d *gorm.DB) {
chDB = d
}
// SetChConnForTest sets the package-level native ClickHouse connection for testing.
func SetChConnForTest(c driver.Conn) {
ChConn = c
}
+12
View File
@@ -0,0 +1,12 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
const (
errRedisHashSetFailed = "failed to set redis hash: %w"
errRedisHashDeleteFailed = "failed to delete redis hash field: %w"
errUnmarshalDataFailed = "failed to unmarshal data: %w"
errMarshalDataFailed = "failed to marshal data: %w"
errRedisKeySetFailed = "failed to set redis key: %w"
)
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package idgen 提供分布式 ID 生成器
package idgen
import (
"fmt"
"log"
"github.com/Rain-kl/Wavelet/pkg/config"
"github.com/bwmarrin/snowflake"
)
// 2025-12-01 00:00:00 UTC 的毫秒时间戳
const epoch int64 = 1764547200000
const maxNegativeIDRetries = 3
var node *snowflake.Node
func init() {
snowflake.Epoch = epoch
nodeID := config.Config.App.NodeID
var err error
node, err = snowflake.NewNode(nodeID)
if err != nil {
log.Fatalf("[Snowflake] init failed: %v\n", err)
}
log.Printf("[Snowflake] initialized with node ID: %d, epoch: 2025-12-01\n", nodeID)
}
// NextUint64ID 生成下一个分布式唯一 ID。
// 理论上不应出现负值;若出现则最多重试 maxNegativeIDRetries 次,仍失败则 panic。
func NextUint64ID() uint64 {
for attempt := 1; attempt <= maxNegativeIDRetries; attempt++ {
id := node.Generate().Int64()
if id >= 0 {
return uint64(id)
}
log.Printf("[Snowflake] generated negative ID: %d (attempt %d/%d)", id, attempt, maxNegativeIDRetries)
}
panic(fmt.Sprintf("[Snowflake] generated negative ID after %d attempts", maxNegativeIDRetries))
}
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package idgen
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNextUint64ID(t *testing.T) {
id := NextUint64ID()
assert.NotZero(t, id)
}
+99
View File
@@ -0,0 +1,99 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"strconv"
"time"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
const (
defaultLogRetentionDays = 30
partitionLeadMonths = 2
userAccessLogTable = "w_user_access_logs"
)
// CleanupSummary 汇总本次清理结果。
type CleanupSummary struct {
ActiveDatabase string `json:"active_database"`
RetentionDays int `json:"retention_days"`
Deleted int64 `json:"deleted"`
}
// CleanupExpired 按当前日志库保留天数删除过期用户访问日志,并预建 PG 分区。
func CleanupExpired(ctx context.Context) (CleanupSummary, error) {
active, err := ActiveDatabase(ctx)
if err != nil {
return CleanupSummary{}, err
}
days := retentionDaysForDatabase(ctx, active)
summary := CleanupSummary{ActiveDatabase: active, RetentionDays: days}
store, err := Active(ctx)
if err != nil {
return summary, err
}
now := time.Now().UTC()
if err := store.UserAccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
logger.WarnF(ctx, "logstore: ensure partitions during cleanup failed: %v", err)
}
cutoff := now.AddDate(0, 0, -days)
// 先 DROP 完全过期的整月分区,再对边界月逐行 DeleteBefore。
if err := store.UserAccessLogs.DropExpiredPartitions(ctx, cutoff); err != nil {
return summary, fmt.Errorf("drop expired partitions: %w", err)
}
deleted, err := store.UserAccessLogs.DeleteBefore(ctx, cutoff)
if err != nil {
return summary, fmt.Errorf("delete expired user access logs: %w", err)
}
summary.Deleted = deleted
if err := store.UserAccessLogs.DropEmptyPartitions(ctx, now); err != nil {
logger.WarnF(ctx, "drop empty log partitions failed: %v", err)
}
return summary, nil
}
func retentionDaysForDatabase(ctx context.Context, dbName string) int {
key := "log_retention_days_postgres"
switch dbName {
case dbNameSQLite:
key = "log_retention_days_sqlite"
case dbNameClickHouse:
key = "log_retention_days_clickhouse"
}
v, err := getConfig(ctx, key)
if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "读取日志保留天数配置失败(key=%s),回退默认 %d 天: %v", key, defaultLogRetentionDays, err)
}
return defaultLogRetentionDays
}
days, perr := strconv.Atoi(v)
if perr != nil || days <= 0 {
logger.ErrorF(ctx, "日志保留天数配置非法(key=%s, value=%q),回退默认 %d 天", key, v, defaultLogRetentionDays)
return defaultLogRetentionDays
}
return days
}
func partitionStatementsRange(from, to time.Time) []string {
var out []string
start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC)
end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 1, 0)
for ; start.Before(end); start = start.AddDate(0, 1, 0) {
monthEnd := start.AddDate(0, 1, 0)
suffix := start.Format("200601")
fromDay := start.Format("2006-01-02")
toDay := monthEnd.Format("2006-01-02")
out = append(out, fmt.Sprintf(
"CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')",
userAccessLogTable, suffix, userAccessLogTable, fromDay, toDay))
}
return out
}
+153
View File
@@ -0,0 +1,153 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"fmt"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
db "github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/Rain-kl/Wavelet/pkg/persistence/analytics"
)
type clickhouseUserAccessLogStore struct {
skipFreeze bool
}
func newClickHouseUserAccessLogStore() *clickhouseUserAccessLogStore {
return &clickhouseUserAccessLogStore{}
}
var (
_ UserAccessLogStore = (*clickhouseUserAccessLogStore)(nil)
_ StatusStore = (*clickhouseUserAccessLogStore)(nil)
)
func (s *clickhouseUserAccessLogStore) ActiveDatabase(_ context.Context) (string, error) {
return dbNameClickHouse, nil
}
func (s *clickhouseUserAccessLogStore) ensureWritable(ctx context.Context) error {
if !s.skipFreeze && Migrating(ctx) {
return ErrMigrating
}
return nil
}
func (s *clickhouseUserAccessLogStore) BatchInsert(ctx context.Context, logs []analytics.UserAccessLog) error {
if len(logs) == 0 {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analytics.BatchInsert(ctx, logs)
}
func (s *clickhouseUserAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analytics.DeleteAllUserAccessLogs(ctx)
}
func (s *clickhouseUserAccessLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analytics.DeleteUserAccessLogsBefore(ctx, cutoff)
}
func (s *clickhouseUserAccessLogStore) Count(ctx context.Context, filter analytics.AccessLogFilter) (uint64, error) {
return analytics.CountAccessLogs(ctx, filter)
}
func (s *clickhouseUserAccessLogStore) List(ctx context.Context, filter analytics.AccessLogFilter, page, pageSize int) ([]analytics.UserAccessLog, uint64, error) {
return analytics.ListAccessLogs(ctx, filter, page, pageSize)
}
func (s *clickhouseUserAccessLogStore) GetDailyTrend(ctx context.Context, days int) ([]analytics.DailyTrend, error) {
return analytics.GetDailyTrend(ctx, days)
}
func (s *clickhouseUserAccessLogStore) GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analytics.BrowserShare, error) {
return analytics.GetBrowserDistribution(ctx, startTime)
}
func (s *clickhouseUserAccessLogStore) GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analytics.TopUser, error) {
return analytics.GetTopActiveUsers(ctx, startTime, limit)
}
func (s *clickhouseUserAccessLogStore) EnsurePartitions(_ context.Context, _, _ time.Time) error {
return nil
}
func (s *clickhouseUserAccessLogStore) DropEmptyPartitions(_ context.Context, _ time.Time) error {
return nil
}
func (s *clickhouseUserAccessLogStore) DropExpiredPartitions(_ context.Context, _ time.Time) error {
return nil
}
func (s *clickhouseUserAccessLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
if db.ChConn == nil {
return time.Time{}, time.Time{}, fmt.Errorf("clickhouse connection is not initialized")
}
table := analytics.UserAccessLog{}.TableName()
var minTime, maxTime *time.Time
if err := db.ChConn.QueryRow(ctx, "SELECT min(created_at), max(created_at) FROM "+table).Scan(&minTime, &maxTime); err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("query migration range %s: %w", table, err)
}
if minTime == nil || maxTime == nil {
return time.Time{}, time.Time{}, nil
}
return minTime.UTC(), maxTime.UTC(), nil
}
func (s *clickhouseUserAccessLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analytics.UserAccessLog, error) {
if db.ChConn == nil {
return nil, fmt.Errorf("clickhouse connection is not initialized")
}
if limit <= 0 {
limit = migrationPageSize
}
table := analytics.UserAccessLog{}.TableName()
columns := analytics.UserAccessLog{}.InsertColumns()
rows, err := db.ChConn.Query(ctx, fmt.Sprintf(
"SELECT %s FROM %s WHERE id > ? ORDER BY id ASC LIMIT ?",
columns, table,
), afterID, limit)
if err != nil {
return nil, fmt.Errorf("list user access logs for migration: %w", err)
}
defer func() { _ = rows.Close() }()
return scanUserAccessLogs(rows)
}
func scanUserAccessLogs(rows driver.Rows) ([]analytics.UserAccessLog, error) {
var result []analytics.UserAccessLog
for rows.Next() {
var item analytics.UserAccessLog
if err := rows.Scan(
&item.ID,
&item.UserID,
&item.Path,
&item.Method,
&item.IP,
&item.UserAgent,
&item.Headers,
&item.Status,
&item.Latency,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan user access log row: %w", err)
}
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
+327
View File
@@ -0,0 +1,327 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence/analytics"
"github.com/Rain-kl/Wavelet/pkg/persistence/idgen"
"gorm.io/gorm"
)
const (
insertBatchSize = 500
migrationPageSize = 100
defaultPageSize = 20
defaultTopN = 10
topUserAgents = 100
dayDuration = 24 * time.Hour
)
type gormLogStore struct {
db *gorm.DB
skipFreeze bool
}
func newGormStore(db *gorm.DB) *gormLogStore { return &gormLogStore{db: db} }
type userAccessLogGormStore struct {
*gormLogStore
}
func newUserAccessLogGormStore(db *gorm.DB) *userAccessLogGormStore {
return &userAccessLogGormStore{gormLogStore: newGormStore(db)}
}
var (
_ UserAccessLogStore = (*userAccessLogGormStore)(nil)
_ StatusStore = (*userAccessLogGormStore)(nil)
)
func (s *gormLogStore) ActiveDatabase(_ context.Context) (string, error) {
if isPostgresDialect(s.db) {
return dbNamePostgres, nil
}
return dbNameSQLite, nil
}
func (s *gormLogStore) ensureWritable(ctx context.Context) error {
if !s.skipFreeze && Migrating(ctx) {
return ErrMigrating
}
return nil
}
func (s *userAccessLogGormStore) BatchInsert(ctx context.Context, logs []analytics.UserAccessLog) error {
if len(logs) == 0 {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
for i := range logs {
if logs[i].ID == 0 {
logs[i].ID = idgen.NextUint64ID()
}
}
return s.db.WithContext(ctx).CreateInBatches(logs, insertBatchSize).Error
}
func (s *userAccessLogGormStore) DeleteAll(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
res := s.db.WithContext(ctx).Where("1 = 1").Delete(&analytics.UserAccessLog{})
return res.RowsAffected, res.Error
}
func (s *userAccessLogGormStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
res := s.db.WithContext(ctx).Where("created_at < ?", cutoff).Delete(&analytics.UserAccessLog{})
if res.Error != nil && isMissingRelation(res.Error) {
return 0, nil
}
return res.RowsAffected, res.Error
}
func (s *userAccessLogGormStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analytics.UserAccessLog, error) {
var rows []analytics.UserAccessLog
q := s.db.WithContext(ctx).Model(&analytics.UserAccessLog{}).
Where("id > ?", afterID).
Order("id ASC").
Limit(limitOr(limit, migrationPageSize))
if err := q.Find(&rows).Error; err != nil {
return nil, err
}
return rows, nil
}
func (s *userAccessLogGormStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
return gormMigrationRange(ctx, s.db, "created_at", analytics.UserAccessLog{}, func(v *analytics.UserAccessLog) time.Time {
return v.CreatedAt
})
}
func (s *userAccessLogGormStore) Count(ctx context.Context, filter analytics.AccessLogFilter) (uint64, error) {
where, args, ok := buildUserAccessLogWhere(filter)
if !ok {
return 0, nil
}
var total int64
if err := s.db.WithContext(ctx).Model(&analytics.UserAccessLog{}).Where(where, args...).Count(&total).Error; err != nil {
return 0, err
}
return countToUint64(total), nil
}
func (s *userAccessLogGormStore) List(ctx context.Context, filter analytics.AccessLogFilter, page, pageSize int) ([]analytics.UserAccessLog, uint64, error) {
where, args, ok := buildUserAccessLogWhere(filter)
if !ok {
return []analytics.UserAccessLog{}, 0, nil
}
var total int64
if err := s.db.WithContext(ctx).Model(&analytics.UserAccessLog{}).Where(where, args...).Count(&total).Error; err != nil {
return nil, 0, err
}
if total == 0 {
return []analytics.UserAccessLog{}, 0, nil
}
var rows []analytics.UserAccessLog
q := s.db.WithContext(ctx).Where(where, args...).Order("created_at DESC, id DESC")
if err := q.Limit(limitOr(pageSize, defaultPageSize)).Offset(offsetOf(page, pageSize)).Find(&rows).Error; err != nil {
return nil, 0, err
}
return rows, countToUint64(total), nil
}
func buildUserAccessLogWhere(filter analytics.AccessLogFilter) (string, []any, bool) {
if filter.UserIDs != nil && len(filter.UserIDs) == 0 {
return "", nil, false
}
var parts []string
var args []any
if filter.UserIDs != nil {
parts = append(parts, "user_id IN ?")
args = append(args, filter.UserIDs)
}
if trimmed := strings.TrimSpace(filter.Path); trimmed != "" {
parts = append(parts, "path LIKE ?")
args = append(args, "%"+trimmed+"%")
}
if filter.StartTime != nil {
parts = append(parts, "created_at >= ?")
args = append(args, *filter.StartTime)
}
if filter.EndTime != nil {
parts = append(parts, "created_at <= ?")
args = append(args, *filter.EndTime)
}
if len(parts) == 0 {
return "1 = 1", args, true
}
return strings.Join(parts, " AND "), args, true
}
func (s *userAccessLogGormStore) GetDailyTrend(ctx context.Context, days int) ([]analytics.DailyTrend, error) {
if days <= 0 {
days = 7
}
start := time.Now().AddDate(0, 0, -(days - 1)).Truncate(dayDuration)
type row struct {
Date string
Cnt uint64
}
var rows []row
err := s.db.WithContext(ctx).Model(&analytics.UserAccessLog{}).
Select(dailyTrendDateSQL(s.db)+" AS date, COUNT(*) AS cnt").
Where("created_at >= ?", start).
Group("date").Order("date ASC").Scan(&rows).Error
if err != nil {
return nil, err
}
counts := make(map[string]uint64, len(rows))
for _, r := range rows {
counts[r.Date] = r.Cnt
}
out := make([]analytics.DailyTrend, 0, days)
for i := 0; i < days; i++ {
d := start.AddDate(0, 0, i).Format("2006-01-02")
out = append(out, analytics.DailyTrend{Date: d, Count: counts[d]})
}
return out, nil
}
func (s *userAccessLogGormStore) GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analytics.BrowserShare, error) {
type row struct {
UserAgent string
Cnt uint64
}
var rows []row
err := s.db.WithContext(ctx).Model(&analytics.UserAccessLog{}).
Select("user_agent, COUNT(*) AS cnt").
Where("created_at >= ?", startTime).
Group("user_agent").Order("cnt DESC").Limit(topUserAgents).Scan(&rows).Error
if err != nil {
return nil, err
}
counts := make(map[string]uint64)
for _, r := range rows {
counts[analytics.ParseBrowserName(r.UserAgent)] += r.Cnt
}
out := make([]analytics.BrowserShare, 0, len(counts))
for label, count := range counts {
out = append(out, analytics.BrowserShare{Browser: label, Count: count})
}
sort.Slice(out, func(i, j int) bool { return out[i].Count > out[j].Count })
return out, nil
}
func (s *userAccessLogGormStore) GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analytics.TopUser, error) {
type row struct {
UserID uint64
Cnt uint64
}
var rows []row
err := s.db.WithContext(ctx).Model(&analytics.UserAccessLog{}).
Select("user_id, COUNT(*) AS cnt").
Where("user_id <> 0 AND created_at >= ?", startTime).
Group("user_id").Order("cnt DESC").Limit(limitOr(limit, defaultTopN)).Scan(&rows).Error
if err != nil {
return nil, err
}
out := make([]analytics.TopUser, len(rows))
for i, r := range rows {
out[i] = analytics.TopUser{UserID: r.UserID, Count: r.Cnt}
}
return out, nil
}
func (s *userAccessLogGormStore) EnsurePartitions(ctx context.Context, from, to time.Time) error {
if !isPostgresDialect(s.db) {
return nil
}
for _, sql := range partitionStatementsRange(from, to) {
if err := s.db.WithContext(ctx).Exec(sql).Error; err != nil {
return fmt.Errorf("ensure partition: %w", err)
}
}
return nil
}
func gormMigrationRange[T any](
ctx context.Context,
gdb *gorm.DB,
column string,
model T,
timeOf func(*T) time.Time,
) (time.Time, time.Time, error) {
var first, last T
found := false
for _, order := range []string{"ASC", "DESC"} {
out := &first
if order == "DESC" {
out = &last
}
res := gdb.WithContext(ctx).Model(model).Order(column + " " + order).Limit(1).Take(out)
if res.Error != nil && !errors.Is(res.Error, gorm.ErrRecordNotFound) {
return time.Time{}, time.Time{}, fmt.Errorf("query migration range %s: %w", column, res.Error)
}
if res.Error == nil {
found = true
}
}
if !found {
return time.Time{}, time.Time{}, nil
}
return timeOf(&first).UTC(), timeOf(&last).UTC(), nil
}
func limitOr(v, def int) int {
if v <= 0 {
return def
}
return v
}
func offsetOf(page, pageSize int) int {
if page < 1 {
page = 1
}
return (page - 1) * limitOr(pageSize, defaultPageSize)
}
func countToUint64(v int64) uint64 {
if v < 0 {
return 0
}
return uint64(v)
}
func isPostgresDialect(db *gorm.DB) bool {
return db != nil && db.Dialector != nil && db.Name() == "postgres"
}
func dailyTrendDateSQL(db *gorm.DB) string {
if isPostgresDialect(db) {
return "to_char(created_at, 'YYYY-MM-DD')"
}
return "strftime('%Y-%m-%d', created_at)"
}
func isMissingRelation(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "no such table") || strings.Contains(msg, "does not exist")
}
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence/analytics"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func newTestUserAccessStore(t *testing.T) *userAccessLogGormStore {
t.Helper()
gdb, err := gorm.Open(sqlite.Open("file:logstore-"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, gdb.AutoMigrate(&analytics.UserAccessLog{}))
return newUserAccessLogGormStore(gdb)
}
func TestGormUserAccessLogCountList(t *testing.T) {
ua := newTestUserAccessStore(t)
ctx := context.Background()
now := time.Now().UTC().Truncate(time.Second)
require.NoError(t, ua.BatchInsert(ctx, []analytics.UserAccessLog{
{UserID: 10, Path: "/api/v1/users", Method: "GET", Status: 200, CreatedAt: now},
{UserID: 20, Path: "/api/v1/admin", Method: "GET", Status: 200, CreatedAt: now},
{UserID: 10, Path: "/api/v1/other", Method: "POST", Status: 201, CreatedAt: now},
}))
count, err := ua.Count(ctx, analytics.AccessLogFilter{UserIDs: []uint64{10}, Path: "users"})
require.NoError(t, err)
require.Equal(t, uint64(1), count)
rows, total, err := ua.List(ctx, analytics.AccessLogFilter{UserIDs: []uint64{10}, Path: "users"}, 1, 10)
require.NoError(t, err)
require.Equal(t, uint64(1), total)
require.Len(t, rows, 1)
require.Equal(t, "/api/v1/users", rows[0].Path)
require.NotZero(t, rows[0].ID)
}
func TestGormUserAccessLogFreeze(t *testing.T) {
ua := newTestUserAccessStore(t)
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logMigrationKey {
return "migrating", nil
}
return "", nil
})
t.Cleanup(ResetForTest)
err := ua.BatchInsert(context.Background(), []analytics.UserAccessLog{{UserID: 1, CreatedAt: time.Now()}})
require.ErrorIs(t, err, ErrMigrating)
}
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"os/exec"
"strings"
"testing"
)
// apps 禁止直连 analytics 做日志读写;查询过滤器请用 logstore.AccessLogFilter。
var forbiddenImports = []string{
"github.com/Rain-kl/Wavelet/pkg/persistence/analytics",
}
// logstore 的 CH 实现按设计委托 analyticsrepo。
var allowedAnalyticsDelegation = map[string]bool{
"github.com/Rain-kl/Wavelet/pkg/persistence/logstore": true,
}
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
t.Chdir("../../..")
out, err := exec.Command("go", "list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./plugins/domain/...").Output()
if err != nil {
t.Fatalf("go list: %v", err)
}
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
pkg := fields[0]
if !strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/plugins/domain") {
continue
}
for _, imp := range fields[1:] {
for _, forbidden := range forbiddenImports {
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
}
}
}
}
}
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package logstore abstracts user access-log storage across ClickHouse, PostgreSQL and SQLite.
package logstore
import (
"context"
"errors"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence/analytics"
)
// ErrMigrating 表示日志数据库正在迁移,当前禁止写入。
var ErrMigrating = errors.New("log database is migrating, writes are disabled")
// UserAccessLogStore 用户访问日志(w_user_access_logs)。
type UserAccessLogStore interface {
BatchInsert(ctx context.Context, logs []analytics.UserAccessLog) error
DeleteAll(ctx context.Context) (int64, error)
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
Count(ctx context.Context, filter analytics.AccessLogFilter) (uint64, error)
List(ctx context.Context, filter analytics.AccessLogFilter, page, pageSize int) ([]analytics.UserAccessLog, uint64, error)
GetDailyTrend(ctx context.Context, days int) ([]analytics.DailyTrend, error)
GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analytics.BrowserShare, error)
GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analytics.TopUser, error)
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analytics.UserAccessLog, error)
MigrationRange(ctx context.Context) (from, to time.Time, err error)
EnsurePartitions(ctx context.Context, from, to time.Time) error
// DropEmptyPartitions 幂等清理 PG 空分区表:删除 before 月份之前、且无任何数据的按月分区;
// CH/SQLite 为 no-op。
DropEmptyPartitions(ctx context.Context, before time.Time) error
// DropExpiredPartitions 直接删除完全过期的 PG 整月分区(候选为月份早于 cutoff 月的分区,
// 删除前校验分区内无保留期内数据,避免时区偏移下误删;迁移冻结期间拒绝执行);CH/SQLite 为 no-op。
DropExpiredPartitions(ctx context.Context, cutoff time.Time) error
}
// UserAccessLog 用户访问日志实体类型别名
type UserAccessLog = analytics.UserAccessLog
// AccessLogFilter 访问日志查询过滤条件类型别名
type AccessLogFilter = analytics.AccessLogFilter
// DailyTrend 每日趋势数据类型别名
type DailyTrend = analytics.DailyTrend
// BrowserShare 浏览器分布数据类型别名
type BrowserShare = analytics.BrowserShare
// TopUser Top 用户活跃统计类型别名
type TopUser = analytics.TopUser
// StatusStore 日志库状态。
type StatusStore interface {
ActiveDatabase(ctx context.Context) (string, error)
}
// Store 当前生效日志库。
type Store struct {
UserAccessLogs UserAccessLogStore
Status StatusStore
}
@@ -0,0 +1,115 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"fmt"
"strings"
"time"
"gorm.io/gorm"
)
// listPartitionNames 列出 table 在当前 schema 下的全部直接分区表名(pg_inherits)。
func listPartitionNames(ctx context.Context, gdb *gorm.DB, table string) ([]string, error) {
var names []string
if err := gdb.WithContext(ctx).Raw(`
SELECT c.relname
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_class p ON p.oid = i.inhparent
JOIN pg_namespace n ON n.oid = p.relnamespace AND n.nspname = current_schema()
WHERE p.relname = ?`, table).Scan(&names).Error; err != nil {
return nil, fmt.Errorf("list partitions of %s: %w", table, err)
}
return names, nil
}
// partitionNameMonth 解析按月分区表名 <table>_YYYYMM 的所属月份;命名不匹配返回 (零值, false)。
func partitionNameMonth(table, name string) (time.Time, bool) {
suffix, ok := strings.CutPrefix(name, table+"_")
if !ok || len(suffix) != 6 {
return time.Time{}, false
}
m, err := time.Parse("200601", suffix)
if err != nil {
return time.Time{}, false
}
return m, true
}
// dropEligiblePartitionNames 返回 before 月份之前、命名合法的分区表名(是否为空由调用方校验)。
func dropEligiblePartitionNames(table string, names []string, before time.Time) []string {
beforeMonth := time.Date(before.Year(), before.Month(), 1, 0, 0, 0, 0, time.UTC)
out := make([]string, 0, len(names))
for _, name := range names {
month, ok := partitionNameMonth(table, name)
if !ok || !month.Before(beforeMonth) {
continue
}
out = append(out, name)
}
return out
}
// DropEmptyPartitions 幂等清理 PG 空分区表:仅删除 before 月份之前、且无任何数据的分区。
// 非 PG 方言为 no-op。
func (s *gormLogStore) DropEmptyPartitions(ctx context.Context, before time.Time) error {
if !isPostgresDialect(s.db) {
return nil
}
names, err := listPartitionNames(ctx, s.db, userAccessLogTable)
if err != nil {
return err
}
for _, name := range dropEligiblePartitionNames(userAccessLogTable, names, before) {
var one int
if err := s.db.WithContext(ctx).Raw("SELECT 1 FROM " + name + " LIMIT 1").Scan(&one).Error; err != nil {
return fmt.Errorf("check partition %s empty: %w", name, err)
}
if one == 1 {
continue
}
if err := s.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + name).Error; err != nil {
return fmt.Errorf("drop empty partition %s: %w", name, err)
}
}
return nil
}
// DropExpiredPartitions 直接删除完全过期的 PG 整月分区(避免 retention 清理逐行 DELETE)。
// 候选 = 月份早于 cutoff 月(按 cutoff 的 UTC 时刻取月);删除前校验分区内不存在 created_at >= cutoff 的行。
// 迁移冻结期间返回 ErrMigrating。CH/SQLite 为 no-op。
func (s *gormLogStore) DropExpiredPartitions(ctx context.Context, cutoff time.Time) error {
if !isPostgresDialect(s.db) {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
names, err := listPartitionNames(ctx, s.db, userAccessLogTable)
if err != nil {
return err
}
cu := cutoff.UTC()
cutoffMonth := time.Date(cu.Year(), cu.Month(), 1, 0, 0, 0, 0, time.UTC)
for _, name := range names {
month, ok := partitionNameMonth(userAccessLogTable, name)
if !ok || !month.Before(cutoffMonth) {
continue
}
var hasRetained int
if err := s.db.WithContext(ctx).Raw("SELECT 1 FROM "+name+" WHERE created_at >= ? LIMIT 1", cu).Scan(&hasRetained).Error; err != nil {
return fmt.Errorf("check partition %s retained rows: %w", name, err)
}
if hasRetained == 1 {
continue
}
if err := s.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + name).Error; err != nil {
return fmt.Errorf("drop expired partition %s: %w", name, err)
}
}
return nil
}
@@ -0,0 +1,70 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/persistence/analytics"
"github.com/stretchr/testify/require"
)
func TestPartitionNameMonth(t *testing.T) {
cases := []struct {
table string
name string
want string
}{
{"w_user_access_logs", "w_user_access_logs_202612", "2026-12"},
{"w_user_access_logs", "w_user_access_logs_202608", "2026-08"},
{"w_user_access_logs", "of_node_access_logs_202608", ""},
{"w_user_access_logs", "w_user_access_logs_20268", ""},
{"w_user_access_logs", "w_user_access_logs_202613", ""},
{"w_user_access_logs", "w_user_access_logs_default", ""},
}
for _, c := range cases {
got, ok := partitionNameMonth(c.table, c.name)
if c.want == "" {
if ok {
t.Fatalf("partitionNameMonth(%q, %q) ok = true, want false", c.table, c.name)
}
continue
}
if !ok || got.Format("2006-01") != c.want {
t.Fatalf("partitionNameMonth(%q, %q) = %v, want %s", c.table, c.name, got, c.want)
}
}
}
func TestDropEligiblePartitionNames(t *testing.T) {
before := time.Date(2026, 10, 15, 0, 0, 0, 0, time.UTC)
names := []string{
"w_user_access_logs_202608",
"w_user_access_logs_202609",
"w_user_access_logs_202610",
"w_user_access_logs_202611",
"w_user_access_logs_default",
}
got := dropEligiblePartitionNames(userAccessLogTable, names, before)
want := []string{"w_user_access_logs_202608", "w_user_access_logs_202609"}
require.Equal(t, want, got)
first := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
require.Empty(t, dropEligiblePartitionNames(userAccessLogTable, []string{"w_user_access_logs_202610"}, first))
}
func TestDropPartitionHelpersSQLiteNoop(t *testing.T) {
ua := newTestUserAccessStore(t)
ctx := context.Background()
require.NoError(t, ua.BatchInsert(ctx, []analytics.UserAccessLog{
{UserID: 1, Path: "/x", CreatedAt: time.Now().UTC()},
}))
require.NoError(t, ua.DropExpiredPartitions(ctx, time.Now().AddDate(0, 0, -90)))
require.NoError(t, ua.DropEmptyPartitions(ctx, time.Now()))
count, err := ua.Count(ctx, AccessLogFilter{})
require.NoError(t, err)
require.Equal(t, uint64(1), count)
}
+189
View File
@@ -0,0 +1,189 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/Rain-kl/Wavelet/pkg/config"
"github.com/Rain-kl/Wavelet/pkg/logger"
db "github.com/Rain-kl/Wavelet/pkg/persistence"
)
const (
logDatabaseKey = "log_database"
logMigrationKey = "log_db_migration"
)
const (
dbNamePostgres = "postgres"
dbNameSQLite = "sqlite"
dbNameClickHouse = "clickhouse"
)
var errConfigReaderNotWired = errors.New("logstore: config reader not wired")
// ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。
type ConfigReader func(ctx context.Context, key string) (string, error)
const resolveCacheTTL = 1 * time.Second
var (
configReader ConfigReader
storeMu sync.RWMutex
active *Store
activeDB string
lastResolveDB string
lastResolveTime time.Time
)
// SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。
func SetConfigReader(fn ConfigReader) { configReader = fn }
func getConfig(ctx context.Context, key string) (string, error) {
if configReader == nil {
return "", errConfigReaderNotWired
}
return configReader(ctx, key)
}
// Active 返回当前生效的日志库 Store。
func Active(ctx context.Context) (*Store, error) {
current, err := resolveDatabase(ctx)
if err != nil {
return nil, err
}
storeMu.RLock()
if active != nil && activeDB == current {
s := active
storeMu.RUnlock()
return s, nil
}
storeMu.RUnlock()
storeMu.Lock()
defer storeMu.Unlock()
if active != nil && activeDB == current {
return active, nil
}
s, err := buildStore(ctx, current, false)
if err != nil {
return nil, err
}
active = s
activeDB = current
return s, nil
}
// Build 直接按目标构造 store(不经 Active 缓存)。
func Build(ctx context.Context, database string) (*Store, error) {
return buildStore(ctx, database, false)
}
// BuildForMigration 构造迁移目标 store,跳过冻结检查。
func BuildForMigration(ctx context.Context, database string) (*Store, error) {
return buildStore(ctx, database, true)
}
func buildStore(ctx context.Context, database string, skipFreeze bool) (*Store, error) {
switch database {
case dbNameClickHouse:
ual := newClickHouseUserAccessLogStore()
ual.skipFreeze = skipFreeze
return &Store{UserAccessLogs: ual, Status: ual}, nil
case dbNamePostgres, dbNameSQLite:
gdb := db.DB(ctx)
ual := newUserAccessLogGormStore(gdb)
ual.skipFreeze = skipFreeze
return &Store{UserAccessLogs: ual, Status: ual}, nil
default:
return nil, fmt.Errorf("unsupported log database: %s", database)
}
}
// Migrating 返回日志库是否处于迁移冻结状态。
func Migrating(ctx context.Context) bool {
v, err := getConfig(ctx, logMigrationKey)
if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "read log migration config failed: %v", err)
}
return false
}
return v == "migrating"
}
// Init 预热激活 store,并兜底预建当前月及未来分区。
func Init(ctx context.Context) {
s, err := Active(ctx)
if err != nil {
return
}
now := time.Now().UTC()
if err := s.UserAccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
logger.WarnF(ctx, "logstore: ensure startup partitions failed: %v", err)
}
}
// InvalidateCache 清空日志库解析缓存。
func InvalidateCache() {
storeMu.Lock()
defer storeMu.Unlock()
lastResolveTime = time.Time{}
lastResolveDB = ""
}
// ResetForTest 清空缓存的激活 store 与 config reader。
func ResetForTest() {
storeMu.Lock()
active = nil
activeDB = ""
lastResolveDB = ""
lastResolveTime = time.Time{}
storeMu.Unlock()
configReader = nil
}
// ActiveDatabase 返回当前日志主库名。
func ActiveDatabase(ctx context.Context) (string, error) {
return resolveDatabase(ctx)
}
func resolveDatabase(ctx context.Context) (string, error) {
storeMu.RLock()
if active != nil && time.Since(lastResolveTime) < resolveCacheTTL {
name := lastResolveDB
storeMu.RUnlock()
return name, nil
}
storeMu.RUnlock()
v, err := getConfig(ctx, logDatabaseKey)
if err != nil && !errors.Is(err, errConfigReaderNotWired) {
return "", err
}
resolved := v
if resolved == "" {
resolved = dbNameSQLite
if config.Config.Database.Enabled {
resolved = dbNamePostgres
}
if config.Config.ClickHouse.Enabled {
resolved = dbNameClickHouse
}
}
storeMu.Lock()
lastResolveDB = resolved
lastResolveTime = time.Now()
storeMu.Unlock()
return resolved, nil
}
+107
View File
@@ -0,0 +1,107 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"context"
"database/sql"
"embed"
"io/fs"
"log"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/Rain-kl/Wavelet/pkg/config"
"github.com/pressly/goose/v3"
)
const (
clickhouseMigrationDir = "goose/clickhouse"
clickhouseGooseVersionTable = "goose_clickhouse_version"
clickhouseMaxExecTime = 60
clickhouseReadTimeoutFactor = 2
)
// clickhouseMigrationFS contains SQL migrations under goose/clickhouse.
//
//go:embed goose/clickhouse/*.sql
var clickhouseMigrationFS embed.FS
// MigrateClickHouse runs goose migrations against ClickHouse when enabled.
func MigrateClickHouse() Report {
if !config.Config.ClickHouse.Enabled {
return Report{Backend: "ClickHouse"}
}
cfg := config.Config.ClickHouse
sqlDB := clickhouse.OpenDB(&clickhouse.Options{
Addr: cfg.Hosts,
Auth: clickhouse.Auth{
Database: cfg.Database,
Username: cfg.Username,
Password: cfg.Password,
},
Settings: clickhouse.Settings{
"max_execution_time": clickhouseMaxExecTime,
},
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
MaxOpenConns: cfg.MaxOpenConn,
MaxIdleConns: cfg.MaxIdleConn,
ConnMaxLifetime: time.Duration(cfg.ConnMaxLifetime) * time.Second,
ReadTimeout: time.Duration(cfg.DialTimeout*clickhouseReadTimeoutFactor) * time.Second,
BlockBufferSize: cfg.BlockBufferSize,
})
subFS, err := fs.Sub(clickhouseMigrationFS, "goose/clickhouse")
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] get sub fs failed: %v\n", err)
}
provider, err := goose.NewProvider(
"clickhouse",
sqlDB,
subFS,
goose.WithTableName(clickhouseGooseVersionTable),
goose.WithDisableGlobalRegistry(true),
)
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] create goose provider failed: %v\n", err)
}
previousVersion, err := provider.GetDBVersion(context.Background())
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] get goose version failed: %v\n", err)
}
if _, err := provider.Up(context.Background()); err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] goose migrate failed: %v\n", err)
}
currentVersion, err := provider.GetDBVersion(context.Background())
if err != nil {
closeClickHouseDB(sqlDB)
log.Fatalf("[ClickHouse] get migrated goose version failed: %v\n", err)
}
closeClickHouseDB(sqlDB)
log.Println("[ClickHouse] goose migrate success")
return Report{
Backend: "ClickHouse",
Enabled: true,
Version: currentVersion,
Applied: currentVersion != previousVersion,
}
}
func closeClickHouseDB(sqlDB *sql.DB) {
if err := sqlDB.Close(); err != nil {
log.Printf("[ClickHouse] close sql db failed: %v\n", err)
}
}
@@ -0,0 +1,51 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"testing"
"github.com/Rain-kl/Wavelet/pkg/config"
"github.com/pressly/goose/v3"
)
func TestClickHouseMigrationFilesEmbedded(t *testing.T) {
entries, err := clickhouseMigrationFS.ReadDir(clickhouseMigrationDir)
if err != nil {
t.Fatalf("ReadDir(%q) error = %v", clickhouseMigrationDir, err)
}
if len(entries) == 0 {
t.Fatal("expected embedded ClickHouse migrations, got none")
}
found := false
for _, entry := range entries {
if entry.IsDir() {
continue
}
if entry.Name() == "202606190001_create_user_access_logs.sql" {
found = true
break
}
}
if !found {
t.Fatal("expected 202606190001_create_user_access_logs.sql in embedded migrations")
}
}
func TestClickHouseGooseDialect(t *testing.T) {
if err := goose.SetDialect("clickhouse"); err != nil {
t.Fatalf("SetDialect(clickhouse) error = %v", err)
}
}
func TestMigrateClickHouseSkipsWhenDisabled(t *testing.T) {
previousEnabled := config.Config.ClickHouse.Enabled
config.Config.ClickHouse.Enabled = false
t.Cleanup(func() {
config.Config.ClickHouse.Enabled = previousEnabled
})
MigrateClickHouse()
}
@@ -0,0 +1,21 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_user_access_logs
(
id UInt64,
user_id UInt64,
path String,
method String,
ip String,
user_agent String,
headers String,
status Int32,
latency Int64,
created_at DateTime
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created_at)
ORDER BY (created_at, ip, user_id)
SETTINGS index_granularity = 8192;
-- +goose Down
DROP TABLE IF EXISTS w_user_access_logs;
@@ -0,0 +1,184 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS users (
id BIGINT PRIMARY KEY,
username VARCHAR(64) UNIQUE,
password VARCHAR(255),
nickname VARCHAR(255),
email VARCHAR(255),
avatar_url VARCHAR(255),
is_active BOOLEAN DEFAULT TRUE,
is_admin BOOLEAN DEFAULT FALSE,
bio VARCHAR(500),
phone VARCHAR(32),
gender VARCHAR(16),
website VARCHAR(255),
location VARCHAR(255),
last_login_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
CREATE TABLE IF NOT EXISTS auth_sources (
id BIGSERIAL PRIMARY KEY,
name VARCHAR(80) NOT NULL UNIQUE,
type VARCHAR(20) NOT NULL,
display_name VARCHAR(100),
is_active BOOLEAN NOT NULL DEFAULT FALSE,
client_id VARCHAR(255),
client_secret VARCHAR(1024),
openid_discovery_url VARCHAR(1024),
scopes VARCHAR(255),
icon_url VARCHAR(1024),
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
CREATE TABLE IF NOT EXISTS external_accounts (
id BIGSERIAL PRIMARY KEY,
auth_source_id BIGINT,
user_id BIGINT NOT NULL,
external_id VARCHAR(255) NOT NULL,
external_username VARCHAR(255),
email VARCHAR(255),
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
CREATE TABLE IF NOT EXISTS system_configs (
key VARCHAR(64) PRIMARY KEY,
value TEXT NOT NULL,
type VARCHAR(32) NOT NULL DEFAULT 'system',
visibility INTEGER NOT NULL DEFAULT 0,
description VARCHAR(255),
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS uploads (
id BIGINT PRIMARY KEY,
user_id BIGINT NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_path VARCHAR(500) NOT NULL,
file_size BIGINT NOT NULL,
mime_type VARCHAR(100) NOT NULL,
extension VARCHAR(50) NOT NULL,
hash VARCHAR(64),
storage_driver VARCHAR(50) NOT NULL,
type VARCHAR(50) NOT NULL,
status VARCHAR(20) NOT NULL,
metadata JSONB,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
CREATE TABLE IF NOT EXISTS access_tokens (
id BIGSERIAL PRIMARY KEY,
user_id BIGINT NOT NULL,
name VARCHAR(128) NOT NULL,
token_hash VARCHAR(64) NOT NULL UNIQUE,
masked_token VARCHAR(64) NOT NULL,
last_used_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
CREATE TABLE IF NOT EXISTS task_executions (
id BIGINT PRIMARY KEY,
task_id VARCHAR(128) NOT NULL UNIQUE,
task_type VARCHAR(64) NOT NULL,
task_name VARCHAR(128),
status VARCHAR(32) NOT NULL,
retryable BOOLEAN NOT NULL DEFAULT FALSE,
max_retry INTEGER NOT NULL DEFAULT 0,
retry_count INTEGER NOT NULL DEFAULT 0,
log TEXT,
error_message TEXT,
result TEXT,
started_at TIMESTAMPTZ,
finished_at TIMESTAMPTZ,
duration BIGINT,
payload TEXT,
triggered_by VARCHAR(32) NOT NULL DEFAULT 'system',
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
CREATE TABLE IF NOT EXISTS templates (
id BIGSERIAL PRIMARY KEY,
key VARCHAR(80) NOT NULL UNIQUE,
name VARCHAR(100) NOT NULL,
type VARCHAR(20) NOT NULL DEFAULT 'email',
subject VARCHAR(255),
content TEXT NOT NULL,
description VARCHAR(255),
is_system BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_difficulty', '4', 'system', 0, '人机验证 PoW 难度(目标前缀长度)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_ttl_seconds', '600', 'system', 0, '人机验证难题有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_token_ttl_seconds', '1200', 'system', 0, '人机验证兑换凭证有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('server_address', '', 'system', 0, '服务器地址(用于跨域源控制,不设定则允许任意源)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_host', '', 'system', 0, 'SMTP 服务器地址(例如 smtp.example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_port', '587', 'system', 0, 'SMTP 端口(例如 587 或 465)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_username', '', 'system', 0, 'SMTP 账户(如 sender@example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_password', '', 'system', 0, 'SMTP 访问凭证(授权码/密码)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('upload_allowed_extensions', 'jpg,png,webp', 'system', 1, '允许上传的图片扩展名(逗号分隔)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('site_name', 'Wavelet', 'system', 1, '系统平台的展示名称', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_login_enabled', 'true', 'system', 1, '是否允许使用账号密码登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('registration_enabled', 'true', 'system', 1, '控制普通用户是否可以自主注册(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_register_enabled', 'true', 'system', 1, '是否允许通过密码创建本地账号', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('oidc_login_enabled', 'true', 'system', 1, '是否允许使用第三方 OIDC 认证源登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('max_api_keys_per_user', '5', 'business', 1, '限制每个普通用户可以创建的 API Key 最大数量', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_login_verification_enabled', 'false', 'system', 1, '是否开启邮箱登录验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_register_verification_enabled', 'false', 'system', 1, '是否开启邮箱注册验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('menu_display_config', '{}', 'system', 1, '目录显示配置(JSON 字符串,格式为 {url: enabled})', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('search_engine_indexing_enabled', 'false', 'system', 1, '是否允许搜索引擎爬取/检索该站点(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('storage_config', '{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}', 'system', 0, '文件存储驱动及连接配置(JSON)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
INSERT INTO users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (1, 'admin', '12345678', 'Administrator', '', TRUE, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
INSERT INTO templates (key, name, type, subject, content, description, is_system, created_at, updated_at) VALUES
('login_email', '登录验证码邮件', 'email', 'Wavelet 登录验证码', '<h3>Wavelet 登录验证</h3><p>您的登录验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿将验证码泄露给他人。</p>', '用户密码登录时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('register_email', '注册验证码邮件', 'email', 'Wavelet 注册验证码', '<h3>Wavelet 注册验证</h3><p>您的注册验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿泄露给他人。</p>', '用户注册时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS templates;
DROP TABLE IF EXISTS task_executions;
DROP TABLE IF EXISTS access_tokens;
DROP TABLE IF EXISTS uploads;
DROP TABLE IF EXISTS system_configs;
DROP TABLE IF EXISTS external_accounts;
DROP TABLE IF EXISTS auth_sources;
DROP TABLE IF EXISTS users;
@@ -0,0 +1,20 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS schedules (
id BIGINT PRIMARY KEY,
name VARCHAR(128) NOT NULL,
task_type VARCHAR(64) NOT NULL,
cron VARCHAR(64) NOT NULL,
payload TEXT,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
-- Seed initial cleanup task
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
VALUES (1, '清理未使用上传', 'cleanup_unused_uploads', '0 */2 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (id) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS schedules;
@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE access_tokens ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT FALSE;
-- +goose Down
ALTER TABLE access_tokens DROP COLUMN IF EXISTS is_admin;
@@ -0,0 +1,9 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE access_tokens DROP COLUMN IF EXISTS last_used_at;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE access_tokens ADD COLUMN last_used_at TIMESTAMPTZ;
-- +goose StatementEnd
@@ -0,0 +1,9 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE schedules ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (START WITH 100);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE schedules ALTER COLUMN id DROP IDENTITY IF EXISTS;
-- +goose StatementEnd
@@ -0,0 +1,66 @@
-- +goose Up
ALTER TABLE users RENAME TO w_users;
ALTER TABLE auth_sources RENAME TO w_auth_sources;
ALTER TABLE external_accounts RENAME TO w_external_accounts;
ALTER TABLE system_configs RENAME TO w_system_configs;
ALTER TABLE uploads RENAME TO w_uploads;
ALTER TABLE access_tokens RENAME TO w_access_tokens;
ALTER TABLE task_executions RENAME TO w_task_executions;
ALTER TABLE templates RENAME TO w_templates;
ALTER TABLE schedules RENAME TO w_schedules;
-- Rename indexes for consistency
ALTER INDEX IF EXISTS idx_users_email RENAME TO idx_w_users_email;
ALTER INDEX IF EXISTS idx_users_is_active RENAME TO idx_w_users_is_active;
ALTER INDEX IF EXISTS idx_users_last_login_at RENAME TO idx_w_users_last_login_at;
ALTER INDEX IF EXISTS idx_users_created_at RENAME TO idx_w_users_created_at;
ALTER INDEX IF EXISTS idx_auth_sources_is_active RENAME TO idx_w_auth_sources_is_active;
ALTER INDEX IF EXISTS idx_external_accounts_auth_source_id RENAME TO idx_w_external_accounts_auth_source_id;
ALTER INDEX IF EXISTS idx_external_accounts_user_id RENAME TO idx_w_external_accounts_user_id;
ALTER INDEX IF EXISTS idx_external_accounts_source_external RENAME TO idx_w_external_accounts_source_external;
ALTER INDEX IF EXISTS idx_uploads_user_id RENAME TO idx_w_uploads_user_id;
ALTER INDEX IF EXISTS idx_uploads_file_path RENAME TO idx_w_uploads_file_path;
ALTER INDEX IF EXISTS idx_uploads_hash RENAME TO idx_w_uploads_hash;
ALTER INDEX IF EXISTS idx_uploads_type RENAME TO idx_w_uploads_type;
ALTER INDEX IF EXISTS idx_access_tokens_user_id RENAME TO idx_w_access_tokens_user_id;
ALTER INDEX IF EXISTS idx_task_executions_task_type RENAME TO idx_w_task_executions_task_type;
ALTER INDEX IF EXISTS idx_task_executions_status RENAME TO idx_w_task_executions_status;
ALTER INDEX IF EXISTS idx_task_executions_started_at RENAME TO idx_w_task_executions_started_at;
ALTER INDEX IF EXISTS idx_task_executions_created_at RENAME TO idx_w_task_executions_created_at;
ALTER INDEX IF EXISTS idx_templates_is_system RENAME TO idx_w_templates_is_system;
ALTER INDEX IF EXISTS idx_templates_created_at RENAME TO idx_w_templates_created_at;
ALTER INDEX IF EXISTS idx_templates_updated_at RENAME TO idx_w_templates_updated_at;
ALTER INDEX IF EXISTS idx_schedules_is_active RENAME TO idx_w_schedules_is_active;
-- +goose Down
ALTER INDEX IF EXISTS idx_w_schedules_is_active RENAME TO idx_schedules_is_active;
ALTER INDEX IF EXISTS idx_w_templates_updated_at RENAME TO idx_templates_updated_at;
ALTER INDEX IF EXISTS idx_w_templates_created_at RENAME TO idx_templates_created_at;
ALTER INDEX IF EXISTS idx_w_templates_is_system RENAME TO idx_templates_is_system;
ALTER INDEX IF EXISTS idx_w_task_executions_created_at RENAME TO idx_task_executions_created_at;
ALTER INDEX IF EXISTS idx_w_task_executions_started_at RENAME TO idx_task_executions_started_at;
ALTER INDEX IF EXISTS idx_w_task_executions_status RENAME TO idx_task_executions_status;
ALTER INDEX IF EXISTS idx_w_task_executions_task_type RENAME TO idx_task_executions_task_type;
ALTER INDEX IF EXISTS idx_w_access_tokens_user_id RENAME TO idx_access_tokens_user_id;
ALTER INDEX IF EXISTS idx_w_uploads_type RENAME TO idx_uploads_type;
ALTER INDEX IF EXISTS idx_w_uploads_hash RENAME TO idx_uploads_hash;
ALTER INDEX IF EXISTS idx_w_uploads_file_path RENAME TO idx_uploads_file_path;
ALTER INDEX IF EXISTS idx_w_uploads_user_id RENAME TO idx_uploads_user_id;
ALTER INDEX IF EXISTS idx_w_external_accounts_source_external RENAME TO idx_external_accounts_source_external;
ALTER INDEX IF EXISTS idx_w_external_accounts_user_id RENAME TO idx_external_accounts_user_id;
ALTER INDEX IF EXISTS idx_w_external_accounts_auth_source_id RENAME TO idx_external_accounts_auth_source_id;
ALTER INDEX IF EXISTS idx_w_auth_sources_is_active RENAME TO idx_auth_sources_is_active;
ALTER INDEX IF EXISTS idx_w_users_created_at RENAME TO idx_users_created_at;
ALTER INDEX IF EXISTS idx_w_users_last_login_at RENAME TO idx_users_last_login_at;
ALTER INDEX IF EXISTS idx_w_users_is_active RENAME TO idx_users_is_active;
ALTER INDEX IF EXISTS idx_w_users_email RENAME TO idx_users_email;
ALTER TABLE w_schedules RENAME TO schedules;
ALTER TABLE w_templates RENAME TO templates;
ALTER TABLE w_task_executions RENAME TO task_executions;
ALTER TABLE w_access_tokens RENAME TO access_tokens;
ALTER TABLE w_uploads RENAME TO uploads;
ALTER TABLE w_system_configs RENAME TO system_configs;
ALTER TABLE w_external_accounts RENAME TO external_accounts;
ALTER TABLE w_auth_sources RENAME TO auth_sources;
ALTER TABLE w_users RENAME TO users;
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('file_access_whitelist', '["avatar"]', 'system', 1, '免登录访问的文件业务类型白名单 (JSON 数组格式)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'file_access_whitelist';
@@ -0,0 +1,10 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
@@ -0,0 +1,8 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('login_session_ttl_hours', '0', 'system', 0, '登录会话过期时间 (小时,0表示浏览器关闭后自动退出,-1表示永不过期)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'login_session_ttl_hours';
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'update_upstream_repository';
@@ -0,0 +1,6 @@
-- +goose Up
ALTER TABLE w_uploads ADD COLUMN access_mode INTEGER NOT NULL DEFAULT 0;
UPDATE w_uploads SET access_mode = 1 WHERE type = 'avatar';
-- +goose Down
ALTER TABLE w_uploads DROP COLUMN access_mode;
@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE w_system_configs ALTER COLUMN value TYPE TEXT;
-- +goose Down
ALTER TABLE w_system_configs ALTER COLUMN value TYPE VARCHAR(255);
@@ -0,0 +1,15 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES (
'storage_config',
'{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}',
'system',
0,
'文件存储驱动及连接配置(JSON)',
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'storage_config';
@@ -0,0 +1,34 @@
-- +goose Up
CREATE TABLE w_push_events (
id BIGSERIAL PRIMARY KEY,
event_key VARCHAR(80) NOT NULL UNIQUE,
name VARCHAR(100) NOT NULL,
channels TEXT NOT NULL,
targets TEXT NOT NULL,
template TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX idx_w_push_events_enabled ON w_push_events(enabled);
CREATE TABLE w_push_histories (
id BIGSERIAL PRIMARY KEY,
event_key VARCHAR(80) NOT NULL,
channel VARCHAR(50) NOT NULL,
target VARCHAR(255) NOT NULL,
title VARCHAR(255) NOT NULL,
content TEXT NOT NULL,
level VARCHAR(20) NOT NULL,
status VARCHAR(20) NOT NULL,
error_msg TEXT,
created_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX idx_w_push_histories_event ON w_push_histories(event_key);
CREATE INDEX idx_w_push_histories_created ON w_push_histories(created_at);
-- +goose Down
DROP TABLE IF EXISTS w_push_histories;
DROP TABLE IF EXISTS w_push_events;
@@ -0,0 +1,8 @@
-- +goose Up
INSERT INTO w_users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (999, 'system', '*', '系统', '', TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
SELECT setval(pg_get_serial_sequence('w_users', 'id'), COALESCE((SELECT MAX(id) FROM w_users), 1));
-- +goose Down
DELETE FROM w_users WHERE username = 'system';
@@ -0,0 +1,19 @@
-- +goose Up
CREATE TABLE w_push_channels (
id BIGSERIAL PRIMARY KEY,
name VARCHAR(80) NOT NULL UNIQUE,
description VARCHAR(255),
type VARCHAR(50) NOT NULL DEFAULT 'custom',
token VARCHAR(100),
url TEXT NOT NULL,
other TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX idx_w_push_channels_name ON w_push_channels(name);
CREATE INDEX idx_w_push_channels_enabled ON w_push_channels(enabled);
-- +goose Down
DROP TABLE IF EXISTS w_push_channels;
@@ -0,0 +1,11 @@
-- +goose Up
UPDATE w_schedules
SET name = '系统定期垃圾清理',
task_type = 'system_cleanup'
WHERE id = 1;
-- +goose Down
UPDATE w_schedules
SET name = '清理未使用上传',
task_type = 'cleanup_unused_uploads'
WHERE id = 1;
@@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE w_push_events ADD COLUMN task_type VARCHAR(100) NOT NULL DEFAULT '';
CREATE INDEX idx_w_push_events_task_type ON w_push_events(task_type);
-- +goose Down
DROP INDEX IF EXISTS idx_w_push_events_task_type;
ALTER TABLE w_push_events DROP COLUMN task_type;
@@ -0,0 +1,6 @@
-- +goose Up
DELETE FROM w_system_configs WHERE key = 'push_config';
DELETE FROM w_push_events WHERE event_key = 'admin_login';
DELETE FROM w_system_configs WHERE key = 'push_global_token';
-- +goose Down
@@ -0,0 +1,9 @@
-- +goose Up
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
-- +goose Down
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
@@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_upload_stats (
dimension VARCHAR(32) NOT NULL,
stat_key VARCHAR(64) NOT NULL DEFAULT '',
file_count BIGINT NOT NULL DEFAULT 0,
file_size BIGINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dimension, stat_key)
);
-- +goose Down
DROP TABLE IF EXISTS w_upload_stats;
@@ -0,0 +1,67 @@
-- +goose Up
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'type',
COALESCE(NULLIF(type, ''), 'generic'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'category',
CASE
WHEN LOWER(mime_type) LIKE 'image/%'
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
OR LOWER(mime_type) LIKE '%zip%'
OR LOWER(mime_type) LIKE '%tar%'
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
OR LOWER(mime_type) LIKE 'text/%'
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
ELSE '其他'
END,
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY 2
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'trend',
TO_CHAR(created_at, 'YYYY-MM-DD'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY TO_CHAR(created_at, 'YYYY-MM-DD')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = EXCLUDED.file_count,
file_size = EXCLUDED.file_size,
updated_at = CURRENT_TIMESTAMP;
-- +goose Down
DELETE FROM w_upload_stats;
@@ -0,0 +1,7 @@
-- +goose Up
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
ALTER TABLE w_uploads DROP COLUMN IF EXISTS storage_driver;
-- +goose Down
ALTER TABLE w_uploads ADD COLUMN IF NOT EXISTS storage_driver VARCHAR(50) NOT NULL DEFAULT 'local';
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
@@ -0,0 +1,20 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_user_access_logs (
id BIGINT NOT NULL,
user_id BIGINT NOT NULL DEFAULT 0,
path VARCHAR(2048) NOT NULL DEFAULT '',
method VARCHAR(16) NOT NULL DEFAULT '',
ip VARCHAR(128) NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
headers TEXT NOT NULL DEFAULT '',
status INTEGER NOT NULL DEFAULT 0,
latency BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, created_at)
) PARTITION BY RANGE (created_at);
CREATE INDEX IF NOT EXISTS idx_w_user_access_logs_user_id ON w_user_access_logs (user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_w_user_access_logs_created_at ON w_user_access_logs (created_at DESC);
-- +goose Down
DROP TABLE IF EXISTS w_user_access_logs;
@@ -0,0 +1,24 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('log_database', '', 'system', 0, '当前日志主库(postgres/sqlite/clickhouse),由切换任务写入', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('log_db_migration', '', 'system', 0, '日志库迁移冻结标记(空或 migrating)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
SELECT k, '30', 'business', 0, descr, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
FROM (VALUES
('log_retention_days_postgres', 'PostgreSQL 用户访问日志保留天数'),
('log_retention_days_sqlite', 'SQLite 用户访问日志保留天数'),
('log_retention_days_clickhouse', 'ClickHouse 用户访问日志保留天数')
) AS v(k, descr)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key IN (
'log_database',
'log_db_migration',
'log_retention_days_postgres',
'log_retention_days_sqlite',
'log_retention_days_clickhouse'
);
@@ -0,0 +1,40 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_message_channels (
id BIGINT PRIMARY KEY,
name VARCHAR(128) NOT NULL,
type VARCHAR(32) NOT NULL,
owner_scope VARCHAR(16) NOT NULL DEFAULT 'system',
owner_id BIGINT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
credentials TEXT NOT NULL DEFAULT '',
extra TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_w_message_channels_type ON w_message_channels (type);
CREATE TABLE IF NOT EXISTS w_message_bindings (
id BIGINT PRIMARY KEY,
user_id BIGINT NOT NULL,
channel_id BIGINT NOT NULL,
platform_user_id VARCHAR(128) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS uniq_w_message_bindings_channel_platform
ON w_message_bindings (channel_id, platform_user_id);
CREATE INDEX IF NOT EXISTS idx_w_message_bindings_user ON w_message_bindings (user_id);
CREATE TABLE IF NOT EXISTS w_message_pairing_codes (
code VARCHAR(16) PRIMARY KEY,
channel_id BIGINT NOT NULL,
platform_user_id VARCHAR(128) NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_w_message_pairing_lookup
ON w_message_pairing_codes (channel_id, platform_user_id);
-- +goose Down
DROP TABLE IF EXISTS w_message_pairing_codes;
DROP TABLE IF EXISTS w_message_bindings;
DROP TABLE IF EXISTS w_message_channels;
@@ -0,0 +1,184 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS users (
id BIGINT PRIMARY KEY,
username VARCHAR(64) UNIQUE,
password VARCHAR(255),
nickname VARCHAR(255),
email VARCHAR(255),
avatar_url VARCHAR(255),
is_active BOOLEAN DEFAULT TRUE,
is_admin BOOLEAN DEFAULT FALSE,
bio VARCHAR(500),
phone VARCHAR(32),
gender VARCHAR(16),
website VARCHAR(255),
location VARCHAR(255),
last_login_at DATETIME,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
CREATE TABLE IF NOT EXISTS auth_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name VARCHAR(80) NOT NULL UNIQUE,
type VARCHAR(20) NOT NULL,
display_name VARCHAR(100),
is_active BOOLEAN NOT NULL DEFAULT FALSE,
client_id VARCHAR(255),
client_secret VARCHAR(1024),
openid_discovery_url VARCHAR(1024),
scopes VARCHAR(255),
icon_url VARCHAR(1024),
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
CREATE TABLE IF NOT EXISTS external_accounts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
auth_source_id BIGINT,
user_id BIGINT NOT NULL,
external_id VARCHAR(255) NOT NULL,
external_username VARCHAR(255),
email VARCHAR(255),
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
CREATE TABLE IF NOT EXISTS system_configs (
key VARCHAR(64) PRIMARY KEY,
value TEXT NOT NULL,
type VARCHAR(32) NOT NULL DEFAULT 'system',
visibility INTEGER NOT NULL DEFAULT 0,
description VARCHAR(255),
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS uploads (
id BIGINT PRIMARY KEY,
user_id BIGINT NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_path VARCHAR(500) NOT NULL,
file_size BIGINT NOT NULL,
mime_type VARCHAR(100) NOT NULL,
extension VARCHAR(50) NOT NULL,
hash VARCHAR(64),
storage_driver VARCHAR(50) NOT NULL,
type VARCHAR(50) NOT NULL,
status VARCHAR(20) NOT NULL,
metadata JSON,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
CREATE TABLE IF NOT EXISTS access_tokens (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id BIGINT NOT NULL,
name VARCHAR(128) NOT NULL,
token_hash VARCHAR(64) NOT NULL UNIQUE,
masked_token VARCHAR(64) NOT NULL,
last_used_at DATETIME,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
CREATE TABLE IF NOT EXISTS task_executions (
id BIGINT PRIMARY KEY,
task_id VARCHAR(128) NOT NULL UNIQUE,
task_type VARCHAR(64) NOT NULL,
task_name VARCHAR(128),
status VARCHAR(32) NOT NULL,
retryable BOOLEAN NOT NULL DEFAULT FALSE,
max_retry INTEGER NOT NULL DEFAULT 0,
retry_count INTEGER NOT NULL DEFAULT 0,
log TEXT,
error_message TEXT,
result TEXT,
started_at DATETIME,
finished_at DATETIME,
duration BIGINT,
payload TEXT,
triggered_by VARCHAR(32) NOT NULL DEFAULT 'system',
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
CREATE TABLE IF NOT EXISTS templates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
key VARCHAR(80) NOT NULL UNIQUE,
name VARCHAR(100) NOT NULL,
type VARCHAR(20) NOT NULL DEFAULT 'email',
subject VARCHAR(255),
content TEXT NOT NULL,
description VARCHAR(255),
is_system BOOLEAN NOT NULL DEFAULT FALSE,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_difficulty', '4', 'system', 0, '人机验证 PoW 难度(目标前缀长度)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_challenge_ttl_seconds', '600', 'system', 0, '人机验证难题有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('cap_token_ttl_seconds', '1200', 'system', 0, '人机验证兑换凭证有效时间(秒)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('server_address', '', 'system', 0, '服务器地址(用于跨域源控制,不设定则允许任意源)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_host', '', 'system', 0, 'SMTP 服务器地址(例如 smtp.example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_port', '587', 'system', 0, 'SMTP 端口(例如 587 或 465)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_username', '', 'system', 0, 'SMTP 账户(如 sender@example.com)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('smtp_password', '', 'system', 0, 'SMTP 访问凭证(授权码/密码)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('upload_allowed_extensions', 'jpg,png,webp', 'system', 1, '允许上传的图片扩展名(逗号分隔)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('site_name', 'Wavelet', 'system', 1, '系统平台的展示名称', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_login_enabled', 'true', 'system', 1, '是否允许使用账号密码登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('registration_enabled', 'true', 'system', 1, '控制普通用户是否可以自主注册(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('password_register_enabled', 'true', 'system', 1, '是否允许通过密码创建本地账号', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('oidc_login_enabled', 'true', 'system', 1, '是否允许使用第三方 OIDC 认证源登录', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('max_api_keys_per_user', '5', 'business', 1, '限制每个普通用户可以创建的 API Key 最大数量', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_login_verification_enabled', 'false', 'system', 1, '是否开启邮箱登录验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('email_register_verification_enabled', 'false', 'system', 1, '是否开启邮箱注册验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('menu_display_config', '{}', 'system', 1, '目录显示配置(JSON 字符串,格式为 {url: enabled})', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('search_engine_indexing_enabled', 'false', 'system', 1, '是否允许搜索引擎爬取/检索该站点(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('storage_config', '{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}', 'system', 0, '文件存储驱动及连接配置(JSON)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
INSERT INTO users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (1, 'admin', '12345678', 'Administrator', '', TRUE, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
INSERT INTO templates (key, name, type, subject, content, description, is_system, created_at, updated_at) VALUES
('login_email', '登录验证码邮件', 'email', 'Wavelet 登录验证码', '<h3>Wavelet 登录验证</h3><p>您的登录验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿将验证码泄露给他人。</p>', '用户密码登录时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('register_email', '注册验证码邮件', 'email', 'Wavelet 注册验证码', '<h3>Wavelet 注册验证</h3><p>您的注册验证码为:<strong>{{.Code}}</strong>,5分钟内有效,请勿泄露给他人。</p>', '用户注册时发送的验证码邮件模板,支持变量:{{.Code}}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS templates;
DROP TABLE IF EXISTS task_executions;
DROP TABLE IF EXISTS access_tokens;
DROP TABLE IF EXISTS uploads;
DROP TABLE IF EXISTS system_configs;
DROP TABLE IF EXISTS external_accounts;
DROP TABLE IF EXISTS auth_sources;
DROP TABLE IF EXISTS users;
@@ -0,0 +1,20 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS schedules (
id BIGINT PRIMARY KEY,
name VARCHAR(128) NOT NULL,
task_type VARCHAR(64) NOT NULL,
cron VARCHAR(64) NOT NULL,
payload TEXT,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
-- Seed initial cleanup task
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
VALUES (1, '清理未使用上传', 'cleanup_unused_uploads', '0 */2 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (id) DO NOTHING;
-- +goose Down
DROP TABLE IF EXISTS schedules;
@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE access_tokens ADD COLUMN is_admin BOOLEAN NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE access_tokens DROP COLUMN is_admin;
@@ -0,0 +1,9 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE access_tokens DROP COLUMN last_used_at;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE access_tokens ADD COLUMN last_used_at DATETIME;
-- +goose StatementEnd
@@ -0,0 +1,32 @@
-- +goose Up
-- +goose StatementBegin
-- 1. Rename existing schedules table
ALTER TABLE schedules RENAME TO schedules_old;
-- 2. Create new schedules table with AUTOINCREMENT
CREATE TABLE schedules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name VARCHAR(128) NOT NULL,
task_type VARCHAR(64) NOT NULL,
cron VARCHAR(64) NOT NULL,
payload TEXT,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- 3. Copy existing data
INSERT INTO schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at FROM schedules_old;
-- 4. Drop the old table
DROP TABLE schedules_old;
-- 5. Recreate index
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- Re-creating table with AUTOINCREMENT cannot be undone simply without recreating table again.
-- +goose StatementEnd
@@ -0,0 +1,123 @@
-- +goose Up
ALTER TABLE users RENAME TO w_users;
ALTER TABLE auth_sources RENAME TO w_auth_sources;
ALTER TABLE external_accounts RENAME TO w_external_accounts;
ALTER TABLE system_configs RENAME TO w_system_configs;
ALTER TABLE uploads RENAME TO w_uploads;
ALTER TABLE access_tokens RENAME TO w_access_tokens;
ALTER TABLE task_executions RENAME TO w_task_executions;
ALTER TABLE templates RENAME TO w_templates;
ALTER TABLE schedules RENAME TO w_schedules;
-- Drop and Recreate SQLite indexes
DROP INDEX IF EXISTS idx_users_email;
CREATE INDEX IF NOT EXISTS idx_w_users_email ON w_users (email);
DROP INDEX IF EXISTS idx_users_is_active;
CREATE INDEX IF NOT EXISTS idx_w_users_is_active ON w_users (is_active);
DROP INDEX IF EXISTS idx_users_last_login_at;
CREATE INDEX IF NOT EXISTS idx_w_users_last_login_at ON w_users (last_login_at);
DROP INDEX IF EXISTS idx_users_created_at;
CREATE INDEX IF NOT EXISTS idx_w_users_created_at ON w_users (created_at);
DROP INDEX IF EXISTS idx_auth_sources_is_active;
CREATE INDEX IF NOT EXISTS idx_w_auth_sources_is_active ON w_auth_sources (is_active);
DROP INDEX IF EXISTS idx_external_accounts_auth_source_id;
CREATE INDEX IF NOT EXISTS idx_w_external_accounts_auth_source_id ON w_external_accounts (auth_source_id);
DROP INDEX IF EXISTS idx_external_accounts_user_id;
CREATE INDEX IF NOT EXISTS idx_w_external_accounts_user_id ON w_external_accounts (user_id);
DROP INDEX IF EXISTS idx_external_accounts_source_external;
CREATE UNIQUE INDEX IF NOT EXISTS idx_w_external_accounts_source_external ON w_external_accounts (auth_source_id, external_id);
DROP INDEX IF EXISTS idx_uploads_user_id;
CREATE INDEX IF NOT EXISTS idx_w_uploads_user_id ON w_uploads (user_id);
DROP INDEX IF EXISTS idx_uploads_file_path;
CREATE INDEX IF NOT EXISTS idx_w_uploads_file_path ON w_uploads (file_path);
DROP INDEX IF EXISTS idx_uploads_hash;
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash ON w_uploads (hash);
DROP INDEX IF EXISTS idx_uploads_type;
CREATE INDEX IF NOT EXISTS idx_w_uploads_type ON w_uploads (type);
DROP INDEX IF EXISTS idx_access_tokens_user_id;
CREATE INDEX IF NOT EXISTS idx_w_access_tokens_user_id ON w_access_tokens (user_id);
DROP INDEX IF EXISTS idx_task_executions_task_type;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_task_type ON w_task_executions (task_type);
DROP INDEX IF EXISTS idx_task_executions_status;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_status ON w_task_executions (status);
DROP INDEX IF EXISTS idx_task_executions_started_at;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_started_at ON w_task_executions (started_at);
DROP INDEX IF EXISTS idx_task_executions_created_at;
CREATE INDEX IF NOT EXISTS idx_w_task_executions_created_at ON w_task_executions (created_at);
DROP INDEX IF EXISTS idx_templates_is_system;
CREATE INDEX IF NOT EXISTS idx_w_templates_is_system ON w_templates (is_system);
DROP INDEX IF EXISTS idx_templates_created_at;
CREATE INDEX IF NOT EXISTS idx_w_templates_created_at ON w_templates (created_at);
DROP INDEX IF EXISTS idx_templates_updated_at;
CREATE INDEX IF NOT EXISTS idx_w_templates_updated_at ON w_templates (updated_at);
DROP INDEX IF EXISTS idx_schedules_is_active;
CREATE INDEX IF NOT EXISTS idx_w_schedules_is_active ON w_schedules (is_active);
-- +goose Down
ALTER TABLE w_schedules RENAME TO schedules;
ALTER TABLE w_templates RENAME TO templates;
ALTER TABLE w_task_executions RENAME TO task_executions;
ALTER TABLE w_access_tokens RENAME TO access_tokens;
ALTER TABLE w_uploads RENAME TO uploads;
ALTER TABLE w_system_configs RENAME TO system_configs;
ALTER TABLE w_external_accounts RENAME TO external_accounts;
ALTER TABLE w_auth_sources RENAME TO auth_sources;
ALTER TABLE w_users RENAME TO users;
-- Revert indexes
DROP INDEX IF EXISTS idx_w_users_email;
CREATE INDEX IF NOT EXISTS idx_users_email ON users (email);
DROP INDEX IF EXISTS idx_w_users_is_active;
CREATE INDEX IF NOT EXISTS idx_users_is_active ON users (is_active);
DROP INDEX IF EXISTS idx_w_users_last_login_at;
CREATE INDEX IF NOT EXISTS idx_users_last_login_at ON users (last_login_at);
DROP INDEX IF EXISTS idx_w_users_created_at;
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users (created_at);
DROP INDEX IF EXISTS idx_w_auth_sources_is_active;
CREATE INDEX IF NOT EXISTS idx_auth_sources_is_active ON auth_sources (is_active);
DROP INDEX IF EXISTS idx_w_external_accounts_auth_source_id;
CREATE INDEX IF NOT EXISTS idx_external_accounts_auth_source_id ON external_accounts (auth_source_id);
DROP INDEX IF EXISTS idx_w_external_accounts_user_id;
CREATE INDEX IF NOT EXISTS idx_external_accounts_user_id ON external_accounts (user_id);
DROP INDEX IF EXISTS idx_w_external_accounts_source_external;
CREATE UNIQUE INDEX IF NOT EXISTS idx_external_accounts_source_external ON external_accounts (auth_source_id, external_id);
DROP INDEX IF EXISTS idx_w_uploads_user_id;
CREATE INDEX IF NOT EXISTS idx_uploads_user_id ON uploads (user_id);
DROP INDEX IF EXISTS idx_w_uploads_file_path;
CREATE INDEX IF NOT EXISTS idx_uploads_file_path ON uploads (file_path);
DROP INDEX IF EXISTS idx_w_uploads_hash;
CREATE INDEX IF NOT EXISTS idx_uploads_hash ON uploads (hash);
DROP INDEX IF EXISTS idx_w_uploads_type;
CREATE INDEX IF NOT EXISTS idx_uploads_type ON uploads (type);
DROP INDEX IF EXISTS idx_w_access_tokens_user_id;
CREATE INDEX IF NOT EXISTS idx_access_tokens_user_id ON access_tokens (user_id);
DROP INDEX IF EXISTS idx_w_task_executions_task_type;
CREATE INDEX IF NOT EXISTS idx_task_executions_task_type ON task_executions (task_type);
DROP INDEX IF EXISTS idx_w_task_executions_status;
CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions (status);
DROP INDEX IF EXISTS idx_w_task_executions_started_at;
CREATE INDEX IF NOT EXISTS idx_task_executions_started_at ON task_executions (started_at);
DROP INDEX IF EXISTS idx_w_task_executions_created_at;
CREATE INDEX IF NOT EXISTS idx_task_executions_created_at ON task_executions (created_at);
DROP INDEX IF EXISTS idx_w_templates_is_system;
CREATE INDEX IF NOT EXISTS idx_templates_is_system ON templates (is_system);
DROP INDEX IF EXISTS idx_w_templates_created_at;
CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
DROP INDEX IF EXISTS idx_w_templates_updated_at;
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
DROP INDEX IF EXISTS idx_w_schedules_is_active;
CREATE INDEX IF NOT EXISTS idx_schedules_is_active ON schedules (is_active);
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('file_access_whitelist', '["avatar"]', 'system', 1, '免登录访问的文件业务类型白名单 (JSON 数组格式)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'file_access_whitelist';
@@ -0,0 +1,10 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('disk_cache_max_size_mb', '100', 'system', 0, '磁盘缓存最大空间大小 (MB)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_ttl_minutes', '60', 'system', 0, '磁盘缓存默认有效期 (分钟)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('disk_cache_lru_enabled', 'true', 'system', 0, '是否启用 LRU 淘汰机制', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key IN ('disk_cache_max_size_mb', 'disk_cache_ttl_minutes', 'disk_cache_lru_enabled');
@@ -0,0 +1,8 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('login_session_ttl_hours', '0', 'system', 0, '登录会话过期时间 (小时,0表示浏览器关闭后自动退出,-1表示永不过期)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'login_session_ttl_hours';
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES ('update_upstream_repository', 'Rain-kl/Wavelet', 'system', 0, 'GitHub Actions Release 上游仓库(owner/repo 或 GitHub 仓库地址)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'update_upstream_repository';
@@ -0,0 +1,6 @@
-- +goose Up
ALTER TABLE w_uploads ADD COLUMN access_mode INTEGER NOT NULL DEFAULT 0;
UPDATE w_uploads SET access_mode = 1 WHERE type = 'avatar';
-- +goose Down
ALTER TABLE w_uploads DROP COLUMN access_mode;
@@ -0,0 +1,6 @@
-- +goose Up
-- SQLite stores VARCHAR and TEXT with the same TEXT affinity.
SELECT 1;
-- +goose Down
SELECT 1;
@@ -0,0 +1,15 @@
-- +goose Up
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES (
'storage_config',
'{"driver":"local","local":{"root":"."},"s3":{"region":"us-east-1"},"r2":{"region":"auto"},"minio":{"region":"us-east-1","path_style":true},"oss":{},"webdav":{}}',
'system',
0,
'文件存储驱动及连接配置(JSON)',
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
)
ON CONFLICT (key) DO NOTHING;
-- +goose Down
DELETE FROM w_system_configs WHERE key = 'storage_config';
@@ -0,0 +1,34 @@
-- +goose Up
CREATE TABLE w_push_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event_key TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
channels TEXT NOT NULL,
targets TEXT NOT NULL,
template TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
);
CREATE INDEX idx_w_push_events_enabled ON w_push_events(enabled);
CREATE TABLE w_push_histories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event_key TEXT NOT NULL,
channel TEXT NOT NULL,
target TEXT NOT NULL,
title TEXT NOT NULL,
content TEXT NOT NULL,
level TEXT NOT NULL,
status TEXT NOT NULL,
error_msg TEXT,
created_at DATETIME NOT NULL
);
CREATE INDEX idx_w_push_histories_event ON w_push_histories(event_key);
CREATE INDEX idx_w_push_histories_created ON w_push_histories(created_at);
-- +goose Down
DROP TABLE IF EXISTS w_push_histories;
DROP TABLE IF EXISTS w_push_events;
@@ -0,0 +1,7 @@
-- +goose Up
INSERT INTO w_users (id, username, password, nickname, avatar_url, is_active, is_admin, last_login_at, created_at, updated_at)
VALUES (999, 'system', '*', '系统', '', TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (username) DO NOTHING;
-- +goose Down
DELETE FROM w_users WHERE username = 'system';
@@ -0,0 +1,19 @@
-- +goose Up
CREATE TABLE w_push_channels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
description TEXT,
type TEXT NOT NULL DEFAULT 'custom',
token TEXT,
url TEXT NOT NULL,
other TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
);
CREATE INDEX idx_w_push_channels_name ON w_push_channels(name);
CREATE INDEX idx_w_push_channels_enabled ON w_push_channels(enabled);
-- +goose Down
DROP TABLE IF EXISTS w_push_channels;
@@ -0,0 +1,11 @@
-- +goose Up
UPDATE w_schedules
SET name = '系统定期垃圾清理',
task_type = 'system_cleanup'
WHERE id = 1;
-- +goose Down
UPDATE w_schedules
SET name = '清理未使用上传',
task_type = 'cleanup_unused_uploads'
WHERE id = 1;
@@ -0,0 +1,9 @@
-- +goose Up
ALTER TABLE w_push_events ADD COLUMN task_type VARCHAR(100) NOT NULL DEFAULT '';
CREATE INDEX idx_w_push_events_task_type ON w_push_events(task_type);
-- +goose Down
DROP INDEX IF EXISTS idx_w_push_events_task_type;
-- SQLite does not support DROP COLUMN in older versions easily, but standard ALTER TABLE DROP COLUMN works in SQLite 3.35.0+.
-- We can write standard DROP COLUMN.
ALTER TABLE w_push_events DROP COLUMN task_type;
@@ -0,0 +1,6 @@
-- +goose Up
DELETE FROM w_system_configs WHERE key = 'push_config';
DELETE FROM w_push_events WHERE event_key = 'admin_login';
DELETE FROM w_system_configs WHERE key = 'push_global_token';
-- +goose Down
@@ -0,0 +1,9 @@
-- +goose Up
CREATE INDEX IF NOT EXISTS idx_w_uploads_status_created_at ON w_uploads (status, created_at);
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
CREATE INDEX IF NOT EXISTS idx_w_uploads_hash_file_size_status ON w_uploads (hash, file_size, status);
-- +goose Down
DROP INDEX IF EXISTS idx_w_uploads_hash_file_size_status;
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
DROP INDEX IF EXISTS idx_w_uploads_status_created_at;
@@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_upload_stats (
dimension VARCHAR(32) NOT NULL,
stat_key VARCHAR(64) NOT NULL DEFAULT '',
file_count BIGINT NOT NULL DEFAULT 0,
file_size BIGINT NOT NULL DEFAULT 0,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dimension, stat_key)
);
-- +goose Down
DROP TABLE IF EXISTS w_upload_stats;
@@ -0,0 +1,67 @@
-- +goose Up
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT 'total', '', COUNT(*), COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'type',
COALESCE(NULLIF(type, ''), 'generic'),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY COALESCE(NULLIF(type, ''), 'generic')
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'category',
CASE
WHEN LOWER(mime_type) LIKE 'image/%'
OR LOWER(extension) IN ('jpg', 'jpeg', 'png', 'webp', 'gif') THEN '图片'
WHEN LOWER(mime_type) LIKE 'video/%' THEN '视频'
WHEN LOWER(mime_type) LIKE 'audio/%' THEN '音频'
WHEN LOWER(extension) IN ('zip', 'rar', '7z', 'tar', 'gz', 'tgz', 'bz2', 'xz')
OR LOWER(mime_type) LIKE '%zip%'
OR LOWER(mime_type) LIKE '%tar%'
OR LOWER(mime_type) LIKE '%gzip%' THEN '压缩包'
WHEN LOWER(extension) IN ('pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', 'txt', 'md', 'csv', 'json', 'yaml', 'yml', 'xml')
OR LOWER(mime_type) LIKE 'text/%'
OR LOWER(mime_type) = 'application/pdf' THEN '文档'
ELSE '其他'
END,
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY 2
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
INSERT INTO w_upload_stats (dimension, stat_key, file_count, file_size)
SELECT
'trend',
STRFTIME('%Y-%m-%d', created_at),
COUNT(*),
COALESCE(SUM(file_size), 0)
FROM w_uploads
WHERE status != 'deleted'
GROUP BY STRFTIME('%Y-%m-%d', created_at)
ON CONFLICT (dimension, stat_key) DO UPDATE SET
file_count = excluded.file_count,
file_size = excluded.file_size,
updated_at = CURRENT_TIMESTAMP;
-- +goose Down
DELETE FROM w_upload_stats;
@@ -0,0 +1,8 @@
-- +goose Up
DROP INDEX IF EXISTS idx_w_uploads_storage_driver_status;
-- SQLite lacks DROP COLUMN IF EXISTS; goose runs this only after initial schema created the column.
ALTER TABLE w_uploads DROP COLUMN storage_driver;
-- +goose Down
ALTER TABLE w_uploads ADD COLUMN storage_driver VARCHAR(50) NOT NULL DEFAULT 'local';
CREATE INDEX IF NOT EXISTS idx_w_uploads_storage_driver_status ON w_uploads (storage_driver, status);
@@ -0,0 +1,22 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_user_access_logs (
id INTEGER NOT NULL,
user_id INTEGER NOT NULL DEFAULT 0,
path TEXT NOT NULL DEFAULT '',
method TEXT NOT NULL DEFAULT '',
ip TEXT NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
headers TEXT NOT NULL DEFAULT '',
status INTEGER NOT NULL DEFAULT 0,
latency INTEGER NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, created_at)
);
CREATE INDEX IF NOT EXISTS idx_w_user_access_logs_user_id ON w_user_access_logs (user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_w_user_access_logs_created_at ON w_user_access_logs (created_at DESC);
-- +goose Down
DROP TABLE IF EXISTS w_user_access_logs;
DROP INDEX IF EXISTS idx_w_user_access_logs_user_id;
DROP INDEX IF EXISTS idx_w_user_access_logs_created_at;
@@ -0,0 +1,17 @@
-- +goose Up
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('log_database', '', 'system', 0, '当前日志主库(postgres/sqlite/clickhouse),由切换任务写入', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('log_db_migration', '', 'system', 0, '日志库迁移冻结标记(空或 migrating)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('log_retention_days_postgres', '30', 'business', 0, 'PostgreSQL 用户访问日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('log_retention_days_sqlite', '30', 'business', 0, 'SQLite 用户访问日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('log_retention_days_clickhouse', '30', 'business', 0, 'ClickHouse 用户访问日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
-- +goose Down
DELETE FROM w_system_configs WHERE key IN (
'log_database',
'log_db_migration',
'log_retention_days_postgres',
'log_retention_days_sqlite',
'log_retention_days_clickhouse'
);
@@ -0,0 +1,40 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_message_channels (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
owner_scope TEXT NOT NULL DEFAULT 'system',
owner_id INTEGER NULL,
enabled INTEGER NOT NULL DEFAULT 1,
credentials TEXT NOT NULL DEFAULT '',
extra TEXT NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_w_message_channels_type ON w_message_channels (type);
CREATE TABLE IF NOT EXISTS w_message_bindings (
id INTEGER PRIMARY KEY,
user_id INTEGER NOT NULL,
channel_id INTEGER NOT NULL,
platform_user_id TEXT NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX IF NOT EXISTS uniq_w_message_bindings_channel_platform
ON w_message_bindings (channel_id, platform_user_id);
CREATE INDEX IF NOT EXISTS idx_w_message_bindings_user ON w_message_bindings (user_id);
CREATE TABLE IF NOT EXISTS w_message_pairing_codes (
code TEXT PRIMARY KEY,
channel_id INTEGER NOT NULL,
platform_user_id TEXT NOT NULL,
expires_at DATETIME NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_w_message_pairing_lookup
ON w_message_pairing_codes (channel_id, platform_user_id);
-- +goose Down
DROP TABLE IF EXISTS w_message_pairing_codes;
DROP TABLE IF EXISTS w_message_bindings;
DROP TABLE IF EXISTS w_message_channels;
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package migrator 提供数据库迁移功能
package migrator
import (
"context"
"embed"
"log"
"github.com/Rain-kl/Wavelet/pkg/config"
"github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/pressly/goose/v3"
)
// migrationFS contains SQL migrations under goose/<dialect>.
//
//go:embed goose/postgres/*.sql goose/sqlite/*.sql
var migrationFS embed.FS
// dbType 返回当前数据库类型名称(用于日志输出)
func dbType() string {
if !config.Config.Database.Enabled {
return "SQLite"
}
return "PostgreSQL"
}
const (
dialectSqlite = "sqlite3"
dialectPostgres = "postgres"
cascadeSuffix = " CASCADE"
)
// Report describes the database migration state observed during startup.
type Report struct {
Backend string
Enabled bool
Version int64
Applied bool
}
func gooseDialect() string {
if !config.Config.Database.Enabled {
return dialectSqlite
}
return dialectPostgres
}
func migrationDir() string {
if !config.Config.Database.Enabled {
return "goose/sqlite"
}
return "goose/postgres"
}
// Migrate 执行数据库迁移
func Migrate() Report {
gormDB := db.DB(context.Background())
if gormDB == nil {
log.Fatalf("[%s] database not initialized\n", dbType())
}
sqlDB, err := gormDB.DB()
if err != nil {
log.Fatalf("[%s] load sql db failed: %v\n", dbType(), err)
}
goose.SetBaseFS(migrationFS)
if err := goose.SetDialect(gooseDialect()); err != nil {
log.Fatalf("[%s] set goose dialect failed: %v\n", dbType(), err)
}
previousVersion, err := goose.GetDBVersion(sqlDB)
if err != nil {
log.Fatalf("[%s] get goose version failed: %v\n", dbType(), err)
}
if err := goose.Up(sqlDB, migrationDir()); err != nil {
log.Fatalf("[%s] goose migrate failed: %v\n", dbType(), err)
}
clearSystemConfigCache()
currentVersion, err := goose.GetDBVersion(sqlDB)
if err != nil {
log.Fatalf("[%s] get migrated goose version failed: %v\n", dbType(), err)
}
log.Printf("[%s] goose migrate success\n", dbType())
return Report{
Backend: dbType(),
Enabled: true,
Version: currentVersion,
Applied: currentVersion != previousVersion,
}
}
func clearSystemConfigCache() {
}
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package migrator
import (
"testing"
"github.com/Rain-kl/Wavelet/pkg/config"
db "github.com/Rain-kl/Wavelet/pkg/persistence"
"github.com/alicebob/miniredis/v2"
"github.com/glebarez/sqlite"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
"gorm.io/gorm"
)
const expectedMigratedSystemConfigCount = 35
func TestMigrateInitializesSQLiteDatabase(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
if err != nil {
t.Fatalf("gorm.Open(sqlite) error = %v", err)
}
mr, err := miniredis.Run()
if err != nil {
t.Fatalf("miniredis.Run() error = %v", err)
}
redisClient := redis.NewClient(&redis.Options{
Addr: mr.Addr(),
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeDisabled,
},
})
previousDBEnabled := config.Config.Database.Enabled
config.Config.Database.Enabled = false
db.SetDB(sqliteDB)
t.Cleanup(func() {
config.Config.Database.Enabled = previousDBEnabled
db.SetDB(nil)
_ = redisClient.Close()
mr.Close()
})
Migrate()
var systemConfigCount int64
if err := sqliteDB.Table("w_system_configs").Count(&systemConfigCount).Error; err != nil {
t.Fatalf("Migrate() count w_system_configs error = %v", err)
}
if systemConfigCount != expectedMigratedSystemConfigCount {
t.Errorf("Migrate() w_system_configs count = %d, want %d", systemConfigCount, expectedMigratedSystemConfigCount)
}
var adminCount int64
if err := sqliteDB.Table("w_users").Where("username = ?", "admin").Count(&adminCount).Error; err != nil {
t.Fatalf("Migrate() count admin user error = %v", err)
}
if adminCount != 1 {
t.Errorf("Migrate() admin user count = %d, want %d", adminCount, 1)
}
var templateCount int64
if err := sqliteDB.Table("w_templates").Count(&templateCount).Error; err != nil {
t.Fatalf("Migrate() count templates error = %v", err)
}
if templateCount != 2 {
t.Errorf("Migrate() templates count = %d, want %d", templateCount, 2)
}
}
+217
View File
@@ -0,0 +1,217 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"log"
"net"
"net/url"
"strconv"
"time"
"github.com/Rain-kl/Wavelet/pkg/config"
"github.com/glebarez/sqlite"
"go.opentelemetry.io/otel/attribute"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/plugin/dbresolver"
"gorm.io/plugin/opentelemetry/tracing"
)
var (
db *gorm.DB
)
func init() {
if !config.Config.Database.Enabled {
// PostgreSQL 禁用,使用 SQLite
initSQLite()
return
}
initPostgres()
}
// initSQLite 初始化 SQLite 数据库(PostgreSQL 禁用时的后备方案)
func initSQLite() {
sqlitePath := config.Config.Database.SQLitePath
if sqlitePath == "" {
sqlitePath = "./data/wavelet.db"
}
var err error
db, err = gorm.Open(sqlite.Open(sqlitePath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: &gormZapLogger{
logLevel: parseLogLevel(config.Config.Database.LogLevel),
slowThreshold: config.Config.Database.SlowThreshold,
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
},
})
if err != nil {
log.Fatalf("[SQLite] init connection failed: %v\n", err)
}
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", sqlitePath),
attribute.String("db.system", "SQLite"),
),
),
); err != nil {
log.Fatalf("[SQLite] init trace failed: %v\n", err)
}
log.Printf("[SQLite] initialized (path: %s)\n", sqlitePath)
}
// initPostgres 初始化 PostgreSQL 数据库
func initPostgres() {
var err error
dbConfig := config.Config.Database
// 构建主库 DSN 并连接
primaryDSN := buildDSN(dbConfig.Host, dbConfig.Port, dbConfig.Username, dbConfig.Password)
pgConfig := postgres.Config{
DSN: primaryDSN,
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
}
db, err = gorm.Open(postgres.New(pgConfig), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: &gormZapLogger{
logLevel: parseLogLevel(config.Config.Database.LogLevel),
slowThreshold: config.Config.Database.SlowThreshold,
ignoreRecordNotFoundError: config.Config.App.IsProduction(),
},
})
if err != nil {
log.Fatalf("[PostgreSQL] init connection failed: %v\n", err)
}
// Trace 注入
if err = db.Use(
tracing.NewPlugin(
tracing.WithoutMetrics(),
tracing.WithAttributes(
attribute.String("db.instance", dbConfig.Database),
attribute.String("db.ip", dbConfig.Host),
attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))),
attribute.String("db.system", "PostgreSQL"),
),
),
); err != nil {
log.Fatalf("[PostgreSQL] init trace failed: %v\n", err)
}
if len(dbConfig.Replicas) > 0 {
var replicaDialectors []gorm.Dialector
for _, replica := range dbConfig.Replicas {
username := replica.Username
if username == "" {
username = dbConfig.Username
}
password := replica.Password
if password == "" {
password = dbConfig.Password
}
replicaDSN := buildDSN(replica.Host, replica.Port, username, password)
replicaDialectors = append(replicaDialectors, postgres.New(postgres.Config{
DSN: replicaDSN,
PreferSimpleProtocol: dbConfig.PreferSimpleProtocol,
}))
}
resolver := dbresolver.Register(dbresolver.Config{
Replicas: replicaDialectors,
Policy: dbresolver.RandomPolicy{},
})
resolver.SetMaxIdleConns(dbConfig.MaxIdleConn).
SetMaxOpenConns(dbConfig.MaxOpenConn).
SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second).
SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
if err = db.Use(resolver); err != nil {
log.Fatalf("[PostgreSQL] init dbresolver failed: %v\n", err)
}
log.Printf("[PostgreSQL] initialized in Primary-Replica mode (%d replicas)\n", len(dbConfig.Replicas))
} else {
log.Println("[PostgreSQL] initialized in Standalone mode")
}
// 获取通用数据库对象设置连接池
sqlDB, err := db.DB()
if err != nil {
log.Fatalf("[PostgreSQL] load sql db failed: %v\n", err)
}
sqlDB.SetMaxIdleConns(dbConfig.MaxIdleConn)
sqlDB.SetMaxOpenConns(dbConfig.MaxOpenConn)
sqlDB.SetConnMaxLifetime(time.Duration(dbConfig.ConnMaxLifetime) * time.Second)
sqlDB.SetConnMaxIdleTime(time.Duration(dbConfig.ConnMaxIdleTime) * time.Second)
}
// buildDSN 构建 PostgreSQL DSN
func buildDSN(host string, port int, username, password string) string {
cfg := config.Config.Database
pqURL := &url.URL{
Scheme: "postgres",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
Path: cfg.Database,
}
if username != "" {
pqURL.User = url.UserPassword(username, password)
}
query := pqURL.Query()
sslMode := cfg.SSLMode
if sslMode == "" {
sslMode = "disable"
}
query.Set("sslmode", sslMode)
if cfg.ApplicationName != "" {
query.Set("application_name", cfg.ApplicationName)
}
if cfg.SearchPath != "" {
query.Set("search_path", cfg.SearchPath)
}
if cfg.DefaultQueryExecMode != "" {
query.Set("default_query_exec_mode", cfg.DefaultQueryExecMode)
}
if cfg.StatementCacheCapacity > 0 {
query.Set("statement_cache_capacity", strconv.Itoa(cfg.StatementCacheCapacity))
}
rawQuery := query.Encode()
if cfg.TimeZone != "" {
if rawQuery != "" {
rawQuery += "&"
}
rawQuery += "TimeZone=" + cfg.TimeZone
}
pqURL.RawQuery = rawQuery
return pqURL.String()
}
// DB 返回带上下文追踪的 GORM 数据库实例
func DB(ctx context.Context) *gorm.DB {
if db == nil {
return nil
}
return db.WithContext(ctx)
}
// SetDB sets the package-level database instance for testing.
func SetDB(d *gorm.DB) {
db = d
}
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
gormLogger "gorm.io/gorm/logger"
)
// nanoToMilli 纳秒转毫秒的除数
const nanoToMilli = 1e6
type gormZapLogger struct {
logLevel gormLogger.LogLevel
ignoreRecordNotFoundError bool
slowThreshold time.Duration
}
func (l *gormZapLogger) LogMode(level gormLogger.LogLevel) gormLogger.Interface {
clone := *l
clone.logLevel = level
return &clone
}
func (l *gormZapLogger) Info(ctx context.Context, fmt string, args ...interface{}) {
if l.logLevel >= gormLogger.Info {
logger.InfoF(ctx, fmt, args...)
}
}
func (l *gormZapLogger) Warn(ctx context.Context, fmt string, args ...interface{}) {
if l.logLevel >= gormLogger.Warn {
logger.WarnF(ctx, fmt, args...)
}
}
func (l *gormZapLogger) Error(ctx context.Context, fmt string, args ...interface{}) {
if l.logLevel >= gormLogger.Error {
logger.ErrorF(ctx, fmt, args...)
}
}
func (l *gormZapLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
elapsed := time.Since(begin)
switch {
case err != nil && l.logLevel >= gormLogger.Error && (!errors.Is(err, gorm.ErrRecordNotFound) || !l.ignoreRecordNotFoundError):
_, rows := fc()
logger.ErrorF(ctx, "database query failed: %s [%.3fms] [rows:%v]", err, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
case elapsed > l.slowThreshold && l.slowThreshold != 0 && l.logLevel >= gormLogger.Warn:
_, rows := fc()
slowLog := fmt.Sprintf("SLOW SQL >= %v", l.slowThreshold)
logger.WarnF(ctx, "%s [%.3fms] [rows:%v]", slowLog, float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows))
case l.logLevel == gormLogger.Info:
sql, rows := fc()
logger.DebugF(ctx, "[%.3fms] [rows:%v] %s", float64(elapsed.Nanoseconds())/nanoToMilli, formatRows(rows), sql)
}
}
func formatRows(rows int64) interface{} {
if rows == -1 {
return "-"
}
return rows
}
func parseLogLevel(level string) gormLogger.LogLevel {
level = strings.ToLower(level)
switch level {
case "silent":
return gormLogger.Silent
case "error":
return gormLogger.Error
case "warn":
return gormLogger.Warn
case "info":
return gormLogger.Info
case "debug":
return gormLogger.Info
default:
return gormLogger.Info
}
}
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"testing"
gormLogger "gorm.io/gorm/logger"
)
func TestParseLogLevel(t *testing.T) {
t.Parallel()
tests := []struct {
name string
configuredLevel string
want gormLogger.LogLevel
}{
{
name: "debug enables SQL trace processing",
configuredLevel: "debug",
want: gormLogger.Info,
},
{
name: "development preserves configured level",
configuredLevel: "warn",
want: gormLogger.Warn,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := parseLogLevel(tt.configuredLevel); got != tt.want {
t.Fatalf("parseLogLevel() = %v, want %v", got, tt.want)
}
})
}
}
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"time"
"github.com/Rain-kl/Wavelet/pkg/config"
"github.com/redis/go-redis/extra/redisotel/v9"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
"go.opentelemetry.io/otel/attribute"
)
var (
// Redis 全局 Redis 客户端实例
Redis redis.UniversalClient
)
func init() {
cfg := config.Config.Redis
if !cfg.Enabled {
log.Println("[Redis] is disabled, skipping Redis initialization")
return
}
if cfg.ClusterMode {
// Cluster 模式
Redis = redis.NewClusterClient(&redis.ClusterOptions{
Addrs: cfg.Addrs,
Username: cfg.Username,
Password: cfg.Password,
PoolSize: cfg.PoolSize,
MinIdleConns: cfg.MinIdleConn,
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
MaxRetries: cfg.MaxRetries,
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
})
log.Println("[Redis] initialized in Cluster mode")
} else {
// Standalone 或 Sentinel 模式
options := &redis.UniversalOptions{
Addrs: cfg.Addrs,
MasterName: cfg.MasterName, // 非空时启用 Sentinel
Username: cfg.Username,
Password: cfg.Password,
DB: cfg.DB,
PoolSize: cfg.PoolSize,
MinIdleConns: cfg.MinIdleConn,
DialTimeout: time.Duration(cfg.DialTimeout) * time.Second,
ReadTimeout: time.Duration(cfg.ReadTimeout) * time.Second,
WriteTimeout: time.Duration(cfg.WriteTimeout) * time.Second,
MaxRetries: cfg.MaxRetries,
PoolTimeout: time.Duration(cfg.PoolTimeout) * time.Second,
ConnMaxIdleTime: time.Duration(cfg.ConnMaxIdleTime) * time.Second,
MaintNotificationsConfig: redisMaintNotificationsConfig(cfg.MaintNotifications),
}
if cfg.MasterName != "" {
client := redis.NewFailoverClient(options.Failover())
// FailoverOptions 暂不暴露该配置,在首次建连前写入客户端选项。
client.Options().MaintNotificationsConfig = redisMaintNotificationsConfig(cfg.MaintNotifications)
Redis = client
log.Println("[Redis] initialized in Sentinel mode")
} else {
Redis = redis.NewUniversalClient(options)
log.Println("[Redis] initialized in Standalone mode")
}
}
// OpenTelemetry 追踪(UniversalClient 兼容)
if err := redisotel.InstrumentTracing(
Redis,
redisotel.WithAttributes(
attribute.String("db.instance", fmt.Sprintf("%v", cfg.DB)),
attribute.String("db.ip", strings.Join(cfg.Addrs, ",")),
attribute.String("db.system", "Redis"),
),
); err != nil {
log.Fatalf("[Redis] failed to init trace: %v\n", err)
}
// 测试连接
_, err := Redis.Ping(context.Background()).Result()
if err != nil {
log.Fatalf("[Redis] failed to connect to redis: %v\n", err)
}
}
func redisMaintNotificationsConfig(enabled bool) *maintnotifications.Config {
mode := maintnotifications.ModeDisabled
if enabled {
mode = maintnotifications.ModeAuto
}
return &maintnotifications.Config{Mode: mode}
}
// PrefixedKey 返回带前缀的 Key
func PrefixedKey(key string) string {
prefix := config.Config.Redis.KeyPrefix
if prefix == "" {
return key
}
return prefix + key
}
// HSetJSON 将泛型数据序列化为 JSON 并设置到 Redis Hash
// ctx: 上下文
// hashKey: Redis Hash key
// fieldKey: Hash field key
// data: 要存储的数据(泛型)
func HSetJSON[T any](ctx context.Context, hashKey, fieldKey string, data T) error {
jsonData, err := json.Marshal(data)
if err != nil {
return err
}
if err := Redis.HSet(ctx, PrefixedKey(hashKey), fieldKey, jsonData).Err(); err != nil {
return fmt.Errorf(errRedisHashSetFailed, err)
}
return nil
}
// HDel removes one or more fields from a Redis Hash.
func HDel(ctx context.Context, hashKey string, fieldKeys ...string) error {
if Redis == nil || len(fieldKeys) == 0 {
return nil
}
if err := Redis.HDel(ctx, PrefixedKey(hashKey), fieldKeys...).Err(); err != nil {
return fmt.Errorf(errRedisHashDeleteFailed, err)
}
return nil
}
// HGetJSON 从 Redis Hash 获取数据并反序列化为泛型类型
// ctx: 上下文
// hashKey: Redis Hash key
// fieldKey: Hash field key
// data: 用于接收数据的指针(泛型)
func HGetJSON[T any](ctx context.Context, hashKey, fieldKey string, data *T) error {
val, err := Redis.HGet(ctx, PrefixedKey(hashKey), fieldKey).Result()
if err != nil {
return err
}
if err := json.Unmarshal([]byte(val), data); err != nil {
return fmt.Errorf(errUnmarshalDataFailed, err)
}
return nil
}
// GetJSON 从Redis获取数据并反序列化为泛型类型
// ctx: 上下文
// key: Redis key
// data: 用于接收数据的指针(泛型)
func GetJSON[T any](ctx context.Context, key string, data *T) error {
val, err := Redis.Get(ctx, PrefixedKey(key)).Bytes()
if err != nil {
return err
}
if err := json.Unmarshal(val, data); err != nil {
return fmt.Errorf(errUnmarshalDataFailed, err)
}
return nil
}
// SetJSON 将泛型数据序列化为JSON并设置到Redis
// ctx: 上下文
// key: Redis key
// data: 要存储的数据(泛型)
// expiration: 过期时间
func SetJSON[T any](ctx context.Context, key string, data T, expiration time.Duration) error {
jsonData, err := json.Marshal(data)
if err != nil {
return fmt.Errorf(errMarshalDataFailed, err)
}
if err := Redis.Set(ctx, PrefixedKey(key), jsonData, expiration).Err(); err != nil {
return fmt.Errorf(errRedisKeySetFailed, err)
}
return nil
}
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"testing"
"github.com/redis/go-redis/v9/maintnotifications"
)
func TestRedisMaintNotificationsConfig(t *testing.T) {
for _, test := range []struct {
name string
enabled bool
want maintnotifications.Mode
}{
{name: "disabled by default", enabled: false, want: maintnotifications.ModeDisabled},
{name: "auto when enabled", enabled: true, want: maintnotifications.ModeAuto},
} {
t.Run(test.name, func(t *testing.T) {
cfg := redisMaintNotificationsConfig(test.enabled)
if cfg.Mode != test.want {
t.Fatalf("maintenance notifications mode = %v, want %v", cfg.Mode, test.want)
}
})
}
}
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package response
import (
"net/http"
"github.com/gin-gonic/gin"
)
// AbortBadRequest 以 400 中断请求并将错误挂载到 Gin Error 链,供全局中间件统一记录 Trace 并响应。
func AbortBadRequest(c *gin.Context, msg string) {
AbortWithError(c, http.StatusBadRequest, msg)
}
// AbortUnauthorized 以 401 中断请求并将错误挂载到 Gin Error 链。
func AbortUnauthorized(c *gin.Context, msg string) {
AbortWithError(c, http.StatusUnauthorized, msg)
}
// AbortForbidden 以 403 中断请求并将错误挂载到 Gin Error 链。
func AbortForbidden(c *gin.Context, msg string) {
AbortWithError(c, http.StatusForbidden, msg)
}
// AbortNotFound 以 404 中断请求并将错误挂载到 Gin Error 链。
func AbortNotFound(c *gin.Context, msg string) {
AbortWithError(c, http.StatusNotFound, msg)
}
// AbortInternal 以 500 中断请求并将错误挂载到 Gin Error 链。
func AbortInternal(c *gin.Context, msg string) {
AbortWithError(c, http.StatusInternalServerError, msg)
}
// AbortTooManyRequests 以 429 中断请求并将错误挂载到 Gin Error 链。
func AbortTooManyRequests(c *gin.Context, msg string) {
AbortWithError(c, http.StatusTooManyRequests, msg)
}
// AbortConflict 以 409 中断请求并将错误挂载到 Gin Error 链。
func AbortConflict(c *gin.Context, msg string) {
AbortWithError(c, http.StatusConflict, msg)
}
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package response
import (
"errors"
"net/http"
"github.com/gin-gonic/gin"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
// ErrorHandlerMiddleware 捕获 c.Errors 并统一格式化为 JSON 返回给客户端,同时将其记录到 Span 异常中。
// 与 AbortWithError / AbortBadRequest 等配合使用,是全局 OTel 友好错误响应的唯一出口。
func ErrorHandlerMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
c.Next()
if len(c.Errors) == 0 || c.Writer.Written() {
return
}
err := c.Errors.Last().Err
span := trace.SpanFromContext(c.Request.Context())
if span.IsRecording() {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
}
var apiErr *APIError
if errors.As(err, &apiErr) {
c.JSON(apiErr.Code, Err(apiErr.Msg))
return
}
c.JSON(http.StatusInternalServerError, Err("内部系统错误"))
}
}
+168
View File
@@ -0,0 +1,168 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package response
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
)
func init() {
gin.SetMode(gin.TestMode)
}
func TestAbortWithError(t *testing.T) {
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
AbortWithError(c, http.StatusBadRequest, "invalid input")
require.Len(t, c.Errors, 1)
var apiErr *APIError
require.True(t, errors.As(c.Errors.Last().Err, &apiErr))
assert.Equal(t, http.StatusBadRequest, apiErr.Code)
assert.Equal(t, "invalid input", apiErr.Msg)
assert.True(t, c.IsAborted())
}
func TestErrorHandlerMiddleware_APIErrorStatusCodes(t *testing.T) {
cases := []struct {
name string
statusCode int
message string
abort func(*gin.Context, string)
}{
{"400 Bad Request", http.StatusBadRequest, "bad request", AbortBadRequest},
{"401 Unauthorized", http.StatusUnauthorized, "unauthorized", AbortUnauthorized},
{"403 Forbidden", http.StatusForbidden, "forbidden", AbortForbidden},
{"404 Not Found", http.StatusNotFound, "not found", AbortNotFound},
{"409 Conflict", http.StatusConflict, "conflict", AbortConflict},
{"429 Too Many Requests", http.StatusTooManyRequests, "too many requests", AbortTooManyRequests},
{"500 Internal Server Error", http.StatusInternalServerError, "internal error", AbortInternal},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := gin.New()
r.Use(ErrorHandlerMiddleware())
r.GET("/test", func(c *gin.Context) {
tc.abort(c, tc.message)
})
req := httptest.NewRequest(http.MethodGet, "/test", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
assert.Equal(t, tc.statusCode, w.Code)
assert.Equal(t, "application/json; charset=utf-8", w.Header().Get("Content-Type"))
var body Response[any]
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
assert.Equal(t, tc.message, body.ErrorMsg)
assert.Nil(t, body.Data)
})
}
}
func TestErrorHandlerMiddleware_SkipsWhenNoErrors(t *testing.T) {
r := gin.New()
r.Use(ErrorHandlerMiddleware())
r.GET("/ok", func(c *gin.Context) {
c.JSON(http.StatusOK, OK("success"))
})
req := httptest.NewRequest(http.MethodGet, "/ok", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
assert.Equal(t, http.StatusOK, w.Code)
var body Response[string]
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
assert.Equal(t, "success", body.Data)
assert.Empty(t, body.ErrorMsg)
}
func TestErrorHandlerMiddleware_SkipsWhenResponseAlreadyWritten(t *testing.T) {
r := gin.New()
r.Use(ErrorHandlerMiddleware())
r.GET("/written", func(c *gin.Context) {
c.JSON(http.StatusOK, OKNil())
_ = c.Error(NewError(http.StatusBadRequest, "should not overwrite"))
})
req := httptest.NewRequest(http.MethodGet, "/written", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
assert.Equal(t, http.StatusOK, w.Code)
var body Response[any]
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
assert.Empty(t, body.ErrorMsg)
assert.Nil(t, body.Data)
}
func TestErrorHandlerMiddleware_FallbackForNonAPIError(t *testing.T) {
r := gin.New()
r.Use(ErrorHandlerMiddleware())
r.GET("/plain", func(c *gin.Context) {
_ = c.Error(errors.New("plain error"))
})
req := httptest.NewRequest(http.MethodGet, "/plain", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
assert.Equal(t, http.StatusInternalServerError, w.Code)
var body Response[any]
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
assert.Equal(t, "内部系统错误", body.ErrorMsg)
assert.Nil(t, body.Data)
}
func TestErrorHandlerMiddleware_RecordsSpanOnAPIError(t *testing.T) {
sr := tracetest.NewSpanRecorder()
tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr))
otel.SetTracerProvider(tp)
defer otel.SetTracerProvider(trace.NewNoopTracerProvider())
tracer := tp.Tracer("test")
ctx, span := tracer.Start(context.Background(), "request")
r := gin.New()
r.Use(ErrorHandlerMiddleware())
r.GET("/err", func(c *gin.Context) {
c.Request = c.Request.WithContext(ctx)
AbortBadRequest(c, "bad request")
})
req := httptest.NewRequest(http.MethodGet, "/err", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
span.End()
require.Equal(t, http.StatusBadRequest, w.Code)
spans := sr.Ended()
require.Len(t, spans, 1)
assert.Equal(t, codes.Error, spans[0].Status().Code)
assert.Equal(t, "bad request", spans[0].Status().Description)
require.NotEmpty(t, spans[0].Events())
}
+57
View File
@@ -0,0 +1,57 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package response provides shared HTTP API response structures.
package response
import "github.com/gin-gonic/gin"
// Response 通用响应体
type Response[T any] struct {
ErrorMsg string `json:"error_msg"`
Data T `json:"data"`
}
// Any 用于 Swagger 文档的响应类型(非泛型)
// swag 不支持泛型,使用此类型替代 Response[T]
type Any struct {
ErrorMsg string `json:"error_msg" example:""`
Data interface{} `json:"data"`
}
// APIError 统一的 API 业务错误类型,可被全局错误处理中间件捕获
type APIError struct {
Code int
Msg string
}
func (e *APIError) Error() string {
return e.Msg
}
// NewError 实例化一个 APIError
func NewError(code int, msg string) *APIError {
return &APIError{Code: code, Msg: msg}
}
// AbortWithError 将 API 错误挂载到 Gin Context 并中断执行流
func AbortWithError(c *gin.Context, code int, msg string) {
_ = c.Error(NewError(code, msg))
c.Abort()
}
// OK 构造成功响应
func OK[T any](data T) Response[T] {
return Response[T]{Data: data}
}
// OKNil 构造成功响应(data 为 null)
func OKNil() Response[any] {
return Response[any]{Data: nil}
}
// Err 构造错误响应
func Err(msg string) Response[any] {
return Response[any]{ErrorMsg: msg, Data: nil}
}
+6
View File
@@ -0,0 +1,6 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package shared 提供跨模块共享的常量、错误定义和通用类型。
package shared
+11
View File
@@ -0,0 +1,11 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package shared
// 通用业务错误消息常量
const (
BannedAccount = "账号已被封禁"
UnAuthorized = "未登录"
)
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package task 定义异步任务类型与调度常量
package task
// 任务队列名称
const (
QueueDefault = "default"
)
// DefaultMaxRetry 任务默认最大重试次数
const DefaultMaxRetry = 3
+17
View File
@@ -0,0 +1,17 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
const (
errUnknownTaskType = "未知的任务类型: %s"
errCreateTaskExecutionFailed = "创建任务执行记录失败: %w"
errTaskEnqueueFailed = "任务入队失败: %w"
errTaskExecutionNotFound = "任务执行记录不存在: %w"
errRetryOnlyFailedTask = "只有失败的任务才能重试,当前状态: %s"
errTaskNotRetryable = "该任务不支持重试"
errTaskMaxRetryExceeded = "已达到最大重试次数 %d"
errCreateRetryExecutionFailed = "创建重试任务执行记录失败: %w"
errRetryTaskEnqueueFailed = "重试任务入队失败: %w"
errUnregisteredTaskHandler = "未注册的任务处理器: %s"
)
+482
View File
@@ -0,0 +1,482 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/Rain-kl/Wavelet/pkg/persistence/idgen"
otel_trace "github.com/Rain-kl/Wavelet/pkg/trace"
"github.com/hibiken/asynq"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
// handlerRegistry 已注册的任务处理器
var handlerRegistry = make(map[string]TaskHandler)
// CompletedHandler is called when a task execution completes.
type CompletedHandler func(ctx context.Context, execution *TaskExecution, result *TaskResult, execErr error)
var taskCompletedHandlers []CompletedHandler
// OnTaskCompleted registers a handler for task completion events.
// Handlers must be registered during application bootstrap before processing tasks.
func OnTaskCompleted(handler CompletedHandler) {
taskCompletedHandlers = append(taskCompletedHandlers, handler)
}
// RegisterHandler 注册任务处理器
// 传入任务类型标识(对应 constants.go 中的 AsynqTask 常量)和 TaskHandler 实现
func RegisterHandler(asynqTaskType string, handler TaskHandler) {
handlerRegistry[asynqTaskType] = handler
}
// getHandler 获取已注册的处理器
func getHandler(asynqTaskType string) (TaskHandler, bool) {
h, ok := handlerRegistry[asynqTaskType]
return h, ok
}
// ValidateAndNormalizePayload 校验并标准化任务参数。
// 如果 Handler 实现了 PayloadValidator,调用其 ValidatePayload 方法;
// 否则直接返回原始 payload。
func ValidateAndNormalizePayload(asynqTaskType string, payload []byte) ([]byte, error) {
handler, ok := getHandler(asynqTaskType)
if !ok {
return payload, nil
}
if validator, ok := handler.(PayloadValidator); ok {
return validator.ValidatePayload(payload)
}
return payload, nil
}
// contextKey 用于 context 存取 taskID
type contextKey string
const taskIDKey contextKey = "task_execution_task_id"
const traceEnvelopeVersion = 1
type traceEnvelope struct {
WaveletTraceEnvelope bool `json:"_wavelet_trace_envelope"`
Version int `json:"version"`
TraceContext map[string]string `json:"trace_context,omitempty"`
Payload []byte `json:"payload"`
}
// withTaskID 将 taskID 注入 context
func withTaskID(ctx context.Context, taskID string) context.Context {
return context.WithValue(ctx, taskIDKey, taskID)
}
// GetTaskID 从 context 中获取 taskID
func GetTaskID(ctx context.Context) string {
if v, ok := ctx.Value(taskIDKey).(string); ok {
return v
}
return ""
}
// IsFinalAttempt 判断当前任务执行是否为最后一次重试尝试(若再次失败即为最终失败)
func IsFinalAttempt(ctx context.Context) bool {
retryCount, hasRetryCount := asynq.GetRetryCount(ctx)
maxRetry, hasMaxRetry := asynq.GetMaxRetry(ctx)
if !hasRetryCount || !hasMaxRetry {
return true
}
return retryCount >= maxRetry
}
// AppendLog 追加日志到任务执行记录
// 在 TaskHandler.Execute 中调用,日志会自动追加到 TaskExecution.Log 字段
func AppendLog(ctx context.Context, format string, args ...interface{}) {
taskID := GetTaskID(ctx)
if taskID == "" {
// 上下文中没有 taskID,降级到普通日志
logger.InfoF(ctx, format, args...)
return
}
logLine := fmt.Sprintf(format, args...)
if err := appendTaskExecutionLog(ctx, taskID, logLine); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 追加任务日志失败 taskID=%s: %v", taskID, err)
}
}
// DispatchTask 下发任务(创建 TaskExecution 记录 → 入队 Asynq)
func DispatchTask(ctx context.Context, taskType string, payload []byte, triggeredBy string) (string, error) {
meta := GetTaskMeta(taskType)
if meta == nil {
return "", fmt.Errorf(errUnknownTaskType, taskType)
}
// 生成唯一的 TaskID
taskID := generateTaskID(taskType, triggeredBy)
// 创建任务执行记录
execution := &TaskExecution{
TaskID: taskID,
TaskType: meta.AsynqTask,
TaskName: meta.Name,
Status: TaskExecutionStatusPending,
Retryable: meta.Retryable,
MaxRetry: meta.MaxRetry,
RetryCount: 0,
Payload: string(payload),
TriggeredBy: triggeredBy,
}
if err := createTaskExecution(ctx, execution); err != nil {
return "", fmt.Errorf(errCreateTaskExecutionFailed, err)
}
// 入队 Asynq
taskInfo := asynq.NewTask(meta.AsynqTask, injectTaskTraceContext(ctx, payload))
if _, err := AsynqClient.Enqueue(
taskInfo,
asynq.TaskID(taskID),
asynq.MaxRetry(meta.MaxRetry),
asynq.Queue(meta.Queue),
); err != nil {
// 入队失败,更新执行记录状态
execution.Status = TaskExecutionStatusFailed
execution.ErrorMessage = fmt.Sprintf("入队失败: %v", err)
now := time.Now()
execution.StartedAt = &now
execution.FinishedAt = &now
_ = updateTaskExecution(ctx, execution)
return "", fmt.Errorf(errTaskEnqueueFailed, err)
}
if err := appendTaskExecutionLog(ctx, taskID, fmt.Sprintf("[系统] 任务已成功入队,等待调度执行 (队列: %s, 最大重试次数: %d)", meta.Queue, meta.MaxRetry)); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 追加入队日志失败 taskID=%s: %v", taskID, err)
}
return taskID, nil
}
// RetryTask 重试失败的任务
func RetryTask(ctx context.Context, id uint64) (string, error) {
execution, err := getTaskExecutionByID(ctx, id)
if err != nil {
return "", fmt.Errorf(errTaskExecutionNotFound, err)
}
if execution.Status != TaskExecutionStatusFailed {
return "", fmt.Errorf(errRetryOnlyFailedTask, execution.Status)
}
if !execution.Retryable {
return "", errors.New(errTaskNotRetryable)
}
// 生成新的 TaskID
newTaskID := generateRetryTaskID(execution.TaskID, execution.RetryCount+1)
// 创建新的执行记录
newExecution := &TaskExecution{
TaskID: newTaskID,
TaskType: execution.TaskType,
TaskName: execution.TaskName,
Status: TaskExecutionStatusPending,
Retryable: execution.Retryable,
MaxRetry: execution.MaxRetry,
RetryCount: execution.RetryCount + 1,
Payload: execution.Payload,
TriggeredBy: "retry",
}
if err := createTaskExecution(ctx, newExecution); err != nil {
return "", fmt.Errorf(errCreateRetryExecutionFailed, err)
}
meta := GetTaskMeta(execution.TaskType)
queueName := QueueDefault
if meta != nil {
queueName = meta.Queue
}
// 入队 Asynq
taskInfo := asynq.NewTask(execution.TaskType, injectTaskTraceContext(ctx, []byte(execution.Payload)))
if _, err := AsynqClient.Enqueue(
taskInfo,
asynq.TaskID(newTaskID),
asynq.MaxRetry(execution.MaxRetry),
asynq.Queue(queueName),
); err != nil {
newExecution.Status = TaskExecutionStatusFailed
newExecution.ErrorMessage = fmt.Sprintf("重试入队失败: %v", err)
now := time.Now()
newExecution.StartedAt = &now
newExecution.FinishedAt = &now
_ = updateTaskExecution(ctx, newExecution)
return "", fmt.Errorf(errRetryTaskEnqueueFailed, err)
}
if err := appendTaskExecutionLog(ctx, newTaskID, fmt.Sprintf("[系统] 手动触发重试,已重新创建任务并入队 (原任务ID: %s, 重试次数: %d/%d)", execution.TaskID, execution.RetryCount+1, execution.MaxRetry)); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 追加重试日志失败 taskID=%s: %v", newTaskID, err)
}
return newTaskID, nil
}
// ProcessTask Asynq 实际调用的统一处理函数
// Worker 注册时统一使用此函数,内部自动分发到对应的 TaskHandler
func ProcessTask(ctx context.Context, t *asynq.Task) error {
taskPayload := t.Payload()
ctx, taskPayload, hasRemoteTraceContext := extractTaskTraceContext(ctx, taskPayload)
// 初始化 Trace
ctx, span := otel_trace.Start(ctx, "TaskProcess_"+t.Type(), trace.WithSpanKind(trace.SpanKindConsumer))
defer span.End()
// 添加任务信息到 Span
span.SetAttributes(
attribute.String("task.type", t.Type()),
attribute.Int("task.payload_size", len(taskPayload)),
attribute.Bool("task.trace_context_propagated", hasRemoteTraceContext),
attribute.String("task.id", t.ResultWriter().TaskID()),
)
taskID := t.ResultWriter().TaskID()
// 注入 taskID 到 context
ctx = withTaskID(ctx, taskID)
// 查找处理器
handler, ok := getHandler(t.Type())
if !ok {
err := fmt.Errorf(errUnregisteredTaskHandler, t.Type())
logger.ErrorF(ctx, "[TaskExecutor] %v", err)
span.SetStatus(codes.Error, err.Error())
return err
}
// 加载或动态创建执行记录
now := time.Now()
execution, err := getOrCreateTaskExecution(ctx, taskID, t, taskPayload, now)
if err == nil {
updateExecutionOnStart(ctx, execution, now)
}
if execution != nil {
AppendLog(ctx, "[系统] 开始执行异步任务 [名称: %s, 类型: %s],重试次数: %d/%d",
execution.TaskName, t.Type(), execution.RetryCount, execution.MaxRetry)
} else {
AppendLog(ctx, "[系统] 开始执行异步任务 [类型: %s]", t.Type())
}
// 开始计时
start := time.Now()
// 执行业务逻辑
result, execErr := handler.Execute(ctx, taskPayload)
// 计算耗时并归档记录
duration := time.Since(start)
finishTime := time.Now()
completeTaskExecution(ctx, execution, t, duration, finishTime, result, execErr, span)
if execution == nil && execErr != nil {
span.SetStatus(codes.Error, execErr.Error())
return execErr
}
return execErr
}
func injectTaskTraceContext(ctx context.Context, payload []byte) []byte {
carrier := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(ctx, carrier)
if len(carrier) == 0 {
return payload
}
envelope := traceEnvelope{
WaveletTraceEnvelope: true,
Version: traceEnvelopeVersion,
TraceContext: map[string]string(carrier),
Payload: payload,
}
data, err := json.Marshal(envelope)
if err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 序列化任务 Trace 上下文失败: %v", err)
return payload
}
return data
}
func extractTaskTraceContext(ctx context.Context, payload []byte) (context.Context, []byte, bool) {
var envelope traceEnvelope
if err := json.Unmarshal(payload, &envelope); err != nil {
return ctx, payload, false
}
if !envelope.WaveletTraceEnvelope || envelope.Version != traceEnvelopeVersion {
return ctx, payload, false
}
if len(envelope.TraceContext) == 0 {
return ctx, envelope.Payload, false
}
extractedCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(envelope.TraceContext))
return extractedCtx, envelope.Payload, true
}
func updateExecutionOnStart(ctx context.Context, execution *TaskExecution, now time.Time) {
if execution == nil {
return
}
dirty := false
if retryCount, hasRetry := asynq.GetRetryCount(ctx); hasRetry && execution.RetryCount != retryCount {
execution.RetryCount = retryCount
dirty = true
}
if execution.Status != TaskExecutionStatusRunning {
execution.Status = TaskExecutionStatusRunning
execution.StartedAt = &now
dirty = true
}
if dirty {
if updateErr := updateTaskExecution(ctx, execution); updateErr != nil {
logger.ErrorF(ctx, "[TaskExecutor] 更新执行状态失败 taskID=%s: %v", execution.TaskID, updateErr)
}
}
}
// getOrCreateTaskExecution 获取已有的任务执行记录,如果不存在则针对已知任务类型动态创建记录
func getOrCreateTaskExecution(ctx context.Context, taskID string, t *asynq.Task, payload []byte, now time.Time) (*TaskExecution, error) {
execution, err := getTaskExecutionByTaskID(ctx, taskID)
if err == nil {
return execution, nil
}
meta := GetTaskMetaByAsynqTask(t.Type())
if meta == nil {
return nil, err
}
execution = &TaskExecution{
TaskID: taskID,
TaskType: meta.AsynqTask,
TaskName: meta.Name,
Status: TaskExecutionStatusRunning,
Retryable: meta.Retryable,
MaxRetry: meta.MaxRetry,
RetryCount: 0,
Payload: string(payload),
TriggeredBy: "schedule",
StartedAt: &now,
}
if createErr := createTaskExecution(ctx, execution); createErr != nil {
logger.ErrorF(ctx, "[TaskExecutor] 动态创建执行记录失败 taskID=%s: %v", taskID, createErr)
return nil, createErr
}
return execution, nil
}
// completeTaskExecution 完成并更新任务执行记录的状态和执行结果
func completeTaskExecution(ctx context.Context, execution *TaskExecution, t *asynq.Task, duration time.Duration, finishTime time.Time, result *TaskResult, execErr error, span trace.Span) {
if execution == nil {
return
}
execution.Duration = duration.Milliseconds()
execution.FinishedAt = &finishTime
if execErr != nil {
handleFailedTask(ctx, execution, t, duration, execErr, span)
} else {
handleSuccessfulTask(ctx, execution, t, duration, result)
}
if err := updateTaskExecution(ctx, execution); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 更新执行记录失败 taskID=%s: %v", execution.TaskID, err)
}
if shouldFlushTaskExecutionLog(ctx, execErr) {
if err := flushTaskExecutionLog(ctx, execution.TaskID); err != nil {
logger.ErrorF(ctx, "[TaskExecutor] 持久化任务日志失败 taskID=%s: %v", execution.TaskID, err)
}
}
notifyTaskCompleted(ctx, execution, result, execErr)
}
func notifyTaskCompleted(ctx context.Context, execution *TaskExecution, result *TaskResult, execErr error) {
if len(taskCompletedHandlers) == 0 {
return
}
asyncCtx := context.WithoutCancel(ctx)
for _, handler := range taskCompletedHandlers {
go handler(asyncCtx, execution, result, execErr)
}
}
func shouldFlushTaskExecutionLog(ctx context.Context, execErr error) bool {
if isTerminalTaskExecutionError(execErr) {
return true
}
retryCount, hasRetryCount := asynq.GetRetryCount(ctx)
maxRetry, hasMaxRetry := asynq.GetMaxRetry(ctx)
if !hasRetryCount || !hasMaxRetry {
return true
}
return retryCount >= maxRetry
}
func isTerminalTaskExecutionError(execErr error) bool {
return execErr == nil || errors.Is(execErr, asynq.SkipRetry)
}
func handleFailedTask(ctx context.Context, execution *TaskExecution, t *asynq.Task, duration time.Duration, execErr error, span trace.Span) {
execution.Status = TaskExecutionStatusFailed
execution.ErrorMessage = execErr.Error()
logger.ErrorF(ctx, "[TaskExecutor] 任务处理失败 Type: %s TaskID: %s Duration: %d ms Error: %v", t.Type(), execution.TaskID, duration.Milliseconds(), execErr)
span.SetStatus(codes.Error, execErr.Error())
span.RecordError(execErr)
AppendLog(ctx, "[系统] 任务执行失败,耗时: %d ms,错误原因: %v", duration.Milliseconds(), execErr)
}
func handleSuccessfulTask(ctx context.Context, execution *TaskExecution, t *asynq.Task, duration time.Duration, result *TaskResult) {
execution.Status = TaskExecutionStatusSucceeded
execution.ErrorMessage = "" // 清除历史重试失败遗留的错误信息
if result != nil {
execution.Result = result.Message
if result.Detail != "" {
execution.Result = fmt.Sprintf("%s\n%s", result.Message, result.Detail)
}
}
logger.InfoF(ctx, "[TaskExecutor] 任务处理完成 Type: %s TaskID: %s Duration: %d ms", t.Type(), execution.TaskID, duration.Milliseconds())
resultMsg := "成功"
if result != nil {
resultMsg = result.Message
}
AppendLog(ctx, "[系统] 任务执行成功,耗时: %d ms,执行结果: %s", duration.Milliseconds(), resultMsg)
}
// generateTaskID 生成任务 ID
func generateTaskID(taskType string, triggeredBy string) string {
return fmt.Sprintf("%s_%s_%d", triggeredBy, taskType, idgen.NextUint64ID())
}
// generateRetryTaskID 生成重试任务 ID
func generateRetryTaskID(originalTaskID string, retryCount int) string {
return fmt.Sprintf("retry_%d_%s", retryCount, originalTaskID)
}
+406
View File
@@ -0,0 +1,406 @@
// Copyright 2025 linux.do
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package task
import (
"context"
"errors"
"fmt"
"testing"
"time"
"github.com/Rain-kl/Wavelet/pkg/testhelper"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
// mockHandler 用于测试的模拟任务处理器
type mockHandler struct {
executeFunc func(ctx context.Context, payload []byte) (*TaskResult, error)
}
func (h *mockHandler) Execute(ctx context.Context, payload []byte) (*TaskResult, error) {
if h.executeFunc != nil {
return h.executeFunc(ctx, payload)
}
return &TaskResult{Message: "mock success"}, nil
}
// successHandler 返回成功的处理器
func successHandler() *mockHandler {
return &mockHandler{
executeFunc: func(ctx context.Context, payload []byte) (*TaskResult, error) {
AppendLog(ctx, "执行成功,处理了 %d 条数据", 100)
return &TaskResult{Message: "处理完成,共 100 条"}, nil
},
}
}
// failHandler 返回失败的处理器
func failHandler() *mockHandler {
return &mockHandler{
executeFunc: func(ctx context.Context, payload []byte) (*TaskResult, error) {
AppendLog(ctx, "开始执行任务")
return nil, fmt.Errorf("模拟执行失败: 数据库连接超时")
},
}
}
const testTaskType = "test:mock_task"
func setupTest(t *testing.T) func() {
_, mr, cleanup := testhelper.SetupTestEnvironment(t)
AsynqClient = asynq.NewClient(asynq.RedisClientOpt{
Addr: mr.Addr(),
})
// 注册测试用 handler
RegisterHandler(testTaskType, successHandler())
return func() {
if AsynqClient != nil {
_ = AsynqClient.Close()
AsynqClient = nil
}
cleanup()
}
}
func TestRegisterAndGetHandler(t *testing.T) {
_ = testTaskType
cleanup := setupTest(t)
defer cleanup()
// 验证 handler 已注册
h, ok := getHandler(testTaskType)
assert.True(t, ok, "handler should be registered")
assert.NotNil(t, h)
// 未注册的 handler
_, ok = getHandler("nonexistent")
assert.False(t, ok, "non-existent handler should return false")
}
func TestGetTaskIDFromContext(t *testing.T) {
ctx := context.Background()
// 空 context
taskID := GetTaskID(ctx)
assert.Equal(t, "", taskID)
// 注入 taskID
ctx = withTaskID(ctx, "test_task_123")
taskID = GetTaskID(ctx)
assert.Equal(t, "test_task_123", taskID)
}
func TestAppendLogWithoutTaskID(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 没有 taskID 的 context,应降级到普通日志,不报错
AppendLog(ctx, "这条日志应该降级处理,不会报错")
}
func TestAppendLogWithTaskID(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 先创建一条执行记录
execution := &TaskExecution{
TaskID: "log_test_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: TaskExecutionStatusRunning,
TriggeredBy: "manual",
}
err := CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 注入 taskID 并追加日志
ctx = withTaskID(ctx, "log_test_001")
AppendLog(ctx, "第一条日志")
AppendLog(ctx, "处理了 %d 条数据", 50)
// 验证日志
found, err := GetTaskExecutionByTaskID(ctx, "log_test_001")
require.NoError(t, err)
assert.Contains(t, found.Log, "第一条日志")
assert.Contains(t, found.Log, "处理了 50 条数据")
}
func TestTaskTraceContextEnvelope(t *testing.T) {
payload := []byte(`{"hello":"wavelet"}`)
traceID := "4bf92f3577b34da6a3ce929d0e0e4736"
parentCtx := otel.GetTextMapPropagator().Extract(
context.Background(),
propagation.MapCarrier{
"traceparent": "00-" + traceID + "-00f067aa0ba902b7-01",
},
)
wrappedPayload := injectTaskTraceContext(parentCtx, payload)
require.NotEqual(t, string(payload), string(wrappedPayload))
gotCtx, gotPayload, ok := extractTaskTraceContext(context.Background(), wrappedPayload)
require.True(t, ok)
assert.Equal(t, payload, gotPayload)
assert.Equal(t, traceID, trace.SpanContextFromContext(gotCtx).TraceID().String())
}
func TestTaskTraceContextEnvelopeKeepsLegacyPayload(t *testing.T) {
payload := []byte(`{"legacy":true}`)
gotCtx, gotPayload, ok := extractTaskTraceContext(context.Background(), payload)
require.False(t, ok)
assert.Equal(t, context.Background(), gotCtx)
assert.Equal(t, payload, gotPayload)
}
func TestTaskTraceContextEnvelopeSkipsEmptyContext(t *testing.T) {
payload := []byte(`{"background":true}`)
wrappedPayload := injectTaskTraceContext(context.Background(), payload)
assert.Equal(t, payload, wrappedPayload)
}
func TestProcessTaskSuccess(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 注册成功 handler
RegisterHandler(testTaskType, successHandler())
// 创建执行记录
execution := &TaskExecution{
TaskID: "process_success_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: TaskExecutionStatusPending,
Retryable: true,
MaxRetry: 3,
TriggeredBy: "manual",
}
err := CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 通过 asynq 的 Task 不能直接设置 taskID,ProcessTask 通过 t.ResultWriter().TaskID() 获取
// 但 asynq.Task 在没有经过 asynq server 的情况下 ResultWriter 可能为 nil
// 我们需要在 ProcessTask 内部改用 taskID 注入的方式测试
// 为了测试 ProcessTask,我们直接模拟调用 handler
// 直接通过 handler 测试
handler, ok := getHandler(testTaskType)
require.True(t, ok)
ctx = withTaskID(ctx, "process_success_001")
result, err := handler.Execute(ctx, nil)
require.NoError(t, err)
assert.Equal(t, "处理完成,共 100 条", result.Message)
// 验证日志被追加
found, err := GetTaskExecutionByTaskID(ctx, "process_success_001")
require.NoError(t, err)
assert.Contains(t, found.Log, "执行成功,处理了 100 条数据")
}
func TestProcessTaskFailure(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 注册失败 handler
RegisterHandler(testTaskType, failHandler())
// 创建执行记录
execution := &TaskExecution{
TaskID: "process_fail_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: TaskExecutionStatusPending,
Retryable: true,
MaxRetry: 3,
TriggeredBy: "manual",
}
err := CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 直接调用 handler
handler, ok := getHandler(testTaskType)
require.True(t, ok)
ctx = withTaskID(ctx, "process_fail_001")
_, err = handler.Execute(ctx, nil)
assert.Error(t, err)
assert.Contains(t, err.Error(), "模拟执行失败")
// 验证日志
found, err := GetTaskExecutionByTaskID(ctx, "process_fail_001")
require.NoError(t, err)
assert.Contains(t, found.Log, "开始执行任务")
}
func TestCompleteTaskExecutionFlushesLog(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
execution := &TaskExecution{
TaskID: "complete_flush_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: TaskExecutionStatusRunning,
TriggeredBy: "manual",
}
err := CreateTaskExecution(ctx, execution)
require.NoError(t, err)
ctx = withTaskID(ctx, execution.TaskID)
AppendLog(ctx, "任务执行中的日志")
finishTime := time.Now()
completeTaskExecution(
ctx,
execution,
asynq.NewTask(testTaskType, nil),
100*time.Millisecond,
finishTime,
&TaskResult{Message: "处理完成"},
nil,
trace.SpanFromContext(ctx),
)
found, err := GetTaskExecutionByTaskID(ctx, execution.TaskID)
require.NoError(t, err)
assert.Equal(t, TaskExecutionStatusSucceeded, found.Status)
assert.Contains(t, found.Log, "任务执行中的日志")
assert.Contains(t, found.Log, "任务执行成功")
}
func TestPermanentErrorIsTerminalForLogFlush(t *testing.T) {
assert.True(t, isTerminalTaskExecutionError(PermanentError("配置无效")))
assert.False(t, isTerminalTaskExecutionError(errors.New("temporary failure")))
}
func TestRetryTask(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 创建一条失败的执行记录(可重试)
now := time.Now()
execution := &TaskExecution{
TaskID: "retry_test_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: TaskExecutionStatusFailed,
Retryable: true,
MaxRetry: 3,
RetryCount: 0,
ErrorMessage: "首次执行失败",
StartedAt: &now,
FinishedAt: &now,
Duration: 100,
TriggeredBy: "manual",
}
err := CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 重试
newTaskID, err := RetryTask(ctx, execution.ID)
require.NoError(t, err)
assert.NotEmpty(t, newTaskID)
assert.Contains(t, newTaskID, "retry_1_")
// 验证新记录
newExecution, err := GetTaskExecutionByTaskID(ctx, newTaskID)
require.NoError(t, err)
assert.Equal(t, TaskExecutionStatusPending, newExecution.Status)
assert.Equal(t, 1, newExecution.RetryCount)
assert.Equal(t, "retry", newExecution.TriggeredBy)
assert.Equal(t, execution.TaskType, newExecution.TaskType)
assert.True(t, newExecution.Retryable)
// 原记录不变
original, err := GetTaskExecutionByID(ctx, execution.ID)
require.NoError(t, err)
assert.Equal(t, TaskExecutionStatusFailed, original.Status)
assert.Equal(t, 0, original.RetryCount)
}
func TestRetryTaskNotFailed(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
// 创建一条成功的记录
execution := &TaskExecution{
TaskID: "retry_not_failed_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: TaskExecutionStatusSucceeded,
Retryable: true,
MaxRetry: 3,
TriggeredBy: "manual",
}
err := CreateTaskExecution(ctx, execution)
require.NoError(t, err)
// 尝试重试成功的任务
_, err = RetryTask(ctx, execution.ID)
assert.Error(t, err)
assert.Contains(t, err.Error(), "只有失败的任务才能重试")
}
func TestRetryTaskNotRetryable(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
execution := &TaskExecution{
TaskID: "retry_not_allowed_001",
TaskType: testTaskType,
TaskName: "测试任务",
Status: TaskExecutionStatusFailed,
Retryable: false,
MaxRetry: 0,
TriggeredBy: "manual",
}
err := CreateTaskExecution(ctx, execution)
require.NoError(t, err)
_, err = RetryTask(ctx, execution.ID)
assert.Error(t, err)
assert.Contains(t, err.Error(), "不支持重试")
}
func TestRetryTaskNonExistent(t *testing.T) {
cleanup := setupTest(t)
defer cleanup()
ctx := context.Background()
_, err := RetryTask(ctx, 99999999)
assert.Error(t, err)
assert.Contains(t, err.Error(), "不存在")
}
func TestGenerateTaskID(t *testing.T) {
id1 := generateTaskID("test_type", "manual")
id2 := generateTaskID("test_type", "manual")
// 两个 ID 应不同(包含 Snowflake ID)
assert.NotEqual(t, id1, id2)
assert.Contains(t, id1, "manual_test_type_")
}
func TestGenerateRetryTaskID(t *testing.T) {
id := generateRetryTaskID("original_task_123", 2)
assert.Equal(t, "retry_2_original_task_123", id)
}

Some files were not shown because too many files have changed in this diff Show More