feat(log): decouple log storage from ClickHouse with switchable logstore

- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
+63
View File
@@ -7,6 +7,9 @@ package bootstrap
import (
"context"
"errors"
"fmt"
"log"
"sync"
admin_push "github.com/Rain-kl/Wavelet/internal/apps/admin/push"
@@ -14,11 +17,15 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
ofgeoip "github.com/Rain-kl/Wavelet/internal/apps/openflare/geoip"
"github.com/Rain-kl/Wavelet/internal/apps/risk_control"
"github.com/Rain-kl/Wavelet/internal/infra/config"
taskhandlers "github.com/Rain-kl/Wavelet/internal/infra/task/handlers"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/platform/lifecycle"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/cache/ram"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
)
// Options selects role-specific runtime bootstrap steps for the current process.
@@ -129,6 +136,21 @@ func RegisterAll() {
// Call from cmd entry points after wiring registration and database migration, not from router.
func Init(ctx context.Context, opts Options) {
initRuntimeOnce.Do(func() {
if err := validateAndSeedLogDatabase(ctx); err != nil {
logger.ErrorF(ctx, "[Bootstrap] 日志主库配置校验失败: %v", err)
log.Fatalf("[Bootstrap] 日志主库配置校验失败: %v", err)
}
// 注入 logstore 配置读取(避免 logstore ↔ repository 循环依赖),并预热激活 store。
logstore.SetConfigReader(func(ctx context.Context, key string) (string, error) {
cfg, err := repository.GetSystemConfigByKey(ctx, key)
if err != nil {
return "", err
}
return cfg.Value, nil
})
logstore.Init(ctx)
// Register config cache loader
RegisterCache(repository.ConfigCacheType, CacheRegistry{
Loader: repository.ConfigLoader{},
@@ -152,6 +174,47 @@ func Init(ctx context.Context, opts Options) {
})
}
// validateAndSeedLogDatabase 校验日志主库标记与运行配置的一致性,首次启动 seed。
func validateAndSeedLogDatabase(ctx context.Context) error {
cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("读取日志主库配置失败: %w", err)
}
current := cfg.Value
if current == "" {
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
current = "sqlite"
if config.Config.Database.Enabled {
current = "postgres"
}
if config.Config.ClickHouse.Enabled {
current = "clickhouse"
}
// 行缺失时 UpdateSystemConfigFields 仅为 UPDATE 无法插入,改用可创建可更新的 SaveOrUpdateSystemConfig。
if err := repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDatabase, current); err != nil {
return fmt.Errorf("初始化日志主库配置失败: %w", err)
}
return nil
}
switch current {
case "clickhouse":
if !config.Config.ClickHouse.Enabled {
return errors.New("当前日志主库为 ClickHouse 但 ClickHouse 未启用。请先重新启用 ClickHouse 配置并启动,在任务管理运行『切换日志数据库』迁移到 PostgreSQL/SQLite 后再禁用 ClickHouse")
}
case "postgres":
if !config.Config.Database.Enabled {
return errors.New("当前日志主库为 PostgreSQL 但 PostgreSQL 未启用(当前为 SQLite 主库)。请运行『切换日志数据库』迁回 SQLite 或启用 PostgreSQL")
}
case "sqlite":
if config.Config.Database.Enabled {
return errors.New("当前日志主库为 SQLite 但当前主库为 PostgreSQL。请运行『切换日志数据库』迁移到 PostgreSQL")
}
default:
return fmt.Errorf("未知的日志主库配置: %s", current)
}
return nil
}
// Stop stops all batch writers and background resources.
func Stop(ctx context.Context) {
lifecycle.Stop(ctx)