feat(log): decouple log storage from ClickHouse with switchable logstore

- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
2026-10-02 06:56:36 +08:00 · 2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
@@ -7,6 +7,9 @@ package bootstrap

 import (
 	"context"
+	"errors"
+	"fmt"
+	"log"
 	"sync"

 	admin_push "github.com/Rain-kl/Wavelet/internal/apps/admin/push"
@@ -14,11 +17,15 @@ import (
 	"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
 	ofgeoip "github.com/Rain-kl/Wavelet/internal/apps/openflare/geoip"
 	"github.com/Rain-kl/Wavelet/internal/apps/risk_control"
+	"github.com/Rain-kl/Wavelet/internal/infra/config"
 	taskhandlers "github.com/Rain-kl/Wavelet/internal/infra/task/handlers"
+	"github.com/Rain-kl/Wavelet/internal/model"
 	"github.com/Rain-kl/Wavelet/internal/platform/lifecycle"
 	"github.com/Rain-kl/Wavelet/internal/repository"
+	"github.com/Rain-kl/Wavelet/internal/repository/logstore"
 	"github.com/Rain-kl/Wavelet/pkg/cache/ram"
 	"github.com/Rain-kl/Wavelet/pkg/logger"
+	"gorm.io/gorm"
 )

 // Options selects role-specific runtime bootstrap steps for the current process.
@@ -129,6 +136,21 @@ func RegisterAll() {
 // Call from cmd entry points after wiring registration and database migration, not from router.
 func Init(ctx context.Context, opts Options) {
 	initRuntimeOnce.Do(func() {
+		if err := validateAndSeedLogDatabase(ctx); err != nil {
+			logger.ErrorF(ctx, "[Bootstrap] 日志主库配置校验失败: %v", err)
+			log.Fatalf("[Bootstrap] 日志主库配置校验失败: %v", err)
+		}
+
+		// 注入 logstore 配置读取（避免 logstore ↔ repository 循环依赖），并预热激活 store。
+		logstore.SetConfigReader(func(ctx context.Context, key string) (string, error) {
+			cfg, err := repository.GetSystemConfigByKey(ctx, key)
+			if err != nil {
+				return "", err
+			}
+			return cfg.Value, nil
+		})
+		logstore.Init(ctx)
+
 		// Register config cache loader
 		RegisterCache(repository.ConfigCacheType, CacheRegistry{
 			Loader: repository.ConfigLoader{},
@@ -152,6 +174,47 @@ func Init(ctx context.Context, opts Options) {
 	})
 }

+// validateAndSeedLogDatabase 校验日志主库标记与运行配置的一致性，首次启动 seed。
+func validateAndSeedLogDatabase(ctx context.Context) error {
+	cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
+	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+		return fmt.Errorf("读取日志主库配置失败: %w", err)
+	}
+	current := cfg.Value
+	if current == "" {
+		// 首次启动 seed：CH 启用 → clickhouse；否则随主库。
+		current = "sqlite"
+		if config.Config.Database.Enabled {
+			current = "postgres"
+		}
+		if config.Config.ClickHouse.Enabled {
+			current = "clickhouse"
+		}
+		// 行缺失时 UpdateSystemConfigFields 仅为 UPDATE 无法插入，改用可创建可更新的 SaveOrUpdateSystemConfig。
+		if err := repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDatabase, current); err != nil {
+			return fmt.Errorf("初始化日志主库配置失败: %w", err)
+		}
+		return nil
+	}
+	switch current {
+	case "clickhouse":
+		if !config.Config.ClickHouse.Enabled {
+			return errors.New("当前日志主库为 ClickHouse 但 ClickHouse 未启用。请先重新启用 ClickHouse 配置并启动，在任务管理运行『切换日志数据库』迁移到 PostgreSQL/SQLite 后再禁用 ClickHouse")
+		}
+	case "postgres":
+		if !config.Config.Database.Enabled {
+			return errors.New("当前日志主库为 PostgreSQL 但 PostgreSQL 未启用（当前为 SQLite 主库）。请运行『切换日志数据库』迁回 SQLite 或启用 PostgreSQL")
+		}
+	case "sqlite":
+		if config.Config.Database.Enabled {
+			return errors.New("当前日志主库为 SQLite 但当前主库为 PostgreSQL。请运行『切换日志数据库』迁移到 PostgreSQL")
+		}
+	default:
+		return fmt.Errorf("未知的日志主库配置: %s", current)
+	}
+	return nil
+}
+
 // Stop stops all batch writers and background resources.
 func Stop(ctx context.Context) {
 	lifecycle.Stop(ctx)
@@ -5,10 +5,15 @@ package bootstrap

 import (
 	"context"
+	"strings"
 	"testing"

 	admin_push "github.com/Rain-kl/Wavelet/internal/apps/admin/push"
+	"github.com/Rain-kl/Wavelet/internal/infra/config"
+	db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
 	"github.com/Rain-kl/Wavelet/internal/model"
+	"github.com/Rain-kl/Wavelet/internal/repository"
+	"github.com/Rain-kl/Wavelet/internal/repository/logstore"
 	"github.com/Rain-kl/Wavelet/internal/testhelper"
 )

@@ -50,3 +55,167 @@ func TestInitSyncsPushEventsOnce(t *testing.T) {
 		t.Fatalf("admin_login name = %q, want %q", adminLogin.Name, "管理员登录")
 	}
 }
+
+func TestValidateAndSeedLogDatabaseSeedsDefault(t *testing.T) {
+	_, _, cleanup := testhelper.SetupTestEnvironment(t)
+	defer cleanup()
+
+	prevDB := config.Config.Database.Enabled
+	prevCH := config.Config.ClickHouse.Enabled
+	t.Cleanup(func() {
+		config.Config.Database.Enabled = prevDB
+		config.Config.ClickHouse.Enabled = prevCH
+	})
+
+	tests := []struct {
+		name      string
+		dbEnabled bool
+		chEnabled bool
+		want      string
+	}{
+		{name: "sqlite default", dbEnabled: false, chEnabled: false, want: "sqlite"},
+		{name: "postgres default", dbEnabled: true, chEnabled: false, want: "postgres"},
+		{name: "clickhouse default", dbEnabled: true, chEnabled: true, want: "clickhouse"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config.Config.Database.Enabled = tt.dbEnabled
+			config.Config.ClickHouse.Enabled = tt.chEnabled
+
+			ctx := context.Background()
+			// 清掉标记行，模拟首次启动。
+			if err := db.DB(ctx).Where("key = ?", model.ConfigKeyLogDatabase).Delete(&model.SystemConfig{}).Error; err != nil {
+				t.Fatalf("delete log_database marker failed: %v", err)
+			}
+			repository.ResetSystemConfigRAMCacheForTest()
+
+			if err := validateAndSeedLogDatabase(ctx); err != nil {
+				t.Fatalf("validateAndSeedLogDatabase() error = %v", err)
+			}
+
+			repository.ResetSystemConfigRAMCacheForTest()
+			cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
+			if err != nil {
+				t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err)
+			}
+			if cfg.Value != tt.want {
+				t.Fatalf("seeded log_database = %q, want %q", cfg.Value, tt.want)
+			}
+		})
+	}
+}
+
+func TestValidateAndSeedLogDatabaseUpdatesEmptyMarker(t *testing.T) {
+	_, _, cleanup := testhelper.SetupTestEnvironment(t)
+	defer cleanup()
+
+	ctx := context.Background()
+	// 标记行已存在但值为空，等同首次启动，应写入默认值（走更新路径）。
+	if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: "", Type: "system"}); err != nil {
+		t.Fatalf("create empty marker failed: %v", err)
+	}
+	repository.ResetSystemConfigRAMCacheForTest()
+
+	if err := validateAndSeedLogDatabase(ctx); err != nil {
+		t.Fatalf("validateAndSeedLogDatabase() error = %v", err)
+	}
+
+	repository.ResetSystemConfigRAMCacheForTest()
+	cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
+	if err != nil {
+		t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err)
+	}
+	want := "sqlite"
+	if config.Config.Database.Enabled {
+		want = "postgres"
+	}
+	if config.Config.ClickHouse.Enabled {
+		want = "clickhouse"
+	}
+	if cfg.Value != want {
+		t.Fatalf("log_database = %q, want %q", cfg.Value, want)
+	}
+}
+
+func TestValidateAndSeedLogDatabaseRejectsInconsistentConfig(t *testing.T) {
+	_, _, cleanup := testhelper.SetupTestEnvironment(t)
+	defer cleanup()
+
+	prevDB := config.Config.Database.Enabled
+	prevCH := config.Config.ClickHouse.Enabled
+	t.Cleanup(func() {
+		config.Config.Database.Enabled = prevDB
+		config.Config.ClickHouse.Enabled = prevCH
+	})
+
+	seedMarker := func(t *testing.T, value string) {
+		t.Helper()
+		ctx := context.Background()
+		if err := db.DB(ctx).Where("key = ?", model.ConfigKeyLogDatabase).Delete(&model.SystemConfig{}).Error; err != nil {
+			t.Fatalf("delete log_database marker failed: %v", err)
+		}
+		if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: value, Type: "system"}); err != nil {
+			t.Fatalf("create log_database marker failed: %v", err)
+		}
+		repository.ResetSystemConfigRAMCacheForTest()
+	}
+
+	tests := []struct {
+		name      string
+		marker    string
+		dbEnabled bool
+		chEnabled bool
+		wantErr   string
+	}{
+		{name: "clickhouse marker but disabled", marker: "clickhouse", dbEnabled: true, chEnabled: false, wantErr: "ClickHouse 未启用"},
+		{name: "postgres marker but disabled", marker: "postgres", dbEnabled: false, chEnabled: false, wantErr: "PostgreSQL 未启用"},
+		{name: "sqlite marker but postgres primary", marker: "sqlite", dbEnabled: true, chEnabled: false, wantErr: "SQLite"},
+		{name: "unknown marker", marker: "mysql", dbEnabled: false, chEnabled: false, wantErr: "未知的日志主库配置"},
+		{name: "consistent sqlite", marker: "sqlite", dbEnabled: false, chEnabled: false, wantErr: ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config.Config.Database.Enabled = tt.dbEnabled
+			config.Config.ClickHouse.Enabled = tt.chEnabled
+			seedMarker(t, tt.marker)
+
+			err := validateAndSeedLogDatabase(context.Background())
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Fatalf("validateAndSeedLogDatabase() error = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatalf("validateAndSeedLogDatabase() = nil, want error containing %q", tt.wantErr)
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Fatalf("validateAndSeedLogDatabase() error = %q, want contains %q", err.Error(), tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestInitWiresLogstoreConfigReader(t *testing.T) {
+	ResetInitRuntimeOnceForTest()
+	t.Cleanup(ResetInitRuntimeOnceForTest)
+	logstore.ResetForTest()
+	t.Cleanup(logstore.ResetForTest)
+
+	_, _, cleanup := testhelper.SetupTestEnvironment(t)
+	defer cleanup()
+
+	ctx := context.Background()
+	// 插入迁移标记：bootstrap 注入的 reader 应能经 repository 读到该值（区分未装配时的兜底行为）。
+	if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDBMigration, Value: "migrating", Type: "system"}); err != nil {
+		t.Fatalf("create log_db_migration marker failed: %v", err)
+	}
+	repository.ResetSystemConfigRAMCacheForTest()
+
+	Init(ctx, Options{})
+
+	repository.ResetSystemConfigRAMCacheForTest()
+	if !logstore.Migrating(ctx) {
+		t.Fatal("logstore config reader not wired after bootstrap.Init: Migrating() = false, want true")
+	}
+}