feat(log): decouple log storage from ClickHouse with switchable logstore

- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
@@ -17,6 +17,11 @@ import (
)
func createSystemConfig(ctx context.Context, req CreateSystemConfigRequest) error {
// 防御:受保护 key(log_database / log_db_migration)仅允许内部写入,Handler 已拦截。
if isProtectedConfigKey(req.Key) {
return errors.New(protectedConfigKeyMessage)
}
exists, err := repository.SystemConfigExists(ctx, req.Key)
if err != nil {
return err
@@ -27,6 +27,17 @@ import (
const maskedConfigValue = "******"
// protectedConfigKeyMessage 命中受保护 key 时返回给管理员的业务错误文案。
const protectedConfigKeyMessage = "该配置项由系统任务管理,禁止手动修改"
// protectedConfigKeys 仅允许内部(迁移任务/bootstrap)写入的 key。
var protectedConfigKeys = map[string]bool{
model.ConfigKeyLogDatabase: true,
model.ConfigKeyLogDBMigration: true,
}
func isProtectedConfigKey(key string) bool { return protectedConfigKeys[key] }
// CreateSystemConfigRequest 创建系统配置请求
type CreateSystemConfigRequest struct {
Key string `json:"key" binding:"required,max=64"`
@@ -64,11 +75,21 @@ func CreateSystemConfig(c *gin.Context) {
return
}
// 与 PUT 路径一致:log_database / log_db_migration 仅允许内部(迁移任务/bootstrap)写入。
if isProtectedConfigKey(req.Key) {
response.AbortBadRequest(c, protectedConfigKeyMessage)
return
}
if err := createSystemConfig(c.Request.Context(), req); err != nil {
if err.Error() == ConfigKeyExists {
response.AbortBadRequest(c, ConfigKeyExists)
return
}
if err.Error() == protectedConfigKeyMessage {
response.AbortBadRequest(c, protectedConfigKeyMessage)
return
}
response.AbortInternal(c, err.Error())
return
}
@@ -155,6 +176,10 @@ func UpdateSystemConfig(c *gin.Context) {
}
key := c.Param("key")
if isProtectedConfigKey(key) {
response.AbortBadRequest(c, protectedConfigKeyMessage)
return
}
if err := updateSystemConfig(c.Request.Context(), key, req); err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
response.AbortNotFound(c, SystemConfigNotFound)
@@ -27,7 +27,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/shared/response"
)
const expectedDefaultConfigsCount = 34
const expectedDefaultConfigsCount = 37
func setupTestRouter(authUser *model.User) *gin.Engine {
r := testhelper.NewTestGinEngine()
@@ -168,8 +168,8 @@ func TestListSystemConfigs(t *testing.T) {
var configs []model.SystemConfig
_ = json.Unmarshal(dataBytes, &configs)
if len(configs) != 5 {
t.Errorf("expected 5 business configs, got %d: %v", len(configs), configs)
if len(configs) != 8 {
t.Errorf("expected 8 business configs, got %d: %v", len(configs), configs)
}
})
}