mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 23:26:38 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -8,11 +8,10 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter"
|
||||
"github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/platform/lifecycle"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
@@ -26,12 +25,10 @@ var (
|
||||
logWriter *batchwriter.Writer[*analytics.UserAccessLog]
|
||||
)
|
||||
|
||||
// InitLogWriter initializes the ClickHouse access-log batch writer.
|
||||
// InitLogWriter initializes the user access-log batch writer.
|
||||
// The active log store is resolved via logstore at flush time, so the writer
|
||||
// runs for PG/SQLite as well as ClickHouse.
|
||||
func InitLogWriter(ctx context.Context) {
|
||||
if !config.Config.ClickHouse.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
logWriterMu.Lock()
|
||||
defer logWriterMu.Unlock()
|
||||
if logWriter != nil {
|
||||
@@ -49,7 +46,11 @@ func InitLogWriter(ctx context.Context) {
|
||||
}
|
||||
rows = append(rows, *item)
|
||||
}
|
||||
return analyticsrepo.BatchInsert(ctx, rows)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.UserAccessLogs.BatchInsert(ctx, rows)
|
||||
},
|
||||
batchwriter.WithDropHandler[*analytics.UserAccessLog](func(item *analytics.UserAccessLog) {
|
||||
path := ""
|
||||
@@ -59,7 +60,7 @@ func InitLogWriter(ctx context.Context) {
|
||||
logger.WarnF(context.Background(), "[RiskControl] Log queue full, dropping log item for path: %s", path)
|
||||
}),
|
||||
batchwriter.WithFlushErrorHandler[*analytics.UserAccessLog](func(ctx context.Context, items []*analytics.UserAccessLog, err error) {
|
||||
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed (batch=%d): %v", len(items), err)
|
||||
logger.ErrorF(ctx, "[RiskControl] Send log batch failed (batch=%d): %v", len(items), err)
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -72,7 +73,7 @@ func InitLogWriter(ctx context.Context) {
|
||||
lifecycle.OnShutdown("risk_control_log_writer", StopLogWriter)
|
||||
}
|
||||
|
||||
// StopLogWriter stops the ClickHouse access-log batch writer and drains pending logs.
|
||||
// StopLogWriter stops the user access-log batch writer and drains pending logs.
|
||||
func StopLogWriter(ctx context.Context) error {
|
||||
writer := currentLogWriter()
|
||||
if writer == nil {
|
||||
@@ -81,6 +82,38 @@ func StopLogWriter(ctx context.Context) error {
|
||||
return writer.Stop(ctx)
|
||||
}
|
||||
|
||||
// DrainLogWriter 等待用户访问日志 writer 的在途批次落库:队列 Depth 归零后
|
||||
// 再保持一个 flush 周期(1s)持续为空才返回;不停止 writer(迁移冻结后由
|
||||
// ensureWritable 拒绝新写入)。writer 未初始化时直接返回 nil。
|
||||
func DrainLogWriter(ctx context.Context) error {
|
||||
writer := currentLogWriter()
|
||||
if writer == nil {
|
||||
return nil
|
||||
}
|
||||
ticker := time.NewTicker(drainPollInterval)
|
||||
defer ticker.Stop()
|
||||
var quietSince time.Time
|
||||
for {
|
||||
if writer.Stats().Depth == 0 {
|
||||
if quietSince.IsZero() {
|
||||
quietSince = time.Now()
|
||||
} else if time.Since(quietSince) >= batchwriter.DefaultConfig().FlushInterval {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
quietSince = time.Time{}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// drainPollInterval 用户访问日志队列轮询间隔。
|
||||
const drainPollInterval = 50 * time.Millisecond
|
||||
|
||||
// IsBufferFull reports whether the access-log queue has no remaining capacity.
|
||||
func IsBufferFull() bool {
|
||||
writer := currentLogWriter()
|
||||
|
||||
@@ -13,11 +13,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||
"github.com/Rain-kl/Wavelet/internal/shared/response"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -79,8 +80,9 @@ func hashAuditLogSensitiveValue(value string) string {
|
||||
// RiskControlMiddleware 全局日志采集中间件
|
||||
func RiskControlMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// 如果未启用 ClickHouse,直接放行
|
||||
if !config.Config.ClickHouse.Enabled {
|
||||
// 日志库迁移冻结期跳过采集,不阻断业务请求。
|
||||
if logstore.Migrating(c.Request.Context()) {
|
||||
logger.WarnF(c.Request.Context(), "[RiskControl] log DB migrating, skip audit log")
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user