mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 15:26:36 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -11,11 +11,14 @@ import (
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/redis/go-redis/v9/maintnotifications"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -307,6 +310,24 @@ func getSeedConfigsPart2() []model.SystemConfig {
|
||||
Type: configTypeBusiness,
|
||||
Description: "Pages 每个项目最大历史部署保留数(0 表示不限制)",
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyLogRetentionDaysPostgres,
|
||||
Value: "90",
|
||||
Type: configTypeBusiness,
|
||||
Description: "PostgreSQL 日志保留天数(访问日志与可观测统一)",
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyLogRetentionDaysSQLite,
|
||||
Value: "90",
|
||||
Type: configTypeBusiness,
|
||||
Description: "SQLite 日志保留天数",
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyLogRetentionDaysClickHouse,
|
||||
Value: "90",
|
||||
Type: configTypeBusiness,
|
||||
Description: "ClickHouse 日志保留天数",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,3 +373,55 @@ func seedDefaultConfigs(t *testing.T, tx *gorm.DB) {
|
||||
_ = db.HSetJSON(context.Background(), repository.SystemConfigRedisHashKey, config.Key, &config)
|
||||
}
|
||||
}
|
||||
|
||||
// SetupLogStoresForTest 将 logstore 指向测试已通过 db.SetDB 注入的 sqlite 库,
|
||||
// 并注册立即 flush 的 hooks,使 repository 层日志写入对后续读取立即可见。
|
||||
// 调用方必须先 db.SetDB(sqliteDB)(并迁移业务表),本函数负责迁移日志分析表。
|
||||
func SetupLogStoresForTest(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
gdb := db.DB(context.Background())
|
||||
require.NoError(t, gdb.AutoMigrate(
|
||||
&analyticsmodel.NodeAccessLog{},
|
||||
&analyticsmodel.NodeMetricSnapshot{},
|
||||
&analyticsmodel.NodeEdgeHealth{},
|
||||
&analyticsmodel.NodeObsFrps{},
|
||||
&analyticsmodel.NodeObsFrpc{},
|
||||
))
|
||||
|
||||
logstore.ResetForTest()
|
||||
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == model.ConfigKeyLogDatabase {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
store, err := logstore.Active(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
|
||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||
_ = store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs)
|
||||
},
|
||||
})
|
||||
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{
|
||||
QueueMetricSnapshot: func(record analyticsmodel.NodeMetricSnapshot) {
|
||||
_ = store.Observability.BatchInsertNodeMetricSnapshots(context.Background(), []analyticsmodel.NodeMetricSnapshot{record})
|
||||
},
|
||||
QueueEdgeHealth: func(record analyticsmodel.NodeEdgeHealth) {
|
||||
_ = store.Observability.BatchInsertNodeEdgeHealth(context.Background(), []analyticsmodel.NodeEdgeHealth{record})
|
||||
},
|
||||
QueueNodeObsFrps: func(record analyticsmodel.NodeObsFrps) {
|
||||
_ = store.Observability.BatchInsertNodeObsFrps(context.Background(), []analyticsmodel.NodeObsFrps{record})
|
||||
},
|
||||
QueueNodeObsFrpc: func(record analyticsmodel.NodeObsFrpc) {
|
||||
_ = store.Observability.BatchInsertNodeObsFrpc(context.Background(), []analyticsmodel.NodeObsFrpc{record})
|
||||
},
|
||||
})
|
||||
|
||||
t.Cleanup(func() {
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
|
||||
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{})
|
||||
logstore.ResetForTest()
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user