mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-08 00:26:37 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -6,21 +6,60 @@ package repository
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// accessLogTestDBSeq 保证每个测试获得独立的 sqlite 内存库(cache=shared 下同名 DSN 复用同一库)。
|
||||
var accessLogTestDBSeq int64
|
||||
|
||||
func setupOpenFlareAccessLogTestEnvironment(t *testing.T) (context.Context, func()) {
|
||||
t.Helper()
|
||||
store := NewMemoryAccessLogStore()
|
||||
reset := SetAccessLogStoreForTest(store)
|
||||
return context.Background(), func() {
|
||||
reset()
|
||||
dsn := fmt.Sprintf("file:repo-access-log-test-%d?mode=memory&cache=shared", atomic.AddInt64(&accessLogTestDBSeq, 1))
|
||||
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gdb.AutoMigrate(&analyticsmodel.NodeAccessLog{}))
|
||||
db.SetDB(gdb)
|
||||
|
||||
logstore.ResetForTest()
|
||||
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == "log_database" {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
|
||||
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{})
|
||||
|
||||
ctx := context.Background()
|
||||
store, err := logstore.Active(ctx)
|
||||
require.NoError(t, err)
|
||||
// 写入入口只入队;测试环境立即 flush,保证后续查询可见。
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
|
||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||
_ = store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs)
|
||||
},
|
||||
})
|
||||
return ctx, func() {
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
|
||||
logstore.ResetForTest()
|
||||
db.SetDB(nil)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +92,7 @@ func TestListOpenFlareAccessLogsPaginated(t *testing.T) {
|
||||
require.NoError(t, InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{record}))
|
||||
}
|
||||
|
||||
// 0-based 分页与 CH ListNodeAccessLogs 一致:page=1 size=5 → OFFSET 5 → /path-05..09。
|
||||
query := model.OpenFlareAccessLogQuery{
|
||||
NodeID: "node-page",
|
||||
Since: now.Add(-24 * time.Hour),
|
||||
@@ -81,6 +121,7 @@ func TestCountOpenFlareAccessLogs(t *testing.T) {
|
||||
totalRecords, totalIPs, _, err := CountOpenFlareAccessLogs(ctx, query)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(5), totalRecords)
|
||||
// GORM 与 CH 一致:distinct IP 排除空 remote_addr(CH uniqExactIf(remote_addr, remote_addr != ''))。
|
||||
assert.Equal(t, int64(3), totalIPs)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user