tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-11 09:46:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(openflare): migrate of_node_access_logs to ClickHouse

Browse Source 
Move node access log storage from PostgreSQL/SQLite to ClickHouse
(database: openflare). System startup now requires a healthy ClickHouse
connection and auto-initializes the schema. Agent heartbeat writes access
logs via batch insert; goose migration drops the legacy relational table.
This commit is contained in:
ryan
2026-06-19 11:45:48 +08:00
parent d78449cbc9
commit 425ca89765
24 changed files with 1224 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openflare-server/internal/apps/openflare/tasks/database_cleanup_test.go
+22 -19
View File
@@ -24,12 +24,13 @@ func setupDatabaseCleanupTestDB(t *testing.T) context.Context {
})
require.NoError(t, err)
require.NoError(t, sqliteDB.AutoMigrate(
&model.OpenFlareAccessLog{},
&model.OpenFlareMetricSnapshot{},
&model.OpenFlareRequestReport{},
))
db.SetDB(sqliteDB)
resetAccessLogStore := model.SetAccessLogStoreForTest(model.NewMemoryAccessLogStore())
t.Cleanup(func() {
resetAccessLogStore()
db.SetDB(nil)
})
return context.Background()
@@ -69,22 +70,24 @@ func TestCleanupDatabaseObservabilityDeletesAllRowsWhenRetentionMissing(t *testi
ctx := setupDatabaseCleanupTestDB(t)
now := time.Now().UTC()
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareAccessLog{
NodeID: "node-a",
LoggedAt: now.Add(-3 * time.Hour),
RemoteAddr: "203.0.113.1",
Host: "example.com",
Path: "/one",
StatusCode: 200,
}).Error)
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareAccessLog{
NodeID: "node-a",
LoggedAt: now.Add(-2 * time.Hour),
RemoteAddr: "203.0.113.2",
Host: "example.com",
Path: "/two",
StatusCode: 502,
}).Error)
require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
{
NodeID: "node-a",
LoggedAt: now.Add(-3 * time.Hour),
RemoteAddr: "203.0.113.1",
Host: "example.com",
Path: "/one",
StatusCode: 200,
},
{
NodeID: "node-a",
LoggedAt: now.Add(-2 * time.Hour),
RemoteAddr: "203.0.113.2",
Host: "example.com",
Path: "/two",
StatusCode: 502,
},
}))
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
Target: DatabaseCleanupTargetAccessLogs,
@@ -102,14 +105,14 @@ func TestRunDatabaseAutoCleanupOnceDeletesAllObservabilityTargets(t *testing.T)
ctx := setupDatabaseCleanupTestDB(t)
now := time.Now().UTC()
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareAccessLog{
require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{{
NodeID: "node-a",
LoggedAt: now.Add(-48 * time.Hour),
RemoteAddr: "203.0.113.10",
Host: "example.com",
Path: "/access",
StatusCode: 200,
}).Error)
}}))
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareMetricSnapshot{
NodeID: "node-a",
CapturedAt: now.Add(-48 * time.Hour),