tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-11 01:36:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(clickhouse): wire batchwriter into business ingestion paths

Browse Source 
Migrate risk_control audit logs to internal/db/batchwriter and add
openflare/chwriter with per-table async flush for observability
timeseries and node access logs.

Replace single-row ClickHouse inserts and pre-insert SELECT count()
dedup with repository BatchInsert* APIs, in-process TTL dedup for
metric/report snapshots, and bootstrap initialization on API startup.
This commit is contained in:
ryan
2026-06-19 17:13:46 +08:00
parent ca6c20ebd9
commit 9491b2a744
11 changed files with 633 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/model/openflare_access_log_store.go
+4 -2
View File
@@ -8,6 +8,7 @@ import (
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
@@ -63,7 +64,7 @@ func NewMemoryAccessLogStore() accessLogStore {
type clickhouseAccessLogStore struct{}
func (clickhouseAccessLogStore) InsertBatch(ctx context.Context, records []*OpenFlareAccessLog) error {
func (clickhouseAccessLogStore) InsertBatch(_ context.Context, records []*OpenFlareAccessLog) error {
logs := make([]analyticsmodel.NodeAccessLog, 0, len(records))
for _, record := range records {
if record == nil {
@@ -71,7 +72,8 @@ func (clickhouseAccessLogStore) InsertBatch(ctx context.Context, records []*Open
}
logs = append(logs, toAnalyticsNodeAccessLog(record))
}
return analyticsrepo.BatchInsertNodeAccessLogs(ctx, logs)
chwriter.QueueNodeAccessLogs(logs)
return nil
}
func (clickhouseAccessLogStore) List(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLog, error) {