mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-12 18:26:36 +08:00
feat(clickhouse): wire batchwriter into business ingestion paths
Migrate risk_control audit logs to internal/db/batchwriter and add openflare/chwriter with per-table async flush for observability timeseries and node access logs. Replace single-row ClickHouse inserts and pre-insert SELECT count() dedup with repository BatchInsert* APIs, in-process TTL dedup for metric/report snapshots, and bootstrap initialization on API startup.
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package chwriter
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const dedupTTL = 2 * time.Minute
|
||||
|
||||
type dedupSet struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]time.Time
|
||||
}
|
||||
|
||||
func newDedupSet() *dedupSet {
|
||||
return &dedupSet{keys: make(map[string]time.Time)}
|
||||
}
|
||||
|
||||
// markIfNew records key when it has not been seen within dedupTTL.
|
||||
func (s *dedupSet) markIfNew(key string) bool {
|
||||
if key == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
for existing, expiresAt := range s.keys {
|
||||
if now.After(expiresAt) {
|
||||
delete(s.keys, existing)
|
||||
}
|
||||
}
|
||||
if expiresAt, exists := s.keys[key]; exists && now.Before(expiresAt) {
|
||||
return false
|
||||
}
|
||||
s.keys[key] = now.Add(dedupTTL)
|
||||
return true
|
||||
}
|
||||
Reference in New Issue
Block a user