mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-03 23:06:36 +08:00
fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)
Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer metrics; model insert hooks; latest-per-node and hourly metric/openresty rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.
This commit is contained in:
@@ -6,6 +6,7 @@ package risk_control
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db/batchwriter"
|
||||
@@ -15,6 +16,11 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
// Bound visibility lag for sparse access-log traffic when MinBatchSize is not met.
|
||||
accessLogMaxFlushWait = 3 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
logWriterMu sync.RWMutex
|
||||
logWriter *batchwriter.Writer[*analytics.UserAccessLog]
|
||||
@@ -33,6 +39,8 @@ func InitLogWriter(ctx context.Context) {
|
||||
}
|
||||
|
||||
cfg := batchwriter.DefaultConfig()
|
||||
cfg.Name = "user_access_logs"
|
||||
cfg.MaxFlushWait = accessLogMaxFlushWait
|
||||
writer, err := batchwriter.New[*analytics.UserAccessLog](cfg, func(ctx context.Context, items []*analytics.UserAccessLog) error {
|
||||
rows := make([]analytics.UserAccessLog, 0, len(items))
|
||||
for _, item := range items {
|
||||
@@ -50,8 +58,8 @@ func InitLogWriter(ctx context.Context) {
|
||||
}
|
||||
logger.WarnF(context.Background(), "[RiskControl] Log queue full, dropping log item for path: %s", path)
|
||||
}),
|
||||
batchwriter.WithFlushErrorHandler[*analytics.UserAccessLog](func(ctx context.Context, batchSize int, err error) {
|
||||
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed (batch=%d): %v", batchSize, err)
|
||||
batchwriter.WithFlushErrorHandler[*analytics.UserAccessLog](func(ctx context.Context, items []*analytics.UserAccessLog, err error) {
|
||||
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed (batch=%d): %v", len(items), err)
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -82,6 +90,16 @@ func IsBufferFull() bool {
|
||||
return writer.IsFull()
|
||||
}
|
||||
|
||||
// LogWriterStats returns queue depth and failure counters for the access-log writer.
|
||||
// When the writer is not initialized, it returns a zero-value Stats with the expected name.
|
||||
func LogWriterStats() batchwriter.Stats {
|
||||
writer := currentLogWriter()
|
||||
if writer == nil {
|
||||
return batchwriter.Stats{Name: "user_access_logs"}
|
||||
}
|
||||
return writer.Stats()
|
||||
}
|
||||
|
||||
// QueueAccessLog enqueues an access log without blocking.
|
||||
func QueueAccessLog(logItem *analytics.UserAccessLog) {
|
||||
writer := currentLogWriter()
|
||||
@@ -108,4 +126,4 @@ func currentLogWriter() *batchwriter.Writer[*analytics.UserAccessLog] {
|
||||
logWriterMu.RLock()
|
||||
defer logWriterMu.RUnlock()
|
||||
return logWriter
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package risk_control
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAccessLogMaxFlushWaitInRange(t *testing.T) {
|
||||
t.Parallel()
|
||||
if accessLogMaxFlushWait < 2*time.Second || accessLogMaxFlushWait > 5*time.Second {
|
||||
t.Fatalf("accessLogMaxFlushWait = %v, want in [2s, 5s]", accessLogMaxFlushWait)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogWriterStatsWhenNil(t *testing.T) {
|
||||
t.Parallel()
|
||||
reset := SetLogWriterForTest(nil)
|
||||
t.Cleanup(reset)
|
||||
|
||||
stats := LogWriterStats()
|
||||
if stats.Name != "user_access_logs" {
|
||||
t.Fatalf("LogWriterStats().Name = %q, want user_access_logs", stats.Name)
|
||||
}
|
||||
if stats.Running {
|
||||
t.Fatal("LogWriterStats().Running = true for nil writer, want false")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user