fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer
metrics; model insert hooks; latest-per-node and hourly metric/openresty
rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.
This commit is contained in:
ryan
2026-07-10 10:34:04 +08:00
parent 9b3555c569
commit 160e63558f
37 changed files with 1779 additions and 269 deletions
+21 -3
View File
@@ -6,6 +6,7 @@ package risk_control
import (
"context"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/config"
"github.com/Rain-kl/Wavelet/internal/db/batchwriter"
@@ -15,6 +16,11 @@ import (
"github.com/Rain-kl/Wavelet/pkg/logger"
)
const (
// Bound visibility lag for sparse access-log traffic when MinBatchSize is not met.
accessLogMaxFlushWait = 3 * time.Second
)
var (
logWriterMu sync.RWMutex
logWriter *batchwriter.Writer[*analytics.UserAccessLog]
@@ -33,6 +39,8 @@ func InitLogWriter(ctx context.Context) {
}
cfg := batchwriter.DefaultConfig()
cfg.Name = "user_access_logs"
cfg.MaxFlushWait = accessLogMaxFlushWait
writer, err := batchwriter.New[*analytics.UserAccessLog](cfg, func(ctx context.Context, items []*analytics.UserAccessLog) error {
rows := make([]analytics.UserAccessLog, 0, len(items))
for _, item := range items {
@@ -50,8 +58,8 @@ func InitLogWriter(ctx context.Context) {
}
logger.WarnF(context.Background(), "[RiskControl] Log queue full, dropping log item for path: %s", path)
}),
batchwriter.WithFlushErrorHandler[*analytics.UserAccessLog](func(ctx context.Context, batchSize int, err error) {
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed (batch=%d): %v", batchSize, err)
batchwriter.WithFlushErrorHandler[*analytics.UserAccessLog](func(ctx context.Context, items []*analytics.UserAccessLog, err error) {
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed (batch=%d): %v", len(items), err)
}),
)
if err != nil {
@@ -82,6 +90,16 @@ func IsBufferFull() bool {
return writer.IsFull()
}
// LogWriterStats returns queue depth and failure counters for the access-log writer.
// When the writer is not initialized, it returns a zero-value Stats with the expected name.
func LogWriterStats() batchwriter.Stats {
writer := currentLogWriter()
if writer == nil {
return batchwriter.Stats{Name: "user_access_logs"}
}
return writer.Stats()
}
// QueueAccessLog enqueues an access log without blocking.
func QueueAccessLog(logItem *analytics.UserAccessLog) {
writer := currentLogWriter()
@@ -108,4 +126,4 @@ func currentLogWriter() *batchwriter.Writer[*analytics.UserAccessLog] {
logWriterMu.RLock()
defer logWriterMu.RUnlock()
return logWriter
}
}
+30
View File
@@ -0,0 +1,30 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package risk_control
import (
"testing"
"time"
)
func TestAccessLogMaxFlushWaitInRange(t *testing.T) {
t.Parallel()
if accessLogMaxFlushWait < 2*time.Second || accessLogMaxFlushWait > 5*time.Second {
t.Fatalf("accessLogMaxFlushWait = %v, want in [2s, 5s]", accessLogMaxFlushWait)
}
}
func TestLogWriterStatsWhenNil(t *testing.T) {
t.Parallel()
reset := SetLogWriterForTest(nil)
t.Cleanup(reset)
stats := LogWriterStats()
if stats.Name != "user_access_logs" {
t.Fatalf("LogWriterStats().Name = %q, want user_access_logs", stats.Name)
}
if stats.Running {
t.Fatal("LogWriterStats().Running = true for nil writer, want false")
}
}