fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer
metrics; model insert hooks; latest-per-node and hourly metric/openresty
rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.

internal/db/batchwriter/writer.go

@@ -9,22 +9,34 @@ package batchwriter
 import (
 	"context"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
 // FlushFunc persists a batch of queued items. It is invoked from the worker goroutine.
 type FlushFunc[T any] func(ctx context.Context, items []T) error
 
-// FlushErrorHandler is called when FlushFunc returns an error. The batch is discarded
-// after the handler returns; the worker continues processing.
-type FlushErrorHandler func(ctx context.Context, batchSize int, err error)
+// FlushErrorHandler is called when FlushFunc returns an error after optional retries.
+// The batch is discarded after the handler returns; the worker continues processing.
+// Handlers receive the failed items so callers can release dedup keys or re-queue.
+type FlushErrorHandler[T any] func(ctx context.Context, items []T, err error)
+
+// Stats is a point-in-time snapshot of Writer queue and failure counters.
+type Stats struct {
+	Name        string `json:"name"`
+	Depth       int    `json:"depth"`
+	Cap         int    `json:"cap"`
+	Drops       int64  `json:"drops"`
+	FlushErrors int64  `json:"flush_errors"`
+	Running     bool   `json:"running"`
+}
 
 // Writer buffers items and flushes them by size or interval.
 type Writer[T any] struct {
 	cfg   Config
 	flush FlushFunc[T]
 
-	onFlushError FlushErrorHandler
+	onFlushError FlushErrorHandler[T]
 	onDrop       func(T)
 
 	startOnce sync.Once
@@ -34,13 +46,16 @@ type Writer[T any] struct {
 	ch        chan T
 	workerCtx context.Context
 	done      chan struct{}
+
+	drops       atomic.Int64
+	flushErrors atomic.Int64
 }
 
 // Option configures optional Writer callbacks.
 type Option[T any] func(*Writer[T])
 
 // WithFlushErrorHandler registers a callback for flush failures.
-func WithFlushErrorHandler[T any](handler FlushErrorHandler) Option[T] {
+func WithFlushErrorHandler[T any](handler FlushErrorHandler[T]) Option[T] {
 	return func(w *Writer[T]) {
 		w.onFlushError = handler
 	}
@@ -168,6 +183,18 @@ func (w *Writer[T]) Cap() int {
 	return w.cfg.QueueSize
 }
 
+// Stats returns a point-in-time snapshot of queue depth and failure counters.
+func (w *Writer[T]) Stats() Stats {
+	return Stats{
+		Name:        w.cfg.Name,
+		Depth:       w.Len(),
+		Cap:         w.Cap(),
+		Drops:       w.drops.Load(),
+		FlushErrors: w.flushErrors.Load(),
+		Running:     w.Running(),
+	}
+}
+
 func (w *Writer[T]) run() {
 	ticker := time.NewTicker(w.cfg.FlushInterval)
 	defer ticker.Stop()
@@ -180,8 +207,9 @@ func (w *Writer[T]) run() {
 		}
 		items := append([]T(nil), batch...)
 		if err := w.flush(w.workerCtx, items); err != nil {
+			w.flushErrors.Add(1)
 			if w.onFlushError != nil {
-				w.onFlushError(w.workerCtx, len(items), err)
+				w.onFlushError(w.workerCtx, items, err)
 			}
 		}
 		batch = batch[:0]
@@ -228,8 +256,9 @@ func (w *Writer[T]) shouldFlushOnInterval(batchLen int, batchStartedAt time.Time
 }
 
 func (w *Writer[T]) notifyDrop(item T) {
+	w.drops.Add(1)
 	if w.onDrop == nil {
 		return
 	}
 	w.onDrop(item)
-}
+}

internal/db/batchwriter/writer_test.go

@@ -36,7 +36,7 @@ func TestWriterFlushesOnMaxBatchSize(t *testing.T) {
 	t.Parallel()
 
 	var (
-		mu     sync.Mutex
+		mu      sync.Mutex
 		batches [][]int
 	)
 	cfg := DefaultConfig()
@@ -385,23 +385,24 @@ func TestWriterInvokesFlushErrorHandler(t *testing.T) {
 	t.Parallel()
 
 	cfg := DefaultConfig()
+	cfg.Name = "test-flush-err"
 	cfg.MaxBatchSize = 1
 	cfg.FlushInterval = time.Hour
 
 	flushErr := errors.New("flush failed")
 	var (
-		mu        sync.Mutex
-		errCount  int
-		batchSize int
+		mu       sync.Mutex
+		errCount int
+		gotItems []int
 	)
 
 	writer, err := New[int](cfg, func(context.Context, []int) error {
 		return flushErr
-	}, WithFlushErrorHandler[int](func(_ context.Context, size int, err error) {
+	}, WithFlushErrorHandler[int](func(_ context.Context, items []int, err error) {
 		mu.Lock()
 		defer mu.Unlock()
 		errCount++
-		batchSize = size
+		gotItems = append([]int(nil), items...)
 		if !errors.Is(err, flushErr) {
 			t.Errorf("flush error = %v, want %v", err, flushErr)
 		}
@@ -434,13 +435,61 @@ func TestWriterInvokesFlushErrorHandler(t *testing.T) {
 
 	mu.Lock()
 	gotCount := errCount
-	gotSize := batchSize
+	items := gotItems
 	mu.Unlock()
 
 	if gotCount != 1 {
 		t.Fatalf("flush error handler count = %d, want 1", gotCount)
 	}
-	if gotSize != 1 {
-		t.Fatalf("flush error handler batch size = %d, want 1", gotSize)
+	if diff := cmp.Diff([]int{7}, items); diff != "" {
+		t.Fatalf("flush error handler items mismatch (-want +got):\n%s", diff)
 	}
-}
+
+	stats := writer.Stats()
+	if stats.FlushErrors != 1 {
+		t.Fatalf("Stats().FlushErrors = %d, want 1", stats.FlushErrors)
+	}
+	if stats.Name != "test-flush-err" {
+		t.Fatalf("Stats().Name = %q, want test-flush-err", stats.Name)
+	}
+}
+
+func TestWriterStatsTracksDrops(t *testing.T) {
+	t.Parallel()
+
+	cfg := DefaultConfig()
+	cfg.Name = "test-drops"
+	cfg.QueueSize = 1
+	cfg.MaxBatchSize = 10
+	cfg.FlushInterval = time.Hour
+
+	writer, err := New[int](cfg, func(context.Context, []int) error { return nil })
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+
+	writer.Start(context.Background())
+	t.Cleanup(func() {
+		stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		_ = writer.Stop(stopCtx)
+	})
+
+	if !writer.TryEnqueue(1) {
+		t.Fatal("TryEnqueue(1) = false, want true")
+	}
+	if writer.TryEnqueue(2) {
+		t.Fatal("TryEnqueue(2) = true, want false")
+	}
+
+	stats := writer.Stats()
+	if stats.Drops != 1 {
+		t.Fatalf("Stats().Drops = %d, want 1", stats.Drops)
+	}
+	if stats.Cap != 1 {
+		t.Fatalf("Stats().Cap = %d, want 1", stats.Cap)
+	}
+	if !stats.Running {
+		t.Fatal("Stats().Running = false, want true")
+	}
+}