fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer
metrics; model insert hooks; latest-per-node and hourly metric/openresty
rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.

internal/repository/analytics/access_log_test.go

@@ -99,6 +99,9 @@ type mockConn struct {
 	batchQuery    string
 	prepareCalled bool
 	preparedQuery string
+	queries       []string
+	queryArgs     [][]any
+	queryFn       func(ctx context.Context, query string, args ...any) (driver.Rows, error)
 }
 
 func (m *mockConn) Contributors() []string { return nil }
@@ -107,8 +110,13 @@ func (m *mockConn) ServerVersion() (*driver.ServerVersion, error) { return nil,
 
 func (m *mockConn) Select(_ context.Context, _ any, _ string, _ ...any) error { return nil }
 
-func (m *mockConn) Query(_ context.Context, _ string, _ ...any) (driver.Rows, error) {
-	return nil, nil
+func (m *mockConn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) {
+	m.queries = append(m.queries, query)
+	m.queryArgs = append(m.queryArgs, args)
+	if m.queryFn != nil {
+		return m.queryFn(ctx, query, args...)
+	}
+	return &mockRows{}, nil
 }
 
 func (m *mockConn) QueryRow(_ context.Context, _ string, _ ...any) driver.Row { return nil }
@@ -158,4 +166,96 @@ func (m *mockBatch) Rows() int { return len(m.rows) }
 
 func (m *mockBatch) Columns() []column.Interface { return nil }
 
-func (m *mockBatch) Close() error { return nil }
+func (m *mockBatch) Close() error { return nil }
+
+// mockRows is an empty driver.Rows implementation for query-path unit tests.
+type mockRows struct {
+	index int
+	data  [][]any
+	err   error
+}
+
+func (m *mockRows) Next() bool {
+	if m.err != nil {
+		return false
+	}
+	if m.index >= len(m.data) {
+		return false
+	}
+	m.index++
+	return true
+}
+
+func (m *mockRows) Scan(dest ...any) error {
+	if m.err != nil {
+		return m.err
+	}
+	if m.index == 0 || m.index > len(m.data) {
+		return nil
+	}
+	row := m.data[m.index-1]
+	for i := range dest {
+		if i >= len(row) {
+			break
+		}
+		if err := assignMockScanValue(dest[i], row[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *mockRows) ScanStruct(_ any) error { return nil }
+
+func (m *mockRows) ColumnTypes() []driver.ColumnType { return nil }
+
+func (m *mockRows) Totals(_ ...any) error { return nil }
+
+func (m *mockRows) Columns() []string { return nil }
+
+func (m *mockRows) Close() error { return nil }
+
+func (m *mockRows) Err() error { return m.err }
+
+func (m *mockRows) HasData() bool { return len(m.data) > 0 }
+
+func assignMockScanValue(dest any, value any) error {
+	switch d := dest.(type) {
+	case *string:
+		if v, ok := value.(string); ok {
+			*d = v
+		}
+	case *uint64:
+		switch v := value.(type) {
+		case uint64:
+			*d = v
+		case int:
+			*d = uint64(v)
+		case int64:
+			*d = uint64(v)
+		}
+	case *int64:
+		switch v := value.(type) {
+		case int64:
+			*d = v
+		case int:
+			*d = int64(v)
+		case uint64:
+			*d = int64(v)
+		}
+	case *float64:
+		switch v := value.(type) {
+		case float64:
+			*d = v
+		case float32:
+			*d = float64(v)
+		case int:
+			*d = float64(v)
+		}
+	case *time.Time:
+		if v, ok := value.(time.Time); ok {
+			*d = v
+		}
+	}
+	return nil
+}

internal/repository/analytics/clickhouse_maintenance.go

@@ -6,21 +6,48 @@ package analytics
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 )
 
+// DDL TTL days for analytics tables (must match goose ClickHouse migrations).
+const (
+	// TableTTLDaysNodeAccessLogs is the of_node_access_logs TTL (90 days).
+	TableTTLDaysNodeAccessLogs = 90
+	// TableTTLDaysNodeMetricSnapshots is the of_node_metric_snapshots TTL (30 days).
+	TableTTLDaysNodeMetricSnapshots = 30
+	// TableTTLDaysNodeRequestReports is the of_node_request_reports TTL (30 days).
+	TableTTLDaysNodeRequestReports = 30
+	// TableTTLDaysNodeObs is the of_node_obs_* TTL (30 days).
+	TableTTLDaysNodeObs = 30
+	// TableTTLDaysUserAccessLogs is the w_user_access_logs TTL (180 days).
+	TableTTLDaysUserAccessLogs = 180
+)
+
 const (
 	// CleanupModeTTLMaterialize expires rows via table TTL instead of ALTER DELETE mutations.
+	// This is not a hard delete: deleted_count must stay 0; use EligibleCount as an estimate.
 	CleanupModeTTLMaterialize = "ttl_materialize"
-	// CleanupModeTruncate removes all rows via TRUNCATE TABLE.
+	// CleanupModeTruncate removes all rows via TRUNCATE TABLE (hard delete).
 	CleanupModeTruncate = "truncate"
 )
 
 // CleanupOutcome describes a non-mutation ClickHouse cleanup operation.
+//
+// For CleanupModeTruncate:
+//   - DeletedCount and EligibleCount are the rows removed by TRUNCATE.
+//
+// For CleanupModeTTLMaterialize:
+//   - DeletedCount is always 0 (MATERIALIZE TTL is async / not a counted hard delete).
+//   - EligibleCount is an estimate of rows already past the table TTL policy (not an
+//     arbitrary user cutoff younger than the DDL TTL).
+//   - TableTTLDays is the DDL TTL used for the estimate and materialize.
 type CleanupOutcome struct {
 	EligibleCount int64
+	DeletedCount  int64
 	Mode          string
+	TableTTLDays  int
 }
 
 func countClickHouseRows(ctx context.Context, conn driver.Conn, countSQL string, countArgs []any) (int64, error) {
@@ -39,21 +66,41 @@ func materializeTableTTL(ctx context.Context, conn driver.Conn, tableName string
 	return nil
 }
 
-func expireRowsViaTTL(ctx context.Context, conn driver.Conn, tableName string, countSQL string, countArgs []any) (CleanupOutcome, error) {
+// tableTTLCutoff returns the UTC instant at which rows become eligible under a fixed day TTL.
+func tableTTLCutoff(tableTTLDays int, now time.Time) time.Time {
+	if tableTTLDays < 1 {
+		tableTTLDays = 1
+	}
+	return now.UTC().Add(-time.Duration(tableTTLDays) * 24 * time.Hour)
+}
+
+// materializeExpiredByTableTTL force-materializes table TTL and estimates rows past that policy.
+//
+// countSQL must count only rows older than the table TTL (callers pass tableTTLCutoff args).
+// Node-scoped filters may be used for the estimate only; MATERIALIZE is always table-global.
+func materializeExpiredByTableTTL(
+	ctx context.Context,
+	conn driver.Conn,
+	tableName string,
+	tableTTLDays int,
+	countSQL string,
+	countArgs []any,
+) (CleanupOutcome, error) {
+	outcome := CleanupOutcome{
+		Mode:         CleanupModeTTLMaterialize,
+		TableTTLDays: tableTTLDays,
+	}
 	count, err := countClickHouseRows(ctx, conn, countSQL, countArgs)
 	if err != nil {
 		return CleanupOutcome{}, err
 	}
-	if count == 0 {
-		return CleanupOutcome{Mode: CleanupModeTTLMaterialize}, nil
-	}
+	outcome.EligibleCount = count
+	// Always force materialize so ClickHouse applies the DDL TTL policy promptly.
+	// EligibleCount is informational only; MATERIALIZE does not return a deleted row count.
 	if err := materializeTableTTL(ctx, conn, tableName); err != nil {
 		return CleanupOutcome{}, err
 	}
-	return CleanupOutcome{
-		EligibleCount: count,
-		Mode:          CleanupModeTTLMaterialize,
-	}, nil
+	return outcome, nil
 }
 
 func truncateClickHouseTable(ctx context.Context, conn driver.Conn, tableName string) (CleanupOutcome, error) {
@@ -69,6 +116,7 @@ func truncateClickHouseTable(ctx context.Context, conn driver.Conn, tableName st
 	}
 	return CleanupOutcome{
 		EligibleCount: count,
+		DeletedCount:  count,
 		Mode:          CleanupModeTruncate,
 	}, nil
-}
+}

internal/repository/analytics/clickhouse_maintenance_test.go

@@ -0,0 +1,37 @@
+// Copyright 2026 Arctel.net
+// SPDX-License-Identifier: Apache-2.0
+
+package analytics
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTableTTLCutoff(t *testing.T) {
+	now := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
+	got := tableTTLCutoff(30, now)
+	assert.Equal(t, now.Add(-30*24*time.Hour), got)
+
+	got = tableTTLCutoff(90, now)
+	assert.Equal(t, now.Add(-90*24*time.Hour), got)
+
+	// Invalid TTL floors to 1 day.
+	got = tableTTLCutoff(0, now)
+	assert.Equal(t, now.Add(-24*time.Hour), got)
+}
+
+func TestCleanupModeConstants(t *testing.T) {
+	assert.Equal(t, "ttl_materialize", CleanupModeTTLMaterialize)
+	assert.Equal(t, "truncate", CleanupModeTruncate)
+}
+
+func TestTableTTLDaysMatchDDL(t *testing.T) {
+	assert.Equal(t, 90, TableTTLDaysNodeAccessLogs)
+	assert.Equal(t, 30, TableTTLDaysNodeMetricSnapshots)
+	assert.Equal(t, 30, TableTTLDaysNodeRequestReports)
+	assert.Equal(t, 30, TableTTLDaysNodeObs)
+	assert.Equal(t, 180, TableTTLDaysUserAccessLogs)
+}

internal/repository/analytics/clickhouse_stats.go

@@ -9,16 +9,20 @@ import (
 
 	"github.com/Rain-kl/Wavelet/internal/config"
 	"github.com/Rain-kl/Wavelet/internal/db"
+	"github.com/Rain-kl/Wavelet/internal/db/batchwriter"
 )
 
-// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure.
+// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
+// and in-process batch writer queue health.
 type ClickHouseOperationalStats struct {
-	Database          string `json:"database"`
-	ActiveParts       int64  `json:"active_parts"`
-	TotalRows         int64  `json:"total_rows"`
-	PendingMutations  int64  `json:"pending_mutations"`
-	AsyncInsertQueue  int64  `json:"async_insert_queue"`
-	AsyncInsertBytes  int64  `json:"async_insert_bytes"`
+	Database         string `json:"database"`
+	ActiveParts      int64  `json:"active_parts"`
+	TotalRows        int64  `json:"total_rows"`
+	PendingMutations int64  `json:"pending_mutations"`
+	AsyncInsertQueue int64  `json:"async_insert_queue"`
+	AsyncInsertBytes int64  `json:"async_insert_bytes"`
+	// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
+	BatchWriters []batchwriter.Stats `json:"batch_writers,omitempty"`
 }
 
 // GetClickHouseOperationalStats returns operational metrics for the configured database.
@@ -67,4 +71,4 @@ WHERE database = ?`
 	}
 
 	return stats, nil
-}
+}

internal/repository/analytics/node_access_log_delete.go

@@ -9,7 +9,7 @@ import (
 	"time"
 )
 
-// DeleteAllNodeAccessLogs deletes all node access logs.
+// DeleteAllNodeAccessLogs hard-deletes all node access logs via TRUNCATE.
 func DeleteAllNodeAccessLogs(ctx context.Context) (int64, error) {
 	conn, err := nodeAccessLogConn()
 	if err != nil {
@@ -19,47 +19,69 @@ func DeleteAllNodeAccessLogs(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }
 
-// DeleteNodeAccessLogsBefore expires logs older than cutoff via table TTL.
-func DeleteNodeAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := nodeAccessLogConn()
+// DeleteNodeAccessLogsBefore force-materializes of_node_access_logs table TTL.
+//
+// The cutoff argument is kept for call-site compatibility and is not used to select rows:
+// ClickHouse MATERIALIZE TTL only enforces the DDL policy (TableTTLDaysNodeAccessLogs).
+// Returns an estimate of rows past table TTL as the int64 (not a hard-deleted count).
+// Callers that need honest API fields should prefer MaterializeNodeAccessLogsTTL.
+func DeleteNodeAccessLogsBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeAccessLogsTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeAccessLogsTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeAccessLogsTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := nodeAccessLogConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeAccessLogTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeAccessLogs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
 		[]any{cutoff},
 	)
+}
+
+// DeleteNodeAccessLogsByNodeBefore force-materializes table-global TTL.
+//
+// Node-scoped hard delete is not supported: MATERIALIZE TTL is table-global.
+// The returned count is an estimate of rows for nodeID past table TTL only.
+func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeAccessLogsTTLByNode(ctx, nodeID)
 	if err != nil {
 		return 0, err
 	}
 	return outcome.EligibleCount, nil
 }
 
-// DeleteNodeAccessLogsByNodeBefore expires logs for a node older than cutoff via table TTL.
-func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
+// MaterializeNodeAccessLogsTTLByNode materializes table-global TTL and estimates node-scoped rows past TTL.
+func MaterializeNodeAccessLogsTTLByNode(ctx context.Context, nodeID string) (CleanupOutcome, error) {
 	conn, err := nodeAccessLogConn()
 	if err != nil {
-		return 0, err
+		return CleanupOutcome{}, err
 	}
 	tableName := nodeAccessLogTableName()
-	before = before.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeAccessLogs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
-		[]any{nodeID, before},
+		[]any{nodeID, cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
-}
+}

internal/repository/analytics/node_observability.go

@@ -45,6 +45,27 @@ ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
 	return scanNodeMetricSnapshotRows(rows)
 }
 
+// ListLatestNodeMetricSnapshots returns the latest metric snapshot per node_id.
+// Uses ClickHouse LIMIT 1 BY so dashboard health does not depend on a global raw LIMIT.
+func ListLatestNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return nil, err
+	}
+	clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
+	sql := fmt.Sprintf(`
+SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
+FROM %s
+WHERE %s
+ORDER BY %s%s`, nodeMetricSnapshotTableName(), clause, nodeObservabilityCapturedAtOrderClause(), clickHouseLimit1ByNodeIDClause)
+	rows, err := conn.Query(ctx, sql, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list latest node metric snapshots: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	return scanNodeMetricSnapshotRows(rows)
+}
+
 // ListNodeRequestReports returns request reports matching filter.
 func ListNodeRequestReports(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeRequestReport, error) {
 	conn, err := observabilityConn()
@@ -70,6 +91,27 @@ ORDER BY %s`, tableName, clause, nodeObservabilityWindowEndedAtOrderClause())
 	return scanNodeRequestReportRows(rows)
 }
 
+// ListLatestNodeRequestReports returns the latest request report per node_id.
+// Uses ClickHouse LIMIT 1 BY so dashboard traffic health is not skewed by a global raw LIMIT.
+func ListLatestNodeRequestReports(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeRequestReport, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return nil, err
+	}
+	clause, args := buildNodeObservabilityFilterClause(filter, "window_ended_at")
+	sql := fmt.Sprintf(`
+SELECT id, node_id, window_started_at, window_ended_at, request_count, error_count, unique_visitor_count, status_codes_json, top_domains_json, source_countries_json, created_at
+FROM %s
+WHERE %s
+ORDER BY %s%s`, nodeRequestReportTableName(), clause, nodeObservabilityWindowEndedAtOrderClause(), clickHouseLimit1ByNodeIDClause)
+	rows, err := conn.Query(ctx, sql, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list latest node request reports: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	return scanNodeRequestReportRows(rows)
+}
+
 // ListNodeObsOpenresty returns OpenResty observations matching filter.
 func ListNodeObsOpenresty(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsOpenresty, error) {
 	conn, err := observabilityConn()
@@ -248,6 +290,10 @@ func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error)
 const nodeTrafficHourlyTableName = "of_node_traffic_hourly"
 
 // NodeTrafficHourly is an hourly traffic rollup row.
+//
+// UniqueVisitorCount is a peak per-window estimate from short request reports
+// (MV uses max()), not true distinct visitors across the hour. SummingMergeTree
+// may still inflate residual unmerged parts; do not present as exact UV.
 type NodeTrafficHourly struct {
 	NodeID             string
 	Hour               time.Time
@@ -258,7 +304,8 @@ type NodeTrafficHourly struct {
 
 // NodeMetricHourly is an hourly metric snapshot aggregation row.
 //
-// Disk and host network counters are cumulative; deltas use consecutive
+// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
+// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
 // lagInFrame samples per node (negative deltas after counter reset are dropped).
 type NodeMetricHourly struct {
 	Hour                      time.Time
@@ -292,7 +339,7 @@ SELECT
 	hour,
 	sum(request_count) AS request_count,
 	sum(error_count) AS error_count,
-	sum(unique_visitor_count) AS unique_visitor_count
+	max(unique_visitor_count) AS unique_visitor_count
 FROM %s
 WHERE %s
 GROUP BY node_id, hour
@@ -322,8 +369,43 @@ ORDER BY hour ASC`, nodeTrafficHourlyTableName, clause)
 }
 
 // ListNodeMetricHourly returns hourly metric snapshot aggregates matching filter.
-// Capacity uses sample averages; network/disk use consecutive counter deltas via lagInFrame.
+// Prefers of_node_metric_capacity_hourly rollups; falls back to raw lagInFrame on empty/error.
 func ListNodeMetricHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
+	if rows, err := listNodeMetricHourlyFromRollup(ctx, filter); err == nil && len(rows) > 0 {
+		return rows, nil
+	}
+	return listNodeMetricHourlyFromRaw(ctx, filter)
+}
+
+func listNodeMetricHourlyFromRollup(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return nil, err
+	}
+	clause, args := buildNodeObservabilityFilterClause(filter, "hour")
+	sql := fmt.Sprintf(`
+SELECT
+	hour,
+	if(sum(cpu_usage_count) > 0, sum(cpu_usage_sum) / sum(cpu_usage_count), 0) AS average_cpu_usage_percent,
+	if(sum(memory_usage_count) > 0, sum(memory_usage_sum) / sum(memory_usage_count), 0) AS average_memory_usage_percent,
+	sum(greatest(network_rx_max - network_rx_min, 0)) AS network_rx_bytes,
+	sum(greatest(network_tx_max - network_tx_min, 0)) AS network_tx_bytes,
+	sum(greatest(disk_read_max - disk_read_min, 0)) AS disk_read_bytes,
+	sum(greatest(disk_write_max - disk_write_min, 0)) AS disk_write_bytes,
+	toUInt64(uniqExact(node_id)) AS reported_nodes
+FROM %s
+WHERE %s
+GROUP BY hour
+ORDER BY hour ASC`, nodeMetricCapacityHourlyTableName(), clause)
+	rows, err := conn.Query(ctx, sql, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list node metric hourly from rollup: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	return scanNodeMetricHourlyRows(rows)
+}
+
+func listNodeMetricHourlyFromRaw(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
 	conn, err := observabilityConn()
 	if err != nil {
 		return nil, err
@@ -372,7 +454,10 @@ ORDER BY hour ASC`, tableName, clause)
 		return nil, fmt.Errorf("list node metric hourly: %w", err)
 	}
 	defer func() { _ = rows.Close() }()
+	return scanNodeMetricHourlyRows(rows)
+}
 
+func scanNodeMetricHourlyRows(rows driver.Rows) ([]NodeMetricHourly, error) {
 	result := make([]NodeMetricHourly, 0)
 	for rows.Next() {
 		var (
@@ -407,7 +492,39 @@ ORDER BY hour ASC`, tableName, clause)
 }
 
 // ListNodeOpenrestyHourly returns hourly OpenResty observation aggregates matching filter.
+// Prefers of_node_openresty_hourly rollups; falls back to raw lagInFrame on empty/error.
 func ListNodeOpenrestyHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeOpenrestyHourly, error) {
+	if rows, err := listNodeOpenrestyHourlyFromRollup(ctx, filter); err == nil && len(rows) > 0 {
+		return rows, nil
+	}
+	return listNodeOpenrestyHourlyFromRaw(ctx, filter)
+}
+
+func listNodeOpenrestyHourlyFromRollup(ctx context.Context, filter NodeObservabilityFilter) ([]NodeOpenrestyHourly, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return nil, err
+	}
+	clause, args := buildNodeObservabilityFilterClause(filter, "hour")
+	sql := fmt.Sprintf(`
+SELECT
+	hour,
+	sum(greatest(openresty_rx_max - openresty_rx_min, 0)) AS openresty_rx_bytes,
+	sum(greatest(openresty_tx_max - openresty_tx_min, 0)) AS openresty_tx_bytes,
+	toUInt64(uniqExact(node_id)) AS reported_nodes
+FROM %s
+WHERE %s
+GROUP BY hour
+ORDER BY hour ASC`, nodeOpenrestyHourlyTableName(), clause)
+	rows, err := conn.Query(ctx, sql, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list node openresty hourly from rollup: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	return scanNodeOpenrestyHourlyRows(rows)
+}
+
+func listNodeOpenrestyHourlyFromRaw(ctx context.Context, filter NodeObservabilityFilter) ([]NodeOpenrestyHourly, error) {
 	conn, err := observabilityConn()
 	if err != nil {
 		return nil, err
@@ -442,7 +559,10 @@ ORDER BY hour ASC`, tableName, clause)
 		return nil, fmt.Errorf("list node openresty hourly: %w", err)
 	}
 	defer func() { _ = rows.Close() }()
+	return scanNodeOpenrestyHourlyRows(rows)
+}
 
+func scanNodeOpenrestyHourlyRows(rows driver.Rows) ([]NodeOpenrestyHourly, error) {
 	result := make([]NodeOpenrestyHourly, 0)
 	for rows.Next() {
 		var (

internal/repository/analytics/node_observability_delete.go

@@ -9,7 +9,7 @@ import (
 	"time"
 )
 
-// DeleteAllNodeMetricSnapshots deletes all node metric snapshots.
+// DeleteAllNodeMetricSnapshots hard-deletes all node metric snapshots via TRUNCATE.
 func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -19,31 +19,39 @@ func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }
 
-// DeleteNodeMetricSnapshotsBefore expires metric snapshots captured before cutoff via table TTL.
-func DeleteNodeMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeMetricSnapshotsBefore force-materializes of_node_metric_snapshots table TTL.
+// cutoff is ignored; see MaterializeNodeMetricSnapshotsTTL.
+func DeleteNodeMetricSnapshotsBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeMetricSnapshotsTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeMetricSnapshotsTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeMetricSnapshotsTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeMetricSnapshotTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeMetricSnapshots
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }
 
-// DeleteAllNodeRequestReports deletes all node request reports.
+// DeleteAllNodeRequestReports hard-deletes all node request reports via TRUNCATE.
 func DeleteAllNodeRequestReports(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -53,31 +61,39 @@ func DeleteAllNodeRequestReports(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }
 
-// DeleteNodeRequestReportsBefore expires request reports ending before cutoff via table TTL.
-func DeleteNodeRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeRequestReportsBefore force-materializes of_node_request_reports table TTL.
+// cutoff is ignored; see MaterializeNodeRequestReportsTTL.
+func DeleteNodeRequestReportsBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeRequestReportsTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeRequestReportsTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeRequestReportsTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeRequestReportTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeRequestReports
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE window_ended_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }
 
-// DeleteAllNodeObsOpenresty deletes all OpenResty observations.
+// DeleteAllNodeObsOpenresty hard-deletes all OpenResty observations via TRUNCATE.
 func DeleteAllNodeObsOpenresty(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -87,31 +103,39 @@ func DeleteAllNodeObsOpenresty(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }
 
-// DeleteNodeObsOpenrestyBefore expires OpenResty observations captured before cutoff via table TTL.
-func DeleteNodeObsOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeObsOpenrestyBefore force-materializes of_node_obs_openresty table TTL.
+// cutoff is ignored; see MaterializeNodeObsOpenrestyTTL.
+func DeleteNodeObsOpenrestyBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeObsOpenrestyTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeObsOpenrestyTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeObsOpenrestyTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeObsOpenrestyTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeObs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }
 
-// DeleteAllNodeObsFrps deletes all FRPS observations.
+// DeleteAllNodeObsFrps hard-deletes all FRPS observations via TRUNCATE.
 func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -121,31 +145,39 @@ func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }
 
-// DeleteNodeObsFrpsBefore expires FRPS observations captured before cutoff via table TTL.
-func DeleteNodeObsFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeObsFrpsBefore force-materializes of_node_obs_frps table TTL.
+// cutoff is ignored; see MaterializeNodeObsFrpsTTL.
+func DeleteNodeObsFrpsBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeObsFrpsTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeObsFrpsTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeObsFrpsTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeObsFrpsTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeObs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }
 
-// DeleteAllNodeObsFrpc deletes all FRPC observations.
+// DeleteAllNodeObsFrpc hard-deletes all FRPC observations via TRUNCATE.
 func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -155,26 +187,34 @@ func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
+	return outcome.DeletedCount, nil
+}
+
+// DeleteNodeObsFrpcBefore force-materializes of_node_obs_frpc table TTL.
+// cutoff is ignored; see MaterializeNodeObsFrpcTTL.
+func DeleteNodeObsFrpcBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeObsFrpcTTL(ctx)
+	if err != nil {
+		return 0, err
+	}
 	return outcome.EligibleCount, nil
 }
 
-// DeleteNodeObsFrpcBefore expires FRPC observations captured before cutoff via table TTL.
-func DeleteNodeObsFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
+// MaterializeNodeObsFrpcTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeObsFrpcTTL(ctx context.Context) (CleanupOutcome, error) {
 	conn, err := observabilityConn()
 	if err != nil {
-		return 0, err
+		return CleanupOutcome{}, err
 	}
 	tableName := nodeObsFrpcTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeObs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
-}
+}

internal/repository/analytics/node_observability_filter.go

@@ -61,3 +61,14 @@ func nodeObsFrpsTableName() string {
 func nodeObsFrpcTableName() string {
 	return "of_node_obs_frpc"
 }
+
+func nodeMetricCapacityHourlyTableName() string {
+	return "of_node_metric_capacity_hourly"
+}
+
+func nodeOpenrestyHourlyTableName() string {
+	return "of_node_openresty_hourly"
+}
+
+// clickHouseLimit1ByNodeIDClause selects the first row per node_id after ORDER BY.
+const clickHouseLimit1ByNodeIDClause = " LIMIT 1 BY node_id"

internal/repository/analytics/node_observability_latest_test.go

@@ -0,0 +1,166 @@
+// Copyright 2026 Arctel.net
+// SPDX-License-Identifier: Apache-2.0
+
+package analytics
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/Rain-kl/Wavelet/internal/db"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListLatestNodeMetricSnapshots_UsesLimit1ByNodeID(t *testing.T) {
+	ctx := context.Background()
+	mock := &mockConn{}
+	db.SetChConnForTest(mock)
+	t.Cleanup(func() { db.SetChConnForTest(nil) })
+
+	since := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
+	_, err := ListLatestNodeMetricSnapshots(ctx, NodeObservabilityFilter{Since: since})
+	require.NoError(t, err)
+	require.Len(t, mock.queries, 1)
+	assert.Contains(t, mock.queries[0], "LIMIT 1 BY node_id")
+	assert.Contains(t, mock.queries[0], nodeMetricSnapshotTableName())
+	assert.Contains(t, mock.queries[0], "captured_at DESC")
+	assert.NotContains(t, mock.queries[0], "LIMIT ?")
+	require.Len(t, mock.queryArgs, 1)
+	require.Len(t, mock.queryArgs[0], 1)
+	assert.Equal(t, since, mock.queryArgs[0][0])
+}
+
+func TestListLatestNodeRequestReports_UsesLimit1ByNodeID(t *testing.T) {
+	ctx := context.Background()
+	mock := &mockConn{}
+	db.SetChConnForTest(mock)
+	t.Cleanup(func() { db.SetChConnForTest(nil) })
+
+	_, err := ListLatestNodeRequestReports(ctx, NodeObservabilityFilter{NodeID: "node-a"})
+	require.NoError(t, err)
+	require.Len(t, mock.queries, 1)
+	assert.Contains(t, mock.queries[0], "LIMIT 1 BY node_id")
+	assert.Contains(t, mock.queries[0], nodeRequestReportTableName())
+	assert.Contains(t, mock.queries[0], "window_ended_at DESC")
+	require.Len(t, mock.queryArgs, 1)
+	require.Len(t, mock.queryArgs[0], 1)
+	assert.Equal(t, "node-a", mock.queryArgs[0][0])
+}
+
+func TestListNodeMetricHourly_PrefersRollup(t *testing.T) {
+	ctx := context.Background()
+	hour := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
+	mock := &mockConn{
+		queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
+			if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
+				return &mockRows{data: [][]any{{
+					hour, 42.5, 60.0, int64(100), int64(200), int64(10), int64(20), uint64(2),
+				}}}, nil
+			}
+			return nil, errors.New("raw path should not be used when rollup has rows")
+		},
+	}
+	db.SetChConnForTest(mock)
+	t.Cleanup(func() { db.SetChConnForTest(nil) })
+
+	rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{})
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, 42.5, rows[0].AverageCPUUsagePercent)
+	assert.Equal(t, 60.0, rows[0].AverageMemoryUsagePercent)
+	assert.Equal(t, int64(100), rows[0].NetworkRxBytes)
+	assert.Equal(t, 2, rows[0].ReportedNodes)
+	require.Len(t, mock.queries, 1)
+	assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
+}
+
+func TestListNodeMetricHourly_FallsBackToRawOnRollupError(t *testing.T) {
+	ctx := context.Background()
+	hour := time.Date(2026, 7, 10, 13, 0, 0, 0, time.UTC)
+	mock := &mockConn{
+		queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
+			if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
+				return nil, errors.New("rollup missing")
+			}
+			if strings.Contains(query, nodeMetricSnapshotTableName()) {
+				return &mockRows{data: [][]any{{
+					hour, 10.0, 20.0, int64(1), int64(2), int64(3), int64(4), uint64(1),
+				}}}, nil
+			}
+			return &mockRows{}, nil
+		},
+	}
+	db.SetChConnForTest(mock)
+	t.Cleanup(func() { db.SetChConnForTest(nil) })
+
+	rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{})
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, 10.0, rows[0].AverageCPUUsagePercent)
+	assert.Equal(t, int64(3), rows[0].DiskReadBytes)
+	require.GreaterOrEqual(t, len(mock.queries), 2)
+	assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
+	assert.Contains(t, mock.queries[1], "lagInFrame")
+}
+
+func TestListNodeOpenrestyHourly_PrefersRollup(t *testing.T) {
+	ctx := context.Background()
+	hour := time.Date(2026, 7, 10, 14, 0, 0, 0, time.UTC)
+	mock := &mockConn{
+		queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
+			if strings.Contains(query, nodeOpenrestyHourlyTableName()) {
+				return &mockRows{data: [][]any{{
+					hour, int64(50), int64(70), uint64(3),
+				}}}, nil
+			}
+			return nil, errors.New("raw path should not be used when rollup has rows")
+		},
+	}
+	db.SetChConnForTest(mock)
+	t.Cleanup(func() { db.SetChConnForTest(nil) })
+
+	rows, err := ListNodeOpenrestyHourly(ctx, NodeObservabilityFilter{})
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, int64(50), rows[0].OpenrestyRxBytes)
+	assert.Equal(t, int64(70), rows[0].OpenrestyTxBytes)
+	assert.Equal(t, 3, rows[0].ReportedNodes)
+}
+
+func TestListNodeOpenrestyHourly_FallsBackToRawOnEmptyRollup(t *testing.T) {
+	ctx := context.Background()
+	hour := time.Date(2026, 7, 10, 15, 0, 0, 0, time.UTC)
+	mock := &mockConn{
+		queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
+			if strings.Contains(query, nodeOpenrestyHourlyTableName()) {
+				return &mockRows{}, nil
+			}
+			if strings.Contains(query, nodeObsOpenrestyTableName()) {
+				return &mockRows{data: [][]any{{
+					hour, int64(9), int64(8), uint64(1),
+				}}}, nil
+			}
+			return &mockRows{}, nil
+		},
+	}
+	db.SetChConnForTest(mock)
+	t.Cleanup(func() { db.SetChConnForTest(nil) })
+
+	rows, err := ListNodeOpenrestyHourly(ctx, NodeObservabilityFilter{})
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, int64(9), rows[0].OpenrestyRxBytes)
+	require.GreaterOrEqual(t, len(mock.queries), 2)
+	assert.Contains(t, mock.queries[1], "lagInFrame")
+}
+
+func TestNonNegativeCounterRange(t *testing.T) {
+	assert.Equal(t, int64(0), nonNegativeCounterRange(10, 10))
+	assert.Equal(t, int64(0), nonNegativeCounterRange(5, 10))
+	assert.Equal(t, int64(15), nonNegativeCounterRange(25, 10))
+}