fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer metrics; model insert hooks; latest-per-node and hourly metric/openresty rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.
2026-10-04 07:06:36 +08:00 · 2026-07-10 10:34:04 +08:00
parent 9b3555c569
commit 160e63558f
37 changed files with 1779 additions and 269 deletions
@@ -0,0 +1,80 @@
+-- +goose Up
+-- Hourly capacity rollups (avg CPU/memory + counter min/max for in-hour delta approximation).
+-- Network/disk counters are cumulative; max-min within an hour approximates that hour's delta
+-- (cross-hour continuity is intentionally approximate for dashboard trends).
+CREATE TABLE IF NOT EXISTS of_node_metric_capacity_hourly
+(
+    node_id              String,
+    hour                 DateTime,
+    cpu_usage_sum        SimpleAggregateFunction(sum, Float64),
+    cpu_usage_count      SimpleAggregateFunction(sum, UInt64),
+    memory_usage_sum     SimpleAggregateFunction(sum, Float64),
+    memory_usage_count   SimpleAggregateFunction(sum, UInt64),
+    network_rx_min       SimpleAggregateFunction(min, Int64),
+    network_rx_max       SimpleAggregateFunction(max, Int64),
+    network_tx_min       SimpleAggregateFunction(min, Int64),
+    network_tx_max       SimpleAggregateFunction(max, Int64),
+    disk_read_min        SimpleAggregateFunction(min, Int64),
+    disk_read_max        SimpleAggregateFunction(max, Int64),
+    disk_write_min       SimpleAggregateFunction(min, Int64),
+    disk_write_max       SimpleAggregateFunction(max, Int64)
+)
+ENGINE = AggregatingMergeTree()
+PARTITION BY toYYYYMM(hour)
+ORDER BY (node_id, hour)
+TTL hour + INTERVAL 30 DAY;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_metric_capacity_hourly_mv
+TO of_node_metric_capacity_hourly
+AS
+SELECT
+    node_id,
+    toStartOfHour(captured_at) AS hour,
+    sum(cpu_usage_percent) AS cpu_usage_sum,
+    toUInt64(count()) AS cpu_usage_count,
+    sum(if(memory_total_bytes > 0, (memory_used_bytes * 100.0) / memory_total_bytes, 0)) AS memory_usage_sum,
+    toUInt64(countIf(memory_total_bytes > 0)) AS memory_usage_count,
+    min(network_rx_bytes) AS network_rx_min,
+    max(network_rx_bytes) AS network_rx_max,
+    min(network_tx_bytes) AS network_tx_min,
+    max(network_tx_bytes) AS network_tx_max,
+    min(disk_read_bytes) AS disk_read_min,
+    max(disk_read_bytes) AS disk_read_max,
+    min(disk_write_bytes) AS disk_write_min,
+    max(disk_write_bytes) AS disk_write_max
+FROM of_node_metric_snapshots
+GROUP BY node_id, hour;
+
+-- Hourly OpenResty counter rollups (min/max per node-hour for delta approximation).
+CREATE TABLE IF NOT EXISTS of_node_openresty_hourly
+(
+    node_id            String,
+    hour               DateTime,
+    openresty_rx_min   SimpleAggregateFunction(min, Int64),
+    openresty_rx_max   SimpleAggregateFunction(max, Int64),
+    openresty_tx_min   SimpleAggregateFunction(min, Int64),
+    openresty_tx_max   SimpleAggregateFunction(max, Int64)
+)
+ENGINE = AggregatingMergeTree()
+PARTITION BY toYYYYMM(hour)
+ORDER BY (node_id, hour)
+TTL hour + INTERVAL 30 DAY;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_openresty_hourly_mv
+TO of_node_openresty_hourly
+AS
+SELECT
+    node_id,
+    toStartOfHour(captured_at) AS hour,
+    min(openresty_rx_bytes) AS openresty_rx_min,
+    max(openresty_rx_bytes) AS openresty_rx_max,
+    min(openresty_tx_bytes) AS openresty_tx_min,
+    max(openresty_tx_bytes) AS openresty_tx_max
+FROM of_node_obs_openresty
+GROUP BY node_id, hour;
+
+-- +goose Down
+DROP VIEW IF EXISTS of_node_openresty_hourly_mv;
+DROP TABLE IF EXISTS of_node_openresty_hourly;
+DROP VIEW IF EXISTS of_node_metric_capacity_hourly_mv;
+DROP TABLE IF EXISTS of_node_metric_capacity_hourly;
@@ -0,0 +1,42 @@
+-- +goose Up
+-- Hourly traffic rollups: 30d TTL + UV aggregation semantics.
+--
+-- unique_visitor_count on of_node_request_reports is per short report window
+-- (agent local distinct count for that window only). Summing those values in the
+-- MV (and again via SummingMergeTree part merges) invents a "true UV" number that
+-- double-counts visitors across windows. Prefer max() as a peak-window estimate;
+-- still NOT distinct visitors across the hour — UI/API must not overclaim.
+
+ALTER TABLE of_node_traffic_hourly
+    MODIFY TTL toDateTime(hour) + INTERVAL 30 DAY;
+
+DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
+
+CREATE MATERIALIZED VIEW of_node_traffic_hourly_mv
+TO of_node_traffic_hourly
+AS
+SELECT
+    node_id,
+    toStartOfHour(window_ended_at) AS hour,
+    sum(request_count) AS request_count,
+    sum(error_count) AS error_count,
+    -- Peak per-window UV estimate for the hour; not true cross-window distinct UV.
+    max(unique_visitor_count) AS unique_visitor_count
+FROM of_node_request_reports
+GROUP BY node_id, hour;
+
+-- +goose Down
+-- TTL reverse is not safe without table rewrite; restore prior MV definition only.
+DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
+
+CREATE MATERIALIZED VIEW of_node_traffic_hourly_mv
+TO of_node_traffic_hourly
+AS
+SELECT
+    node_id,
+    toStartOfHour(window_ended_at) AS hour,
+    sum(request_count) AS request_count,
+    sum(error_count) AS error_count,
+    sum(unique_visitor_count) AS unique_visitor_count
+FROM of_node_request_reports
+GROUP BY node_id, hour;