fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer
metrics; model insert hooks; latest-per-node and hourly metric/openresty
rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.
This commit is contained in:
ryan
2026-07-10 10:34:04 +08:00
parent 9b3555c569
commit 160e63558f
37 changed files with 1779 additions and 269 deletions
+13 -3
View File
@@ -16,10 +16,18 @@ import (
)
const (
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
// async_insert 仅挂在运行时 ChConn(写路径)上,不进入 migrator OpenDB:
// 迁移/DDL 需要同步可见结果,且不应走异步 insert 缓冲。
//
// 为何启用:batchwriter 仍可能在短间隔内写出相对小的块;服务端 async_insert
// 把多次 INSERT 合并成更大 part,减轻 3c6g 上 background merge 的 CPU 压力。
// wait_for_async_insert=1:调用方在 flush 返回前等待落盘,避免进程崩溃丢批。
// max_data_size / busy_timeout:约 10MB 或 ~2s 触发刷出,在延迟与 part 数之间折中。
clickhouseAsyncInsertMaxDataSize = 10_000_000
clickhouseAsyncInsertBusyTimeoutMs = 1000
clickhouseAsyncInsertBusyTimeoutMs = 2000
)
var (
@@ -52,6 +60,8 @@ func init() {
log.Println("[ClickHouse] connection established successfully")
}
// buildClickHouseOptions builds the runtime native client options (queries + batch inserts).
// Migrator uses a separate clickhouse.OpenDB path without async_insert settings.
func buildClickHouseOptions() *clickhouse.Options {
cfg := config.Config.ClickHouse