fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer
metrics; model insert hooks; latest-per-node and hourly metric/openresty
rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.
This commit is contained in:
ryan
2026-07-10 10:34:04 +08:00
parent 9b3555c569
commit 160e63558f
37 changed files with 1779 additions and 269 deletions
+6 -3
View File
@@ -134,11 +134,14 @@ func applyClickHouseDefaults(c *configModel) {
if c.ClickHouse.Username == "" {
c.ClickHouse.Username = "default"
}
// Pool / buffer defaults target small control-plane hosts (e.g. 3c6g):
// oversized open/idle pools waste RAM and amplify concurrent CH pressure;
// large block buffers add client memory without helping our small batch inserts.
if c.ClickHouse.MaxIdleConn <= 0 {
c.ClickHouse.MaxIdleConn = 20
c.ClickHouse.MaxIdleConn = 8
}
if c.ClickHouse.MaxOpenConn <= 0 {
c.ClickHouse.MaxOpenConn = 50
c.ClickHouse.MaxOpenConn = 16
}
if c.ClickHouse.ConnMaxLifetime <= 0 {
c.ClickHouse.ConnMaxLifetime = 3600
@@ -147,7 +150,7 @@ func applyClickHouseDefaults(c *configModel) {
c.ClickHouse.DialTimeout = 5
}
if c.ClickHouse.BlockBufferSize == 0 {
c.ClickHouse.BlockBufferSize = 100
c.ClickHouse.BlockBufferSize = 32
}
}
+7 -6
View File
@@ -71,18 +71,19 @@ type databaseReplicaConfig struct {
Password string `mapstructure:"password"`
}
// clickhouse 配置
// clickHouseConfig ClickHouse 原生客户端配置。
// 连接池 / block_buffer 默认值按小型控制面主机(如 3c6g)收敛,见 applyClickHouseDefaults。
type clickHouseConfig struct {
Enabled bool `mapstructure:"enabled"`
Hosts []string `mapstructure:"hosts"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Database string `mapstructure:"database"`
MaxIdleConn int `mapstructure:"max_idle_conn"`
MaxOpenConn int `mapstructure:"max_open_conn"`
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"`
DialTimeout int `mapstructure:"dial_timeout"`
BlockBufferSize uint8 `mapstructure:"block_buffer_size"`
MaxIdleConn int `mapstructure:"max_idle_conn"` // 默认 8
MaxOpenConn int `mapstructure:"max_open_conn"` // 默认 16
ConnMaxLifetime int `mapstructure:"conn_max_lifetime"` // 秒
DialTimeout int `mapstructure:"dial_timeout"` // 秒
BlockBufferSize uint8 `mapstructure:"block_buffer_size"` // 默认 32
}
// redisConfig Redis配置