perf(clickhouse): Phase 2 legacy governance — TTL cleanup, unified pool, MV, ops API

- Replace retention ALTER DELETE with MATERIALIZE TTL; use TRUNCATE for delete-all
- Remove GORM ClickHouse pool; migrate user access log reads to ChConn
- Drop query-side trim(remote_addr); enable wait_for_async_insert=1
- Add of_node_traffic_hourly MV and dashboard traffic trend fallback
- Add GET /admin/status/clickhouse operational metrics endpoint
This commit is contained in:
ryan
2026-07-02 15:47:38 +08:00
parent 38946d1af5
commit 58624db397
30 changed files with 1351 additions and 436 deletions
+1
View File
@@ -20,6 +20,7 @@ sidebar: false
### 修改
- ClickHouse 遗留治理 Phase 2:保留期清理改为 TTL `MATERIALIZE TTL`(全量清理使用 `TRUNCATE`),消除定时 `ALTER DELETE` mutation;移除 GORM 双连接池并统一 `ChConn` 读路径;查询侧去除 `trim(remote_addr)`;`wait_for_async_insert` 调整为 1;新增 `/admin/status/clickhouse` 运维指标与 `of_node_traffic_hourly` 预聚合 MV。
- ClickHouse 写入路径优化:移除 Agent 心跳路径中的同步 `ALTER DELETE` 保留清理;`batchwriter` 新增 `MinBatchSize` 抑制过小批次定时 flush;可观测 writer 批次提升至 500、flush 间隔 5s,并为 OpenResty/FRPS/FRPC 补全去重。
- ClickHouse 客户端启用 `async_insert` 异步写入缓冲,并调高 `block_buffer_size` 与连接池默认值,降低小 part 与连接争用。
- Dashboard 与节点可观测 API 消除无 `LIMIT` 全表扫描、增加短 TTL 内存缓存,前端轮询间隔分别调整为 60s/30s。
+219 -5
View File
@@ -309,6 +309,7 @@ const docTemplate = `{
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "认证源 ID 或名称",
"name": "id",
"in": "path",
@@ -386,6 +387,7 @@ const docTemplate = `{
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "认证源 ID 或名称",
"name": "id",
"in": "path",
@@ -453,6 +455,7 @@ const docTemplate = `{
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "认证源 ID 或名称",
"name": "id",
"in": "path",
@@ -1363,6 +1366,7 @@ const docTemplate = `{
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "通道ID",
"name": "id",
"in": "path",
@@ -1416,6 +1420,7 @@ const docTemplate = `{
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "通道ID",
"name": "id",
"in": "path",
@@ -1872,6 +1877,67 @@ const docTemplate = `{
}
}
},
"/api/v1/admin/status/clickhouse": {
"get": {
"security": [
{
"SessionCookie": []
}
],
"description": "返回 ClickHouse parts、mutation、async_insert 队列等运维指标,需要管理员权限",
"produces": [
"application/json"
],
"tags": [
"admin"
],
"summary": "获取 ClickHouse 运行指标",
"responses": {
"200": {
"description": "获取成功",
"schema": {
"allOf": [
{
"$ref": "#/definitions/response.Any"
},
{
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/analytics.ClickHouseOperationalStats"
}
}
}
]
}
},
"400": {
"description": "ClickHouse 未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"403": {
"description": "无管理员权限",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"500": {
"description": "内部错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
}
}
}
},
"/api/v1/admin/system-configs": {
"get": {
"security": [
@@ -3368,6 +3434,7 @@ const docTemplate = `{
},
{
"type": "integer",
"format": "int64",
"description": "上传用户 ID",
"name": "user_id",
"in": "query"
@@ -3691,6 +3758,11 @@ const docTemplate = `{
],
"summary": "获取用户列表",
"parameters": [
{
"type": "string",
"name": "email",
"in": "query"
},
{
"minimum": 1,
"type": "integer",
@@ -3909,6 +3981,92 @@ const docTemplate = `{
}
}
},
"put": {
"security": [
{
"SessionCookie": []
}
],
"description": "更新指定用户的昵称、邮箱、管理员权限,并可选重置密码,需要管理员权限",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"admin"
],
"summary": "更新用户信息",
"parameters": [
{
"type": "integer",
"description": "用户 ID",
"name": "id",
"in": "path",
"required": true
},
{
"description": "更新参数",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/user.updateUserRequest"
}
}
],
"responses": {
"200": {
"description": "更新成功",
"schema": {
"allOf": [
{
"$ref": "#/definitions/response.Any"
},
{
"type": "object",
"properties": {
"data": {
"type": "string"
}
}
}
]
}
},
"400": {
"description": "参数错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"403": {
"description": "无管理员权限或尝试修改自身权限",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"404": {
"description": "用户不存在",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"500": {
"description": "内部错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
}
}
},
"delete": {
"security": [
{
@@ -11144,6 +11302,7 @@ const docTemplate = `{
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "外部帐号绑定记录 ID",
"name": "id",
"in": "path",
@@ -12948,6 +13107,29 @@ const docTemplate = `{
}
}
},
"analytics.ClickHouseOperationalStats": {
"type": "object",
"properties": {
"active_parts": {
"type": "integer"
},
"async_insert_bytes": {
"type": "integer"
},
"async_insert_queue": {
"type": "integer"
},
"database": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
}
}
},
"apply_log.CleanupInput": {
"type": "object",
"properties": {
@@ -13775,19 +13957,22 @@ const docTemplate = `{
"source_countries": {
"type": "object",
"additionalProperties": {
"type": "integer"
"type": "integer",
"format": "int64"
}
},
"status_codes": {
"type": "object",
"additionalProperties": {
"type": "integer"
"type": "integer",
"format": "int64"
}
},
"top_domains": {
"type": "object",
"additionalProperties": {
"type": "integer"
"type": "integer",
"format": "int64"
}
},
"unique_visitor_count": {
@@ -14217,7 +14402,7 @@ const docTemplate = `{
"type": "string"
},
"id": {
"type": "integer"
"type": "string"
},
"is_active": {
"type": "boolean"
@@ -14252,7 +14437,7 @@ const docTemplate = `{
"type": "string"
},
"id": {
"type": "integer"
"type": "string"
},
"is_active": {
"type": "boolean"
@@ -15092,6 +15277,11 @@ const docTemplate = `{
"UploadStatusPending": "待使用",
"UploadStatusUsed": "已使用"
},
"x-enum-descriptions": [
"待使用",
"已使用",
"已删除"
],
"x-enum-varnames": [
"UploadStatusPending",
"UploadStatusUsed",
@@ -18085,6 +18275,30 @@ const docTemplate = `{
}
}
},
"user.updateUserRequest": {
"type": "object",
"required": [
"email"
],
"properties": {
"email": {
"type": "string",
"maxLength": 255
},
"is_admin": {
"type": "boolean"
},
"nickname": {
"type": "string",
"maxLength": 64
},
"password": {
"type": "string",
"maxLength": 64,
"minLength": 8
}
}
},
"user.updateUserStatusRequest": {
"type": "object",
"properties": {
+1 -1
View File
@@ -1,7 +1,7 @@
# ClickHouse CPU 性能优化计划
> PLAN_ID: `63ba981b`
> 状态: 已完成
> 状态: 已完成(含 Phase 2 遗留治理)
> 目标: 完成 P0–P2 优化,降低 ClickHouse CPU 占用
## 背景
+219 -5
View File
@@ -302,6 +302,7 @@
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "认证源 ID 或名称",
"name": "id",
"in": "path",
@@ -379,6 +380,7 @@
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "认证源 ID 或名称",
"name": "id",
"in": "path",
@@ -446,6 +448,7 @@
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "认证源 ID 或名称",
"name": "id",
"in": "path",
@@ -1356,6 +1359,7 @@
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "通道ID",
"name": "id",
"in": "path",
@@ -1409,6 +1413,7 @@
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "通道ID",
"name": "id",
"in": "path",
@@ -1865,6 +1870,67 @@
}
}
},
"/api/v1/admin/status/clickhouse": {
"get": {
"security": [
{
"SessionCookie": []
}
],
"description": "返回 ClickHouse parts、mutation、async_insert 队列等运维指标,需要管理员权限",
"produces": [
"application/json"
],
"tags": [
"admin"
],
"summary": "获取 ClickHouse 运行指标",
"responses": {
"200": {
"description": "获取成功",
"schema": {
"allOf": [
{
"$ref": "#/definitions/response.Any"
},
{
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/analytics.ClickHouseOperationalStats"
}
}
}
]
}
},
"400": {
"description": "ClickHouse 未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"403": {
"description": "无管理员权限",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"500": {
"description": "内部错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
}
}
}
},
"/api/v1/admin/system-configs": {
"get": {
"security": [
@@ -3361,6 +3427,7 @@
},
{
"type": "integer",
"format": "int64",
"description": "上传用户 ID",
"name": "user_id",
"in": "query"
@@ -3684,6 +3751,11 @@
],
"summary": "获取用户列表",
"parameters": [
{
"type": "string",
"name": "email",
"in": "query"
},
{
"minimum": 1,
"type": "integer",
@@ -3902,6 +3974,92 @@
}
}
},
"put": {
"security": [
{
"SessionCookie": []
}
],
"description": "更新指定用户的昵称、邮箱、管理员权限,并可选重置密码,需要管理员权限",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"admin"
],
"summary": "更新用户信息",
"parameters": [
{
"type": "integer",
"description": "用户 ID",
"name": "id",
"in": "path",
"required": true
},
{
"description": "更新参数",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/user.updateUserRequest"
}
}
],
"responses": {
"200": {
"description": "更新成功",
"schema": {
"allOf": [
{
"$ref": "#/definitions/response.Any"
},
{
"type": "object",
"properties": {
"data": {
"type": "string"
}
}
}
]
}
},
"400": {
"description": "参数错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"403": {
"description": "无管理员权限或尝试修改自身权限",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"404": {
"description": "用户不存在",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"500": {
"description": "内部错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
}
}
},
"delete": {
"security": [
{
@@ -11137,6 +11295,7 @@
"parameters": [
{
"type": "integer",
"format": "int64",
"description": "外部帐号绑定记录 ID",
"name": "id",
"in": "path",
@@ -12941,6 +13100,29 @@
}
}
},
"analytics.ClickHouseOperationalStats": {
"type": "object",
"properties": {
"active_parts": {
"type": "integer"
},
"async_insert_bytes": {
"type": "integer"
},
"async_insert_queue": {
"type": "integer"
},
"database": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
}
}
},
"apply_log.CleanupInput": {
"type": "object",
"properties": {
@@ -13768,19 +13950,22 @@
"source_countries": {
"type": "object",
"additionalProperties": {
"type": "integer"
"type": "integer",
"format": "int64"
}
},
"status_codes": {
"type": "object",
"additionalProperties": {
"type": "integer"
"type": "integer",
"format": "int64"
}
},
"top_domains": {
"type": "object",
"additionalProperties": {
"type": "integer"
"type": "integer",
"format": "int64"
}
},
"unique_visitor_count": {
@@ -14210,7 +14395,7 @@
"type": "string"
},
"id": {
"type": "integer"
"type": "string"
},
"is_active": {
"type": "boolean"
@@ -14245,7 +14430,7 @@
"type": "string"
},
"id": {
"type": "integer"
"type": "string"
},
"is_active": {
"type": "boolean"
@@ -15085,6 +15270,11 @@
"UploadStatusPending": "待使用",
"UploadStatusUsed": "已使用"
},
"x-enum-descriptions": [
"待使用",
"已使用",
"已删除"
],
"x-enum-varnames": [
"UploadStatusPending",
"UploadStatusUsed",
@@ -18078,6 +18268,30 @@
}
}
},
"user.updateUserRequest": {
"type": "object",
"required": [
"email"
],
"properties": {
"email": {
"type": "string",
"maxLength": 255
},
"is_admin": {
"type": "boolean"
},
"nickname": {
"type": "string",
"maxLength": 64
},
"password": {
"type": "string",
"maxLength": 64,
"minLength": 8
}
}
},
"user.updateUserStatusRequest": {
"type": "object",
"properties": {
+140 -2
View File
@@ -169,6 +169,21 @@ definitions:
$ref: '#/definitions/github_com_Rain-kl_Wavelet_pkg_protocol.WAFIPGroup'
type: array
type: object
analytics.ClickHouseOperationalStats:
properties:
active_parts:
type: integer
async_insert_bytes:
type: integer
async_insert_queue:
type: integer
database:
type: string
pending_mutations:
type: integer
total_rows:
type: integer
type: object
apply_log.CleanupInput:
properties:
delete_all:
@@ -713,14 +728,17 @@ definitions:
type: integer
source_countries:
additionalProperties:
format: int64
type: integer
type: object
status_codes:
additionalProperties:
format: int64
type: integer
type: object
top_domains:
additionalProperties:
format: int64
type: integer
type: object
unique_visitor_count:
@@ -1004,7 +1022,7 @@ definitions:
created_by:
type: string
id:
type: integer
type: string
is_active:
type: boolean
main_config:
@@ -1027,7 +1045,7 @@ definitions:
created_by:
type: string
id:
type: integer
type: string
is_active:
type: boolean
version:
@@ -1593,6 +1611,10 @@ definitions:
UploadStatusDeleted: 已删除
UploadStatusPending: 待使用
UploadStatusUsed: 已使用
x-enum-descriptions:
- 待使用
- 已使用
- 已删除
x-enum-varnames:
- UploadStatusPending
- UploadStatusUsed
@@ -3577,6 +3599,23 @@ definitions:
website:
type: string
type: object
user.updateUserRequest:
properties:
email:
maxLength: 255
type: string
is_admin:
type: boolean
nickname:
maxLength: 64
type: string
password:
maxLength: 64
minLength: 8
type: string
required:
- email
type: object
user.updateUserStatusRequest:
properties:
is_active:
@@ -4085,6 +4124,7 @@ paths:
description: 删除指定认证源及其关联的所有外部帐号绑定记录,警告:删除后相关用户将无法通过该源登录,需要管理员权限
parameters:
- description: 认证源 ID 或名称
format: int64
in: path
name: id
required: true
@@ -4124,6 +4164,7 @@ paths:
description: 更新指定 ID 的认证源配置。若 client_secret 字段为空,则保留原有密钥不变,需要管理员权限
parameters:
- description: 认证源 ID 或名称
format: int64
in: path
name: id
required: true
@@ -4174,6 +4215,7 @@ paths:
description: 启用或禁用指定认证源。尝试启用时将验证 Client ID 和 Client Secret 是否已配置,需要管理员权限
parameters:
- description: 认证源 ID 或名称
format: int64
in: path
name: id
required: true
@@ -4670,6 +4712,7 @@ paths:
description: 根据ID删除消息通道,需要管理员权限
parameters:
- description: 通道ID
format: int64
in: path
name: id
required: true
@@ -4692,6 +4735,7 @@ paths:
description: 修改消息通道配置,需要管理员权限
parameters:
- description: 通道ID
format: int64
in: path
name: id
required: true
@@ -5016,6 +5060,42 @@ paths:
summary: 获取系统状态信息
tags:
- admin
/api/v1/admin/status/clickhouse:
get:
description: 返回 ClickHouse parts、mutation、async_insert 队列等运维指标,需要管理员权限
produces:
- application/json
responses:
"200":
description: 获取成功
schema:
allOf:
- $ref: '#/definitions/response.Any'
- properties:
data:
$ref: '#/definitions/analytics.ClickHouseOperationalStats'
type: object
"400":
description: ClickHouse 未启用
schema:
$ref: '#/definitions/response.Any'
"401":
description: 未登录
schema:
$ref: '#/definitions/response.Any'
"403":
description: 无管理员权限
schema:
$ref: '#/definitions/response.Any'
"500":
description: 内部错误
schema:
$ref: '#/definitions/response.Any'
security:
- SessionCookie: []
summary: 获取 ClickHouse 运行指标
tags:
- admin
/api/v1/admin/system-configs:
get:
description: 返回所有系统配置列表,支持按配置类型(system/business)过滤,需要管理员权限
@@ -5912,6 +5992,7 @@ paths:
name: extension
type: string
- description: 上传用户 ID
format: int64
in: query
name: user_id
type: integer
@@ -6108,6 +6189,9 @@ paths:
get:
description: 分页返回用户列表,支持按用户 ID 和用户名筛选,需要管理员权限
parameters:
- in: query
name: email
type: string
- in: query
minimum: 1
name: page
@@ -6291,6 +6375,59 @@ paths:
summary: 获取用户详情
tags:
- admin
put:
consumes:
- application/json
description: 更新指定用户的昵称、邮箱、管理员权限,并可选重置密码,需要管理员权限
parameters:
- description: 用户 ID
in: path
name: id
required: true
type: integer
- description: 更新参数
in: body
name: request
required: true
schema:
$ref: '#/definitions/user.updateUserRequest'
produces:
- application/json
responses:
"200":
description: 更新成功
schema:
allOf:
- $ref: '#/definitions/response.Any'
- properties:
data:
type: string
type: object
"400":
description: 参数错误
schema:
$ref: '#/definitions/response.Any'
"401":
description: 未登录
schema:
$ref: '#/definitions/response.Any'
"403":
description: 无管理员权限或尝试修改自身权限
schema:
$ref: '#/definitions/response.Any'
"404":
description: 用户不存在
schema:
$ref: '#/definitions/response.Any'
"500":
description: 内部错误
schema:
$ref: '#/definitions/response.Any'
security:
- SessionCookie: []
summary: 更新用户信息
tags:
- admin
/api/v1/admin/users/{id}/status:
put:
consumes:
@@ -10642,6 +10779,7 @@ paths:
description: 解除当前登录用户与指定外部帐号的绑定关系,需要登录
parameters:
- description: 外部帐号绑定记录 ID
format: int64
in: path
name: id
required: true