tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-03 07:06:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(clickhouse): Phase 2 legacy governance — TTL cleanup, unified pool, MV, ops API

Browse Source 
- Replace retention ALTER DELETE with MATERIALIZE TTL; use TRUNCATE for delete-all
- Remove GORM ClickHouse pool; migrate user access log reads to ChConn
- Drop query-side trim(remote_addr); enable wait_for_async_insert=1
- Add of_node_traffic_hourly MV and dashboard traffic trend fallback
- Add GET /admin/status/clickhouse operational metrics endpoint
This commit is contained in:
ryan
2026-07-02 15:47:38 +08:00
parent 38946d1af5
commit 58624db397
30 changed files with 1351 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/repository/analytics/node_access_log_delete.go
+44 -39
View File
@@ -11,50 +11,55 @@ import (
// DeleteAllNodeAccessLogs deletes all node access logs.
func DeleteAllNodeAccessLogs(ctx context.Context) (int64, error) {
tableName := nodeAccessLogTableName()
return deleteNodeAccessLogsWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
}
// DeleteNodeAccessLogsBefore deletes logs older than cutoff.
func DeleteNodeAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
tableName := nodeAccessLogTableName()
cutoff = cutoff.UTC()
return deleteNodeAccessLogsWithCount(
ctx,
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
[]any{cutoff},
fmt.Sprintf("ALTER TABLE %s DELETE WHERE logged_at < ?", tableName),
cutoff,
)
}
// DeleteNodeAccessLogsByNodeBefore deletes logs for a node older than cutoff.
func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
tableName := nodeAccessLogTableName()
before = before.UTC()
return deleteNodeAccessLogsWithCount(
ctx,
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
[]any{nodeID, before},
fmt.Sprintf("ALTER TABLE %s DELETE WHERE node_id = ? AND logged_at < ?", tableName),
nodeID, before,
)
}
func deleteNodeAccessLogsWithCount(ctx context.Context, countSQL string, countArgs []any, deleteSQL string, deleteArgs ...any) (int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, err
}
var count uint64
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
return 0, fmt.Errorf("count node access logs for delete: %w", err)
outcome, err := truncateClickHouseTable(ctx, conn, nodeAccessLogTableName())
if err != nil {
return 0, err
}
if count == 0 {
return 0, nil
return outcome.EligibleCount, nil
}
// DeleteNodeAccessLogsBefore expires logs older than cutoff via table TTL.
func DeleteNodeAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, err
}
if err := conn.Exec(ctx, deleteSQL, deleteArgs...); err != nil {
return 0, fmt.Errorf("delete node access logs: %w", err)
tableName := nodeAccessLogTableName()
cutoff = cutoff.UTC()
outcome, err := expireRowsViaTTL(
ctx,
conn,
tableName,
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
[]any{cutoff},
)
if err != nil {
return 0, err
}
return safeInt64Count(count), nil
return outcome.EligibleCount, nil
}
// DeleteNodeAccessLogsByNodeBefore expires logs for a node older than cutoff via table TTL.
func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, err
}
tableName := nodeAccessLogTableName()
before = before.UTC()
outcome, err := expireRowsViaTTL(
ctx,
conn,
tableName,
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
[]any{nodeID, before},
)
if err != nil {
return 0, err
}
return outcome.EligibleCount, nil
}