fix(clickhouse): harden R/W path P0–P3 (cleanup, durability, rollups)

Honest TTL cleanup semantics; enqueue-safe dedup with flush retry and writer metrics; model insert hooks; latest-per-node and hourly metric/openresty rollups; small-host pool/async defaults, traffic hourly TTL, and UV labeling.
2026-10-02 06:56:36 +08:00 · 2026-07-10 10:34:04 +08:00
parent 9b3555c569
commit 160e63558f
37 changed files with 1779 additions and 269 deletions
@@ -9,7 +9,7 @@ import (
 	"time"
 )

-// DeleteAllNodeMetricSnapshots deletes all node metric snapshots.
+// DeleteAllNodeMetricSnapshots hard-deletes all node metric snapshots via TRUNCATE.
 func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -19,31 +19,39 @@ func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }

-// DeleteNodeMetricSnapshotsBefore expires metric snapshots captured before cutoff via table TTL.
-func DeleteNodeMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeMetricSnapshotsBefore force-materializes of_node_metric_snapshots table TTL.
+// cutoff is ignored; see MaterializeNodeMetricSnapshotsTTL.
+func DeleteNodeMetricSnapshotsBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeMetricSnapshotsTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeMetricSnapshotsTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeMetricSnapshotsTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeMetricSnapshotTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeMetricSnapshots
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }

-// DeleteAllNodeRequestReports deletes all node request reports.
+// DeleteAllNodeRequestReports hard-deletes all node request reports via TRUNCATE.
 func DeleteAllNodeRequestReports(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -53,31 +61,39 @@ func DeleteAllNodeRequestReports(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }

-// DeleteNodeRequestReportsBefore expires request reports ending before cutoff via table TTL.
-func DeleteNodeRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeRequestReportsBefore force-materializes of_node_request_reports table TTL.
+// cutoff is ignored; see MaterializeNodeRequestReportsTTL.
+func DeleteNodeRequestReportsBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeRequestReportsTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeRequestReportsTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeRequestReportsTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeRequestReportTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeRequestReports
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE window_ended_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }

-// DeleteAllNodeObsOpenresty deletes all OpenResty observations.
+// DeleteAllNodeObsOpenresty hard-deletes all OpenResty observations via TRUNCATE.
 func DeleteAllNodeObsOpenresty(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -87,31 +103,39 @@ func DeleteAllNodeObsOpenresty(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }

-// DeleteNodeObsOpenrestyBefore expires OpenResty observations captured before cutoff via table TTL.
-func DeleteNodeObsOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeObsOpenrestyBefore force-materializes of_node_obs_openresty table TTL.
+// cutoff is ignored; see MaterializeNodeObsOpenrestyTTL.
+func DeleteNodeObsOpenrestyBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeObsOpenrestyTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeObsOpenrestyTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeObsOpenrestyTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeObsOpenrestyTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeObs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }

-// DeleteAllNodeObsFrps deletes all FRPS observations.
+// DeleteAllNodeObsFrps hard-deletes all FRPS observations via TRUNCATE.
 func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -121,31 +145,39 @@ func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return outcome.EligibleCount, nil
+	return outcome.DeletedCount, nil
 }

-// DeleteNodeObsFrpsBefore expires FRPS observations captured before cutoff via table TTL.
-func DeleteNodeObsFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
-	conn, err := observabilityConn()
+// DeleteNodeObsFrpsBefore force-materializes of_node_obs_frps table TTL.
+// cutoff is ignored; see MaterializeNodeObsFrpsTTL.
+func DeleteNodeObsFrpsBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeObsFrpsTTL(ctx)
 	if err != nil {
 		return 0, err
 	}
+	return outcome.EligibleCount, nil
+}
+
+// MaterializeNodeObsFrpsTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeObsFrpsTTL(ctx context.Context) (CleanupOutcome, error) {
+	conn, err := observabilityConn()
+	if err != nil {
+		return CleanupOutcome{}, err
+	}
 	tableName := nodeObsFrpsTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeObs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
 }

-// DeleteAllNodeObsFrpc deletes all FRPC observations.
+// DeleteAllNodeObsFrpc hard-deletes all FRPC observations via TRUNCATE.
 func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
 	conn, err := observabilityConn()
 	if err != nil {
@@ -155,26 +187,34 @@ func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
+	return outcome.DeletedCount, nil
+}
+
+// DeleteNodeObsFrpcBefore force-materializes of_node_obs_frpc table TTL.
+// cutoff is ignored; see MaterializeNodeObsFrpcTTL.
+func DeleteNodeObsFrpcBefore(ctx context.Context, _ time.Time) (int64, error) {
+	outcome, err := MaterializeNodeObsFrpcTTL(ctx)
+	if err != nil {
+		return 0, err
+	}
 	return outcome.EligibleCount, nil
 }

-// DeleteNodeObsFrpcBefore expires FRPC observations captured before cutoff via table TTL.
-func DeleteNodeObsFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
+// MaterializeNodeObsFrpcTTL force-materializes table TTL and reports an honest outcome.
+func MaterializeNodeObsFrpcTTL(ctx context.Context) (CleanupOutcome, error) {
 	conn, err := observabilityConn()
 	if err != nil {
-		return 0, err
+		return CleanupOutcome{}, err
 	}
 	tableName := nodeObsFrpcTableName()
-	cutoff = cutoff.UTC()
-	outcome, err := expireRowsViaTTL(
+	ttlDays := TableTTLDaysNodeObs
+	cutoff := tableTTLCutoff(ttlDays, time.Now())
+	return materializeExpiredByTableTTL(
 		ctx,
 		conn,
 		tableName,
+		ttlDays,
 		fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
 		[]any{cutoff},
 	)
-	if err != nil {
-		return 0, err
-	}
-	return outcome.EligibleCount, nil
-}
+}