perf(clickhouse): P1 TTL migrations, ORDER BY tune, remote_addr normalization

internal/apps/openflare/tasks/database_cleanup.go

@@ -21,12 +21,21 @@ const (
 	DatabaseCleanupTargetMetricSnapshots = "node_metric_snapshots"
 	// DatabaseCleanupTargetRequestReports is the API cleanup target for request reports.
 	DatabaseCleanupTargetRequestReports = "node_request_reports"
+	// DatabaseCleanupTargetObsOpenresty is the API cleanup target for OpenResty observations.
+	DatabaseCleanupTargetObsOpenresty = "node_obs_openresty"
+	// DatabaseCleanupTargetObsFrps is the API cleanup target for FRPS observations.
+	DatabaseCleanupTargetObsFrps = "node_obs_frps"
+	// DatabaseCleanupTargetObsFrpc is the API cleanup target for FRPC observations.
+	DatabaseCleanupTargetObsFrpc = "node_obs_frpc"
 )
 
 var databaseCleanupTargets = map[string]string{
 	DatabaseCleanupTargetAccessLogs:      "访问日志",
 	DatabaseCleanupTargetMetricSnapshots: "性能快照",
 	DatabaseCleanupTargetRequestReports:  "请求聚合",
+	DatabaseCleanupTargetObsOpenresty:      "OpenResty 观测",
+	DatabaseCleanupTargetObsFrps:           "FRPS 观测",
+	DatabaseCleanupTargetObsFrpc:           "FRPC 观测",
 }
 
 // DatabaseCleanupInput describes a manual observability cleanup request.
@@ -111,6 +120,9 @@ func RunDatabaseAutoCleanupOnce(ctx context.Context, now time.Time) (*DatabaseAu
 		DatabaseCleanupTargetAccessLogs,
 		DatabaseCleanupTargetMetricSnapshots,
 		DatabaseCleanupTargetRequestReports,
+		DatabaseCleanupTargetObsOpenresty,
+		DatabaseCleanupTargetObsFrps,
+		DatabaseCleanupTargetObsFrpc,
 	} {
 		result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
 			Target:        target,
@@ -137,6 +149,12 @@ func deleteAllObservabilityRows(ctx context.Context, target string) (int64, erro
 		return model.DeleteAllOpenFlareMetricSnapshots(ctx)
 	case DatabaseCleanupTargetRequestReports:
 		return model.DeleteAllOpenFlareRequestReports(ctx)
+	case DatabaseCleanupTargetObsOpenresty:
+		return model.DeleteAllOpenFlareNodeObservationOpenresty(ctx)
+	case DatabaseCleanupTargetObsFrps:
+		return model.DeleteAllOpenFlareNodeObservationFrps(ctx)
+	case DatabaseCleanupTargetObsFrpc:
+		return model.DeleteAllOpenFlareNodeObservationFrpc(ctx)
 	default:
 		return 0, errors.New("unsupported cleanup target")
 	}
@@ -150,6 +168,12 @@ func deleteObservabilityRowsBefore(ctx context.Context, target string, cutoff ti
 		return model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
 	case DatabaseCleanupTargetRequestReports:
 		return model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
+	case DatabaseCleanupTargetObsOpenresty:
+		return model.DeleteOpenFlareNodeObservationOpenrestyBefore(ctx, cutoff)
+	case DatabaseCleanupTargetObsFrps:
+		return model.DeleteOpenFlareNodeObservationFrpsBefore(ctx, cutoff)
+	case DatabaseCleanupTargetObsFrpc:
+		return model.DeleteOpenFlareNodeObservationFrpcBefore(ctx, cutoff)
 	default:
 		return 0, errors.New("unsupported cleanup target")
 	}

internal/apps/openflare/tasks/database_cleanup_test.go

@@ -131,7 +131,7 @@ func TestRunDatabaseAutoCleanupOnceDeletesAllObservabilityTargets(t *testing.T)
 	summary, err := RunDatabaseAutoCleanupOnce(ctx, now)
 	require.NoError(t, err)
 	require.NotNil(t, summary)
-	require.Len(t, summary.Results, 3)
+	require.Len(t, summary.Results, 6)
 
 	accessLogs, err := model.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
 	require.NoError(t, err)

internal/db/migrator/goose/clickhouse/202607020001_optimize_analytics_tables.sql

@@ -0,0 +1,23 @@
+-- +goose Up
+-- Add TTL policies to analytics tables so ClickHouse can expire rows automatically.
+ALTER TABLE w_user_access_logs MODIFY TTL created_at + INTERVAL 180 DAY;
+
+ALTER TABLE of_node_access_logs MODIFY TTL logged_at + INTERVAL 90 DAY;
+
+ALTER TABLE of_node_metric_snapshots MODIFY TTL captured_at + INTERVAL 30 DAY;
+
+ALTER TABLE of_node_request_reports MODIFY TTL window_ended_at + INTERVAL 30 DAY;
+
+ALTER TABLE of_node_obs_openresty MODIFY TTL captured_at + INTERVAL 30 DAY;
+
+ALTER TABLE of_node_obs_frps MODIFY TTL captured_at + INTERVAL 30 DAY;
+
+ALTER TABLE of_node_obs_frpc MODIFY TTL captured_at + INTERVAL 30 DAY;
+
+-- Narrow ORDER BY for node access logs to match common filter patterns.
+-- Requires ClickHouse 24.10+ (MODIFY ORDER BY). On older versions this statement
+-- may fail and require manual table recreation; TTL changes above are still safe.
+ALTER TABLE of_node_access_logs MODIFY ORDER BY (node_id, logged_at, status_code);
+
+-- +goose Down
+-- TTL and ORDER BY changes cannot be safely reversed without recreating tables.

internal/model/openflare_observability.go

@@ -396,6 +396,36 @@ func DeleteAllOpenFlareRequestReports(ctx context.Context) (int64, error) {
 	return currentObservabilityStore().DeleteAllRequestReports(ctx)
 }
 
+// DeleteOpenFlareNodeObservationOpenrestyBefore deletes OpenResty observations captured before cutoff.
+func DeleteOpenFlareNodeObservationOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
+	return currentObservabilityStore().DeleteNodeObservationOpenrestyBefore(ctx, cutoff)
+}
+
+// DeleteAllOpenFlareNodeObservationOpenresty deletes all OpenResty observations.
+func DeleteAllOpenFlareNodeObservationOpenresty(ctx context.Context) (int64, error) {
+	return currentObservabilityStore().DeleteAllNodeObservationOpenresty(ctx)
+}
+
+// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
+func DeleteOpenFlareNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
+	return currentObservabilityStore().DeleteNodeObservationFrpsBefore(ctx, cutoff)
+}
+
+// DeleteAllOpenFlareNodeObservationFrps deletes all FRPS observations.
+func DeleteAllOpenFlareNodeObservationFrps(ctx context.Context) (int64, error) {
+	return currentObservabilityStore().DeleteAllNodeObservationFrps(ctx)
+}
+
+// DeleteOpenFlareNodeObservationFrpcBefore deletes FRPC observations captured before cutoff.
+func DeleteOpenFlareNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
+	return currentObservabilityStore().DeleteNodeObservationFrpcBefore(ctx, cutoff)
+}
+
+// DeleteAllOpenFlareNodeObservationFrpc deletes all FRPC observations.
+func DeleteAllOpenFlareNodeObservationFrpc(ctx context.Context) (int64, error) {
+	return currentObservabilityStore().DeleteAllNodeObservationFrpc(ctx)
+}
+
 // DeleteOpenFlareHealthEventsByNodeID deletes all health events for a node.
 func DeleteOpenFlareHealthEventsByNodeID(ctx context.Context, nodeID string) (int64, error) {
 	conn := db.DB(ctx)

internal/repository/analytics/node_access_log_writer.go

@@ -6,6 +6,7 @@ package analytics
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/Rain-kl/Wavelet/internal/db"
@@ -41,7 +42,7 @@ func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAc
 			id,
 			logItem.NodeID,
 			logItem.LoggedAt.UTC(),
-			logItem.RemoteAddr,
+			strings.TrimSpace(logItem.RemoteAddr),
 			logItem.Region,
 			logItem.Host,
 			logItem.Path,