perf(clickhouse): P0 write path — remove heartbeat DELETE, batchwriter MinBatchSize, tune chwriter

internal/apps/openflare/agent/observability.go

@@ -24,8 +24,6 @@ const (
 	healthSeverityInfo           = "info"
 	healthSeverityWarning        = "warning"
 	healthSeverityCritical       = "critical"
-	nodeAccessLogRetentionDays   = 90
-	nodeAccessLogRetentionWindow = nodeAccessLogRetentionDays * 24 * time.Hour
 	accessLogPathMaxLength       = 100
 	healthEventMessageMaxLength  = 4096
 )
@@ -241,11 +239,7 @@ func persistNodeAccessLogs(ctx context.Context, nodeID string, records []*model.
 	if len(records) == 0 {
 		return nil
 	}
-	if err := model.InsertOpenFlareAccessLogsBatch(ctx, records); err != nil {
-		return err
-	}
-	_, err := model.DeleteOpenFlareAccessLogsByNodeBefore(ctx, nodeID, reportedAt.Add(-nodeAccessLogRetentionWindow))
-	return err
+	return model.InsertOpenFlareAccessLogsBatch(ctx, records)
 }
 
 func reconcileNodeHealthEvents(tx *gorm.DB, nodeID string, events []NodeHealthEvent, reportedAt time.Time) error {

internal/apps/openflare/chwriter/writer.go

@@ -21,8 +21,8 @@ import (
 
 const (
 	observabilityQueueSize    = 5_000
-	observabilityMaxBatchSize = 200
-	observabilityFlushEvery   = 2 * time.Second
+	observabilityMaxBatchSize = 500
+	observabilityFlushEvery   = 5 * time.Second
 
 	nodeAccessLogQueueSize    = 10_000
 	nodeAccessLogMaxBatchSize = 1_000
@@ -41,6 +41,9 @@ var (
 
 	metricSnapshotDedup *dedupSet
 	requestReportDedup  *dedupSet
+	openrestyDedup      *dedupSet
+	frpsDedup           *dedupSet
+	frpcDedup           *dedupSet
 )
 
 // Init starts OpenFlare ClickHouse batch writers. Safe to call multiple times.
@@ -52,6 +55,9 @@ func Init(ctx context.Context) {
 	initOnce.Do(func() {
 		metricSnapshotDedup = newDedupSet()
 		requestReportDedup = newDedupSet()
+		openrestyDedup = newDedupSet()
+		frpsDedup = newDedupSet()
+		frpcDedup = newDedupSet()
 
 		metricSnapshotWriter = mustNewObservabilityWriter("metric_snapshots", analyticsrepo.BatchInsertNodeMetricSnapshots)
 		requestReportWriter = mustNewObservabilityWriter("request_reports", analyticsrepo.BatchInsertNodeRequestReports)
@@ -130,6 +136,10 @@ func QueueOpenrestyObservation(observation analyticsmodel.NodeObsOpenresty) {
 	if openrestyWriter == nil {
 		return
 	}
+	key := fmt.Sprintf("%s|%d", observation.NodeID, observation.CapturedAt.UTC().UnixNano())
+	if !openrestyDedup.markIfNew(key) {
+		return
+	}
 	openrestyWriter.TryEnqueue(observation)
 }
 
@@ -138,6 +148,10 @@ func QueueFrpsObservation(observation analyticsmodel.NodeObsFrps) {
 	if frpsWriter == nil {
 		return
 	}
+	key := fmt.Sprintf("%s|%d", observation.NodeID, observation.CapturedAt.UTC().UnixNano())
+	if !frpsDedup.markIfNew(key) {
+		return
+	}
 	frpsWriter.TryEnqueue(observation)
 }
 
@@ -146,6 +160,10 @@ func QueueFrpcObservation(observation analyticsmodel.NodeObsFrpc) {
 	if frpcWriter == nil {
 		return
 	}
+	key := fmt.Sprintf("%s|%d", observation.NodeID, observation.CapturedAt.UTC().UnixNano())
+	if !frpcDedup.markIfNew(key) {
+		return
+	}
 	frpcWriter.TryEnqueue(observation)
 }