fix(observability): restore 24h capacity/network/disk trends via CH hourly agg

Node and dashboard 24h capacity, network, and disk IO charts only used the
latest limited raw snapshots (120/500 rows), so historical hour buckets stayed
empty. Prefer ClickHouse hourly aggregates with counter deltas, and fall back
to raw snapshots when aggregation is unavailable.

internal/apps/openflare/dashboard/logics.go

@@ -137,21 +137,11 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
 	if err != nil {
 		return nil, err
 	}
-	trafficTrend := observability.BuildTrafficTrendPoints(now, reports)
-	if trafficHourly, hourlyErr := model.ListOpenFlareTrafficHourlySince(ctx, "", since); hourlyErr == nil && len(trafficHourly) > 0 {
-		trafficTrend = observability.BuildTrafficTrendPointsFromHourly(now, trafficHourly)
-	}
-
 	view := &OverviewView{
 		GeneratedAt:   now,
 		Nodes:         make([]NodeHealth, 0, len(nodes)),
 		Distributions: observability.BuildTrafficDistributions(reports, accessLogRegions, dashboardDistributionLimit),
-		Trends: observability.NodeTrends{
-			Traffic24h:  trafficTrend,
-			Capacity24h: observability.BuildCapacityTrendPoints(now, snapshots),
-			Network24h:  observability.BuildNetworkTrendPoints(now, snapshots, openrestySnapshots),
-			DiskIO24h:   observability.BuildDiskIOTrendPoints(now, snapshots),
-		},
+		Trends:        observability.BuildNodeTrends(ctx, now, "", snapshots, openrestySnapshots, reports),
 	}
 
 	var cpuNodeCount int

internal/apps/openflare/observability/analytics.go

@@ -4,6 +4,7 @@
 package observability
 
 import (
+	"context"
 	"encoding/json"
 	"sort"
 	"strings"
@@ -13,6 +14,7 @@ import (
 )
 
 const observabilityTrendBuckets = 24
+const unknownTrendNodeKey = "__unknown__"
 
 const (
 	healthEventStatusActive   = "active"
@@ -133,6 +135,12 @@ type diskCounterState struct {
 	seen  bool
 }
 
+type networkCounterState struct {
+	rx   int64
+	tx   int64
+	seen bool
+}
+
 func buildTrafficWindowSummary(report *model.OpenFlareRequestReport) *TrafficWindowSummary {
 	if report == nil {
 		return nil
@@ -257,6 +265,44 @@ func buildHealthSummary(
 	return summary
 }
 
+// BuildNodeTrends builds 24h trend series, preferring ClickHouse hourly aggregates
+// over limited raw snapshot windows so capacity/network/disk charts stay complete.
+func BuildNodeTrends(
+	ctx context.Context,
+	now time.Time,
+	nodeID string,
+	snapshots []*model.OpenFlareMetricSnapshot,
+	openrestyObs []*model.OpenFlareNodeObservationOpenresty,
+	reports []*model.OpenFlareRequestReport,
+) NodeTrends {
+	trendSince := now.Add(-24 * time.Hour)
+	trafficTrend := BuildTrafficTrendPoints(now, reports)
+	if trafficHourly, err := model.ListOpenFlareTrafficHourlySince(ctx, nodeID, trendSince); err == nil && len(trafficHourly) > 0 {
+		trafficTrend = BuildTrafficTrendPointsFromHourly(now, trafficHourly)
+	}
+
+	capacityTrend := BuildCapacityTrendPoints(now, snapshots)
+	networkTrend := BuildNetworkTrendPoints(now, snapshots, openrestyObs)
+	diskIOTrend := BuildDiskIOTrendPoints(now, snapshots)
+
+	metricHourly, metricErr := model.ListOpenFlareMetricHourlySince(ctx, nodeID, trendSince)
+	if metricErr == nil && len(metricHourly) > 0 {
+		capacityTrend = BuildCapacityTrendPointsFromHourly(now, metricHourly)
+		diskIOTrend = BuildDiskIOTrendPointsFromHourly(now, metricHourly)
+	}
+	openrestyHourly, openrestyErr := model.ListOpenFlareOpenrestyHourlySince(ctx, nodeID, trendSince)
+	if metricErr == nil && openrestyErr == nil && (len(metricHourly) > 0 || len(openrestyHourly) > 0) {
+		networkTrend = BuildNetworkTrendPointsFromHourly(now, metricHourly, openrestyHourly)
+	}
+
+	return NodeTrends{
+		Traffic24h:  trafficTrend,
+		Capacity24h: capacityTrend,
+		Network24h:  networkTrend,
+		DiskIO24h:   diskIOTrend,
+	}
+}
+
 // BuildTrafficTrendPointsFromHourly builds 24h traffic trend buckets from hourly rollups.
 func BuildTrafficTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlareTrafficHourly) []TrafficTrendPoint {
 	start := trendWindowStart(now)
@@ -336,7 +382,30 @@ func BuildCapacityTrendPoints(now time.Time, snapshots []*model.OpenFlareMetricS
 	return points
 }
 
+// BuildCapacityTrendPointsFromHourly builds 24h capacity trend buckets from hourly aggregates.
+func BuildCapacityTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlareMetricHourly) []CapacityTrendPoint {
+	start := trendWindowStart(now)
+	points := make([]CapacityTrendPoint, observabilityTrendBuckets)
+	for index := range points {
+		points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
+	}
+	for _, row := range hourly {
+		if row == nil {
+			continue
+		}
+		index, ok := trendBucketIndex(row.Hour, start)
+		if !ok {
+			continue
+		}
+		points[index].AverageCPUUsagePercent = row.AverageCPUUsagePercent
+		points[index].AverageMemoryUsagePercent = row.AverageMemoryUsagePercent
+		points[index].ReportedNodes = row.ReportedNodes
+	}
+	return points
+}
+
 // BuildNetworkTrendPoints builds 24h network trend buckets.
+// Host and OpenResty counters are cumulative; values are consecutive deltas.
 func BuildNetworkTrendPoints(
 	now time.Time,
 	snapshots []*model.OpenFlareMetricSnapshot,
@@ -349,24 +418,70 @@ func BuildNetworkTrendPoints(
 		points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
 		accumulators[index].nodes = make(map[string]struct{})
 	}
+	sort.Slice(snapshots, func(i int, j int) bool {
+		if snapshots[i].CapturedAt.Equal(snapshots[j].CapturedAt) {
+			return snapshots[i].NodeID < snapshots[j].NodeID
+		}
+		return snapshots[i].CapturedAt.Before(snapshots[j].CapturedAt)
+	})
+	previousHostByNode := make(map[string]networkCounterState, len(snapshots))
 	for _, snapshot := range snapshots {
+		if snapshot == nil {
+			continue
+		}
+		nodeKey := snapshot.NodeID
+		if nodeKey == "" {
+			nodeKey = unknownTrendNodeKey
+		}
+		previous := previousHostByNode[nodeKey]
+		previousHostByNode[nodeKey] = networkCounterState{
+			rx:   snapshot.NetworkRxBytes,
+			tx:   snapshot.NetworkTxBytes,
+			seen: true,
+		}
+		if !previous.seen {
+			continue
+		}
 		index, ok := trendBucketIndex(snapshot.CapturedAt, start)
 		if !ok {
 			continue
 		}
-		points[index].NetworkRxBytes += snapshot.NetworkRxBytes
-		points[index].NetworkTxBytes += snapshot.NetworkTxBytes
+		points[index].NetworkRxBytes += nonNegativeDelta(snapshot.NetworkRxBytes, previous.rx)
+		points[index].NetworkTxBytes += nonNegativeDelta(snapshot.NetworkTxBytes, previous.tx)
 		if snapshot.NodeID != "" {
 			accumulators[index].nodes[snapshot.NodeID] = struct{}{}
 		}
 	}
+	sort.Slice(openrestyObs, func(i int, j int) bool {
+		if openrestyObs[i].CapturedAt.Equal(openrestyObs[j].CapturedAt) {
+			return openrestyObs[i].NodeID < openrestyObs[j].NodeID
+		}
+		return openrestyObs[i].CapturedAt.Before(openrestyObs[j].CapturedAt)
+	})
+	previousOpenrestyByNode := make(map[string]networkCounterState, len(openrestyObs))
 	for _, obs := range openrestyObs {
+		if obs == nil {
+			continue
+		}
+		nodeKey := obs.NodeID
+		if nodeKey == "" {
+			nodeKey = unknownTrendNodeKey
+		}
+		previous := previousOpenrestyByNode[nodeKey]
+		previousOpenrestyByNode[nodeKey] = networkCounterState{
+			rx:   obs.OpenrestyRxBytes,
+			tx:   obs.OpenrestyTxBytes,
+			seen: true,
+		}
+		if !previous.seen {
+			continue
+		}
 		index, ok := trendBucketIndex(obs.CapturedAt, start)
 		if !ok {
 			continue
 		}
-		points[index].OpenrestyRxBytes += obs.OpenrestyRxBytes
-		points[index].OpenrestyTxBytes += obs.OpenrestyTxBytes
+		points[index].OpenrestyRxBytes += nonNegativeDelta(obs.OpenrestyRxBytes, previous.rx)
+		points[index].OpenrestyTxBytes += nonNegativeDelta(obs.OpenrestyTxBytes, previous.tx)
 		if obs.NodeID != "" {
 			accumulators[index].nodes[obs.NodeID] = struct{}{}
 		}
@@ -377,6 +492,48 @@ func BuildNetworkTrendPoints(
 	return points
 }
 
+// BuildNetworkTrendPointsFromHourly builds 24h network trend buckets from hourly aggregates.
+func BuildNetworkTrendPointsFromHourly(
+	now time.Time,
+	metricHourly []*model.OpenFlareMetricHourly,
+	openrestyHourly []*model.OpenFlareOpenrestyHourly,
+) []NetworkTrendPoint {
+	start := trendWindowStart(now)
+	points := make([]NetworkTrendPoint, observabilityTrendBuckets)
+	for index := range points {
+		points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
+	}
+	for _, row := range metricHourly {
+		if row == nil {
+			continue
+		}
+		index, ok := trendBucketIndex(row.Hour, start)
+		if !ok {
+			continue
+		}
+		points[index].NetworkRxBytes += row.NetworkRxBytes
+		points[index].NetworkTxBytes += row.NetworkTxBytes
+		if row.ReportedNodes > points[index].ReportedNodes {
+			points[index].ReportedNodes = row.ReportedNodes
+		}
+	}
+	for _, row := range openrestyHourly {
+		if row == nil {
+			continue
+		}
+		index, ok := trendBucketIndex(row.Hour, start)
+		if !ok {
+			continue
+		}
+		points[index].OpenrestyRxBytes += row.OpenrestyRxBytes
+		points[index].OpenrestyTxBytes += row.OpenrestyTxBytes
+		if row.ReportedNodes > points[index].ReportedNodes {
+			points[index].ReportedNodes = row.ReportedNodes
+		}
+	}
+	return points
+}
+
 // BuildDiskIOTrendPoints builds 24h disk IO trend buckets.
 func BuildDiskIOTrendPoints(now time.Time, snapshots []*model.OpenFlareMetricSnapshot) []DiskIOTrendPoint {
 	start := trendWindowStart(now)
@@ -396,7 +553,7 @@ func BuildDiskIOTrendPoints(now time.Time, snapshots []*model.OpenFlareMetricSna
 	for _, snapshot := range snapshots {
 		nodeKey := snapshot.NodeID
 		if nodeKey == "" {
-			nodeKey = "__unknown__"
+			nodeKey = unknownTrendNodeKey
 		}
 		previous := previousByNode[nodeKey]
 		previousByNode[nodeKey] = diskCounterState{
@@ -411,16 +568,8 @@ func BuildDiskIOTrendPoints(now time.Time, snapshots []*model.OpenFlareMetricSna
 		if !ok {
 			continue
 		}
-		readDelta := snapshot.DiskReadBytes - previous.read
-		writeDelta := snapshot.DiskWriteBytes - previous.write
-		if readDelta < 0 {
-			readDelta = 0
-		}
-		if writeDelta < 0 {
-			writeDelta = 0
-		}
-		points[index].DiskReadBytes += readDelta
-		points[index].DiskWriteBytes += writeDelta
+		points[index].DiskReadBytes += nonNegativeDelta(snapshot.DiskReadBytes, previous.read)
+		points[index].DiskWriteBytes += nonNegativeDelta(snapshot.DiskWriteBytes, previous.write)
 		if snapshot.NodeID != "" {
 			accumulators[index].nodes[snapshot.NodeID] = struct{}{}
 		}
@@ -431,6 +580,36 @@ func BuildDiskIOTrendPoints(now time.Time, snapshots []*model.OpenFlareMetricSna
 	return points
 }
 
+// BuildDiskIOTrendPointsFromHourly builds 24h disk IO trend buckets from hourly aggregates.
+func BuildDiskIOTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlareMetricHourly) []DiskIOTrendPoint {
+	start := trendWindowStart(now)
+	points := make([]DiskIOTrendPoint, observabilityTrendBuckets)
+	for index := range points {
+		points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
+	}
+	for _, row := range hourly {
+		if row == nil {
+			continue
+		}
+		index, ok := trendBucketIndex(row.Hour, start)
+		if !ok {
+			continue
+		}
+		points[index].DiskReadBytes += row.DiskReadBytes
+		points[index].DiskWriteBytes += row.DiskWriteBytes
+		points[index].ReportedNodes = row.ReportedNodes
+	}
+	return points
+}
+
+func nonNegativeDelta(current int64, previous int64) int64 {
+	delta := current - previous
+	if delta < 0 {
+		return 0
+	}
+	return delta
+}
+
 func latestMetricSnapshot(snapshots []*model.OpenFlareMetricSnapshot) *model.OpenFlareMetricSnapshot {
 	var latest *model.OpenFlareMetricSnapshot
 	for _, snapshot := range snapshots {

internal/apps/openflare/observability/analytics_test.go

@@ -132,3 +132,84 @@ func TestBuildTrafficWindowSummaryNilWithoutReport(t *testing.T) {
 		t.Fatalf("buildTrafficWindowSummary(nil) = %#v, want nil", summary)
 	}
 }
+
+func TestBuildCapacityTrendPointsFromHourlyFillsBuckets(t *testing.T) {
+	t.Parallel()
+
+	now := time.Date(2026, 7, 10, 9, 30, 0, 0, time.UTC)
+	hourly := []*model.OpenFlareMetricHourly{
+		{
+			Hour:                      now.Add(-3 * time.Hour).Truncate(time.Hour),
+			AverageCPUUsagePercent:    42.5,
+			AverageMemoryUsagePercent: 61.2,
+			ReportedNodes:             1,
+		},
+		{
+			Hour:                      now.Truncate(time.Hour),
+			AverageCPUUsagePercent:    12.0,
+			AverageMemoryUsagePercent: 50.0,
+			ReportedNodes:             2,
+		},
+	}
+
+	points := BuildCapacityTrendPointsFromHourly(now, hourly)
+	if len(points) != observabilityTrendBuckets {
+		t.Fatalf("len = %d, want %d", len(points), observabilityTrendBuckets)
+	}
+	if points[len(points)-4].AverageCPUUsagePercent != 42.5 {
+		t.Fatalf("hour-3 cpu = %v, want 42.5", points[len(points)-4].AverageCPUUsagePercent)
+	}
+	if points[len(points)-1].ReportedNodes != 2 {
+		t.Fatalf("current hour reported_nodes = %d, want 2", points[len(points)-1].ReportedNodes)
+	}
+}
+
+func TestBuildNetworkTrendPointsUsesCounterDeltas(t *testing.T) {
+	t.Parallel()
+
+	now := time.Date(2026, 7, 10, 9, 30, 0, 0, time.UTC)
+	base := now.Truncate(time.Hour)
+	snapshots := []*model.OpenFlareMetricSnapshot{
+		{NodeID: "n1", CapturedAt: base.Add(10 * time.Minute), NetworkRxBytes: 1000, NetworkTxBytes: 2000},
+		{NodeID: "n1", CapturedAt: base.Add(20 * time.Minute), NetworkRxBytes: 1500, NetworkTxBytes: 2600},
+	}
+	openrestyObs := []*model.OpenFlareNodeObservationOpenresty{
+		{NodeID: "n1", CapturedAt: base.Add(10 * time.Minute), OpenrestyRxBytes: 100, OpenrestyTxBytes: 200},
+		{NodeID: "n1", CapturedAt: base.Add(20 * time.Minute), OpenrestyRxBytes: 180, OpenrestyTxBytes: 250},
+	}
+
+	points := BuildNetworkTrendPoints(now, snapshots, openrestyObs)
+	current := points[len(points)-1]
+	if current.NetworkRxBytes != 500 {
+		t.Fatalf("network_rx_bytes = %d, want 500", current.NetworkRxBytes)
+	}
+	if current.NetworkTxBytes != 600 {
+		t.Fatalf("network_tx_bytes = %d, want 600", current.NetworkTxBytes)
+	}
+	if current.OpenrestyRxBytes != 80 {
+		t.Fatalf("openresty_rx_bytes = %d, want 80", current.OpenrestyRxBytes)
+	}
+	if current.OpenrestyTxBytes != 50 {
+		t.Fatalf("openresty_tx_bytes = %d, want 50", current.OpenrestyTxBytes)
+	}
+}
+
+func TestBuildDiskIOTrendPointsFromHourlyFillsBuckets(t *testing.T) {
+	t.Parallel()
+
+	now := time.Date(2026, 7, 10, 9, 30, 0, 0, time.UTC)
+	hourly := []*model.OpenFlareMetricHourly{
+		{
+			Hour:            now.Add(-1 * time.Hour).Truncate(time.Hour),
+			DiskReadBytes:   1024,
+			DiskWriteBytes:  2048,
+			ReportedNodes:   1,
+		},
+	}
+
+	points := BuildDiskIOTrendPointsFromHourly(now, hourly)
+	prev := points[len(points)-2]
+	if prev.DiskReadBytes != 1024 || prev.DiskWriteBytes != 2048 {
+		t.Fatalf("previous hour disk io = %#v, want read=1024 write=2048", prev)
+	}
+}

internal/apps/openflare/observability/node_logics.go

@@ -134,11 +134,6 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
 	if err != nil {
 		return nil, err
 	}
-	trafficTrend := BuildTrafficTrendPoints(now, reports)
-	if trafficHourly, hourlyErr := model.ListOpenFlareTrafficHourlySince(ctx, node.NodeID, now.Add(-24*time.Hour)); hourlyErr == nil && len(trafficHourly) > 0 {
-		trafficTrend = BuildTrafficTrendPointsFromHourly(now, trafficHourly)
-	}
-
 	view := &NodeView{
 		NodeID:          node.NodeID,
 		Profile:         profile,
@@ -150,12 +145,7 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
 			Distributions: BuildTrafficDistributions(reports, accessLogRegions, defaultTrafficDistributionLimit),
 			Health:        buildHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events),
 		},
-		Trends: NodeTrends{
-			Traffic24h:  trafficTrend,
-			Capacity24h: BuildCapacityTrendPoints(now, snapshots),
-			Network24h:  BuildNetworkTrendPoints(now, snapshots, openrestyObs),
-			DiskIO24h:   BuildDiskIOTrendPoints(now, snapshots),
-		},
+		Trends: BuildNodeTrends(ctx, now, node.NodeID, snapshots, openrestyObs, reports),
 	}
 	if node.NodeType == "tunnel_relay" {
 		frpsObs, frpsErr := model.ListOpenFlareNodeObservationFrps(ctx, node.NodeID, time.Time{}, 1)