perf(clickhouse): P0 dashboard/observability query limits, cache, slower polling

This commit is contained in:
ryan
2026-07-02 15:20:19 +08:00
parent 28c14eb054
commit 3e910f1961
6 changed files with 90 additions and 22 deletions
@@ -289,7 +289,7 @@ export function NodeObservability({
const observabilityQuery = useQuery({ const observabilityQuery = useQuery({
queryKey: ['openflare', 'node-observability', nodeId], queryKey: ['openflare', 'node-observability', nodeId],
queryFn: () => NodeService.getObservability(nodeId, { hours: 24, limit: 48 }), queryFn: () => NodeService.getObservability(nodeId, { hours: 24, limit: 48 }),
refetchInterval: 10000, refetchInterval: 30000,
}); });
const cleanupMutation = useMutation({ const cleanupMutation = useMutation({
+1 -1
View File
@@ -28,7 +28,7 @@ export default function OpenFlareDashboardPage() {
const overviewQuery = useQuery({ const overviewQuery = useQuery({
queryKey: dashboardQueryKey, queryKey: dashboardQueryKey,
queryFn: () => DashboardService.getOverview(), queryFn: () => DashboardService.getOverview(),
refetchInterval: 30_000, refetchInterval: 60_000,
}); });
const overview = overviewQuery.data; const overview = overviewQuery.data;
@@ -0,0 +1,33 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package dashboard
import (
"sync"
"time"
)
const overviewCacheTTL = 30 * time.Second
var overviewCache struct {
mu sync.Mutex
payload *OverviewPayload
expiresAt time.Time
}
func getCachedOverview() (*OverviewPayload, bool) {
overviewCache.mu.Lock()
defer overviewCache.mu.Unlock()
if overviewCache.payload == nil || time.Now().After(overviewCache.expiresAt) {
return nil, false
}
return overviewCache.payload, true
}
func setCachedOverview(payload *OverviewPayload) {
overviewCache.mu.Lock()
defer overviewCache.mu.Unlock()
overviewCache.payload = payload
overviewCache.expiresAt = time.Now().Add(overviewCacheTTL)
}
@@ -18,6 +18,7 @@ const (
nodeStatusPending = "pending" nodeStatusPending = "pending"
dashboardDistributionLimit = 8 dashboardDistributionLimit = 8
dashboardOverviewSnapshotLimit = 500
highCPUUsagePercentThreshold = 80 highCPUUsagePercentThreshold = 80
highMemoryUsagePercentThreshold = 85 highMemoryUsagePercentThreshold = 85
highStorageUsagePercentThreshold = 85 highStorageUsagePercentThreshold = 85
+9 -4
View File
@@ -97,11 +97,16 @@ type trendsPayload struct {
// GetOverview aggregates dashboard overview data from nodes and observability tables. // GetOverview aggregates dashboard overview data from nodes and observability tables.
func GetOverview(ctx context.Context) (*OverviewPayload, error) { func GetOverview(ctx context.Context) (*OverviewPayload, error) {
if payload, ok := getCachedOverview(); ok {
return payload, nil
}
view, err := buildOverviewView(ctx) view, err := buildOverviewView(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return compressOverview(view), nil payload := compressOverview(view)
setCachedOverview(payload)
return payload, nil
} }
func buildOverviewView(ctx context.Context) (*OverviewView, error) { func buildOverviewView(ctx context.Context) (*OverviewView, error) {
@@ -112,11 +117,11 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
snapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", since, 0) snapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", since, dashboardOverviewSnapshotLimit)
if err != nil { if err != nil {
return nil, err return nil, err
} }
reports, err := model.ListOpenFlareRequestReportsSince(ctx, "", since, 0) reports, err := model.ListOpenFlareRequestReportsSince(ctx, "", since, dashboardOverviewSnapshotLimit)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -128,7 +133,7 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
openrestySnapshots, err := model.ListOpenFlareNodeObservationOpenresty(ctx, "", since, 0) openrestySnapshots, err := model.ListOpenFlareNodeObservationOpenresty(ctx, "", since, dashboardOverviewSnapshotLimit)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -7,6 +7,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"sync"
"time" "time"
"github.com/Rain-kl/Wavelet/internal/model" "github.com/Rain-kl/Wavelet/internal/model"
@@ -18,8 +19,19 @@ const (
defaultObservabilityLimit = 120 defaultObservabilityLimit = 120
maxObservabilityLimit = 500 maxObservabilityLimit = 500
defaultTrafficDistributionLimit = 8 defaultTrafficDistributionLimit = 8
nodeObservabilityCacheTTL = 15 * time.Second
) )
var nodeObservabilityCache struct {
mu sync.Mutex
views map[string]cachedNodeObservability
}
type cachedNodeObservability struct {
view *NodeView
expiresAt time.Time
}
// NodeQuery filters node observability data. // NodeQuery filters node observability data.
type NodeQuery struct { type NodeQuery struct {
Hours int `json:"hours"` Hours int `json:"hours"`
@@ -87,6 +99,9 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
if err != nil { if err != nil {
return nil, err return nil, err
} }
if view, ok := getCachedNodeObservability(node.NodeID); ok {
return view, nil
}
limit := normalizeObservabilityLimit(query.Limit) limit := normalizeObservabilityLimit(query.Limit)
since := now.Add(-normalizeObservabilityWindow(query.Hours)) since := now.Add(-normalizeObservabilityWindow(query.Hours))
@@ -115,18 +130,6 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
if err != nil { if err != nil {
return nil, err return nil, err
} }
trendSnapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, node.NodeID, now.Add(-24*time.Hour), 0)
if err != nil {
return nil, err
}
trendOpenresty, err := model.ListOpenFlareNodeObservationOpenresty(ctx, node.NodeID, now.Add(-24*time.Hour), 0)
if err != nil {
return nil, err
}
trendReports, err := model.ListOpenFlareRequestReportsSince(ctx, node.NodeID, now.Add(-24*time.Hour), 0)
if err != nil {
return nil, err
}
events, err := model.ListOpenFlareHealthEvents(ctx, node.NodeID, false, limit) events, err := model.ListOpenFlareHealthEvents(ctx, node.NodeID, false, limit)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -144,10 +147,10 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
Health: buildHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events), Health: buildHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events),
}, },
Trends: NodeTrends{ Trends: NodeTrends{
Traffic24h: BuildTrafficTrendPoints(now, trendReports), Traffic24h: BuildTrafficTrendPoints(now, reports),
Capacity24h: BuildCapacityTrendPoints(now, trendSnapshots), Capacity24h: BuildCapacityTrendPoints(now, snapshots),
Network24h: BuildNetworkTrendPoints(now, trendSnapshots, trendOpenresty), Network24h: BuildNetworkTrendPoints(now, snapshots, openrestyObs),
DiskIO24h: BuildDiskIOTrendPoints(now, trendSnapshots), DiskIO24h: BuildDiskIOTrendPoints(now, snapshots),
}, },
} }
if node.NodeType == "tunnel_relay" { if node.NodeType == "tunnel_relay" {
@@ -161,9 +164,35 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
} }
view.RelayDashboard = buildRelayDashboardSnapshot(node, latestFrps) view.RelayDashboard = buildRelayDashboardSnapshot(node, latestFrps)
} }
setCachedNodeObservability(node.NodeID, view)
return view, nil return view, nil
} }
func getCachedNodeObservability(nodeID string) (*NodeView, bool) {
nodeObservabilityCache.mu.Lock()
defer nodeObservabilityCache.mu.Unlock()
if nodeObservabilityCache.views == nil {
return nil, false
}
entry, ok := nodeObservabilityCache.views[nodeID]
if !ok || time.Now().After(entry.expiresAt) {
return nil, false
}
return entry.view, true
}
func setCachedNodeObservability(nodeID string, view *NodeView) {
nodeObservabilityCache.mu.Lock()
defer nodeObservabilityCache.mu.Unlock()
if nodeObservabilityCache.views == nil {
nodeObservabilityCache.views = make(map[string]cachedNodeObservability)
}
nodeObservabilityCache.views[nodeID] = cachedNodeObservability{
view: view,
expiresAt: time.Now().Add(nodeObservabilityCacheTTL),
}
}
// CleanupHealthEvents removes all health events for a node. // CleanupHealthEvents removes all health events for a node.
func CleanupHealthEvents(ctx context.Context, id uint) (*HealthEventCleanupResult, error) { func CleanupHealthEvents(ctx context.Context, id uint) (*HealthEventCleanupResult, error) {
node, err := model.GetOpenFlareNodeByID(ctx, id) node, err := model.GetOpenFlareNodeByID(ctx, id)