Files
OpenFlare/internal/apps/openflare/dashboard/logics.go
T
ryan 3187934f72 fix(openflare): ClickHouse count scan and dashboard traffic metrics
- Scan ClickHouse count()/countIf() aggregates as uint64 before int64 conversion
- Fix access log list count, region stats, and delete pre-count queries
- Aggregate dashboard 24h traffic totals from hourly trend buckets
- Show current hour value in traffic trend chart summary
- Rename docker-compose service to openflare
- Remove redundant config preview section from performance page
2026-06-19 17:34:19 +08:00

404 lines
13 KiB
Go

// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package dashboard
import (
"context"
"sort"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/observability"
"github.com/Rain-kl/Wavelet/internal/model"
)
// Summary is the dashboard node summary section.
type Summary struct {
TotalNodes int `json:"total_nodes"`
OnlineNodes int `json:"online_nodes"`
OfflineNodes int `json:"offline_nodes"`
PendingNodes int `json:"pending_nodes"`
UnhealthyNodes int `json:"unhealthy_nodes"`
}
// Traffic is the dashboard traffic section.
type Traffic struct {
RequestCount int64 `json:"request_count"`
UniqueVisitors int64 `json:"unique_visitors"`
ErrorCount int64 `json:"error_count"`
EstimatedQPS float64 `json:"estimated_qps"`
ReportedNodes int `json:"reported_nodes"`
}
// Capacity is the dashboard capacity section.
type Capacity struct {
AverageCPUUsagePercent float64 `json:"average_cpu_usage_percent"`
AverageMemoryUsagePercent float64 `json:"average_memory_usage_percent"`
HighCPUNodes int `json:"high_cpu_nodes"`
HighMemoryNodes int `json:"high_memory_nodes"`
HighStorageNodes int `json:"high_storage_nodes"`
}
// NodeHealth is a dashboard node health row.
type NodeHealth struct {
ID uint `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
GeoName string `json:"geo_name"`
GeoLatitude *float64 `json:"geo_latitude"`
GeoLongitude *float64 `json:"geo_longitude"`
Status string `json:"status"`
OpenrestyStatus string `json:"openresty_status"`
CurrentVersion string `json:"current_version"`
LastSeenAt any `json:"last_seen_at"`
ActiveEventCount int `json:"active_event_count"`
CPUUsagePercent float64 `json:"cpu_usage_percent"`
MemoryUsagePercent float64 `json:"memory_usage_percent"`
StorageUsagePercent float64 `json:"storage_usage_percent"`
RequestCount int64 `json:"request_count"`
ErrorCount int64 `json:"error_count"`
UniqueVisitorCount int64 `json:"unique_visitor_count"`
}
// OverviewView is the expanded dashboard overview payload.
type OverviewView struct {
GeneratedAt time.Time `json:"generated_at"`
Summary Summary `json:"summary"`
Traffic Traffic `json:"traffic"`
Capacity Capacity `json:"capacity"`
Distributions observability.TrafficDistributions `json:"distributions"`
Trends observability.NodeTrends `json:"trends"`
Nodes []NodeHealth `json:"nodes"`
}
// OverviewPayload is the compact legacy dashboard overview response.
type OverviewPayload struct {
GeneratedAt any `json:"generated_at"`
Summary Summary `json:"summary"`
Traffic Traffic `json:"traffic"`
Capacity Capacity `json:"capacity"`
Distributions distributionsPayload `json:"distributions"`
Trends trendsPayload `json:"trends"`
Nodes [][]any `json:"nodes"`
}
type distributionsPayload struct {
StatusCodes [][]any `json:"status_codes"`
TopDomains [][]any `json:"top_domains"`
SourceCountries [][]any `json:"source_countries"`
}
type trendsPayload struct {
Traffic24h [][]any `json:"traffic_24h"`
Capacity24h [][]any `json:"capacity_24h"`
Network24h [][]any `json:"network_24h"`
DiskIO24h [][]any `json:"disk_io_24h"`
}
// GetOverview aggregates dashboard overview data from nodes and observability tables.
func GetOverview(ctx context.Context) (*OverviewPayload, error) {
view, err := buildOverviewView(ctx)
if err != nil {
return nil, err
}
return compressOverview(view), nil
}
func buildOverviewView(ctx context.Context) (*OverviewView, error) {
now := time.Now()
since := now.Add(-24 * time.Hour)
nodes, err := model.ListOpenFlareNodes(ctx)
if err != nil {
return nil, err
}
snapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", since, 0)
if err != nil {
return nil, err
}
reports, err := model.ListOpenFlareRequestReportsSince(ctx, "", since, 0)
if err != nil {
return nil, err
}
accessLogRegions, err := model.ListOpenFlareAccessLogRegionCounts(ctx, "", since, dashboardDistributionLimit)
if err != nil {
return nil, err
}
activeEvents, err := model.ListOpenFlareActiveHealthEvents(ctx)
if err != nil {
return nil, err
}
openrestySnapshots, err := model.ListOpenFlareNodeObservationOpenresty(ctx, "", since, 0)
if err != nil {
return nil, err
}
view := &OverviewView{
GeneratedAt: now,
Nodes: make([]NodeHealth, 0, len(nodes)),
Distributions: observability.BuildTrafficDistributions(reports, accessLogRegions, dashboardDistributionLimit),
Trends: observability.NodeTrends{
Traffic24h: observability.BuildTrafficTrendPoints(now, reports),
Capacity24h: observability.BuildCapacityTrendPoints(now, snapshots),
Network24h: observability.BuildNetworkTrendPoints(now, snapshots, openrestySnapshots),
DiskIO24h: observability.BuildDiskIOTrendPoints(now, snapshots),
},
}
var cpuNodeCount int
var memoryNodeCount int
latestSnapshots := observability.LatestMetricSnapshotsByNode(snapshots)
latestTrafficReports := observability.LatestTrafficReportsByNode(reports)
activeEventsByNode := observability.ActiveHealthEventsByNode(activeEvents)
for _, node := range nodes {
computedStatus := computeNodeStatus(&node)
switch computedStatus {
case nodeStatusOnline:
view.Summary.OnlineNodes++
case nodeStatusOffline:
view.Summary.OfflineNodes++
case nodeStatusPending:
view.Summary.PendingNodes++
}
if node.OpenrestyStatus == "unhealthy" {
view.Summary.UnhealthyNodes++
}
latestSnapshot := latestSnapshots[node.NodeID]
latestTraffic := latestTrafficReports[node.NodeID]
nodeActiveEvents := activeEventsByNode[node.NodeID]
nodeHealth := NodeHealth{
ID: node.ID,
NodeID: node.NodeID,
Name: node.Name,
GeoName: node.GeoName,
GeoLatitude: node.GeoLatitude,
GeoLongitude: node.GeoLongitude,
Status: computedStatus,
OpenrestyStatus: node.OpenrestyStatus,
CurrentVersion: node.CurrentVersion,
LastSeenAt: nodeViewLastSeenAt(&node),
ActiveEventCount: len(nodeActiveEvents),
}
cpuNodeCount, memoryNodeCount = applyNodeSnapshotMetrics(&nodeHealth, latestSnapshot, view, cpuNodeCount, memoryNodeCount)
applyNodeTrafficMetrics(&nodeHealth, latestTraffic)
view.Nodes = append(view.Nodes, nodeHealth)
}
applyTrafficTotalsFromTrend(&view.Traffic, view.Trends.Traffic24h)
applyTrafficRuntimeMetrics(&view.Traffic, latestTrafficReports)
view.Summary.TotalNodes = len(nodes)
if cpuNodeCount > 0 {
view.Capacity.AverageCPUUsagePercent /= float64(cpuNodeCount)
}
if memoryNodeCount > 0 {
view.Capacity.AverageMemoryUsagePercent /= float64(memoryNodeCount)
}
sort.Slice(view.Nodes, func(i int, j int) bool {
if view.Nodes[i].ActiveEventCount == view.Nodes[j].ActiveEventCount {
return view.Nodes[i].CPUUsagePercent > view.Nodes[j].CPUUsagePercent
}
return view.Nodes[i].ActiveEventCount > view.Nodes[j].ActiveEventCount
})
return view, nil
}
func applyNodeSnapshotMetrics(nodeHealth *NodeHealth, snapshot *model.OpenFlareMetricSnapshot, view *OverviewView, cpuNodeCount, memoryNodeCount int) (int, int) {
if snapshot == nil {
return cpuNodeCount, memoryNodeCount
}
nodeHealth.CPUUsagePercent = snapshot.CPUUsagePercent
nodeHealth.MemoryUsagePercent = observability.Percentage(snapshot.MemoryUsedBytes, snapshot.MemoryTotalBytes)
nodeHealth.StorageUsagePercent = observability.Percentage(snapshot.StorageUsedBytes, snapshot.StorageTotalBytes)
if snapshot.CPUUsagePercent > 0 {
view.Capacity.AverageCPUUsagePercent += snapshot.CPUUsagePercent
cpuNodeCount++
}
if nodeHealth.MemoryUsagePercent > 0 {
view.Capacity.AverageMemoryUsagePercent += nodeHealth.MemoryUsagePercent
memoryNodeCount++
}
if snapshot.CPUUsagePercent >= highCPUUsagePercentThreshold {
view.Capacity.HighCPUNodes++
}
if nodeHealth.MemoryUsagePercent >= highMemoryUsagePercentThreshold {
view.Capacity.HighMemoryNodes++
}
if nodeHealth.StorageUsagePercent >= highStorageUsagePercentThreshold {
view.Capacity.HighStorageNodes++
}
return cpuNodeCount, memoryNodeCount
}
func applyNodeTrafficMetrics(nodeHealth *NodeHealth, traffic *model.OpenFlareRequestReport) {
if traffic == nil {
return
}
nodeHealth.RequestCount = traffic.RequestCount
nodeHealth.ErrorCount = traffic.ErrorCount
nodeHealth.UniqueVisitorCount = traffic.UniqueVisitorCount
}
func applyTrafficTotalsFromTrend(traffic *Traffic, points []observability.TrafficTrendPoint) {
if traffic == nil {
return
}
traffic.RequestCount = 0
traffic.ErrorCount = 0
for _, point := range points {
traffic.RequestCount += point.RequestCount
traffic.ErrorCount += point.ErrorCount
}
}
func applyTrafficRuntimeMetrics(traffic *Traffic, latestReports map[string]*model.OpenFlareRequestReport) {
if traffic == nil {
return
}
traffic.UniqueVisitors = 0
traffic.EstimatedQPS = 0
traffic.ReportedNodes = 0
for _, report := range latestReports {
if report == nil {
continue
}
traffic.UniqueVisitors += report.UniqueVisitorCount
traffic.ReportedNodes++
if duration := report.WindowEndedAt.Sub(report.WindowStartedAt).Seconds(); duration > 0 {
traffic.EstimatedQPS += float64(report.RequestCount) / duration
}
}
}
func compressOverview(view *OverviewView) *OverviewPayload {
if view == nil {
return &OverviewPayload{
Distributions: distributionsPayload{
StatusCodes: [][]any{},
TopDomains: [][]any{},
SourceCountries: [][]any{},
},
Trends: trendsPayload{
Traffic24h: [][]any{},
Capacity24h: [][]any{},
Network24h: [][]any{},
DiskIO24h: [][]any{},
},
Nodes: [][]any{},
}
}
return &OverviewPayload{
GeneratedAt: view.GeneratedAt,
Summary: view.Summary,
Traffic: view.Traffic,
Capacity: view.Capacity,
Distributions: distributionsPayload{
StatusCodes: compressDistributionItems(view.Distributions.StatusCodes),
TopDomains: compressDistributionItems(view.Distributions.TopDomains),
SourceCountries: compressDistributionItems(view.Distributions.SourceCountries),
},
Trends: trendsPayload{
Traffic24h: compressTrafficTrendPoints(view.Trends.Traffic24h),
Capacity24h: compressCapacityTrendPoints(view.Trends.Capacity24h),
Network24h: compressNetworkTrendPoints(view.Trends.Network24h),
DiskIO24h: compressDiskIOTrendPoints(view.Trends.DiskIO24h),
},
Nodes: compressDashboardNodes(view.Nodes),
}
}
func compressDistributionItems(items []observability.DistributionItem) [][]any {
rows := make([][]any, 0, len(items))
for _, item := range items {
rows = append(rows, []any{item.Key, item.Value})
}
return rows
}
func compressTrafficTrendPoints(points []observability.TrafficTrendPoint) [][]any {
rows := make([][]any, 0, len(points))
for _, point := range points {
rows = append(rows, []any{
point.BucketStartedAt,
point.RequestCount,
point.ErrorCount,
point.UniqueVisitorCount,
})
}
return rows
}
func compressCapacityTrendPoints(points []observability.CapacityTrendPoint) [][]any {
rows := make([][]any, 0, len(points))
for _, point := range points {
rows = append(rows, []any{
point.BucketStartedAt,
point.AverageCPUUsagePercent,
point.AverageMemoryUsagePercent,
point.ReportedNodes,
})
}
return rows
}
func compressNetworkTrendPoints(points []observability.NetworkTrendPoint) [][]any {
rows := make([][]any, 0, len(points))
for _, point := range points {
rows = append(rows, []any{
point.BucketStartedAt,
point.NetworkRxBytes,
point.NetworkTxBytes,
point.OpenrestyRxBytes,
point.OpenrestyTxBytes,
point.ReportedNodes,
})
}
return rows
}
func compressDiskIOTrendPoints(points []observability.DiskIOTrendPoint) [][]any {
rows := make([][]any, 0, len(points))
for _, point := range points {
rows = append(rows, []any{
point.BucketStartedAt,
point.DiskReadBytes,
point.DiskWriteBytes,
point.ReportedNodes,
})
}
return rows
}
func compressDashboardNodes(nodes []NodeHealth) [][]any {
rows := make([][]any, 0, len(nodes))
for _, node := range nodes {
rows = append(rows, []any{
node.ID,
node.NodeID,
node.Name,
node.GeoName,
node.GeoLatitude,
node.GeoLongitude,
node.Status,
node.OpenrestyStatus,
node.CurrentVersion,
node.LastSeenAt,
node.ActiveEventCount,
node.CPUUsagePercent,
node.MemoryUsagePercent,
node.StorageUsagePercent,
node.RequestCount,
node.ErrorCount,
node.UniqueVisitorCount,
})
}
return rows
}