mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-08 00:26:37 +08:00
feat: add dashboard overview health summary
This commit is contained in:
ryan
@@ -0,0 +1,260 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"atsflare/model"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type DashboardOverviewView struct {
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Summary DashboardSummary `json:"summary"`
|
||||
Traffic DashboardTraffic `json:"traffic"`
|
||||
Capacity DashboardCapacity `json:"capacity"`
|
||||
Config DashboardConfig `json:"config"`
|
||||
Nodes []DashboardNodeHealth `json:"nodes"`
|
||||
ActiveAlerts []DashboardAlert `json:"active_alerts"`
|
||||
}
|
||||
|
||||
type DashboardSummary struct {
|
||||
TotalNodes int `json:"total_nodes"`
|
||||
OnlineNodes int `json:"online_nodes"`
|
||||
OfflineNodes int `json:"offline_nodes"`
|
||||
PendingNodes int `json:"pending_nodes"`
|
||||
UnhealthyNodes int `json:"unhealthy_nodes"`
|
||||
ActiveAlerts int `json:"active_alerts"`
|
||||
LaggingNodes int `json:"lagging_nodes"`
|
||||
}
|
||||
|
||||
type DashboardTraffic struct {
|
||||
RequestCount int64 `json:"request_count"`
|
||||
UniqueVisitors int64 `json:"unique_visitors"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
EstimatedQPS float64 `json:"estimated_qps"`
|
||||
ReportedNodes int `json:"reported_nodes"`
|
||||
}
|
||||
|
||||
type DashboardCapacity struct {
|
||||
AverageCPUUsagePercent float64 `json:"average_cpu_usage_percent"`
|
||||
AverageMemoryUsagePercent float64 `json:"average_memory_usage_percent"`
|
||||
HighCPUNodes int `json:"high_cpu_nodes"`
|
||||
HighMemoryNodes int `json:"high_memory_nodes"`
|
||||
HighStorageNodes int `json:"high_storage_nodes"`
|
||||
}
|
||||
|
||||
type DashboardConfig struct {
|
||||
ActiveVersion string `json:"active_version"`
|
||||
LaggingNodes int `json:"lagging_nodes"`
|
||||
PendingNodes int `json:"pending_nodes"`
|
||||
}
|
||||
|
||||
type DashboardNodeHealth struct {
|
||||
ID uint `json:"id"`
|
||||
NodeID string `json:"node_id"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
OpenrestyStatus string `json:"openresty_status"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
LastSeenAt time.Time `json:"last_seen_at"`
|
||||
ActiveEventCount int `json:"active_event_count"`
|
||||
CPUUsagePercent float64 `json:"cpu_usage_percent"`
|
||||
MemoryUsagePercent float64 `json:"memory_usage_percent"`
|
||||
StorageUsagePercent float64 `json:"storage_usage_percent"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
UniqueVisitorCount int64 `json:"unique_visitor_count"`
|
||||
}
|
||||
|
||||
type DashboardAlert struct {
|
||||
NodeID string `json:"node_id"`
|
||||
NodeName string `json:"node_name"`
|
||||
EventType string `json:"event_type"`
|
||||
Severity string `json:"severity"`
|
||||
Message string `json:"message"`
|
||||
LastTriggeredAt time.Time `json:"last_triggered_at"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
func GetDashboardOverview() (*DashboardOverviewView, error) {
|
||||
now := time.Now()
|
||||
since := now.Add(-24 * time.Hour)
|
||||
|
||||
nodes, err := model.ListNodes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
activeVersion := ""
|
||||
if version, versionErr := model.GetActiveConfigVersion(); versionErr == nil {
|
||||
activeVersion = version.Version
|
||||
} else if !errors.Is(versionErr, gorm.ErrRecordNotFound) {
|
||||
return nil, versionErr
|
||||
}
|
||||
|
||||
view := &DashboardOverviewView{
|
||||
GeneratedAt: now,
|
||||
Nodes: make([]DashboardNodeHealth, 0, len(nodes)),
|
||||
}
|
||||
|
||||
var cpuNodeCount int
|
||||
var memoryNodeCount int
|
||||
|
||||
for _, node := range nodes {
|
||||
computedStatus := computeNodeStatus(node)
|
||||
switch computedStatus {
|
||||
case NodeStatusOnline:
|
||||
view.Summary.OnlineNodes++
|
||||
case NodeStatusOffline:
|
||||
view.Summary.OfflineNodes++
|
||||
case NodeStatusPending:
|
||||
view.Summary.PendingNodes++
|
||||
}
|
||||
if node.OpenrestyStatus == OpenrestyStatusUnhealthy {
|
||||
view.Summary.UnhealthyNodes++
|
||||
}
|
||||
if activeVersion != "" && node.CurrentVersion != "" && node.CurrentVersion != activeVersion {
|
||||
view.Summary.LaggingNodes++
|
||||
}
|
||||
if activeVersion != "" && node.CurrentVersion == "" && computedStatus != NodeStatusPending {
|
||||
view.Summary.LaggingNodes++
|
||||
}
|
||||
|
||||
latestSnapshot := latestMetricSnapshotForNode(node.NodeID, since)
|
||||
latestTraffic := latestTrafficReportForNode(node.NodeID, since)
|
||||
activeEvents, eventErr := model.ListNodeHealthEvents(node.NodeID, true, 20)
|
||||
if eventErr != nil {
|
||||
return nil, eventErr
|
||||
}
|
||||
|
||||
nodeHealth := DashboardNodeHealth{
|
||||
ID: node.ID,
|
||||
NodeID: node.NodeID,
|
||||
Name: node.Name,
|
||||
Status: computedStatus,
|
||||
OpenrestyStatus: node.OpenrestyStatus,
|
||||
CurrentVersion: node.CurrentVersion,
|
||||
LastSeenAt: node.LastSeenAt,
|
||||
ActiveEventCount: len(activeEvents),
|
||||
}
|
||||
|
||||
for _, event := range activeEvents {
|
||||
view.ActiveAlerts = append(view.ActiveAlerts, DashboardAlert{
|
||||
NodeID: node.NodeID,
|
||||
NodeName: node.Name,
|
||||
EventType: event.EventType,
|
||||
Severity: event.Severity,
|
||||
Message: event.Message,
|
||||
LastTriggeredAt: event.LastTriggeredAt,
|
||||
Status: event.Status,
|
||||
})
|
||||
}
|
||||
|
||||
if latestSnapshot != nil {
|
||||
nodeHealth.CPUUsagePercent = latestSnapshot.CPUUsagePercent
|
||||
nodeHealth.MemoryUsagePercent = percentage(latestSnapshot.MemoryUsedBytes, latestSnapshot.MemoryTotalBytes)
|
||||
nodeHealth.StorageUsagePercent = percentage(latestSnapshot.StorageUsedBytes, latestSnapshot.StorageTotalBytes)
|
||||
if latestSnapshot.CPUUsagePercent > 0 {
|
||||
view.Capacity.AverageCPUUsagePercent += latestSnapshot.CPUUsagePercent
|
||||
cpuNodeCount++
|
||||
}
|
||||
if nodeHealth.MemoryUsagePercent > 0 {
|
||||
view.Capacity.AverageMemoryUsagePercent += nodeHealth.MemoryUsagePercent
|
||||
memoryNodeCount++
|
||||
}
|
||||
if latestSnapshot.CPUUsagePercent >= 80 {
|
||||
view.Capacity.HighCPUNodes++
|
||||
}
|
||||
if nodeHealth.MemoryUsagePercent >= 85 {
|
||||
view.Capacity.HighMemoryNodes++
|
||||
}
|
||||
if nodeHealth.StorageUsagePercent >= 85 {
|
||||
view.Capacity.HighStorageNodes++
|
||||
}
|
||||
}
|
||||
|
||||
if latestTraffic != nil {
|
||||
nodeHealth.RequestCount = latestTraffic.RequestCount
|
||||
nodeHealth.ErrorCount = latestTraffic.ErrorCount
|
||||
nodeHealth.UniqueVisitorCount = latestTraffic.UniqueVisitorCount
|
||||
view.Traffic.RequestCount += latestTraffic.RequestCount
|
||||
view.Traffic.UniqueVisitors += latestTraffic.UniqueVisitorCount
|
||||
view.Traffic.ErrorCount += latestTraffic.ErrorCount
|
||||
if duration := latestTraffic.WindowEndedAt.Sub(latestTraffic.WindowStartedAt).Seconds(); duration > 0 {
|
||||
view.Traffic.EstimatedQPS += float64(latestTraffic.RequestCount) / duration
|
||||
}
|
||||
view.Traffic.ReportedNodes++
|
||||
}
|
||||
|
||||
view.Summary.ActiveAlerts += len(activeEvents)
|
||||
view.Nodes = append(view.Nodes, nodeHealth)
|
||||
}
|
||||
|
||||
view.Summary.TotalNodes = len(nodes)
|
||||
view.Config.ActiveVersion = activeVersion
|
||||
view.Config.LaggingNodes = view.Summary.LaggingNodes
|
||||
view.Config.PendingNodes = view.Summary.PendingNodes
|
||||
|
||||
if cpuNodeCount > 0 {
|
||||
view.Capacity.AverageCPUUsagePercent /= float64(cpuNodeCount)
|
||||
}
|
||||
if memoryNodeCount > 0 {
|
||||
view.Capacity.AverageMemoryUsagePercent /= float64(memoryNodeCount)
|
||||
}
|
||||
|
||||
sort.Slice(view.ActiveAlerts, func(i int, j int) bool {
|
||||
if severityWeight(view.ActiveAlerts[i].Severity) == severityWeight(view.ActiveAlerts[j].Severity) {
|
||||
return view.ActiveAlerts[i].LastTriggeredAt.After(view.ActiveAlerts[j].LastTriggeredAt)
|
||||
}
|
||||
return severityWeight(view.ActiveAlerts[i].Severity) > severityWeight(view.ActiveAlerts[j].Severity)
|
||||
})
|
||||
|
||||
sort.Slice(view.Nodes, func(i int, j int) bool {
|
||||
if view.Nodes[i].ActiveEventCount == view.Nodes[j].ActiveEventCount {
|
||||
return view.Nodes[i].CPUUsagePercent > view.Nodes[j].CPUUsagePercent
|
||||
}
|
||||
return view.Nodes[i].ActiveEventCount > view.Nodes[j].ActiveEventCount
|
||||
})
|
||||
|
||||
if len(view.ActiveAlerts) > 8 {
|
||||
view.ActiveAlerts = view.ActiveAlerts[:8]
|
||||
}
|
||||
|
||||
return view, nil
|
||||
}
|
||||
|
||||
func latestMetricSnapshotForNode(nodeID string, since time.Time) *model.NodeMetricSnapshot {
|
||||
snapshots, err := model.ListNodeMetricSnapshots(nodeID, since, 1)
|
||||
if err != nil || len(snapshots) == 0 {
|
||||
return nil
|
||||
}
|
||||
return snapshots[0]
|
||||
}
|
||||
|
||||
func latestTrafficReportForNode(nodeID string, since time.Time) *model.NodeRequestReport {
|
||||
reports, err := model.ListNodeRequestReports(nodeID, since, 1)
|
||||
if err != nil || len(reports) == 0 {
|
||||
return nil
|
||||
}
|
||||
return reports[0]
|
||||
}
|
||||
|
||||
func percentage(used int64, total int64) float64 {
|
||||
if used <= 0 || total <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (float64(used) / float64(total)) * 100
|
||||
}
|
||||
|
||||
func severityWeight(severity string) int {
|
||||
switch severity {
|
||||
case NodeHealthSeverityCritical:
|
||||
return 3
|
||||
case NodeHealthSeverityWarning:
|
||||
return 2
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
@@ -590,3 +590,129 @@ func TestGetNodeObservabilityAllowsMissingProfile(t *testing.T) {
|
||||
t.Fatalf("expected nil profile when profile not reported, got %+v", view.Profile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDashboardOverview(t *testing.T) {
|
||||
setupServiceTestDB(t)
|
||||
|
||||
now := time.Now()
|
||||
if err := model.DB.Create(&model.ConfigVersion{
|
||||
Version: "20260314-001",
|
||||
SnapshotJSON: "{}",
|
||||
MainConfig: "worker_processes auto;",
|
||||
RenderedConfig: "server { listen 80; }",
|
||||
Checksum: "checksum-dashboard",
|
||||
IsActive: true,
|
||||
CreatedBy: "root",
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("failed to seed active config version: %v", err)
|
||||
}
|
||||
|
||||
nodes := []*model.Node{
|
||||
{
|
||||
NodeID: "node-dashboard-a",
|
||||
Name: "edge-a",
|
||||
IP: "10.0.0.71",
|
||||
AgentToken: "token-a",
|
||||
AgentVersion: "v0.6.0",
|
||||
NginxVersion: "1.27.1.2",
|
||||
OpenrestyStatus: OpenrestyStatusHealthy,
|
||||
Status: NodeStatusOnline,
|
||||
CurrentVersion: "20260314-001",
|
||||
LastSeenAt: now,
|
||||
},
|
||||
{
|
||||
NodeID: "node-dashboard-b",
|
||||
Name: "edge-b",
|
||||
IP: "10.0.0.72",
|
||||
AgentToken: "token-b",
|
||||
AgentVersion: "v0.6.0",
|
||||
NginxVersion: "1.27.1.2",
|
||||
OpenrestyStatus: OpenrestyStatusUnhealthy,
|
||||
Status: NodeStatusOnline,
|
||||
CurrentVersion: "20260313-001",
|
||||
LastSeenAt: now,
|
||||
},
|
||||
}
|
||||
for _, node := range nodes {
|
||||
if err := node.Insert(); err != nil {
|
||||
t.Fatalf("failed to insert node %s: %v", node.NodeID, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := (&model.NodeMetricSnapshot{
|
||||
NodeID: "node-dashboard-a",
|
||||
CapturedAt: now,
|
||||
CPUUsagePercent: 45,
|
||||
MemoryUsedBytes: 4 * 1024 * 1024 * 1024,
|
||||
MemoryTotalBytes: 8 * 1024 * 1024 * 1024,
|
||||
StorageUsedBytes: 50 * 1024 * 1024 * 1024,
|
||||
StorageTotalBytes: 100 * 1024 * 1024 * 1024,
|
||||
}).Insert(); err != nil {
|
||||
t.Fatalf("failed to insert node a metric snapshot: %v", err)
|
||||
}
|
||||
if err := (&model.NodeMetricSnapshot{
|
||||
NodeID: "node-dashboard-b",
|
||||
CapturedAt: now,
|
||||
CPUUsagePercent: 92,
|
||||
MemoryUsedBytes: 15 * 1024 * 1024 * 1024,
|
||||
MemoryTotalBytes: 16 * 1024 * 1024 * 1024,
|
||||
StorageUsedBytes: 95 * 1024 * 1024 * 1024,
|
||||
StorageTotalBytes: 100 * 1024 * 1024 * 1024,
|
||||
}).Insert(); err != nil {
|
||||
t.Fatalf("failed to insert node b metric snapshot: %v", err)
|
||||
}
|
||||
|
||||
if err := (&model.NodeRequestReport{
|
||||
NodeID: "node-dashboard-a",
|
||||
WindowStartedAt: now.Add(-time.Minute),
|
||||
WindowEndedAt: now,
|
||||
RequestCount: 600,
|
||||
ErrorCount: 6,
|
||||
UniqueVisitorCount: 120,
|
||||
}).Insert(); err != nil {
|
||||
t.Fatalf("failed to insert node a traffic report: %v", err)
|
||||
}
|
||||
if err := (&model.NodeRequestReport{
|
||||
NodeID: "node-dashboard-b",
|
||||
WindowStartedAt: now.Add(-time.Minute),
|
||||
WindowEndedAt: now,
|
||||
RequestCount: 300,
|
||||
ErrorCount: 30,
|
||||
UniqueVisitorCount: 80,
|
||||
}).Insert(); err != nil {
|
||||
t.Fatalf("failed to insert node b traffic report: %v", err)
|
||||
}
|
||||
|
||||
if err := model.DB.Create(&model.NodeHealthEvent{
|
||||
NodeID: "node-dashboard-b",
|
||||
EventType: "openresty_unhealthy",
|
||||
Severity: NodeHealthSeverityCritical,
|
||||
Status: NodeHealthEventStatusActive,
|
||||
Message: "reload failed",
|
||||
FirstTriggeredAt: now.Add(-time.Minute),
|
||||
LastTriggeredAt: now,
|
||||
ReportedAt: now,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("failed to insert dashboard health event: %v", err)
|
||||
}
|
||||
|
||||
view, err := GetDashboardOverview()
|
||||
if err != nil {
|
||||
t.Fatalf("GetDashboardOverview failed: %v", err)
|
||||
}
|
||||
if view.Summary.TotalNodes != 2 || view.Summary.OnlineNodes != 2 {
|
||||
t.Fatalf("unexpected dashboard summary: %+v", view.Summary)
|
||||
}
|
||||
if view.Summary.UnhealthyNodes != 1 || view.Summary.ActiveAlerts != 1 {
|
||||
t.Fatalf("unexpected unhealthy/alert summary: %+v", view.Summary)
|
||||
}
|
||||
if view.Traffic.RequestCount != 900 || view.Traffic.ErrorCount != 36 {
|
||||
t.Fatalf("unexpected dashboard traffic: %+v", view.Traffic)
|
||||
}
|
||||
if view.Config.ActiveVersion != "20260314-001" || view.Config.LaggingNodes != 1 {
|
||||
t.Fatalf("unexpected dashboard config summary: %+v", view.Config)
|
||||
}
|
||||
if len(view.Nodes) != 2 || len(view.ActiveAlerts) != 1 {
|
||||
t.Fatalf("unexpected dashboard nodes/alerts: %+v %+v", view.Nodes, view.ActiveAlerts)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user