mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 23:26:38 +08:00
feat: add phase-one node observability ingestion
This commit is contained in:
ryan
@@ -3,9 +3,9 @@ package model
|
||||
import (
|
||||
"atsflare/common"
|
||||
"github.com/glebarez/sqlite"
|
||||
"log/slog"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
@@ -90,10 +90,26 @@ func InitDB() (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = db.AutoMigrate(&NodeSystemProfile{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = db.AutoMigrate(&ApplyLog{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = db.AutoMigrate(&NodeMetricSnapshot{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = db.AutoMigrate(&NodeRequestReport{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = db.AutoMigrate(&NodeHealthEvent{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = db.AutoMigrate(&TLSCertificate{})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type NodeHealthEvent struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
EventType string `json:"event_type" gorm:"index;size:64;not null"`
|
||||
Severity string `json:"severity" gorm:"size:16;not null"`
|
||||
Status string `json:"status" gorm:"index;size:16;not null"`
|
||||
Message string `json:"message" gorm:"size:2048"`
|
||||
FirstTriggeredAt time.Time `json:"first_triggered_at" gorm:"index"`
|
||||
LastTriggeredAt time.Time `json:"last_triggered_at" gorm:"index"`
|
||||
ReportedAt time.Time `json:"reported_at" gorm:"index"`
|
||||
ResolvedAt *time.Time `json:"resolved_at" gorm:"index"`
|
||||
RawJSON string `json:"raw_json" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func GetActiveNodeHealthEvent(nodeID string, eventType string) (*NodeHealthEvent, error) {
|
||||
event := &NodeHealthEvent{}
|
||||
err := DB.Where("node_id = ? AND event_type = ? AND status = ?", nodeID, eventType, "active").First(event).Error
|
||||
return event, err
|
||||
}
|
||||
|
||||
func ListNodeHealthEvents(nodeID string, activeOnly bool, limit int) (events []*NodeHealthEvent, err error) {
|
||||
query := DB.Where("node_id = ?", nodeID).Order("last_triggered_at desc")
|
||||
if activeOnly {
|
||||
query = query.Where("status = ?", "active")
|
||||
}
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
err = query.Find(&events).Error
|
||||
return events, err
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type NodeMetricSnapshot struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
CapturedAt time.Time `json:"captured_at" gorm:"index"`
|
||||
CPUUsagePercent float64 `json:"cpu_usage_percent"`
|
||||
MemoryUsedBytes int64 `json:"memory_used_bytes"`
|
||||
MemoryTotalBytes int64 `json:"memory_total_bytes"`
|
||||
StorageUsedBytes int64 `json:"storage_used_bytes"`
|
||||
StorageTotalBytes int64 `json:"storage_total_bytes"`
|
||||
DiskReadBytes int64 `json:"disk_read_bytes"`
|
||||
DiskWriteBytes int64 `json:"disk_write_bytes"`
|
||||
NetworkRxBytes int64 `json:"network_rx_bytes"`
|
||||
NetworkTxBytes int64 `json:"network_tx_bytes"`
|
||||
OpenrestyRxBytes int64 `json:"openresty_rx_bytes"`
|
||||
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
|
||||
OpenrestyConnections int64 `json:"openresty_connections"`
|
||||
RawJSON string `json:"raw_json" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (snapshot *NodeMetricSnapshot) Insert() error {
|
||||
return DB.Create(snapshot).Error
|
||||
}
|
||||
|
||||
func ListNodeMetricSnapshots(nodeID string, since time.Time, limit int) (snapshots []*NodeMetricSnapshot, err error) {
|
||||
query := DB.Where("node_id = ?", nodeID).Order("captured_at desc")
|
||||
if !since.IsZero() {
|
||||
query = query.Where("captured_at >= ?", since)
|
||||
}
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
err = query.Find(&snapshots).Error
|
||||
return snapshots, err
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type NodeRequestReport struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
WindowStartedAt time.Time `json:"window_started_at" gorm:"index"`
|
||||
WindowEndedAt time.Time `json:"window_ended_at" gorm:"index"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
UniqueVisitorCount int64 `json:"unique_visitor_count"`
|
||||
StatusCodesJSON string `json:"status_codes_json" gorm:"type:text"`
|
||||
TopDomainsJSON string `json:"top_domains_json" gorm:"type:text"`
|
||||
SourceCountriesJSON string `json:"source_countries_json" gorm:"type:text"`
|
||||
RawJSON string `json:"raw_json" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (report *NodeRequestReport) Insert() error {
|
||||
return DB.Create(report).Error
|
||||
}
|
||||
|
||||
func ListNodeRequestReports(nodeID string, since time.Time, limit int) (reports []*NodeRequestReport, err error) {
|
||||
query := DB.Where("node_id = ?", nodeID).Order("window_ended_at desc")
|
||||
if !since.IsZero() {
|
||||
query = query.Where("window_ended_at >= ?", since)
|
||||
}
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
err = query.Find(&reports).Error
|
||||
return reports, err
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
type NodeSystemProfile struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
NodeID string `json:"node_id" gorm:"uniqueIndex;size:64;not null"`
|
||||
Hostname string `json:"hostname" gorm:"size:255"`
|
||||
OSName string `json:"os_name" gorm:"size:128"`
|
||||
OSVersion string `json:"os_version" gorm:"size:128"`
|
||||
KernelVersion string `json:"kernel_version" gorm:"size:128"`
|
||||
Architecture string `json:"architecture" gorm:"size:64"`
|
||||
CPUModel string `json:"cpu_model" gorm:"size:255"`
|
||||
CPUCores int `json:"cpu_cores"`
|
||||
TotalMemoryBytes int64 `json:"total_memory_bytes"`
|
||||
TotalDiskBytes int64 `json:"total_disk_bytes"`
|
||||
UptimeSeconds int64 `json:"uptime_seconds"`
|
||||
ReportedAt time.Time `json:"reported_at" gorm:"index"`
|
||||
RawJSON string `json:"raw_json" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func GetNodeSystemProfile(nodeID string) (*NodeSystemProfile, error) {
|
||||
profile := &NodeSystemProfile{}
|
||||
err := DB.Where("node_id = ?", nodeID).First(profile).Error
|
||||
return profile, err
|
||||
}
|
||||
|
||||
func UpsertNodeSystemProfile(profile *NodeSystemProfile) error {
|
||||
if profile == nil {
|
||||
return nil
|
||||
}
|
||||
return DB.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "node_id"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"hostname",
|
||||
"os_name",
|
||||
"os_version",
|
||||
"kernel_version",
|
||||
"architecture",
|
||||
"cpu_model",
|
||||
"cpu_cores",
|
||||
"total_memory_bytes",
|
||||
"total_disk_bytes",
|
||||
"uptime_seconds",
|
||||
"reported_at",
|
||||
"raw_json",
|
||||
"updated_at",
|
||||
}),
|
||||
}).Create(profile).Error
|
||||
}
|
||||
@@ -24,15 +24,19 @@ const (
|
||||
)
|
||||
|
||||
type AgentNodePayload struct {
|
||||
NodeID string `json:"node_id"`
|
||||
Name string `json:"name"`
|
||||
IP string `json:"ip"`
|
||||
AgentVersion string `json:"agent_version"`
|
||||
NginxVersion string `json:"nginx_version"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
LastError string `json:"last_error"`
|
||||
OpenrestyStatus string `json:"openresty_status"`
|
||||
OpenrestyMessage string `json:"openresty_message"`
|
||||
NodeID string `json:"node_id"`
|
||||
Name string `json:"name"`
|
||||
IP string `json:"ip"`
|
||||
AgentVersion string `json:"agent_version"`
|
||||
NginxVersion string `json:"nginx_version"`
|
||||
CurrentVersion string `json:"current_version"`
|
||||
LastError string `json:"last_error"`
|
||||
OpenrestyStatus string `json:"openresty_status"`
|
||||
OpenrestyMessage string `json:"openresty_message"`
|
||||
Profile *AgentNodeSystemProfile `json:"profile,omitempty"`
|
||||
Snapshot *AgentNodeMetricSnapshot `json:"snapshot,omitempty"`
|
||||
TrafficReport *AgentNodeTrafficReport `json:"traffic_report,omitempty"`
|
||||
HealthEvents []AgentNodeHealthEvent `json:"health_events"`
|
||||
}
|
||||
|
||||
type ApplyLogPayload struct {
|
||||
@@ -134,6 +138,7 @@ func HeartbeatNode(node *model.Node, payload AgentNodePayload) (*HeartbeatRespon
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
persistHeartbeatObservability(node.NodeID, payload, node.LastSeenAt)
|
||||
activeConfig, err := GetActiveConfigMetaForAgent()
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
|
||||
@@ -325,3 +325,171 @@ func TestListNodeViewsDoesNotPersistComputedStatus(t *testing.T) {
|
||||
t.Fatalf("expected list query to avoid persisting computed status, got %s", storedNode.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) {
|
||||
setupServiceTestDB(t)
|
||||
|
||||
node := &model.Node{
|
||||
NodeID: "node-observe-1",
|
||||
Name: "observe-edge-1",
|
||||
IP: "10.0.0.31",
|
||||
AgentToken: "token-observe",
|
||||
AgentVersion: "v0.6.0",
|
||||
NginxVersion: "1.27.1.2",
|
||||
Status: NodeStatusOnline,
|
||||
}
|
||||
if err := node.Insert(); err != nil {
|
||||
t.Fatalf("failed to seed node: %v", err)
|
||||
}
|
||||
|
||||
_, err := HeartbeatNode(node, AgentNodePayload{
|
||||
NodeID: node.NodeID,
|
||||
Name: node.Name,
|
||||
IP: node.IP,
|
||||
AgentVersion: node.AgentVersion,
|
||||
NginxVersion: node.NginxVersion,
|
||||
Profile: &AgentNodeSystemProfile{
|
||||
Hostname: "observe-edge-1",
|
||||
OSName: "Ubuntu",
|
||||
OSVersion: "24.04",
|
||||
KernelVersion: "6.8.0",
|
||||
Architecture: "amd64",
|
||||
CPUModel: "Intel Xeon",
|
||||
CPUCores: 8,
|
||||
TotalMemoryBytes: 16 * 1024 * 1024 * 1024,
|
||||
TotalDiskBytes: 200 * 1024 * 1024 * 1024,
|
||||
UptimeSeconds: 3600,
|
||||
ReportedAtUnix: time.Now().Add(-time.Minute).Unix(),
|
||||
},
|
||||
Snapshot: &AgentNodeMetricSnapshot{
|
||||
CapturedAtUnix: time.Now().Add(-30 * time.Second).Unix(),
|
||||
CPUUsagePercent: 42.5,
|
||||
MemoryUsedBytes: 8 * 1024 * 1024 * 1024,
|
||||
MemoryTotalBytes: 16 * 1024 * 1024 * 1024,
|
||||
StorageUsedBytes: 70 * 1024 * 1024 * 1024,
|
||||
StorageTotalBytes: 200 * 1024 * 1024 * 1024,
|
||||
DiskReadBytes: 1024,
|
||||
DiskWriteBytes: 2048,
|
||||
NetworkRxBytes: 4096,
|
||||
NetworkTxBytes: 8192,
|
||||
OpenrestyConnections: 128,
|
||||
},
|
||||
TrafficReport: &AgentNodeTrafficReport{
|
||||
WindowStartedAtUnix: time.Now().Add(-time.Minute).Unix(),
|
||||
WindowEndedAtUnix: time.Now().Unix(),
|
||||
RequestCount: 1200,
|
||||
ErrorCount: 12,
|
||||
UniqueVisitorCount: 320,
|
||||
StatusCodes: map[string]int64{"200": 1100, "502": 12},
|
||||
TopDomains: map[string]int64{"example.com": 900},
|
||||
SourceCountries: map[string]int64{"CN": 700, "US": 200},
|
||||
},
|
||||
HealthEvents: []AgentNodeHealthEvent{
|
||||
{
|
||||
EventType: "openresty_unhealthy",
|
||||
Severity: NodeHealthSeverityCritical,
|
||||
Message: "reload failed",
|
||||
TriggeredAtUnix: time.Now().Add(-2 * time.Minute).Unix(),
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected heartbeat to succeed: %v", err)
|
||||
}
|
||||
|
||||
profile, err := model.GetNodeSystemProfile(node.NodeID)
|
||||
if err != nil {
|
||||
t.Fatalf("expected node profile to persist: %v", err)
|
||||
}
|
||||
if profile.OSName != "Ubuntu" || profile.CPUCores != 8 {
|
||||
t.Fatalf("unexpected system profile: %+v", profile)
|
||||
}
|
||||
|
||||
snapshots, err := model.ListNodeMetricSnapshots(node.NodeID, time.Time{}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("expected node snapshots query to succeed: %v", err)
|
||||
}
|
||||
if len(snapshots) != 1 || snapshots[0].OpenrestyConnections != 128 {
|
||||
t.Fatalf("unexpected metric snapshots: %+v", snapshots)
|
||||
}
|
||||
|
||||
reports, err := model.ListNodeRequestReports(node.NodeID, time.Time{}, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("expected node request reports query to succeed: %v", err)
|
||||
}
|
||||
if len(reports) != 1 || reports[0].RequestCount != 1200 {
|
||||
t.Fatalf("unexpected request reports: %+v", reports)
|
||||
}
|
||||
|
||||
events, err := model.ListNodeHealthEvents(node.NodeID, true, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("expected node health events query to succeed: %v", err)
|
||||
}
|
||||
if len(events) != 1 || events[0].EventType != "openresty_unhealthy" {
|
||||
t.Fatalf("unexpected active health events: %+v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatNodeResolvesMissingHealthEvents(t *testing.T) {
|
||||
setupServiceTestDB(t)
|
||||
|
||||
node := &model.Node{
|
||||
NodeID: "node-event-1",
|
||||
Name: "event-edge-1",
|
||||
IP: "10.0.0.41",
|
||||
AgentToken: "token-event",
|
||||
AgentVersion: "v0.6.0",
|
||||
NginxVersion: "1.27.1.2",
|
||||
Status: NodeStatusOnline,
|
||||
}
|
||||
if err := node.Insert(); err != nil {
|
||||
t.Fatalf("failed to seed node: %v", err)
|
||||
}
|
||||
|
||||
_, err := HeartbeatNode(node, AgentNodePayload{
|
||||
NodeID: node.NodeID,
|
||||
Name: node.Name,
|
||||
IP: node.IP,
|
||||
AgentVersion: node.AgentVersion,
|
||||
NginxVersion: node.NginxVersion,
|
||||
HealthEvents: []AgentNodeHealthEvent{
|
||||
{
|
||||
EventType: "sync_error",
|
||||
Severity: NodeHealthSeverityWarning,
|
||||
Message: "checksum mismatch",
|
||||
TriggeredAtUnix: time.Now().Add(-time.Minute).Unix(),
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected first heartbeat to succeed: %v", err)
|
||||
}
|
||||
|
||||
_, err = HeartbeatNode(node, AgentNodePayload{
|
||||
NodeID: node.NodeID,
|
||||
Name: node.Name,
|
||||
IP: node.IP,
|
||||
AgentVersion: node.AgentVersion,
|
||||
NginxVersion: node.NginxVersion,
|
||||
HealthEvents: []AgentNodeHealthEvent{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected second heartbeat to succeed: %v", err)
|
||||
}
|
||||
|
||||
activeEvents, err := model.ListNodeHealthEvents(node.NodeID, true, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("expected active node health events query to succeed: %v", err)
|
||||
}
|
||||
if len(activeEvents) != 0 {
|
||||
t.Fatalf("expected no active health events, got %+v", activeEvents)
|
||||
}
|
||||
|
||||
allEvents, err := model.ListNodeHealthEvents(node.NodeID, false, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("expected all node health events query to succeed: %v", err)
|
||||
}
|
||||
if len(allEvents) != 1 || allEvents[0].Status != NodeHealthEventStatusResolved || allEvents[0].ResolvedAt == nil {
|
||||
t.Fatalf("expected resolved health event record, got %+v", allEvents)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"atsflare/model"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
NodeHealthEventStatusActive = "active"
|
||||
NodeHealthEventStatusResolved = "resolved"
|
||||
NodeHealthSeverityInfo = "info"
|
||||
NodeHealthSeverityWarning = "warning"
|
||||
NodeHealthSeverityCritical = "critical"
|
||||
)
|
||||
|
||||
type AgentNodeSystemProfile struct {
|
||||
Hostname string `json:"hostname"`
|
||||
OSName string `json:"os_name"`
|
||||
OSVersion string `json:"os_version"`
|
||||
KernelVersion string `json:"kernel_version"`
|
||||
Architecture string `json:"architecture"`
|
||||
CPUModel string `json:"cpu_model"`
|
||||
CPUCores int `json:"cpu_cores"`
|
||||
TotalMemoryBytes int64 `json:"total_memory_bytes"`
|
||||
TotalDiskBytes int64 `json:"total_disk_bytes"`
|
||||
UptimeSeconds int64 `json:"uptime_seconds"`
|
||||
ReportedAtUnix int64 `json:"reported_at_unix"`
|
||||
}
|
||||
|
||||
type AgentNodeMetricSnapshot struct {
|
||||
CapturedAtUnix int64 `json:"captured_at_unix"`
|
||||
CPUUsagePercent float64 `json:"cpu_usage_percent"`
|
||||
MemoryUsedBytes int64 `json:"memory_used_bytes"`
|
||||
MemoryTotalBytes int64 `json:"memory_total_bytes"`
|
||||
StorageUsedBytes int64 `json:"storage_used_bytes"`
|
||||
StorageTotalBytes int64 `json:"storage_total_bytes"`
|
||||
DiskReadBytes int64 `json:"disk_read_bytes"`
|
||||
DiskWriteBytes int64 `json:"disk_write_bytes"`
|
||||
NetworkRxBytes int64 `json:"network_rx_bytes"`
|
||||
NetworkTxBytes int64 `json:"network_tx_bytes"`
|
||||
OpenrestyRxBytes int64 `json:"openresty_rx_bytes"`
|
||||
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
|
||||
OpenrestyConnections int64 `json:"openresty_connections"`
|
||||
}
|
||||
|
||||
type AgentNodeTrafficReport struct {
|
||||
WindowStartedAtUnix int64 `json:"window_started_at_unix"`
|
||||
WindowEndedAtUnix int64 `json:"window_ended_at_unix"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
UniqueVisitorCount int64 `json:"unique_visitor_count"`
|
||||
StatusCodes map[string]int64 `json:"status_codes"`
|
||||
TopDomains map[string]int64 `json:"top_domains"`
|
||||
SourceCountries map[string]int64 `json:"source_countries"`
|
||||
}
|
||||
|
||||
type AgentNodeHealthEvent struct {
|
||||
EventType string `json:"event_type"`
|
||||
Severity string `json:"severity"`
|
||||
Message string `json:"message"`
|
||||
TriggeredAtUnix int64 `json:"triggered_at_unix"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
}
|
||||
|
||||
func persistHeartbeatObservability(nodeID string, payload AgentNodePayload, reportedAt time.Time) {
|
||||
if strings.TrimSpace(nodeID) == "" {
|
||||
return
|
||||
}
|
||||
if payload.Profile == nil && payload.Snapshot == nil && payload.TrafficReport == nil && payload.HealthEvents == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := model.DB.Transaction(func(tx *gorm.DB) error {
|
||||
if err := persistNodeSystemProfile(tx, nodeID, payload.Profile, reportedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := persistNodeMetricSnapshot(tx, nodeID, payload.Snapshot, reportedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := persistNodeTrafficReport(tx, nodeID, payload.TrafficReport, reportedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if payload.HealthEvents != nil {
|
||||
if err := reconcileNodeHealthEvents(tx, nodeID, payload.HealthEvents, reportedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
slog.Error("persist heartbeat observability failed", "node_id", nodeID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func persistNodeSystemProfile(tx *gorm.DB, nodeID string, profile *AgentNodeSystemProfile, reportedAt time.Time) error {
|
||||
if profile == nil {
|
||||
return nil
|
||||
}
|
||||
record := &model.NodeSystemProfile{
|
||||
NodeID: nodeID,
|
||||
Hostname: strings.TrimSpace(profile.Hostname),
|
||||
OSName: strings.TrimSpace(profile.OSName),
|
||||
OSVersion: strings.TrimSpace(profile.OSVersion),
|
||||
KernelVersion: strings.TrimSpace(profile.KernelVersion),
|
||||
Architecture: strings.TrimSpace(profile.Architecture),
|
||||
CPUModel: strings.TrimSpace(profile.CPUModel),
|
||||
CPUCores: profile.CPUCores,
|
||||
TotalMemoryBytes: profile.TotalMemoryBytes,
|
||||
TotalDiskBytes: profile.TotalDiskBytes,
|
||||
UptimeSeconds: profile.UptimeSeconds,
|
||||
ReportedAt: timeFromUnix(profile.ReportedAtUnix, reportedAt),
|
||||
RawJSON: marshalJSON(profile),
|
||||
}
|
||||
return tx.Model(&model.NodeSystemProfile{}).Where("node_id = ?", nodeID).Assign(record).FirstOrCreate(record).Error
|
||||
}
|
||||
|
||||
func persistNodeMetricSnapshot(tx *gorm.DB, nodeID string, snapshot *AgentNodeMetricSnapshot, reportedAt time.Time) error {
|
||||
if snapshot == nil {
|
||||
return nil
|
||||
}
|
||||
record := &model.NodeMetricSnapshot{
|
||||
NodeID: nodeID,
|
||||
CapturedAt: timeFromUnix(snapshot.CapturedAtUnix, reportedAt),
|
||||
CPUUsagePercent: snapshot.CPUUsagePercent,
|
||||
MemoryUsedBytes: snapshot.MemoryUsedBytes,
|
||||
MemoryTotalBytes: snapshot.MemoryTotalBytes,
|
||||
StorageUsedBytes: snapshot.StorageUsedBytes,
|
||||
StorageTotalBytes: snapshot.StorageTotalBytes,
|
||||
DiskReadBytes: snapshot.DiskReadBytes,
|
||||
DiskWriteBytes: snapshot.DiskWriteBytes,
|
||||
NetworkRxBytes: snapshot.NetworkRxBytes,
|
||||
NetworkTxBytes: snapshot.NetworkTxBytes,
|
||||
OpenrestyRxBytes: snapshot.OpenrestyRxBytes,
|
||||
OpenrestyTxBytes: snapshot.OpenrestyTxBytes,
|
||||
OpenrestyConnections: snapshot.OpenrestyConnections,
|
||||
RawJSON: marshalJSON(snapshot),
|
||||
}
|
||||
return tx.Create(record).Error
|
||||
}
|
||||
|
||||
func persistNodeTrafficReport(tx *gorm.DB, nodeID string, report *AgentNodeTrafficReport, reportedAt time.Time) error {
|
||||
if report == nil {
|
||||
return nil
|
||||
}
|
||||
if report.WindowEndedAtUnix > 0 && report.WindowStartedAtUnix > report.WindowEndedAtUnix {
|
||||
return errors.New("traffic report window_started_at_unix 不能大于 window_ended_at_unix")
|
||||
}
|
||||
record := &model.NodeRequestReport{
|
||||
NodeID: nodeID,
|
||||
WindowStartedAt: timeFromUnix(report.WindowStartedAtUnix, reportedAt),
|
||||
WindowEndedAt: timeFromUnix(report.WindowEndedAtUnix, reportedAt),
|
||||
RequestCount: report.RequestCount,
|
||||
ErrorCount: report.ErrorCount,
|
||||
UniqueVisitorCount: report.UniqueVisitorCount,
|
||||
StatusCodesJSON: marshalJSON(report.StatusCodes),
|
||||
TopDomainsJSON: marshalJSON(report.TopDomains),
|
||||
SourceCountriesJSON: marshalJSON(report.SourceCountries),
|
||||
RawJSON: marshalJSON(report),
|
||||
}
|
||||
return tx.Create(record).Error
|
||||
}
|
||||
|
||||
func reconcileNodeHealthEvents(tx *gorm.DB, nodeID string, events []AgentNodeHealthEvent, reportedAt time.Time) error {
|
||||
activeTypes := make(map[string]AgentNodeHealthEvent, len(events))
|
||||
for _, event := range events {
|
||||
eventType := normalizeHealthEventType(event.EventType)
|
||||
if eventType == "" {
|
||||
continue
|
||||
}
|
||||
event.EventType = eventType
|
||||
event.Severity = normalizeHealthSeverity(event.Severity)
|
||||
if event.TriggeredAtUnix <= 0 {
|
||||
event.TriggeredAtUnix = reportedAt.Unix()
|
||||
}
|
||||
activeTypes[eventType] = event
|
||||
}
|
||||
|
||||
var activeEvents []*model.NodeHealthEvent
|
||||
if err := tx.Where("node_id = ? AND status = ?", nodeID, NodeHealthEventStatusActive).Find(&activeEvents).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
activeByType := make(map[string]*model.NodeHealthEvent, len(activeEvents))
|
||||
for _, event := range activeEvents {
|
||||
activeByType[event.EventType] = event
|
||||
}
|
||||
|
||||
for eventType, event := range activeTypes {
|
||||
triggeredAt := timeFromUnix(event.TriggeredAtUnix, reportedAt)
|
||||
if existing, ok := activeByType[eventType]; ok {
|
||||
existing.Severity = event.Severity
|
||||
existing.Message = strings.TrimSpace(event.Message)
|
||||
existing.LastTriggeredAt = triggeredAt
|
||||
existing.ReportedAt = reportedAt
|
||||
existing.RawJSON = marshalJSON(event)
|
||||
existing.ResolvedAt = nil
|
||||
if err := tx.Save(existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
record := &model.NodeHealthEvent{
|
||||
NodeID: nodeID,
|
||||
EventType: eventType,
|
||||
Severity: event.Severity,
|
||||
Status: NodeHealthEventStatusActive,
|
||||
Message: strings.TrimSpace(event.Message),
|
||||
FirstTriggeredAt: triggeredAt,
|
||||
LastTriggeredAt: triggeredAt,
|
||||
ReportedAt: reportedAt,
|
||||
RawJSON: marshalJSON(event),
|
||||
}
|
||||
if err := tx.Create(record).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, existing := range activeEvents {
|
||||
if _, ok := activeTypes[existing.EventType]; ok {
|
||||
continue
|
||||
}
|
||||
resolvedAt := reportedAt
|
||||
existing.Status = NodeHealthEventStatusResolved
|
||||
existing.ReportedAt = reportedAt
|
||||
existing.ResolvedAt = &resolvedAt
|
||||
if err := tx.Save(existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeHealthEventType(eventType string) string {
|
||||
eventType = strings.TrimSpace(strings.ToLower(eventType))
|
||||
eventType = strings.ReplaceAll(eventType, " ", "_")
|
||||
return eventType
|
||||
}
|
||||
|
||||
func normalizeHealthSeverity(severity string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(severity)) {
|
||||
case NodeHealthSeverityCritical:
|
||||
return NodeHealthSeverityCritical
|
||||
case NodeHealthSeverityInfo:
|
||||
return NodeHealthSeverityInfo
|
||||
default:
|
||||
return NodeHealthSeverityWarning
|
||||
}
|
||||
}
|
||||
|
||||
func timeFromUnix(unixSeconds int64, fallback time.Time) time.Time {
|
||||
if unixSeconds <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return time.Unix(unixSeconds, 0).UTC()
|
||||
}
|
||||
|
||||
func marshalJSON(value any) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
Reference in New Issue
Block a user