mirror of
https://github.com/Sagit-chu/flvx.git
synced 2026-10-13 04:36:37 +08:00
feat(monitoring): add node/tunnel metrics, service monitors, and health checks
- Add NodeMetric/TunnelMetric/ServiceMonitor models and repository methods - Implement metrics ingestion service with per-minute bucket aggregation - Add health checker for node connectivity monitoring - Wire node metrics from WebSocket SystemInfo messages - Add tunnel metrics ingestion from flow upload endpoint - Create monitoring REST API endpoints for nodes, tunnels, services - Implement service monitor CRUD and execution (TCP/ICMP checks) - Add MonitorPermission for non-admin access control - Create frontend monitor page with node/tunnel/service views - Add tunnel metrics ingestion from agent flow reports - Include schema migration for tunnel_metric unique index - Fix tunnel entry port conflict validation to use transaction Entire-Checkpoint: 030821a7c8e3
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/store/model"
|
||||
"go-backend/internal/store/repo"
|
||||
)
|
||||
|
||||
type SystemInfo struct {
|
||||
Uptime uint64 `json:"uptime"`
|
||||
BytesReceived uint64 `json:"bytes_received"`
|
||||
BytesTransmitted uint64 `json:"bytes_transmitted"`
|
||||
CPUUsage float64 `json:"cpu_usage"`
|
||||
MemoryUsage float64 `json:"memory_usage"`
|
||||
DiskUsage float64 `json:"disk_usage"`
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
TCPConns int64 `json:"tcp_conns"`
|
||||
UDPConns int64 `json:"udp_conns"`
|
||||
NetInSpeed int64 `json:"net_in_speed"`
|
||||
NetOutSpeed int64 `json:"net_out_speed"`
|
||||
}
|
||||
|
||||
type IngestionService struct {
|
||||
repo *repo.Repository
|
||||
nodeBuffer []*model.NodeMetric
|
||||
nodeBufferMu sync.Mutex
|
||||
flushInterval time.Duration
|
||||
retentionDays int
|
||||
}
|
||||
|
||||
func NewIngestionService(repo *repo.Repository) *IngestionService {
|
||||
return &IngestionService{
|
||||
repo: repo,
|
||||
nodeBuffer: make([]*model.NodeMetric, 0, 500),
|
||||
flushInterval: 30 * time.Second,
|
||||
retentionDays: 7,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) Start(ctx context.Context) {
|
||||
flushTicker := time.NewTicker(s.flushInterval)
|
||||
defer flushTicker.Stop()
|
||||
|
||||
pruneTicker := time.NewTicker(1 * time.Hour)
|
||||
defer pruneTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.flushNodeMetrics()
|
||||
return
|
||||
case <-flushTicker.C:
|
||||
s.flushNodeMetrics()
|
||||
case <-pruneTicker.C:
|
||||
s.pruneMetrics()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) RecordNodeMetric(nodeID int64, info SystemInfo) {
|
||||
m := &model.NodeMetric{
|
||||
NodeID: nodeID,
|
||||
Timestamp: time.Now().UnixMilli(),
|
||||
CPUUsage: info.CPUUsage,
|
||||
MemUsage: info.MemoryUsage,
|
||||
DiskUsage: info.DiskUsage,
|
||||
NetInBytes: int64(info.BytesReceived),
|
||||
NetOutBytes: int64(info.BytesTransmitted),
|
||||
NetInSpeed: info.NetInSpeed,
|
||||
NetOutSpeed: info.NetOutSpeed,
|
||||
Load1: info.Load1,
|
||||
Load5: info.Load5,
|
||||
Load15: info.Load15,
|
||||
TCPConns: info.TCPConns,
|
||||
UDPConns: info.UDPConns,
|
||||
Uptime: int64(info.Uptime),
|
||||
}
|
||||
|
||||
s.nodeBufferMu.Lock()
|
||||
s.nodeBuffer = append(s.nodeBuffer, m)
|
||||
shouldFlush := len(s.nodeBuffer) >= 200
|
||||
s.nodeBufferMu.Unlock()
|
||||
|
||||
if shouldFlush {
|
||||
go s.flushNodeMetrics()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) flushNodeMetrics() {
|
||||
s.nodeBufferMu.Lock()
|
||||
if len(s.nodeBuffer) == 0 {
|
||||
s.nodeBufferMu.Unlock()
|
||||
return
|
||||
}
|
||||
buffer := s.nodeBuffer
|
||||
s.nodeBuffer = make([]*model.NodeMetric, 0, 500)
|
||||
s.nodeBufferMu.Unlock()
|
||||
|
||||
if s.repo == nil {
|
||||
return
|
||||
}
|
||||
if err := s.repo.InsertNodeMetricBatch(buffer); err != nil {
|
||||
log.Printf("monitoring write failed op=node_metric.flush count=%d err=%v", len(buffer), err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) pruneMetrics() {
|
||||
cutoff := time.Now().Add(-time.Duration(s.retentionDays) * 24 * time.Hour).UnixMilli()
|
||||
if s.repo == nil {
|
||||
return
|
||||
}
|
||||
if err := s.repo.PruneNodeMetrics(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=node_metric cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
if err := s.repo.PruneTunnelMetrics(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=tunnel_metric cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
if err := s.repo.PruneServiceMonitorResults(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=service_monitor_result cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) GetLatestMetric(nodeID int64) (*model.NodeMetric, error) {
|
||||
return s.repo.GetLatestNodeMetric(nodeID)
|
||||
}
|
||||
|
||||
func (s *IngestionService) GetMetrics(nodeID int64, startMs, endMs int64) ([]model.NodeMetric, error) {
|
||||
return s.repo.GetNodeMetrics(nodeID, startMs, endMs)
|
||||
}
|
||||
Reference in New Issue
Block a user