mirror of
https://github.com/Sagit-chu/flvx.git
synced 2026-10-11 03:36:37 +08:00
feat(monitoring): add node/tunnel metrics, service monitors, and health checks
- Add NodeMetric/TunnelMetric/ServiceMonitor models and repository methods - Implement metrics ingestion service with per-minute bucket aggregation - Add health checker for node connectivity monitoring - Wire node metrics from WebSocket SystemInfo messages - Add tunnel metrics ingestion from flow upload endpoint - Create monitoring REST API endpoints for nodes, tunnels, services - Implement service monitor CRUD and execution (TCP/ICMP checks) - Add MonitorPermission for non-admin access control - Create frontend monitor page with node/tunnel/service views - Add tunnel metrics ingestion from agent flow reports - Include schema migration for tunnel_metric unique index - Fix tunnel entry port conflict validation to use transaction Entire-Checkpoint: 030821a7c8e3
This commit is contained in:
sagitchu
@@ -0,0 +1,135 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/store/model"
|
||||
"go-backend/internal/store/repo"
|
||||
)
|
||||
|
||||
type SystemInfo struct {
|
||||
Uptime uint64 `json:"uptime"`
|
||||
BytesReceived uint64 `json:"bytes_received"`
|
||||
BytesTransmitted uint64 `json:"bytes_transmitted"`
|
||||
CPUUsage float64 `json:"cpu_usage"`
|
||||
MemoryUsage float64 `json:"memory_usage"`
|
||||
DiskUsage float64 `json:"disk_usage"`
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
TCPConns int64 `json:"tcp_conns"`
|
||||
UDPConns int64 `json:"udp_conns"`
|
||||
NetInSpeed int64 `json:"net_in_speed"`
|
||||
NetOutSpeed int64 `json:"net_out_speed"`
|
||||
}
|
||||
|
||||
type IngestionService struct {
|
||||
repo *repo.Repository
|
||||
nodeBuffer []*model.NodeMetric
|
||||
nodeBufferMu sync.Mutex
|
||||
flushInterval time.Duration
|
||||
retentionDays int
|
||||
}
|
||||
|
||||
func NewIngestionService(repo *repo.Repository) *IngestionService {
|
||||
return &IngestionService{
|
||||
repo: repo,
|
||||
nodeBuffer: make([]*model.NodeMetric, 0, 500),
|
||||
flushInterval: 30 * time.Second,
|
||||
retentionDays: 7,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) Start(ctx context.Context) {
|
||||
flushTicker := time.NewTicker(s.flushInterval)
|
||||
defer flushTicker.Stop()
|
||||
|
||||
pruneTicker := time.NewTicker(1 * time.Hour)
|
||||
defer pruneTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.flushNodeMetrics()
|
||||
return
|
||||
case <-flushTicker.C:
|
||||
s.flushNodeMetrics()
|
||||
case <-pruneTicker.C:
|
||||
s.pruneMetrics()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) RecordNodeMetric(nodeID int64, info SystemInfo) {
|
||||
m := &model.NodeMetric{
|
||||
NodeID: nodeID,
|
||||
Timestamp: time.Now().UnixMilli(),
|
||||
CPUUsage: info.CPUUsage,
|
||||
MemUsage: info.MemoryUsage,
|
||||
DiskUsage: info.DiskUsage,
|
||||
NetInBytes: int64(info.BytesReceived),
|
||||
NetOutBytes: int64(info.BytesTransmitted),
|
||||
NetInSpeed: info.NetInSpeed,
|
||||
NetOutSpeed: info.NetOutSpeed,
|
||||
Load1: info.Load1,
|
||||
Load5: info.Load5,
|
||||
Load15: info.Load15,
|
||||
TCPConns: info.TCPConns,
|
||||
UDPConns: info.UDPConns,
|
||||
Uptime: int64(info.Uptime),
|
||||
}
|
||||
|
||||
s.nodeBufferMu.Lock()
|
||||
s.nodeBuffer = append(s.nodeBuffer, m)
|
||||
shouldFlush := len(s.nodeBuffer) >= 200
|
||||
s.nodeBufferMu.Unlock()
|
||||
|
||||
if shouldFlush {
|
||||
go s.flushNodeMetrics()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) flushNodeMetrics() {
|
||||
s.nodeBufferMu.Lock()
|
||||
if len(s.nodeBuffer) == 0 {
|
||||
s.nodeBufferMu.Unlock()
|
||||
return
|
||||
}
|
||||
buffer := s.nodeBuffer
|
||||
s.nodeBuffer = make([]*model.NodeMetric, 0, 500)
|
||||
s.nodeBufferMu.Unlock()
|
||||
|
||||
if s.repo == nil {
|
||||
return
|
||||
}
|
||||
if err := s.repo.InsertNodeMetricBatch(buffer); err != nil {
|
||||
log.Printf("monitoring write failed op=node_metric.flush count=%d err=%v", len(buffer), err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) pruneMetrics() {
|
||||
cutoff := time.Now().Add(-time.Duration(s.retentionDays) * 24 * time.Hour).UnixMilli()
|
||||
if s.repo == nil {
|
||||
return
|
||||
}
|
||||
if err := s.repo.PruneNodeMetrics(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=node_metric cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
if err := s.repo.PruneTunnelMetrics(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=tunnel_metric cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
if err := s.repo.PruneServiceMonitorResults(cutoff); err != nil {
|
||||
log.Printf("monitoring prune failed op=service_monitor_result cutoff=%d err=%v", cutoff, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *IngestionService) GetLatestMetric(nodeID int64) (*model.NodeMetric, error) {
|
||||
return s.repo.GetLatestNodeMetric(nodeID)
|
||||
}
|
||||
|
||||
func (s *IngestionService) GetMetrics(nodeID int64, startMs, endMs int64) ([]model.NodeMetric, error) {
|
||||
return s.repo.GetNodeMetrics(nodeID, startMs, endMs)
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/store/repo"
|
||||
)
|
||||
|
||||
func TestRecordNodeMetric(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
|
||||
info := SystemInfo{
|
||||
Uptime: 86400,
|
||||
BytesReceived: 1024000,
|
||||
BytesTransmitted: 2048000,
|
||||
CPUUsage: 45.5,
|
||||
MemoryUsage: 60.2,
|
||||
DiskUsage: 30.1,
|
||||
Load1: 1.5,
|
||||
Load5: 1.2,
|
||||
Load15: 0.9,
|
||||
TCPConns: 100,
|
||||
UDPConns: 50,
|
||||
NetInSpeed: 51200,
|
||||
NetOutSpeed: 102400,
|
||||
}
|
||||
|
||||
svc.RecordNodeMetric(1, info)
|
||||
svc.flushNodeMetrics()
|
||||
|
||||
metrics, err := r.GetNodeMetrics(1, 0, time.Now().UnixMilli()+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get metrics: %v", err)
|
||||
}
|
||||
if len(metrics) != 1 {
|
||||
t.Fatalf("expected 1 metric, got %d", len(metrics))
|
||||
}
|
||||
|
||||
m := metrics[0]
|
||||
if m.CPUUsage != 45.5 {
|
||||
t.Fatalf("expected CPUUsage 45.5, got %f", m.CPUUsage)
|
||||
}
|
||||
if m.MemUsage != 60.2 {
|
||||
t.Fatalf("expected MemUsage 60.2, got %f", m.MemUsage)
|
||||
}
|
||||
if m.DiskUsage != 30.1 {
|
||||
t.Fatalf("expected DiskUsage 30.1, got %f", m.DiskUsage)
|
||||
}
|
||||
if m.Load1 != 1.5 {
|
||||
t.Fatalf("expected Load1 1.5, got %f", m.Load1)
|
||||
}
|
||||
if m.TCPConns != 100 {
|
||||
t.Fatalf("expected TCPConns 100, got %d", m.TCPConns)
|
||||
}
|
||||
if m.UDPConns != 50 {
|
||||
t.Fatalf("expected UDPConns 50, got %d", m.UDPConns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordNodeMetricAutoFlush(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
|
||||
info := SystemInfo{
|
||||
CPUUsage: 50.0,
|
||||
MemoryUsage: 60.0,
|
||||
DiskUsage: 30.0,
|
||||
}
|
||||
|
||||
for i := 0; i < 250; i++ {
|
||||
svc.RecordNodeMetric(1, info)
|
||||
}
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
metrics, err := r.GetNodeMetrics(1, 0, time.Now().UnixMilli()+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get metrics: %v", err)
|
||||
}
|
||||
if len(metrics) < 200 {
|
||||
t.Fatalf("expected at least 200 metrics after auto-flush, got %d", len(metrics))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestionServiceStart(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
svc.flushInterval = 100 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
info := SystemInfo{
|
||||
CPUUsage: 45.0,
|
||||
MemoryUsage: 55.0,
|
||||
DiskUsage: 35.0,
|
||||
}
|
||||
|
||||
go svc.Start(ctx)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
svc.RecordNodeMetric(1, info)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
<-ctx.Done()
|
||||
|
||||
metrics, err := r.GetNodeMetrics(1, 0, time.Now().UnixMilli()+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get metrics: %v", err)
|
||||
}
|
||||
if len(metrics) == 0 {
|
||||
t.Fatalf("expected metrics after service run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLatestMetric(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
|
||||
info1 := SystemInfo{CPUUsage: 40.0, MemoryUsage: 50.0, DiskUsage: 30.0}
|
||||
svc.RecordNodeMetric(1, info1)
|
||||
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
|
||||
info2 := SystemInfo{CPUUsage: 60.0, MemoryUsage: 70.0, DiskUsage: 40.0}
|
||||
svc.RecordNodeMetric(1, info2)
|
||||
|
||||
svc.flushNodeMetrics()
|
||||
|
||||
latest, err := svc.GetLatestMetric(1)
|
||||
if err != nil {
|
||||
t.Fatalf("get latest: %v", err)
|
||||
}
|
||||
if latest == nil {
|
||||
t.Fatalf("expected latest metric")
|
||||
}
|
||||
if latest.CPUUsage != 60.0 {
|
||||
t.Fatalf("expected latest CPUUsage 60.0, got %f", latest.CPUUsage)
|
||||
}
|
||||
|
||||
_ = now
|
||||
|
||||
latestNone, err := svc.GetLatestMetric(999)
|
||||
if err != nil {
|
||||
t.Fatalf("get latest for non-existent: %v", err)
|
||||
}
|
||||
if latestNone != nil {
|
||||
t.Fatalf("expected nil for non-existent node")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetMetricsWithTimeRange(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
info := SystemInfo{
|
||||
CPUUsage: float64(40 + i*5),
|
||||
MemoryUsage: 50.0,
|
||||
DiskUsage: 30.0,
|
||||
}
|
||||
svc.RecordNodeMetric(1, info)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
svc.flushNodeMetrics()
|
||||
|
||||
metrics, err := svc.GetMetrics(1, 0, now+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get metrics: %v", err)
|
||||
}
|
||||
if len(metrics) != 5 {
|
||||
t.Fatalf("expected 5 metrics, got %d", len(metrics))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneMetrics(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
svc.retentionDays = 1
|
||||
|
||||
info := SystemInfo{CPUUsage: 50.0, MemoryUsage: 60.0, DiskUsage: 30.0}
|
||||
|
||||
svc.RecordNodeMetric(1, info)
|
||||
svc.flushNodeMetrics()
|
||||
|
||||
svc.pruneMetrics()
|
||||
|
||||
metrics, err := r.GetNodeMetrics(1, 0, time.Now().UnixMilli()+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get metrics: %v", err)
|
||||
}
|
||||
if len(metrics) != 1 {
|
||||
t.Fatalf("expected 1 metric (not pruned), got %d", len(metrics))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultipleNodes(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
|
||||
info := SystemInfo{
|
||||
CPUUsage: 50.0,
|
||||
MemoryUsage: 60.0,
|
||||
DiskUsage: 30.0,
|
||||
}
|
||||
|
||||
svc.RecordNodeMetric(1, info)
|
||||
svc.RecordNodeMetric(2, info)
|
||||
svc.RecordNodeMetric(3, info)
|
||||
|
||||
svc.flushNodeMetrics()
|
||||
|
||||
for nodeID := int64(1); nodeID <= 3; nodeID++ {
|
||||
metrics, err := r.GetNodeMetrics(nodeID, 0, time.Now().UnixMilli()+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get metrics for node %d: %v", nodeID, err)
|
||||
}
|
||||
if len(metrics) != 1 {
|
||||
t.Fatalf("expected 1 metric for node %d, got %d", nodeID, len(metrics))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroValues(t *testing.T) {
|
||||
r, err := repo.Open(":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open repo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
svc := NewIngestionService(r)
|
||||
|
||||
info := SystemInfo{}
|
||||
|
||||
svc.RecordNodeMetric(1, info)
|
||||
svc.flushNodeMetrics()
|
||||
|
||||
metrics, err := r.GetNodeMetrics(1, 0, time.Now().UnixMilli()+1000)
|
||||
if err != nil {
|
||||
t.Fatalf("get metrics: %v", err)
|
||||
}
|
||||
if len(metrics) != 1 {
|
||||
t.Fatalf("expected 1 metric, got %d", len(metrics))
|
||||
}
|
||||
|
||||
m := metrics[0]
|
||||
if m.CPUUsage != 0 || m.MemUsage != 0 || m.DiskUsage != 0 {
|
||||
t.Fatalf("expected zero values, got CPU=%f Mem=%f Disk=%f", m.CPUUsage, m.MemUsage, m.DiskUsage)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user