mirror of
https://github.com/Sagit-chu/flvx.git
synced 2026-10-13 12:46:36 +08:00
feat(monitoring): add node/tunnel metrics, service monitors, and health checks
- Add NodeMetric/TunnelMetric/ServiceMonitor models and repository methods - Implement metrics ingestion service with per-minute bucket aggregation - Add health checker for node connectivity monitoring - Wire node metrics from WebSocket SystemInfo messages - Add tunnel metrics ingestion from flow upload endpoint - Create monitoring REST API endpoints for nodes, tunnels, services - Implement service monitor CRUD and execution (TCP/ICMP checks) - Add MonitorPermission for non-admin access control - Create frontend monitor page with node/tunnel/service views - Add tunnel metrics ingestion from agent flow reports - Include schema migration for tunnel_metric unique index - Fix tunnel entry port conflict validation to use transaction Entire-Checkpoint: 030821a7c8e3
This commit is contained in:
sagitchu
@@ -0,0 +1,360 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go-backend/internal/monitoring"
|
||||
"go-backend/internal/store/model"
|
||||
"go-backend/internal/store/repo"
|
||||
"go-backend/internal/ws"
|
||||
)
|
||||
|
||||
type nodeCommander interface {
|
||||
SendCommand(nodeID int64, cmdType string, data interface{}, timeout time.Duration) (ws.CommandResult, error)
|
||||
}
|
||||
|
||||
type Checker struct {
|
||||
repo *repo.Repository
|
||||
commander nodeCommander
|
||||
lastRun map[int64]int64
|
||||
inFlight map[int64]struct{}
|
||||
|
||||
mu sync.RWMutex
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func NewChecker(repo *repo.Repository, commander nodeCommander) *Checker {
|
||||
return &Checker{
|
||||
repo: repo,
|
||||
commander: commander,
|
||||
lastRun: make(map[int64]int64),
|
||||
inFlight: make(map[int64]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Checker) Start(ctx context.Context) {
|
||||
c.mu.Lock()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
c.cancel = cancel
|
||||
c.mu.Unlock()
|
||||
|
||||
c.runChecks(ctx)
|
||||
|
||||
for {
|
||||
limits := c.loadServiceMonitorLimits()
|
||||
scanInterval := time.Duration(limits.CheckerScanIntervalSec) * time.Second
|
||||
if scanInterval <= 0 {
|
||||
scanInterval = 30 * time.Second
|
||||
}
|
||||
|
||||
timer := time.NewTimer(scanInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
c.runChecks(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Checker) Stop() {
|
||||
c.mu.Lock()
|
||||
if c.cancel != nil {
|
||||
c.cancel()
|
||||
}
|
||||
c.mu.Unlock()
|
||||
c.wg.Wait()
|
||||
}
|
||||
|
||||
func (c *Checker) RunOnce(m *model.ServiceMonitor) (*model.ServiceMonitorResult, error) {
|
||||
if c == nil {
|
||||
return nil, errors.New("checker not initialized")
|
||||
}
|
||||
if m == nil {
|
||||
return nil, errors.New("monitor is nil")
|
||||
}
|
||||
limits := c.loadServiceMonitorLimits()
|
||||
return c.executeCheck(m, time.Now().UnixMilli(), limits), nil
|
||||
}
|
||||
|
||||
func (c *Checker) runChecks(ctx context.Context) {
|
||||
if c == nil || c.repo == nil {
|
||||
return
|
||||
}
|
||||
|
||||
limits := c.loadServiceMonitorLimits()
|
||||
monitors, err := c.repo.ListEnabledServiceMonitors()
|
||||
if err != nil {
|
||||
log.Printf("service monitor scheduler failed op=list_enabled err=%v", err)
|
||||
return
|
||||
}
|
||||
if len(monitors) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Use persisted result timestamps to avoid restart bursts.
|
||||
latest, err := c.repo.GetLatestServiceMonitorResults()
|
||||
if err != nil {
|
||||
log.Printf("service monitor scheduler failed op=get_latest_results err=%v", err)
|
||||
latest = nil
|
||||
}
|
||||
persistedLast := make(map[int64]int64, len(latest))
|
||||
for _, r := range latest {
|
||||
if r.MonitorID <= 0 || r.Timestamp <= 0 {
|
||||
continue
|
||||
}
|
||||
persistedLast[r.MonitorID] = r.Timestamp
|
||||
}
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
due := make([]model.ServiceMonitor, 0, len(monitors))
|
||||
for _, m := range monitors {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
intervalSec := m.IntervalSec
|
||||
if intervalSec <= 0 {
|
||||
intervalSec = limits.DefaultIntervalSec
|
||||
}
|
||||
if intervalSec < limits.MinIntervalSec {
|
||||
intervalSec = limits.MinIntervalSec
|
||||
}
|
||||
intervalMs := int64(intervalSec) * 1000
|
||||
|
||||
c.mu.Lock()
|
||||
if _, ok := c.inFlight[m.ID]; ok {
|
||||
c.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
lastSeen := persistedLast[m.ID]
|
||||
if v := c.lastRun[m.ID]; v > lastSeen {
|
||||
lastSeen = v
|
||||
}
|
||||
if lastSeen > 0 && intervalMs > 0 && now-lastSeen < intervalMs {
|
||||
c.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
c.inFlight[m.ID] = struct{}{}
|
||||
// Use now as a best-effort guard against overlapping scans; the final
|
||||
// timestamp is updated again when the result is persisted.
|
||||
c.lastRun[m.ID] = now
|
||||
c.mu.Unlock()
|
||||
|
||||
due = append(due, m)
|
||||
}
|
||||
if len(due) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
workerLimit := limits.WorkerLimit
|
||||
if workerLimit <= 0 {
|
||||
workerLimit = 1
|
||||
}
|
||||
if workerLimit > len(due) {
|
||||
workerLimit = len(due)
|
||||
}
|
||||
|
||||
jobs := make(chan model.ServiceMonitor, len(due))
|
||||
for _, m := range due {
|
||||
jobs <- m
|
||||
}
|
||||
close(jobs)
|
||||
|
||||
for i := 0; i < workerLimit; i++ {
|
||||
c.wg.Add(1)
|
||||
go func() {
|
||||
defer c.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case m, ok := <-jobs:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
ts := time.Now().UnixMilli()
|
||||
result := c.executeCheck(&m, ts, limits)
|
||||
if err := c.repo.InsertServiceMonitorResult(result); err != nil {
|
||||
log.Printf("monitoring write failed op=service_monitor_result.insert monitor_id=%d err=%v", result.MonitorID, err)
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
c.lastRun[m.ID] = result.Timestamp
|
||||
delete(c.inFlight, m.ID)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Checker) executeCheck(m *model.ServiceMonitor, timestamp int64, limits monitoring.ServiceMonitorLimits) *model.ServiceMonitorResult {
|
||||
result := &model.ServiceMonitorResult{
|
||||
MonitorID: m.ID,
|
||||
NodeID: m.NodeID,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
|
||||
timeoutSec := m.TimeoutSec
|
||||
if timeoutSec <= 0 {
|
||||
timeoutSec = limits.DefaultTimeoutSec
|
||||
}
|
||||
if timeoutSec < limits.MinTimeoutSec {
|
||||
timeoutSec = limits.MinTimeoutSec
|
||||
}
|
||||
if timeoutSec > limits.MaxTimeoutSec {
|
||||
timeoutSec = limits.MaxTimeoutSec
|
||||
}
|
||||
|
||||
timeout := time.Duration(timeoutSec) * time.Second
|
||||
|
||||
// When nodeId is set, run checks on the specified node.
|
||||
if m.NodeID > 0 {
|
||||
c.checkOnNode(m, timeoutSec, timeout, result)
|
||||
return result
|
||||
}
|
||||
|
||||
switch strings.ToLower(strings.TrimSpace(m.Type)) {
|
||||
case "tcp":
|
||||
c.checkTCP(m.Target, timeout, result)
|
||||
case "icmp":
|
||||
result.Success = 0
|
||||
result.ErrorMessage = "ICMP 监控必须指定执行节点"
|
||||
default:
|
||||
result.Success = 0
|
||||
result.ErrorMessage = fmt.Sprintf("不支持的检查类型: %s", m.Type)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (c *Checker) loadServiceMonitorLimits() monitoring.ServiceMonitorLimits {
|
||||
defaults := monitoring.DefaultServiceMonitorLimits()
|
||||
if c == nil || c.repo == nil {
|
||||
return defaults
|
||||
}
|
||||
cfg, err := c.repo.GetConfigsByNames([]string{
|
||||
monitoring.ConfigServiceMonitorCheckerScanIntervalSec,
|
||||
monitoring.ConfigServiceMonitorWorkerLimit,
|
||||
monitoring.ConfigServiceMonitorMinIntervalSec,
|
||||
monitoring.ConfigServiceMonitorDefaultIntervalSec,
|
||||
monitoring.ConfigServiceMonitorMinTimeoutSec,
|
||||
monitoring.ConfigServiceMonitorDefaultTimeoutSec,
|
||||
monitoring.ConfigServiceMonitorMaxTimeoutSec,
|
||||
})
|
||||
if err != nil {
|
||||
return defaults
|
||||
}
|
||||
return monitoring.ServiceMonitorLimitsFromConfigMap(cfg)
|
||||
}
|
||||
|
||||
type serviceMonitorCheckRequest struct {
|
||||
MonitorID int64 `json:"monitorId"`
|
||||
Type string `json:"type"`
|
||||
Target string `json:"target"`
|
||||
TimeoutSec int `json:"timeoutSec"`
|
||||
}
|
||||
|
||||
func (c *Checker) checkOnNode(m *model.ServiceMonitor, timeoutSec int, timeout time.Duration, result *model.ServiceMonitorResult) {
|
||||
if c == nil || m == nil || result == nil {
|
||||
return
|
||||
}
|
||||
if c.commander == nil {
|
||||
result.Success = 0
|
||||
result.ErrorMessage = "节点检查不可用"
|
||||
return
|
||||
}
|
||||
|
||||
checkType := strings.ToLower(strings.TrimSpace(m.Type))
|
||||
if checkType != "tcp" && checkType != "icmp" {
|
||||
result.Success = 0
|
||||
result.ErrorMessage = fmt.Sprintf("不支持的检查类型: %s", m.Type)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(m.Target) == "" {
|
||||
result.Success = 0
|
||||
result.ErrorMessage = "检查目标为空"
|
||||
return
|
||||
}
|
||||
|
||||
req := serviceMonitorCheckRequest{
|
||||
MonitorID: m.ID,
|
||||
Type: checkType,
|
||||
Target: m.Target,
|
||||
TimeoutSec: timeoutSec,
|
||||
}
|
||||
|
||||
cmdTimeout := timeout
|
||||
if cmdTimeout < 2*time.Second {
|
||||
cmdTimeout = 2 * time.Second
|
||||
}
|
||||
cmdTimeout = cmdTimeout + 2*time.Second
|
||||
|
||||
cmdRes, err := c.commander.SendCommand(m.NodeID, "ServiceMonitorCheck", req, cmdTimeout)
|
||||
if err != nil {
|
||||
result.Success = 0
|
||||
result.ErrorMessage = err.Error()
|
||||
return
|
||||
}
|
||||
if cmdRes.Data == nil {
|
||||
result.Success = 0
|
||||
result.ErrorMessage = "节点返回为空"
|
||||
return
|
||||
}
|
||||
|
||||
if v, ok := cmdRes.Data["success"]; ok {
|
||||
if b, ok := v.(bool); ok {
|
||||
if b {
|
||||
result.Success = 1
|
||||
} else {
|
||||
result.Success = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
if v, ok := cmdRes.Data["latencyMs"]; ok {
|
||||
if f, ok := v.(float64); ok {
|
||||
result.LatencyMs = f
|
||||
}
|
||||
}
|
||||
if v, ok := cmdRes.Data["statusCode"]; ok {
|
||||
if f, ok := v.(float64); ok {
|
||||
result.StatusCode = int(f)
|
||||
}
|
||||
}
|
||||
if v, ok := cmdRes.Data["errorMessage"]; ok {
|
||||
if s, ok := v.(string); ok {
|
||||
result.ErrorMessage = s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Checker) checkTCP(target string, timeout time.Duration, result *model.ServiceMonitorResult) {
|
||||
start := time.Now()
|
||||
|
||||
conn, err := net.DialTimeout("tcp", target, timeout)
|
||||
latency := time.Since(start)
|
||||
|
||||
result.LatencyMs = float64(latency.Milliseconds())
|
||||
|
||||
if err != nil {
|
||||
result.Success = 0
|
||||
result.ErrorMessage = err.Error()
|
||||
return
|
||||
}
|
||||
_ = conn.Close()
|
||||
result.Success = 1
|
||||
}
|
||||
Reference in New Issue
Block a user