flvx/go-backend/internal/health/checker.go

package health

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"go-backend/internal/monitoring"
	"go-backend/internal/store/model"
	"go-backend/internal/store/repo"
	"go-backend/internal/ws"
)

type nodeCommander interface {
	SendCommand(nodeID int64, cmdType string, data interface{}, timeout time.Duration) (ws.CommandResult, error)
}

const serviceMonitorReportInterval = 30 * time.Second // DB write interval per monitor

type Checker struct {
	repo      *repo.Repository
	commander nodeCommander
	lastRun   map[int64]int64
	inFlight  map[int64]struct{}

	// In-memory latest result per monitor (for real-time API reads)
	latestResults map[int64]*model.ServiceMonitorResult
	lastDBWrite   map[int64]int64 // last DB write timestamp per monitorID

	mu       sync.RWMutex
	cancel   context.CancelFunc
	wg       sync.WaitGroup
	checking int32 // atomic flag: 1 = runChecks running, 0 = idle
}

func NewChecker(repo *repo.Repository, commander nodeCommander) *Checker {
	return &Checker{
		repo:          repo,
		commander:     commander,
		lastRun:       make(map[int64]int64),
		inFlight:      make(map[int64]struct{}),
		latestResults: make(map[int64]*model.ServiceMonitorResult),
		lastDBWrite:   make(map[int64]int64),
	}
}

// GetLatestCached returns the in-memory latest results (updated every 1s).
// Returns nil if no results are cached.
func (c *Checker) GetLatestCached() []*model.ServiceMonitorResult {
	c.mu.RLock()
	defer c.mu.RUnlock()
	results := make([]*model.ServiceMonitorResult, 0, len(c.latestResults))
	for _, r := range c.latestResults {
		results = append(results, r)
	}
	return results
}

func (c *Checker) Start(ctx context.Context) {
	c.mu.Lock()
	ctx, cancel := context.WithCancel(ctx)
	c.cancel = cancel
	c.mu.Unlock()

	c.runChecks(ctx)

	for {
		limits := c.loadServiceMonitorLimits()
		scanInterval := time.Duration(limits.CheckerScanIntervalSec) * time.Second
		if scanInterval <= 0 {
			scanInterval = 1 * time.Second
		}

		timer := time.NewTimer(scanInterval)
		select {
		case <-ctx.Done():
			timer.Stop()
			return
		case <-timer.C:
			c.runChecks(ctx)
		}
	}
}

func (c *Checker) Stop() {
	c.mu.Lock()
	if c.cancel != nil {
		c.cancel()
	}
	c.mu.Unlock()
	c.wg.Wait()
}

func (c *Checker) RunOnce(m *model.ServiceMonitor) (*model.ServiceMonitorResult, error) {
	if c == nil {
		return nil, errors.New("checker not initialized")
	}
	if m == nil {
		return nil, errors.New("monitor is nil")
	}
	limits := c.loadServiceMonitorLimits()
	return c.executeCheck(m, time.Now().UnixMilli(), limits), nil
}

func (c *Checker) runChecks(ctx context.Context) {
	// Skip if previous round is still running (interval < timeout guard)
	if !atomic.CompareAndSwapInt32(&c.checking, 0, 1) {
		return
	}
	defer atomic.StoreInt32(&c.checking, 0)

	if c == nil || c.repo == nil {
		return
	}

	limits := c.loadServiceMonitorLimits()
	monitors, err := c.repo.ListEnabledServiceMonitors()
	if err != nil {
		log.Printf("service monitor scheduler failed op=list_enabled err=%v", err)
		return
	}
	if len(monitors) == 0 {
		return
	}

	// Use persisted result timestamps to avoid restart bursts.
	latest, err := c.repo.GetLatestServiceMonitorResults()
	if err != nil {
		log.Printf("service monitor scheduler failed op=get_latest_results err=%v", err)
		latest = nil
	}
	persistedLast := make(map[int64]int64, len(latest))
	for _, r := range latest {
		if r.MonitorID <= 0 || r.Timestamp <= 0 {
			continue
		}
		persistedLast[r.MonitorID] = r.Timestamp
	}

	now := time.Now().UnixMilli()
	due := make([]model.ServiceMonitor, 0, len(monitors))
	for _, m := range monitors {
		select {
		case <-ctx.Done():
			return
		default:
		}

		intervalSec := m.IntervalSec
		if intervalSec <= 0 {
			intervalSec = limits.DefaultIntervalSec
		}
		if intervalSec < limits.MinIntervalSec {
			intervalSec = limits.MinIntervalSec
		}
		intervalMs := int64(intervalSec) * 1000

		c.mu.Lock()
		if _, ok := c.inFlight[m.ID]; ok {
			c.mu.Unlock()
			continue
		}

		lastSeen := persistedLast[m.ID]
		if v := c.lastRun[m.ID]; v > lastSeen {
			lastSeen = v
		}
		if lastSeen > 0 && intervalMs > 0 && now-lastSeen < intervalMs {
			c.mu.Unlock()
			continue
		}

		c.inFlight[m.ID] = struct{}{}
		// Use now as a best-effort guard against overlapping scans; the final
		// timestamp is updated again when the result is persisted.
		c.lastRun[m.ID] = now
		c.mu.Unlock()

		due = append(due, m)
	}
	if len(due) == 0 {
		return
	}

	workerLimit := limits.WorkerLimit
	if workerLimit <= 0 {
		workerLimit = 1
	}
	if workerLimit > len(due) {
		workerLimit = len(due)
	}

	jobs := make(chan model.ServiceMonitor, len(due))
	for _, m := range due {
		jobs <- m
	}
	close(jobs)

	reportIntervalMs := int64(serviceMonitorReportInterval / time.Millisecond)

	for i := 0; i < workerLimit; i++ {
		c.wg.Add(1)
		go func() {
			defer c.wg.Done()
			for {
				select {
				case <-ctx.Done():
					return
				case m, ok := <-jobs:
					if !ok {
						return
					}
					ts := time.Now().UnixMilli()
					result := c.executeCheck(&m, ts, limits)

					// Always update in-memory cache for real-time reads
					c.mu.Lock()
					c.latestResults[m.ID] = result
					c.lastRun[m.ID] = result.Timestamp
					delete(c.inFlight, m.ID)

					// Only write to DB every 30s per monitor
					lastWrite := c.lastDBWrite[m.ID]
					writeToDB := ts-lastWrite >= reportIntervalMs
					if writeToDB {
						c.lastDBWrite[m.ID] = ts
					}
					c.mu.Unlock()

					if writeToDB {
						if err := c.repo.InsertServiceMonitorResult(result); err != nil {
							log.Printf("monitoring write failed op=service_monitor_result.insert monitor_id=%d err=%v", result.MonitorID, err)
						}
					}
				}
			}
		}()
	}
}

func (c *Checker) executeCheck(m *model.ServiceMonitor, timestamp int64, limits monitoring.ServiceMonitorLimits) *model.ServiceMonitorResult {
	result := &model.ServiceMonitorResult{
		MonitorID: m.ID,
		NodeID:    m.NodeID,
		Timestamp: timestamp,
	}

	timeoutSec := m.TimeoutSec
	if timeoutSec <= 0 {
		timeoutSec = limits.DefaultTimeoutSec
	}
	if timeoutSec < limits.MinTimeoutSec {
		timeoutSec = limits.MinTimeoutSec
	}
	if timeoutSec > limits.MaxTimeoutSec {
		timeoutSec = limits.MaxTimeoutSec
	}

	timeout := time.Duration(timeoutSec) * time.Second

	// When nodeId is set, run checks on the specified node.
	if m.NodeID > 0 {
		c.checkOnNode(m, timeoutSec, timeout, result)
		return result
	}

	switch strings.ToLower(strings.TrimSpace(m.Type)) {
	case "tcp":
		c.checkTCP(m.Target, timeout, result)
	case "icmp":
		result.Success = 0
		result.ErrorMessage = "ICMP 监控必须指定执行节点"
	default:
		result.Success = 0
		result.ErrorMessage = fmt.Sprintf("不支持的检查类型: %s", m.Type)
	}

	return result
}

func (c *Checker) loadServiceMonitorLimits() monitoring.ServiceMonitorLimits {
	defaults := monitoring.DefaultServiceMonitorLimits()
	if c == nil || c.repo == nil {
		return defaults
	}
	cfg, err := c.repo.GetConfigsByNames([]string{
		monitoring.ConfigServiceMonitorCheckerScanIntervalSec,
		monitoring.ConfigServiceMonitorWorkerLimit,
		monitoring.ConfigServiceMonitorMinIntervalSec,
		monitoring.ConfigServiceMonitorDefaultIntervalSec,
		monitoring.ConfigServiceMonitorMinTimeoutSec,
		monitoring.ConfigServiceMonitorDefaultTimeoutSec,
		monitoring.ConfigServiceMonitorMaxTimeoutSec,
	})
	if err != nil {
		return defaults
	}
	return monitoring.ServiceMonitorLimitsFromConfigMap(cfg)
}

type serviceMonitorCheckRequest struct {
	MonitorID  int64  `json:"monitorId"`
	Type       string `json:"type"`
	Target     string `json:"target"`
	TimeoutSec int    `json:"timeoutSec"`
}

func (c *Checker) checkOnNode(m *model.ServiceMonitor, timeoutSec int, timeout time.Duration, result *model.ServiceMonitorResult) {
	if c == nil || m == nil || result == nil {
		return
	}
	if c.commander == nil {
		result.Success = 0
		result.ErrorMessage = "节点检查不可用"
		return
	}

	checkType := strings.ToLower(strings.TrimSpace(m.Type))
	if checkType != "tcp" && checkType != "icmp" {
		result.Success = 0
		result.ErrorMessage = fmt.Sprintf("不支持的检查类型: %s", m.Type)
		return
	}
	if strings.TrimSpace(m.Target) == "" {
		result.Success = 0
		result.ErrorMessage = "检查目标为空"
		return
	}

	req := serviceMonitorCheckRequest{
		MonitorID:  m.ID,
		Type:       checkType,
		Target:     m.Target,
		TimeoutSec: timeoutSec,
	}

	cmdTimeout := timeout
	if cmdTimeout < 2*time.Second {
		cmdTimeout = 2 * time.Second
	}
	cmdTimeout = cmdTimeout + 2*time.Second

	cmdRes, err := c.commander.SendCommand(m.NodeID, "ServiceMonitorCheck", req, cmdTimeout)
	if err != nil {
		result.Success = 0
		result.ErrorMessage = err.Error()
		return
	}
	if cmdRes.Data == nil {
		result.Success = 0
		result.ErrorMessage = "节点返回为空"
		return
	}

	if v, ok := cmdRes.Data["success"]; ok {
		if b, ok := v.(bool); ok {
			if b {
				result.Success = 1
			} else {
				result.Success = 0
			}
		}
	}
	if v, ok := cmdRes.Data["latencyMs"]; ok {
		if f, ok := v.(float64); ok {
			result.LatencyMs = f
		}
	}
	if v, ok := cmdRes.Data["statusCode"]; ok {
		if f, ok := v.(float64); ok {
			result.StatusCode = int(f)
		}
	}
	if v, ok := cmdRes.Data["errorMessage"]; ok {
		if s, ok := v.(string); ok {
			result.ErrorMessage = s
		}
	}
}

func (c *Checker) checkTCP(target string, timeout time.Duration, result *model.ServiceMonitorResult) {
	start := time.Now()

	conn, err := net.DialTimeout("tcp", target, timeout)
	latency := time.Since(start)

	result.LatencyMs = float64(latency.Milliseconds())

	if err != nil {
		result.Success = 0
		result.ErrorMessage = err.Error()
		return
	}
	_ = conn.Close()
	result.Success = 1
}