[功能] 添加节点健康事件清理功能，优化节点观测数据管理

2026-10-08 16:46:37 +08:00 · 2026-03-18 23:11:48 +08:00
parent 4945b8b44f
commit 782304012c
16 changed files with 3473 additions and 433 deletions
@@ -105,13 +105,27 @@ func (s *Service) sync(ctx context.Context, startup bool, target *protocol.Activ
 		}
 		snapshot.CurrentVersion = target.Version
 		snapshot.CurrentChecksum = target.Checksum
+		clearBlockedTarget(snapshot)
 		snapshot.LastError = ""
 		slog.Debug("sync finished without changes", "mode", mode, "version", target.Version)
 		return s.stateStore.Save(snapshot)
 	}
+	if isBlockedTarget(snapshot, target.Version, target.Checksum) {
+		slog.Warn("skipping blocked config version after previous failed apply", "mode", mode, "version", target.Version, "checksum", target.Checksum)
+		if startup {
+			if err = s.ensureRuntimeForCurrentConfig(ctx, mode, snapshot, currentChecksum); err != nil {
+				return err
+			}
+			return s.stateStore.Save(snapshot)
+		}
+		return nil
+	}
+	if hasBlockedTarget(snapshot) {
+		clearBlockedTarget(snapshot)
+	}
 	if snapshot.CurrentVersion == target.Version && snapshot.CurrentChecksum == target.Checksum && !startup {
 		slog.Debug("skipping config fetch because state already records target version/checksum", "version", target.Version, "checksum", target.Checksum)
-		return nil
+		return s.stateStore.Save(snapshot)
 	}

 	config, err := s.client.GetActiveConfig(ctx)
@@ -139,6 +153,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
 		}
 		snapshot.CurrentVersion = config.Version
 		snapshot.CurrentChecksum = config.Checksum
+		clearBlockedTarget(snapshot)
 		snapshot.LastError = ""
 		slog.Debug("sync finished without changes", "mode", mode, "version", config.Version)
 		return s.stateStore.Save(snapshot)
@@ -146,9 +161,22 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
 	if target != nil && (target.Version != config.Version || target.Checksum != config.Checksum) {
 		slog.Warn("active config changed between heartbeat and fetch", "heartbeat_version", target.Version, "heartbeat_checksum", target.Checksum, "fetched_version", config.Version, "fetched_checksum", config.Checksum)
 	}
+	if isBlockedTarget(snapshot, config.Version, config.Checksum) {
+		slog.Warn("skipping blocked config after fetch because the same version previously failed", "mode", mode, "version", config.Version, "checksum", config.Checksum)
+		if startup {
+			if err := s.ensureRuntimeForCurrentConfig(ctx, mode, snapshot, currentChecksum); err != nil {
+				return err
+			}
+			return s.stateStore.Save(snapshot)
+		}
+		return nil
+	}
+	if hasBlockedTarget(snapshot) {
+		clearBlockedTarget(snapshot)
+	}
 	if snapshot.CurrentVersion == config.Version && snapshot.CurrentChecksum == config.Checksum && !startup {
 		slog.Debug("skipping apply because state already records target version/checksum", "version", config.Version, "checksum", config.Checksum)
-		return nil
+		return s.stateStore.Save(snapshot)
 	}
 	routeConfig := config.RouteConfig
 	if routeConfig == "" {
@@ -172,6 +200,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
 		slog.Info("openresty config applied successfully", "mode", mode, "version", config.Version)
 		snapshot.CurrentVersion = config.Version
 		snapshot.CurrentChecksum = config.Checksum
+		clearBlockedTarget(snapshot)
 		snapshot.LastError = ""
 		snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
 		snapshot.OpenrestyMessage = ""
@@ -184,6 +213,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
 			message = "apply rolled back to previous config"
 		}
 		slog.Warn("openresty config apply rolled back", "mode", mode, "version", config.Version, "message", message)
+		markBlockedTarget(snapshot, config.Version, config.Checksum, message)
 		snapshot.LastError = message
 		snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
 		snapshot.OpenrestyMessage = message
@@ -193,6 +223,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
 			message = "openresty apply failed"
 		}
 		slog.Error("apply openresty config failed", "mode", mode, "version", config.Version, "message", message)
+		markBlockedTarget(snapshot, config.Version, config.Checksum, message)
 		snapshot.LastError = message
 		snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
 		snapshot.OpenrestyMessage = message
@@ -230,6 +261,56 @@ func outcomeError(version string, message string) error {
 	return fmt.Errorf("apply version %s failed: %s", version, trimmed)
 }

+func (s *Service) ensureRuntimeForCurrentConfig(ctx context.Context, mode string, snapshot *state.Snapshot, currentChecksum string) error {
+	if strings.TrimSpace(currentChecksum) == "" {
+		slog.Warn("blocked config cannot be retried and no local checksum is available for runtime recovery", "mode", mode, "blocked_version", snapshot.BlockedVersion)
+		return nil
+	}
+	slog.Info("ensuring runtime with current local config while active target remains blocked", "mode", mode, "current_version", snapshot.CurrentVersion, "current_checksum", currentChecksum, "blocked_version", snapshot.BlockedVersion)
+	if err := s.nginxManager.EnsureRuntime(ctx, true); err != nil {
+		snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
+		snapshot.OpenrestyMessage = err.Error()
+		_ = s.stateStore.Save(snapshot)
+		return err
+	}
+	snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
+	if strings.TrimSpace(snapshot.OpenrestyMessage) == strings.TrimSpace(snapshot.BlockedReason) {
+		snapshot.OpenrestyMessage = ""
+	}
+	return nil
+}
+
+func markBlockedTarget(snapshot *state.Snapshot, version string, checksum string, reason string) {
+	if snapshot == nil {
+		return
+	}
+	snapshot.BlockedVersion = strings.TrimSpace(version)
+	snapshot.BlockedChecksum = strings.TrimSpace(checksum)
+	snapshot.BlockedReason = strings.TrimSpace(reason)
+}
+
+func clearBlockedTarget(snapshot *state.Snapshot) {
+	if snapshot == nil {
+		return
+	}
+	snapshot.BlockedVersion = ""
+	snapshot.BlockedChecksum = ""
+	snapshot.BlockedReason = ""
+}
+
+func hasBlockedTarget(snapshot *state.Snapshot) bool {
+	return snapshot != nil && (strings.TrimSpace(snapshot.BlockedVersion) != "" || strings.TrimSpace(snapshot.BlockedChecksum) != "")
+}
+
+func isBlockedTarget(snapshot *state.Snapshot, version string, checksum string) bool {
+	if snapshot == nil {
+		return false
+	}
+	return strings.TrimSpace(snapshot.BlockedVersion) == strings.TrimSpace(version) &&
+		strings.TrimSpace(snapshot.BlockedChecksum) == strings.TrimSpace(checksum) &&
+		(strings.TrimSpace(version) != "" || strings.TrimSpace(checksum) != "")
+}
+
 func checksumString(content string) string {
 	sum := sha256.Sum256([]byte(content))
 	return hex.EncodeToString(sum[:])