mirror of
https://github.com/Sagit-chu/flvx.git
synced 2026-09-28 07:36:38 +08:00
sagit
58d2e89147
* fix: reduce reconnect redeploy and metrics load Throttle node-online redeploy retries and lower the agent metric cadence so brief reconnect churn no longer fans out into repeated runtime syncs and backend connection pressure. * docs: add follow-up implementation design notes Document the planned flow upload batching work and the local remote-address toggle so the next changesets can implement them against an agreed design.
622 lines
17 KiB
Go
622 lines
17 KiB
Go
package handler
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"go-backend/internal/http/response"
|
|
)
|
|
|
|
// failedForward tracks a forward that failed redeployment, for retry.
|
|
type failedForward struct {
|
|
id int64
|
|
forward *forwardRecord
|
|
err error
|
|
}
|
|
|
|
const (
|
|
githubRepo = "Sagit-chu/flvx"
|
|
githubAPIBase = "https://api.github.com"
|
|
githubHTMLBase = "https://github.com"
|
|
upgradeTimeout = 5 * time.Minute
|
|
batchWorkers = 5
|
|
|
|
releaseChannelStable = "stable"
|
|
releaseChannelDev = "dev"
|
|
|
|
defaultGithubProxyEnabled = true
|
|
defaultGithubProxyURL = "https://gcode.hostcentral.cc"
|
|
)
|
|
|
|
var (
|
|
stableVersionPattern = regexp.MustCompile(`^\d+(?:\.\d+)+$`)
|
|
testKeywordPattern = regexp.MustCompile(`(?i)(alpha|beta|rc)`)
|
|
)
|
|
|
|
const nodeOnlineRedeployCooldown = 30 * time.Second
|
|
|
|
type githubRelease struct {
|
|
TagName string `json:"tag_name"`
|
|
Name string `json:"name"`
|
|
PublishedAt string `json:"published_at"`
|
|
Prerelease bool `json:"prerelease"`
|
|
Draft bool `json:"draft"`
|
|
}
|
|
|
|
func normalizeReleaseChannel(channel string) string {
|
|
switch strings.ToLower(strings.TrimSpace(channel)) {
|
|
case releaseChannelDev:
|
|
return releaseChannelDev
|
|
default:
|
|
return releaseChannelStable
|
|
}
|
|
}
|
|
|
|
func releaseChannelFromTag(tag string) string {
|
|
normalized := strings.ToLower(strings.TrimSpace(tag))
|
|
if normalized == "" {
|
|
return releaseChannelDev
|
|
}
|
|
if testKeywordPattern.MatchString(normalized) {
|
|
return releaseChannelDev
|
|
}
|
|
if stableVersionPattern.MatchString(normalized) {
|
|
return releaseChannelStable
|
|
}
|
|
|
|
return releaseChannelDev
|
|
}
|
|
|
|
func releaseChannelLabel(channel string) string {
|
|
if normalizeReleaseChannel(channel) == releaseChannelDev {
|
|
return "测试版"
|
|
}
|
|
|
|
return "正式版"
|
|
}
|
|
|
|
func (h *Handler) getGithubProxyConfig() (enabled bool, proxyURL string) {
|
|
enabled = defaultGithubProxyEnabled
|
|
proxyURL = defaultGithubProxyURL
|
|
|
|
if h == nil || h.repo == nil {
|
|
return
|
|
}
|
|
|
|
if enabledCfg, err := h.repo.GetConfigByName("github_proxy_enabled"); err == nil && enabledCfg != nil {
|
|
enabled = enabledCfg.Value != "false"
|
|
}
|
|
|
|
if urlCfg, err := h.repo.GetConfigByName("github_proxy_url"); err == nil && urlCfg != nil && urlCfg.Value != "" {
|
|
proxyURL = strings.TrimSpace(urlCfg.Value)
|
|
if !strings.HasPrefix(proxyURL, "http://") && !strings.HasPrefix(proxyURL, "https://") {
|
|
proxyURL = "https://" + proxyURL
|
|
}
|
|
proxyURL = strings.TrimSuffix(proxyURL, "/")
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (h *Handler) buildGithubDownloadURL(version, filename string) string {
|
|
enabled, proxyURL := h.getGithubProxyConfig()
|
|
base := fmt.Sprintf("%s/%s/releases/download/%s/%s", githubHTMLBase, githubRepo, version, filename)
|
|
|
|
if enabled {
|
|
return fmt.Sprintf("%s/%s", proxyURL, base)
|
|
}
|
|
return base
|
|
}
|
|
|
|
func fetchGitHubReleases(perPage int) ([]githubRelease, error) {
|
|
if perPage <= 0 {
|
|
perPage = 20
|
|
}
|
|
|
|
client := &http.Client{Timeout: 15 * time.Second}
|
|
resp, err := client.Get(fmt.Sprintf("%s/repos/%s/releases?per_page=%d", githubAPIBase, githubRepo, perPage))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("请求GitHub API失败: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
return nil, fmt.Errorf("GitHub API返回 %d: %s", resp.StatusCode, string(body))
|
|
}
|
|
|
|
var releases []githubRelease
|
|
if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
|
|
return nil, fmt.Errorf("解析GitHub API响应失败: %v", err)
|
|
}
|
|
|
|
return releases, nil
|
|
}
|
|
|
|
func resolveLatestReleaseByChannel(channel string) (string, error) {
|
|
normalizedChannel := normalizeReleaseChannel(channel)
|
|
releases, err := fetchGitHubReleases(50)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
for _, r := range releases {
|
|
if r.Draft {
|
|
continue
|
|
}
|
|
tag := strings.TrimSpace(r.TagName)
|
|
if tag == "" {
|
|
continue
|
|
}
|
|
if releaseChannelFromTag(tag) == normalizedChannel {
|
|
return tag, nil
|
|
}
|
|
}
|
|
|
|
return "", fmt.Errorf("未找到%s版本号", releaseChannelLabel(normalizedChannel))
|
|
}
|
|
|
|
func (h *Handler) nodeUpgrade(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
response.WriteJSON(w, response.ErrDefault("请求失败"))
|
|
return
|
|
}
|
|
|
|
var req struct {
|
|
ID int64 `json:"id"`
|
|
Version string `json:"version"`
|
|
Channel string `json:"channel"`
|
|
}
|
|
if err := decodeJSON(r.Body, &req); err != nil {
|
|
response.WriteJSON(w, response.ErrDefault("请求参数错误"))
|
|
return
|
|
}
|
|
if req.ID <= 0 {
|
|
response.WriteJSON(w, response.ErrDefault("节点ID无效"))
|
|
return
|
|
}
|
|
|
|
channel := normalizeReleaseChannel(req.Channel)
|
|
version := strings.TrimSpace(req.Version)
|
|
if version == "" {
|
|
var err error
|
|
version, err = resolveLatestReleaseByChannel(channel)
|
|
if err != nil {
|
|
response.WriteJSON(w, response.Err(-2, fmt.Sprintf("获取最新%s失败: %v", releaseChannelLabel(channel), err)))
|
|
return
|
|
}
|
|
}
|
|
|
|
downloadURL := h.buildGithubDownloadURL(version, "gost-{ARCH}")
|
|
checksumURL := h.buildGithubDownloadURL(version, "gost-{ARCH}.sha256")
|
|
|
|
result, err := h.wsServer.SendCommand(req.ID, "UpgradeAgent", map[string]interface{}{
|
|
"downloadUrl": downloadURL,
|
|
"checksumUrl": checksumURL,
|
|
}, upgradeTimeout)
|
|
if err != nil {
|
|
response.WriteJSON(w, response.Err(-2, fmt.Sprintf("升级失败: %v", err)))
|
|
return
|
|
}
|
|
h.markNodePendingUpgradeRedeploy(req.ID)
|
|
|
|
response.WriteJSON(w, response.OK(map[string]interface{}{
|
|
"version": version,
|
|
"message": result.Message,
|
|
}))
|
|
}
|
|
|
|
func resolveLatestRelease() (string, error) {
|
|
return resolveLatestReleaseByChannel(releaseChannelStable)
|
|
}
|
|
|
|
func resolveLatestReleaseAPI() (string, error) {
|
|
return resolveLatestReleaseByChannel(releaseChannelStable)
|
|
}
|
|
|
|
func (h *Handler) nodeBatchUpgrade(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
response.WriteJSON(w, response.ErrDefault("请求失败"))
|
|
return
|
|
}
|
|
|
|
var req struct {
|
|
IDs []int64 `json:"ids"`
|
|
Version string `json:"version"`
|
|
Channel string `json:"channel"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
response.WriteJSON(w, response.ErrDefault("请求参数错误"))
|
|
return
|
|
}
|
|
if len(req.IDs) == 0 {
|
|
response.WriteJSON(w, response.ErrDefault("ids不能为空"))
|
|
return
|
|
}
|
|
|
|
channel := normalizeReleaseChannel(req.Channel)
|
|
version := strings.TrimSpace(req.Version)
|
|
if version == "" {
|
|
var err error
|
|
version, err = resolveLatestReleaseByChannel(channel)
|
|
if err != nil {
|
|
response.WriteJSON(w, response.Err(-2, fmt.Sprintf("获取最新%s失败: %v", releaseChannelLabel(channel), err)))
|
|
return
|
|
}
|
|
}
|
|
|
|
downloadURL := h.buildGithubDownloadURL(version, "gost-{ARCH}")
|
|
checksumURL := h.buildGithubDownloadURL(version, "gost-{ARCH}.sha256")
|
|
|
|
type upgradeResult struct {
|
|
ID int64 `json:"id"`
|
|
Success bool `json:"success"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
results := make([]upgradeResult, len(req.IDs))
|
|
sem := make(chan struct{}, batchWorkers)
|
|
var wg sync.WaitGroup
|
|
|
|
for i, id := range req.IDs {
|
|
wg.Add(1)
|
|
go func(index int, nodeID int64) {
|
|
defer wg.Done()
|
|
sem <- struct{}{}
|
|
defer func() { <-sem }()
|
|
|
|
result, err := h.wsServer.SendCommand(nodeID, "UpgradeAgent", map[string]interface{}{
|
|
"downloadUrl": downloadURL,
|
|
"checksumUrl": checksumURL,
|
|
}, upgradeTimeout)
|
|
if err != nil {
|
|
results[index] = upgradeResult{ID: nodeID, Success: false, Message: err.Error()}
|
|
return
|
|
}
|
|
h.markNodePendingUpgradeRedeploy(nodeID)
|
|
results[index] = upgradeResult{ID: nodeID, Success: true, Message: result.Message}
|
|
}(i, id)
|
|
}
|
|
wg.Wait()
|
|
|
|
response.WriteJSON(w, response.OK(map[string]interface{}{
|
|
"version": version,
|
|
"results": results,
|
|
}))
|
|
}
|
|
|
|
func (h *Handler) listReleases(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
response.WriteJSON(w, response.ErrDefault("请求失败"))
|
|
return
|
|
}
|
|
|
|
var req struct {
|
|
Channel string `json:"channel"`
|
|
}
|
|
if err := decodeJSON(r.Body, &req); err != nil && err != io.EOF {
|
|
response.WriteJSON(w, response.ErrDefault("请求参数错误"))
|
|
return
|
|
}
|
|
|
|
channel := normalizeReleaseChannel(req.Channel)
|
|
|
|
releases, err := fetchGitHubReleases(50)
|
|
if err != nil {
|
|
response.WriteJSON(w, response.Err(-2, fmt.Sprintf("获取版本列表失败: %v", err)))
|
|
return
|
|
}
|
|
|
|
type releaseItem struct {
|
|
Version string `json:"version"`
|
|
Name string `json:"name"`
|
|
PublishedAt string `json:"publishedAt"`
|
|
Prerelease bool `json:"prerelease"`
|
|
Channel string `json:"channel"`
|
|
}
|
|
|
|
items := make([]releaseItem, 0, len(releases))
|
|
for _, r := range releases {
|
|
if r.Draft {
|
|
continue
|
|
}
|
|
tag := strings.TrimSpace(r.TagName)
|
|
if tag == "" {
|
|
continue
|
|
}
|
|
itemChannel := releaseChannelFromTag(tag)
|
|
if itemChannel != channel {
|
|
continue
|
|
}
|
|
items = append(items, releaseItem{
|
|
Version: tag,
|
|
Name: r.Name,
|
|
PublishedAt: r.PublishedAt,
|
|
Prerelease: itemChannel == releaseChannelDev,
|
|
Channel: itemChannel,
|
|
})
|
|
}
|
|
|
|
response.WriteJSON(w, response.OK(items))
|
|
}
|
|
|
|
func (h *Handler) nodeRollback(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
response.WriteJSON(w, response.ErrDefault("请求失败"))
|
|
return
|
|
}
|
|
|
|
var req struct {
|
|
ID int64 `json:"id"`
|
|
}
|
|
if err := decodeJSON(r.Body, &req); err != nil {
|
|
response.WriteJSON(w, response.ErrDefault("请求参数错误"))
|
|
return
|
|
}
|
|
if req.ID <= 0 {
|
|
response.WriteJSON(w, response.ErrDefault("节点ID无效"))
|
|
return
|
|
}
|
|
|
|
result, err := h.wsServer.SendCommand(req.ID, "RollbackAgent", map[string]interface{}{}, 30*time.Second)
|
|
if err != nil {
|
|
response.WriteJSON(w, response.Err(-2, fmt.Sprintf("回退失败: %v", err)))
|
|
return
|
|
}
|
|
|
|
response.WriteJSON(w, response.OK(map[string]interface{}{
|
|
"message": result.Message,
|
|
}))
|
|
}
|
|
|
|
func (h *Handler) markNodePendingUpgradeRedeploy(nodeID int64) {
|
|
if h == nil || nodeID <= 0 {
|
|
return
|
|
}
|
|
h.upgradeMu.Lock()
|
|
h.pendingUpgradeRedeploy[nodeID] = struct{}{}
|
|
h.upgradeMu.Unlock()
|
|
}
|
|
|
|
func (h *Handler) consumeNodePendingUpgradeRedeploy(nodeID int64) bool {
|
|
if h == nil || nodeID <= 0 {
|
|
return false
|
|
}
|
|
h.upgradeMu.Lock()
|
|
_, ok := h.pendingUpgradeRedeploy[nodeID]
|
|
if ok {
|
|
delete(h.pendingUpgradeRedeploy, nodeID)
|
|
}
|
|
h.upgradeMu.Unlock()
|
|
return ok
|
|
}
|
|
|
|
func (h *Handler) onNodeOnline(nodeID int64) {
|
|
if !h.startNodeOnlineRedeploy(nodeID, time.Now()) {
|
|
return
|
|
}
|
|
defer h.finishNodeOnlineRedeploy(nodeID)
|
|
|
|
// Reconcile node runtime on the first reconnect, but suppress rapid flapping
|
|
// so websocket churn does not trigger repeated full redeploy storms.
|
|
if !h.redeployNodeRuntimeAfterUpgrade(nodeID) {
|
|
h.markNodePendingUpgradeRedeploy(nodeID)
|
|
}
|
|
}
|
|
|
|
func (h *Handler) startNodeOnlineRedeploy(nodeID int64, now time.Time) bool {
|
|
if h == nil || nodeID <= 0 {
|
|
return false
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now()
|
|
}
|
|
|
|
h.upgradeMu.Lock()
|
|
defer h.upgradeMu.Unlock()
|
|
if h.pendingUpgradeRedeploy == nil {
|
|
h.pendingUpgradeRedeploy = make(map[int64]struct{})
|
|
}
|
|
if h.nodeOnlineRedeployAt == nil {
|
|
h.nodeOnlineRedeployAt = make(map[int64]time.Time)
|
|
}
|
|
if h.nodeOnlineRedeployQueued == nil {
|
|
h.nodeOnlineRedeployQueued = make(map[int64]struct{})
|
|
}
|
|
if h.nodeOnlineRedeploying == nil {
|
|
h.nodeOnlineRedeploying = make(map[int64]struct{})
|
|
}
|
|
|
|
_, pendingUpgrade := h.pendingUpgradeRedeploy[nodeID]
|
|
lastRedeployAt := h.nodeOnlineRedeployAt[nodeID]
|
|
_, inFlight := h.nodeOnlineRedeploying[nodeID]
|
|
if fireAt, start := nextNodeOnlineRedeployFireAt(lastRedeployAt, now, pendingUpgrade, inFlight); !start {
|
|
h.queueNodeOnlineRedeployLocked(nodeID, fireAt)
|
|
return false
|
|
}
|
|
|
|
delete(h.pendingUpgradeRedeploy, nodeID)
|
|
h.nodeOnlineRedeployAt[nodeID] = now
|
|
h.nodeOnlineRedeploying[nodeID] = struct{}{}
|
|
return true
|
|
}
|
|
|
|
func nextNodeOnlineRedeployFireAt(lastRedeployAt, now time.Time, pendingUpgrade bool, inFlight bool) (time.Time, bool) {
|
|
if now.IsZero() {
|
|
now = time.Now()
|
|
}
|
|
if inFlight {
|
|
fireAt := now.Add(nodeOnlineRedeployCooldown)
|
|
if !lastRedeployAt.IsZero() {
|
|
cooldownAt := lastRedeployAt.Add(nodeOnlineRedeployCooldown)
|
|
if cooldownAt.After(now) {
|
|
fireAt = cooldownAt
|
|
}
|
|
}
|
|
return fireAt, false
|
|
}
|
|
if !pendingUpgrade && !lastRedeployAt.IsZero() && now.Sub(lastRedeployAt) < nodeOnlineRedeployCooldown {
|
|
return lastRedeployAt.Add(nodeOnlineRedeployCooldown), false
|
|
}
|
|
return time.Time{}, true
|
|
}
|
|
|
|
func (h *Handler) queueNodeOnlineRedeployLocked(nodeID int64, fireAt time.Time) {
|
|
if h == nil || nodeID <= 0 {
|
|
return
|
|
}
|
|
if h.nodeOnlineRedeployQueued == nil {
|
|
h.nodeOnlineRedeployQueued = make(map[int64]struct{})
|
|
}
|
|
if _, queued := h.nodeOnlineRedeployQueued[nodeID]; queued {
|
|
return
|
|
}
|
|
if fireAt.IsZero() {
|
|
fireAt = time.Now().Add(nodeOnlineRedeployCooldown)
|
|
}
|
|
delay := time.Until(fireAt)
|
|
if delay < 0 {
|
|
delay = 0
|
|
}
|
|
h.nodeOnlineRedeployQueued[nodeID] = struct{}{}
|
|
time.AfterFunc(delay, func() {
|
|
h.upgradeMu.Lock()
|
|
delete(h.nodeOnlineRedeployQueued, nodeID)
|
|
h.upgradeMu.Unlock()
|
|
h.onNodeOnline(nodeID)
|
|
})
|
|
}
|
|
|
|
func (h *Handler) finishNodeOnlineRedeploy(nodeID int64) {
|
|
if h == nil || nodeID <= 0 {
|
|
return
|
|
}
|
|
h.upgradeMu.Lock()
|
|
delete(h.nodeOnlineRedeploying, nodeID)
|
|
h.upgradeMu.Unlock()
|
|
}
|
|
|
|
func (h *Handler) redeployNodeRuntimeAfterUpgrade(nodeID int64) bool {
|
|
tunnelIDs, err := h.repo.ListActiveTunnelIDsByNode(nodeID)
|
|
if err != nil {
|
|
fmt.Printf("post-upgrade redeploy: list tunnels for node %d failed: %v\n", nodeID, err)
|
|
return false
|
|
}
|
|
forwardIDs, err := h.repo.ListForwardIDsByNode(nodeID)
|
|
if err != nil {
|
|
fmt.Printf("post-upgrade redeploy: list forwards for node %d failed: %v\n", nodeID, err)
|
|
return false
|
|
}
|
|
|
|
// First pass: deploy everything
|
|
tunnelFailed := make(map[int64]struct{})
|
|
for _, tunnelID := range tunnelIDs {
|
|
if err := h.redeployTunnelAndForwards(tunnelID); err != nil {
|
|
tunnelFailed[tunnelID] = struct{}{}
|
|
fmt.Printf("post-upgrade redeploy: tunnel %d failed on node %d: %v\n", tunnelID, nodeID, err)
|
|
}
|
|
}
|
|
|
|
// Collect forwards that failed independently (not skipped due to tunnel failure)
|
|
var failedForwards []failedForward
|
|
|
|
for _, forwardID := range forwardIDs {
|
|
forward, getErr := h.getForwardRecord(forwardID)
|
|
if getErr != nil || forward == nil {
|
|
continue
|
|
}
|
|
if _, skipped := tunnelFailed[forward.TunnelID]; skipped {
|
|
continue
|
|
}
|
|
if err := h.syncForwardServices(forward, "UpdateService", true); err != nil {
|
|
failedForwards = append(failedForwards, failedForward{id: forwardID, forward: forward, err: err})
|
|
fmt.Printf("post-upgrade redeploy: forward %d failed on node %d: %v\n", forwardID, nodeID, err)
|
|
}
|
|
}
|
|
|
|
// Retry failed items with exponential backoff (max 3 attempts)
|
|
return h.retryFailedRedeploys(nodeID, tunnelFailed, failedForwards)
|
|
}
|
|
|
|
// isRetryableError returns true if the error looks transient and worth retrying.
|
|
func isRetryableError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
msg := strings.ToLower(err.Error())
|
|
// Skip non-retryable errors: not-found, already-exists, validation errors
|
|
if strings.Contains(msg, "not found") || strings.Contains(msg, "不存在") {
|
|
return false
|
|
}
|
|
if strings.Contains(msg, "already exists") || strings.Contains(msg, "已存在") {
|
|
return false
|
|
}
|
|
// Everything else (timeout, connection lost, port in use, etc.) is retryable
|
|
return true
|
|
}
|
|
|
|
// retryFailedRedeploys retries failed tunnels and forwards with exponential backoff.
|
|
func (h *Handler) retryFailedRedeploys(nodeID int64, tunnelFailed map[int64]struct{}, failedForwards []failedForward) bool {
|
|
if len(tunnelFailed) == 0 && len(failedForwards) == 0 {
|
|
return true
|
|
}
|
|
|
|
const maxRetries = 3
|
|
baseDelay := time.Second
|
|
|
|
for attempt := 1; attempt <= maxRetries; attempt++ {
|
|
delay := baseDelay * time.Duration(1<<uint(attempt-1)) // 1s, 2s, 4s
|
|
time.Sleep(delay)
|
|
|
|
// Retry failed tunnels
|
|
for tunnelID := range tunnelFailed {
|
|
if err := h.redeployTunnelAndForwards(tunnelID); err == nil {
|
|
delete(tunnelFailed, tunnelID)
|
|
fmt.Printf("post-upgrade redeploy retry: tunnel %d succeeded on node %d (attempt %d)\n", tunnelID, nodeID, attempt)
|
|
} else if !isRetryableError(err) {
|
|
delete(tunnelFailed, tunnelID) // Non-retryable, don't retry again
|
|
} else {
|
|
fmt.Printf("post-upgrade redeploy retry: tunnel %d still failing on node %d (attempt %d): %v\n", tunnelID, nodeID, attempt, err)
|
|
}
|
|
}
|
|
|
|
// Retry failed forwards
|
|
var stillFailed []failedForward
|
|
for _, ff := range failedForwards {
|
|
if _, skipped := tunnelFailed[ff.forward.TunnelID]; skipped {
|
|
stillFailed = append(stillFailed, ff) // Tunnel still failed, skip forward
|
|
continue
|
|
}
|
|
if err := h.syncForwardServices(ff.forward, "UpdateService", true); err == nil {
|
|
fmt.Printf("post-upgrade redeploy retry: forward %d succeeded on node %d (attempt %d)\n", ff.id, nodeID, attempt)
|
|
} else if !isRetryableError(err) {
|
|
// Non-retryable, drop it
|
|
} else {
|
|
stillFailed = append(stillFailed, ff)
|
|
fmt.Printf("post-upgrade redeploy retry: forward %d still failing on node %d (attempt %d): %v\n", ff.id, nodeID, attempt, err)
|
|
}
|
|
}
|
|
failedForwards = stillFailed
|
|
|
|
if len(tunnelFailed) == 0 && len(failedForwards) == 0 {
|
|
fmt.Printf("post-upgrade redeploy retry: all items recovered on node %d\n", nodeID)
|
|
return true
|
|
}
|
|
}
|
|
|
|
// Final summary
|
|
for tunnelID := range tunnelFailed {
|
|
fmt.Printf("post-upgrade redeploy: tunnel %d permanently failed on node %d after retries\n", tunnelID, nodeID)
|
|
}
|
|
for _, ff := range failedForwards {
|
|
fmt.Printf("post-upgrade redeploy: forward %d permanently failed on node %d after retries\n", ff.id, nodeID)
|
|
}
|
|
return false
|
|
}
|