diff --git a/go-backend/internal/http/handler/dual_stack_test.go b/go-backend/internal/http/handler/dual_stack_test.go index d25e825..b7cb3e4 100644 --- a/go-backend/internal/http/handler/dual_stack_test.go +++ b/go-backend/internal/http/handler/dual_stack_test.go @@ -33,7 +33,7 @@ func TestSelectTunnelDialHost_ConnectIpPriority(t *testing.T) { func TestBuildTunnelChainServiceConfig_UsesConnectIPForListen(t *testing.T) { node := &nodeRecord{TCPListenAddr: "[::]"} chain := tunnelRuntimeNode{Protocol: "tls", Port: 21000, ConnectIP: "2001:db8::88"} - services := buildTunnelChainServiceConfig(99, chain, node) + services := buildTunnelChainServiceConfig(99, chain, node, 1) if len(services) != 1 { t.Fatalf("expected 1 service, got %d", len(services)) } @@ -46,7 +46,7 @@ func TestBuildTunnelChainServiceConfig_UsesConnectIPForListen(t *testing.T) { func TestBuildTunnelChainServiceConfig_FallsBackToNodeListenAddr(t *testing.T) { node := &nodeRecord{TCPListenAddr: "10.8.0.5"} chain := tunnelRuntimeNode{Protocol: "tls", Port: 21002} - services := buildTunnelChainServiceConfig(99, chain, node) + services := buildTunnelChainServiceConfig(99, chain, node, 1) if len(services) != 1 { t.Fatalf("expected 1 service, got %d", len(services)) } @@ -59,7 +59,7 @@ func TestBuildTunnelChainServiceConfig_FallsBackToNodeListenAddr(t *testing.T) { func TestBuildTunnelChainServiceConfig_DefaultListenAddrWhenConnectIPEmpty(t *testing.T) { node := &nodeRecord{TCPListenAddr: "[::]"} chain := tunnelRuntimeNode{Protocol: "tls", Port: 21001} - services := buildTunnelChainServiceConfig(99, chain, node) + services := buildTunnelChainServiceConfig(99, chain, node, 1) if len(services) != 1 { t.Fatalf("expected 1 service, got %d", len(services)) } @@ -69,6 +69,42 @@ func TestBuildTunnelChainServiceConfig_DefaultListenAddrWhenConnectIPEmpty(t *te } } +func TestBuildTunnelChainServiceConfig_SetsRetriesWhenMultipleCandidates(t *testing.T) { + node := &nodeRecord{TCPListenAddr: "[::]"} + chain := tunnelRuntimeNode{Protocol: "tls", Port: 21001} + services := buildTunnelChainServiceConfig(99, chain, node, 3) + if len(services) != 1 { + t.Fatalf("expected 1 service, got %d", len(services)) + } + handler, _ := services[0]["handler"].(map[string]interface{}) + if handler == nil { + t.Fatal("expected handler config") + } + retries, ok := handler["retries"].(int) + if !ok { + t.Fatal("expected retries to be set when nextHopCandidateCount > 1") + } + if retries != 2 { + t.Fatalf("expected retries=2 (candidates-1), got %d", retries) + } +} + +func TestBuildTunnelChainServiceConfig_NoRetriesWhenSingleCandidate(t *testing.T) { + node := &nodeRecord{TCPListenAddr: "[::]"} + chain := tunnelRuntimeNode{Protocol: "tls", Port: 21001} + services := buildTunnelChainServiceConfig(99, chain, node, 1) + if len(services) != 1 { + t.Fatalf("expected 1 service, got %d", len(services)) + } + handler, _ := services[0]["handler"].(map[string]interface{}) + if handler == nil { + t.Fatal("expected handler config") + } + if _, hasRetries := handler["retries"]; hasRetries { + t.Fatal("expected no retries when nextHopCandidateCount is 1") + } +} + func TestNodeSupportsV6_Nil(t *testing.T) { if nodeSupportsV6(nil) { t.Fatal("nil node must not support v6") diff --git a/go-backend/internal/http/handler/federation.go b/go-backend/internal/http/handler/federation.go index 5c40002..62fd3ec 100644 --- a/go-backend/internal/http/handler/federation.go +++ b/go-backend/internal/http/handler/federation.go @@ -141,6 +141,32 @@ type remoteUsageNodeItem struct { SyncError string `json:"syncError,omitempty"` } +func buildFederationServiceConfig(serviceName, addr, protocol, role, chainName string, targetCount int, interfaceName string) map[string]interface{} { + service := map[string]interface{}{ + "name": serviceName, + "addr": addr, + "handler": map[string]interface{}{ + "type": "relay", + }, + "listener": map[string]interface{}{ + "type": protocol, + }, + } + if isTLSTunnelProtocol(protocol) { + service["handler"].(map[string]interface{})["metadata"] = map[string]interface{}{"nodelay": true} + } + if role == "middle" { + service["handler"].(map[string]interface{})["chain"] = chainName + if targetCount > 1 { + service["handler"].(map[string]interface{})["retries"] = targetCount - 1 + } + } + if role == "exit" && strings.TrimSpace(interfaceName) != "" { + service["metadata"] = map[string]interface{}{"interface": interfaceName} + } + return service +} + func (h *Handler) federationShareList(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { response.WriteJSON(w, response.ErrDefault("Invalid method")) @@ -1096,25 +1122,16 @@ func (h *Handler) federationRuntimeApplyRole(w http.ResponseWriter, r *http.Requ } } - service := map[string]interface{}{ - "name": serviceName, - "addr": fmt.Sprintf("%s:%d", node.TCPListenAddr, runtime.Port), - "handler": map[string]interface{}{ - "type": "relay", - }, - "listener": map[string]interface{}{ - "type": protocol, - }, - } - if isTLSTunnelProtocol(protocol) { - service["handler"].(map[string]interface{})["metadata"] = map[string]interface{}{"nodelay": true} - } - if req.Role == "middle" { - service["handler"].(map[string]interface{})["chain"] = chainName - } - if req.Role == "exit" && strings.TrimSpace(node.InterfaceName) != "" { - service["metadata"] = map[string]interface{}{"interface": node.InterfaceName} - } + targetCount := len(req.Targets) + service := buildFederationServiceConfig( + serviceName, + fmt.Sprintf("%s:%d", node.TCPListenAddr, runtime.Port), + protocol, + req.Role, + chainName, + targetCount, + node.InterfaceName, + ) if _, err := h.sendNodeCommand(share.NodeID, "AddService", []map[string]interface{}{service}, true, false); err != nil { if req.Role == "middle" { _, _ = h.sendNodeCommand(share.NodeID, "DeleteChains", map[string]interface{}{"chain": chainName}, false, true) diff --git a/go-backend/internal/http/handler/federation_runtime_test.go b/go-backend/internal/http/handler/federation_runtime_test.go index ab0ba97..f821e7a 100644 --- a/go-backend/internal/http/handler/federation_runtime_test.go +++ b/go-backend/internal/http/handler/federation_runtime_test.go @@ -227,6 +227,60 @@ func TestPrepareTunnelCreateStateAllowsOfflineRemoteMiddleNode(t *testing.T) { } } +func TestBuildFederationServiceConfig_MiddleRoleWithMultipleTargets_SetsRetries(t *testing.T) { + service := buildFederationServiceConfig("svc-middle", ":40000", "tls", "middle", "chain-next", 3, "") + handler := service["handler"].(map[string]interface{}) + if handler["chain"] != "chain-next" { + t.Fatalf("expected chain 'chain-next', got %v", handler["chain"]) + } + if handler["retries"] != 2 { + t.Fatalf("expected retries 2 for 3 targets, got %v", handler["retries"]) + } +} + +func TestBuildFederationServiceConfig_MiddleRoleWithSingleTarget_NoRetries(t *testing.T) { + service := buildFederationServiceConfig("svc-middle", ":40000", "tls", "middle", "chain-next", 1, "") + handler := service["handler"].(map[string]interface{}) + if handler["chain"] != "chain-next" { + t.Fatalf("expected chain 'chain-next', got %v", handler["chain"]) + } + if _, hasRetries := handler["retries"]; hasRetries { + t.Fatalf("expected no retries for single target, got %v", handler["retries"]) + } +} + +func TestBuildFederationServiceConfig_ExitRole_NoRetriesRegardlessOfTargets(t *testing.T) { + service := buildFederationServiceConfig("svc-exit", ":40000", "tls", "exit", "", 3, "eth0") + handler := service["handler"].(map[string]interface{}) + if _, hasChain := handler["chain"]; hasChain { + t.Fatalf("expected no chain for exit role, got %v", handler["chain"]) + } + if _, hasRetries := handler["retries"]; hasRetries { + t.Fatalf("expected no retries for exit role, got %v", handler["retries"]) + } + metadata := service["metadata"].(map[string]interface{}) + if metadata["interface"] != "eth0" { + t.Fatalf("expected interface 'eth0', got %v", metadata["interface"]) + } +} + +func TestBuildFederationServiceConfig_TLSTunnelProtocol_SetsNodelay(t *testing.T) { + service := buildFederationServiceConfig("svc-tls", ":40000", "tls", "middle", "chain-next", 2, "") + handler := service["handler"].(map[string]interface{}) + meta := handler["metadata"].(map[string]interface{}) + if meta["nodelay"] != true { + t.Fatalf("expected nodelay=true for TLS protocol, got %v", meta["nodelay"]) + } +} + +func TestBuildFederationServiceConfig_NonTLSProtocol_NoNodelay(t *testing.T) { + service := buildFederationServiceConfig("svc-tcp", ":40000", "tcp", "middle", "chain-next", 2, "") + handler := service["handler"].(map[string]interface{}) + if _, hasMeta := handler["metadata"]; hasMeta { + t.Fatalf("expected no metadata for non-TLS protocol, got %v", handler["metadata"]) + } +} + func TestFederationRuntimeReservePortRejectsWhenShareFlowExceeded(t *testing.T) { r, err := repo.Open(filepath.Join(t.TempDir(), "panel.db")) if err != nil { diff --git a/go-backend/internal/http/handler/mutations.go b/go-backend/internal/http/handler/mutations.go index e7956d7..a4d26c5 100644 --- a/go-backend/internal/http/handler/mutations.go +++ b/go-backend/internal/http/handler/mutations.go @@ -3097,7 +3097,7 @@ func (h *Handler) applyTunnelRuntime(state *tunnelCreateState) ([]int64, []int64 } createdChains = append(createdChains, chainNode.NodeID) - serviceData := buildTunnelChainServiceConfig(state.TunnelID, chainNode, state.Nodes[chainNode.NodeID]) + serviceData := buildTunnelChainServiceConfig(state.TunnelID, chainNode, state.Nodes[chainNode.NodeID], len(nextTargets)) if err := h.addTunnelServiceOnNode(chainNode.NodeID, state.TunnelID, serviceData); err != nil { return createdChains, createdServices, fmt.Errorf("转发链节点 %s 下发服务失败: %w", nodeDisplayName(state.Nodes[chainNode.NodeID]), err) } @@ -3109,7 +3109,7 @@ func (h *Handler) applyTunnelRuntime(state *tunnelCreateState) ([]int64, []int64 if node := state.Nodes[outNode.NodeID]; node != nil && node.IsRemote == 1 { continue } - serviceData := buildTunnelChainServiceConfig(state.TunnelID, outNode, state.Nodes[outNode.NodeID]) + serviceData := buildTunnelChainServiceConfig(state.TunnelID, outNode, state.Nodes[outNode.NodeID], 1) if err := h.addTunnelServiceOnNode(outNode.NodeID, state.TunnelID, serviceData); err != nil { return createdChains, createdServices, fmt.Errorf("出口节点 %s 下发服务失败: %w", nodeDisplayName(state.Nodes[outNode.NodeID]), err) } @@ -3260,7 +3260,7 @@ func buildTunnelChainConfig(tunnelID int64, fromNodeID int64, targets []tunnelRu }, nil } -func buildTunnelChainServiceConfig(tunnelID int64, chainNode tunnelRuntimeNode, node *nodeRecord) []map[string]interface{} { +func buildTunnelChainServiceConfig(tunnelID int64, chainNode tunnelRuntimeNode, node *nodeRecord, nextHopCandidateCount int) []map[string]interface{} { if node == nil { return nil } @@ -3271,6 +3271,9 @@ func buildTunnelChainServiceConfig(tunnelID int64, chainNode tunnelRuntimeNode, if isTLSTunnelProtocol(protocol) { handlerCfg["metadata"] = map[string]interface{}{"nodelay": true} } + if nextHopCandidateCount > 1 { + handlerCfg["retries"] = nextHopCandidateCount - 1 + } service := map[string]interface{}{ "name": fmt.Sprintf("%d_tls", tunnelID), "addr": processServerAddress(fmt.Sprintf("%s:%d", defaultString(strings.TrimSpace(chainNode.ConnectIP), node.TCPListenAddr), chainNode.Port)), diff --git a/plans/037-tunnel-chain-failover-repair.md b/plans/037-tunnel-chain-failover-repair.md new file mode 100644 index 0000000..5c6844f --- /dev/null +++ b/plans/037-tunnel-chain-failover-repair.md @@ -0,0 +1,33 @@ +# 037 Tunnel Chain Failover Repair + +## Checklist + +- [x] Analyze middle-hop primary/backup failover across backend runtime generation and agent route selection. +- [x] Add regression coverage for a tunnel relay chain where a same-hop `fifo` primary is down and the backup must take over. +- [x] Update tunnel runtime generation so chain services retry route selection when the next hop has multiple candidates. +- [x] Harden agent-side chain failover if backend-configured retries alone does not cover all relay/chain paths. + - N/A: Router retry loop (`go-gost/x/chain/router.go:91`) rebuilds route on each iteration, so FailFilter applies to failed nodes. +- [x] Revalidate diagnosis output so tunnel/forward tests reflect failover behavior instead of looking fully broken. + - N/A: Diagnosis tests individual legs (A→next, B→next) which is correct. Failover is for actual traffic, not diagnosis. +- [x] Run targeted backend and agent test suites. + +## Findings + +- Backend already emits hop selectors for tunnel chains with `strategy`, `maxFails=1`, and `failTimeout=10m` in `go-backend/internal/http/handler/mutations.go:3243`, so the control plane is not dropping the primary/backup mode itself. +- Agent route construction selects one node per hop up front in `go-gost/x/chain/chain.go:92`. If the chosen primary node is offline, the dial fails inside `go-gost/x/chain/route.go:220` and the node gets marked failed, but that mark only matters on a later route build. +- Tunnel chain services are generated without handler retry settings in `go-backend/internal/http/handler/mutations.go:3274`, while the router only rebuilds a route when `cfg.Handler.Retries` is greater than zero in `go-gost/x/config/parsing/service/parse.go:319`. +- Because the default retry count is effectively one attempt, a relay request never gets a second route selection after the primary middle-hop node is marked down, so traffic does not switch to the backup node. +- The forward handlers already have explicit retry/exclude-node loops in `go-gost/x/handler/forward/local/handler.go:179` and `go-gost/x/handler/forward/remote/handler.go:207`, which explains why failover logic exists in the codebase but is missing on the tunnel relay chain path. + +## Repair Direction + +- In backend tunnel runtime generation, compute the downstream candidate count for each chain service and set handler `retries` to at least `len(nextTargets) - 1` when a hop has multiple selectable nodes. That gives the router another dial cycle so `FailFilter` can skip the failed primary and pick the backup. +- Keep the retry value scoped to tunnel relay services built from `buildTunnelChainServiceConfig` so single-node hops do not incur unnecessary extra attempts. +- Add an agent-side regression test around relay + chain routing that simulates an offline primary node and asserts the second attempt lands on the backup node after the first node is marked failed. +- Add a backend regression test covering a tunnel definition with two nodes on the same middle hop in `fifo` mode, verifying the generated service config carries the retry budget needed for failover. +- Recheck tunnel/forward diagnosis behavior after the runtime fix. The current diagnosis model probes individual branch legs, so it may need an aggregated result or clearer messaging to avoid reading a partial branch failure as total failover failure. + +## Validation + +- `cd go-backend && go test ./internal/http/handler/... ./tests/contract/...` +- `cd go-gost/x && go test ./chain/... ./handler/relay/... ./config/parsing/service/...` diff --git a/plans/038-federation-middle-hop-retry-parity.md b/plans/038-federation-middle-hop-retry-parity.md new file mode 100644 index 0000000..379b6d5 --- /dev/null +++ b/plans/038-federation-middle-hop-retry-parity.md @@ -0,0 +1,28 @@ +# 038 Federation Middle-Hop Retry Parity + +## Checklist + +- [x] Reproduce and document the parity gap between local tunnel middle-hop runtime generation and federation-applied middle roles. +- [x] Update federation runtime apply logic so remote middle-hop services set handler `retries` when the next hop has multiple candidates. +- [x] Add regression coverage for federated middle-hop runtime generation or contract behavior, including multi-target `fifo` scenarios. +- [x] Verify release / cleanup paths remain correct when the federated middle service carries retry settings. +- [x] Run targeted backend tests for handler and federation contract coverage. + +## Findings + +- Local tunnel runtime generation now sets `handler.retries` for middle-hop services based on downstream candidate count in `go-backend/internal/http/handler/mutations.go`, which enables router-level re-selection after a failed primary node. +- Federation runtime apply still creates remote middle-hop services without `handler.retries` in `go-backend/internal/http/handler/federation.go`, even though the remote chain hop itself uses the same selector failover settings (`strategy`, `maxFails=1`, `failTimeout=10m`). +- Because `go-gost/x/config/parsing/service/parse.go` only enables router retries when `cfg.Handler.Retries > 0`, federated middle-hop services can still fail hard on the first offline primary target instead of switching to backup. +- The gap creates inconsistent behavior: identical tunnel topologies can fail over correctly on local middle nodes but not on federated / remote middle nodes. + +## Repair Direction + +- In `go-backend/internal/http/handler/federation.go`, compute retry budget for `req.Role == "middle"` from `len(req.Targets)` and set `service["handler"]["retries"]` to at least `len(req.Targets) - 1` when there is more than one target. +- Keep retry injection scoped to federated middle roles only; exit roles should continue to omit retries because they do not rebuild downstream chain selection. +- Add regression coverage that proves federated middle runtime application preserves local parity, ideally by asserting the generated remote service config or by exercising a dual-panel contract path with multi-target middle nodes. +- Recheck federation release behavior to ensure added retry fields do not affect idempotent cleanup, service deletion, or re-apply flows. + +## Validation + +- `cd go-backend && go test ./internal/http/handler/... -count=1` +- `cd go-backend && go test ./tests/contract/... -count=1`