fix(relay): authenticate endpoints and fail closed on missing routes (#569)

* fix(relay): authenticate endpoints and fail closed on missing routes

* fix(tunnel): retain chains until receiver replacement is acknowledged

* docs: record final relay security review and validation
This commit is contained in:
sagit
2026-11-08 23:46:18 +08:00
committed by GitHub
parent 46bfb86c1b
commit fadc2b6c12
51 changed files with 5562 additions and 277 deletions
@@ -1972,7 +1972,12 @@ func buildForwardServiceConfigs(baseName string, forward *forwardRecord, tunnel
service["listener"].(map[string]interface{})["metadata"] = listenerMetadata
}
if tunnel != nil && tunnel.Type == 2 {
service["handler"].(map[string]interface{})["chain"] = fmt.Sprintf("chains_%d", forward.TunnelID)
handlerConfig := service["handler"].(map[string]interface{})
handlerConfig["chain"] = fmt.Sprintf("chains_%d", forward.TunnelID)
if handlerConfig["metadata"] == nil {
handlerConfig["metadata"] = map[string]interface{}{}
}
handlerConfig["metadata"].(map[string]interface{})["chain.strict"] = true
}
if tunnel != nil && tunnel.Type == 1 && strings.TrimSpace(node.InterfaceName) != "" {
serviceMetadata := ensureServiceMetadata(service)
+75 -33
View File
@@ -17,6 +17,7 @@ import (
"go-backend/internal/http/client"
"go-backend/internal/http/response"
"go-backend/internal/relaysecurity"
"go-backend/internal/store/repo"
)
@@ -68,18 +69,20 @@ type federationRuntimeReservePortRequest struct {
}
type federationRuntimeTarget struct {
Host string `json:"host"`
Port int `json:"port"`
Protocol string `json:"protocol"`
RelayAuth *relaysecurity.Credentials `json:"relayAuth,omitempty"`
Host string `json:"host"`
Port int `json:"port"`
Protocol string `json:"protocol"`
}
type federationRuntimeApplyRoleRequest struct {
ReservationID string `json:"reservationId"`
ResourceKey string `json:"resourceKey"`
Role string `json:"role"`
Protocol string `json:"protocol"`
Strategy string `json:"strategy"`
Targets []federationRuntimeTarget `json:"targets"`
RelayAuth *relaysecurity.Credentials `json:"relayAuth,omitempty"`
ReservationID string `json:"reservationId"`
ResourceKey string `json:"resourceKey"`
Role string `json:"role"`
Protocol string `json:"protocol"`
Strategy string `json:"strategy"`
Targets []federationRuntimeTarget `json:"targets"`
}
type federationRuntimeReleaseRoleRequest struct {
@@ -115,6 +118,13 @@ func buildFederationMiddleChainConfig(chainName string, runtimeID int64, protoco
connector := map[string]interface{}{
"type": "relay",
}
if target.RelayAuth != nil {
if err := target.RelayAuth.Validate(); err != nil {
return nil, err
}
connector["auth"] = target.RelayAuth.Config()
}
if isTCPTunnelProtocol(targetProtocol) {
connector["metadata"] = map[string]interface{}{
"nodelay": true,
@@ -221,7 +231,7 @@ type remoteUsageNodeItem struct {
SyncError string `json:"syncError,omitempty"`
}
func buildFederationServiceConfig(serviceName, addr, protocol, role, chainName string, targetCount int, interfaceName string) map[string]interface{} {
func buildFederationServiceConfig(serviceName, addr, protocol, role, chainName string, targetCount int, interfaceName string, credentials *relaysecurity.Credentials) map[string]interface{} {
service := map[string]interface{}{
"name": serviceName,
"addr": addr,
@@ -230,6 +240,10 @@ func buildFederationServiceConfig(serviceName, addr, protocol, role, chainName s
},
"listener": buildTunnelListenerConfig(protocol),
}
if credentials != nil {
service["handler"].(map[string]interface{})["auth"] = credentials.Config()
}
if isTCPTunnelProtocol(protocol) {
service["handler"].(map[string]interface{})["metadata"] = map[string]interface{}{
"nodelay": true,
@@ -243,6 +257,13 @@ func buildFederationServiceConfig(serviceName, addr, protocol, role, chainName s
}
}
if role == "middle" {
handler := service["handler"].(map[string]interface{})
md, _ := handler["metadata"].(map[string]interface{})
if md == nil {
md = map[string]interface{}{}
handler["metadata"] = md
}
md["chain.strict"] = true
service["handler"].(map[string]interface{})["chain"] = chainName
if targetCount > 1 {
service["handler"].(map[string]interface{})["retries"] = targetCount - 1
@@ -915,17 +936,18 @@ func (h *Handler) federationConnect(w http.ResponseWriter, r *http.Request) {
}
response.WriteJSON(w, response.OK(map[string]interface{}{
"shareId": share.ID,
"shareName": share.Name,
"nodeId": share.NodeID,
"nodeName": nodeInfo.Name,
"serverIp": nodeInfo.ServerIP,
"status": nodeInfo.Status,
"maxBandwidth": share.MaxBandwidth,
"currentFlow": share.CurrentFlow,
"expiryTime": share.ExpiryTime,
"portRangeStart": share.PortRangeStart,
"portRangeEnd": share.PortRangeEnd,
"shareId": share.ID,
"relayAuthVersion": relaysecurity.Version,
"shareName": share.Name,
"nodeId": share.NodeID,
"nodeName": nodeInfo.Name,
"serverIp": nodeInfo.ServerIP,
"status": nodeInfo.Status,
"maxBandwidth": share.MaxBandwidth,
"currentFlow": share.CurrentFlow,
"expiryTime": share.ExpiryTime,
"portRangeStart": share.PortRangeStart,
"portRangeEnd": share.PortRangeEnd,
}))
}
@@ -1079,9 +1101,10 @@ func (h *Handler) federationRuntimeReservePort(w http.ResponseWriter, r *http.Re
}
if existing != nil && existing.Status == 1 {
response.WriteJSON(w, response.OK(map[string]interface{}{
"reservationId": existing.ReservationID,
"allocatedPort": existing.Port,
"bindingId": existing.BindingID,
"reservationId": existing.ReservationID,
"allocatedPort": existing.Port,
"relayAuthVersion": relaysecurity.Version,
"bindingId": existing.BindingID,
}))
return
}
@@ -1103,6 +1126,8 @@ func (h *Handler) federationRuntimeReservePort(w http.ResponseWriter, r *http.Re
existing.ServiceName = ""
existing.Strategy = "round"
existing.Target = ""
existing.RelayAuth = ""
existing.RelayAuthVersion = 0
existing.Applied = 0
existing.Status = 1
existing.UpdatedTime = now
@@ -1111,9 +1136,10 @@ func (h *Handler) federationRuntimeReservePort(w http.ResponseWriter, r *http.Re
return
}
response.WriteJSON(w, response.OK(map[string]interface{}{
"reservationId": existing.ReservationID,
"allocatedPort": existing.Port,
"bindingId": existing.BindingID,
"reservationId": existing.ReservationID,
"allocatedPort": existing.Port,
"relayAuthVersion": relaysecurity.Version,
"bindingId": existing.BindingID,
}))
return
}
@@ -1142,9 +1168,10 @@ func (h *Handler) federationRuntimeReservePort(w http.ResponseWriter, r *http.Re
}
response.WriteJSON(w, response.OK(map[string]interface{}{
"reservationId": runtime.ReservationID,
"allocatedPort": runtime.Port,
"bindingId": runtime.BindingID,
"reservationId": runtime.ReservationID,
"allocatedPort": runtime.Port,
"relayAuthVersion": relaysecurity.Version,
"bindingId": runtime.BindingID,
}))
}
@@ -1211,6 +1238,20 @@ func (h *Handler) federationRuntimeApplyRole(w http.ResponseWriter, r *http.Requ
response.WriteJSON(w, response.ErrDefault("Reserved port is outside share range"))
return
}
if _, err := decodeRuntimeRelayAuth(runtime); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
if err := validateFederationRelayAuth(runtime.RelayAuth, req.RelayAuth, req.Targets); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
if req.RelayAuth != nil {
encoded, _ := json.Marshal(req.RelayAuth)
runtime.RelayAuth = string(encoded)
runtime.RelayAuthVersion = relaysecurity.Version
}
if strings.TrimSpace(runtime.BindingID) == "" {
runtime.BindingID = randomToken(24)
}
@@ -1230,9 +1271,10 @@ func (h *Handler) federationRuntimeApplyRole(w http.ResponseWriter, r *http.Requ
}
response.WriteJSON(w, response.OK(map[string]interface{}{
"bindingId": runtime.BindingID,
"reservationId": runtime.ReservationID,
"allocatedPort": runtime.Port,
"relayAuthVersion": relayAuthVersion(req.RelayAuth),
"bindingId": runtime.BindingID,
"reservationId": runtime.ReservationID,
"allocatedPort": runtime.Port,
}))
}
@@ -8,6 +8,7 @@ import (
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
@@ -495,19 +496,22 @@ func TestPeerResourcePendingRetryLeavesAppliedSiblingAlone(t *testing.T) {
}
func TestPeerResourceLegacyReleaseAcknowledgmentIsAtomic(t *testing.T) {
a := newCleanupAgent(t)
var failCompletion atomic.Bool
a := newCleanupAgent(t, func(r *repo.Repository) {
if err := r.DB().Callback().Update().Before("gorm:update").Register("fail-runtime-release", func(tx *gorm.DB) {
if failCompletion.Load() && tx.Statement.Table == "peer_share_runtime" {
tx.AddError(errors.New("simulated completion failure"))
}
}); err != nil {
t.Fatal(err)
}
})
shareID := a.addRuntime(t, "70_1_0", 1, 1, 1, time.Now())
share, err := a.h.repo.GetPeerShare(shareID)
if err != nil {
t.Fatal(err)
}
if err := a.h.repo.DB().Callback().Update().Before("gorm:update").Register("fail-runtime-release", func(tx *gorm.DB) {
if tx.Statement.Table == "peer_share_runtime" {
tx.AddError(errors.New("simulated completion failure"))
}
}); err != nil {
t.Fatal(err)
}
failCompletion.Store(true)
got := resourceTestCommand(t, a.h, share, "DeleteService", map[string]interface{}{"services": []string{"70_1_0", "70_1_0_tcp", "70_1_0_udp"}})
if got.Code == 0 {
t.Fatal("completion database failure reported success")
@@ -525,9 +529,7 @@ func TestPeerResourceLegacyReleaseAcknowledgmentIsAtomic(t *testing.T) {
if !proof || !pending {
t.Fatalf("failure lost ownership or retry marker: %+v", items)
}
if err := a.h.repo.DB().Callback().Update().Remove("fail-runtime-release"); err != nil {
t.Fatal(err)
}
failCompletion.Store(false)
if err := a.h.retryPendingPeerShareResourcesOnNode(1); err != nil {
t.Fatal(err)
}
@@ -228,7 +228,7 @@ func TestPrepareTunnelCreateStateAllowsOfflineRemoteMiddleNode(t *testing.T) {
}
func TestBuildFederationServiceConfig_MiddleRoleWithMultipleTargets_SetsRetries(t *testing.T) {
service := buildFederationServiceConfig("svc-middle", ":40000", "tls", "middle", "chain-next", 3, "")
service := buildFederationServiceConfig("svc-middle", ":40000", "tls", "middle", "chain-next", 3, "", nil)
handler := service["handler"].(map[string]interface{})
if handler["chain"] != "chain-next" {
t.Fatalf("expected chain 'chain-next', got %v", handler["chain"])
@@ -239,7 +239,7 @@ func TestBuildFederationServiceConfig_MiddleRoleWithMultipleTargets_SetsRetries(
}
func TestBuildFederationServiceConfig_MiddleRoleWithSingleTarget_NoRetries(t *testing.T) {
service := buildFederationServiceConfig("svc-middle", ":40000", "tls", "middle", "chain-next", 1, "")
service := buildFederationServiceConfig("svc-middle", ":40000", "tls", "middle", "chain-next", 1, "", nil)
handler := service["handler"].(map[string]interface{})
if handler["chain"] != "chain-next" {
t.Fatalf("expected chain 'chain-next', got %v", handler["chain"])
@@ -250,7 +250,7 @@ func TestBuildFederationServiceConfig_MiddleRoleWithSingleTarget_NoRetries(t *te
}
func TestBuildFederationServiceConfig_ExitRole_NoRetriesRegardlessOfTargets(t *testing.T) {
service := buildFederationServiceConfig("svc-exit", ":40000", "tls", "exit", "", 3, "eth0")
service := buildFederationServiceConfig("svc-exit", ":40000", "tls", "exit", "", 3, "eth0", nil)
handler := service["handler"].(map[string]interface{})
if _, hasChain := handler["chain"]; hasChain {
t.Fatalf("expected no chain for exit role, got %v", handler["chain"])
@@ -265,7 +265,7 @@ func TestBuildFederationServiceConfig_ExitRole_NoRetriesRegardlessOfTargets(t *t
}
func TestBuildFederationServiceConfig_TLSTunnelProtocol_SetsNodelay(t *testing.T) {
service := buildFederationServiceConfig("svc-tls", ":40000", "tls", "middle", "chain-next", 2, "")
service := buildFederationServiceConfig("svc-tls", ":40000", "tls", "middle", "chain-next", 2, "", nil)
handler := service["handler"].(map[string]interface{})
meta := handler["metadata"].(map[string]interface{})
if meta["nodelay"] != true {
@@ -274,10 +274,14 @@ func TestBuildFederationServiceConfig_TLSTunnelProtocol_SetsNodelay(t *testing.T
}
func TestBuildFederationServiceConfig_NonTLSProtocol_NoNodelay(t *testing.T) {
service := buildFederationServiceConfig("svc-tcp", ":40000", "tcp", "middle", "chain-next", 2, "")
service := buildFederationServiceConfig("svc-tcp", ":40000", "tcp", "middle", "chain-next", 2, "", nil)
handler := service["handler"].(map[string]interface{})
if _, hasMeta := handler["metadata"]; hasMeta {
t.Fatalf("expected no metadata for non-TLS protocol, got %v", handler["metadata"])
metadata, _ := handler["metadata"].(map[string]interface{})
if _, hasNodelay := metadata["nodelay"]; hasNodelay {
t.Fatal("non-TLS protocol must not set nodelay")
}
if metadata["chain.strict"] != true {
t.Fatal("middle relay must require its configured chain")
}
}
@@ -305,11 +305,17 @@ func TestFederationRemoteUsageListSyncErrorFallback(t *testing.T) {
h := New(r, "test-jwt-secret")
now := time.Now().UnixMilli()
// Model an unavailable provider entirely on loopback instead of relying on
// external DNS or network restrictions to produce the sync error.
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "provider unavailable", http.StatusServiceUnavailable)
}))
defer provider.Close()
if err := r.DB().Exec(`
INSERT INTO node(name, secret, server_ip, server_ip_v4, server_ip_v6, port, interface_name, version, http, tls, socks, created_time, updated_time, status, tcp_listen_addr, udp_listen_addr, inx, is_remote, remote_url, remote_token, remote_config)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, "sync-error-node", "sync-error-secret", "10.50.60.70", "10.50.60.70", "", "32000-32010", "", "v1", 1, 1, 1, now, now, 1, "[::]", "[::]", 0, 1, "http://unreachable.invalid:9999", "bad-token", `{"shareId":42,"maxBandwidth":5368709120,"currentFlow":999999,"portRangeStart":32000,"portRangeEnd":32010}`).Error; err != nil {
`, "sync-error-node", "sync-error-secret", "10.50.60.70", "10.50.60.70", "", "32000-32010", "", "v1", 1, 1, 1, now, now, 1, "[::]", "[::]", 0, 1, provider.URL, "bad-token", `{"shareId":42,"maxBandwidth":5368709120,"currentFlow":999999,"portRangeStart":32000,"portRangeEnd":32010}`).Error; err != nil {
t.Fatalf("insert remote node: %v", err)
}
@@ -348,7 +354,7 @@ func TestFederationRemoteUsageListSyncErrorFallback(t *testing.T) {
syncErr, _ := first["syncError"].(string)
if syncErr == "" {
t.Fatalf("expected non-empty syncError field on unreachable provider")
t.Fatalf("expected non-empty syncError field on unavailable provider")
}
}
@@ -858,11 +864,15 @@ func TestFederationRemoteUsageList(t *testing.T) {
h := New(r, "test-jwt-secret")
now := time.Now().UnixMilli()
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "provider unavailable", http.StatusServiceUnavailable)
}))
defer provider.Close()
if err := r.DB().Exec(`
INSERT INTO node(name, secret, server_ip, server_ip_v4, server_ip_v6, port, interface_name, version, http, tls, socks, created_time, updated_time, status, tcp_listen_addr, udp_listen_addr, inx, is_remote, remote_url, remote_token, remote_config)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, "remote-consumer-node", "remote-consumer-secret", "10.30.40.50", "10.30.40.50", "", "31000-31010", "", "v1", 1, 1, 1, now, now, 1, "[::]", "[::]", 0, 1, "http://peer.example", "peer-token", `{"shareId":88,"maxBandwidth":2147483648,"currentFlow":1073741824,"portRangeStart":31000,"portRangeEnd":31010}`).Error; err != nil {
`, "remote-consumer-node", "remote-consumer-secret", "10.30.40.50", "10.30.40.50", "", "31000-31010", "", "v1", 1, 1, 1, now, now, 1, "[::]", "[::]", 0, 1, provider.URL, "peer-token", `{"shareId":88,"maxBandwidth":2147483648,"currentFlow":1073741824,"portRangeStart":31000,"portRangeEnd":31010}`).Error; err != nil {
t.Fatalf("insert remote node: %v", err)
}
nodeID := mustLastInsertID(t, r, "remote-consumer-node")
@@ -882,8 +892,8 @@ func TestFederationRemoteUsageList(t *testing.T) {
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?),
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
tunnelAID, nodeID, 2, 1, "http://peer.example", "rk-a", "rb-a", 31001, 1, now, now,
tunnelBID, nodeID, 3, 0, "http://peer.example", "rk-b", "rb-b", 31002, 1, now, now,
tunnelAID, nodeID, 2, 1, provider.URL, "rk-a", "rb-a", 31001, 1, now, now,
tunnelBID, nodeID, 3, 0, provider.URL, "rk-b", "rb-b", 31002, 1, now, now,
).Error; err != nil {
t.Fatalf("insert federation bindings: %v", err)
}
@@ -37,7 +37,7 @@ type cleanupAgent struct {
commands []cleanupAgentCommand
}
func newCleanupAgent(t *testing.T) *cleanupAgent {
func newCleanupAgent(t *testing.T, setup ...func(*repo.Repository)) *cleanupAgent {
t.Helper()
r, err := repo.Open(filepath.Join(t.TempDir(), "cleanup.db"))
if err != nil {
@@ -48,6 +48,11 @@ func newCleanupAgent(t *testing.T) *cleanupAgent {
if err := r.DB().Create(&model.Node{ID: 1, Name: "relay", Secret: secret, ServerIP: "127.0.0.1", Port: "30000", Status: 1}).Error; err != nil {
t.Fatal(err)
}
// Register test-local database callbacks before WebSocket goroutines can
// execute them. Keep registries immutable until the fixture is closed.
for _, configure := range setup {
configure(r)
}
server := ws.NewServer(r, "test-jwt-secret")
online := make(chan struct{})
server.SetNodeOnlineHook(func(int64) { close(online) })
@@ -39,6 +39,9 @@ type Handler struct {
healthCheck *health.Checker
nftablesManager nftablesRuntimeManager
tunnelLifecycleMu sync.Mutex
tunnelLifecycleLocks map[int64]*tunnelLifecycleLock
peerResourceMu sync.Mutex
captchaMu sync.Mutex
captchaTokens map[string]int64
+3
View File
@@ -48,6 +48,9 @@ func (h *Handler) runFederationCleanupRetryLoop(ctx context.Context) {
if err := h.retryPendingFederationRuntimeCleanup(); err != nil {
log.Printf("federation cleanup remains pending: %v", err)
}
if err := h.retryPendingTunnelRuntimeCleanup(); err != nil {
log.Printf("tunnel runtime cleanup remains pending: %v", err)
}
h.retryPendingPeerShareOperations()
select {
case <-ctx.Done():
+331 -150
View File
@@ -19,6 +19,7 @@ import (
"go-backend/internal/http/client"
"go-backend/internal/http/response"
"go-backend/internal/relaysecurity"
"go-backend/internal/security"
"go-backend/internal/store/model"
"go-backend/internal/store/repo"
@@ -848,10 +849,70 @@ func (h *Handler) tunnelCreate(w http.ResponseWriter, r *http.Request) {
return
}
var tunnelInIP sql.NullString
if trimmed := strings.TrimSpace(inIP); trimmed != "" {
tunnelInIP = sql.NullString{String: trimmed, Valid: true}
}
tunnelProtocol := "tls"
if len(runtimeState.InNodes) > 0 && strings.TrimSpace(runtimeState.InNodes[0].Protocol) != "" {
tunnelProtocol = strings.TrimSpace(runtimeState.InNodes[0].Protocol)
}
tunnel := model.Tunnel{
RelayAuthMode: relaysecurity.Legacy,
Name: name,
TrafficRatio: trafficRatio,
Type: typeVal,
Protocol: tunnelProtocol,
Flow: flow,
CreatedTime: now,
UpdatedTime: now,
Status: status,
InIP: tunnelInIP,
Inx: inx,
IPPreference: ipPreference,
ProbeTargetHost: probeTargetHost,
ProbeTargetPort: probeTargetPort,
}
if typeVal == 2 {
tunnel.RelayAuthMode = relaysecurity.Required
}
if err := tx.Create(&tunnel).Error; err != nil {
response.WriteJSON(w, response.Err(-2, err.Error()))
return
}
tunnelID := tunnel.ID
unlock, locked := h.tryLockNewTunnelLifecycle(tunnelID)
if !locked {
response.WriteJSON(w, response.ErrDefault("隧道正在变更,请重试"))
return
}
defer unlock()
runtimeState.TunnelID = tunnelID
runtimeState.RelayAuthMode = tunnel.RelayAuthMode
if err := h.resolveTunnelRelayAuth(tx, runtimeState, true); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
if typeVal == 2 {
if err := h.persistPreparedTunnelChainsTx(tx, runtimeState); err != nil {
response.WriteJSON(w, response.Err(-2, err.Error()))
return
}
}
// The lifecycle lock is held before publishing this ID. Network calls below
// must not monopolize SQLite's only connection or a database transaction.
if err := tx.Commit().Error; err != nil {
response.WriteJSON(w, response.Err(-2, err.Error()))
return
}
failCreate := func(err error, code int) {
err = errors.Join(err, h.deleteTunnelByID(tunnelID))
response.WriteJSON(w, response.Err(code, err.Error()))
}
if len(runtimeState.InNodes) > 0 {
firstNodeID := runtimeState.InNodes[0].NodeID
isRemote, rUrl, rToken, _ := h.repo.GetNodeRemoteFieldsTx(tx, firstNodeID)
if isRemote == 1 {
entryNode := runtimeState.Nodes[firstNodeID]
if entryNode != nil && entryNode.IsRemote == 1 {
fc := client.NewFederationClient()
targetProto := "tcp"
@@ -878,17 +939,12 @@ func (h *Handler) tunnelCreate(w http.ResponseWriter, r *http.Request) {
if targetPort > 0 && targetAddr != "" {
inNodeRec := runtimeState.Nodes[firstNodeID]
if err := validateRemoteNodePort(inNodeRec, targetPort); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
failCreate(err, -1)
return
}
domainCfg, _ := h.repo.GetConfigByName("panel_domain")
localDomain := ""
if domainCfg != nil {
localDomain = domainCfg.Value
}
_, err := fc.CreateTunnel(rUrl.String, rToken.String, localDomain, targetProto, targetPort, targetAddr)
_, err := fc.CreateTunnel(entryNode.RemoteURL, entryNode.RemoteToken, localDomain, targetProto, targetPort, targetAddr)
if err != nil {
response.WriteJSON(w, response.ErrDefault("Remote tunnel creation failed: "+err.Error()))
failCreate(fmt.Errorf("remote tunnel creation failed: %w", err), -1)
return
}
}
@@ -896,66 +952,53 @@ func (h *Handler) tunnelCreate(w http.ResponseWriter, r *http.Request) {
}
}
var tunnelInIP sql.NullString
if trimmed := strings.TrimSpace(inIP); trimmed != "" {
tunnelInIP = sql.NullString{String: trimmed, Valid: true}
}
tunnelProtocol := "tls"
if len(runtimeState.InNodes) > 0 && strings.TrimSpace(runtimeState.InNodes[0].Protocol) != "" {
tunnelProtocol = strings.TrimSpace(runtimeState.InNodes[0].Protocol)
}
tunnel := model.Tunnel{
Name: name,
TrafficRatio: trafficRatio,
Type: typeVal,
Protocol: tunnelProtocol,
Flow: flow,
CreatedTime: now,
UpdatedTime: now,
Status: status,
InIP: tunnelInIP,
Inx: inx,
IPPreference: ipPreference,
ProbeTargetHost: probeTargetHost,
ProbeTargetPort: probeTargetPort,
}
if err := tx.Create(&tunnel).Error; err != nil {
response.WriteJSON(w, response.Err(-2, err.Error()))
return
}
tunnelID := tunnel.ID
runtimeState.TunnelID = tunnelID
var federationBindings []repo.FederationTunnelBinding
var federationReleaseRefs []federationRuntimeReleaseRef
federationBindings, federationReleaseRefs, err = h.applyFederationRuntime(runtimeState, localDomain)
if err != nil {
tx.Rollback()
err = errors.Join(err, h.releaseFederationRuntimeRefs(federationReleaseRefs))
response.WriteJSON(w, response.ErrDefault(err.Error()))
failCreate(err, -1)
return
}
tx = h.repo.BeginTx()
if tx.Error != nil {
failCreate(errors.Join(tx.Error, h.releaseFederationRuntimeRefs(federationReleaseRefs)), -2)
return
}
if typeVal == 2 {
if err := h.repo.DeleteChainTunnelsByTunnelTx(tx, tunnelID); err != nil {
tx.Rollback()
failCreate(errors.Join(err, h.releaseFederationRuntimeRefs(federationReleaseRefs)), -2)
return
}
}
applyTunnelPortsToRequest(req, runtimeState)
if err := h.replaceTunnelChainsTx(tx, tunnelID, req); err != nil {
tx.Rollback()
err = errors.Join(err, h.releaseFederationRuntimeRefs(federationReleaseRefs))
response.WriteJSON(w, response.Err(-2, err.Error()))
failCreate(err, -2)
return
}
if err := h.repo.ReplaceFederationTunnelBindingsTx(tx, tunnelID, federationBindings); err != nil {
tx.Rollback()
err = errors.Join(err, h.releaseFederationRuntimeRefs(federationReleaseRefs))
response.WriteJSON(w, response.Err(-2, err.Error()))
failCreate(err, -2)
return
}
if err := tx.Commit().Error; err != nil {
tx.Rollback()
err = errors.Join(err, h.releaseFederationRuntimeRefs(federationReleaseRefs))
response.WriteJSON(w, response.Err(-2, err.Error()))
failCreate(err, -2)
return
}
if typeVal == 2 {
createdChains, createdServices, applyErr := h.applyTunnelRuntime(runtimeState)
if applyErr != nil {
if isTunnelRuntimePending(applyErr) {
response.WriteJSON(w, tunnelRuntimePendingResponse(tunnelID, applyErr))
return
}
h.rollbackTunnelRuntime(createdChains, createdServices, tunnelID, tunnelProtocol)
if cleanupErr := h.cleanupFederationRuntime(tunnelID); cleanupErr != nil {
applyErr = errors.Join(applyErr, cleanupErr)
@@ -969,31 +1012,6 @@ func (h *Handler) tunnelCreate(w http.ResponseWriter, r *http.Request) {
response.WriteJSON(w, response.OKEmpty())
}
func (h *Handler) cleanupTunnelRuntime(tunnelID int64) {
tunnel, err := h.getTunnelRecord(tunnelID)
if err != nil || tunnel.Type != 2 {
return
}
chainRows, err := h.listChainNodesForTunnel(tunnelID)
if err != nil {
return
}
chainName := fmt.Sprintf("chains_%d", tunnelID)
serviceNames := tunnelRuntimeServiceNames(tunnelID)
for _, row := range chainRows {
if row.ChainType == 1 {
_, _ = h.sendNodeCommand(row.NodeID, "DeleteChains", map[string]interface{}{"chain": chainName}, false, true)
} else if row.ChainType == 2 {
_, _ = h.sendNodeCommand(row.NodeID, "DeleteChains", map[string]interface{}{"chain": chainName}, false, true)
_, _ = h.sendNodeCommand(row.NodeID, "DeleteService", map[string]interface{}{"services": serviceNames}, false, true)
} else if row.ChainType == 3 {
_, _ = h.sendNodeCommand(row.NodeID, "DeleteService", map[string]interface{}{"services": serviceNames}, false, true)
}
}
}
func tunnelRuntimeServiceNames(tunnelID int64) []string {
return []string{
fmt.Sprintf("tunnel_%d", tunnelID),
@@ -1044,20 +1062,6 @@ func removedTunnelRuntimeNodeIDs(oldRows, newRows []chainNodeRecord, needsRuntim
return removed
}
func (h *Handler) cleanupObsoleteTunnelRuntime(tunnelID int64, oldRows, newRows []chainNodeRecord) {
if h == nil || tunnelID <= 0 || len(oldRows) == 0 {
return
}
chainName := fmt.Sprintf("chains_%d", tunnelID)
for _, nodeID := range removedTunnelRuntimeNodeIDs(oldRows, newRows, tunnelRuntimeNeedsChain) {
_, _ = h.sendNodeCommand(nodeID, "DeleteChains", map[string]interface{}{"chain": chainName}, false, true)
}
serviceNames := tunnelRuntimeServiceNames(tunnelID)
for _, nodeID := range removedTunnelRuntimeNodeIDs(oldRows, newRows, tunnelRuntimeNeedsService) {
_, _ = h.sendNodeCommand(nodeID, "DeleteService", map[string]interface{}{"services": serviceNames}, false, true)
}
}
func (h *Handler) tunnelGet(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
response.WriteJSON(w, response.ErrDefault("请求失败"))
@@ -1101,6 +1105,8 @@ func (h *Handler) tunnelUpdate(w http.ResponseWriter, r *http.Request) {
response.WriteJSON(w, response.ErrDefault("隧道ID不能为空"))
return
}
unlock := h.lockTunnelLifecycle(id)
defer unlock()
typeVal := asInt(req["type"], 1)
ipPreference := asString(req["ipPreference"])
_, hasProbeTargetHost := req["probeTargetHost"]
@@ -1156,6 +1162,10 @@ func (h *Handler) tunnelUpdate(w http.ResponseWriter, r *http.Request) {
return
}
runtimeState.TunnelID = id
runtimeState.RelayAuthMode = oldTunnel.RelayAuthMode
if typeVal == 2 && oldTunnel.Type != 2 {
runtimeState.RelayAuthMode = relaysecurity.Required
}
runtimeState.IPPreference = ipPreference
entryNodeIDs := make([]int64, 0, len(runtimeState.InNodes))
for _, inNode := range runtimeState.InNodes {
@@ -1199,12 +1209,18 @@ func (h *Handler) tunnelUpdate(w http.ResponseWriter, r *http.Request) {
response.WriteJSON(w, response.Err(-2, err.Error()))
return
}
if err := h.cleanupFederationRuntime(id); err != nil {
if err := h.resolveTunnelRelayAuth(nil, runtimeState, true); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
if oldTunnel.Type == 2 && typeVal != 2 {
h.cleanupTunnelRuntime(id)
if err := h.preflightFederationRelayAuth(runtimeState, localDomain); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
if err := h.cleanupFederationRuntime(id); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
var federationBindings []repo.FederationTunnelBinding
@@ -1279,6 +1295,19 @@ func (h *Handler) tunnelUpdate(w http.ResponseWriter, r *http.Request) {
return
}
if err := h.repo.SetTunnelRelayAuthModeTx(tx, id, runtimeState.RelayAuthMode); err != nil {
tx.Rollback()
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
if oldTunnel.Type == 2 {
if err := h.queueObsoleteTunnelRuntimeTx(tx, id, oldChainRows, tunnelStateRuntimeRows(runtimeState)); err != nil {
tx.Rollback()
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
}
if err := tx.Commit().Error; err != nil {
tx.Rollback()
err = errors.Join(err, h.releaseFederationRuntimeRefs(federationReleaseRefs))
@@ -1292,13 +1321,17 @@ func (h *Handler) tunnelUpdate(w http.ResponseWriter, r *http.Request) {
h.syncTunnelForwardsEntryPorts(id, newEntryNodeIDs)
}
var runtimePending error
if typeVal == 2 {
applyRuntime := h.applyTunnelRuntime
if oldTunnel != nil && oldTunnel.Type == 2 {
applyRuntime = h.applyTunnelRuntimeUpsert
}
createdChains, createdServices, applyErr := applyRuntime(runtimeState)
if applyErr != nil {
if isTunnelRuntimePending(applyErr) {
runtimePending = applyErr
}
if applyErr != nil && runtimePending == nil {
updateProtocol := "tls"
if len(runtimeState.InNodes) > 0 && strings.TrimSpace(runtimeState.InNodes[0].Protocol) != "" {
updateProtocol = strings.TrimSpace(runtimeState.InNodes[0].Protocol)
@@ -1307,16 +1340,12 @@ func (h *Handler) tunnelUpdate(w http.ResponseWriter, r *http.Request) {
h.rollbackTunnelRuntime(createdChains, createdServices, id, updateProtocol)
}
applyErr = errors.Join(applyErr, h.cleanupFederationRuntime(id))
if len(federationReleaseRefs) == 0 && shouldDeferTunnelRuntimeApplyError(applyErr) {
response.WriteJSON(w, response.OKEmpty())
return
}
response.WriteJSON(w, response.ErrDefault(applyErr.Error()))
return
}
newChainRows, _ := h.listChainNodesForTunnel(id)
h.cleanupObsoleteTunnelRuntime(id, oldChainRows, newChainRows)
runtimePending = errors.Join(runtimePending, h.acknowledgeTunnelServiceReplacements(id, createdServices))
}
runtimePending = errors.Join(runtimePending, h.retryTunnelRuntimeCleanupLocked(id))
oldType := 0
if oldTunnel != nil {
@@ -1325,14 +1354,22 @@ func (h *Handler) tunnelUpdate(w http.ResponseWriter, r *http.Request) {
if tunnelForwardRuntimeNeedsSync(oldType, typeVal, oldEntryNodeIDs, newEntryNodeIDs) {
forwards, fwdErr := h.listForwardsByTunnel(id)
if fwdErr != nil {
response.WriteJSON(w, response.OKEmpty())
response.WriteJSON(w, tunnelRuntimePendingResponse(id, errors.Join(runtimePending, fwdErr)))
return
}
var forwardErrors []error
for i := range forwards {
_ = h.syncForwardServices(&forwards[i], "UpdateService", true)
if err := h.syncForwardServices(&forwards[i], "UpdateService", true); err != nil {
forwardErrors = append(forwardErrors, err)
}
}
runtimePending = errors.Join(runtimePending, errors.Join(forwardErrors...))
}
if runtimePending != nil {
response.WriteJSON(w, tunnelRuntimePendingResponse(id, runtimePending))
return
}
response.WriteJSON(w, response.OKEmpty())
}
@@ -1494,6 +1531,15 @@ func (h *Handler) validateForwardPortAvailabilityTx(tx *gorm.DB, node *nodeRecor
if h == nil || h.repo == nil || tx == nil || node == nil || port <= 0 {
return nil
}
pendingPorts, err := h.repo.PendingTunnelCleanupPortsTx(tx, node.ID, 0)
if err != nil {
return err
}
for _, reserved := range pendingPorts {
if reserved == port {
return fmt.Errorf("节点 %s 端口 %d 仍在等待旧运行时清理", node.Name, port)
}
}
occupied, err := h.repo.HasOtherForwardOnNodePortTx(tx, node.ID, port, currentForwardID)
if err != nil {
return err
@@ -1691,11 +1737,12 @@ func (h *Handler) tunnelDelete(w http.ResponseWriter, r *http.Request) {
if id <= 0 {
return
}
unlock := h.lockTunnelLifecycle(id)
defer unlock()
if err := h.cleanupFederationRuntime(id); err != nil {
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
h.cleanupTunnelRuntime(id)
if err := h.deleteTunnelByID(id); err != nil {
response.WriteJSON(w, response.Err(-2, err.Error()))
return
@@ -1756,24 +1803,27 @@ func (h *Handler) tunnelBatchDelete(w http.ResponseWriter, r *http.Request) {
fail := 0
failures := make([]batchFailureDetail, 0)
for _, id := range ids {
tunnelName, _ := h.repo.GetTunnelName(id)
if _, err := h.getTunnelRecord(id); err != nil {
fail++
failures = appendBatchFailure(failures, id, tunnelName, err)
continue
}
if err := h.cleanupFederationRuntime(id); err != nil {
fail++
failures = appendBatchFailure(failures, id, tunnelName, err)
continue
}
h.cleanupTunnelRuntime(id)
if err := h.deleteTunnelByID(id); err != nil {
fail++
failures = appendBatchFailure(failures, id, tunnelName, err)
} else {
success++
}
func() {
unlock := h.lockTunnelLifecycle(id)
defer unlock()
tunnelName, _ := h.repo.GetTunnelName(id)
if _, err := h.getTunnelRecord(id); err != nil {
fail++
failures = appendBatchFailure(failures, id, tunnelName, err)
return
}
if err := h.cleanupFederationRuntime(id); err != nil {
fail++
failures = appendBatchFailure(failures, id, tunnelName, err)
return
}
if err := h.deleteTunnelByID(id); err != nil {
fail++
failures = appendBatchFailure(failures, id, tunnelName, err)
} else {
success++
}
}()
}
response.WriteJSON(w, response.OK(batchOperationResult{SuccessCount: success, FailCount: fail, Failures: failures}))
}
@@ -1792,14 +1842,15 @@ func (h *Handler) reconstructTunnelState(tunnelID int64) (*tunnelCreateState, er
ipPreference := h.repo.GetTunnelIPPreference(tunnelID)
state := &tunnelCreateState{
TunnelID: tunnelID,
Type: tunnel.Type,
IPPreference: ipPreference,
InNodes: make([]tunnelRuntimeNode, 0),
ChainHops: make([][]tunnelRuntimeNode, 0),
OutNodes: make([]tunnelRuntimeNode, 0),
Nodes: make(map[int64]*nodeRecord),
NodeIDList: make([]int64, 0),
RelayAuthMode: tunnel.RelayAuthMode,
TunnelID: tunnelID,
Type: tunnel.Type,
IPPreference: ipPreference,
InNodes: make([]tunnelRuntimeNode, 0),
ChainHops: make([][]tunnelRuntimeNode, 0),
OutNodes: make([]tunnelRuntimeNode, 0),
Nodes: make(map[int64]*nodeRecord),
NodeIDList: make([]int64, 0),
}
inNodes, chainHops, outNodes := splitChainNodeGroups(chainRows)
@@ -1856,10 +1907,23 @@ func (h *Handler) reconstructTunnelState(tunnelID int64) (*tunnelCreateState, er
state.Nodes[id] = node
}
if err := h.resolveTunnelRelayAuth(nil, state, false); err != nil {
return nil, err
}
return state, nil
}
func (h *Handler) redeployTunnelAndForwards(tunnelID int64) error {
unlock := h.lockTunnelLifecycle(tunnelID)
defer unlock()
return h.redeployTunnelAndForwardsLocked(tunnelID)
}
func (h *Handler) redeployTunnelAndForwardsLocked(tunnelID int64) error {
if err := h.retryTunnelRuntimeCleanupLocked(tunnelID); err != nil {
return err
}
tunnel, err := h.getTunnelRecord(tunnelID)
if err != nil {
return err
@@ -1873,28 +1937,44 @@ func (h *Handler) redeployTunnelAndForwards(tunnelID int64) error {
if err := validateTunnelRuntimeBeforeApply(state); err != nil {
return err
}
if err := h.cleanupFederationRuntime(tunnelID); err != nil {
existingBindings, err := h.repo.ListActiveFederationTunnelBindingsByTunnel(tunnelID)
if err != nil {
return err
}
h.cleanupTunnelRuntime(tunnelID)
// Reserve/apply use stable resource keys and upsert the peer runtime.
// A failed refresh must never release bindings that were already live.
federationBindings, federationReleaseRefs, fedErr := h.applyFederationRuntime(state, h.federationLocalDomain())
newRefs := make([]federationRuntimeReleaseRef, 0, len(federationReleaseRefs))
for _, ref := range federationReleaseRefs {
existing := false
for _, binding := range existingBindings {
if strings.TrimSpace(binding.RemoteURL) == strings.TrimSpace(ref.RemoteURL) && binding.ResourceKey == ref.ResourceKey {
existing = true
break
}
}
if !existing {
newRefs = append(newRefs, ref)
}
}
if fedErr != nil {
return errors.Join(fedErr, h.releaseFederationRuntimeRefs(federationReleaseRefs))
return errors.Join(fedErr, h.releaseFederationRuntimeRefs(newRefs))
}
tx := h.repo.BeginTx()
if tx.Error != nil {
return errors.Join(tx.Error, h.releaseFederationRuntimeRefs(federationReleaseRefs))
return errors.Join(tx.Error, h.releaseFederationRuntimeRefs(newRefs))
}
if replaceErr := h.repo.ReplaceFederationTunnelBindingsTx(tx, tunnelID, federationBindings); replaceErr != nil {
tx.Rollback()
return errors.Join(replaceErr, h.releaseFederationRuntimeRefs(federationReleaseRefs))
return errors.Join(replaceErr, h.releaseFederationRuntimeRefs(newRefs))
}
if commitErr := tx.Commit().Error; commitErr != nil {
return errors.Join(commitErr, h.releaseFederationRuntimeRefs(federationReleaseRefs))
return errors.Join(commitErr, h.releaseFederationRuntimeRefs(newRefs))
}
_, _, applyErr := h.applyTunnelRuntime(state)
_, services, applyErr := h.applyTunnelRuntimeUpsert(state)
applyErr = errors.Join(applyErr, h.acknowledgeTunnelServiceReplacements(tunnelID, services))
applyErr = errors.Join(applyErr, h.retryTunnelRuntimeCleanupLocked(tunnelID))
if applyErr != nil {
applyErr = errors.Join(applyErr, h.cleanupFederationRuntime(tunnelID))
return applyErr
}
}
@@ -3349,6 +3429,7 @@ func (h *Handler) syncPermissionsByTunnelGroup(tunnelGroupID int64) error {
}
type tunnelRuntimeNode struct {
RelayAuth *relaysecurity.Credentials
NodeID int64
Protocol string
Strategy string
@@ -3359,14 +3440,15 @@ type tunnelRuntimeNode struct {
}
type tunnelCreateState struct {
TunnelID int64
Type int
IPPreference string // "" = auto, "v4" = prefer IPv4, "v6" = prefer IPv6
InNodes []tunnelRuntimeNode
ChainHops [][]tunnelRuntimeNode
OutNodes []tunnelRuntimeNode
Nodes map[int64]*nodeRecord
NodeIDList []int64
RelayAuthMode string
TunnelID int64
Type int
IPPreference string // "" = auto, "v4" = prefer IPv4, "v6" = prefer IPv6
InNodes []tunnelRuntimeNode
ChainHops [][]tunnelRuntimeNode
OutNodes []tunnelRuntimeNode
Nodes map[int64]*nodeRecord
NodeIDList []int64
}
func (h *Handler) prepareTunnelCreateState(tx *gorm.DB, req map[string]interface{}, tunnelType int, excludeTunnelID int64) (*tunnelCreateState, error) {
@@ -3540,6 +3622,21 @@ func (h *Handler) prepareTunnelCreateState(tx *gorm.DB, req map[string]interface
}
}
for _, row := range tunnelStateRuntimeRows(state) {
if row.Port <= 0 {
continue
}
ports, err := h.repo.PendingTunnelCleanupPortsTx(tx, row.NodeID, excludeTunnelID)
if err != nil {
return nil, err
}
for _, port := range ports {
if port == row.Port {
return nil, errors.New("节点端口仍在等待旧运行时清理")
}
}
}
return state, nil
}
@@ -3733,6 +3830,10 @@ func (h *Handler) applyFederationRuntime(state *tunnelCreateState, localDomain s
if h == nil || state == nil {
return bindings, releaseRefs, nil
}
if err := h.preflightFederationRelayAuth(state, localDomain); err != nil {
return bindings, releaseRefs, err
}
fc := client.NewFederationClient()
now := time.Now().UnixMilli()
@@ -3768,6 +3869,9 @@ func (h *Handler) applyFederationRuntime(state *tunnelCreateState, localDomain s
ReservationID: reserveRes.ReservationID, ResourceKey: resourceKey,
})
if outNode.RelayAuth != nil && reserveRes.RelayAuthVersion != relaysecurity.Version {
return bindings, releaseRefs, errors.New("remote reservation does not support required relay authentication")
}
state.OutNodes[outIdx].Port = reserveRes.AllocatedPort
outNode = state.OutNodes[outIdx]
@@ -3775,6 +3879,7 @@ func (h *Handler) applyFederationRuntime(state *tunnelCreateState, localDomain s
ReservationID: reserveRes.ReservationID,
ResourceKey: resourceKey,
Role: "exit",
RelayAuth: outNode.RelayAuth,
Protocol: defaultString(outNode.Protocol, "tls"),
Strategy: defaultString(outNode.Strategy, "round"),
}
@@ -3836,6 +3941,9 @@ func (h *Handler) applyFederationRuntime(state *tunnelCreateState, localDomain s
ReservationID: reserveRes.ReservationID, ResourceKey: resourceKey,
})
if chainNode.RelayAuth != nil && reserveRes.RelayAuthVersion != relaysecurity.Version {
return bindings, releaseRefs, errors.New("remote reservation does not support required relay authentication")
}
state.ChainHops[hopIdx][nodeIdx].Port = reserveRes.AllocatedPort
chainNode = state.ChainHops[hopIdx][nodeIdx]
@@ -3859,9 +3967,10 @@ func (h *Handler) applyFederationRuntime(state *tunnelCreateState, localDomain s
return bindings, releaseRefs, errors.New("节点端口不能为空")
}
applyTargets = append(applyTargets, client.RuntimeTarget{
Host: host,
Port: target.Port,
Protocol: defaultString(target.Protocol, "tls"),
Host: host,
RelayAuth: target.RelayAuth,
Port: target.Port,
Protocol: defaultString(target.Protocol, "tls"),
})
}
@@ -3869,6 +3978,7 @@ func (h *Handler) applyFederationRuntime(state *tunnelCreateState, localDomain s
ReservationID: reserveRes.ReservationID,
ResourceKey: resourceKey,
Role: "middle",
RelayAuth: chainNode.RelayAuth,
Protocol: defaultString(chainNode.Protocol, "tls"),
Strategy: runtimeStrategyForTargets(chainNode, nextTargets),
Targets: applyTargets,
@@ -4033,6 +4143,7 @@ func (h *Handler) applyTunnelRuntimeWithMode(state *tunnelCreateState, upsert bo
return createdChains, createdServices, nil
}
var pending []error
for _, inNode := range state.InNodes {
targets := state.OutNodes
if len(state.ChainHops) > 0 {
@@ -4046,6 +4157,7 @@ func (h *Handler) applyTunnelRuntimeWithMode(state *tunnelCreateState, upsert bo
}
if err := h.applyTunnelChainOnNode(inNode.NodeID, chainData, upsert); err != nil {
if shouldDeferTunnelRuntimeApplyError(err) {
pending = append(pending, err)
continue
}
return createdChains, createdServices, fmt.Errorf("入口节点 %s 下发转发链失败: %w", nodeDisplayName(state.Nodes[inNode.NodeID]), err)
@@ -4062,7 +4174,11 @@ func (h *Handler) applyTunnelRuntimeWithMode(state *tunnelCreateState, upsert bo
nextTargets = h.orderBestExitTargets(state.TunnelID, chainNode.NodeID, nextTargets)
}
node := state.Nodes[chainNode.NodeID]
if node != nil && (node.IsRemote == 1 || node.Status != 1) {
if node != nil && node.IsRemote == 1 {
continue
}
if node != nil && node.Status != 1 {
pending = append(pending, fmt.Errorf("节点 %s 不在线", nodeDisplayName(node)))
continue
}
chainData, err := buildTunnelChainConfig(state.TunnelID, chainNode.NodeID, nextTargets, state.Nodes, state.IPPreference)
@@ -4071,6 +4187,7 @@ func (h *Handler) applyTunnelRuntimeWithMode(state *tunnelCreateState, upsert bo
}
if err := h.applyTunnelChainOnNode(chainNode.NodeID, chainData, upsert); err != nil {
if shouldDeferTunnelRuntimeApplyError(err) {
pending = append(pending, err)
continue
}
return createdChains, createdServices, fmt.Errorf("转发链节点 %s 下发转发链失败: %w", nodeDisplayName(state.Nodes[chainNode.NodeID]), err)
@@ -4080,6 +4197,7 @@ func (h *Handler) applyTunnelRuntimeWithMode(state *tunnelCreateState, upsert bo
serviceData := buildTunnelChainServiceConfig(state.TunnelID, chainNode, state.Nodes[chainNode.NodeID], len(nextTargets))
if err := h.addTunnelServiceOnNodeWithMode(chainNode.NodeID, state.TunnelID, serviceData, upsert); err != nil {
if shouldDeferTunnelRuntimeApplyError(err) {
pending = append(pending, err)
continue
}
return createdChains, createdServices, fmt.Errorf("转发链节点 %s 下发服务失败: %w", nodeDisplayName(state.Nodes[chainNode.NodeID]), err)
@@ -4090,12 +4208,17 @@ func (h *Handler) applyTunnelRuntimeWithMode(state *tunnelCreateState, upsert bo
for _, outNode := range state.OutNodes {
node := state.Nodes[outNode.NodeID]
if node != nil && (node.IsRemote == 1 || node.Status != 1) {
if node != nil && node.IsRemote == 1 {
continue
}
if node != nil && node.Status != 1 {
pending = append(pending, fmt.Errorf("节点 %s 不在线", nodeDisplayName(node)))
continue
}
serviceData := buildTunnelChainServiceConfig(state.TunnelID, outNode, state.Nodes[outNode.NodeID], 1)
if err := h.addTunnelServiceOnNodeWithMode(outNode.NodeID, state.TunnelID, serviceData, upsert); err != nil {
if shouldDeferTunnelRuntimeApplyError(err) {
pending = append(pending, err)
continue
}
return createdChains, createdServices, fmt.Errorf("出口节点 %s 下发服务失败: %w", nodeDisplayName(state.Nodes[outNode.NodeID]), err)
@@ -4103,6 +4226,9 @@ func (h *Handler) applyTunnelRuntimeWithMode(state *tunnelCreateState, upsert bo
createdServices = append(createdServices, outNode.NodeID)
}
if len(pending) > 0 {
return createdChains, createdServices, &tunnelRuntimePendingError{cause: errors.Join(pending...)}
}
return createdChains, createdServices, nil
}
@@ -4123,7 +4249,15 @@ func (h *Handler) applyBestExitChainOrder(tunnelID, ownerNodeID int64, outNodes
log.Printf("best_exit: invalid chain update input tunnel=%d owner=%d exits=%d", tunnelID, ownerNodeID, len(outNodes))
return fmt.Errorf("invalid best exit chain update input tunnel=%d owner=%d exits=%d", tunnelID, ownerNodeID, len(outNodes))
}
unlock := h.lockTunnelLifecycle(tunnelID)
defer unlock()
if err := h.validateBestExitSnapshot(tunnelID, ownerNodeID, outNodes, ipPreference); err != nil {
return err
}
targets := chainRecordsToRuntimeTargets(outNodes)
if err := h.resolveBestExitRelayAuth(tunnelID, targets); err != nil {
return err
}
orderedIDs := make([]int64, 0, len(scores))
for _, score := range scores {
if score.ExitNodeID > 0 {
@@ -4204,9 +4338,10 @@ func (h *Handler) applyRemoteBestExitChainOrder(tunnelID, ownerNodeID int64, own
return errors.New("节点端口不能为空")
}
applyTargets = append(applyTargets, client.RuntimeTarget{
Host: host,
Port: target.Port,
Protocol: defaultString(target.Protocol, "tls"),
Host: host,
RelayAuth: target.RelayAuth,
Port: target.Port,
Protocol: defaultString(target.Protocol, "tls"),
})
}
@@ -4228,8 +4363,30 @@ func (h *Handler) applyRemoteBestExitChainOrder(tunnelID, ownerNodeID int64, own
}
}
var credentials *relaysecurity.Credentials
tunnel, loadErr := h.repo.GetTunnelRecord(tunnelID)
if loadErr != nil {
return loadErr
}
if tunnel == nil {
return errors.New("tunnel not found")
}
if tunnel.RelayAuthMode == relaysecurity.Required {
credentials, err = h.repo.RelayEndpointCredentials(nil, tunnelID, ownerNodeID, 2, false)
if err != nil {
return err
}
info, e := client.NewFederationClient().Connect(remoteURL, remoteToken, h.federationLocalDomain())
if e != nil {
return e
}
if info.RelayAuthVersion != relaysecurity.Version {
return errors.New("remote does not support required relay authentication")
}
}
_, err = client.NewFederationClient().ApplyRole(remoteURL, remoteToken, h.federationLocalDomain(), client.RuntimeApplyRoleRequest{
ResourceKey: strings.TrimSpace(binding.ResourceKey),
RelayAuth: credentials,
Role: "middle",
Protocol: defaultString(ownerRuntimeNode.Protocol, "tls"),
Strategy: runtimeStrategyForTargets(ownerRuntimeNode, targets),
@@ -4279,19 +4436,20 @@ func (h *Handler) addTunnelServiceOnNodeWithMode(nodeID, tunnelID int64, service
if h == nil {
return errors.New("invalid tunnel service context")
}
if upsert {
// Update failure must not trigger deletion of an existing listener.
_, err := h.sendNodeCommand(nodeID, "UpdateService", serviceData, true, false)
return err
}
serviceName := fmt.Sprintf("%d_tls", tunnelID)
if len(serviceData) > 0 {
if name, ok := serviceData[0]["name"].(string); ok && strings.TrimSpace(name) != "" {
serviceName = strings.TrimSpace(name)
}
}
command := "AddService"
if upsert {
command = "UpdateService"
}
return retryTunnelServiceAddWithCleanup(
func() error {
_, err := h.sendNodeCommand(nodeID, command, serviceData, true, false)
_, err := h.sendNodeCommand(nodeID, "AddService", serviceData, true, false)
return err
},
func() error {
@@ -4382,6 +4540,12 @@ func buildTunnelChainConfig(tunnelID int64, fromNodeID int64, targets []tunnelRu
connector := map[string]interface{}{
"type": "relay",
}
if target.RelayAuth != nil {
if err := target.RelayAuth.Validate(); err != nil {
return nil, err
}
connector["auth"] = target.RelayAuth.Config()
}
connectorMetadata := map[string]interface{}{}
if isTCPTunnelProtocol(protocol) {
connectorMetadata["nodelay"] = true
@@ -4454,6 +4618,9 @@ func buildTunnelChainServiceConfig(tunnelID int64, chainNode tunnelRuntimeNode,
handlerCfg := map[string]interface{}{
"type": "relay",
}
if chainNode.RelayAuth != nil {
handlerCfg["auth"] = chainNode.RelayAuth.Config()
}
handlerMetadata := map[string]interface{}{}
if isTCPTunnelProtocol(protocol) {
handlerMetadata["nodelay"] = true
@@ -4482,6 +4649,8 @@ func buildTunnelChainServiceConfig(tunnelID int64, chainNode tunnelRuntimeNode,
"listener": buildTunnelListenerConfig(protocol),
}
if chainNode.ChainType == 2 {
handlerMetadata["chain.strict"] = true
handlerCfg["metadata"] = handlerMetadata
service["handler"].(map[string]interface{})["chain"] = fmt.Sprintf("chains_%d", tunnelID)
}
if chainNode.ChainType == 3 && strings.TrimSpace(node.InterfaceName) != "" {
@@ -4781,6 +4950,9 @@ func (h *Handler) deleteTunnelByID(id int64) error {
return err
}
if err := h.retryTunnelRuntimeCleanupLocked(id); err != nil {
return &tunnelDeletionCleanupPendingError{cause: err}
}
shareID, port, ok := parsePeerShareInfoFromFederationTunnelName(tunnelName)
if !ok {
return nil
@@ -5472,6 +5644,15 @@ func (h *Handler) validateForwardPortAvailability(node *nodeRecord, port int, cu
if h == nil || h.repo == nil || node == nil || port <= 0 {
return nil
}
pendingPorts, err := h.repo.PendingTunnelCleanupPortsTx(nil, node.ID, 0)
if err != nil {
return err
}
for _, reserved := range pendingPorts {
if reserved == port {
return fmt.Errorf("节点 %s 端口 %d 仍在等待旧运行时清理", node.Name, port)
}
}
occupied, err := h.repo.HasOtherForwardOnNodePort(node.ID, port, currentForwardID)
if err != nil {
return err
@@ -29,12 +29,19 @@ func (h *Handler) applyPeerShareRoleRuntime(runtime *repo.PeerShareRuntime) erro
if err != nil {
return err
}
credentials, err := decodeRuntimeRelayAuth(runtime)
if err != nil {
return err
}
var targets []federationRuntimeTarget
if strings.TrimSpace(runtime.Target) != "" {
if err := json.Unmarshal([]byte(runtime.Target), &targets); err != nil {
return err
}
}
if err := validateFederationRelayAuth(runtime.RelayAuth, credentials, targets); err != nil {
return err
}
var chain map[string]interface{}
if runtime.Role == "middle" {
chain, err = buildFederationMiddleChainConfig(runtime.ChainName, runtime.ID, runtime.Protocol, runtime.Strategy, targets, node.InterfaceName)
@@ -42,7 +49,7 @@ func (h *Handler) applyPeerShareRoleRuntime(runtime *repo.PeerShareRuntime) erro
return err
}
}
service := buildFederationServiceConfig(runtime.ServiceName, fmt.Sprintf("%s:%d", node.TCPListenAddr, runtime.Port), runtime.Protocol, runtime.Role, runtime.ChainName, len(targets), node.InterfaceName)
service := buildFederationServiceConfig(runtime.ServiceName, fmt.Sprintf("%s:%d", node.TCPListenAddr, runtime.Port), runtime.Protocol, runtime.Role, runtime.ChainName, len(targets), node.InterfaceName, credentials)
runtime.Applied = 0
runtime.UpdatedTime = time.Now().UnixMilli()
if err := h.repo.UpdatePeerShareRuntime(runtime); err != nil {
@@ -145,18 +145,20 @@ func TestPeerShareReleaseOfflineKeepsPortAndReconnectDeletes(t *testing.T) {
}
func TestPeerShareApplyPersistsBeforeCommand(t *testing.T) {
agent := newCleanupAgent(t)
runtime := roleRuntimeFixture(t, agent, "exit")
callback := "test:fail-role-desired-write"
if err := agent.h.repo.DB().Callback().Update().Before("gorm:update").Register(callback, func(tx *gorm.DB) {
if tx.Statement.Table == "peer_share_runtime" {
tx.AddError(errors.New("desired-state write failed"))
var failDesiredWrite atomic.Bool
agent := newCleanupAgent(t, func(r *repo.Repository) {
if err := r.DB().Callback().Update().Before("gorm:update").Register("test:fail-role-desired-write", func(tx *gorm.DB) {
if failDesiredWrite.Load() && tx.Statement.Table == "peer_share_runtime" {
tx.AddError(errors.New("desired-state write failed"))
}
}); err != nil {
t.Fatal(err)
}
}); err != nil {
t.Fatal(err)
}
// This database is test-local; keep the callback registered through the
// websocket teardown so callback mutation cannot race node status writes.
})
runtime := roleRuntimeFixture(t, agent, "exit")
failDesiredWrite.Store(true)
// Only the atomic flag changes while the WebSocket is running; GORM's
// callback registry stays immutable through node-status teardown writes.
if code := roleRuntimeRequest(t, agent.h, `{"reservationId":"role-reservation","role":"exit"}`, false); code == 0 {
t.Fatal("failed ownership persistence reported success")
}
@@ -0,0 +1,125 @@
package handler
import (
"encoding/json"
"fmt"
"go-backend/internal/http/client"
"go-backend/internal/relaysecurity"
"go-backend/internal/store/repo"
"gorm.io/gorm"
)
func (h *Handler) resolveTunnelRelayAuth(tx *gorm.DB, state *tunnelCreateState, create bool) error {
if state.Type != 2 || state.RelayAuthMode == relaysecurity.Legacy {
return nil
}
if state.RelayAuthMode != relaysecurity.Required {
return fmt.Errorf("unsupported relay authentication mode")
}
for i := range state.OutNodes {
c, err := h.repo.RelayEndpointCredentials(tx, state.TunnelID, state.OutNodes[i].NodeID, 3, create)
if err != nil {
return err
}
state.OutNodes[i].RelayAuth = c
}
for hop := range state.ChainHops {
for i := range state.ChainHops[hop] {
c, err := h.repo.RelayEndpointCredentials(tx, state.TunnelID, state.ChainHops[hop][i].NodeID, 2, create)
if err != nil {
return err
}
state.ChainHops[hop][i].RelayAuth = c
}
}
return nil
}
func (h *Handler) resolveBestExitRelayAuth(tunnelID int64, targets []tunnelRuntimeNode) error {
tunnel, err := h.repo.GetTunnelRecord(tunnelID)
if err != nil {
return err
}
if tunnel == nil {
return fmt.Errorf("tunnel not found")
}
return h.resolveTunnelRelayAuth(nil, &tunnelCreateState{TunnelID: tunnelID, Type: tunnel.Type, RelayAuthMode: tunnel.RelayAuthMode, OutNodes: targets}, false)
}
func relayAuthVersion(c *relaysecurity.Credentials) int {
if c == nil {
return 0
}
return c.Version
}
func decodePersistedRelayAuth(raw string) (*relaysecurity.Credentials, error) {
if raw == "" {
return nil, nil
}
var c relaysecurity.Credentials
if err := json.Unmarshal([]byte(raw), &c); err != nil {
return nil, fmt.Errorf("invalid persisted relay authentication")
}
if err := c.Validate(); err != nil {
return nil, err
}
return &c, nil
}
func validateFederationRelayAuth(persisted string, c *relaysecurity.Credentials, targets []federationRuntimeTarget) error {
old, err := decodePersistedRelayAuth(persisted)
if err != nil {
return err
}
if old != nil && (c == nil || *old != *c) {
return fmt.Errorf("authenticated relay cannot be downgraded or rotated by reapply")
}
if c != nil {
if err := c.Validate(); err != nil {
return err
}
for _, target := range targets {
if err := target.RelayAuth.Validate(); err != nil {
return err
}
}
} else {
for _, target := range targets {
if target.RelayAuth != nil {
if err := target.RelayAuth.Validate(); err != nil {
return err
}
}
}
}
return nil
}
func (h *Handler) preflightFederationRelayAuth(state *tunnelCreateState, domain string) error {
if state.RelayAuthMode != relaysecurity.Required || state.Type != 2 {
return nil
}
for _, node := range state.Nodes {
if node == nil || node.IsRemote != 1 {
continue
}
info, err := client.NewFederationClient().Connect(node.RemoteURL, node.RemoteToken, domain)
if err != nil {
return fmt.Errorf("relay authentication capability check failed for remote node %d", node.ID)
}
if info.RelayAuthVersion != relaysecurity.Version {
return fmt.Errorf("remote node %d does not support required relay authentication", node.ID)
}
}
return nil
}
func decodeRuntimeRelayAuth(runtime *repo.PeerShareRuntime) (*relaysecurity.Credentials, error) {
c, err := decodePersistedRelayAuth(runtime.RelayAuth)
if err != nil {
return nil, err
}
if runtime.RelayAuthVersion != relayAuthVersion(c) {
return nil, fmt.Errorf("persisted relay authentication version or credentials unavailable")
}
return c, nil
}
@@ -371,6 +371,8 @@ func summarizeTunnelDeleteRuleFailures(failures []batchFailureDetail) string {
}
func (h *Handler) processTunnelDeleteWithForwards(tunnelID int64, action string, targetTunnelID int64) (tunnelDeleteWithForwardsResult, []batchFailureDetail, error) {
unlock := h.lockTunnelLifecycle(tunnelID, targetTunnelID)
defer unlock()
preview, err := h.buildTunnelDeletePreview(tunnelID)
if err != nil {
return tunnelDeleteWithForwardsResult{}, nil, err
@@ -431,9 +433,14 @@ func (h *Handler) processTunnelDeleteReplaceAction(tunnelID, targetTunnelID int6
}
if err := h.deleteTunnelAndCleanup(tunnelID); err != nil {
h.rollbackTunnelForwardMigrationPlans(plans)
_ = h.redeployTunnelAndForwards(tunnelID)
return tunnelDeleteWithForwardsResult{}, nil, err
var pending *tunnelDeletionCleanupPendingError
if errors.As(err, &pending) {
warnings = append(warnings, err.Error())
} else {
h.rollbackTunnelForwardMigrationPlans(plans)
_ = h.redeployTunnelAndForwardsLocked(tunnelID)
return tunnelDeleteWithForwardsResult{}, nil, err
}
}
result.MigratedCount = len(plans)
@@ -639,8 +646,9 @@ func (h *Handler) rollbackTunnelForwardMigrationPlans(plans []tunnelForwardMigra
}
func (h *Handler) deleteTunnelAndCleanup(tunnelID int64) error {
h.cleanupTunnelRuntime(tunnelID)
h.cleanupFederationRuntime(tunnelID)
if err := h.cleanupFederationRuntime(tunnelID); err != nil {
return err
}
if err := h.deleteTunnelByID(tunnelID); err != nil {
return err
}
@@ -0,0 +1,153 @@
package handler
import (
"database/sql"
"errors"
"go-backend/internal/http/response"
"gorm.io/gorm"
"slices"
"sort"
"sync"
)
type tunnelLifecycleLock struct {
mu sync.Mutex
users int
}
// lockTunnelLifecycle serializes snapshots, persistence and node acknowledgements.
// Multiple tunnels (forward migration) are locked in ID order to avoid deadlocks.
// Callers must not acquire these locks recursively or hold a database transaction.
func (h *Handler) lockTunnelLifecycle(ids ...int64) func() {
unique := make(map[int64]bool, len(ids))
ordered := make([]int64, 0, len(ids))
for _, id := range ids {
if id > 0 && !unique[id] {
unique[id] = true
ordered = append(ordered, id)
}
}
sort.Slice(ordered, func(i, j int) bool { return ordered[i] < ordered[j] })
locks := make([]*tunnelLifecycleLock, 0, len(ordered))
for _, id := range ordered {
h.tunnelLifecycleMu.Lock()
if h.tunnelLifecycleLocks == nil {
h.tunnelLifecycleLocks = make(map[int64]*tunnelLifecycleLock)
}
lock := h.tunnelLifecycleLocks[id]
if lock == nil {
lock = &tunnelLifecycleLock{}
h.tunnelLifecycleLocks[id] = lock
}
lock.users++
h.tunnelLifecycleMu.Unlock()
lock.mu.Lock()
locks = append(locks, lock)
}
return func() {
for i := len(locks) - 1; i >= 0; i-- {
h.tunnelLifecycleMu.Lock()
locks[i].mu.Unlock()
locks[i].users--
if locks[i].users == 0 {
delete(h.tunnelLifecycleLocks, ordered[i])
}
h.tunnelLifecycleMu.Unlock()
}
}
}
// Probes run without the lifecycle lock. Recheck their topology under the lock
// so a late result cannot restore a removed middle hop or deleted tunnel.
func (h *Handler) validateBestExitSnapshot(tunnelID, ownerNodeID int64, exits []chainNodeRecord, ipPreference string) error {
tunnel, err := h.getTunnelRecord(tunnelID)
if err != nil {
return err
}
if tunnel.Type != 2 || h.repo.GetTunnelIPPreference(tunnelID) != ipPreference {
return errors.New("best exit topology changed; retry next probe")
}
rows, err := h.listChainNodesForTunnel(tunnelID)
if err != nil {
return err
}
ins, hops, currentExits := splitChainNodeGroups(rows)
if !slices.Equal(exits, currentExits) {
return errors.New("best exit targets changed; retry next probe")
}
for _, owner := range bestExitChainOwners(ins, hops) {
if owner.NodeID == ownerNodeID {
return nil
}
}
return errors.New("best exit owner changed; retry next probe")
}
// A newly allocated ID is not visible yet. Claim it without waiting while the
// insert transaction is open: an overlapping request for that ID must not make
// creation wait on a lifecycle lock whose owner is waiting for this transaction.
func (h *Handler) tryLockNewTunnelLifecycle(id int64) (func(), bool) {
h.tunnelLifecycleMu.Lock()
defer h.tunnelLifecycleMu.Unlock()
if h.tunnelLifecycleLocks == nil {
h.tunnelLifecycleLocks = make(map[int64]*tunnelLifecycleLock)
}
if h.tunnelLifecycleLocks[id] != nil {
return nil, false
}
lock := &tunnelLifecycleLock{users: 1}
lock.mu.Lock()
h.tunnelLifecycleLocks[id] = lock
return func() {
h.tunnelLifecycleMu.Lock()
defer h.tunnelLifecycleMu.Unlock()
lock.mu.Unlock()
lock.users--
if lock.users == 0 {
delete(h.tunnelLifecycleLocks, id)
}
}, true
}
// Pending is not successful application. HTTP create/update retain their
// accepted-config contract, while redeploy returns this error to its batch result.
type tunnelRuntimePendingError struct{ cause error }
func (e *tunnelRuntimePendingError) Error() string { return "运行时待同步: " + e.cause.Error() }
func isTunnelRuntimePending(err error) bool {
var pending *tunnelRuntimePendingError
return errors.As(err, &pending)
}
func tunnelRuntimePendingResponse(tunnelID int64, err error) response.R {
message := "配置已保存,部分节点运行时待同步:" + err.Error()
out := response.OK(map[string]interface{}{"runtimeStatus": "pending", "tunnelId": tunnelID, "message": message})
out.Msg = message
return out
}
// Reserve locally allocated ports with the initial desired state, before any
// remote acknowledgment. Remote ports remain unallocated until their provider
// returns; the second transaction replaces these rows with the final bindings.
func (h *Handler) persistPreparedTunnelChainsTx(tx *gorm.DB, state *tunnelCreateState) error {
save := func(node tunnelRuntimeNode, kind string, inx int) error {
return h.repo.CreateChainTunnelTx(tx, state.TunnelID, kind, node.NodeID, sql.NullInt64{Int64: int64(node.Port), Valid: node.Port > 0}, defaultString(node.Strategy, "round"), inx, defaultString(node.Protocol, "tls"), node.ConnectIP)
}
for i, node := range state.InNodes {
if err := save(node, "1", i+1); err != nil {
return err
}
}
for i, node := range state.OutNodes {
if err := save(node, "3", i+1); err != nil {
return err
}
}
for i, hop := range state.ChainHops {
for _, node := range hop {
if err := save(node, "2", i+1); err != nil {
return err
}
}
}
return nil
}
@@ -0,0 +1,180 @@
package handler
import (
"errors"
"fmt"
"go-backend/internal/store/model"
"gorm.io/gorm"
)
func tunnelStateRuntimeRows(state *tunnelCreateState) []chainNodeRecord {
if state.Type != 2 {
return nil
}
var rows []chainNodeRecord
for _, n := range state.InNodes {
rows = append(rows, chainNodeRecord{NodeID: n.NodeID, ChainType: 1})
}
for _, hop := range state.ChainHops {
for _, n := range hop {
rows = append(rows, chainNodeRecord{NodeID: n.NodeID, ChainType: 2, Port: n.Port})
}
}
for _, n := range state.OutNodes {
rows = append(rows, chainNodeRecord{NodeID: n.NodeID, ChainType: 3, Port: n.Port})
}
return rows
}
func (h *Handler) queueObsoleteTunnelRuntimeTx(tx *gorm.DB, tunnelID int64, oldRows, newRows []chainNodeRecord) error {
var items []model.TunnelRuntimeCleanup
for _, nodeID := range removedTunnelRuntimeNodeIDs(oldRows, newRows, tunnelRuntimeNeedsService) {
port := 0
for _, row := range oldRows {
if row.NodeID == nodeID {
port = row.Port
break
}
}
items = append(items, model.TunnelRuntimeCleanup{TunnelID: tunnelID, NodeID: nodeID, Kind: "service", Port: port})
}
for _, old := range oldRows {
if !tunnelRuntimeNeedsService(old) {
continue
}
for _, current := range newRows {
if current.NodeID == old.NodeID && tunnelRuntimeNeedsService(current) && (current.Port != old.Port || current.ChainType != old.ChainType) {
items = append(items, model.TunnelRuntimeCleanup{TunnelID: tunnelID, NodeID: old.NodeID, Kind: "service", Port: old.Port})
}
}
}
for _, nodeID := range removedTunnelRuntimeNodeIDs(oldRows, newRows, tunnelRuntimeNeedsChain) {
items = append(items, model.TunnelRuntimeCleanup{TunnelID: tunnelID, NodeID: nodeID, Kind: "chain"})
}
return h.repo.QueueTunnelRuntimeCleanupTx(tx, items)
}
// Caller holds the tunnel lifecycle lock. Recheck current desired roles before
// every retry so a delayed deletion cannot remove a re-added runtime.
func (h *Handler) retryTunnelRuntimeCleanupLocked(tunnelID int64) error {
items, err := h.repo.ListTunnelRuntimeCleanup(tunnelID)
if err != nil {
return err
}
if len(items) == 0 {
return nil
}
tunnel, err := h.repo.GetTunnelRecord(tunnelID)
if err != nil {
return err
}
var rows []chainNodeRecord
if tunnel != nil && tunnel.Type == 2 {
rows, err = h.listChainNodesForTunnel(tunnelID)
if err != nil {
return err
}
}
var failures []error
liveServices := map[int64]bool{}
for _, item := range items {
desired := false
for _, row := range rows {
if row.NodeID == item.NodeID && ((item.Kind == "service" && tunnelRuntimeNeedsService(row)) || (item.Kind == "chain" && tunnelRuntimeNeedsChain(row))) {
desired = true
break
}
}
if desired && item.Kind == "service" {
// Desired state does not prove that the old receiver was replaced.
// Keep this barrier (even at the same port) and protect its old
// chain until acknowledgeTunnelServiceReplacements observes ACK.
liveServices[item.NodeID] = true
continue
}
if !desired {
if item.Kind == "chain" && liveServices[item.NodeID] {
continue
}
var commandErr error
if item.Kind == "service" {
_, commandErr = h.sendNodeCommand(item.NodeID, "DeleteService", map[string]interface{}{"services": tunnelRuntimeServiceNames(tunnelID)}, false, true)
} else {
_, commandErr = h.sendNodeCommand(item.NodeID, "DeleteChains", map[string]interface{}{"chain": fmt.Sprintf("chains_%d", tunnelID)}, false, true)
}
if commandErr != nil {
if item.Kind == "service" {
liveServices[item.NodeID] = true
}
failures = append(failures, fmt.Errorf("节点 %d 旧运行时清理失败: %w", item.NodeID, commandErr))
continue
}
}
if err := h.repo.DeleteTunnelRuntimeCleanup(item.ID); err != nil {
if item.Kind == "service" {
liveServices[item.NodeID] = true
}
failures = append(failures, err)
}
}
return errors.Join(failures...)
}
func (h *Handler) acknowledgeTunnelServiceReplacements(tunnelID int64, nodeIDs []int64) error {
if len(nodeIDs) == 0 {
return nil
}
items, err := h.repo.ListTunnelRuntimeCleanup(tunnelID)
if err != nil {
return err
}
for _, item := range items {
if item.Kind == "service" {
for _, nodeID := range nodeIDs {
if item.NodeID == nodeID {
if err := h.repo.DeleteTunnelRuntimeCleanup(item.ID); err != nil {
return err
}
break
}
}
}
}
return nil
}
func (h *Handler) retryPendingTunnelRuntimeCleanup() error {
return h.retryPendingTunnelRuntimeCleanupOnNode(0)
}
func (h *Handler) retryPendingTunnelRuntimeCleanupOnNode(nodeID int64) error {
items, err := h.repo.ListTunnelRuntimeCleanup(0)
if err != nil {
return err
}
seen := map[int64]bool{}
var failures []error
for _, item := range items {
if nodeID > 0 && item.NodeID != nodeID {
continue
}
if seen[item.TunnelID] {
continue
}
seen[item.TunnelID] = true
unlock := h.lockTunnelLifecycle(item.TunnelID)
err := h.retryTunnelRuntimeCleanupLocked(item.TunnelID)
unlock()
if err != nil {
failures = append(failures, err)
}
}
return errors.Join(failures...)
}
// The desired tunnel has already been deleted; callers must not roll back a
// completed forward migration merely because asynchronous cleanup is pending.
type tunnelDeletionCleanupPendingError struct{ cause error }
func (e *tunnelDeletionCleanupPendingError) Error() string {
return "隧道配置已删除,运行时清理待重试: " + e.cause.Error()
}
func (e *tunnelDeletionCleanupPendingError) Unwrap() error { return e.cause }
+5 -1
View File
@@ -411,6 +411,9 @@ func (h *Handler) onNodeOnline(nodeID int64) {
// A fresh agent needs a full restore. Shared failures remain persisted and
// are retried by maintenance without restarting acknowledged services.
if err := h.retryPendingTunnelRuntimeCleanupOnNode(nodeID); err != nil {
fmt.Printf("reconnect obsolete tunnel cleanup remains pending: %v\n", err)
}
h.reconcileSharedNodeRuntime(nodeID)
if !h.redeployLocalNodeRuntime(nodeID) {
h.scheduleNodeLocalRuntimeRetry(nodeID)
@@ -568,9 +571,10 @@ func (h *Handler) finishNodeOnlineRedeploy(nodeID int64) {
}
func (h *Handler) redeployNodeRuntimeAfterUpgrade(nodeID int64) bool {
cleanupOK := h.retryPendingTunnelRuntimeCleanupOnNode(nodeID) == nil
sharedOK := h.reconcileSharedNodeRuntime(nodeID)
localOK := h.redeployLocalNodeRuntime(nodeID)
return sharedOK && localOK
return cleanupOK && sharedOK && localOK
}
func (h *Handler) reconcileSharedNodeRuntime(nodeID int64) bool {