Merge pull request #14 from Sagit-chu/opencode/cosmic-pixel

fix(gost): implement failover for multi-node forwarding rules
This commit is contained in:
sagit
2026-02-04 12:30:49 +08:00
committed by GitHub
8 changed files with 396 additions and 202 deletions
+20
View File
@@ -109,3 +109,23 @@ func LoggerFromContext(ctx context.Context) logger.Logger {
v, _ := ctx.Value(keyLogger).(logger.Logger) v, _ := ctx.Value(keyLogger).(logger.Logger)
return v return v
} }
// excludeNodesKey saves the list of node addresses to exclude during selection.
// This is used for failover retry logic - when a node fails, it gets added to
// the exclude list so the next Select() call will skip it.
type excludeNodesKey struct{}
var (
keyExcludeNodes = &excludeNodesKey{}
)
// ContextWithExcludeNodes returns a context with the list of node addresses to exclude.
func ContextWithExcludeNodes(ctx context.Context, nodes []string) context.Context {
return context.WithValue(ctx, keyExcludeNodes, nodes)
}
// ExcludeNodesFromContext returns the list of node addresses to exclude from selection.
func ExcludeNodesFromContext(ctx context.Context) []string {
v, _ := ctx.Value(keyExcludeNodes).([]string)
return v
}
+69 -36
View File
@@ -176,51 +176,84 @@ func (h *forwardHandler) Handle(ctx context.Context, conn net.Conn, opts ...hand
} }
} }
target := &chain.Node{} // Determine max retry attempts
if h.hop != nil { maxRetries := h.md.maxRetries
target = h.hop.Select(ctx, if maxRetries <= 0 {
hop.ProtocolSelectOption(proto), // Default: try all available nodes
) if nl, ok := h.hop.(hop.NodeList); ok {
} maxRetries = len(nl.Nodes())
if target == nil { }
err := errors.New("node not available") if maxRetries <= 0 {
return err maxRetries = 1
}
} }
addr := target.Addr var triedNodes []string
if opts := target.Options(); opts != nil { var lastErr error
switch opts.Network { var cc net.Conn
case "unix":
network = opts.Network for attempt := 0; attempt < maxRetries; attempt++ {
default: // Select a target node, excluding previously tried nodes
if _, _, err := net.SplitHostPort(addr); err != nil { selectCtx := ctxvalue.ContextWithExcludeNodes(ctx, triedNodes)
addr += ":0" target := &chain.Node{}
if h.hop != nil {
target = h.hop.Select(selectCtx,
hop.ProtocolSelectOption(proto),
)
}
if target == nil {
if lastErr != nil {
return lastErr
}
return errors.New("node not available")
}
// Track this node as tried
triedNodes = append(triedNodes, target.Addr)
addr := target.Addr
if opts := target.Options(); opts != nil {
switch opts.Network {
case "unix":
network = opts.Network
default:
if _, _, err := net.SplitHostPort(addr); err != nil {
addr += ":0"
}
} }
} }
}
ro.Network = network ro.Network = network
ro.Host = addr ro.Host = addr
var buf bytes.Buffer var buf bytes.Buffer
cc, err := h.options.Router.Dial(ctxvalue.ContextWithBuffer(ctx, &buf), network, addr) cc, err = h.options.Router.Dial(ctxvalue.ContextWithBuffer(ctx, &buf), network, addr)
ro.Route = buf.String() ro.Route = buf.String()
if err != nil { if err != nil {
// TODO: the router itself may be failed due to the failed node in the router, // Mark node as failed for future selections
// the dead marker may be a wrong operation. if marker := target.Marker(); marker != nil {
if marker := target.Marker(); marker != nil { marker.Mark()
marker.Mark() }
lastErr = err
// Try next node
continue
} }
return err
}
if marker := target.Marker(); marker != nil {
marker.Reset()
}
defer cc.Close()
xnet.Transport(conn, cc) // Success - reset marker and proceed
if marker := target.Marker(); marker != nil {
marker.Reset()
}
defer cc.Close()
return nil xnet.Transport(conn, cc)
return nil
}
// All retries exhausted
if lastErr != nil {
return lastErr
}
return errors.New("all nodes failed")
} }
func (h *forwardHandler) checkRateLimit(addr net.Addr) bool { func (h *forwardHandler) checkRateLimit(addr net.Addr) bool {
@@ -25,6 +25,12 @@ type metadata struct {
privateKey crypto.PrivateKey privateKey crypto.PrivateKey
alpn string alpn string
mitmBypass bypass.Bypass mitmBypass bypass.Bypass
// maxRetries specifies the maximum number of failover retry attempts.
// When a target node fails, the handler will try the next available node.
// 0 means use the total number of available nodes (try all nodes once).
// Default: 0 (try all available nodes)
maxRetries int
} }
func (h *forwardHandler) parseMetadata(md mdata.Metadata) (err error) { func (h *forwardHandler) parseMetadata(md mdata.Metadata) (err error) {
@@ -56,5 +62,8 @@ func (h *forwardHandler) parseMetadata(md mdata.Metadata) (err error) {
h.md.alpn = mdutil.GetString(md, "mitm.alpn") h.md.alpn = mdutil.GetString(md, "mitm.alpn")
h.md.mitmBypass = registry.BypassRegistry().Get(mdutil.GetString(md, "mitm.bypass")) h.md.mitmBypass = registry.BypassRegistry().Get(mdutil.GetString(md, "mitm.bypass"))
// maxRetries: 0 means try all available nodes (default behavior)
h.md.maxRetries = mdutil.GetInt(md, "maxRetries", "retry.max")
return return
} }
+86 -51
View File
@@ -204,68 +204,103 @@ func (h *forwardHandler) Handle(ctx context.Context, conn net.Conn, opts ...hand
} }
} }
var target *chain.Node // Determine max retry attempts
if host != "" { maxRetries := h.md.maxRetries
target = &chain.Node{ if maxRetries <= 0 {
Addr: host, // Default: try all available nodes
if nl, ok := h.hop.(hop.NodeList); ok {
maxRetries = len(nl.Nodes())
} }
} if maxRetries <= 0 {
if h.hop != nil { maxRetries = 1
target = h.hop.Select(ctx,
hop.ProtocolSelectOption(proto),
)
}
if target == nil {
err := errors.New("node not available")
log.Error(err)
return err
}
if opts := target.Options(); opts != nil {
switch opts.Network {
case "unix":
network = opts.Network
default:
} }
} }
ro.Network = network var triedNodes []string
ro.Host = target.Addr var lastErr error
var cc net.Conn
log = log.WithFields(map[string]any{ for attempt := 0; attempt < maxRetries; attempt++ {
"node": target.Name, // Select a target node, excluding previously tried nodes
"dst": fmt.Sprintf("%s/%s", target.Addr, network), selectCtx := ctxvalue.ContextWithExcludeNodes(ctx, triedNodes)
}) var target *chain.Node
if host != "" {
target = &chain.Node{
Addr: host,
}
}
if h.hop != nil {
target = h.hop.Select(selectCtx,
hop.ProtocolSelectOption(proto),
)
}
if target == nil {
if lastErr != nil {
return lastErr
}
err := errors.New("node not available")
log.Error(err)
return err
}
log.Debugf("%s >> %s", conn.RemoteAddr(), target.Addr) // Track this node as tried
triedNodes = append(triedNodes, target.Addr)
var buf bytes.Buffer if opts := target.Options(); opts != nil {
cc, err := h.options.Router.Dial(ctxvalue.ContextWithBuffer(ctx, &buf), network, target.Addr) switch opts.Network {
ro.Route = buf.String() case "unix":
if err != nil { network = opts.Network
log.Error(err) default:
// TODO: the router itself may be failed due to the failed node in the router, }
// the dead marker may be a wrong operation. }
ro.Network = network
ro.Host = target.Addr
targetLog := log.WithFields(map[string]any{
"node": target.Name,
"dst": fmt.Sprintf("%s/%s", target.Addr, network),
})
targetLog.Debugf("%s >> %s", conn.RemoteAddr(), target.Addr)
var buf bytes.Buffer
cc, err = h.options.Router.Dial(ctxvalue.ContextWithBuffer(ctx, &buf), network, target.Addr)
ro.Route = buf.String()
if err != nil {
targetLog.Error(err)
// Mark node as failed for future selections
if marker := target.Marker(); marker != nil {
marker.Mark()
}
lastErr = err
// Try next node
continue
}
// Success - reset marker and proceed
if marker := target.Marker(); marker != nil { if marker := target.Marker(); marker != nil {
marker.Mark() marker.Reset()
} }
return err defer cc.Close()
}
defer cc.Close() cc = proxyproto.WrapClientConn(h.md.proxyProtocol, conn.RemoteAddr(), convertAddr(conn.LocalAddr()), cc)
if marker := target.Marker(); marker != nil {
marker.Reset() t := time.Now()
targetLog.Infof("%s <-> %s", conn.RemoteAddr(), target.Addr)
xnet.Transport(conn, cc)
targetLog.WithFields(map[string]any{
"duration": time.Since(t),
}).Infof("%s >-< %s", conn.RemoteAddr(), target.Addr)
return nil
} }
cc = proxyproto.WrapClientConn(h.md.proxyProtocol, conn.RemoteAddr(), convertAddr(conn.LocalAddr()), cc) // All retries exhausted
if lastErr != nil {
t := time.Now() return lastErr
log.Infof("%s <-> %s", conn.RemoteAddr(), target.Addr) }
xnet.Transport(conn, cc) return errors.New("all nodes failed")
log.WithFields(map[string]any{
"duration": time.Since(t),
}).Infof("%s >-< %s", conn.RemoteAddr(), target.Addr)
return nil
} }
func (h *forwardHandler) checkRateLimit(addr net.Addr) bool { func (h *forwardHandler) checkRateLimit(addr net.Addr) bool {
@@ -26,6 +26,12 @@ type metadata struct {
privateKey crypto.PrivateKey privateKey crypto.PrivateKey
alpn string alpn string
mitmBypass bypass.Bypass mitmBypass bypass.Bypass
// maxRetries specifies the maximum number of failover retry attempts.
// When a target node fails, the handler will try the next available node.
// 0 means use the total number of available nodes (try all nodes once).
// Default: 0 (try all available nodes)
maxRetries int
} }
func (h *forwardHandler) parseMetadata(md mdata.Metadata) (err error) { func (h *forwardHandler) parseMetadata(md mdata.Metadata) (err error) {
@@ -57,5 +63,9 @@ func (h *forwardHandler) parseMetadata(md mdata.Metadata) (err error) {
} }
h.md.alpn = mdutil.GetString(md, "mitm.alpn") h.md.alpn = mdutil.GetString(md, "mitm.alpn")
h.md.mitmBypass = registry.BypassRegistry().Get(mdutil.GetString(md, "mitm.bypass")) h.md.mitmBypass = registry.BypassRegistry().Get(mdutil.GetString(md, "mitm.bypass"))
// maxRetries: 0 means try all available nodes (default behavior)
h.md.maxRetries = mdutil.GetInt(md, "maxRetries", "retry.max")
return return
} }
+20 -3
View File
@@ -18,6 +18,7 @@ import (
"github.com/go-gost/core/selector" "github.com/go-gost/core/selector"
"github.com/go-gost/x/config" "github.com/go-gost/x/config"
node_parser "github.com/go-gost/x/config/parsing/node" node_parser "github.com/go-gost/x/config/parsing/node"
ctxvalue "github.com/go-gost/x/ctx"
"github.com/go-gost/x/internal/loader" "github.com/go-gost/x/internal/loader"
) )
@@ -141,11 +142,25 @@ func (p *chainHop) Select(ctx context.Context, opts ...hop.SelectOption) *chain.
return nil return nil
} }
// Get list of nodes to exclude (for failover retry)
excludeNodes := ctxvalue.ExcludeNodesFromContext(ctx)
excludeSet := make(map[string]bool)
for _, addr := range excludeNodes {
excludeSet[addr] = true
}
var nodes []*chain.Node var nodes []*chain.Node
for _, node := range p.Nodes() { for _, node := range p.Nodes() {
if node == nil { if node == nil {
continue continue
} }
// Skip nodes in the exclude list (failover retry)
if excludeSet[node.Addr] || excludeSet[node.Name] {
log.Debugf("node %s(%s) excluded for failover retry", node.Name, node.Addr)
continue
}
// node level bypass // node level bypass
if node.Options().Bypass != nil && if node.Options().Bypass != nil &&
node.Options().Bypass.Contains(ctx, options.Network, options.Addr, bypass.WithHostOpton(options.Host)) { node.Options().Bypass.Contains(ctx, options.Network, options.Addr, bypass.WithHostOpton(options.Host)) {
@@ -177,9 +192,6 @@ func (p *chainHop) Select(ctx context.Context, opts ...hop.SelectOption) *chain.
if len(nodes) == 0 { if len(nodes) == 0 {
return nil return nil
} }
if len(nodes) == 1 {
return nodes[0]
}
sort.Slice(nodes, func(i, j int) bool { sort.Slice(nodes, func(i, j int) bool {
return nodes[i].Options().Priority > nodes[j].Options().Priority return nodes[i].Options().Priority > nodes[j].Options().Priority
@@ -189,9 +201,14 @@ func (p *chainHop) Select(ctx context.Context, opts ...hop.SelectOption) *chain.
return nodes[0] return nodes[0]
} }
// Always go through selector for proper FailFilter evaluation,
// even when there's only one node. This ensures failed nodes
// can be filtered out properly.
if s := p.options.selector; s != nil { if s := p.options.selector; s != nil {
return s.Select(ctx, nodes...) return s.Select(ctx, nodes...)
} }
// Fallback: return first node if no selector configured
return nodes[0] return nodes[0]
} }
+177 -108
View File
@@ -247,64 +247,100 @@ func (h *Sniffer) dial(ctx context.Context, conn net.Conn, req *http.Request, ho
} }
} }
node = &chain.Node{ // Determine max retry attempts
Addr: host, maxRetries := 1
if nl, ok := ho.Hop.(hop.NodeList); ok {
maxRetries = len(nl.Nodes())
} }
if ho.Hop != nil { if maxRetries <= 0 {
node = ho.Hop.Select(ctx, maxRetries = 1
hop.ClientIPSelectOption(net.ParseIP(ro.ClientIP)),
hop.ProtocolSelectOption(sniffing.ProtoHTTP),
hop.HostSelectOption(host),
hop.MethodSelectOption(req.Method),
hop.PathSelectOption(req.URL.Path),
hop.QuerySelectOption(req.URL.Query()),
hop.HeaderSelectOption(req.Header),
)
}
if node == nil {
ho.Log.Warnf("node for %s not found", host)
res.StatusCode = http.StatusBadGateway
ro.HTTP.StatusCode = res.StatusCode
res.Write(conn)
return nil, nil, errors.New("node not available")
} }
ro.Host = node.Addr var triedNodes []string
ho.Log = ho.Log.WithFields(map[string]any{ var lastErr error
"node": node.Name,
"dst": node.Addr,
})
ho.Log.Debugf("find node for host %s -> %s(%s)", host, node.Name, node.Addr)
cc, err = dial(ctx, "tcp", node.Addr) for attempt := 0; attempt < maxRetries; attempt++ {
if err != nil { // Select a node, excluding previously tried nodes
// TODO: the router itself may be failed due to the failed node in the router, selectCtx := ctxvalue.ContextWithExcludeNodes(ctx, triedNodes)
// the dead marker may be a wrong operation.
if marker := node.Marker(); marker != nil { node = &chain.Node{
marker.Mark() Addr: host,
} }
ho.Log.Warnf("connect to node %s(%s) failed: %v", node.Name, node.Addr, err) if ho.Hop != nil {
res.Write(conn) node = ho.Hop.Select(selectCtx,
return hop.ClientIPSelectOption(net.ParseIP(ro.ClientIP)),
} hop.ProtocolSelectOption(sniffing.ProtoHTTP),
if marker := node.Marker(); marker != nil { hop.HostSelectOption(host),
marker.Reset() hop.MethodSelectOption(req.Method),
} hop.PathSelectOption(req.URL.Path),
hop.QuerySelectOption(req.URL.Query()),
if tlsSettings := node.Options().TLS; tlsSettings != nil { hop.HeaderSelectOption(req.Header),
cfg := &tls.Config{ )
ServerName: tlsSettings.ServerName,
InsecureSkipVerify: !tlsSettings.Secure,
} }
tls_util.SetTLSOptions(cfg, &config.TLSOptions{ if node == nil {
MinVersion: tlsSettings.Options.MinVersion, if lastErr != nil {
MaxVersion: tlsSettings.Options.MaxVersion, ho.Log.Warnf("node for %s not found after retries", host)
CipherSuites: tlsSettings.Options.CipherSuites, res.StatusCode = http.StatusBadGateway
ALPN: tlsSettings.Options.ALPN, ro.HTTP.StatusCode = res.StatusCode
res.Write(conn)
return nil, nil, lastErr
}
ho.Log.Warnf("node for %s not found", host)
res.StatusCode = http.StatusBadGateway
ro.HTTP.StatusCode = res.StatusCode
res.Write(conn)
return nil, nil, errors.New("node not available")
}
// Track this node as tried
triedNodes = append(triedNodes, node.Addr)
ro.Host = node.Addr
ho.Log = ho.Log.WithFields(map[string]any{
"node": node.Name,
"dst": node.Addr,
}) })
cc = tls.Client(cc, cfg) ho.Log.Debugf("find node for host %s -> %s(%s)", host, node.Name, node.Addr)
cc, err = dial(ctx, "tcp", node.Addr)
if err != nil {
// Mark node as failed for future selections
if marker := node.Marker(); marker != nil {
marker.Mark()
}
ho.Log.Warnf("connect to node %s(%s) failed: %v, trying next node", node.Name, node.Addr, err)
lastErr = err
continue
}
// Success - reset marker
if marker := node.Marker(); marker != nil {
marker.Reset()
}
if tlsSettings := node.Options().TLS; tlsSettings != nil {
cfg := &tls.Config{
ServerName: tlsSettings.ServerName,
InsecureSkipVerify: !tlsSettings.Secure,
}
tls_util.SetTLSOptions(cfg, &config.TLSOptions{
MinVersion: tlsSettings.Options.MinVersion,
MaxVersion: tlsSettings.Options.MaxVersion,
CipherSuites: tlsSettings.Options.CipherSuites,
ALPN: tlsSettings.Options.ALPN,
})
cc = tls.Client(cc, cfg)
}
return node, cc, nil
} }
return
// All retries exhausted
ho.Log.Warnf("all nodes failed for host %s", host)
res.Write(conn)
if lastErr != nil {
return nil, nil, lastErr
}
return nil, nil, errors.New("all nodes failed")
} }
func (h *Sniffer) serveH2(ctx context.Context, conn net.Conn, ho *HandleOptions) error { func (h *Sniffer) serveH2(ctx context.Context, conn net.Conn, ho *HandleOptions) error {
@@ -847,74 +883,107 @@ func (h *Sniffer) dialTLS(ctx context.Context, host string, ho *HandleOptions) (
return return
} }
if host != "" {
node = &chain.Node{
Addr: host,
}
}
ro := ho.RecorderObject ro := ho.RecorderObject
if ho.Hop != nil {
node = ho.Hop.Select(ctx, // Determine max retry attempts
hop.ClientIPSelectOption(net.ParseIP(ro.ClientIP)), maxRetries := 1
hop.HostSelectOption(host), if nl, ok := ho.Hop.(hop.NodeList); ok {
hop.ProtocolSelectOption(sniffing.ProtoTLS), maxRetries = len(nl.Nodes())
)
} }
if node == nil { if maxRetries <= 0 {
err = errors.New("node not available") maxRetries = 1
return
} }
addr := node.Addr var triedNodes []string
if opts := node.Options(); opts != nil { var lastErr error
switch opts.Network {
case "unix": for attempt := 0; attempt < maxRetries; attempt++ {
ro.Network = opts.Network // Select a node, excluding previously tried nodes
default: selectCtx := ctxvalue.ContextWithExcludeNodes(ctx, triedNodes)
if _, _, err := net.SplitHostPort(addr); err != nil {
addr += ":443" node = nil
if host != "" {
node = &chain.Node{
Addr: host,
} }
} }
} if ho.Hop != nil {
ro.Host = addr node = ho.Hop.Select(selectCtx,
hop.ClientIPSelectOption(net.ParseIP(ro.ClientIP)),
ho.Log = ho.Log.WithFields(map[string]any{ hop.HostSelectOption(host),
"host": host, hop.ProtocolSelectOption(sniffing.ProtoTLS),
"node": node.Name, )
"dst": fmt.Sprintf("%s/%s", addr, ro.Network),
})
ho.Log.Debugf("find node for host %s -> %s(%s)", host, node.Name, addr)
cc, err = dial(ctx, ro.Network, addr)
if err != nil {
// TODO: the router itself may be failed due to the failed node in the router,
// the dead marker may be a wrong operation.
if marker := node.Marker(); marker != nil {
marker.Mark()
} }
ho.Log.Warnf("connect to node %s(%s) failed: %v", node.Name, node.Addr, err) if node == nil {
return if lastErr != nil {
} ho.Log.Warnf("node for %s not found after retries", host)
return nil, nil, lastErr
if marker := node.Marker(); marker != nil { }
marker.Reset() ho.Log.Warnf("node for %s not found", host)
} return nil, nil, errors.New("node not available")
if tlsSettings := node.Options().TLS; tlsSettings != nil {
cfg := &tls.Config{
ServerName: tlsSettings.ServerName,
InsecureSkipVerify: !tlsSettings.Secure,
} }
tls_util.SetTLSOptions(cfg, &config.TLSOptions{
MinVersion: tlsSettings.Options.MinVersion, // Track this node as tried
MaxVersion: tlsSettings.Options.MaxVersion, triedNodes = append(triedNodes, node.Addr)
CipherSuites: tlsSettings.Options.CipherSuites,
ALPN: tlsSettings.Options.ALPN, addr := node.Addr
if opts := node.Options(); opts != nil {
switch opts.Network {
case "unix":
ro.Network = opts.Network
default:
if _, _, err := net.SplitHostPort(addr); err != nil {
addr += ":443"
}
}
}
ro.Host = addr
ho.Log = ho.Log.WithFields(map[string]any{
"host": host,
"node": node.Name,
"dst": fmt.Sprintf("%s/%s", addr, ro.Network),
}) })
cc = tls.Client(cc, cfg) ho.Log.Debugf("find node for host %s -> %s(%s)", host, node.Name, addr)
cc, err = dial(ctx, ro.Network, addr)
if err != nil {
// Mark node as failed for future selections
if marker := node.Marker(); marker != nil {
marker.Mark()
}
ho.Log.Warnf("connect to node %s(%s) failed: %v, trying next node", node.Name, node.Addr, err)
lastErr = err
continue
}
// Success - reset marker
if marker := node.Marker(); marker != nil {
marker.Reset()
}
if tlsSettings := node.Options().TLS; tlsSettings != nil {
cfg := &tls.Config{
ServerName: tlsSettings.ServerName,
InsecureSkipVerify: !tlsSettings.Secure,
}
tls_util.SetTLSOptions(cfg, &config.TLSOptions{
MinVersion: tlsSettings.Options.MinVersion,
MaxVersion: tlsSettings.Options.MaxVersion,
CipherSuites: tlsSettings.Options.CipherSuites,
ALPN: tlsSettings.Options.ALPN,
})
cc = tls.Client(cc, cfg)
}
return node, cc, nil
} }
return
// All retries exhausted
ho.Log.Warnf("all nodes failed for host %s", host)
if lastErr != nil {
return nil, nil, lastErr
}
return nil, nil, errors.New("all nodes failed")
} }
func (h *Sniffer) terminateTLS(ctx context.Context, conn, cc net.Conn, clientHello *dissector.ClientHelloInfo, ho *HandleOptions) error { func (h *Sniffer) terminateTLS(ctx context.Context, conn, cc net.Conn, clientHello *dissector.ClientHelloInfo, ho *HandleOptions) error {
+5 -4
View File
@@ -5,8 +5,8 @@ import (
"time" "time"
"github.com/go-gost/core/metadata" "github.com/go-gost/core/metadata"
mdutil "github.com/go-gost/x/metadata/util"
"github.com/go-gost/core/selector" "github.com/go-gost/core/selector"
mdutil "github.com/go-gost/x/metadata/util"
) )
type failFilter[T any] struct { type failFilter[T any] struct {
@@ -24,10 +24,11 @@ func FailFilter[T any](maxFails int, timeout time.Duration) selector.Filter[T] {
} }
// Filter filters dead objects. // Filter filters dead objects.
// Note: We intentionally do NOT skip filtering when len(vs) <= 1.
// This ensures that even a single dead node gets filtered out,
// allowing the caller to know that no healthy nodes are available
// and potentially trigger failover behavior.
func (f *failFilter[T]) Filter(ctx context.Context, vs ...T) []T { func (f *failFilter[T]) Filter(ctx context.Context, vs ...T) []T {
if len(vs) <= 1 {
return vs
}
var l []T var l []T
for _, v := range vs { for _, v := range vs {
maxFails := f.maxFails maxFails := f.maxFails