// Copyright 2026 Arctel.net // SPDX-License-Identifier: Apache-2.0 package pages import ( "context" "crypto/sha256" "crypto/tls" "encoding/hex" "errors" "fmt" "io" "math" "net" "net/http" "net/netip" "net/url" "os" "path" "strings" "time" "github.com/Rain-kl/Wavelet/pkg/httppool" "github.com/Rain-kl/Wavelet/pkg/pagesarchive" ) const ( // RemoteNetworkPolicyPublic only permits publicly routable targets and // performs DNS validation again for every connection. RemoteNetworkPolicyPublic = "public" // RemoteNetworkPolicyTrustedInternal permits private targets and self-signed // TLS certificates. It is an explicit administrator trust boundary. RemoteNetworkPolicyTrustedInternal = "trusted_internal" remoteSourceDownloadTimeout = 10 * time.Minute remoteSourceResponseHeaderTimeout = 30 * time.Second remoteSourceDialTimeout = 30 * time.Second remoteSourceDialKeepAlive = 30 * time.Second remoteSourceMaxRedirects = 5 remoteSourceMagicSniffBytes = 512 remoteSourceMaxSafeLabelBytes = 255 remoteSourceFallbackLabel = "package" remoteSourceUserAgent = "OpenFlare Pages Source/2" remoteSourceSchemeHTTP = "http" remoteSourceSchemeHTTPS = "https" ) type remoteProviderError string func (providerError remoteProviderError) Error() string { return string(providerError) } const ( errRemoteProviderInvalidPolicy remoteProviderError = "远程来源网络策略无效" errRemoteProviderInvalidLimit remoteProviderError = "远程来源部署包大小限制无效" errRemoteProviderBlockedAddress remoteProviderError = "远程来源 public 策略禁止访问非公网地址" errRemoteProviderResolveFailed remoteProviderError = "远程来源地址解析失败" errRemoteProviderRedirectLimit remoteProviderError = "远程来源重定向次数超过限制" errRemoteProviderDownloadFailed remoteProviderError = errPagesPackageURLDownloadFailed errRemoteProviderTooLarge remoteProviderError = errPagesPackageURLTooLarge errRemoteProviderEmpty remoteProviderError = errPagesPackageEmpty errRemoteProviderUnsupported remoteProviderError = errPagesPackageUnsupported errRemoteProviderCleanupFailed remoteProviderError = "清理远程来源临时文件失败" ) var remoteSourceNonPublicPrefixes = []netip.Prefix{ // IPv4 special-use, private, link-local, documentation, multicast and // reserved ranges. A conservative deny list is intentional for SSRF safety. netip.MustParsePrefix("0.0.0.0/8"), netip.MustParsePrefix("10.0.0.0/8"), netip.MustParsePrefix("100.64.0.0/10"), netip.MustParsePrefix("127.0.0.0/8"), netip.MustParsePrefix("169.254.0.0/16"), netip.MustParsePrefix("172.16.0.0/12"), netip.MustParsePrefix("192.0.0.0/24"), netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("192.88.99.0/24"), netip.MustParsePrefix("192.168.0.0/16"), netip.MustParsePrefix("198.18.0.0/15"), netip.MustParsePrefix("198.51.100.0/24"), netip.MustParsePrefix("203.0.113.0/24"), netip.MustParsePrefix("224.0.0.0/4"), netip.MustParsePrefix("240.0.0.0/4"), // IPv6 protocol-assignment, documentation and transition ranges that are // not acceptable as direct public artifact origins. netip.MustParsePrefix("2001::/23"), netip.MustParsePrefix("2001:db8::/32"), netip.MustParsePrefix("2002::/16"), netip.MustParsePrefix("3fff::/20"), } var remoteSourcePublicIPv6Prefix = netip.MustParsePrefix("2000::/3") // RemoteSourceRequest describes one immutable Remote URL package fetch. type RemoteSourceRequest struct { URL string NetworkPolicy string MaxPackageBytes int64 } // SourceCandidate is a constrained, immutable archive downloaded to a // provider-owned temporary file. The caller owns the file after a successful // fetch and must call Cleanup when processing finishes. type SourceCandidate struct { TempPath string Checksum string PackageSize int64 Format pagesarchive.Format SafeLabel string } // Cleanup removes the candidate temporary file. It is safe to call repeatedly. func (candidate *SourceCandidate) Cleanup() error { if candidate == nil || candidate.TempPath == "" { return nil } tempPath := candidate.TempPath err := os.Remove(tempPath) if err == nil || errors.Is(err, os.ErrNotExist) { candidate.TempPath = "" return nil } return errRemoteProviderCleanupFailed } type remoteSourceResolver interface { LookupNetIP(context.Context, string, string) ([]netip.Addr, error) } type remoteSourceDependencies struct { resolver remoteSourceResolver dialContext func(context.Context, string, string) (net.Conn, error) createTemp func(string, string) (*os.File, error) } // FetchRemoteSource downloads a Remote URL package without writing deployment // state. Errors are reduced to safe domain messages and never contain the raw // URL, query, response headers or response body. func FetchRemoteSource(ctx context.Context, request RemoteSourceRequest) (*SourceCandidate, error) { dialer := &net.Dialer{ Timeout: remoteSourceDialTimeout, KeepAlive: remoteSourceDialKeepAlive, } dependencies := remoteSourceDependencies{ resolver: net.DefaultResolver, dialContext: dialer.DialContext, createTemp: os.CreateTemp, } return fetchRemoteSource(ctx, request, dependencies) } func fetchRemoteSource(ctx context.Context, request RemoteSourceRequest, dependencies remoteSourceDependencies) (*SourceCandidate, error) { if request.MaxPackageBytes <= 0 { return nil, errRemoteProviderInvalidLimit } if dependencies.dialContext == nil || dependencies.createTemp == nil { return nil, errRemoteProviderDownloadFailed } policy, err := normalizeRemoteNetworkPolicy(request.NetworkPolicy) if err != nil { return nil, err } parsed, err := parseRemoteSourceURL(request.URL) if err != nil { return nil, err } if err := validateRemoteSourceTarget(ctx, parsed, policy, dependencies.resolver); err != nil { return nil, sanitizeRemoteProviderError(ctx, err) } safeLabel, namedFormat := remoteSourceLabel(parsed) client := newRemoteSourceClient(policy, dependencies) defer client.CloseIdleConnections() response, err := requestRemoteSource(ctx, client, parsed) if err != nil { return nil, err } defer func() { _ = response.Body.Close() }() if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { return nil, fmt.Errorf("%w: HTTP %d", errRemoteProviderDownloadFailed, response.StatusCode) } if response.ContentLength > request.MaxPackageBytes { return nil, errRemoteProviderTooLarge } tempPath, checksum, packageSize, err := streamRemoteSourcePackage( response.Body, request.MaxPackageBytes, dependencies.createTemp, ) if err != nil { return nil, sanitizeRemoteProviderError(ctx, err) } format, safeLabel, err := detectRemoteSourceFormat(tempPath, safeLabel, namedFormat) if err != nil { if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { return nil, errRemoteProviderCleanupFailed } return nil, err } return &SourceCandidate{ TempPath: tempPath, Checksum: checksum, PackageSize: packageSize, Format: format, SafeLabel: safeLabel, }, nil } func normalizeRemoteNetworkPolicy(policy string) (string, error) { switch strings.TrimSpace(policy) { case "", RemoteNetworkPolicyPublic: return RemoteNetworkPolicyPublic, nil case RemoteNetworkPolicyTrustedInternal: return RemoteNetworkPolicyTrustedInternal, nil default: return "", errRemoteProviderInvalidPolicy } } func newRemoteSourceClient(policy string, dependencies remoteSourceDependencies) *http.Client { tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} dialContext := dependencies.dialContext if policy == RemoteNetworkPolicyPublic { dialContext = newPublicRemoteSourceDialer(dependencies.resolver, dependencies.dialContext) } else { // trusted_internal is an explicit administrator-selected boundary for // private artifact services using an internal CA or self-signed cert. tlsConfig.InsecureSkipVerify = true //nolint:gosec // required trusted_internal semantics } client := &http.Client{ Timeout: remoteSourceDownloadTimeout, Transport: httppool.NewTransport(httppool.TransportOptions{ Proxy: nil, DialContext: dialContext, TLSClientConfig: tlsConfig, ResponseHeaderTimeout: remoteSourceResponseHeaderTimeout, TraceFilter: remoteSourceTraceFilter, }), } client.CheckRedirect = func(next *http.Request, previous []*http.Request) error { if len(previous) > remoteSourceMaxRedirects { return errRemoteProviderRedirectLimit } stripRemoteSourceRedirectHeaders(next) if err := validateRemoteSourceTarget(next.Context(), next.URL, policy, dependencies.resolver); err != nil { return err } applyRemoteSourceHeaders(next) return nil } return client } func requestRemoteSource(ctx context.Context, client *http.Client, parsed *url.URL) (*http.Response, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) if err != nil { return nil, errors.New(errPagesSourceRemoteURLInvalid) } applyRemoteSourceHeaders(request) response, err := client.Do(request) //nolint:gosec // scheme and every dial target are validated above if err == nil { return response, nil } if response != nil && response.Body != nil { _ = response.Body.Close() } return nil, sanitizeRemoteProviderError(ctx, err) } func applyRemoteSourceHeaders(request *http.Request) { request.Header.Set("User-Agent", remoteSourceUserAgent) request.Header.Set("Accept", "application/octet-stream,application/zip,application/x-tar,application/gzip,*/*;q=0.1") // Preserve the artifact bytes exactly as stored. Automatic HTTP gzip // decompression would change the checksum, size and archive format. request.Header.Set("Accept-Encoding", "identity") } func stripRemoteSourceRedirectHeaders(request *http.Request) { request.Header.Del("Authorization") request.Header.Del("Cookie") request.Header.Del("Proxy-Authorization") request.Header.Del("Referer") } func remoteSourceTraceFilter(request *http.Request) bool { // otelhttp records url.full. Signed query strings must never enter traces. return request.URL == nil || request.URL.RawQuery == "" } func validateRemoteSourceTarget(ctx context.Context, target *url.URL, policy string, resolver remoteSourceResolver) error { if target == nil || target.User != nil || target.Fragment != "" || target.Opaque != "" { return errors.New(errPagesSourceRemoteURLInvalid) } scheme := strings.ToLower(strings.TrimSpace(target.Scheme)) if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(target.Hostname()) == "" { return errors.New(errPagesSourceRemoteURLInvalid) } if policy != RemoteNetworkPolicyPublic { return nil } _, err := resolvePublicRemoteSourceIPs(ctx, resolver, target.Hostname()) return err } func newPublicRemoteSourceDialer( resolver remoteSourceResolver, directDial func(context.Context, string, string) (net.Conn, error), ) func(context.Context, string, string) (net.Conn, error) { return func(ctx context.Context, network string, address string) (net.Conn, error) { host, port, err := net.SplitHostPort(address) if err != nil { return nil, errRemoteProviderDownloadFailed } addresses, err := resolvePublicRemoteSourceIPs(ctx, resolver, host) if err != nil { return nil, err } for _, address := range addresses { if !remoteSourceIPMatchesNetwork(address, network) { continue } connection, dialErr := directDial(ctx, network, net.JoinHostPort(address.String(), port)) if dialErr == nil { return connection, nil } } return nil, errRemoteProviderDownloadFailed } } func resolvePublicRemoteSourceIPs(ctx context.Context, resolver remoteSourceResolver, host string) ([]netip.Addr, error) { if strings.Contains(host, "%") { return nil, errRemoteProviderBlockedAddress } if literal, parseErr := netip.ParseAddr(host); parseErr == nil { if !isPublicRemoteSourceIP(literal) { return nil, errRemoteProviderBlockedAddress } return []netip.Addr{literal}, nil } if resolver == nil { return nil, errRemoteProviderResolveFailed } addresses, err := resolver.LookupNetIP(ctx, "ip", host) if err != nil || len(addresses) == 0 { return nil, errRemoteProviderResolveFailed } for _, address := range addresses { if !isPublicRemoteSourceIP(address) { return nil, errRemoteProviderBlockedAddress } } return addresses, nil } func isPublicRemoteSourceIP(address netip.Addr) bool { if !address.IsValid() || address.Zone() != "" { return false } address = address.Unmap() if !address.IsGlobalUnicast() { return false } if address.Is6() && !remoteSourcePublicIPv6Prefix.Contains(address) { return false } for _, prefix := range remoteSourceNonPublicPrefixes { if prefix.Contains(address) { return false } } return true } func remoteSourceIPMatchesNetwork(address netip.Addr, network string) bool { switch network { case "tcp4": return address.Unmap().Is4() case "tcp6": return address.Unmap().Is6() default: return true } } func streamRemoteSourcePackage( body io.Reader, maxPackageBytes int64, createTemp func(string, string) (*os.File, error), ) (tempPath string, checksum string, packageSize int64, err error) { if createTemp == nil { return "", "", 0, errRemoteProviderDownloadFailed } tempFile, err := createTemp("", "openflare-pages-source-*") if err != nil { return "", "", 0, errRemoteProviderDownloadFailed } createdTempPath := tempFile.Name() tempPath = createdTempPath defer func() { closeErr := tempFile.Close() if err == nil && closeErr != nil { err = errRemoteProviderDownloadFailed } if err != nil { if removeErr := os.Remove(createdTempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { err = errRemoteProviderCleanupFailed } } }() hasher := sha256.New() readLimit := maxPackageBytes if readLimit < math.MaxInt64 { readLimit++ } packageSize, err = io.Copy(io.MultiWriter(tempFile, hasher), io.LimitReader(body, readLimit)) if err != nil { return "", "", 0, errRemoteProviderDownloadFailed } if packageSize > maxPackageBytes { return "", "", 0, errRemoteProviderTooLarge } if packageSize == 0 { return "", "", 0, errRemoteProviderEmpty } checksum = hex.EncodeToString(hasher.Sum(nil)) return tempPath, checksum, packageSize, nil } func detectRemoteSourceFormat( tempPath string, safeLabel string, namedFormat pagesarchive.Format, ) (pagesarchive.Format, string, error) { if namedFormat != "" { return namedFormat, safeLabel, nil } tempFile, err := os.Open(tempPath) //nolint:gosec // path is a provider-created temporary file if err != nil { return "", safeLabel, errRemoteProviderDownloadFailed } defer func() { _ = tempFile.Close() }() head := make([]byte, remoteSourceMagicSniffBytes) readBytes, readErr := io.ReadFull(tempFile, head) if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { return "", safeLabel, errRemoteProviderDownloadFailed } format, ok := pagesarchive.DetectFormatFromBytes(head[:readBytes]) if !ok { return "", safeLabel, errRemoteProviderUnsupported } return format, appendRemoteSourceLabelExtension(safeLabel, format), nil } func remoteSourceLabel(parsed *url.URL) (string, pagesarchive.Format) { baseName := path.Base(parsed.Path) if baseName == "" || baseName == "." || baseName == "/" { baseName = remoteSourceFallbackLabel } safeLabel := sanitizeRemoteSourceLabel(baseName) format, _ := pagesarchive.DetectFormatFromName(safeLabel) return limitRemoteSourceLabel(safeLabel, format), format } func sanitizeRemoteSourceLabel(label string) string { var builder strings.Builder lastReplacement := false for _, character := range label { if isRemoteSourceLabelCharacter(character) { builder.WriteRune(character) lastReplacement = false continue } if !lastReplacement { builder.WriteByte('-') lastReplacement = true } } safeLabel := strings.TrimSpace(builder.String()) if safeLabel == "" || strings.Trim(safeLabel, "._-") == "" { return remoteSourceFallbackLabel } return safeLabel } func isRemoteSourceLabelCharacter(character rune) bool { return character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || character == '.' || character == '-' || character == '_' } func limitRemoteSourceLabel(label string, format pagesarchive.Format) string { if len(label) <= remoteSourceMaxSafeLabelBytes { return label } if format == "" { return strings.TrimRight(label[:remoteSourceMaxSafeLabelBytes], ".-_") } extension := "." + pagesarchive.Extension(format) prefixLength := remoteSourceMaxSafeLabelBytes - len(extension) prefix := strings.TrimRight(label[:prefixLength], ".-_") if prefix == "" { prefix = remoteSourceFallbackLabel } return prefix + extension } func appendRemoteSourceLabelExtension(label string, format pagesarchive.Format) string { extension := "." + pagesarchive.Extension(format) maxPrefixLength := remoteSourceMaxSafeLabelBytes - len(extension) if len(label) > maxPrefixLength { label = strings.TrimRight(label[:maxPrefixLength], ".-_") } if label == "" { label = remoteSourceFallbackLabel } return label + extension } func sanitizeRemoteProviderError(ctx context.Context, err error) error { if ctxErr := ctx.Err(); ctxErr != nil { return fmt.Errorf("%w: %w", errRemoteProviderDownloadFailed, ctxErr) } for _, safeError := range []error{ errRemoteProviderInvalidPolicy, errRemoteProviderInvalidLimit, errRemoteProviderBlockedAddress, errRemoteProviderResolveFailed, errRemoteProviderRedirectLimit, errRemoteProviderTooLarge, errRemoteProviderEmpty, errRemoteProviderUnsupported, errRemoteProviderCleanupFailed, } { if errors.Is(err, safeError) { return safeError } } return errRemoteProviderDownloadFailed }