OpenFlare/internal/apps/openflare/pages/source_provider.go

// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0

package pages

import (
	"context"
	"crypto/sha256"
	"crypto/tls"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"math"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"os"
	"path"
	"strings"
	"time"

	"github.com/Rain-kl/Wavelet/pkg/httppool"
	"github.com/Rain-kl/Wavelet/pkg/pagesarchive"
)

const (
	remoteSourceDownloadTimeout       = 10 * time.Minute
	remoteSourceResponseHeaderTimeout = 30 * time.Second
	remoteSourceDialTimeout           = 30 * time.Second
	remoteSourceDialKeepAlive         = 30 * time.Second
	remoteSourceMaxRedirects          = 5
	remoteSourceMagicSniffBytes       = 512
	remoteSourceMaxSafeLabelBytes     = 255
	remoteSourceFallbackLabel         = "package"
	remoteSourceUserAgent             = "OpenFlare Pages Source/2"
	remoteSourceSchemeHTTP            = "http"
	remoteSourceSchemeHTTPS           = "https"
)

type remoteProviderError string

func (providerError remoteProviderError) Error() string {
	return string(providerError)
}

	const (
	errRemoteProviderInvalidLimit  remoteProviderError = "远程来源部署包大小限制无效"
	errRemoteProviderRedirectLimit remoteProviderError = "远程来源重定向次数超过限制"
	errRemoteProviderDownloadFailed remoteProviderError = errPagesPackageURLDownloadFailed
	errRemoteProviderTooLarge       remoteProviderError = errPagesPackageURLTooLarge
	errRemoteProviderEmpty          remoteProviderError = errPagesPackageEmpty
	errRemoteProviderUnsupported    remoteProviderError = errPagesPackageUnsupported
	errRemoteProviderCleanupFailed  remoteProviderError = "清理远程来源临时文件失败"
)

// RemoteSourceRequest describes one immutable Remote URL package fetch.
type RemoteSourceRequest struct {
	URL             string
	AllowInsecure   bool
	MaxPackageBytes int64
}

// SourceCandidate is a constrained, immutable archive downloaded to a
// provider-owned temporary file. The caller owns the file after a successful
// fetch and must call Cleanup when processing finishes.
type SourceCandidate struct {
	TempPath    string
	Checksum    string
	PackageSize int64
	Format      pagesarchive.Format
	SafeLabel   string
}

// Cleanup removes the candidate temporary file. It is safe to call repeatedly.
func (candidate *SourceCandidate) Cleanup() error {
	if candidate == nil || candidate.TempPath == "" {
		return nil
	}
	tempPath := candidate.TempPath
	err := os.Remove(tempPath)
	if err == nil || errors.Is(err, os.ErrNotExist) {
		candidate.TempPath = ""
		return nil
	}
	return errRemoteProviderCleanupFailed
}

type remoteSourceResolver interface {
	LookupNetIP(context.Context, string, string) ([]netip.Addr, error)
}

type remoteSourceDependencies struct {
	resolver    remoteSourceResolver
	dialContext func(context.Context, string, string) (net.Conn, error)
	createTemp  func(string, string) (*os.File, error)
}

// FetchRemoteSource downloads a Remote URL package without writing deployment
// state. Errors are reduced to safe domain messages and never contain the raw
// URL, query, response headers or response body.
func FetchRemoteSource(ctx context.Context, request RemoteSourceRequest) (*SourceCandidate, error) {
	dialer := &net.Dialer{
		Timeout:   remoteSourceDialTimeout,
		KeepAlive: remoteSourceDialKeepAlive,
	}
	dependencies := remoteSourceDependencies{
		resolver:    net.DefaultResolver,
		dialContext: dialer.DialContext,
		createTemp:  os.CreateTemp,
	}
	return fetchRemoteSource(ctx, request, dependencies)
}

func fetchRemoteSource(ctx context.Context, request RemoteSourceRequest, dependencies remoteSourceDependencies) (*SourceCandidate, error) {
	if request.MaxPackageBytes <= 0 {
		return nil, errRemoteProviderInvalidLimit
	}
	if dependencies.dialContext == nil || dependencies.createTemp == nil {
		return nil, errRemoteProviderDownloadFailed
	}
	parsed, err := parseRemoteSourceURL(request.URL)
	if err != nil {
		return nil, err
	}
	if err := validateRemoteSourceTarget(ctx, parsed); err != nil {
		return nil, sanitizeRemoteProviderError(ctx, err)
	}

	safeLabel, namedFormat := remoteSourceLabel(parsed)
	client := newRemoteSourceClient(request.AllowInsecure, dependencies)
	defer client.CloseIdleConnections()
	response, err := requestRemoteSource(ctx, client, parsed)
	if err != nil {
		return nil, err
	}
	defer func() { _ = response.Body.Close() }()

	if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
		return nil, fmt.Errorf("%w: HTTP %d", errRemoteProviderDownloadFailed, response.StatusCode)
	}
	if response.ContentLength > request.MaxPackageBytes {
		return nil, errRemoteProviderTooLarge
	}

	tempPath, checksum, packageSize, err := streamRemoteSourcePackage(
		response.Body,
		request.MaxPackageBytes,
		dependencies.createTemp,
	)
	if err != nil {
		return nil, sanitizeRemoteProviderError(ctx, err)
	}
	format, safeLabel, err := detectRemoteSourceFormat(tempPath, safeLabel, namedFormat)
	if err != nil {
		if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
			return nil, errRemoteProviderCleanupFailed
		}
		return nil, err
	}

	return &SourceCandidate{
		TempPath:    tempPath,
		Checksum:    checksum,
		PackageSize: packageSize,
		Format:      format,
		SafeLabel:   safeLabel,
	}, nil
}

func newRemoteSourceClient(allowInsecure bool, dependencies remoteSourceDependencies) *http.Client {
	tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
	if allowInsecure {
		// Explicit administrator choice for self-signed or private CA endpoints.
		tlsConfig.InsecureSkipVerify = true //nolint:gosec // required allow_insecure semantics
	}

	client := &http.Client{
		Timeout: remoteSourceDownloadTimeout,
		Transport: httppool.NewTransport(httppool.TransportOptions{
			Proxy:                 nil,
			DialContext:           dependencies.dialContext,
			TLSClientConfig:       tlsConfig,
			ResponseHeaderTimeout: remoteSourceResponseHeaderTimeout,
			TraceFilter:           remoteSourceTraceFilter,
		}),
	}
	client.CheckRedirect = func(next *http.Request, previous []*http.Request) error {
		if len(previous) > remoteSourceMaxRedirects {
			return errRemoteProviderRedirectLimit
		}
		stripRemoteSourceRedirectHeaders(next)
		if err := validateRemoteSourceTarget(next.Context(), next.URL); err != nil {
			return err
		}
		applyRemoteSourceHeaders(next)
		return nil
	}
	return client
}

func requestRemoteSource(ctx context.Context, client *http.Client, parsed *url.URL) (*http.Response, error) {
	request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
	if err != nil {
		return nil, errors.New(errPagesSourceRemoteURLInvalid)
	}
	applyRemoteSourceHeaders(request)
	response, err := client.Do(request) //nolint:gosec // scheme and every dial target are validated above
	if err == nil {
		return response, nil
	}
	if response != nil && response.Body != nil {
		_ = response.Body.Close()
	}
	return nil, sanitizeRemoteProviderError(ctx, err)
}

func applyRemoteSourceHeaders(request *http.Request) {
	request.Header.Set("User-Agent", remoteSourceUserAgent)
	request.Header.Set("Accept", "application/octet-stream,application/zip,application/x-tar,application/gzip,*/*;q=0.1")
	// Preserve the artifact bytes exactly as stored. Automatic HTTP gzip
	// decompression would change the checksum, size and archive format.
	request.Header.Set("Accept-Encoding", "identity")
}

func stripRemoteSourceRedirectHeaders(request *http.Request) {
	request.Header.Del("Authorization")
	request.Header.Del("Cookie")
	request.Header.Del("Proxy-Authorization")
	request.Header.Del("Referer")
}

func remoteSourceTraceFilter(request *http.Request) bool {
	// otelhttp records url.full. Signed query strings must never enter traces.
	return request.URL == nil || request.URL.RawQuery == ""
}

func validateRemoteSourceTarget(_ context.Context, target *url.URL) error {
	if target == nil || target.User != nil || target.Fragment != "" || target.Opaque != "" {
		return errors.New(errPagesSourceRemoteURLInvalid)
	}
	scheme := strings.ToLower(strings.TrimSpace(target.Scheme))
	if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(target.Hostname()) == "" {
		return errors.New(errPagesSourceRemoteURLInvalid)
	}
	return nil
}

func streamRemoteSourcePackage(
	body io.Reader,
	maxPackageBytes int64,
	createTemp func(string, string) (*os.File, error),
) (tempPath string, checksum string, packageSize int64, err error) {
	if createTemp == nil {
		return "", "", 0, errRemoteProviderDownloadFailed
	}
	tempFile, err := createTemp("", "openflare-pages-source-*")
	if err != nil {
		return "", "", 0, errRemoteProviderDownloadFailed
	}
	createdTempPath := tempFile.Name()
	tempPath = createdTempPath
	defer func() {
		closeErr := tempFile.Close()
		if err == nil && closeErr != nil {
			err = errRemoteProviderDownloadFailed
		}
		if err != nil {
			if removeErr := os.Remove(createdTempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
				err = errRemoteProviderCleanupFailed
			}
		}
	}()

	hasher := sha256.New()
	readLimit := maxPackageBytes
	if readLimit < math.MaxInt64 {
		readLimit++
	}
	packageSize, err = io.Copy(io.MultiWriter(tempFile, hasher), io.LimitReader(body, readLimit))
	if err != nil {
		return "", "", 0, errRemoteProviderDownloadFailed
	}
	if packageSize > maxPackageBytes {
		return "", "", 0, errRemoteProviderTooLarge
	}
	if packageSize == 0 {
		return "", "", 0, errRemoteProviderEmpty
	}
	checksum = hex.EncodeToString(hasher.Sum(nil))
	return tempPath, checksum, packageSize, nil
}

func detectRemoteSourceFormat(
	tempPath string,
	safeLabel string,
	namedFormat pagesarchive.Format,
) (pagesarchive.Format, string, error) {
	if namedFormat != "" {
		return namedFormat, safeLabel, nil
	}
	tempFile, err := os.Open(tempPath) //nolint:gosec // path is a provider-created temporary file
	if err != nil {
		return "", safeLabel, errRemoteProviderDownloadFailed
	}
	defer func() { _ = tempFile.Close() }()

	head := make([]byte, remoteSourceMagicSniffBytes)
	readBytes, readErr := io.ReadFull(tempFile, head)
	if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) {
		return "", safeLabel, errRemoteProviderDownloadFailed
	}
	format, ok := pagesarchive.DetectFormatFromBytes(head[:readBytes])
	if !ok {
		return "", safeLabel, errRemoteProviderUnsupported
	}
	return format, appendRemoteSourceLabelExtension(safeLabel, format), nil
}

func remoteSourceLabel(parsed *url.URL) (string, pagesarchive.Format) {
	baseName := path.Base(parsed.Path)
	if baseName == "" || baseName == "." || baseName == "/" {
		baseName = remoteSourceFallbackLabel
	}
	safeLabel := sanitizeRemoteSourceLabel(baseName)
	format, _ := pagesarchive.DetectFormatFromName(safeLabel)
	return limitRemoteSourceLabel(safeLabel, format), format
}

func sanitizeRemoteSourceLabel(label string) string {
	var builder strings.Builder
	lastReplacement := false
	for _, character := range label {
		if isRemoteSourceLabelCharacter(character) {
			builder.WriteRune(character)
			lastReplacement = false
			continue
		}
		if !lastReplacement {
			builder.WriteByte('-')
			lastReplacement = true
		}
	}
	safeLabel := strings.TrimSpace(builder.String())
	if safeLabel == "" || strings.Trim(safeLabel, "._-") == "" {
		return remoteSourceFallbackLabel
	}
	return safeLabel
}

func isRemoteSourceLabelCharacter(character rune) bool {
	return character >= 'a' && character <= 'z' ||
		character >= 'A' && character <= 'Z' ||
		character >= '0' && character <= '9' ||
		character == '.' || character == '-' || character == '_'
}

func limitRemoteSourceLabel(label string, format pagesarchive.Format) string {
	if len(label) <= remoteSourceMaxSafeLabelBytes {
		return label
	}
	if format == "" {
		return strings.TrimRight(label[:remoteSourceMaxSafeLabelBytes], ".-_")
	}
	extension := "." + pagesarchive.Extension(format)
	prefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
	prefix := strings.TrimRight(label[:prefixLength], ".-_")
	if prefix == "" {
		prefix = remoteSourceFallbackLabel
	}
	return prefix + extension
}

func appendRemoteSourceLabelExtension(label string, format pagesarchive.Format) string {
	extension := "." + pagesarchive.Extension(format)
	maxPrefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
	if len(label) > maxPrefixLength {
		label = strings.TrimRight(label[:maxPrefixLength], ".-_")
	}
	if label == "" {
		label = remoteSourceFallbackLabel
	}
	return label + extension
}

func sanitizeRemoteProviderError(ctx context.Context, err error) error {
	if ctxErr := ctx.Err(); ctxErr != nil {
		return fmt.Errorf("%w: %w", errRemoteProviderDownloadFailed, ctxErr)
	}
	for _, safeError := range []error{
		errRemoteProviderInvalidLimit,
		errRemoteProviderRedirectLimit,
		errRemoteProviderTooLarge,
		errRemoteProviderEmpty,
		errRemoteProviderUnsupported,
		errRemoteProviderCleanupFailed,
	} {
		if errors.Is(err, safeError) {
			return safeError
		}
	}
	return errRemoteProviderDownloadFailed
}