refactor(backend): rename OpenFlare directory to lowercase openflare

2026-10-10 17:26:38 +08:00 · 2026-08-30 17:43:23 +08:00
parent 06d5fedbfc
commit c93ff6674f
543 changed files with 819 additions and 819 deletions
@@ -0,0 +1,404 @@
+// Copyright 2026 Arctel.net
+// SPDX-License-Identifier: Apache-2.0
+
+package pages
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/tls"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"Wavelet/openflare/share/pagesarchive"
+	"Wavelet/pkg/httppool"
+)
+
+const (
+	remoteSourceDownloadTimeout       = 10 * time.Minute
+	remoteSourceResponseHeaderTimeout = 30 * time.Second
+	remoteSourceDialTimeout           = 30 * time.Second
+	remoteSourceDialKeepAlive         = 30 * time.Second
+	remoteSourceMaxRedirects          = 5
+	remoteSourceMagicSniffBytes       = 512
+	remoteSourceMaxSafeLabelBytes     = 255
+	remoteSourceFallbackLabel         = "package"
+	remoteSourceUserAgent             = "OpenFlare Pages Source/2"
+	remoteSourceSchemeHTTP            = "http"
+	remoteSourceSchemeHTTPS           = "https"
+)
+
+type remoteProviderError string
+
+func (providerError remoteProviderError) Error() string {
+	return string(providerError)
+}
+
+const (
+	errRemoteProviderInvalidLimit   remoteProviderError = "远程来源部署包大小限制无效"
+	errRemoteProviderRedirectLimit  remoteProviderError = "远程来源重定向次数超过限制"
+	errRemoteProviderDownloadFailed remoteProviderError = errPagesPackageURLDownloadFailed
+	errRemoteProviderTooLarge       remoteProviderError = errPagesPackageURLTooLarge
+	errRemoteProviderEmpty          remoteProviderError = errPagesPackageEmpty
+	errRemoteProviderUnsupported    remoteProviderError = errPagesPackageUnsupported
+	errRemoteProviderCleanupFailed  remoteProviderError = "清理远程来源临时文件失败"
+)
+
+// RemoteSourceRequest describes one immutable Remote URL package fetch.
+type RemoteSourceRequest struct {
+	URL             string
+	AllowInsecure   bool
+	MaxPackageBytes int64
+}
+
+// SourceCandidate is a constrained, immutable archive downloaded to a
+// provider-owned temporary file. The caller owns the file after a successful
+// fetch and must call Cleanup when processing finishes.
+type SourceCandidate struct {
+	TempPath    string
+	Checksum    string
+	PackageSize int64
+	Format      pagesarchive.Format
+	SafeLabel   string
+}
+
+// Cleanup removes the candidate temporary file. It is safe to call repeatedly.
+func (candidate *SourceCandidate) Cleanup() error {
+	if candidate == nil || candidate.TempPath == "" {
+		return nil
+	}
+	tempPath := candidate.TempPath
+	err := os.Remove(tempPath)
+	if err == nil || errors.Is(err, os.ErrNotExist) {
+		candidate.TempPath = ""
+		return nil
+	}
+	return errRemoteProviderCleanupFailed
+}
+
+type remoteSourceResolver interface {
+	LookupNetIP(context.Context, string, string) ([]netip.Addr, error)
+}
+
+type remoteSourceDependencies struct {
+	resolver    remoteSourceResolver
+	dialContext func(context.Context, string, string) (net.Conn, error)
+	createTemp  func(string, string) (*os.File, error)
+}
+
+// FetchRemoteSource downloads a Remote URL package without writing deployment
+// state. Errors are reduced to safe domain messages and never contain the raw
+// URL, query, response headers or response body.
+func FetchRemoteSource(ctx context.Context, request RemoteSourceRequest) (*SourceCandidate, error) {
+	dialer := &net.Dialer{
+		Timeout:   remoteSourceDialTimeout,
+		KeepAlive: remoteSourceDialKeepAlive,
+	}
+	dependencies := remoteSourceDependencies{
+		resolver:    net.DefaultResolver,
+		dialContext: dialer.DialContext,
+		createTemp:  os.CreateTemp,
+	}
+	return fetchRemoteSource(ctx, request, dependencies)
+}
+
+func fetchRemoteSource(ctx context.Context, request RemoteSourceRequest, dependencies remoteSourceDependencies) (*SourceCandidate, error) {
+	if request.MaxPackageBytes <= 0 {
+		return nil, errRemoteProviderInvalidLimit
+	}
+	if dependencies.dialContext == nil || dependencies.createTemp == nil {
+		return nil, errRemoteProviderDownloadFailed
+	}
+	parsed, err := parseRemoteSourceURL(request.URL)
+	if err != nil {
+		return nil, err
+	}
+	if err := validateRemoteSourceTarget(ctx, parsed); err != nil {
+		return nil, sanitizeRemoteProviderError(ctx, err)
+	}
+
+	safeLabel, namedFormat := remoteSourceLabel(parsed)
+	client := newRemoteSourceClient(request.AllowInsecure, dependencies)
+	defer client.CloseIdleConnections()
+	response, err := requestRemoteSource(ctx, client, parsed)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = response.Body.Close() }()
+
+	if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
+		return nil, fmt.Errorf("%w: HTTP %d", errRemoteProviderDownloadFailed, response.StatusCode)
+	}
+	if response.ContentLength > request.MaxPackageBytes {
+		return nil, errRemoteProviderTooLarge
+	}
+
+	tempPath, checksum, packageSize, err := streamRemoteSourcePackage(
+		response.Body,
+		request.MaxPackageBytes,
+		dependencies.createTemp,
+	)
+	if err != nil {
+		return nil, sanitizeRemoteProviderError(ctx, err)
+	}
+	format, safeLabel, err := detectRemoteSourceFormat(tempPath, safeLabel, namedFormat)
+	if err != nil {
+		if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
+			return nil, errRemoteProviderCleanupFailed
+		}
+		return nil, err
+	}
+
+	return &SourceCandidate{
+		TempPath:    tempPath,
+		Checksum:    checksum,
+		PackageSize: packageSize,
+		Format:      format,
+		SafeLabel:   safeLabel,
+	}, nil
+}
+
+func newRemoteSourceClient(allowInsecure bool, dependencies remoteSourceDependencies) *http.Client {
+	tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
+	if allowInsecure {
+		// Explicit administrator choice for self-signed or private CA endpoints.
+		tlsConfig.InsecureSkipVerify = true //nolint:gosec // required allow_insecure semantics
+	}
+
+	client := &http.Client{
+		Timeout: remoteSourceDownloadTimeout,
+		Transport: httppool.NewTransport(httppool.TransportOptions{
+			Proxy:                 nil,
+			DialContext:           dependencies.dialContext,
+			TLSClientConfig:       tlsConfig,
+			ResponseHeaderTimeout: remoteSourceResponseHeaderTimeout,
+			TraceFilter:           remoteSourceTraceFilter,
+		}),
+	}
+	client.CheckRedirect = func(next *http.Request, previous []*http.Request) error {
+		if len(previous) > remoteSourceMaxRedirects {
+			return errRemoteProviderRedirectLimit
+		}
+		stripRemoteSourceRedirectHeaders(next)
+		if err := validateRemoteSourceTarget(next.Context(), next.URL); err != nil {
+			return err
+		}
+		applyRemoteSourceHeaders(next)
+		return nil
+	}
+	return client
+}
+
+func requestRemoteSource(ctx context.Context, client *http.Client, parsed *url.URL) (*http.Response, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
+	if err != nil {
+		return nil, errors.New(errPagesSourceRemoteURLInvalid)
+	}
+	applyRemoteSourceHeaders(request)
+	response, err := client.Do(request) //nolint:gosec // scheme and every dial target are validated above
+	if err == nil {
+		return response, nil
+	}
+	if response != nil && response.Body != nil {
+		_ = response.Body.Close()
+	}
+	return nil, sanitizeRemoteProviderError(ctx, err)
+}
+
+func applyRemoteSourceHeaders(request *http.Request) {
+	request.Header.Set("User-Agent", remoteSourceUserAgent)
+	request.Header.Set("Accept", "application/octet-stream,application/zip,application/x-tar,application/gzip,*/*;q=0.1")
+	// Preserve the artifact bytes exactly as stored. Automatic HTTP gzip
+	// decompression would change the checksum, size and archive format.
+	request.Header.Set("Accept-Encoding", "identity")
+}
+
+func stripRemoteSourceRedirectHeaders(request *http.Request) {
+	request.Header.Del("Authorization")
+	request.Header.Del("Cookie")
+	request.Header.Del("Proxy-Authorization")
+	request.Header.Del("Referer")
+}
+
+func remoteSourceTraceFilter(request *http.Request) bool {
+	// otelhttp records url.full. Signed query strings must never enter traces.
+	return request.URL == nil || request.URL.RawQuery == ""
+}
+
+func validateRemoteSourceTarget(_ context.Context, target *url.URL) error {
+	if target == nil || target.User != nil || target.Fragment != "" || target.Opaque != "" {
+		return errors.New(errPagesSourceRemoteURLInvalid)
+	}
+	scheme := strings.ToLower(strings.TrimSpace(target.Scheme))
+	if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(target.Hostname()) == "" {
+		return errors.New(errPagesSourceRemoteURLInvalid)
+	}
+	return nil
+}
+
+func streamRemoteSourcePackage(
+	body io.Reader,
+	maxPackageBytes int64,
+	createTemp func(string, string) (*os.File, error),
+) (tempPath string, checksum string, packageSize int64, err error) {
+	if createTemp == nil {
+		return "", "", 0, errRemoteProviderDownloadFailed
+	}
+	tempFile, err := createTemp("", "openflare-pages-source-*")
+	if err != nil {
+		return "", "", 0, errRemoteProviderDownloadFailed
+	}
+	createdTempPath := tempFile.Name()
+	tempPath = createdTempPath
+	defer func() {
+		closeErr := tempFile.Close()
+		if err == nil && closeErr != nil {
+			err = errRemoteProviderDownloadFailed
+		}
+		if err != nil {
+			if removeErr := os.Remove(createdTempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
+				err = errRemoteProviderCleanupFailed
+			}
+		}
+	}()
+
+	hasher := sha256.New()
+	readLimit := maxPackageBytes
+	if readLimit < math.MaxInt64 {
+		readLimit++
+	}
+	packageSize, err = io.Copy(io.MultiWriter(tempFile, hasher), io.LimitReader(body, readLimit))
+	if err != nil {
+		return "", "", 0, errRemoteProviderDownloadFailed
+	}
+	if packageSize > maxPackageBytes {
+		return "", "", 0, errRemoteProviderTooLarge
+	}
+	if packageSize == 0 {
+		return "", "", 0, errRemoteProviderEmpty
+	}
+	checksum = hex.EncodeToString(hasher.Sum(nil))
+	return tempPath, checksum, packageSize, nil
+}
+
+func detectRemoteSourceFormat(
+	tempPath string,
+	safeLabel string,
+	namedFormat pagesarchive.Format,
+) (pagesarchive.Format, string, error) {
+	if namedFormat != "" {
+		return namedFormat, safeLabel, nil
+	}
+	tempFile, err := os.Open(tempPath) //nolint:gosec // path is a provider-created temporary file
+	if err != nil {
+		return "", safeLabel, errRemoteProviderDownloadFailed
+	}
+	defer func() { _ = tempFile.Close() }()
+
+	head := make([]byte, remoteSourceMagicSniffBytes)
+	readBytes, readErr := io.ReadFull(tempFile, head)
+	if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) {
+		return "", safeLabel, errRemoteProviderDownloadFailed
+	}
+	format, ok := pagesarchive.DetectFormatFromBytes(head[:readBytes])
+	if !ok {
+		return "", safeLabel, errRemoteProviderUnsupported
+	}
+	return format, appendRemoteSourceLabelExtension(safeLabel, format), nil
+}
+
+func remoteSourceLabel(parsed *url.URL) (string, pagesarchive.Format) {
+	baseName := path.Base(parsed.Path)
+	if baseName == "" || baseName == "." || baseName == "/" {
+		baseName = remoteSourceFallbackLabel
+	}
+	safeLabel := sanitizeRemoteSourceLabel(baseName)
+	format, _ := pagesarchive.DetectFormatFromName(safeLabel)
+	return limitRemoteSourceLabel(safeLabel, format), format
+}
+
+func sanitizeRemoteSourceLabel(label string) string {
+	var builder strings.Builder
+	lastReplacement := false
+	for _, character := range label {
+		if isRemoteSourceLabelCharacter(character) {
+			builder.WriteRune(character)
+			lastReplacement = false
+			continue
+		}
+		if !lastReplacement {
+			builder.WriteByte('-')
+			lastReplacement = true
+		}
+	}
+	safeLabel := strings.TrimSpace(builder.String())
+	if safeLabel == "" || strings.Trim(safeLabel, "._-") == "" {
+		return remoteSourceFallbackLabel
+	}
+	return safeLabel
+}
+
+func isRemoteSourceLabelCharacter(character rune) bool {
+	return character >= 'a' && character <= 'z' ||
+		character >= 'A' && character <= 'Z' ||
+		character >= '0' && character <= '9' ||
+		character == '.' || character == '-' || character == '_'
+}
+
+func limitRemoteSourceLabel(label string, format pagesarchive.Format) string {
+	if len(label) <= remoteSourceMaxSafeLabelBytes {
+		return label
+	}
+	if format == "" {
+		return strings.TrimRight(label[:remoteSourceMaxSafeLabelBytes], ".-_")
+	}
+	extension := "." + pagesarchive.Extension(format)
+	prefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
+	prefix := strings.TrimRight(label[:prefixLength], ".-_")
+	if prefix == "" {
+		prefix = remoteSourceFallbackLabel
+	}
+	return prefix + extension
+}
+
+func appendRemoteSourceLabelExtension(label string, format pagesarchive.Format) string {
+	extension := "." + pagesarchive.Extension(format)
+	maxPrefixLength := remoteSourceMaxSafeLabelBytes - len(extension)
+	if len(label) > maxPrefixLength {
+		label = strings.TrimRight(label[:maxPrefixLength], ".-_")
+	}
+	if label == "" {
+		label = remoteSourceFallbackLabel
+	}
+	return label + extension
+}
+
+func sanitizeRemoteProviderError(ctx context.Context, err error) error {
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return fmt.Errorf("%w: %w", errRemoteProviderDownloadFailed, ctxErr)
+	}
+	for _, safeError := range []error{
+		errRemoteProviderInvalidLimit,
+		errRemoteProviderRedirectLimit,
+		errRemoteProviderTooLarge,
+		errRemoteProviderEmpty,
+		errRemoteProviderUnsupported,
+		errRemoteProviderCleanupFailed,
+	} {
+		if errors.Is(err, safeError) {
+			return safeError
+		}
+	}
+	return errRemoteProviderDownloadFailed
+}