refactor(backend): rename OpenFlare directory to lowercase openflare

This commit is contained in:
ryan
2026-08-30 17:43:23 +08:00
parent 06d5fedbfc
commit c93ff6674f
543 changed files with 819 additions and 819 deletions
@@ -0,0 +1,110 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"errors"
"fmt"
"io"
"math"
"os"
"path/filepath"
)
// Limits bounds archive inspection / extraction work.
type Limits struct {
// MaxFiles is the maximum number of regular files allowed.
MaxFiles int
// MaxFileBytes is the maximum size of a single extracted file.
MaxFileBytes int64
// MaxTotalBytes is the maximum sum of all extracted file sizes.
MaxTotalBytes int64
}
// FileEntry is a regular file discovered inside a deployment package.
type FileEntry struct {
Path string
Size int64
// Checksum is retained for API/schema compatibility and is left empty.
// Integrity is enforced via the whole-package SHA-256 on the deployment record.
Checksum string
}
// Manifest is the inspected content of a Pages deployment package.
type Manifest struct {
Files []FileEntry
FileCount int
TotalSize int64
}
// Entry describes one archive member for extraction.
type Entry struct {
// Name is the original path inside the archive.
Name string
// IsDir marks directory entries.
IsDir bool
// IsSymlink marks symbolic links (unsupported for Pages).
IsSymlink bool
// IsHardlink marks hard links (unsupported for Pages).
IsHardlink bool
// IsSpecial marks device, FIFO, socket, and other non-regular entries.
IsSpecial bool
// Size is the archive-declared uncompressed size; 0 means an empty member.
Size uint64
// Open returns a reader for the entry body. Caller must Close it.
Open func() (io.ReadCloser, error)
}
// copyLimited copies actual bytes from src. maxBytes < 0 disables the byte cap;
// maxBytes == 0 permits only an empty stream.
func copyLimited(dst io.Writer, src io.Reader, maxBytes int64) (int64, error) {
if maxBytes < 0 {
return io.Copy(dst, src)
}
readLimit := maxBytes
if maxBytes < math.MaxInt64 {
readLimit++
}
written, err := io.Copy(dst, io.LimitReader(src, readLimit))
if err != nil {
return written, err
}
if written > maxBytes {
return written, errors.New("pages file size out of bounds")
}
return written, nil
}
func copyAndVerifySize(dst io.Writer, src io.Reader, declaredSize uint64, maxBytes int64) (int64, error) {
if declaredSize > uint64(math.MaxInt64) {
return 0, errors.New("pages file size out of bounds")
}
written, err := copyLimited(dst, src, maxBytes)
if err != nil {
return written, err
}
//nolint:gosec // declaredSize is bounded to MaxInt64 above
if written != int64(declaredSize) {
return written, fmt.Errorf("pages declared size %d does not match actual %d", declaredSize, written)
}
return written, nil
}
func writeEntryFile(targetPath string, src io.Reader, declaredSize uint64, maxBytes int64, perm os.FileMode) (int64, error) {
if err := os.MkdirAll(filepath.Dir(targetPath), dirPerm); err != nil {
return 0, err
}
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) //nolint:gosec // caller validates path under release dir
if err != nil {
return 0, err
}
written, copyErr := copyAndVerifySize(target, src, declaredSize, maxBytes)
closeErr := target.Close()
if err := errors.Join(copyErr, closeErr); err != nil {
_ = os.Remove(targetPath)
return written, err
}
return written, nil
}
@@ -0,0 +1,255 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"archive/tar"
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
)
// formatDetectHeadBytes is the sniff window used for archive format detection.
const formatDetectHeadBytes = 512
// ExtractOptions controls package extraction.
type ExtractOptions struct {
// Limits bounds actual files and sizes during extraction when EnforceLimits is true.
Limits Limits
// StripCommonRoot strips a single shared top-level directory when present.
StripCommonRoot bool
// EnforceLimits enables MaxFiles / MaxFileBytes / MaxTotalBytes checks.
// Path, member type, and declared/actual-size validation always remain enabled.
EnforceLimits bool
}
// ExtractBytes extracts a deployment package into destDir.
func ExtractBytes(data []byte, format Format, destDir string, opts ExtractOptions) error {
if format == "" {
var err error
format, err = DetectFormat("", data)
if err != nil {
return err
}
}
return extractFromReaderAt(bytes.NewReader(data), int64(len(data)), format, destDir, opts)
}
// ExtractFile opens path and extracts it into destDir without buffering the
// whole archive or tar member bodies in memory.
func ExtractFile(filePath string, format Format, destDir string, opts ExtractOptions) error {
file, err := os.Open(filePath) //nolint:gosec // controlled path
if err != nil {
return err
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return err
}
if format == "" {
head := make([]byte, formatDetectHeadBytes)
n, readErr := file.ReadAt(head, 0)
if readErr != nil && readErr != io.EOF {
return readErr
}
format, err = DetectFormat(filePath, head[:n])
if err != nil {
return err
}
}
return extractFromReaderAt(file, info.Size(), format, destDir, opts)
}
func extractFromReaderAt(ra io.ReaderAt, size int64, format Format, destDir string, opts ExtractOptions) error {
if isTarFamily(format) {
return extractTarFamilyAt(ra, size, format, destDir, opts)
}
entries, err := listRandomAccessEntriesAt(ra, size, format)
if err != nil {
return err
}
return extractEntries(entries, destDir, opts)
}
func extractEntries(entries []Entry, destDir string, opts ExtractOptions) error {
limits := Limits{}
if opts.EnforceLimits {
limits = normalizeLimits(opts.Limits)
}
commonPrefix, err := commonRootForEntries(entries, opts.StripCommonRoot)
if err != nil {
return err
}
measured := &measuredArchive{files: make([]measuredFile, 0, len(entries))}
for _, entry := range entries {
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return err
}
if skip {
continue
}
normalizedPath = StripPrefix(normalizedPath, commonPrefix)
if normalizedPath == "" {
continue
}
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, opts.EnforceLimits); err != nil {
return err
}
if entry.Open == nil {
return fmt.Errorf("%s: pages archive member cannot be opened", normalizedPath)
}
src, err := entry.Open()
if err != nil {
return fmt.Errorf("%s: %w", normalizedPath, err)
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, opts.EnforceLimits)
targetPath, err := safeExtractionTarget(destDir, normalizedPath)
if err != nil {
_ = src.Close()
return err
}
actual, writeErr := writeEntryFile(targetPath, src, entry.Size, maxBytes, filePerm)
closeErr := src.Close()
if err := errors.Join(writeErr, closeErr); err != nil {
return fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
if measured.fileCount == 0 {
return errors.New("pages package is empty")
}
return nil
}
func extractTarFamilyAt(ra io.ReaderAt, size int64, format Format, destDir string, opts ExtractOptions) error {
limits := Limits{}
if opts.EnforceLimits {
limits = normalizeLimits(opts.Limits)
}
firstPass, err := scanTarFamilyAt(ra, size, format, limits, opts.EnforceLimits)
if err != nil {
return err
}
if firstPass.fileCount == 0 {
return errors.New("pages package is empty")
}
commonPrefix := ""
if opts.StripCommonRoot {
paths := make([]string, 0, len(firstPass.files))
for _, file := range firstPass.files {
paths = append(paths, file.path)
}
commonPrefix = FindCommonRootPrefix(paths)
}
tarReader, closeReader, err := openTarFamilyReader(io.NewSectionReader(ra, 0, size), format)
if err != nil {
return err
}
secondPass, extractErr := extractTarReader(tarReader, destDir, commonPrefix, limits, opts.EnforceLimits)
if closeErr := closeReader(); closeErr != nil {
extractErr = errors.Join(extractErr, closeErr)
}
if extractErr != nil {
return extractErr
}
if secondPass.fileCount != firstPass.fileCount || secondPass.totalSize != firstPass.totalSize {
return errors.New("pages tar package changed between validation and extraction")
}
return nil
}
func extractTarReader(
tarReader *tar.Reader,
destDir string,
commonPrefix string,
limits Limits,
enforceLimits bool,
) (*measuredArchive, error) {
measured := &measuredArchive{files: make([]measuredFile, 0)}
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("read tar pages package: %w", err)
}
entry := entryFromTarHeader(header)
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return nil, err
}
if skip {
continue
}
normalizedPath = StripPrefix(normalizedPath, commonPrefix)
if normalizedPath == "" {
continue
}
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, enforceLimits); err != nil {
return nil, err
}
targetPath, err := safeExtractionTarget(destDir, normalizedPath)
if err != nil {
return nil, err
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, enforceLimits)
actual, err := writeEntryFile(targetPath, tarReader, entry.Size, maxBytes, filePerm)
if err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
return measured, nil
}
func commonRootForEntries(entries []Entry, strip bool) (string, error) {
paths := make([]string, 0, len(entries))
for _, entry := range entries {
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return "", err
}
if !skip {
paths = append(paths, normalizedPath)
}
}
if !strip {
return "", nil
}
return FindCommonRootPrefix(paths), nil
}
func safeExtractionTarget(destDir, relativePath string) (string, error) {
targetPath := filepath.Join(destDir, filepath.FromSlash(relativePath))
if !isWithinDir(destDir, targetPath) {
return "", fmt.Errorf("pages package path escapes directory: %s", relativePath)
}
return targetPath, nil
}
func isWithinDir(baseDir, targetPath string) bool {
cleanBase := filepath.Clean(baseDir)
cleanTarget := filepath.Clean(targetPath)
rel, err := filepath.Rel(cleanBase, cleanTarget)
if err != nil {
return false
}
return rel != ".." && !hasParentRel(rel)
}
func hasParentRel(rel string) bool {
if rel == ".." {
return true
}
return len(rel) >= 3 && (rel[:3] == "../" || rel[:3] == "..\\")
}
@@ -0,0 +1,191 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package pagesarchive provides multi-format archive detection, inspection and
// extraction helpers for OpenFlare Pages deployment packages.
package pagesarchive
import (
"bytes"
"errors"
"path/filepath"
"strings"
)
// Format identifies a supported Pages deployment package archive format.
type Format string
const (
// FormatZip is a ZIP archive.
FormatZip Format = "zip"
// FormatTar is an uncompressed tar archive.
FormatTar Format = "tar"
// FormatTarGz is a gzip-compressed tar archive.
FormatTarGz Format = "tar.gz"
// FormatTarXz is an xz-compressed tar archive.
FormatTarXz Format = "tar.xz"
// FormatTarBz2 is a bzip2-compressed tar archive.
FormatTarBz2 Format = "tar.bz2"
// FormatSevenZip is a 7z archive.
FormatSevenZip Format = "7z"
ustarMagicOffset = 257
ustarMagicMinLen = 262
)
// DetectFormatFromName returns the archive format inferred from a file name.
// Returns an empty Format and false when the extension is unsupported.
func DetectFormatFromName(fileName string) (Format, bool) {
name := strings.ToLower(strings.TrimSpace(fileName))
switch {
case strings.HasSuffix(name, ".tar.gz"), strings.HasSuffix(name, ".tgz"):
return FormatTarGz, true
case strings.HasSuffix(name, ".tar.xz"), strings.HasSuffix(name, ".txz"):
return FormatTarXz, true
case strings.HasSuffix(name, ".tar.bz2"), strings.HasSuffix(name, ".tbz2"), strings.HasSuffix(name, ".tbz"):
return FormatTarBz2, true
case strings.HasSuffix(name, ".tar"):
return FormatTar, true
case strings.HasSuffix(name, ".7z"):
return FormatSevenZip, true
case strings.HasSuffix(name, ".zip"):
return FormatZip, true
default:
return "", false
}
}
// DetectFormatFromBytes returns the archive format inferred from magic bytes.
// Prefer DetectFormatFromName when a reliable file name is available.
func DetectFormatFromBytes(data []byte) (Format, bool) {
if isZipMagic(data) {
return FormatZip, true
}
if isSevenZipMagic(data) {
return FormatSevenZip, true
}
if isXZMagic(data) {
return FormatTarXz, true
}
if isGzipMagic(data) {
return FormatTarGz, true
}
if isBzip2Magic(data) {
return FormatTarBz2, true
}
if looksLikeTar(data) {
return FormatTar, true
}
return "", false
}
// DetectFormat prefers the file name when present, otherwise magic bytes.
func DetectFormat(fileName string, data []byte) (Format, error) {
if format, ok := DetectFormatFromName(fileName); ok {
return format, nil
}
if format, ok := DetectFormatFromBytes(data); ok {
return format, nil
}
return "", errors.New("unsupported pages package format")
}
// Extension returns the canonical file extension for a format (without leading dot).
func Extension(format Format) string {
switch format {
case FormatZip:
return "zip"
case FormatTar:
return "tar"
case FormatTarGz:
return "tar.gz"
case FormatTarXz:
return "tar.xz"
case FormatTarBz2:
return "tar.bz2"
case FormatSevenZip:
return "7z"
default:
return "bin"
}
}
// MIMEType returns a reasonable content type for the archive format.
func MIMEType(format Format) string {
switch format {
case FormatZip:
return "application/zip"
case FormatTar:
return "application/x-tar"
case FormatTarGz:
return "application/gzip"
case FormatTarXz:
return "application/x-xz"
case FormatTarBz2:
return "application/x-bzip2"
case FormatSevenZip:
return "application/x-7z-compressed"
default:
return "application/octet-stream"
}
}
// SupportedExtensions lists human-readable extensions for UI copy and accept attributes.
func SupportedExtensions() []string {
return []string{".zip", ".tar.gz", ".tgz", ".tar.xz", ".txz", ".tar.bz2", ".tbz2", ".tar", ".7z"}
}
// AcceptAttribute returns a comma-separated accept list for file inputs.
func AcceptAttribute() string {
return strings.Join(SupportedExtensions(), ",")
}
// NormalizeNameExtension returns a storage-safe extension for the given format/name.
func NormalizeNameExtension(fileName string, format Format) string {
if format != "" {
return Extension(format)
}
if formatFromName, ok := DetectFormatFromName(fileName); ok {
return Extension(formatFromName)
}
ext := strings.TrimPrefix(filepath.Ext(fileName), ".")
if ext == "" {
return "bin"
}
return strings.ToLower(ext)
}
func isZipMagic(data []byte) bool {
return len(data) >= 4 &&
data[0] == 0x50 && data[1] == 0x4b &&
(data[2] == 0x03 || data[2] == 0x05 || data[2] == 0x07) &&
(data[3] == 0x04 || data[3] == 0x06 || data[3] == 0x08)
}
func isSevenZipMagic(data []byte) bool {
return len(data) >= 6 &&
data[0] == 0x37 && data[1] == 0x7a && data[2] == 0xbc &&
data[3] == 0xaf && data[4] == 0x27 && data[5] == 0x1c
}
func isXZMagic(data []byte) bool {
return len(data) >= 6 &&
data[0] == 0xfd && data[1] == 0x37 && data[2] == 0x7a &&
data[3] == 0x58 && data[4] == 0x5a && data[5] == 0x00
}
func isGzipMagic(data []byte) bool {
return len(data) >= 2 && data[0] == 0x1f && data[1] == 0x8b
}
func isBzip2Magic(data []byte) bool {
return len(data) >= 3 && data[0] == 0x42 && data[1] == 0x5a && data[2] == 0x68
}
func looksLikeTar(data []byte) bool {
// POSIX ustar magic at offset 257 ("ustar\0" or "ustar ").
if len(data) < ustarMagicMinLen {
return false
}
return bytes.Equal(data[ustarMagicOffset:ustarMagicOffset+5], []byte("ustar"))
}
@@ -0,0 +1,303 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"archive/tar"
"bytes"
"errors"
"fmt"
"io"
"math"
"os"
"path"
)
// InspectOptions controls package inspection.
type InspectOptions struct {
// RootDir is an optional project root subdirectory that must contain EntryFile.
RootDir string
// EntryFile is the required entry file name (e.g. index.html).
EntryFile string
// Limits bounds files and actual extracted sizes.
Limits Limits
// VerifySizes is retained for source compatibility. Inspection now always
// streams regular members and verifies actual bytes against declared sizes.
VerifySizes bool
}
type measuredFile struct {
path string
size int64
}
type measuredArchive struct {
files []measuredFile
fileCount int
totalSize int64
}
// InspectFile opens path and inspects it as a Pages deployment package without
// loading the whole archive or any tar member body into memory.
func InspectFile(filePath string, format Format, opts InspectOptions) (*Manifest, error) {
file, err := os.Open(filePath) //nolint:gosec // filePath is a controlled temp upload path
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return nil, err
}
if format == "" {
head := make([]byte, formatDetectHeadBytes)
n, readErr := file.ReadAt(head, 0)
if readErr != nil && readErr != io.EOF {
return nil, readErr
}
format, err = DetectFormat(filePath, head[:n])
if err != nil {
return nil, err
}
}
return inspectFromReaderAt(file, info.Size(), format, opts)
}
// InspectBytes inspects an in-memory deployment package.
func InspectBytes(data []byte, format Format, opts InspectOptions) (*Manifest, error) {
if format == "" {
var err error
format, err = DetectFormat("", data)
if err != nil {
return nil, err
}
}
return inspectFromReaderAt(bytes.NewReader(data), int64(len(data)), format, opts)
}
func inspectFromReaderAt(ra io.ReaderAt, size int64, format Format, opts InspectOptions) (*Manifest, error) {
limits := normalizeLimits(opts.Limits)
var (
measured *measuredArchive
err error
)
if isTarFamily(format) {
measured, err = scanTarFamilyAt(ra, size, format, limits, true)
} else {
var entries []Entry
entries, err = listRandomAccessEntriesAt(ra, size, format)
if err == nil {
measured, err = inspectRandomAccessEntries(entries, limits)
}
}
if err != nil {
return nil, err
}
return buildMeasuredManifest(measured, opts)
}
func inspectRandomAccessEntries(entries []Entry, limits Limits) (*measuredArchive, error) {
measured := &measuredArchive{files: make([]measuredFile, 0, len(entries))}
for _, entry := range entries {
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return nil, err
}
if skip {
continue
}
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, true); err != nil {
return nil, err
}
if entry.Open == nil {
return nil, fmt.Errorf("%s: pages archive member cannot be opened", normalizedPath)
}
src, err := entry.Open()
if err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, true)
actual, copyErr := copyAndVerifySize(io.Discard, src, entry.Size, maxBytes)
closeErr := src.Close()
if err := errors.Join(copyErr, closeErr); err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
return measured, nil
}
func scanTarFamilyAt(
ra io.ReaderAt,
size int64,
format Format,
limits Limits,
enforceLimits bool,
) (*measuredArchive, error) {
if size < 0 {
return nil, errors.New("invalid pages package size")
}
tarReader, closeReader, err := openTarFamilyReader(io.NewSectionReader(ra, 0, size), format)
if err != nil {
return nil, err
}
measured, scanErr := scanTarReader(tarReader, limits, enforceLimits)
if closeErr := closeReader(); closeErr != nil {
scanErr = errors.Join(scanErr, closeErr)
}
return measured, scanErr
}
func scanTarReader(tarReader *tar.Reader, limits Limits, enforceLimits bool) (*measuredArchive, error) {
measured := &measuredArchive{files: make([]measuredFile, 0)}
for {
header, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("read tar pages package: %w", err)
}
entry := entryFromTarHeader(header)
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return nil, err
}
if skip {
continue
}
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, enforceLimits); err != nil {
return nil, err
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, enforceLimits)
actual, err := copyAndVerifySize(io.Discard, tarReader, entry.Size, maxBytes)
if err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
return measured, nil
}
func buildMeasuredManifest(measured *measuredArchive, opts InspectOptions) (*Manifest, error) {
if measured == nil || measured.fileCount == 0 {
return nil, errors.New("pages package is empty")
}
targetEntryPath, err := resolveTargetEntryPath(opts.RootDir, opts.EntryFile)
if err != nil {
return nil, err
}
paths := make([]string, 0, len(measured.files))
for _, file := range measured.files {
paths = append(paths, file.path)
}
commonPrefix := FindCommonRootPrefix(paths)
manifest := &Manifest{
Files: make([]FileEntry, 0, measured.fileCount),
FileCount: measured.fileCount,
TotalSize: measured.totalSize,
}
entrySeen := false
for _, file := range measured.files {
normalizedPath := StripPrefix(file.path, commonPrefix)
if normalizedPath == targetEntryPath {
entrySeen = true
}
manifest.Files = append(manifest.Files, FileEntry{
Path: normalizedPath,
Size: file.size,
})
}
if !entrySeen {
return nil, fmt.Errorf("pages package is missing entry file %s", targetEntryPath)
}
return manifest, nil
}
func prepareMeasuredFile(measured *measuredArchive, normalizedPath string, declaredSize uint64, limits Limits, enforceLimits bool) error {
if declaredSize > uint64(math.MaxInt64) {
return fmt.Errorf("%s: pages file size out of bounds", normalizedPath)
}
if !enforceLimits {
return nil
}
if measured.fileCount >= limits.MaxFiles {
return fmt.Errorf("pages deployment file count exceeds %d", limits.MaxFiles)
}
if exceedsFileByteLimit(declaredSize, limits.MaxFileBytes) {
return fmt.Errorf("pages file too large: %s", normalizedPath)
}
remaining := limits.MaxTotalBytes - measured.totalSize
if remaining < 0 || declaredSize > uint64(remaining) { //nolint:gosec // remaining is checked non-negative
return errors.New("pages extracted size exceeds limit")
}
return nil
}
func appendMeasuredFile(measured *measuredArchive, normalizedPath string, actual int64) {
measured.files = append(measured.files, measuredFile{path: normalizedPath, size: actual})
measured.fileCount++
measured.totalSize += actual
}
func effectiveFileLimit(limits Limits, totalSize int64, enforceLimits bool) int64 {
if !enforceLimits {
return -1
}
remaining := limits.MaxTotalBytes - totalSize
if remaining < limits.MaxFileBytes {
return remaining
}
return limits.MaxFileBytes
}
func resolveTargetEntryPath(rootDir, entryFile string) (string, error) {
normalizedRoot, err := NormalizeLogicalPath(rootDir, true)
if err != nil {
return "", fmt.Errorf("invalid pages root directory: %w", err)
}
if entryFile == "" {
entryFile = "index.html"
}
normalizedEntry, err := NormalizeLogicalPath(entryFile, false)
if err != nil {
return "", fmt.Errorf("invalid pages entry file: %w", err)
}
if normalizedRoot == "" {
return normalizedEntry, nil
}
return path.Join(normalizedRoot, normalizedEntry), nil
}
func validateArchiveEntry(entry Entry) (string, bool, error) {
normalizedPath, skip, err := NormalizeEntryPath(entry.Name)
if err != nil {
return "", false, err
}
if entry.IsSymlink {
return "", false, fmt.Errorf("pages package contains unsupported symlink: %s", normalizedPath)
}
if entry.IsHardlink {
return "", false, fmt.Errorf("pages package contains unsupported hardlink: %s", normalizedPath)
}
if entry.IsSpecial {
return "", false, fmt.Errorf("pages package contains unsupported special entry: %s", normalizedPath)
}
if skip || entry.IsDir {
return normalizedPath, true, nil
}
return normalizedPath, false, nil
}
func isTarFamily(format Format) bool {
switch format {
case FormatTar, FormatTarGz, FormatTarXz, FormatTarBz2:
return true
case FormatZip, FormatSevenZip:
return false
default:
return false
}
}
@@ -0,0 +1,38 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
const (
defaultMaxFiles = 1000
defaultMaxFileBytes = 100 * 1024 * 1024
defaultMaxTotalBytes = 100 * 1024 * 1024
dirPerm = 0o750
filePerm = 0o644
)
// normalizeLimits applies defaults for control-plane inspection.
// Callers that already validated the package should use EnforceLimits=false instead.
func normalizeLimits(limits Limits) Limits {
if limits.MaxFiles <= 0 {
limits.MaxFiles = defaultMaxFiles
}
if limits.MaxFileBytes <= 0 {
limits.MaxFileBytes = defaultMaxFileBytes
}
if limits.MaxTotalBytes <= 0 {
limits.MaxTotalBytes = defaultMaxTotalBytes
}
return limits
}
func exceedsFileByteLimit(size uint64, maxBytes int64) bool {
// maxBytes <= 0 means unlimited (trusted extract path).
if maxBytes <= 0 {
return false
}
if size == 0 {
return false
}
return size > uint64(maxBytes) //nolint:gosec // maxBytes is positive
}
@@ -0,0 +1,157 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"archive/tar"
"archive/zip"
"compress/bzip2"
"compress/gzip"
"errors"
"fmt"
"io"
"os"
"github.com/bodgit/sevenzip"
"github.com/ulikunitz/xz"
)
type archiveFile interface {
Name() string
Mode() os.FileMode
IsDir() bool
Size() uint64
Open() (io.ReadCloser, error)
}
type zipArchiveFile struct {
file *zip.File
}
func (z zipArchiveFile) Name() string { return z.file.Name }
func (z zipArchiveFile) Mode() os.FileMode { return z.file.Mode() }
func (z zipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
func (z zipArchiveFile) Size() uint64 { return z.file.UncompressedSize64 }
func (z zipArchiveFile) Open() (io.ReadCloser, error) {
return z.file.Open()
}
type sevenZipArchiveFile struct {
file *sevenzip.File
}
func (z sevenZipArchiveFile) Name() string { return z.file.Name }
func (z sevenZipArchiveFile) Mode() os.FileMode { return z.file.Mode() }
func (z sevenZipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
func (z sevenZipArchiveFile) Size() uint64 { return z.file.UncompressedSize }
func (z sevenZipArchiveFile) Open() (io.ReadCloser, error) {
return z.file.Open()
}
// listRandomAccessEntriesAt lists zip/7z members without reading their bodies.
// Tar-family archives use the sequential streaming paths in inspect.go/extract.go.
func listRandomAccessEntriesAt(ra io.ReaderAt, size int64, format Format) ([]Entry, error) {
if size < 0 {
return nil, errors.New("invalid pages package size")
}
switch format {
case FormatZip:
return listZipEntriesAt(ra, size)
case FormatSevenZip:
return listSevenZipEntriesAt(ra, size)
case FormatTar, FormatTarGz, FormatTarXz, FormatTarBz2:
return nil, fmt.Errorf("unsupported random-access pages package format: %s", format)
default:
return nil, fmt.Errorf("unsupported random-access pages package format: %s", format)
}
}
func entriesFromArchiveFiles(files []archiveFile) []Entry {
entries := make([]Entry, 0, len(files))
for _, item := range files {
file := item
mode := file.Mode()
isDir := file.IsDir()
isSymlink := mode&os.ModeSymlink != 0
isSpecial := !isDir && !isSymlink && !mode.IsRegular()
entries = append(entries, Entry{
Name: file.Name(),
IsDir: isDir,
IsSymlink: isSymlink,
IsSpecial: isSpecial,
Size: file.Size(),
Open: file.Open,
})
}
return entries
}
func listZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
reader, err := zip.NewReader(ra, size)
if err != nil {
return nil, fmt.Errorf("open zip pages package: %w", err)
}
files := make([]archiveFile, 0, len(reader.File))
for _, item := range reader.File {
files = append(files, zipArchiveFile{file: item})
}
return entriesFromArchiveFiles(files), nil
}
func listSevenZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
reader, err := sevenzip.NewReader(ra, size)
if err != nil {
return nil, fmt.Errorf("open 7z pages package: %w", err)
}
files := make([]archiveFile, 0, len(reader.File))
for _, item := range reader.File {
files = append(files, sevenZipArchiveFile{file: item})
}
return entriesFromArchiveFiles(files), nil
}
func openTarFamilyReader(r io.Reader, format Format) (*tar.Reader, func() error, error) {
switch format {
case FormatTar:
return tar.NewReader(r), func() error { return nil }, nil
case FormatTarGz:
gzReader, err := gzip.NewReader(r)
if err != nil {
return nil, nil, fmt.Errorf("open gzip pages package: %w", err)
}
return tar.NewReader(gzReader), gzReader.Close, nil
case FormatTarXz:
xzReader, err := xz.NewReader(r)
if err != nil {
return nil, nil, fmt.Errorf("open xz pages package: %w", err)
}
return tar.NewReader(xzReader), func() error { return nil }, nil
case FormatTarBz2:
return tar.NewReader(bzip2.NewReader(r)), func() error { return nil }, nil
case FormatZip, FormatSevenZip:
return nil, nil, fmt.Errorf("unsupported tar family format: %s", format)
default:
return nil, nil, fmt.Errorf("unsupported tar family format: %s", format)
}
}
func entryFromTarHeader(header *tar.Header) Entry {
entry := Entry{Name: header.Name}
if header.Size > 0 {
entry.Size = uint64(header.Size) //nolint:gosec // archive/tar rejects negative sizes
}
switch header.Typeflag {
case tar.TypeReg, tar.TypeRegA: //nolint:staticcheck // TypeRegA appears in older archives
// Regular file.
case tar.TypeDir:
entry.IsDir = true
case tar.TypeSymlink:
entry.IsSymlink = true
case tar.TypeLink:
entry.IsHardlink = true
default:
entry.IsSpecial = true
}
return entry
}
@@ -0,0 +1,233 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/ulikunitz/xz"
)
func TestDetectFormatFromName(t *testing.T) {
cases := map[string]Format{
"site.zip": FormatZip,
"site.TAR.GZ": FormatTarGz,
"site.tgz": FormatTarGz,
"site.tar.xz": FormatTarXz,
"site.txz": FormatTarXz,
"site.tar.bz2": FormatTarBz2,
"site.tar": FormatTar,
"site.7z": FormatSevenZip,
}
for name, want := range cases {
got, ok := DetectFormatFromName(name)
assert.True(t, ok, name)
assert.Equal(t, want, got, name)
}
_, ok := DetectFormatFromName("site.rar")
assert.False(t, ok)
}
func TestInspectAndExtractZip(t *testing.T) {
data := testZip(t, map[string]string{
"dist/index.html": "<html>ok</html>",
"dist/app.js": "console.log(1)",
})
manifest, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.NoError(t, err)
assert.Equal(t, 2, manifest.FileCount)
paths := make(map[string]struct{}, len(manifest.Files))
for _, file := range manifest.Files {
paths[file.Path] = struct{}{}
assert.Empty(t, file.Checksum, "per-file checksum should not be computed")
assert.Positive(t, file.Size)
}
assert.Contains(t, paths, "index.html")
assert.Contains(t, paths, "app.js")
assert.Equal(t, int64(len("<html>ok</html>")+len("console.log(1)")), manifest.TotalSize)
dest := t.TempDir()
require.NoError(t, ExtractBytes(data, FormatZip, dest, ExtractOptions{
StripCommonRoot: true,
EnforceLimits: true,
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
}))
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
require.NoError(t, err)
assert.Equal(t, "<html>ok</html>", string(body))
}
func TestInspectFileUsesDeclaredSizesWithoutHash(t *testing.T) {
data := testZip(t, map[string]string{
"index.html": "<html>disk</html>",
"asset.css": "body{}",
})
path := filepath.Join(t.TempDir(), "site.zip")
require.NoError(t, os.WriteFile(path, data, 0o600))
manifest, err := InspectFile(path, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.NoError(t, err)
require.Equal(t, 2, manifest.FileCount)
for _, file := range manifest.Files {
assert.Empty(t, file.Checksum)
}
dest := t.TempDir()
require.NoError(t, ExtractFile(path, FormatZip, dest, ExtractOptions{
EnforceLimits: true,
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
}))
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
require.NoError(t, err)
assert.Equal(t, "<html>disk</html>", string(body))
}
func TestInspectVerifySizesOptional(t *testing.T) {
data := testZip(t, map[string]string{
"index.html": "verify-me",
})
manifest, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
VerifySizes: true,
Limits: Limits{MaxFiles: 10, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.NoError(t, err)
require.Len(t, manifest.Files, 1)
assert.Equal(t, int64(len("verify-me")), manifest.Files[0].Size)
assert.Empty(t, manifest.Files[0].Checksum)
}
func TestExtractTrustedSkipsSizeLimits(t *testing.T) {
// Content larger than a tiny limit would fail if limits were enforced.
large := strings.Repeat("x", 64)
data := testZip(t, map[string]string{
"index.html": large,
})
dest := t.TempDir()
require.NoError(t, ExtractBytes(data, FormatZip, dest, ExtractOptions{
// Agent trusts control-plane validation: no size/count re-check.
EnforceLimits: false,
}))
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
require.NoError(t, err)
assert.Equal(t, large, string(body))
}
func TestInspectAndExtractTarGz(t *testing.T) {
data := testTarGz(t, map[string]string{
"index.html": "<html>tar</html>",
"style.css": "body{}",
})
format, err := DetectFormat("site.tar.gz", data)
require.NoError(t, err)
assert.Equal(t, FormatTarGz, format)
manifest, err := InspectBytes(data, format, InspectOptions{
EntryFile: "index.html",
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.NoError(t, err)
assert.Equal(t, 2, manifest.FileCount)
dest := t.TempDir()
require.NoError(t, ExtractBytes(data, format, dest, ExtractOptions{
EnforceLimits: true,
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
}))
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
require.NoError(t, err)
assert.Equal(t, "<html>tar</html>", string(body))
}
func TestInspectTarXz(t *testing.T) {
data := testTarXz(t, map[string]string{
"index.html": "<html>xz</html>",
})
manifest, err := InspectBytes(data, FormatTarXz, InspectOptions{
EntryFile: "index.html",
Limits: Limits{MaxFiles: 10, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.NoError(t, err)
assert.Equal(t, 1, manifest.FileCount)
}
func TestRejectZipSlip(t *testing.T) {
data := testZip(t, map[string]string{
"../evil.txt": "x",
"index.html": "ok",
})
_, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{MaxFiles: 10, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.Error(t, err)
}
func testZip(t *testing.T, files map[string]string) []byte {
t.Helper()
var buffer bytes.Buffer
writer := zip.NewWriter(&buffer)
for name, content := range files {
file, err := writer.Create(name)
require.NoError(t, err)
_, err = file.Write([]byte(content))
require.NoError(t, err)
}
require.NoError(t, writer.Close())
return buffer.Bytes()
}
func testTarGz(t *testing.T, files map[string]string) []byte {
t.Helper()
var buffer bytes.Buffer
gzWriter := gzip.NewWriter(&buffer)
tarWriter := tar.NewWriter(gzWriter)
for name, content := range files {
require.NoError(t, tarWriter.WriteHeader(&tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(content)),
}))
_, err := tarWriter.Write([]byte(content))
require.NoError(t, err)
}
require.NoError(t, tarWriter.Close())
require.NoError(t, gzWriter.Close())
return buffer.Bytes()
}
func testTarXz(t *testing.T, files map[string]string) []byte {
t.Helper()
var buffer bytes.Buffer
xzWriter, err := xz.NewWriter(&buffer)
require.NoError(t, err)
tarWriter := tar.NewWriter(xzWriter)
for name, content := range files {
require.NoError(t, tarWriter.WriteHeader(&tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(content)),
}))
_, err := tarWriter.Write([]byte(content))
require.NoError(t, err)
}
require.NoError(t, tarWriter.Close())
require.NoError(t, xzWriter.Close())
return buffer.Bytes()
}
@@ -0,0 +1,142 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"errors"
"fmt"
"path"
"strings"
"unicode"
"unicode/utf8"
)
// NormalizeLogicalPath validates and normalizes a relative POSIX path.
// Empty input is returned unchanged only when allowEmpty is true.
func NormalizeLogicalPath(raw string, allowEmpty bool) (string, error) {
if raw == "" {
if allowEmpty {
return "", nil
}
return "", errors.New("pages path is required")
}
if err := validateLogicalPathText(raw); err != nil {
return "", err
}
cleaned := path.Clean(raw)
if cleaned == "." || cleaned == "" {
if allowEmpty {
return "", nil
}
return "", errors.New("pages path is required")
}
if strings.HasPrefix(cleaned, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
return "", fmt.Errorf("pages path escapes directory: %s", raw)
}
return cleaned, nil
}
func validateLogicalPathText(raw string) error {
if !utf8.ValidString(raw) {
return errors.New("pages path is not valid UTF-8")
}
if strings.Contains(raw, "\\") {
return fmt.Errorf("pages path must use POSIX separators: %s", raw)
}
if strings.HasPrefix(raw, "/") || path.IsAbs(raw) {
return fmt.Errorf("pages path must be relative: %s", raw)
}
if err := validateLogicalPathRunes(raw); err != nil {
return err
}
return validateLogicalPathSegments(raw)
}
func validateLogicalPathRunes(raw string) error {
for _, r := range raw {
if r == 0 || unicode.IsControl(r) {
return errors.New("pages path contains a control character")
}
if r == '\'' || r == '"' || r == ';' {
return fmt.Errorf("pages path contains an unsupported character: %s", raw)
}
}
return nil
}
func validateLogicalPathSegments(raw string) error {
for segment := range strings.SplitSeq(raw, "/") {
if len(segment) >= 2 && segment[1] == ':' {
return fmt.Errorf("pages path contains a Windows drive: %s", raw)
}
if segment == "." || segment == ".." {
return fmt.Errorf("pages path escapes directory or contains a dot segment: %s", raw)
}
}
return nil
}
// NormalizeEntryPath cleans an archive entry path and rejects zip-slip / absolute paths.
// skip=true means the entry should be ignored (empty path or directory marker).
func NormalizeEntryPath(raw string) (cleaned string, skip bool, err error) {
if raw == "" {
return "", true, nil
}
cleanedPath, normalizeErr := NormalizeLogicalPath(raw, false)
if normalizeErr != nil {
return "", false, fmt.Errorf("invalid pages package path %q: %w", raw, normalizeErr)
}
if strings.HasSuffix(raw, "/") {
return cleanedPath, true, nil
}
return cleanedPath, false, nil
}
// FindCommonRootPrefix returns a trailing-slash directory prefix shared by all file paths.
// When files do not share a single root folder the result is empty.
func FindCommonRootPrefix(paths []string) string {
var firstFilePath string
hasMultipleFiles := false
for _, item := range paths {
normalizedPath, skip, err := NormalizeEntryPath(item)
if err != nil || skip {
continue
}
if firstFilePath == "" {
firstFilePath = normalizedPath
} else {
hasMultipleFiles = true
}
}
if firstFilePath == "" {
return ""
}
parts := strings.Split(firstFilePath, "/")
if len(parts) <= 1 {
return ""
}
commonPrefix := parts[0] + "/"
if !hasMultipleFiles {
return commonPrefix
}
for _, item := range paths {
normalizedPath, skip, err := NormalizeEntryPath(item)
if err != nil || skip {
continue
}
if !strings.HasPrefix(normalizedPath, commonPrefix) {
return ""
}
}
return commonPrefix
}
// StripPrefix removes a directory prefix from a cleaned path when present.
func StripPrefix(normalizedPath, prefix string) string {
if prefix == "" {
return normalizedPath
}
return strings.TrimPrefix(normalizedPath, prefix)
}
@@ -0,0 +1,499 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"archive/tar"
"archive/zip"
"bytes"
"encoding/base64"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var testLimits = Limits{
MaxFiles: 100,
MaxFileBytes: 1 << 20,
MaxTotalBytes: 1 << 20,
}
func TestNormalizeLogicalPathStrict(t *testing.T) {
t.Parallel()
valid := []struct {
name string
raw string
allowEmpty bool
want string
}{
{name: "empty root", allowEmpty: true},
{name: "single file", raw: "index.html", want: "index.html"},
{name: "nested posix", raw: "public/assets/app.js", want: "public/assets/app.js"},
{name: "unicode", raw: "静态/首页.html", want: "静态/首页.html"},
{name: "repeated separator is normalized", raw: "public//app.js", want: "public/app.js"},
}
for _, tt := range valid {
t.Run(tt.name, func(t *testing.T) {
got, err := NormalizeLogicalPath(tt.raw, tt.allowEmpty)
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
invalidUTF8 := string([]byte{'a', '/', 0xff})
invalid := []struct {
name string
raw string
}{
{name: "empty entry"},
{name: "absolute", raw: "/etc/passwd"},
{name: "unc", raw: "//server/share"},
{name: "windows drive", raw: "C:/site/index.html"},
{name: "nested windows drive", raw: "site/C:/index.html"},
{name: "windows separator", raw: `site\index.html`},
{name: "windows unc", raw: `\\server\share`},
{name: "parent segment", raw: "../index.html"},
{name: "nested parent segment", raw: "site/../index.html"},
{name: "current segment", raw: "site/./index.html"},
{name: "nul", raw: "site/\x00index.html"},
{name: "newline", raw: "site/\nindex.html"},
{name: "delete control", raw: "site/\x7findex.html"},
{name: "single quote", raw: "site/'index.html"},
{name: "double quote", raw: `site/"index.html`},
{name: "semicolon", raw: "site/;index.html"},
{name: "invalid utf8", raw: invalidUTF8},
}
for _, tt := range invalid {
t.Run(tt.name, func(t *testing.T) {
_, err := NormalizeLogicalPath(tt.raw, false)
require.Error(t, err)
})
}
cleaned, skip, err := NormalizeEntryPath("")
require.NoError(t, err)
assert.Empty(t, cleaned)
assert.True(t, skip)
cleaned, skip, err = NormalizeEntryPath("assets/")
require.NoError(t, err)
assert.Equal(t, "assets", cleaned)
assert.True(t, skip)
}
func TestSupportedFormatsInspectAndExtract(t *testing.T) {
t.Parallel()
sevenZipData := decodeFixture(t, "N3q8ryccAASgR6WICAAAAAAAAABmAAAAAAAAAN2R8/FiYXIKZm9vCgEEBgACCQQEAAcLAgABAQABAQAMBAQACAoB6bOiBKhlMn4AAAUCGQUAAAAAABERAGIAYQByAAAAZgBvAG8AAAAZAgAAFBIBAACFM3PyY9YBAFgCcvJj1gEVCgEAIICkgSCApIEAAA==")
bzipTarData := decodeFixture(t, "QlpoOTFBWSZTWYp5f6EAAHV//P64A8RQAf/iOm/9cO/v/9AAAgBADlAABAADAAgwAU1RIZJpNCaammmnqbSGTI9Q0BoBpp6mjIaGmmRoaHGRpkxNBkyYTTIGQ0BoDTJoYATQGG1KCntExT01MhoAABoAAHqAAPU9QacVDtN45fA6MmuGVQlWowrijpZgwASITYSPUcpJpoQGMkKq69jMkUR6L86R5j0IySUaZEjazEqhQ9E8vuuxsmWZQLCA84jNsobYNEzuEB1eCPhw8nc2AOz+xrCY5hVxQW1IIokpfSRKi+McvXU+QoYuEg6BD4w8x3K0imi+bULpkLCylCZ4lzoGlTQgibvG67sQcrTCRBTbBCVL7zC0q0qULmK/WOneu94s9cs4s4K98SjY2YvpdZvl42kwtxvvPMheorYQ2pcxyF4sNQYvd4+bgqm5gKXElqnGF3jhxGTeXp9eCUxWVlbi9ikxAik4xxATl7cJrISVWnHwUFiLdhEnKWw0Lhm3ZyKlX7P5Wj7b9TLAmWBaAwH/F3JFOFCQinl/oQ==")
cases := []struct {
name string
format Format
data []byte
entryFile string
wantPath string
}{
{name: "zip", format: FormatZip, data: testZip(t, map[string]string{"bundle/index.html": "zip"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar", format: FormatTar, data: testTar(t, map[string]string{"bundle/index.html": "tar"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar gzip", format: FormatTarGz, data: testTarGz(t, map[string]string{"bundle/index.html": "gzip"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar xz", format: FormatTarXz, data: testTarXz(t, map[string]string{"bundle/index.html": "xz"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar bzip2", format: FormatTarBz2, data: bzipTarData, entryFile: "index.html", wantPath: "index.html"},
{name: "7z", format: FormatSevenZip, data: sevenZipData, entryFile: "foo", wantPath: "foo"},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
manifest, err := InspectBytes(tt.data, tt.format, InspectOptions{
EntryFile: tt.entryFile,
Limits: testLimits,
})
require.NoError(t, err)
assert.Positive(t, manifest.FileCount)
assertManifestContains(t, manifest, tt.wantPath)
destDir := t.TempDir()
require.NoError(t, ExtractBytes(tt.data, tt.format, destDir, ExtractOptions{
Limits: testLimits,
StripCommonRoot: true,
EnforceLimits: true,
}))
_, err = os.Stat(filepath.Join(destDir, filepath.FromSlash(tt.wantPath)))
require.NoError(t, err)
})
}
}
func TestExtractFilePreservesCommonRootAndEnforcesLimits(t *testing.T) {
t.Parallel()
data := testTarGz(t, map[string]string{
"repository/dist/index.html": "pages",
"repository/dist/app.js": "javascript",
})
archivePath := filepath.Join(t.TempDir(), "site.tar.gz")
require.NoError(t, os.WriteFile(archivePath, data, 0o600))
manifest, err := InspectFile(archivePath, FormatTarGz, InspectOptions{
RootDir: "dist",
EntryFile: "index.html",
Limits: testLimits,
})
require.NoError(t, err)
assertManifestContains(t, manifest, "dist/index.html")
destDir := t.TempDir()
require.NoError(t, ExtractFile(archivePath, FormatTarGz, destDir, ExtractOptions{
Limits: testLimits,
StripCommonRoot: true,
EnforceLimits: true,
}))
body, err := os.ReadFile(filepath.Join(destDir, "dist", "index.html")) //nolint:gosec
require.NoError(t, err)
assert.Equal(t, "pages", string(body))
err = ExtractFile(archivePath, FormatTarGz, t.TempDir(), ExtractOptions{
Limits: Limits{
MaxFiles: 10,
MaxFileBytes: 5,
MaxTotalBytes: 1 << 20,
},
StripCommonRoot: true,
EnforceLimits: true,
})
require.ErrorContains(t, err, "file too large")
err = ExtractFile(archivePath, FormatTarGz, t.TempDir(), ExtractOptions{
Limits: Limits{
MaxFiles: 10,
MaxFileBytes: 1 << 20,
MaxTotalBytes: int64(len("pages") + len("javascript") - 1),
},
StripCommonRoot: true,
EnforceLimits: true,
})
require.ErrorContains(t, err, "extracted size exceeds limit")
}
func TestRandomAccessMembersVerifyActualSize(t *testing.T) {
t.Parallel()
entry := Entry{
Name: "index.html",
Size: 1,
Open: func() (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("actual-body")), nil
},
}
_, err := inspectRandomAccessEntries([]Entry{entry}, testLimits)
require.ErrorContains(t, err, "declared size 1 does not match actual 11")
destDir := t.TempDir()
err = extractEntries([]Entry{entry}, destDir, ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
})
require.ErrorContains(t, err, "declared size 1 does not match actual 11")
_, statErr := os.Stat(filepath.Join(destDir, "index.html"))
assert.ErrorIs(t, statErr, os.ErrNotExist, "failed extraction must remove the partial file")
}
func TestActualByteLimitAbortsReaderEarly(t *testing.T) {
t.Parallel()
reader := &countingFillReader{remaining: 1 << 30}
entry := Entry{
Name: "index.html",
Size: 0,
Open: func() (io.ReadCloser, error) {
return io.NopCloser(reader), nil
},
}
_, err := inspectRandomAccessEntries([]Entry{entry}, Limits{
MaxFiles: 1,
MaxFileBytes: 32,
MaxTotalBytes: 32,
})
require.ErrorContains(t, err, "size out of bounds")
assert.LessOrEqual(t, reader.read, int64(33), "inspection must stop after limit+1 actual bytes")
tarData := tarWithDeclaredBodyOnly(t, "index.html", 1<<30)
_, err = InspectBytes(tarData, FormatTar, InspectOptions{
EntryFile: "index.html",
Limits: Limits{
MaxFiles: 1,
MaxFileBytes: 32,
MaxTotalBytes: 32,
},
})
require.ErrorContains(t, err, "file too large")
}
func TestArchiveLimitsUseFilesAndActualTotals(t *testing.T) {
t.Parallel()
data := testZip(t, map[string]string{
"index.html": "1234",
"app.js": "5678",
})
_, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{
MaxFiles: 1,
MaxFileBytes: 8,
MaxTotalBytes: 16,
},
})
require.ErrorContains(t, err, "file count exceeds")
_, err = InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{
MaxFiles: 2,
MaxFileBytes: 8,
MaxTotalBytes: 7,
},
})
require.ErrorContains(t, err, "extracted size exceeds limit")
sevenZipData := decodeFixture(t, "N3q8ryccAASgR6WICAAAAAAAAABmAAAAAAAAAN2R8/FiYXIKZm9vCgEEBgACCQQEAAcLAgABAQABAQAMBAQACAoB6bOiBKhlMn4AAAUCGQUAAAAAABERAGIAYQByAAAAZgBvAG8AAAAZAgAAFBIBAACFM3PyY9YBAFgCcvJj1gEVCgEAIICkgSCApIEAAA==")
_, err = InspectBytes(sevenZipData, FormatSevenZip, InspectOptions{
EntryFile: "foo",
Limits: Limits{
MaxFiles: 10,
MaxFileBytes: 3,
MaxTotalBytes: 32,
},
})
require.ErrorContains(t, err, "file too large")
}
func TestRejectUnsupportedTarMemberTypes(t *testing.T) {
t.Parallel()
cases := []struct {
name string
header tar.Header
wantErr string
}{
{name: "symlink", header: tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "index.html"}, wantErr: "unsupported symlink"},
{name: "hardlink", header: tar.Header{Name: "hard", Typeflag: tar.TypeLink, Linkname: "index.html"}, wantErr: "unsupported hardlink"},
{name: "fifo", header: tar.Header{Name: "pipe", Typeflag: tar.TypeFifo, Mode: 0o600}, wantErr: "unsupported special entry"},
{name: "character device", header: tar.Header{Name: "tty", Typeflag: tar.TypeChar, Mode: 0o600, Devmajor: 1, Devminor: 3}, wantErr: "unsupported special entry"},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
data := tarWithSpecialEntry(t, &tt.header)
_, err := InspectBytes(data, FormatTar, InspectOptions{
EntryFile: "index.html",
Limits: testLimits,
})
require.ErrorContains(t, err, tt.wantErr)
destDir := t.TempDir()
err = ExtractBytes(data, FormatTar, destDir, ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
})
require.ErrorContains(t, err, tt.wantErr)
_, statErr := os.Stat(filepath.Join(destDir, "index.html"))
assert.ErrorIs(t, statErr, os.ErrNotExist, "tar validation pass must reject before writing files")
})
}
}
func TestRejectUnsupportedZipMemberTypes(t *testing.T) {
t.Parallel()
cases := []struct {
name string
mode os.FileMode
wantErr string
}{
{name: "symlink", mode: os.ModeSymlink | 0o777, wantErr: "unsupported symlink"},
{name: "named pipe", mode: os.ModeNamedPipe | 0o600, wantErr: "unsupported special entry"},
{name: "device", mode: os.ModeDevice | 0o600, wantErr: "unsupported special entry"},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
data := zipWithSpecialEntry(t, tt.mode)
_, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: testLimits,
})
require.ErrorContains(t, err, tt.wantErr)
err = ExtractBytes(data, FormatZip, t.TempDir(), ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
})
require.ErrorContains(t, err, tt.wantErr)
})
}
}
func TestTarMetadataHeadersRemainTransparent(t *testing.T) {
t.Parallel()
for _, format := range []tar.Format{tar.FormatPAX, tar.FormatGNU} {
format := format
t.Run(format.String(), func(t *testing.T) {
data := tarWithLongMetadata(t, format)
manifest, err := InspectBytes(data, FormatTar, InspectOptions{
EntryFile: "index.html",
Limits: testLimits,
})
require.NoError(t, err)
assert.Equal(t, 2, manifest.FileCount)
assertManifestContains(t, manifest, "index.html")
destDir := t.TempDir()
require.NoError(t, ExtractBytes(data, FormatTar, destDir, ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
}))
_, err = os.Stat(filepath.Join(destDir, "index.html"))
require.NoError(t, err)
})
}
}
type countingFillReader struct {
remaining int64
read int64
}
func (r *countingFillReader) Read(p []byte) (int, error) {
if r.remaining == 0 {
return 0, io.EOF
}
if int64(len(p)) > r.remaining {
p = p[:r.remaining]
}
for i := range p {
p[i] = 'x'
}
r.remaining -= int64(len(p))
r.read += int64(len(p))
return len(p), nil
}
func decodeFixture(t *testing.T, encoded string) []byte {
t.Helper()
data, err := base64.StdEncoding.DecodeString(encoded)
require.NoError(t, err)
return data
}
func assertManifestContains(t *testing.T, manifest *Manifest, path string) {
t.Helper()
for _, file := range manifest.Files {
if file.Path == path {
return
}
}
require.Failf(t, "manifest path missing", "path %q not found in %#v", path, manifest.Files)
}
func testTar(t *testing.T, files map[string]string) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
for name, content := range files {
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(content)),
}))
_, err := writer.Write([]byte(content))
require.NoError(t, err)
}
require.NoError(t, writer.Close())
return buffer.Bytes()
}
func tarWithDeclaredBodyOnly(t *testing.T, name string, size int64) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: name,
Mode: 0o644,
Size: size,
}))
// Deliberately omit the body and trailer. The limit must reject from the
// header before archive/tar attempts to stream the declared body.
return buffer.Bytes()
}
func tarWithSpecialEntry(t *testing.T, special *tar.Header) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: "index.html",
Mode: 0o644,
Size: 2,
}))
_, err := writer.Write([]byte("ok"))
require.NoError(t, err)
require.NoError(t, writer.WriteHeader(special))
require.NoError(t, writer.Close())
return buffer.Bytes()
}
func zipWithSpecialEntry(t *testing.T, mode os.FileMode) []byte {
t.Helper()
var buffer bytes.Buffer
writer := zip.NewWriter(&buffer)
index, err := writer.Create("index.html")
require.NoError(t, err)
_, err = index.Write([]byte("ok"))
require.NoError(t, err)
header := &zip.FileHeader{Name: "special"}
header.SetMode(mode)
special, err := writer.CreateHeader(header)
require.NoError(t, err)
if mode&os.ModeSymlink != 0 {
_, err = special.Write([]byte("index.html"))
require.NoError(t, err)
}
require.NoError(t, writer.Close())
return buffer.Bytes()
}
func tarWithLongMetadata(t *testing.T, format tar.Format) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
longName := strings.Repeat("long-segment-", 12) + "asset.js"
header := &tar.Header{
Name: longName,
Mode: 0o644,
Size: 1,
Format: format,
}
if format == tar.FormatPAX {
header.PAXRecords = map[string]string{"comment": "metadata is not a deployable member"}
}
require.NoError(t, writer.WriteHeader(header))
_, err := writer.Write([]byte("x"))
require.NoError(t, err)
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: "index.html",
Mode: 0o644,
Size: 2,
Format: format,
}))
_, err = writer.Write([]byte("ok"))
require.NoError(t, err)
require.NoError(t, writer.Close())
return buffer.Bytes()
}