mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-12 10:16:37 +08:00
refactor(backend): rename OpenFlare directory to lowercase openflare
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Limits bounds archive inspection / extraction work.
|
||||
type Limits struct {
|
||||
// MaxFiles is the maximum number of regular files allowed.
|
||||
MaxFiles int
|
||||
// MaxFileBytes is the maximum size of a single extracted file.
|
||||
MaxFileBytes int64
|
||||
// MaxTotalBytes is the maximum sum of all extracted file sizes.
|
||||
MaxTotalBytes int64
|
||||
}
|
||||
|
||||
// FileEntry is a regular file discovered inside a deployment package.
|
||||
type FileEntry struct {
|
||||
Path string
|
||||
Size int64
|
||||
// Checksum is retained for API/schema compatibility and is left empty.
|
||||
// Integrity is enforced via the whole-package SHA-256 on the deployment record.
|
||||
Checksum string
|
||||
}
|
||||
|
||||
// Manifest is the inspected content of a Pages deployment package.
|
||||
type Manifest struct {
|
||||
Files []FileEntry
|
||||
FileCount int
|
||||
TotalSize int64
|
||||
}
|
||||
|
||||
// Entry describes one archive member for extraction.
|
||||
type Entry struct {
|
||||
// Name is the original path inside the archive.
|
||||
Name string
|
||||
// IsDir marks directory entries.
|
||||
IsDir bool
|
||||
// IsSymlink marks symbolic links (unsupported for Pages).
|
||||
IsSymlink bool
|
||||
// IsHardlink marks hard links (unsupported for Pages).
|
||||
IsHardlink bool
|
||||
// IsSpecial marks device, FIFO, socket, and other non-regular entries.
|
||||
IsSpecial bool
|
||||
// Size is the archive-declared uncompressed size; 0 means an empty member.
|
||||
Size uint64
|
||||
// Open returns a reader for the entry body. Caller must Close it.
|
||||
Open func() (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// copyLimited copies actual bytes from src. maxBytes < 0 disables the byte cap;
|
||||
// maxBytes == 0 permits only an empty stream.
|
||||
func copyLimited(dst io.Writer, src io.Reader, maxBytes int64) (int64, error) {
|
||||
if maxBytes < 0 {
|
||||
return io.Copy(dst, src)
|
||||
}
|
||||
|
||||
readLimit := maxBytes
|
||||
if maxBytes < math.MaxInt64 {
|
||||
readLimit++
|
||||
}
|
||||
written, err := io.Copy(dst, io.LimitReader(src, readLimit))
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
if written > maxBytes {
|
||||
return written, errors.New("pages file size out of bounds")
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func copyAndVerifySize(dst io.Writer, src io.Reader, declaredSize uint64, maxBytes int64) (int64, error) {
|
||||
if declaredSize > uint64(math.MaxInt64) {
|
||||
return 0, errors.New("pages file size out of bounds")
|
||||
}
|
||||
written, err := copyLimited(dst, src, maxBytes)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
//nolint:gosec // declaredSize is bounded to MaxInt64 above
|
||||
if written != int64(declaredSize) {
|
||||
return written, fmt.Errorf("pages declared size %d does not match actual %d", declaredSize, written)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func writeEntryFile(targetPath string, src io.Reader, declaredSize uint64, maxBytes int64, perm os.FileMode) (int64, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), dirPerm); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) //nolint:gosec // caller validates path under release dir
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written, copyErr := copyAndVerifySize(target, src, declaredSize, maxBytes)
|
||||
closeErr := target.Close()
|
||||
if err := errors.Join(copyErr, closeErr); err != nil {
|
||||
_ = os.Remove(targetPath)
|
||||
return written, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// formatDetectHeadBytes is the sniff window used for archive format detection.
|
||||
const formatDetectHeadBytes = 512
|
||||
|
||||
// ExtractOptions controls package extraction.
|
||||
type ExtractOptions struct {
|
||||
// Limits bounds actual files and sizes during extraction when EnforceLimits is true.
|
||||
Limits Limits
|
||||
// StripCommonRoot strips a single shared top-level directory when present.
|
||||
StripCommonRoot bool
|
||||
// EnforceLimits enables MaxFiles / MaxFileBytes / MaxTotalBytes checks.
|
||||
// Path, member type, and declared/actual-size validation always remain enabled.
|
||||
EnforceLimits bool
|
||||
}
|
||||
|
||||
// ExtractBytes extracts a deployment package into destDir.
|
||||
func ExtractBytes(data []byte, format Format, destDir string, opts ExtractOptions) error {
|
||||
if format == "" {
|
||||
var err error
|
||||
format, err = DetectFormat("", data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return extractFromReaderAt(bytes.NewReader(data), int64(len(data)), format, destDir, opts)
|
||||
}
|
||||
|
||||
// ExtractFile opens path and extracts it into destDir without buffering the
|
||||
// whole archive or tar member bodies in memory.
|
||||
func ExtractFile(filePath string, format Format, destDir string, opts ExtractOptions) error {
|
||||
file, err := os.Open(filePath) //nolint:gosec // controlled path
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if format == "" {
|
||||
head := make([]byte, formatDetectHeadBytes)
|
||||
n, readErr := file.ReadAt(head, 0)
|
||||
if readErr != nil && readErr != io.EOF {
|
||||
return readErr
|
||||
}
|
||||
format, err = DetectFormat(filePath, head[:n])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return extractFromReaderAt(file, info.Size(), format, destDir, opts)
|
||||
}
|
||||
|
||||
func extractFromReaderAt(ra io.ReaderAt, size int64, format Format, destDir string, opts ExtractOptions) error {
|
||||
if isTarFamily(format) {
|
||||
return extractTarFamilyAt(ra, size, format, destDir, opts)
|
||||
}
|
||||
entries, err := listRandomAccessEntriesAt(ra, size, format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return extractEntries(entries, destDir, opts)
|
||||
}
|
||||
|
||||
func extractEntries(entries []Entry, destDir string, opts ExtractOptions) error {
|
||||
limits := Limits{}
|
||||
if opts.EnforceLimits {
|
||||
limits = normalizeLimits(opts.Limits)
|
||||
}
|
||||
commonPrefix, err := commonRootForEntries(entries, opts.StripCommonRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
measured := &measuredArchive{files: make([]measuredFile, 0, len(entries))}
|
||||
for _, entry := range entries {
|
||||
normalizedPath, skip, err := validateArchiveEntry(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
normalizedPath = StripPrefix(normalizedPath, commonPrefix)
|
||||
if normalizedPath == "" {
|
||||
continue
|
||||
}
|
||||
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, opts.EnforceLimits); err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.Open == nil {
|
||||
return fmt.Errorf("%s: pages archive member cannot be opened", normalizedPath)
|
||||
}
|
||||
src, err := entry.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", normalizedPath, err)
|
||||
}
|
||||
maxBytes := effectiveFileLimit(limits, measured.totalSize, opts.EnforceLimits)
|
||||
targetPath, err := safeExtractionTarget(destDir, normalizedPath)
|
||||
if err != nil {
|
||||
_ = src.Close()
|
||||
return err
|
||||
}
|
||||
actual, writeErr := writeEntryFile(targetPath, src, entry.Size, maxBytes, filePerm)
|
||||
closeErr := src.Close()
|
||||
if err := errors.Join(writeErr, closeErr); err != nil {
|
||||
return fmt.Errorf("%s: %w", normalizedPath, err)
|
||||
}
|
||||
appendMeasuredFile(measured, normalizedPath, actual)
|
||||
}
|
||||
if measured.fileCount == 0 {
|
||||
return errors.New("pages package is empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractTarFamilyAt(ra io.ReaderAt, size int64, format Format, destDir string, opts ExtractOptions) error {
|
||||
limits := Limits{}
|
||||
if opts.EnforceLimits {
|
||||
limits = normalizeLimits(opts.Limits)
|
||||
}
|
||||
firstPass, err := scanTarFamilyAt(ra, size, format, limits, opts.EnforceLimits)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if firstPass.fileCount == 0 {
|
||||
return errors.New("pages package is empty")
|
||||
}
|
||||
commonPrefix := ""
|
||||
if opts.StripCommonRoot {
|
||||
paths := make([]string, 0, len(firstPass.files))
|
||||
for _, file := range firstPass.files {
|
||||
paths = append(paths, file.path)
|
||||
}
|
||||
commonPrefix = FindCommonRootPrefix(paths)
|
||||
}
|
||||
|
||||
tarReader, closeReader, err := openTarFamilyReader(io.NewSectionReader(ra, 0, size), format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
secondPass, extractErr := extractTarReader(tarReader, destDir, commonPrefix, limits, opts.EnforceLimits)
|
||||
if closeErr := closeReader(); closeErr != nil {
|
||||
extractErr = errors.Join(extractErr, closeErr)
|
||||
}
|
||||
if extractErr != nil {
|
||||
return extractErr
|
||||
}
|
||||
if secondPass.fileCount != firstPass.fileCount || secondPass.totalSize != firstPass.totalSize {
|
||||
return errors.New("pages tar package changed between validation and extraction")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractTarReader(
|
||||
tarReader *tar.Reader,
|
||||
destDir string,
|
||||
commonPrefix string,
|
||||
limits Limits,
|
||||
enforceLimits bool,
|
||||
) (*measuredArchive, error) {
|
||||
measured := &measuredArchive{files: make([]measuredFile, 0)}
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read tar pages package: %w", err)
|
||||
}
|
||||
entry := entryFromTarHeader(header)
|
||||
normalizedPath, skip, err := validateArchiveEntry(entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
normalizedPath = StripPrefix(normalizedPath, commonPrefix)
|
||||
if normalizedPath == "" {
|
||||
continue
|
||||
}
|
||||
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, enforceLimits); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetPath, err := safeExtractionTarget(destDir, normalizedPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxBytes := effectiveFileLimit(limits, measured.totalSize, enforceLimits)
|
||||
actual, err := writeEntryFile(targetPath, tarReader, entry.Size, maxBytes, filePerm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
|
||||
}
|
||||
appendMeasuredFile(measured, normalizedPath, actual)
|
||||
}
|
||||
return measured, nil
|
||||
}
|
||||
|
||||
func commonRootForEntries(entries []Entry, strip bool) (string, error) {
|
||||
paths := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
normalizedPath, skip, err := validateArchiveEntry(entry)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !skip {
|
||||
paths = append(paths, normalizedPath)
|
||||
}
|
||||
}
|
||||
if !strip {
|
||||
return "", nil
|
||||
}
|
||||
return FindCommonRootPrefix(paths), nil
|
||||
}
|
||||
|
||||
func safeExtractionTarget(destDir, relativePath string) (string, error) {
|
||||
targetPath := filepath.Join(destDir, filepath.FromSlash(relativePath))
|
||||
if !isWithinDir(destDir, targetPath) {
|
||||
return "", fmt.Errorf("pages package path escapes directory: %s", relativePath)
|
||||
}
|
||||
return targetPath, nil
|
||||
}
|
||||
|
||||
func isWithinDir(baseDir, targetPath string) bool {
|
||||
cleanBase := filepath.Clean(baseDir)
|
||||
cleanTarget := filepath.Clean(targetPath)
|
||||
rel, err := filepath.Rel(cleanBase, cleanTarget)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel != ".." && !hasParentRel(rel)
|
||||
}
|
||||
|
||||
func hasParentRel(rel string) bool {
|
||||
if rel == ".." {
|
||||
return true
|
||||
}
|
||||
return len(rel) >= 3 && (rel[:3] == "../" || rel[:3] == "..\\")
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package pagesarchive provides multi-format archive detection, inspection and
|
||||
// extraction helpers for OpenFlare Pages deployment packages.
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Format identifies a supported Pages deployment package archive format.
|
||||
type Format string
|
||||
|
||||
const (
|
||||
// FormatZip is a ZIP archive.
|
||||
FormatZip Format = "zip"
|
||||
// FormatTar is an uncompressed tar archive.
|
||||
FormatTar Format = "tar"
|
||||
// FormatTarGz is a gzip-compressed tar archive.
|
||||
FormatTarGz Format = "tar.gz"
|
||||
// FormatTarXz is an xz-compressed tar archive.
|
||||
FormatTarXz Format = "tar.xz"
|
||||
// FormatTarBz2 is a bzip2-compressed tar archive.
|
||||
FormatTarBz2 Format = "tar.bz2"
|
||||
// FormatSevenZip is a 7z archive.
|
||||
FormatSevenZip Format = "7z"
|
||||
|
||||
ustarMagicOffset = 257
|
||||
ustarMagicMinLen = 262
|
||||
)
|
||||
|
||||
// DetectFormatFromName returns the archive format inferred from a file name.
|
||||
// Returns an empty Format and false when the extension is unsupported.
|
||||
func DetectFormatFromName(fileName string) (Format, bool) {
|
||||
name := strings.ToLower(strings.TrimSpace(fileName))
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".tar.gz"), strings.HasSuffix(name, ".tgz"):
|
||||
return FormatTarGz, true
|
||||
case strings.HasSuffix(name, ".tar.xz"), strings.HasSuffix(name, ".txz"):
|
||||
return FormatTarXz, true
|
||||
case strings.HasSuffix(name, ".tar.bz2"), strings.HasSuffix(name, ".tbz2"), strings.HasSuffix(name, ".tbz"):
|
||||
return FormatTarBz2, true
|
||||
case strings.HasSuffix(name, ".tar"):
|
||||
return FormatTar, true
|
||||
case strings.HasSuffix(name, ".7z"):
|
||||
return FormatSevenZip, true
|
||||
case strings.HasSuffix(name, ".zip"):
|
||||
return FormatZip, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// DetectFormatFromBytes returns the archive format inferred from magic bytes.
|
||||
// Prefer DetectFormatFromName when a reliable file name is available.
|
||||
func DetectFormatFromBytes(data []byte) (Format, bool) {
|
||||
if isZipMagic(data) {
|
||||
return FormatZip, true
|
||||
}
|
||||
if isSevenZipMagic(data) {
|
||||
return FormatSevenZip, true
|
||||
}
|
||||
if isXZMagic(data) {
|
||||
return FormatTarXz, true
|
||||
}
|
||||
if isGzipMagic(data) {
|
||||
return FormatTarGz, true
|
||||
}
|
||||
if isBzip2Magic(data) {
|
||||
return FormatTarBz2, true
|
||||
}
|
||||
if looksLikeTar(data) {
|
||||
return FormatTar, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// DetectFormat prefers the file name when present, otherwise magic bytes.
|
||||
func DetectFormat(fileName string, data []byte) (Format, error) {
|
||||
if format, ok := DetectFormatFromName(fileName); ok {
|
||||
return format, nil
|
||||
}
|
||||
if format, ok := DetectFormatFromBytes(data); ok {
|
||||
return format, nil
|
||||
}
|
||||
return "", errors.New("unsupported pages package format")
|
||||
}
|
||||
|
||||
// Extension returns the canonical file extension for a format (without leading dot).
|
||||
func Extension(format Format) string {
|
||||
switch format {
|
||||
case FormatZip:
|
||||
return "zip"
|
||||
case FormatTar:
|
||||
return "tar"
|
||||
case FormatTarGz:
|
||||
return "tar.gz"
|
||||
case FormatTarXz:
|
||||
return "tar.xz"
|
||||
case FormatTarBz2:
|
||||
return "tar.bz2"
|
||||
case FormatSevenZip:
|
||||
return "7z"
|
||||
default:
|
||||
return "bin"
|
||||
}
|
||||
}
|
||||
|
||||
// MIMEType returns a reasonable content type for the archive format.
|
||||
func MIMEType(format Format) string {
|
||||
switch format {
|
||||
case FormatZip:
|
||||
return "application/zip"
|
||||
case FormatTar:
|
||||
return "application/x-tar"
|
||||
case FormatTarGz:
|
||||
return "application/gzip"
|
||||
case FormatTarXz:
|
||||
return "application/x-xz"
|
||||
case FormatTarBz2:
|
||||
return "application/x-bzip2"
|
||||
case FormatSevenZip:
|
||||
return "application/x-7z-compressed"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
// SupportedExtensions lists human-readable extensions for UI copy and accept attributes.
|
||||
func SupportedExtensions() []string {
|
||||
return []string{".zip", ".tar.gz", ".tgz", ".tar.xz", ".txz", ".tar.bz2", ".tbz2", ".tar", ".7z"}
|
||||
}
|
||||
|
||||
// AcceptAttribute returns a comma-separated accept list for file inputs.
|
||||
func AcceptAttribute() string {
|
||||
return strings.Join(SupportedExtensions(), ",")
|
||||
}
|
||||
|
||||
// NormalizeNameExtension returns a storage-safe extension for the given format/name.
|
||||
func NormalizeNameExtension(fileName string, format Format) string {
|
||||
if format != "" {
|
||||
return Extension(format)
|
||||
}
|
||||
if formatFromName, ok := DetectFormatFromName(fileName); ok {
|
||||
return Extension(formatFromName)
|
||||
}
|
||||
ext := strings.TrimPrefix(filepath.Ext(fileName), ".")
|
||||
if ext == "" {
|
||||
return "bin"
|
||||
}
|
||||
return strings.ToLower(ext)
|
||||
}
|
||||
|
||||
func isZipMagic(data []byte) bool {
|
||||
return len(data) >= 4 &&
|
||||
data[0] == 0x50 && data[1] == 0x4b &&
|
||||
(data[2] == 0x03 || data[2] == 0x05 || data[2] == 0x07) &&
|
||||
(data[3] == 0x04 || data[3] == 0x06 || data[3] == 0x08)
|
||||
}
|
||||
|
||||
func isSevenZipMagic(data []byte) bool {
|
||||
return len(data) >= 6 &&
|
||||
data[0] == 0x37 && data[1] == 0x7a && data[2] == 0xbc &&
|
||||
data[3] == 0xaf && data[4] == 0x27 && data[5] == 0x1c
|
||||
}
|
||||
|
||||
func isXZMagic(data []byte) bool {
|
||||
return len(data) >= 6 &&
|
||||
data[0] == 0xfd && data[1] == 0x37 && data[2] == 0x7a &&
|
||||
data[3] == 0x58 && data[4] == 0x5a && data[5] == 0x00
|
||||
}
|
||||
|
||||
func isGzipMagic(data []byte) bool {
|
||||
return len(data) >= 2 && data[0] == 0x1f && data[1] == 0x8b
|
||||
}
|
||||
|
||||
func isBzip2Magic(data []byte) bool {
|
||||
return len(data) >= 3 && data[0] == 0x42 && data[1] == 0x5a && data[2] == 0x68
|
||||
}
|
||||
|
||||
func looksLikeTar(data []byte) bool {
|
||||
// POSIX ustar magic at offset 257 ("ustar\0" or "ustar ").
|
||||
if len(data) < ustarMagicMinLen {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(data[ustarMagicOffset:ustarMagicOffset+5], []byte("ustar"))
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
)
|
||||
|
||||
// InspectOptions controls package inspection.
|
||||
type InspectOptions struct {
|
||||
// RootDir is an optional project root subdirectory that must contain EntryFile.
|
||||
RootDir string
|
||||
// EntryFile is the required entry file name (e.g. index.html).
|
||||
EntryFile string
|
||||
// Limits bounds files and actual extracted sizes.
|
||||
Limits Limits
|
||||
// VerifySizes is retained for source compatibility. Inspection now always
|
||||
// streams regular members and verifies actual bytes against declared sizes.
|
||||
VerifySizes bool
|
||||
}
|
||||
|
||||
type measuredFile struct {
|
||||
path string
|
||||
size int64
|
||||
}
|
||||
|
||||
type measuredArchive struct {
|
||||
files []measuredFile
|
||||
fileCount int
|
||||
totalSize int64
|
||||
}
|
||||
|
||||
// InspectFile opens path and inspects it as a Pages deployment package without
|
||||
// loading the whole archive or any tar member body into memory.
|
||||
func InspectFile(filePath string, format Format, opts InspectOptions) (*Manifest, error) {
|
||||
file, err := os.Open(filePath) //nolint:gosec // filePath is a controlled temp upload path
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if format == "" {
|
||||
head := make([]byte, formatDetectHeadBytes)
|
||||
n, readErr := file.ReadAt(head, 0)
|
||||
if readErr != nil && readErr != io.EOF {
|
||||
return nil, readErr
|
||||
}
|
||||
format, err = DetectFormat(filePath, head[:n])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return inspectFromReaderAt(file, info.Size(), format, opts)
|
||||
}
|
||||
|
||||
// InspectBytes inspects an in-memory deployment package.
|
||||
func InspectBytes(data []byte, format Format, opts InspectOptions) (*Manifest, error) {
|
||||
if format == "" {
|
||||
var err error
|
||||
format, err = DetectFormat("", data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return inspectFromReaderAt(bytes.NewReader(data), int64(len(data)), format, opts)
|
||||
}
|
||||
|
||||
func inspectFromReaderAt(ra io.ReaderAt, size int64, format Format, opts InspectOptions) (*Manifest, error) {
|
||||
limits := normalizeLimits(opts.Limits)
|
||||
var (
|
||||
measured *measuredArchive
|
||||
err error
|
||||
)
|
||||
if isTarFamily(format) {
|
||||
measured, err = scanTarFamilyAt(ra, size, format, limits, true)
|
||||
} else {
|
||||
var entries []Entry
|
||||
entries, err = listRandomAccessEntriesAt(ra, size, format)
|
||||
if err == nil {
|
||||
measured, err = inspectRandomAccessEntries(entries, limits)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildMeasuredManifest(measured, opts)
|
||||
}
|
||||
|
||||
func inspectRandomAccessEntries(entries []Entry, limits Limits) (*measuredArchive, error) {
|
||||
measured := &measuredArchive{files: make([]measuredFile, 0, len(entries))}
|
||||
for _, entry := range entries {
|
||||
normalizedPath, skip, err := validateArchiveEntry(entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if entry.Open == nil {
|
||||
return nil, fmt.Errorf("%s: pages archive member cannot be opened", normalizedPath)
|
||||
}
|
||||
src, err := entry.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
|
||||
}
|
||||
maxBytes := effectiveFileLimit(limits, measured.totalSize, true)
|
||||
actual, copyErr := copyAndVerifySize(io.Discard, src, entry.Size, maxBytes)
|
||||
closeErr := src.Close()
|
||||
if err := errors.Join(copyErr, closeErr); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
|
||||
}
|
||||
appendMeasuredFile(measured, normalizedPath, actual)
|
||||
}
|
||||
return measured, nil
|
||||
}
|
||||
|
||||
func scanTarFamilyAt(
|
||||
ra io.ReaderAt,
|
||||
size int64,
|
||||
format Format,
|
||||
limits Limits,
|
||||
enforceLimits bool,
|
||||
) (*measuredArchive, error) {
|
||||
if size < 0 {
|
||||
return nil, errors.New("invalid pages package size")
|
||||
}
|
||||
tarReader, closeReader, err := openTarFamilyReader(io.NewSectionReader(ra, 0, size), format)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
measured, scanErr := scanTarReader(tarReader, limits, enforceLimits)
|
||||
if closeErr := closeReader(); closeErr != nil {
|
||||
scanErr = errors.Join(scanErr, closeErr)
|
||||
}
|
||||
return measured, scanErr
|
||||
}
|
||||
|
||||
func scanTarReader(tarReader *tar.Reader, limits Limits, enforceLimits bool) (*measuredArchive, error) {
|
||||
measured := &measuredArchive{files: make([]measuredFile, 0)}
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read tar pages package: %w", err)
|
||||
}
|
||||
entry := entryFromTarHeader(header)
|
||||
normalizedPath, skip, err := validateArchiveEntry(entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, enforceLimits); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxBytes := effectiveFileLimit(limits, measured.totalSize, enforceLimits)
|
||||
actual, err := copyAndVerifySize(io.Discard, tarReader, entry.Size, maxBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
|
||||
}
|
||||
appendMeasuredFile(measured, normalizedPath, actual)
|
||||
}
|
||||
return measured, nil
|
||||
}
|
||||
|
||||
func buildMeasuredManifest(measured *measuredArchive, opts InspectOptions) (*Manifest, error) {
|
||||
if measured == nil || measured.fileCount == 0 {
|
||||
return nil, errors.New("pages package is empty")
|
||||
}
|
||||
targetEntryPath, err := resolveTargetEntryPath(opts.RootDir, opts.EntryFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := make([]string, 0, len(measured.files))
|
||||
for _, file := range measured.files {
|
||||
paths = append(paths, file.path)
|
||||
}
|
||||
commonPrefix := FindCommonRootPrefix(paths)
|
||||
manifest := &Manifest{
|
||||
Files: make([]FileEntry, 0, measured.fileCount),
|
||||
FileCount: measured.fileCount,
|
||||
TotalSize: measured.totalSize,
|
||||
}
|
||||
entrySeen := false
|
||||
for _, file := range measured.files {
|
||||
normalizedPath := StripPrefix(file.path, commonPrefix)
|
||||
if normalizedPath == targetEntryPath {
|
||||
entrySeen = true
|
||||
}
|
||||
manifest.Files = append(manifest.Files, FileEntry{
|
||||
Path: normalizedPath,
|
||||
Size: file.size,
|
||||
})
|
||||
}
|
||||
if !entrySeen {
|
||||
return nil, fmt.Errorf("pages package is missing entry file %s", targetEntryPath)
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func prepareMeasuredFile(measured *measuredArchive, normalizedPath string, declaredSize uint64, limits Limits, enforceLimits bool) error {
|
||||
if declaredSize > uint64(math.MaxInt64) {
|
||||
return fmt.Errorf("%s: pages file size out of bounds", normalizedPath)
|
||||
}
|
||||
if !enforceLimits {
|
||||
return nil
|
||||
}
|
||||
if measured.fileCount >= limits.MaxFiles {
|
||||
return fmt.Errorf("pages deployment file count exceeds %d", limits.MaxFiles)
|
||||
}
|
||||
if exceedsFileByteLimit(declaredSize, limits.MaxFileBytes) {
|
||||
return fmt.Errorf("pages file too large: %s", normalizedPath)
|
||||
}
|
||||
remaining := limits.MaxTotalBytes - measured.totalSize
|
||||
if remaining < 0 || declaredSize > uint64(remaining) { //nolint:gosec // remaining is checked non-negative
|
||||
return errors.New("pages extracted size exceeds limit")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendMeasuredFile(measured *measuredArchive, normalizedPath string, actual int64) {
|
||||
measured.files = append(measured.files, measuredFile{path: normalizedPath, size: actual})
|
||||
measured.fileCount++
|
||||
measured.totalSize += actual
|
||||
}
|
||||
|
||||
func effectiveFileLimit(limits Limits, totalSize int64, enforceLimits bool) int64 {
|
||||
if !enforceLimits {
|
||||
return -1
|
||||
}
|
||||
remaining := limits.MaxTotalBytes - totalSize
|
||||
if remaining < limits.MaxFileBytes {
|
||||
return remaining
|
||||
}
|
||||
return limits.MaxFileBytes
|
||||
}
|
||||
|
||||
func resolveTargetEntryPath(rootDir, entryFile string) (string, error) {
|
||||
normalizedRoot, err := NormalizeLogicalPath(rootDir, true)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid pages root directory: %w", err)
|
||||
}
|
||||
if entryFile == "" {
|
||||
entryFile = "index.html"
|
||||
}
|
||||
normalizedEntry, err := NormalizeLogicalPath(entryFile, false)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid pages entry file: %w", err)
|
||||
}
|
||||
if normalizedRoot == "" {
|
||||
return normalizedEntry, nil
|
||||
}
|
||||
return path.Join(normalizedRoot, normalizedEntry), nil
|
||||
}
|
||||
|
||||
func validateArchiveEntry(entry Entry) (string, bool, error) {
|
||||
normalizedPath, skip, err := NormalizeEntryPath(entry.Name)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if entry.IsSymlink {
|
||||
return "", false, fmt.Errorf("pages package contains unsupported symlink: %s", normalizedPath)
|
||||
}
|
||||
if entry.IsHardlink {
|
||||
return "", false, fmt.Errorf("pages package contains unsupported hardlink: %s", normalizedPath)
|
||||
}
|
||||
if entry.IsSpecial {
|
||||
return "", false, fmt.Errorf("pages package contains unsupported special entry: %s", normalizedPath)
|
||||
}
|
||||
if skip || entry.IsDir {
|
||||
return normalizedPath, true, nil
|
||||
}
|
||||
return normalizedPath, false, nil
|
||||
}
|
||||
|
||||
func isTarFamily(format Format) bool {
|
||||
switch format {
|
||||
case FormatTar, FormatTarGz, FormatTarXz, FormatTarBz2:
|
||||
return true
|
||||
case FormatZip, FormatSevenZip:
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
const (
|
||||
defaultMaxFiles = 1000
|
||||
defaultMaxFileBytes = 100 * 1024 * 1024
|
||||
defaultMaxTotalBytes = 100 * 1024 * 1024
|
||||
dirPerm = 0o750
|
||||
filePerm = 0o644
|
||||
)
|
||||
|
||||
// normalizeLimits applies defaults for control-plane inspection.
|
||||
// Callers that already validated the package should use EnforceLimits=false instead.
|
||||
func normalizeLimits(limits Limits) Limits {
|
||||
if limits.MaxFiles <= 0 {
|
||||
limits.MaxFiles = defaultMaxFiles
|
||||
}
|
||||
if limits.MaxFileBytes <= 0 {
|
||||
limits.MaxFileBytes = defaultMaxFileBytes
|
||||
}
|
||||
if limits.MaxTotalBytes <= 0 {
|
||||
limits.MaxTotalBytes = defaultMaxTotalBytes
|
||||
}
|
||||
return limits
|
||||
}
|
||||
|
||||
func exceedsFileByteLimit(size uint64, maxBytes int64) bool {
|
||||
// maxBytes <= 0 means unlimited (trusted extract path).
|
||||
if maxBytes <= 0 {
|
||||
return false
|
||||
}
|
||||
if size == 0 {
|
||||
return false
|
||||
}
|
||||
return size > uint64(maxBytes) //nolint:gosec // maxBytes is positive
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/bodgit/sevenzip"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
type archiveFile interface {
|
||||
Name() string
|
||||
Mode() os.FileMode
|
||||
IsDir() bool
|
||||
Size() uint64
|
||||
Open() (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
type zipArchiveFile struct {
|
||||
file *zip.File
|
||||
}
|
||||
|
||||
func (z zipArchiveFile) Name() string { return z.file.Name }
|
||||
func (z zipArchiveFile) Mode() os.FileMode { return z.file.Mode() }
|
||||
func (z zipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
|
||||
func (z zipArchiveFile) Size() uint64 { return z.file.UncompressedSize64 }
|
||||
func (z zipArchiveFile) Open() (io.ReadCloser, error) {
|
||||
return z.file.Open()
|
||||
}
|
||||
|
||||
type sevenZipArchiveFile struct {
|
||||
file *sevenzip.File
|
||||
}
|
||||
|
||||
func (z sevenZipArchiveFile) Name() string { return z.file.Name }
|
||||
func (z sevenZipArchiveFile) Mode() os.FileMode { return z.file.Mode() }
|
||||
func (z sevenZipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
|
||||
func (z sevenZipArchiveFile) Size() uint64 { return z.file.UncompressedSize }
|
||||
func (z sevenZipArchiveFile) Open() (io.ReadCloser, error) {
|
||||
return z.file.Open()
|
||||
}
|
||||
|
||||
// listRandomAccessEntriesAt lists zip/7z members without reading their bodies.
|
||||
// Tar-family archives use the sequential streaming paths in inspect.go/extract.go.
|
||||
func listRandomAccessEntriesAt(ra io.ReaderAt, size int64, format Format) ([]Entry, error) {
|
||||
if size < 0 {
|
||||
return nil, errors.New("invalid pages package size")
|
||||
}
|
||||
switch format {
|
||||
case FormatZip:
|
||||
return listZipEntriesAt(ra, size)
|
||||
case FormatSevenZip:
|
||||
return listSevenZipEntriesAt(ra, size)
|
||||
case FormatTar, FormatTarGz, FormatTarXz, FormatTarBz2:
|
||||
return nil, fmt.Errorf("unsupported random-access pages package format: %s", format)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported random-access pages package format: %s", format)
|
||||
}
|
||||
}
|
||||
|
||||
func entriesFromArchiveFiles(files []archiveFile) []Entry {
|
||||
entries := make([]Entry, 0, len(files))
|
||||
for _, item := range files {
|
||||
file := item
|
||||
mode := file.Mode()
|
||||
isDir := file.IsDir()
|
||||
isSymlink := mode&os.ModeSymlink != 0
|
||||
isSpecial := !isDir && !isSymlink && !mode.IsRegular()
|
||||
entries = append(entries, Entry{
|
||||
Name: file.Name(),
|
||||
IsDir: isDir,
|
||||
IsSymlink: isSymlink,
|
||||
IsSpecial: isSpecial,
|
||||
Size: file.Size(),
|
||||
Open: file.Open,
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func listZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
|
||||
reader, err := zip.NewReader(ra, size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open zip pages package: %w", err)
|
||||
}
|
||||
files := make([]archiveFile, 0, len(reader.File))
|
||||
for _, item := range reader.File {
|
||||
files = append(files, zipArchiveFile{file: item})
|
||||
}
|
||||
return entriesFromArchiveFiles(files), nil
|
||||
}
|
||||
|
||||
func listSevenZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
|
||||
reader, err := sevenzip.NewReader(ra, size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open 7z pages package: %w", err)
|
||||
}
|
||||
files := make([]archiveFile, 0, len(reader.File))
|
||||
for _, item := range reader.File {
|
||||
files = append(files, sevenZipArchiveFile{file: item})
|
||||
}
|
||||
return entriesFromArchiveFiles(files), nil
|
||||
}
|
||||
|
||||
func openTarFamilyReader(r io.Reader, format Format) (*tar.Reader, func() error, error) {
|
||||
switch format {
|
||||
case FormatTar:
|
||||
return tar.NewReader(r), func() error { return nil }, nil
|
||||
case FormatTarGz:
|
||||
gzReader, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("open gzip pages package: %w", err)
|
||||
}
|
||||
return tar.NewReader(gzReader), gzReader.Close, nil
|
||||
case FormatTarXz:
|
||||
xzReader, err := xz.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("open xz pages package: %w", err)
|
||||
}
|
||||
return tar.NewReader(xzReader), func() error { return nil }, nil
|
||||
case FormatTarBz2:
|
||||
return tar.NewReader(bzip2.NewReader(r)), func() error { return nil }, nil
|
||||
case FormatZip, FormatSevenZip:
|
||||
return nil, nil, fmt.Errorf("unsupported tar family format: %s", format)
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unsupported tar family format: %s", format)
|
||||
}
|
||||
}
|
||||
|
||||
func entryFromTarHeader(header *tar.Header) Entry {
|
||||
entry := Entry{Name: header.Name}
|
||||
if header.Size > 0 {
|
||||
entry.Size = uint64(header.Size) //nolint:gosec // archive/tar rejects negative sizes
|
||||
}
|
||||
switch header.Typeflag {
|
||||
case tar.TypeReg, tar.TypeRegA: //nolint:staticcheck // TypeRegA appears in older archives
|
||||
// Regular file.
|
||||
case tar.TypeDir:
|
||||
entry.IsDir = true
|
||||
case tar.TypeSymlink:
|
||||
entry.IsSymlink = true
|
||||
case tar.TypeLink:
|
||||
entry.IsHardlink = true
|
||||
default:
|
||||
entry.IsSpecial = true
|
||||
}
|
||||
return entry
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
func TestDetectFormatFromName(t *testing.T) {
|
||||
cases := map[string]Format{
|
||||
"site.zip": FormatZip,
|
||||
"site.TAR.GZ": FormatTarGz,
|
||||
"site.tgz": FormatTarGz,
|
||||
"site.tar.xz": FormatTarXz,
|
||||
"site.txz": FormatTarXz,
|
||||
"site.tar.bz2": FormatTarBz2,
|
||||
"site.tar": FormatTar,
|
||||
"site.7z": FormatSevenZip,
|
||||
}
|
||||
for name, want := range cases {
|
||||
got, ok := DetectFormatFromName(name)
|
||||
assert.True(t, ok, name)
|
||||
assert.Equal(t, want, got, name)
|
||||
}
|
||||
_, ok := DetectFormatFromName("site.rar")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestInspectAndExtractZip(t *testing.T) {
|
||||
data := testZip(t, map[string]string{
|
||||
"dist/index.html": "<html>ok</html>",
|
||||
"dist/app.js": "console.log(1)",
|
||||
})
|
||||
manifest, err := InspectBytes(data, FormatZip, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, manifest.FileCount)
|
||||
paths := make(map[string]struct{}, len(manifest.Files))
|
||||
for _, file := range manifest.Files {
|
||||
paths[file.Path] = struct{}{}
|
||||
assert.Empty(t, file.Checksum, "per-file checksum should not be computed")
|
||||
assert.Positive(t, file.Size)
|
||||
}
|
||||
assert.Contains(t, paths, "index.html")
|
||||
assert.Contains(t, paths, "app.js")
|
||||
assert.Equal(t, int64(len("<html>ok</html>")+len("console.log(1)")), manifest.TotalSize)
|
||||
|
||||
dest := t.TempDir()
|
||||
require.NoError(t, ExtractBytes(data, FormatZip, dest, ExtractOptions{
|
||||
StripCommonRoot: true,
|
||||
EnforceLimits: true,
|
||||
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
}))
|
||||
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "<html>ok</html>", string(body))
|
||||
}
|
||||
|
||||
func TestInspectFileUsesDeclaredSizesWithoutHash(t *testing.T) {
|
||||
data := testZip(t, map[string]string{
|
||||
"index.html": "<html>disk</html>",
|
||||
"asset.css": "body{}",
|
||||
})
|
||||
path := filepath.Join(t.TempDir(), "site.zip")
|
||||
require.NoError(t, os.WriteFile(path, data, 0o600))
|
||||
|
||||
manifest, err := InspectFile(path, FormatZip, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, manifest.FileCount)
|
||||
for _, file := range manifest.Files {
|
||||
assert.Empty(t, file.Checksum)
|
||||
}
|
||||
|
||||
dest := t.TempDir()
|
||||
require.NoError(t, ExtractFile(path, FormatZip, dest, ExtractOptions{
|
||||
EnforceLimits: true,
|
||||
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
}))
|
||||
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "<html>disk</html>", string(body))
|
||||
}
|
||||
|
||||
func TestInspectVerifySizesOptional(t *testing.T) {
|
||||
data := testZip(t, map[string]string{
|
||||
"index.html": "verify-me",
|
||||
})
|
||||
manifest, err := InspectBytes(data, FormatZip, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
VerifySizes: true,
|
||||
Limits: Limits{MaxFiles: 10, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, manifest.Files, 1)
|
||||
assert.Equal(t, int64(len("verify-me")), manifest.Files[0].Size)
|
||||
assert.Empty(t, manifest.Files[0].Checksum)
|
||||
}
|
||||
|
||||
func TestExtractTrustedSkipsSizeLimits(t *testing.T) {
|
||||
// Content larger than a tiny limit would fail if limits were enforced.
|
||||
large := strings.Repeat("x", 64)
|
||||
data := testZip(t, map[string]string{
|
||||
"index.html": large,
|
||||
})
|
||||
dest := t.TempDir()
|
||||
require.NoError(t, ExtractBytes(data, FormatZip, dest, ExtractOptions{
|
||||
// Agent trusts control-plane validation: no size/count re-check.
|
||||
EnforceLimits: false,
|
||||
}))
|
||||
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, large, string(body))
|
||||
}
|
||||
|
||||
func TestInspectAndExtractTarGz(t *testing.T) {
|
||||
data := testTarGz(t, map[string]string{
|
||||
"index.html": "<html>tar</html>",
|
||||
"style.css": "body{}",
|
||||
})
|
||||
format, err := DetectFormat("site.tar.gz", data)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, FormatTarGz, format)
|
||||
|
||||
manifest, err := InspectBytes(data, format, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, manifest.FileCount)
|
||||
|
||||
dest := t.TempDir()
|
||||
require.NoError(t, ExtractBytes(data, format, dest, ExtractOptions{
|
||||
EnforceLimits: true,
|
||||
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
}))
|
||||
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "<html>tar</html>", string(body))
|
||||
}
|
||||
|
||||
func TestInspectTarXz(t *testing.T) {
|
||||
data := testTarXz(t, map[string]string{
|
||||
"index.html": "<html>xz</html>",
|
||||
})
|
||||
manifest, err := InspectBytes(data, FormatTarXz, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{MaxFiles: 10, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, manifest.FileCount)
|
||||
}
|
||||
|
||||
func TestRejectZipSlip(t *testing.T) {
|
||||
data := testZip(t, map[string]string{
|
||||
"../evil.txt": "x",
|
||||
"index.html": "ok",
|
||||
})
|
||||
_, err := InspectBytes(data, FormatZip, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{MaxFiles: 10, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
|
||||
})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func testZip(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
writer := zip.NewWriter(&buffer)
|
||||
for name, content := range files {
|
||||
file, err := writer.Create(name)
|
||||
require.NoError(t, err)
|
||||
_, err = file.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func testTarGz(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
gzWriter := gzip.NewWriter(&buffer)
|
||||
tarWriter := tar.NewWriter(gzWriter)
|
||||
for name, content := range files {
|
||||
require.NoError(t, tarWriter.WriteHeader(&tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}))
|
||||
_, err := tarWriter.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tarWriter.Close())
|
||||
require.NoError(t, gzWriter.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func testTarXz(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
xzWriter, err := xz.NewWriter(&buffer)
|
||||
require.NoError(t, err)
|
||||
tarWriter := tar.NewWriter(xzWriter)
|
||||
for name, content := range files {
|
||||
require.NoError(t, tarWriter.WriteHeader(&tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}))
|
||||
_, err := tarWriter.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tarWriter.Close())
|
||||
require.NoError(t, xzWriter.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// NormalizeLogicalPath validates and normalizes a relative POSIX path.
|
||||
// Empty input is returned unchanged only when allowEmpty is true.
|
||||
func NormalizeLogicalPath(raw string, allowEmpty bool) (string, error) {
|
||||
if raw == "" {
|
||||
if allowEmpty {
|
||||
return "", nil
|
||||
}
|
||||
return "", errors.New("pages path is required")
|
||||
}
|
||||
if err := validateLogicalPathText(raw); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
cleaned := path.Clean(raw)
|
||||
if cleaned == "." || cleaned == "" {
|
||||
if allowEmpty {
|
||||
return "", nil
|
||||
}
|
||||
return "", errors.New("pages path is required")
|
||||
}
|
||||
if strings.HasPrefix(cleaned, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
|
||||
return "", fmt.Errorf("pages path escapes directory: %s", raw)
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
func validateLogicalPathText(raw string) error {
|
||||
if !utf8.ValidString(raw) {
|
||||
return errors.New("pages path is not valid UTF-8")
|
||||
}
|
||||
if strings.Contains(raw, "\\") {
|
||||
return fmt.Errorf("pages path must use POSIX separators: %s", raw)
|
||||
}
|
||||
if strings.HasPrefix(raw, "/") || path.IsAbs(raw) {
|
||||
return fmt.Errorf("pages path must be relative: %s", raw)
|
||||
}
|
||||
if err := validateLogicalPathRunes(raw); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateLogicalPathSegments(raw)
|
||||
}
|
||||
|
||||
func validateLogicalPathRunes(raw string) error {
|
||||
for _, r := range raw {
|
||||
if r == 0 || unicode.IsControl(r) {
|
||||
return errors.New("pages path contains a control character")
|
||||
}
|
||||
if r == '\'' || r == '"' || r == ';' {
|
||||
return fmt.Errorf("pages path contains an unsupported character: %s", raw)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateLogicalPathSegments(raw string) error {
|
||||
for segment := range strings.SplitSeq(raw, "/") {
|
||||
if len(segment) >= 2 && segment[1] == ':' {
|
||||
return fmt.Errorf("pages path contains a Windows drive: %s", raw)
|
||||
}
|
||||
if segment == "." || segment == ".." {
|
||||
return fmt.Errorf("pages path escapes directory or contains a dot segment: %s", raw)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NormalizeEntryPath cleans an archive entry path and rejects zip-slip / absolute paths.
|
||||
// skip=true means the entry should be ignored (empty path or directory marker).
|
||||
func NormalizeEntryPath(raw string) (cleaned string, skip bool, err error) {
|
||||
if raw == "" {
|
||||
return "", true, nil
|
||||
}
|
||||
cleanedPath, normalizeErr := NormalizeLogicalPath(raw, false)
|
||||
if normalizeErr != nil {
|
||||
return "", false, fmt.Errorf("invalid pages package path %q: %w", raw, normalizeErr)
|
||||
}
|
||||
if strings.HasSuffix(raw, "/") {
|
||||
return cleanedPath, true, nil
|
||||
}
|
||||
return cleanedPath, false, nil
|
||||
}
|
||||
|
||||
// FindCommonRootPrefix returns a trailing-slash directory prefix shared by all file paths.
|
||||
// When files do not share a single root folder the result is empty.
|
||||
func FindCommonRootPrefix(paths []string) string {
|
||||
var firstFilePath string
|
||||
hasMultipleFiles := false
|
||||
for _, item := range paths {
|
||||
normalizedPath, skip, err := NormalizeEntryPath(item)
|
||||
if err != nil || skip {
|
||||
continue
|
||||
}
|
||||
if firstFilePath == "" {
|
||||
firstFilePath = normalizedPath
|
||||
} else {
|
||||
hasMultipleFiles = true
|
||||
}
|
||||
}
|
||||
if firstFilePath == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(firstFilePath, "/")
|
||||
if len(parts) <= 1 {
|
||||
return ""
|
||||
}
|
||||
commonPrefix := parts[0] + "/"
|
||||
if !hasMultipleFiles {
|
||||
return commonPrefix
|
||||
}
|
||||
for _, item := range paths {
|
||||
normalizedPath, skip, err := NormalizeEntryPath(item)
|
||||
if err != nil || skip {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(normalizedPath, commonPrefix) {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return commonPrefix
|
||||
}
|
||||
|
||||
// StripPrefix removes a directory prefix from a cleaned path when present.
|
||||
func StripPrefix(normalizedPath, prefix string) string {
|
||||
if prefix == "" {
|
||||
return normalizedPath
|
||||
}
|
||||
return strings.TrimPrefix(normalizedPath, prefix)
|
||||
}
|
||||
@@ -0,0 +1,499 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package pagesarchive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var testLimits = Limits{
|
||||
MaxFiles: 100,
|
||||
MaxFileBytes: 1 << 20,
|
||||
MaxTotalBytes: 1 << 20,
|
||||
}
|
||||
|
||||
func TestNormalizeLogicalPathStrict(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
valid := []struct {
|
||||
name string
|
||||
raw string
|
||||
allowEmpty bool
|
||||
want string
|
||||
}{
|
||||
{name: "empty root", allowEmpty: true},
|
||||
{name: "single file", raw: "index.html", want: "index.html"},
|
||||
{name: "nested posix", raw: "public/assets/app.js", want: "public/assets/app.js"},
|
||||
{name: "unicode", raw: "静态/首页.html", want: "静态/首页.html"},
|
||||
{name: "repeated separator is normalized", raw: "public//app.js", want: "public/app.js"},
|
||||
}
|
||||
for _, tt := range valid {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NormalizeLogicalPath(tt.raw, tt.allowEmpty)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
|
||||
invalidUTF8 := string([]byte{'a', '/', 0xff})
|
||||
invalid := []struct {
|
||||
name string
|
||||
raw string
|
||||
}{
|
||||
{name: "empty entry"},
|
||||
{name: "absolute", raw: "/etc/passwd"},
|
||||
{name: "unc", raw: "//server/share"},
|
||||
{name: "windows drive", raw: "C:/site/index.html"},
|
||||
{name: "nested windows drive", raw: "site/C:/index.html"},
|
||||
{name: "windows separator", raw: `site\index.html`},
|
||||
{name: "windows unc", raw: `\\server\share`},
|
||||
{name: "parent segment", raw: "../index.html"},
|
||||
{name: "nested parent segment", raw: "site/../index.html"},
|
||||
{name: "current segment", raw: "site/./index.html"},
|
||||
{name: "nul", raw: "site/\x00index.html"},
|
||||
{name: "newline", raw: "site/\nindex.html"},
|
||||
{name: "delete control", raw: "site/\x7findex.html"},
|
||||
{name: "single quote", raw: "site/'index.html"},
|
||||
{name: "double quote", raw: `site/"index.html`},
|
||||
{name: "semicolon", raw: "site/;index.html"},
|
||||
{name: "invalid utf8", raw: invalidUTF8},
|
||||
}
|
||||
for _, tt := range invalid {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := NormalizeLogicalPath(tt.raw, false)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
cleaned, skip, err := NormalizeEntryPath("")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, cleaned)
|
||||
assert.True(t, skip)
|
||||
|
||||
cleaned, skip, err = NormalizeEntryPath("assets/")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "assets", cleaned)
|
||||
assert.True(t, skip)
|
||||
}
|
||||
|
||||
func TestSupportedFormatsInspectAndExtract(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sevenZipData := decodeFixture(t, "N3q8ryccAASgR6WICAAAAAAAAABmAAAAAAAAAN2R8/FiYXIKZm9vCgEEBgACCQQEAAcLAgABAQABAQAMBAQACAoB6bOiBKhlMn4AAAUCGQUAAAAAABERAGIAYQByAAAAZgBvAG8AAAAZAgAAFBIBAACFM3PyY9YBAFgCcvJj1gEVCgEAIICkgSCApIEAAA==")
|
||||
bzipTarData := decodeFixture(t, "QlpoOTFBWSZTWYp5f6EAAHV//P64A8RQAf/iOm/9cO/v/9AAAgBADlAABAADAAgwAU1RIZJpNCaammmnqbSGTI9Q0BoBpp6mjIaGmmRoaHGRpkxNBkyYTTIGQ0BoDTJoYATQGG1KCntExT01MhoAABoAAHqAAPU9QacVDtN45fA6MmuGVQlWowrijpZgwASITYSPUcpJpoQGMkKq69jMkUR6L86R5j0IySUaZEjazEqhQ9E8vuuxsmWZQLCA84jNsobYNEzuEB1eCPhw8nc2AOz+xrCY5hVxQW1IIokpfSRKi+McvXU+QoYuEg6BD4w8x3K0imi+bULpkLCylCZ4lzoGlTQgibvG67sQcrTCRBTbBCVL7zC0q0qULmK/WOneu94s9cs4s4K98SjY2YvpdZvl42kwtxvvPMheorYQ2pcxyF4sNQYvd4+bgqm5gKXElqnGF3jhxGTeXp9eCUxWVlbi9ikxAik4xxATl7cJrISVWnHwUFiLdhEnKWw0Lhm3ZyKlX7P5Wj7b9TLAmWBaAwH/F3JFOFCQinl/oQ==")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
format Format
|
||||
data []byte
|
||||
entryFile string
|
||||
wantPath string
|
||||
}{
|
||||
{name: "zip", format: FormatZip, data: testZip(t, map[string]string{"bundle/index.html": "zip"}), entryFile: "index.html", wantPath: "index.html"},
|
||||
{name: "tar", format: FormatTar, data: testTar(t, map[string]string{"bundle/index.html": "tar"}), entryFile: "index.html", wantPath: "index.html"},
|
||||
{name: "tar gzip", format: FormatTarGz, data: testTarGz(t, map[string]string{"bundle/index.html": "gzip"}), entryFile: "index.html", wantPath: "index.html"},
|
||||
{name: "tar xz", format: FormatTarXz, data: testTarXz(t, map[string]string{"bundle/index.html": "xz"}), entryFile: "index.html", wantPath: "index.html"},
|
||||
{name: "tar bzip2", format: FormatTarBz2, data: bzipTarData, entryFile: "index.html", wantPath: "index.html"},
|
||||
{name: "7z", format: FormatSevenZip, data: sevenZipData, entryFile: "foo", wantPath: "foo"},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
manifest, err := InspectBytes(tt.data, tt.format, InspectOptions{
|
||||
EntryFile: tt.entryFile,
|
||||
Limits: testLimits,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Positive(t, manifest.FileCount)
|
||||
assertManifestContains(t, manifest, tt.wantPath)
|
||||
|
||||
destDir := t.TempDir()
|
||||
require.NoError(t, ExtractBytes(tt.data, tt.format, destDir, ExtractOptions{
|
||||
Limits: testLimits,
|
||||
StripCommonRoot: true,
|
||||
EnforceLimits: true,
|
||||
}))
|
||||
_, err = os.Stat(filepath.Join(destDir, filepath.FromSlash(tt.wantPath)))
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFilePreservesCommonRootAndEnforcesLimits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := testTarGz(t, map[string]string{
|
||||
"repository/dist/index.html": "pages",
|
||||
"repository/dist/app.js": "javascript",
|
||||
})
|
||||
archivePath := filepath.Join(t.TempDir(), "site.tar.gz")
|
||||
require.NoError(t, os.WriteFile(archivePath, data, 0o600))
|
||||
|
||||
manifest, err := InspectFile(archivePath, FormatTarGz, InspectOptions{
|
||||
RootDir: "dist",
|
||||
EntryFile: "index.html",
|
||||
Limits: testLimits,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assertManifestContains(t, manifest, "dist/index.html")
|
||||
|
||||
destDir := t.TempDir()
|
||||
require.NoError(t, ExtractFile(archivePath, FormatTarGz, destDir, ExtractOptions{
|
||||
Limits: testLimits,
|
||||
StripCommonRoot: true,
|
||||
EnforceLimits: true,
|
||||
}))
|
||||
body, err := os.ReadFile(filepath.Join(destDir, "dist", "index.html")) //nolint:gosec
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "pages", string(body))
|
||||
|
||||
err = ExtractFile(archivePath, FormatTarGz, t.TempDir(), ExtractOptions{
|
||||
Limits: Limits{
|
||||
MaxFiles: 10,
|
||||
MaxFileBytes: 5,
|
||||
MaxTotalBytes: 1 << 20,
|
||||
},
|
||||
StripCommonRoot: true,
|
||||
EnforceLimits: true,
|
||||
})
|
||||
require.ErrorContains(t, err, "file too large")
|
||||
|
||||
err = ExtractFile(archivePath, FormatTarGz, t.TempDir(), ExtractOptions{
|
||||
Limits: Limits{
|
||||
MaxFiles: 10,
|
||||
MaxFileBytes: 1 << 20,
|
||||
MaxTotalBytes: int64(len("pages") + len("javascript") - 1),
|
||||
},
|
||||
StripCommonRoot: true,
|
||||
EnforceLimits: true,
|
||||
})
|
||||
require.ErrorContains(t, err, "extracted size exceeds limit")
|
||||
}
|
||||
|
||||
func TestRandomAccessMembersVerifyActualSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
entry := Entry{
|
||||
Name: "index.html",
|
||||
Size: 1,
|
||||
Open: func() (io.ReadCloser, error) {
|
||||
return io.NopCloser(strings.NewReader("actual-body")), nil
|
||||
},
|
||||
}
|
||||
_, err := inspectRandomAccessEntries([]Entry{entry}, testLimits)
|
||||
require.ErrorContains(t, err, "declared size 1 does not match actual 11")
|
||||
|
||||
destDir := t.TempDir()
|
||||
err = extractEntries([]Entry{entry}, destDir, ExtractOptions{
|
||||
Limits: testLimits,
|
||||
EnforceLimits: true,
|
||||
})
|
||||
require.ErrorContains(t, err, "declared size 1 does not match actual 11")
|
||||
_, statErr := os.Stat(filepath.Join(destDir, "index.html"))
|
||||
assert.ErrorIs(t, statErr, os.ErrNotExist, "failed extraction must remove the partial file")
|
||||
}
|
||||
|
||||
func TestActualByteLimitAbortsReaderEarly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
reader := &countingFillReader{remaining: 1 << 30}
|
||||
entry := Entry{
|
||||
Name: "index.html",
|
||||
Size: 0,
|
||||
Open: func() (io.ReadCloser, error) {
|
||||
return io.NopCloser(reader), nil
|
||||
},
|
||||
}
|
||||
_, err := inspectRandomAccessEntries([]Entry{entry}, Limits{
|
||||
MaxFiles: 1,
|
||||
MaxFileBytes: 32,
|
||||
MaxTotalBytes: 32,
|
||||
})
|
||||
require.ErrorContains(t, err, "size out of bounds")
|
||||
assert.LessOrEqual(t, reader.read, int64(33), "inspection must stop after limit+1 actual bytes")
|
||||
|
||||
tarData := tarWithDeclaredBodyOnly(t, "index.html", 1<<30)
|
||||
_, err = InspectBytes(tarData, FormatTar, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{
|
||||
MaxFiles: 1,
|
||||
MaxFileBytes: 32,
|
||||
MaxTotalBytes: 32,
|
||||
},
|
||||
})
|
||||
require.ErrorContains(t, err, "file too large")
|
||||
}
|
||||
|
||||
func TestArchiveLimitsUseFilesAndActualTotals(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := testZip(t, map[string]string{
|
||||
"index.html": "1234",
|
||||
"app.js": "5678",
|
||||
})
|
||||
_, err := InspectBytes(data, FormatZip, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{
|
||||
MaxFiles: 1,
|
||||
MaxFileBytes: 8,
|
||||
MaxTotalBytes: 16,
|
||||
},
|
||||
})
|
||||
require.ErrorContains(t, err, "file count exceeds")
|
||||
|
||||
_, err = InspectBytes(data, FormatZip, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: Limits{
|
||||
MaxFiles: 2,
|
||||
MaxFileBytes: 8,
|
||||
MaxTotalBytes: 7,
|
||||
},
|
||||
})
|
||||
require.ErrorContains(t, err, "extracted size exceeds limit")
|
||||
|
||||
sevenZipData := decodeFixture(t, "N3q8ryccAASgR6WICAAAAAAAAABmAAAAAAAAAN2R8/FiYXIKZm9vCgEEBgACCQQEAAcLAgABAQABAQAMBAQACAoB6bOiBKhlMn4AAAUCGQUAAAAAABERAGIAYQByAAAAZgBvAG8AAAAZAgAAFBIBAACFM3PyY9YBAFgCcvJj1gEVCgEAIICkgSCApIEAAA==")
|
||||
_, err = InspectBytes(sevenZipData, FormatSevenZip, InspectOptions{
|
||||
EntryFile: "foo",
|
||||
Limits: Limits{
|
||||
MaxFiles: 10,
|
||||
MaxFileBytes: 3,
|
||||
MaxTotalBytes: 32,
|
||||
},
|
||||
})
|
||||
require.ErrorContains(t, err, "file too large")
|
||||
}
|
||||
|
||||
func TestRejectUnsupportedTarMemberTypes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
header tar.Header
|
||||
wantErr string
|
||||
}{
|
||||
{name: "symlink", header: tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "index.html"}, wantErr: "unsupported symlink"},
|
||||
{name: "hardlink", header: tar.Header{Name: "hard", Typeflag: tar.TypeLink, Linkname: "index.html"}, wantErr: "unsupported hardlink"},
|
||||
{name: "fifo", header: tar.Header{Name: "pipe", Typeflag: tar.TypeFifo, Mode: 0o600}, wantErr: "unsupported special entry"},
|
||||
{name: "character device", header: tar.Header{Name: "tty", Typeflag: tar.TypeChar, Mode: 0o600, Devmajor: 1, Devminor: 3}, wantErr: "unsupported special entry"},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data := tarWithSpecialEntry(t, &tt.header)
|
||||
_, err := InspectBytes(data, FormatTar, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: testLimits,
|
||||
})
|
||||
require.ErrorContains(t, err, tt.wantErr)
|
||||
|
||||
destDir := t.TempDir()
|
||||
err = ExtractBytes(data, FormatTar, destDir, ExtractOptions{
|
||||
Limits: testLimits,
|
||||
EnforceLimits: true,
|
||||
})
|
||||
require.ErrorContains(t, err, tt.wantErr)
|
||||
_, statErr := os.Stat(filepath.Join(destDir, "index.html"))
|
||||
assert.ErrorIs(t, statErr, os.ErrNotExist, "tar validation pass must reject before writing files")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectUnsupportedZipMemberTypes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
mode os.FileMode
|
||||
wantErr string
|
||||
}{
|
||||
{name: "symlink", mode: os.ModeSymlink | 0o777, wantErr: "unsupported symlink"},
|
||||
{name: "named pipe", mode: os.ModeNamedPipe | 0o600, wantErr: "unsupported special entry"},
|
||||
{name: "device", mode: os.ModeDevice | 0o600, wantErr: "unsupported special entry"},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data := zipWithSpecialEntry(t, tt.mode)
|
||||
_, err := InspectBytes(data, FormatZip, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: testLimits,
|
||||
})
|
||||
require.ErrorContains(t, err, tt.wantErr)
|
||||
|
||||
err = ExtractBytes(data, FormatZip, t.TempDir(), ExtractOptions{
|
||||
Limits: testLimits,
|
||||
EnforceLimits: true,
|
||||
})
|
||||
require.ErrorContains(t, err, tt.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTarMetadataHeadersRemainTransparent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, format := range []tar.Format{tar.FormatPAX, tar.FormatGNU} {
|
||||
format := format
|
||||
t.Run(format.String(), func(t *testing.T) {
|
||||
data := tarWithLongMetadata(t, format)
|
||||
manifest, err := InspectBytes(data, FormatTar, InspectOptions{
|
||||
EntryFile: "index.html",
|
||||
Limits: testLimits,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, manifest.FileCount)
|
||||
assertManifestContains(t, manifest, "index.html")
|
||||
|
||||
destDir := t.TempDir()
|
||||
require.NoError(t, ExtractBytes(data, FormatTar, destDir, ExtractOptions{
|
||||
Limits: testLimits,
|
||||
EnforceLimits: true,
|
||||
}))
|
||||
_, err = os.Stat(filepath.Join(destDir, "index.html"))
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type countingFillReader struct {
|
||||
remaining int64
|
||||
read int64
|
||||
}
|
||||
|
||||
func (r *countingFillReader) Read(p []byte) (int, error) {
|
||||
if r.remaining == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if int64(len(p)) > r.remaining {
|
||||
p = p[:r.remaining]
|
||||
}
|
||||
for i := range p {
|
||||
p[i] = 'x'
|
||||
}
|
||||
r.remaining -= int64(len(p))
|
||||
r.read += int64(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func decodeFixture(t *testing.T, encoded string) []byte {
|
||||
t.Helper()
|
||||
data, err := base64.StdEncoding.DecodeString(encoded)
|
||||
require.NoError(t, err)
|
||||
return data
|
||||
}
|
||||
|
||||
func assertManifestContains(t *testing.T, manifest *Manifest, path string) {
|
||||
t.Helper()
|
||||
for _, file := range manifest.Files {
|
||||
if file.Path == path {
|
||||
return
|
||||
}
|
||||
}
|
||||
require.Failf(t, "manifest path missing", "path %q not found in %#v", path, manifest.Files)
|
||||
}
|
||||
|
||||
func testTar(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
writer := tar.NewWriter(&buffer)
|
||||
for name, content := range files {
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}))
|
||||
_, err := writer.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func tarWithDeclaredBodyOnly(t *testing.T, name string, size int64) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
writer := tar.NewWriter(&buffer)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: size,
|
||||
}))
|
||||
// Deliberately omit the body and trailer. The limit must reject from the
|
||||
// header before archive/tar attempts to stream the declared body.
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func tarWithSpecialEntry(t *testing.T, special *tar.Header) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
writer := tar.NewWriter(&buffer)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{
|
||||
Name: "index.html",
|
||||
Mode: 0o644,
|
||||
Size: 2,
|
||||
}))
|
||||
_, err := writer.Write([]byte("ok"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.WriteHeader(special))
|
||||
require.NoError(t, writer.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func zipWithSpecialEntry(t *testing.T, mode os.FileMode) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
writer := zip.NewWriter(&buffer)
|
||||
index, err := writer.Create("index.html")
|
||||
require.NoError(t, err)
|
||||
_, err = index.Write([]byte("ok"))
|
||||
require.NoError(t, err)
|
||||
|
||||
header := &zip.FileHeader{Name: "special"}
|
||||
header.SetMode(mode)
|
||||
special, err := writer.CreateHeader(header)
|
||||
require.NoError(t, err)
|
||||
if mode&os.ModeSymlink != 0 {
|
||||
_, err = special.Write([]byte("index.html"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func tarWithLongMetadata(t *testing.T, format tar.Format) []byte {
|
||||
t.Helper()
|
||||
var buffer bytes.Buffer
|
||||
writer := tar.NewWriter(&buffer)
|
||||
longName := strings.Repeat("long-segment-", 12) + "asset.js"
|
||||
header := &tar.Header{
|
||||
Name: longName,
|
||||
Mode: 0o644,
|
||||
Size: 1,
|
||||
Format: format,
|
||||
}
|
||||
if format == tar.FormatPAX {
|
||||
header.PAXRecords = map[string]string{"comment": "metadata is not a deployable member"}
|
||||
}
|
||||
require.NoError(t, writer.WriteHeader(header))
|
||||
_, err := writer.Write([]byte("x"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{
|
||||
Name: "index.html",
|
||||
Mode: 0o644,
|
||||
Size: 2,
|
||||
Format: format,
|
||||
}))
|
||||
_, err = writer.Write([]byte("ok"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
return buffer.Bytes()
|
||||
}
|
||||
Reference in New Issue
Block a user