fix(pages): 收紧部署包与 Agent 同步边界

完成 V2 Phase 0 安全与一致性前置:统一真实归档限额、流式拉取、候选裁剪、保留上传删除语义及 Pages 路由引用锁。
This commit is contained in:
deqiying
2026-07-19 16:42:45 +08:00
parent f386674464
commit 4e8ec23264
47 changed files with 4005 additions and 759 deletions
+40 -23
View File
@@ -4,6 +4,7 @@
package pagesarchive
import (
"errors"
"fmt"
"io"
"math"
@@ -45,39 +46,50 @@ type Entry struct {
IsDir bool
// IsSymlink marks symbolic links (unsupported for Pages).
IsSymlink bool
// Size is the declared uncompressed size when known; 0 means empty or unknown.
// IsHardlink marks hard links (unsupported for Pages).
IsHardlink bool
// IsSpecial marks device, FIFO, socket, and other non-regular entries.
IsSpecial bool
// Size is the archive-declared uncompressed size; 0 means an empty member.
Size uint64
// Open returns a reader for the entry body. Caller must Close it.
// May be unavailable for inspect-only tar listings (body not materialized).
Open func() (io.ReadCloser, error)
}
// copyLimited copies src to dst.
// When maxBytes <= 0, size limits are not enforced (trusted extract path).
func copyLimited(dst io.Writer, src io.Reader, declaredSize uint64, maxBytes int64) (int64, error) {
if maxBytes <= 0 {
if declaredSize > 0 {
if declaredSize > uint64(math.MaxInt64) {
return 0, fmt.Errorf("pages file size out of bounds")
}
//nolint:gosec // declaredSize is bounded to MaxInt64 above
return io.CopyN(dst, src, int64(declaredSize))
}
// copyLimited copies actual bytes from src. maxBytes < 0 disables the byte cap;
// maxBytes == 0 permits only an empty stream.
func copyLimited(dst io.Writer, src io.Reader, maxBytes int64) (int64, error) {
if maxBytes < 0 {
return io.Copy(dst, src)
}
if declaredSize > uint64(maxBytes) || declaredSize > uint64(math.MaxInt64) { //nolint:gosec // maxBytes positive
return 0, fmt.Errorf("pages file size out of bounds")
readLimit := maxBytes
if maxBytes < math.MaxInt64 {
readLimit++
}
if declaredSize > 0 {
//nolint:gosec // declaredSize is bounded to MaxInt64 above
return io.CopyN(dst, src, int64(declaredSize))
written, err := io.Copy(dst, io.LimitReader(src, readLimit))
if err != nil {
return written, err
}
limited := io.LimitReader(src, maxBytes+1)
written, err := io.Copy(dst, limited)
if written > maxBytes {
return written, fmt.Errorf("pages file size out of bounds")
}
return written, err
return written, nil
}
func copyAndVerifySize(dst io.Writer, src io.Reader, declaredSize uint64, maxBytes int64) (int64, error) {
if declaredSize > uint64(math.MaxInt64) {
return 0, fmt.Errorf("pages file size out of bounds")
}
written, err := copyLimited(dst, src, maxBytes)
if err != nil {
return written, err
}
//nolint:gosec // declaredSize is bounded to MaxInt64 above
if written != int64(declaredSize) {
return written, fmt.Errorf("pages declared size %d does not match actual %d", declaredSize, written)
}
return written, nil
}
func writeEntryFile(targetPath string, src io.Reader, declaredSize uint64, maxBytes int64, perm os.FileMode) (int64, error) {
@@ -88,6 +100,11 @@ func writeEntryFile(targetPath string, src io.Reader, declaredSize uint64, maxBy
if err != nil {
return 0, err
}
defer func() { _ = target.Close() }()
return copyLimited(target, src, declaredSize, maxBytes)
written, copyErr := copyAndVerifySize(target, src, declaredSize, maxBytes)
closeErr := target.Close()
if err := errors.Join(copyErr, closeErr); err != nil {
_ = os.Remove(targetPath)
return written, err
}
return written, nil
}
+141 -68
View File
@@ -4,7 +4,9 @@
package pagesarchive
import (
"archive/tar"
"bytes"
"errors"
"fmt"
"io"
"os"
@@ -16,14 +18,12 @@ const formatDetectHeadBytes = 512
// ExtractOptions controls package extraction.
type ExtractOptions struct {
// Limits bounds files and sizes during extraction when EnforceLimits is true.
// Limits bounds actual files and sizes during extraction when EnforceLimits is true.
Limits Limits
// StripCommonRoot strips a single shared top-level directory when present.
StripCommonRoot bool
// EnforceLimits enables MaxFiles / MaxFileBytes / MaxTotalBytes checks.
// When false, the caller is assumed to have already validated the package
// (e.g. Agent trusts control-plane inspection). Path-escape and symlink
// guards still apply so local extraction cannot leave destDir.
// Path, member type, and declared/actual-size validation always remain enabled.
EnforceLimits bool
}
@@ -36,16 +36,11 @@ func ExtractBytes(data []byte, format Format, destDir string, opts ExtractOption
return err
}
}
entries, err := listEntriesAt(bytes.NewReader(data), int64(len(data)), format, true)
if err != nil {
return err
}
return extractEntries(entries, destDir, opts)
return extractFromReaderAt(bytes.NewReader(data), int64(len(data)), format, destDir, opts)
}
// ExtractFile opens path and extracts it into destDir without buffering the
// whole archive as an intermediate []byte for zip/7z (ReaderAt). Tar-family
// formats still materialize member bodies so random Open works for extract.
// whole archive or tar member bodies in memory.
func ExtractFile(filePath string, format Format, destDir string, opts ExtractOptions) error {
file, err := os.Open(filePath) //nolint:gosec // controlled path
if err != nil {
@@ -68,7 +63,14 @@ func ExtractFile(filePath string, format Format, destDir string, opts ExtractOpt
return err
}
}
entries, err := listEntriesAt(file, info.Size(), format, true)
return extractFromReaderAt(file, info.Size(), format, destDir, opts)
}
func extractFromReaderAt(ra io.ReaderAt, size int64, format Format, destDir string, opts ExtractOptions) error {
if isTarFamily(format) {
return extractTarFamilyAt(ra, size, format, destDir, opts)
}
entries, err := listRandomAccessEntriesAt(ra, size, format)
if err != nil {
return err
}
@@ -80,88 +82,159 @@ func extractEntries(entries []Entry, destDir string, opts ExtractOptions) error
if opts.EnforceLimits {
limits = normalizeLimits(opts.Limits)
}
commonPrefix := ""
if opts.StripCommonRoot {
commonPrefix = FindCommonRootPrefix(collectFileNames(entries))
commonPrefix, err := commonRootForEntries(entries, opts.StripCommonRoot)
if err != nil {
return err
}
var totalSize int64
var fileCount int
measured := &measuredArchive{files: make([]measuredFile, 0, len(entries))}
for _, entry := range entries {
written, counted, err := extractSingleEntry(entry, destDir, commonPrefix, limits, opts.EnforceLimits)
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return err
}
if !counted {
if skip {
continue
}
fileCount++
if opts.EnforceLimits && fileCount > limits.MaxFiles {
return fmt.Errorf("pages deployment file count exceeds %d", limits.MaxFiles)
normalizedPath = StripPrefix(normalizedPath, commonPrefix)
if normalizedPath == "" {
continue
}
totalSize += written
if opts.EnforceLimits && totalSize > limits.MaxTotalBytes {
return fmt.Errorf("pages extracted size exceeds limit")
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, opts.EnforceLimits); err != nil {
return err
}
if entry.Open == nil {
return fmt.Errorf("%s: pages archive member cannot be opened", normalizedPath)
}
src, err := entry.Open()
if err != nil {
return fmt.Errorf("%s: %w", normalizedPath, err)
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, opts.EnforceLimits)
targetPath, err := safeExtractionTarget(destDir, normalizedPath)
if err != nil {
_ = src.Close()
return err
}
actual, writeErr := writeEntryFile(targetPath, src, entry.Size, maxBytes, filePerm)
closeErr := src.Close()
if err := errors.Join(writeErr, closeErr); err != nil {
return fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
if fileCount == 0 {
if measured.fileCount == 0 {
return fmt.Errorf("pages package is empty")
}
return nil
}
func extractSingleEntry(
entry Entry,
destDir, commonPrefix string,
func extractTarFamilyAt(ra io.ReaderAt, size int64, format Format, destDir string, opts ExtractOptions) error {
limits := Limits{}
if opts.EnforceLimits {
limits = normalizeLimits(opts.Limits)
}
firstPass, err := scanTarFamilyAt(ra, size, format, limits, opts.EnforceLimits)
if err != nil {
return err
}
if firstPass.fileCount == 0 {
return fmt.Errorf("pages package is empty")
}
commonPrefix := ""
if opts.StripCommonRoot {
paths := make([]string, 0, len(firstPass.files))
for _, file := range firstPass.files {
paths = append(paths, file.path)
}
commonPrefix = FindCommonRootPrefix(paths)
}
tarReader, closeReader, err := openTarFamilyReader(io.NewSectionReader(ra, 0, size), format)
if err != nil {
return err
}
secondPass, extractErr := extractTarReader(tarReader, destDir, commonPrefix, limits, opts.EnforceLimits)
if closeErr := closeReader(); closeErr != nil {
extractErr = errors.Join(extractErr, closeErr)
}
if extractErr != nil {
return extractErr
}
if secondPass.fileCount != firstPass.fileCount || secondPass.totalSize != firstPass.totalSize {
return fmt.Errorf("pages tar package changed between validation and extraction")
}
return nil
}
func extractTarReader(
tarReader *tar.Reader,
destDir string,
commonPrefix string,
limits Limits,
enforceLimits bool,
) (written int64, counted bool, err error) {
relativePath, skip, err := NormalizeEntryPath(entry.Name)
if err != nil {
return 0, false, err
) (*measuredArchive, error) {
measured := &measuredArchive{files: make([]measuredFile, 0)}
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read tar pages package: %w", err)
}
entry := entryFromTarHeader(header)
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return nil, err
}
if skip {
continue
}
normalizedPath = StripPrefix(normalizedPath, commonPrefix)
if normalizedPath == "" {
continue
}
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, enforceLimits); err != nil {
return nil, err
}
targetPath, err := safeExtractionTarget(destDir, normalizedPath)
if err != nil {
return nil, err
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, enforceLimits)
actual, err := writeEntryFile(targetPath, tarReader, entry.Size, maxBytes, filePerm)
if err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
if skip {
return 0, false, nil
}
if commonPrefix != "" {
relativePath = StripPrefix(relativePath, commonPrefix)
if relativePath == "" {
return 0, false, nil
return measured, nil
}
func commonRootForEntries(entries []Entry, strip bool) (string, error) {
paths := make([]string, 0, len(entries))
for _, entry := range entries {
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return "", err
}
if !skip {
paths = append(paths, normalizedPath)
}
}
if entry.IsSymlink {
return 0, false, fmt.Errorf("pages package contains unsupported symlink: %s", relativePath)
if !strip {
return "", nil
}
return FindCommonRootPrefix(paths), nil
}
func safeExtractionTarget(destDir, relativePath string) (string, error) {
targetPath := filepath.Join(destDir, filepath.FromSlash(relativePath))
if !isWithinDir(destDir, targetPath) {
return 0, false, fmt.Errorf("pages package path escapes directory: %s", entry.Name)
return "", fmt.Errorf("pages package path escapes directory: %s", relativePath)
}
if entry.IsDir {
if err := os.MkdirAll(targetPath, dirPerm); err != nil {
return 0, false, err
}
return 0, false, nil
}
maxFileBytes := int64(0) // unlimited when not enforcing
if enforceLimits {
if exceedsFileByteLimit(entry.Size, limits.MaxFileBytes) {
return 0, false, fmt.Errorf("pages file too large: %s", relativePath)
}
maxFileBytes = limits.MaxFileBytes
}
src, err := entry.Open()
if err != nil {
return 0, false, fmt.Errorf("%s: %w", relativePath, err)
}
written, writeErr := writeEntryFile(targetPath, src, entry.Size, maxFileBytes, filePerm)
_ = src.Close()
if writeErr != nil {
return 0, false, fmt.Errorf("%s: %w", relativePath, writeErr)
}
return written, true, nil
return targetPath, nil
}
func isWithinDir(baseDir, targetPath string) bool {
+189 -93
View File
@@ -4,13 +4,14 @@
package pagesarchive
import (
"archive/tar"
"bytes"
"errors"
"fmt"
"io"
"math"
"os"
"path"
"strings"
)
// InspectOptions controls package inspection.
@@ -19,17 +20,26 @@ type InspectOptions struct {
RootDir string
// EntryFile is the required entry file name (e.g. index.html).
EntryFile string
// Limits bounds files and sizes.
// Limits bounds files and actual extracted sizes.
Limits Limits
// VerifySizes, when true, streams each regular file and compares the actual
// byte count against the archive-declared size (no content hashing).
// Default false: trust zip central directory / tar header sizes.
// VerifySizes is retained for source compatibility. Inspection now always
// streams regular members and verifies actual bytes against declared sizes.
VerifySizes bool
}
type measuredFile struct {
path string
size int64
}
type measuredArchive struct {
files []measuredFile
fileCount int
totalSize int64
}
// InspectFile opens path and inspects it as a Pages deployment package without
// loading the whole archive into memory. File inventory uses declared sizes;
// per-file content hashes are not computed.
// loading the whole archive or any tar member body into memory.
func InspectFile(filePath string, format Format, opts InspectOptions) (*Manifest, error) {
file, err := os.Open(filePath) //nolint:gosec // filePath is a controlled temp upload path
if err != nil {
@@ -68,55 +78,137 @@ func InspectBytes(data []byte, format Format, opts InspectOptions) (*Manifest, e
}
func inspectFromReaderAt(ra io.ReaderAt, size int64, format Format, opts InspectOptions) (*Manifest, error) {
// Default: zip/7z use central directory only; tar streams headers and discards bodies.
// VerifySizes needs openable tar bodies, so materialize only when requested.
entries, err := listEntriesAt(ra, size, format, opts.VerifySizes)
limits := normalizeLimits(opts.Limits)
var (
measured *measuredArchive
err error
)
if isTarFamily(format) {
measured, err = scanTarFamilyAt(ra, size, format, limits, true)
} else {
var entries []Entry
entries, err = listRandomAccessEntriesAt(ra, size, format)
if err == nil {
measured, err = inspectRandomAccessEntries(entries, limits)
}
}
if err != nil {
return nil, err
}
return buildManifest(entries, opts)
return buildMeasuredManifest(measured, opts)
}
func buildManifest(entries []Entry, opts InspectOptions) (*Manifest, error) {
limits := normalizeLimits(opts.Limits)
commonPrefix := FindCommonRootPrefix(collectFileNames(entries))
targetEntryPath := resolveTargetEntryPath(opts.RootDir, opts.EntryFile)
manifest := &Manifest{Files: make([]FileEntry, 0)}
entrySeen := false
func inspectRandomAccessEntries(entries []Entry, limits Limits) (*measuredArchive, error) {
measured := &measuredArchive{files: make([]measuredFile, 0, len(entries))}
for _, entry := range entries {
normalizedPath, skip, err := prepareEntryPath(entry, commonPrefix)
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return nil, err
}
if skip {
continue
}
if exceedsFileByteLimit(entry.Size, limits.MaxFileBytes) {
return nil, fmt.Errorf("pages file too large: %s", normalizedPath)
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, true); err != nil {
return nil, err
}
if entry.Open == nil {
return nil, fmt.Errorf("%s: pages archive member cannot be opened", normalizedPath)
}
src, err := entry.Open()
if err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, true)
actual, copyErr := copyAndVerifySize(io.Discard, src, entry.Size, maxBytes)
closeErr := src.Close()
if err := errors.Join(copyErr, closeErr); err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
return measured, nil
}
fileEntry, err := inspectRegularFile(entry, normalizedPath, limits, opts.VerifySizes)
func scanTarFamilyAt(
ra io.ReaderAt,
size int64,
format Format,
limits Limits,
enforceLimits bool,
) (*measuredArchive, error) {
if size < 0 {
return nil, fmt.Errorf("invalid pages package size")
}
tarReader, closeReader, err := openTarFamilyReader(io.NewSectionReader(ra, 0, size), format)
if err != nil {
return nil, err
}
measured, scanErr := scanTarReader(tarReader, limits, enforceLimits)
if closeErr := closeReader(); closeErr != nil {
scanErr = errors.Join(scanErr, closeErr)
}
return measured, scanErr
}
func scanTarReader(tarReader *tar.Reader, limits Limits, enforceLimits bool) (*measuredArchive, error) {
measured := &measuredArchive{files: make([]measuredFile, 0)}
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read tar pages package: %w", err)
}
entry := entryFromTarHeader(header)
normalizedPath, skip, err := validateArchiveEntry(entry)
if err != nil {
return nil, err
}
manifest.FileCount++
if manifest.FileCount > limits.MaxFiles {
return nil, fmt.Errorf("pages deployment file count exceeds %d", limits.MaxFiles)
if skip {
continue
}
manifest.TotalSize += fileEntry.Size
if manifest.TotalSize > limits.MaxTotalBytes {
return nil, fmt.Errorf("pages extracted size exceeds limit")
if err := prepareMeasuredFile(measured, normalizedPath, entry.Size, limits, enforceLimits); err != nil {
return nil, err
}
maxBytes := effectiveFileLimit(limits, measured.totalSize, enforceLimits)
actual, err := copyAndVerifySize(io.Discard, tarReader, entry.Size, maxBytes)
if err != nil {
return nil, fmt.Errorf("%s: %w", normalizedPath, err)
}
appendMeasuredFile(measured, normalizedPath, actual)
}
return measured, nil
}
func buildMeasuredManifest(measured *measuredArchive, opts InspectOptions) (*Manifest, error) {
if measured == nil || measured.fileCount == 0 {
return nil, fmt.Errorf("pages package is empty")
}
targetEntryPath, err := resolveTargetEntryPath(opts.RootDir, opts.EntryFile)
if err != nil {
return nil, err
}
paths := make([]string, 0, len(measured.files))
for _, file := range measured.files {
paths = append(paths, file.path)
}
commonPrefix := FindCommonRootPrefix(paths)
manifest := &Manifest{
Files: make([]FileEntry, 0, measured.fileCount),
FileCount: measured.fileCount,
TotalSize: measured.totalSize,
}
entrySeen := false
for _, file := range measured.files {
normalizedPath := StripPrefix(file.path, commonPrefix)
if normalizedPath == targetEntryPath {
entrySeen = true
}
manifest.Files = append(manifest.Files, fileEntry)
}
if manifest.FileCount == 0 {
return nil, fmt.Errorf("pages package is empty")
manifest.Files = append(manifest.Files, FileEntry{
Path: normalizedPath,
Size: file.size,
})
}
if !entrySeen {
return nil, fmt.Errorf("pages package is missing entry file %s", targetEntryPath)
@@ -124,82 +216,86 @@ func buildManifest(entries []Entry, opts InspectOptions) (*Manifest, error) {
return manifest, nil
}
func collectFileNames(entries []Entry) []string {
names := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir || entry.IsSymlink {
continue
}
names = append(names, entry.Name)
func prepareMeasuredFile(measured *measuredArchive, normalizedPath string, declaredSize uint64, limits Limits, enforceLimits bool) error {
if declaredSize > uint64(math.MaxInt64) {
return fmt.Errorf("%s: pages file size out of bounds", normalizedPath)
}
return names
if !enforceLimits {
return nil
}
if measured.fileCount >= limits.MaxFiles {
return fmt.Errorf("pages deployment file count exceeds %d", limits.MaxFiles)
}
if exceedsFileByteLimit(declaredSize, limits.MaxFileBytes) {
return fmt.Errorf("pages file too large: %s", normalizedPath)
}
remaining := limits.MaxTotalBytes - measured.totalSize
if remaining < 0 || declaredSize > uint64(remaining) { //nolint:gosec // remaining is checked non-negative
return fmt.Errorf("pages extracted size exceeds limit")
}
return nil
}
func resolveTargetEntryPath(rootDir, entryFile string) string {
normalizedEntry := strings.TrimSpace(entryFile)
if normalizedEntry == "" {
normalizedEntry = "index.html"
func appendMeasuredFile(measured *measuredArchive, normalizedPath string, actual int64) {
measured.files = append(measured.files, measuredFile{path: normalizedPath, size: actual})
measured.fileCount++
measured.totalSize += actual
}
func effectiveFileLimit(limits Limits, totalSize int64, enforceLimits bool) int64 {
if !enforceLimits {
return -1
}
remaining := limits.MaxTotalBytes - totalSize
if remaining < limits.MaxFileBytes {
return remaining
}
return limits.MaxFileBytes
}
func resolveTargetEntryPath(rootDir, entryFile string) (string, error) {
normalizedRoot, err := NormalizeLogicalPath(rootDir, true)
if err != nil {
return "", fmt.Errorf("invalid pages root directory: %w", err)
}
if entryFile == "" {
entryFile = "index.html"
}
normalizedEntry, err := NormalizeLogicalPath(entryFile, false)
if err != nil {
return "", fmt.Errorf("invalid pages entry file: %w", err)
}
normalizedRoot := strings.Trim(strings.TrimSpace(rootDir), "/")
if normalizedRoot == "" {
return normalizedEntry
return normalizedEntry, nil
}
return path.Join(normalizedRoot, normalizedEntry)
return path.Join(normalizedRoot, normalizedEntry), nil
}
func prepareEntryPath(entry Entry, commonPrefix string) (string, bool, error) {
func validateArchiveEntry(entry Entry) (string, bool, error) {
normalizedPath, skip, err := NormalizeEntryPath(entry.Name)
if err != nil {
return "", false, err
}
if skip || entry.IsDir {
return "", true, nil
}
normalizedPath = StripPrefix(normalizedPath, commonPrefix)
if entry.IsSymlink {
return "", false, fmt.Errorf("pages package contains unsupported symlink: %s", normalizedPath)
}
if entry.IsHardlink {
return "", false, fmt.Errorf("pages package contains unsupported hardlink: %s", normalizedPath)
}
if entry.IsSpecial {
return "", false, fmt.Errorf("pages package contains unsupported special entry: %s", normalizedPath)
}
if skip || entry.IsDir {
return normalizedPath, true, nil
}
return normalizedPath, false, nil
}
func inspectRegularFile(entry Entry, normalizedPath string, limits Limits, verifySizes bool) (FileEntry, error) {
if entry.Size > uint64(math.MaxInt64) {
return FileEntry{}, fmt.Errorf("%s: pages file size out of bounds", normalizedPath)
func isTarFamily(format Format) bool {
switch format {
case FormatTar, FormatTarGz, FormatTarXz, FormatTarBz2:
return true
default:
return false
}
//nolint:gosec // bounded to MaxInt64 above
declaredSize := int64(entry.Size)
if !verifySizes {
return FileEntry{
Path: normalizedPath,
Size: declaredSize,
Checksum: "",
}, nil
}
if entry.Open == nil {
return FileEntry{}, fmt.Errorf("%s: cannot verify size without entry open", normalizedPath)
}
src, err := entry.Open()
if err != nil {
return FileEntry{}, fmt.Errorf("%s: %w", normalizedPath, err)
}
actual, measureErr := measureReader(src, entry.Size, limits.MaxFileBytes)
_ = src.Close()
if measureErr != nil {
return FileEntry{}, fmt.Errorf("%s: %w", normalizedPath, measureErr)
}
if declaredSize > 0 && actual != declaredSize {
return FileEntry{}, fmt.Errorf("%s: declared size %d does not match actual %d", normalizedPath, declaredSize, actual)
}
return FileEntry{
Path: normalizedPath,
Size: actual,
Checksum: "",
}, nil
}
// measureReader counts bytes without hashing, enforcing maxBytes when positive.
func measureReader(src io.Reader, declaredSize uint64, maxBytes int64) (int64, error) {
return copyLimited(io.Discard, src, declaredSize, maxBytes)
}
+49 -172
View File
@@ -6,7 +6,6 @@ package pagesarchive
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"fmt"
@@ -19,8 +18,8 @@ import (
type archiveFile interface {
Name() string
Mode() os.FileMode
IsDir() bool
IsSymlink() bool
Size() uint64
Open() (io.ReadCloser, error)
}
@@ -29,12 +28,10 @@ type zipArchiveFile struct {
file *zip.File
}
func (z zipArchiveFile) Name() string { return z.file.Name }
func (z zipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
func (z zipArchiveFile) IsSymlink() bool {
return z.file.Mode()&os.ModeSymlink != 0
}
func (z zipArchiveFile) Size() uint64 { return z.file.UncompressedSize64 }
func (z zipArchiveFile) Name() string { return z.file.Name }
func (z zipArchiveFile) Mode() os.FileMode { return z.file.Mode() }
func (z zipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
func (z zipArchiveFile) Size() uint64 { return z.file.UncompressedSize64 }
func (z zipArchiveFile) Open() (io.ReadCloser, error) {
return z.file.Open()
}
@@ -43,65 +40,27 @@ type sevenZipArchiveFile struct {
file *sevenzip.File
}
func (z sevenZipArchiveFile) Name() string { return z.file.Name }
func (z sevenZipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
func (z sevenZipArchiveFile) IsSymlink() bool {
return z.file.Mode()&os.ModeSymlink != 0
}
func (z sevenZipArchiveFile) Size() uint64 { return z.file.UncompressedSize }
func (z sevenZipArchiveFile) Name() string { return z.file.Name }
func (z sevenZipArchiveFile) Mode() os.FileMode { return z.file.Mode() }
func (z sevenZipArchiveFile) IsDir() bool { return z.file.FileInfo().IsDir() }
func (z sevenZipArchiveFile) Size() uint64 { return z.file.UncompressedSize }
func (z sevenZipArchiveFile) Open() (io.ReadCloser, error) {
return z.file.Open()
}
// listEntriesAt lists archive members from a random-access source.
// When materializeBodies is true, tar-family streams buffer regular-file bodies so Entry.Open works.
// When false (inspect path), tar bodies are discarded after reading headers; zip/7z only use central directory metadata.
func listEntriesAt(ra io.ReaderAt, size int64, format Format, materializeBodies bool) ([]Entry, error) {
// listRandomAccessEntriesAt lists zip/7z members without reading their bodies.
// Tar-family archives use the sequential streaming paths in inspect.go/extract.go.
func listRandomAccessEntriesAt(ra io.ReaderAt, size int64, format Format) ([]Entry, error) {
if size < 0 {
return nil, fmt.Errorf("invalid pages package size")
}
switch format {
case FormatZip:
return listZipEntriesAt(ra, size)
case FormatTar:
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTar, materializeBodies)
case FormatTarGz:
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarGz, materializeBodies)
case FormatTarXz:
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarXz, materializeBodies)
case FormatTarBz2:
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarBz2, materializeBodies)
case FormatSevenZip:
return listSevenZipEntriesAt(ra, size)
default:
return nil, fmt.Errorf("unsupported pages package format: %s", format)
}
}
func listTarFamily(r io.Reader, format Format, materializeBodies bool) ([]Entry, error) {
switch format {
case FormatTar:
if materializeBodies {
return listTarEntries(r, true)
}
return listTarEntries(r, false)
case FormatTarGz:
gzReader, err := gzip.NewReader(r)
if err != nil {
return nil, fmt.Errorf("open gzip pages package: %w", err)
}
defer func() { _ = gzReader.Close() }()
return listTarEntries(gzReader, materializeBodies)
case FormatTarXz:
xzReader, err := xz.NewReader(r)
if err != nil {
return nil, fmt.Errorf("open xz pages package: %w", err)
}
return listTarEntries(xzReader, materializeBodies)
case FormatTarBz2:
return listTarEntries(bzip2.NewReader(r), materializeBodies)
default:
return nil, fmt.Errorf("unsupported tar family format: %s", format)
return nil, fmt.Errorf("unsupported random-access pages package format: %s", format)
}
}
@@ -109,10 +68,15 @@ func entriesFromArchiveFiles(files []archiveFile) []Entry {
entries := make([]Entry, 0, len(files))
for _, item := range files {
file := item
mode := file.Mode()
isDir := file.IsDir()
isSymlink := mode&os.ModeSymlink != 0
isSpecial := !isDir && !isSymlink && !mode.IsRegular()
entries = append(entries, Entry{
Name: file.Name(),
IsDir: file.IsDir(),
IsSymlink: file.IsSymlink(),
IsDir: isDir,
IsSymlink: isSymlink,
IsSpecial: isSpecial,
Size: file.Size(),
Open: file.Open,
})
@@ -144,132 +108,45 @@ func listSevenZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
return entriesFromArchiveFiles(files), nil
}
func listTarEntries(r io.Reader, materializeBodies bool) ([]Entry, error) {
tarReader := tar.NewReader(r)
type materialised struct {
header *tar.Header
body []byte
}
items := make([]materialised, 0)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
func openTarFamilyReader(r io.Reader, format Format) (*tar.Reader, func() error, error) {
switch format {
case FormatTar:
return tar.NewReader(r), func() error { return nil }, nil
case FormatTarGz:
gzReader, err := gzip.NewReader(r)
if err != nil {
return nil, fmt.Errorf("read tar pages package: %w", err)
return nil, nil, fmt.Errorf("open gzip pages package: %w", err)
}
item, skip, err := readTarHeader(tarReader, header, materializeBodies)
return tar.NewReader(gzReader), gzReader.Close, nil
case FormatTarXz:
xzReader, err := xz.NewReader(r)
if err != nil {
return nil, err
return nil, nil, fmt.Errorf("open xz pages package: %w", err)
}
if skip {
continue
}
items = append(items, item)
}
entries := make([]Entry, 0, len(items))
for _, item := range items {
entries = append(entries, tarEntryFromHeader(item.header, item.body, materializeBodies))
}
return entries, nil
}
func readTarHeader(tarReader *tar.Reader, header *tar.Header, materializeBodies bool) (item struct {
header *tar.Header
body []byte
}, skip bool, err error) {
switch header.Typeflag {
case tar.TypeDir, tar.TypeSymlink, tar.TypeLink:
return struct {
header *tar.Header
body []byte
}{header: header}, false, nil
case tar.TypeReg, tar.TypeRegA: //nolint:staticcheck // TypeRegA still appears in older archives
if !materializeBodies {
if err := discardTarBody(tarReader, header); err != nil {
return item, false, err
}
return struct {
header *tar.Header
body []byte
}{header: header}, false, nil
}
body, readErr := readTarBody(tarReader, header)
if readErr != nil {
return item, false, readErr
}
return struct {
header *tar.Header
body []byte
}{header: header, body: body}, false, nil
return tar.NewReader(xzReader), func() error { return nil }, nil
case FormatTarBz2:
return tar.NewReader(bzip2.NewReader(r)), func() error { return nil }, nil
default:
if header.Size > 0 {
if _, copyErr := io.CopyN(io.Discard, tarReader, header.Size); copyErr != nil {
return item, false, fmt.Errorf("skip tar entry %s: %w", header.Name, copyErr)
}
}
return item, true, nil
return nil, nil, fmt.Errorf("unsupported tar family format: %s", format)
}
}
func discardTarBody(tarReader *tar.Reader, header *tar.Header) error {
if header.Size <= 0 {
_, err := io.Copy(io.Discard, tarReader)
if err != nil {
return fmt.Errorf("discard tar entry %s: %w", header.Name, err)
}
return nil
}
if _, err := io.CopyN(io.Discard, tarReader, header.Size); err != nil {
return fmt.Errorf("discard tar entry %s: %w", header.Name, err)
}
return nil
}
func readTarBody(tarReader *tar.Reader, header *tar.Header) ([]byte, error) {
func entryFromTarHeader(header *tar.Header) Entry {
entry := Entry{Name: header.Name}
if header.Size > 0 {
body := make([]byte, header.Size)
if _, err := io.ReadFull(tarReader, body); err != nil {
return nil, fmt.Errorf("read tar entry %s: %w", header.Name, err)
}
return body, nil
entry.Size = uint64(header.Size) //nolint:gosec // archive/tar rejects negative sizes
}
body, err := io.ReadAll(tarReader)
if err != nil {
return nil, fmt.Errorf("read tar entry %s: %w", header.Name, err)
}
return body, nil
}
func tarEntryFromHeader(header *tar.Header, body []byte, materializeBodies bool) Entry {
size := header.Size
if materializeBodies && int64(len(body)) > size {
size = int64(len(body))
}
entry := Entry{
Name: header.Name,
IsDir: header.Typeflag == tar.TypeDir,
IsSymlink: header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeLink,
Size: uint64(size), //nolint:gosec // non-negative sizes
}
if entry.IsDir || entry.IsSymlink {
entry.Open = func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(nil)), nil
}
return entry
}
if materializeBodies {
bodyCopy := body
entry.Open = func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(bodyCopy)), nil
}
return entry
}
// Inspect path: body not retained; Open is unavailable.
entry.Open = func() (io.ReadCloser, error) {
return nil, fmt.Errorf("tar entry body not materialized: %s", header.Name)
switch header.Typeflag {
case tar.TypeReg, tar.TypeRegA: //nolint:staticcheck // TypeRegA appears in older archives
// Regular file.
case tar.TypeDir:
entry.IsDir = true
case tar.TypeSymlink:
entry.IsSymlink = true
case tar.TypeLink:
entry.IsHardlink = true
default:
entry.IsSpecial = true
}
return entry
}
+74 -18
View File
@@ -6,33 +6,89 @@ package pagesarchive
import (
"fmt"
"path"
"path/filepath"
"strings"
"unicode"
"unicode/utf8"
)
// NormalizeLogicalPath validates and normalizes a relative POSIX path.
// Empty input is returned unchanged only when allowEmpty is true.
func NormalizeLogicalPath(raw string, allowEmpty bool) (string, error) {
if raw == "" {
if allowEmpty {
return "", nil
}
return "", fmt.Errorf("pages path is required")
}
if err := validateLogicalPathText(raw); err != nil {
return "", err
}
cleaned := path.Clean(raw)
if cleaned == "." || cleaned == "" {
if allowEmpty {
return "", nil
}
return "", fmt.Errorf("pages path is required")
}
if strings.HasPrefix(cleaned, "/") || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
return "", fmt.Errorf("pages path escapes directory: %s", raw)
}
return cleaned, nil
}
func validateLogicalPathText(raw string) error {
if !utf8.ValidString(raw) {
return fmt.Errorf("pages path is not valid UTF-8")
}
if strings.Contains(raw, "\\") {
return fmt.Errorf("pages path must use POSIX separators: %s", raw)
}
if strings.HasPrefix(raw, "/") || path.IsAbs(raw) {
return fmt.Errorf("pages path must be relative: %s", raw)
}
if err := validateLogicalPathRunes(raw); err != nil {
return err
}
return validateLogicalPathSegments(raw)
}
func validateLogicalPathRunes(raw string) error {
for _, r := range raw {
if r == 0 || unicode.IsControl(r) {
return fmt.Errorf("pages path contains a control character")
}
if r == '\'' || r == '"' || r == ';' {
return fmt.Errorf("pages path contains an unsupported character: %s", raw)
}
}
return nil
}
func validateLogicalPathSegments(raw string) error {
for _, segment := range strings.Split(raw, "/") {
if len(segment) >= 2 && segment[1] == ':' {
return fmt.Errorf("pages path contains a Windows drive: %s", raw)
}
if segment == "." || segment == ".." {
return fmt.Errorf("pages path escapes directory or contains a dot segment: %s", raw)
}
}
return nil
}
// NormalizeEntryPath cleans an archive entry path and rejects zip-slip / absolute paths.
// skip=true means the entry should be ignored (empty path or directory marker).
func NormalizeEntryPath(raw string) (cleaned string, skip bool, err error) {
name := strings.TrimSpace(filepath.ToSlash(raw))
if name == "" {
if raw == "" {
return "", true, nil
}
if strings.HasSuffix(name, "/") {
return "", true, nil
cleanedPath, normalizeErr := NormalizeLogicalPath(raw, false)
if normalizeErr != nil {
return "", false, fmt.Errorf("invalid pages package path %q: %w", raw, normalizeErr)
}
if strings.HasPrefix(name, "/") || path.IsAbs(name) {
return "", false, fmt.Errorf("pages package contains absolute path: %s", raw)
}
// Reject Windows drive / UNC-style paths that may appear after ToSlash.
if len(name) >= 2 && name[1] == ':' {
return "", false, fmt.Errorf("pages package contains absolute path: %s", raw)
}
cleanedPath := path.Clean(name)
if cleanedPath == "." {
return "", true, nil
}
if cleanedPath == ".." || strings.HasPrefix(cleanedPath, "../") || strings.Contains(cleanedPath, "/../") {
return "", false, fmt.Errorf("pages package path escapes directory: %s", raw)
if strings.HasSuffix(raw, "/") {
return cleanedPath, true, nil
}
return cleanedPath, false, nil
}
+499
View File
@@ -0,0 +1,499 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package pagesarchive
import (
"archive/tar"
"archive/zip"
"bytes"
"encoding/base64"
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var testLimits = Limits{
MaxFiles: 100,
MaxFileBytes: 1 << 20,
MaxTotalBytes: 1 << 20,
}
func TestNormalizeLogicalPathStrict(t *testing.T) {
t.Parallel()
valid := []struct {
name string
raw string
allowEmpty bool
want string
}{
{name: "empty root", allowEmpty: true},
{name: "single file", raw: "index.html", want: "index.html"},
{name: "nested posix", raw: "public/assets/app.js", want: "public/assets/app.js"},
{name: "unicode", raw: "静态/首页.html", want: "静态/首页.html"},
{name: "repeated separator is normalized", raw: "public//app.js", want: "public/app.js"},
}
for _, tt := range valid {
t.Run(tt.name, func(t *testing.T) {
got, err := NormalizeLogicalPath(tt.raw, tt.allowEmpty)
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
invalidUTF8 := string([]byte{'a', '/', 0xff})
invalid := []struct {
name string
raw string
}{
{name: "empty entry"},
{name: "absolute", raw: "/etc/passwd"},
{name: "unc", raw: "//server/share"},
{name: "windows drive", raw: "C:/site/index.html"},
{name: "nested windows drive", raw: "site/C:/index.html"},
{name: "windows separator", raw: `site\index.html`},
{name: "windows unc", raw: `\\server\share`},
{name: "parent segment", raw: "../index.html"},
{name: "nested parent segment", raw: "site/../index.html"},
{name: "current segment", raw: "site/./index.html"},
{name: "nul", raw: "site/\x00index.html"},
{name: "newline", raw: "site/\nindex.html"},
{name: "delete control", raw: "site/\x7findex.html"},
{name: "single quote", raw: "site/'index.html"},
{name: "double quote", raw: `site/"index.html`},
{name: "semicolon", raw: "site/;index.html"},
{name: "invalid utf8", raw: invalidUTF8},
}
for _, tt := range invalid {
t.Run(tt.name, func(t *testing.T) {
_, err := NormalizeLogicalPath(tt.raw, false)
require.Error(t, err)
})
}
cleaned, skip, err := NormalizeEntryPath("")
require.NoError(t, err)
assert.Empty(t, cleaned)
assert.True(t, skip)
cleaned, skip, err = NormalizeEntryPath("assets/")
require.NoError(t, err)
assert.Equal(t, "assets", cleaned)
assert.True(t, skip)
}
func TestSupportedFormatsInspectAndExtract(t *testing.T) {
t.Parallel()
sevenZipData := decodeFixture(t, "N3q8ryccAASgR6WICAAAAAAAAABmAAAAAAAAAN2R8/FiYXIKZm9vCgEEBgACCQQEAAcLAgABAQABAQAMBAQACAoB6bOiBKhlMn4AAAUCGQUAAAAAABERAGIAYQByAAAAZgBvAG8AAAAZAgAAFBIBAACFM3PyY9YBAFgCcvJj1gEVCgEAIICkgSCApIEAAA==")
bzipTarData := decodeFixture(t, "QlpoOTFBWSZTWYp5f6EAAHV//P64A8RQAf/iOm/9cO/v/9AAAgBADlAABAADAAgwAU1RIZJpNCaammmnqbSGTI9Q0BoBpp6mjIaGmmRoaHGRpkxNBkyYTTIGQ0BoDTJoYATQGG1KCntExT01MhoAABoAAHqAAPU9QacVDtN45fA6MmuGVQlWowrijpZgwASITYSPUcpJpoQGMkKq69jMkUR6L86R5j0IySUaZEjazEqhQ9E8vuuxsmWZQLCA84jNsobYNEzuEB1eCPhw8nc2AOz+xrCY5hVxQW1IIokpfSRKi+McvXU+QoYuEg6BD4w8x3K0imi+bULpkLCylCZ4lzoGlTQgibvG67sQcrTCRBTbBCVL7zC0q0qULmK/WOneu94s9cs4s4K98SjY2YvpdZvl42kwtxvvPMheorYQ2pcxyF4sNQYvd4+bgqm5gKXElqnGF3jhxGTeXp9eCUxWVlbi9ikxAik4xxATl7cJrISVWnHwUFiLdhEnKWw0Lhm3ZyKlX7P5Wj7b9TLAmWBaAwH/F3JFOFCQinl/oQ==")
cases := []struct {
name string
format Format
data []byte
entryFile string
wantPath string
}{
{name: "zip", format: FormatZip, data: testZip(t, map[string]string{"bundle/index.html": "zip"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar", format: FormatTar, data: testTar(t, map[string]string{"bundle/index.html": "tar"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar gzip", format: FormatTarGz, data: testTarGz(t, map[string]string{"bundle/index.html": "gzip"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar xz", format: FormatTarXz, data: testTarXz(t, map[string]string{"bundle/index.html": "xz"}), entryFile: "index.html", wantPath: "index.html"},
{name: "tar bzip2", format: FormatTarBz2, data: bzipTarData, entryFile: "index.html", wantPath: "index.html"},
{name: "7z", format: FormatSevenZip, data: sevenZipData, entryFile: "foo", wantPath: "foo"},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
manifest, err := InspectBytes(tt.data, tt.format, InspectOptions{
EntryFile: tt.entryFile,
Limits: testLimits,
})
require.NoError(t, err)
assert.Positive(t, manifest.FileCount)
assertManifestContains(t, manifest, tt.wantPath)
destDir := t.TempDir()
require.NoError(t, ExtractBytes(tt.data, tt.format, destDir, ExtractOptions{
Limits: testLimits,
StripCommonRoot: true,
EnforceLimits: true,
}))
_, err = os.Stat(filepath.Join(destDir, filepath.FromSlash(tt.wantPath)))
require.NoError(t, err)
})
}
}
func TestExtractFilePreservesCommonRootAndEnforcesLimits(t *testing.T) {
t.Parallel()
data := testTarGz(t, map[string]string{
"repository/dist/index.html": "pages",
"repository/dist/app.js": "javascript",
})
archivePath := filepath.Join(t.TempDir(), "site.tar.gz")
require.NoError(t, os.WriteFile(archivePath, data, 0o600))
manifest, err := InspectFile(archivePath, FormatTarGz, InspectOptions{
RootDir: "dist",
EntryFile: "index.html",
Limits: testLimits,
})
require.NoError(t, err)
assertManifestContains(t, manifest, "dist/index.html")
destDir := t.TempDir()
require.NoError(t, ExtractFile(archivePath, FormatTarGz, destDir, ExtractOptions{
Limits: testLimits,
StripCommonRoot: true,
EnforceLimits: true,
}))
body, err := os.ReadFile(filepath.Join(destDir, "dist", "index.html")) //nolint:gosec
require.NoError(t, err)
assert.Equal(t, "pages", string(body))
err = ExtractFile(archivePath, FormatTarGz, t.TempDir(), ExtractOptions{
Limits: Limits{
MaxFiles: 10,
MaxFileBytes: 5,
MaxTotalBytes: 1 << 20,
},
StripCommonRoot: true,
EnforceLimits: true,
})
require.ErrorContains(t, err, "file too large")
err = ExtractFile(archivePath, FormatTarGz, t.TempDir(), ExtractOptions{
Limits: Limits{
MaxFiles: 10,
MaxFileBytes: 1 << 20,
MaxTotalBytes: int64(len("pages") + len("javascript") - 1),
},
StripCommonRoot: true,
EnforceLimits: true,
})
require.ErrorContains(t, err, "extracted size exceeds limit")
}
func TestRandomAccessMembersVerifyActualSize(t *testing.T) {
t.Parallel()
entry := Entry{
Name: "index.html",
Size: 1,
Open: func() (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("actual-body")), nil
},
}
_, err := inspectRandomAccessEntries([]Entry{entry}, testLimits)
require.ErrorContains(t, err, "declared size 1 does not match actual 11")
destDir := t.TempDir()
err = extractEntries([]Entry{entry}, destDir, ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
})
require.ErrorContains(t, err, "declared size 1 does not match actual 11")
_, statErr := os.Stat(filepath.Join(destDir, "index.html"))
assert.ErrorIs(t, statErr, os.ErrNotExist, "failed extraction must remove the partial file")
}
func TestActualByteLimitAbortsReaderEarly(t *testing.T) {
t.Parallel()
reader := &countingFillReader{remaining: 1 << 30}
entry := Entry{
Name: "index.html",
Size: 0,
Open: func() (io.ReadCloser, error) {
return io.NopCloser(reader), nil
},
}
_, err := inspectRandomAccessEntries([]Entry{entry}, Limits{
MaxFiles: 1,
MaxFileBytes: 32,
MaxTotalBytes: 32,
})
require.ErrorContains(t, err, "size out of bounds")
assert.LessOrEqual(t, reader.read, int64(33), "inspection must stop after limit+1 actual bytes")
tarData := tarWithDeclaredBodyOnly(t, "index.html", 1<<30)
_, err = InspectBytes(tarData, FormatTar, InspectOptions{
EntryFile: "index.html",
Limits: Limits{
MaxFiles: 1,
MaxFileBytes: 32,
MaxTotalBytes: 32,
},
})
require.ErrorContains(t, err, "file too large")
}
func TestArchiveLimitsUseFilesAndActualTotals(t *testing.T) {
t.Parallel()
data := testZip(t, map[string]string{
"index.html": "1234",
"app.js": "5678",
})
_, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{
MaxFiles: 1,
MaxFileBytes: 8,
MaxTotalBytes: 16,
},
})
require.ErrorContains(t, err, "file count exceeds")
_, err = InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{
MaxFiles: 2,
MaxFileBytes: 8,
MaxTotalBytes: 7,
},
})
require.ErrorContains(t, err, "extracted size exceeds limit")
sevenZipData := decodeFixture(t, "N3q8ryccAASgR6WICAAAAAAAAABmAAAAAAAAAN2R8/FiYXIKZm9vCgEEBgACCQQEAAcLAgABAQABAQAMBAQACAoB6bOiBKhlMn4AAAUCGQUAAAAAABERAGIAYQByAAAAZgBvAG8AAAAZAgAAFBIBAACFM3PyY9YBAFgCcvJj1gEVCgEAIICkgSCApIEAAA==")
_, err = InspectBytes(sevenZipData, FormatSevenZip, InspectOptions{
EntryFile: "foo",
Limits: Limits{
MaxFiles: 10,
MaxFileBytes: 3,
MaxTotalBytes: 32,
},
})
require.ErrorContains(t, err, "file too large")
}
func TestRejectUnsupportedTarMemberTypes(t *testing.T) {
t.Parallel()
cases := []struct {
name string
header tar.Header
wantErr string
}{
{name: "symlink", header: tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "index.html"}, wantErr: "unsupported symlink"},
{name: "hardlink", header: tar.Header{Name: "hard", Typeflag: tar.TypeLink, Linkname: "index.html"}, wantErr: "unsupported hardlink"},
{name: "fifo", header: tar.Header{Name: "pipe", Typeflag: tar.TypeFifo, Mode: 0o600}, wantErr: "unsupported special entry"},
{name: "character device", header: tar.Header{Name: "tty", Typeflag: tar.TypeChar, Mode: 0o600, Devmajor: 1, Devminor: 3}, wantErr: "unsupported special entry"},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
data := tarWithSpecialEntry(t, &tt.header)
_, err := InspectBytes(data, FormatTar, InspectOptions{
EntryFile: "index.html",
Limits: testLimits,
})
require.ErrorContains(t, err, tt.wantErr)
destDir := t.TempDir()
err = ExtractBytes(data, FormatTar, destDir, ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
})
require.ErrorContains(t, err, tt.wantErr)
_, statErr := os.Stat(filepath.Join(destDir, "index.html"))
assert.ErrorIs(t, statErr, os.ErrNotExist, "tar validation pass must reject before writing files")
})
}
}
func TestRejectUnsupportedZipMemberTypes(t *testing.T) {
t.Parallel()
cases := []struct {
name string
mode os.FileMode
wantErr string
}{
{name: "symlink", mode: os.ModeSymlink | 0o777, wantErr: "unsupported symlink"},
{name: "named pipe", mode: os.ModeNamedPipe | 0o600, wantErr: "unsupported special entry"},
{name: "device", mode: os.ModeDevice | 0o600, wantErr: "unsupported special entry"},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
data := zipWithSpecialEntry(t, tt.mode)
_, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: testLimits,
})
require.ErrorContains(t, err, tt.wantErr)
err = ExtractBytes(data, FormatZip, t.TempDir(), ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
})
require.ErrorContains(t, err, tt.wantErr)
})
}
}
func TestTarMetadataHeadersRemainTransparent(t *testing.T) {
t.Parallel()
for _, format := range []tar.Format{tar.FormatPAX, tar.FormatGNU} {
format := format
t.Run(format.String(), func(t *testing.T) {
data := tarWithLongMetadata(t, format)
manifest, err := InspectBytes(data, FormatTar, InspectOptions{
EntryFile: "index.html",
Limits: testLimits,
})
require.NoError(t, err)
assert.Equal(t, 2, manifest.FileCount)
assertManifestContains(t, manifest, "index.html")
destDir := t.TempDir()
require.NoError(t, ExtractBytes(data, FormatTar, destDir, ExtractOptions{
Limits: testLimits,
EnforceLimits: true,
}))
_, err = os.Stat(filepath.Join(destDir, "index.html"))
require.NoError(t, err)
})
}
}
type countingFillReader struct {
remaining int64
read int64
}
func (r *countingFillReader) Read(p []byte) (int, error) {
if r.remaining == 0 {
return 0, io.EOF
}
if int64(len(p)) > r.remaining {
p = p[:r.remaining]
}
for i := range p {
p[i] = 'x'
}
r.remaining -= int64(len(p))
r.read += int64(len(p))
return len(p), nil
}
func decodeFixture(t *testing.T, encoded string) []byte {
t.Helper()
data, err := base64.StdEncoding.DecodeString(encoded)
require.NoError(t, err)
return data
}
func assertManifestContains(t *testing.T, manifest *Manifest, path string) {
t.Helper()
for _, file := range manifest.Files {
if file.Path == path {
return
}
}
require.Failf(t, "manifest path missing", "path %q not found in %#v", path, manifest.Files)
}
func testTar(t *testing.T, files map[string]string) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
for name, content := range files {
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(content)),
}))
_, err := writer.Write([]byte(content))
require.NoError(t, err)
}
require.NoError(t, writer.Close())
return buffer.Bytes()
}
func tarWithDeclaredBodyOnly(t *testing.T, name string, size int64) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: name,
Mode: 0o644,
Size: size,
}))
// Deliberately omit the body and trailer. The limit must reject from the
// header before archive/tar attempts to stream the declared body.
return buffer.Bytes()
}
func tarWithSpecialEntry(t *testing.T, special *tar.Header) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: "index.html",
Mode: 0o644,
Size: 2,
}))
_, err := writer.Write([]byte("ok"))
require.NoError(t, err)
require.NoError(t, writer.WriteHeader(special))
require.NoError(t, writer.Close())
return buffer.Bytes()
}
func zipWithSpecialEntry(t *testing.T, mode os.FileMode) []byte {
t.Helper()
var buffer bytes.Buffer
writer := zip.NewWriter(&buffer)
index, err := writer.Create("index.html")
require.NoError(t, err)
_, err = index.Write([]byte("ok"))
require.NoError(t, err)
header := &zip.FileHeader{Name: "special"}
header.SetMode(mode)
special, err := writer.CreateHeader(header)
require.NoError(t, err)
if mode&os.ModeSymlink != 0 {
_, err = special.Write([]byte("index.html"))
require.NoError(t, err)
}
require.NoError(t, writer.Close())
return buffer.Bytes()
}
func tarWithLongMetadata(t *testing.T, format tar.Format) []byte {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
longName := strings.Repeat("long-segment-", 12) + "asset.js"
header := &tar.Header{
Name: longName,
Mode: 0o644,
Size: 1,
Format: format,
}
if format == tar.FormatPAX {
header.PAXRecords = map[string]string{"comment": "metadata is not a deployable member"}
}
require.NoError(t, writer.WriteHeader(header))
_, err := writer.Write([]byte("x"))
require.NoError(t, err)
require.NoError(t, writer.WriteHeader(&tar.Header{
Name: "index.html",
Mode: 0o644,
Size: 2,
Format: format,
}))
_, err = writer.Write([]byte("ok"))
require.NoError(t, err)
require.NoError(t, writer.Close())
return buffer.Bytes()
}
+6
View File
@@ -1,3 +1,6 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package protocol defines the communication protocol between OpenFlare server, agent, and relay components.
package protocol
@@ -218,4 +221,7 @@ type PagesProjectLatestHashResponse struct {
ProjectID uint `json:"project_id"`
DeploymentID uint `json:"deployment_id"`
Hash string `json:"hash"`
PackageSize int64 `json:"package_size"`
FileCount int `json:"file_count"`
TotalSize int64 `json:"total_size"`
}
+15
View File
@@ -1,3 +1,6 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package protocol
import (
@@ -47,6 +50,18 @@ func TestAgentProtocolJSONTags(t *testing.T) {
"AccessToken": "agent_token",
},
},
{
name: "PagesProjectLatestHashResponse",
value: PagesProjectLatestHashResponse{},
expected: map[string]string{
"ProjectID": "project_id",
"DeploymentID": "deployment_id",
"Hash": "hash",
"PackageSize": "package_size",
"FileCount": "file_count",
"TotalSize": "total_size",
},
},
}
for _, tc := range cases {