perf(pages): skip per-file hashes during package inspect

Inspect deployment archives via file handles and declared sizes instead of loading the whole package and hashing every member, while keeping whole-package checksums for Agent integrity checks.
2026-10-11 09:46:37 +08:00 · 2026-07-17 18:37:34 +08:00
parent 285f127d48
commit 3b9f4daa4e
7 changed files with 236 additions and 56 deletions
@@ -53,31 +53,55 @@ func (z sevenZipArchiveFile) Open() (io.ReadCloser, error) {
 	return z.file.Open()
 }

-func listEntries(data []byte, format Format) ([]Entry, error) {
+// listEntriesAt lists archive members from a random-access source.
+// When materializeBodies is true, tar-family streams buffer regular-file bodies so Entry.Open works.
+// When false (inspect path), tar bodies are discarded after reading headers; zip/7z only use central directory metadata.
+func listEntriesAt(ra io.ReaderAt, size int64, format Format, materializeBodies bool) ([]Entry, error) {
+	if size < 0 {
+		return nil, fmt.Errorf("invalid pages package size")
+	}
 	switch format {
 	case FormatZip:
-		return listZipEntries(data)
+		return listZipEntriesAt(ra, size)
 	case FormatTar:
-		return listTarEntries(bytes.NewReader(data))
+		return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTar, materializeBodies)
 	case FormatTarGz:
-		gzReader, err := gzip.NewReader(bytes.NewReader(data))
+		return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarGz, materializeBodies)
+	case FormatTarXz:
+		return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarXz, materializeBodies)
+	case FormatTarBz2:
+		return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarBz2, materializeBodies)
+	case FormatSevenZip:
+		return listSevenZipEntriesAt(ra, size)
+	default:
+		return nil, fmt.Errorf("unsupported pages package format: %s", format)
+	}
+}
+
+func listTarFamily(r io.Reader, format Format, materializeBodies bool) ([]Entry, error) {
+	switch format {
+	case FormatTar:
+		if materializeBodies {
+			return listTarEntries(r, true)
+		}
+		return listTarEntries(r, false)
+	case FormatTarGz:
+		gzReader, err := gzip.NewReader(r)
 		if err != nil {
 			return nil, fmt.Errorf("open gzip pages package: %w", err)
 		}
 		defer func() { _ = gzReader.Close() }()
-		return listTarEntries(gzReader)
+		return listTarEntries(gzReader, materializeBodies)
 	case FormatTarXz:
-		xzReader, err := xz.NewReader(bytes.NewReader(data))
+		xzReader, err := xz.NewReader(r)
 		if err != nil {
 			return nil, fmt.Errorf("open xz pages package: %w", err)
 		}
-		return listTarEntries(xzReader)
+		return listTarEntries(xzReader, materializeBodies)
 	case FormatTarBz2:
-		return listTarEntries(bzip2.NewReader(bytes.NewReader(data)))
-	case FormatSevenZip:
-		return listSevenZipEntries(data)
+		return listTarEntries(bzip2.NewReader(r), materializeBodies)
 	default:
-		return nil, fmt.Errorf("unsupported pages package format: %s", format)
+		return nil, fmt.Errorf("unsupported tar family format: %s", format)
 	}
 }

@@ -96,8 +120,8 @@ func entriesFromArchiveFiles(files []archiveFile) []Entry {
 	return entries
 }

-func listZipEntries(data []byte) ([]Entry, error) {
-	reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+func listZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
+	reader, err := zip.NewReader(ra, size)
 	if err != nil {
 		return nil, fmt.Errorf("open zip pages package: %w", err)
 	}
@@ -108,8 +132,8 @@ func listZipEntries(data []byte) ([]Entry, error) {
 	return entriesFromArchiveFiles(files), nil
 }

-func listSevenZipEntries(data []byte) ([]Entry, error) {
-	reader, err := sevenzip.NewReader(bytes.NewReader(data), int64(len(data)))
+func listSevenZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
+	reader, err := sevenzip.NewReader(ra, size)
 	if err != nil {
 		return nil, fmt.Errorf("open 7z pages package: %w", err)
 	}
@@ -120,9 +144,8 @@ func listSevenZipEntries(data []byte) ([]Entry, error) {
 	return entriesFromArchiveFiles(files), nil
 }

-func listTarEntries(r io.Reader) ([]Entry, error) {
+func listTarEntries(r io.Reader, materializeBodies bool) ([]Entry, error) {
 	tarReader := tar.NewReader(r)
-	// Tar is sequential: materialize regular file bodies so entries can be opened later.
 	type materialised struct {
 		header *tar.Header
 		body   []byte
@@ -136,7 +159,7 @@ func listTarEntries(r io.Reader) ([]Entry, error) {
 		if err != nil {
 			return nil, fmt.Errorf("read tar pages package: %w", err)
 		}
-		item, skip, err := materialiseTarHeader(tarReader, header)
+		item, skip, err := readTarHeader(tarReader, header, materializeBodies)
 		if err != nil {
 			return nil, err
 		}
@@ -148,12 +171,12 @@ func listTarEntries(r io.Reader) ([]Entry, error) {

 	entries := make([]Entry, 0, len(items))
 	for _, item := range items {
-		entries = append(entries, tarEntryFromMaterialised(item.header, item.body))
+		entries = append(entries, tarEntryFromHeader(item.header, item.body, materializeBodies))
 	}
 	return entries, nil
 }

-func materialiseTarHeader(tarReader *tar.Reader, header *tar.Header) (item struct {
+func readTarHeader(tarReader *tar.Reader, header *tar.Header, materializeBodies bool) (item struct {
 	header *tar.Header
 	body   []byte
 }, skip bool, err error) {
@@ -164,6 +187,15 @@ func materialiseTarHeader(tarReader *tar.Reader, header *tar.Header) (item struc
 			body   []byte
 		}{header: header}, false, nil
 	case tar.TypeReg, tar.TypeRegA: //nolint:staticcheck // TypeRegA still appears in older archives
+		if !materializeBodies {
+			if err := discardTarBody(tarReader, header); err != nil {
+				return item, false, err
+			}
+			return struct {
+				header *tar.Header
+				body   []byte
+			}{header: header}, false, nil
+		}
 		body, readErr := readTarBody(tarReader, header)
 		if readErr != nil {
 			return item, false, readErr
@@ -182,6 +214,20 @@ func materialiseTarHeader(tarReader *tar.Reader, header *tar.Header) (item struc
 	}
 }

+func discardTarBody(tarReader *tar.Reader, header *tar.Header) error {
+	if header.Size <= 0 {
+		_, err := io.Copy(io.Discard, tarReader)
+		if err != nil {
+			return fmt.Errorf("discard tar entry %s: %w", header.Name, err)
+		}
+		return nil
+	}
+	if _, err := io.CopyN(io.Discard, tarReader, header.Size); err != nil {
+		return fmt.Errorf("discard tar entry %s: %w", header.Name, err)
+	}
+	return nil
+}
+
 func readTarBody(tarReader *tar.Reader, header *tar.Header) ([]byte, error) {
 	if header.Size > 0 {
 		body := make([]byte, header.Size)
@@ -197,9 +243,9 @@ func readTarBody(tarReader *tar.Reader, header *tar.Header) ([]byte, error) {
 	return body, nil
 }

-func tarEntryFromMaterialised(header *tar.Header, body []byte) Entry {
+func tarEntryFromHeader(header *tar.Header, body []byte, materializeBodies bool) Entry {
 	size := header.Size
-	if int64(len(body)) > size {
+	if materializeBodies && int64(len(body)) > size {
 		size = int64(len(body))
 	}
 	entry := Entry{
@@ -207,14 +253,23 @@ func tarEntryFromMaterialised(header *tar.Header, body []byte) Entry {
 		IsDir:     header.Typeflag == tar.TypeDir,
 		IsSymlink: header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeLink,
 		Size:      uint64(size), //nolint:gosec // non-negative sizes
-		Open: func() (io.ReadCloser, error) {
-			return io.NopCloser(bytes.NewReader(body)), nil
-		},
 	}
 	if entry.IsDir || entry.IsSymlink {
 		entry.Open = func() (io.ReadCloser, error) {
 			return io.NopCloser(bytes.NewReader(nil)), nil
 		}
+		return entry
+	}
+	if materializeBodies {
+		bodyCopy := body
+		entry.Open = func() (io.ReadCloser, error) {
+			return io.NopCloser(bytes.NewReader(bodyCopy)), nil
+		}
+		return entry
+	}
+	// Inspect path: body not retained; Open is unavailable.
+	entry.Open = func() (io.ReadCloser, error) {
+		return nil, fmt.Errorf("tar entry body not materialized: %s", header.Name)
 	}
 	return entry
 }