perf(pages): skip per-file hashes during package inspect

Inspect deployment archives via file handles and declared sizes instead of loading the whole package and hashing every member, while keeping whole-package checksums for Agent integrity checks.
This commit is contained in:
ryan
2026-07-17 18:37:34 +08:00
parent 285f127d48
commit 3b9f4daa4e
7 changed files with 236 additions and 56 deletions
+1
View File
@@ -35,6 +35,7 @@ sidebar: false
- 主配置版本与 Pages 部署双轨管理:Agent 以 Pages 项目 ID 为锚点请求最新激活部署包,项目内切换激活版本无需重新发布主配置即可在边缘自动更新。
- Agent 边缘仅保留每个 Pages 项目的最新部署包,新包就绪并切换成功后清理旧 release;拉取 latest 时增加 hash 二次校验与重试以降低激活竞态失败;单个项目同步失败不再阻塞同批其它项目。
- Pages 支持从 URL 导入部署包:填写下载链接后由控制面代为拉取并创建部署。
- 优化 Pages 部署包校验性能:上传时不再为包内每个文件计算哈希,文件清单仅记录路径与声明大小;完整性改由整包校验和保障,并避免将整个压缩包装入内存后再逐文件读取。
## [v3.3.0] - 2026-07-14
+6 -2
View File
@@ -66,7 +66,9 @@ graph TD
* **Pages 部署 (`pages_deployments`)**:
* 记录单次上传生成的不可变快照。包含:部署号 (DeploymentNumber, 递增序列)、SHA-256 Checksum 校验和、部署状态 (uploaded/active)、部署包的本地存储路径、解压后的文件数与总字节数。
* **部署文件清单 (`pages_deployment_files`)**:
* 存储每次部署的完整静态文件树路径、文件大小及单个文件哈希。用于审计和后续校验。
* 存储每次部署的完整静态文件树路径与文件大小(来自压缩包声明的未压缩大小),供控制台展示与统计。
* 不再为包内每个文件计算内容哈希;完整性由**整包** SHA-256(`pages_deployments.checksum`)保证,Agent 拉取时只校验整包 hash。
* 控制面 inspect 通过文件句柄 / 随机访问读取归档索引(zip/7z 中央目录;tar 流式读 header 并丢弃 body),避免将整包 `ReadFile` 进内存,也避免逐文件解压算 hash。
### 2. 路由关联与快照
`proxy_routes` 路由规则通过 `upstream_type = "pages"` 及 `pages_project_id` 关联 Pages 项目。当路由类型为 `pages` 且该项目存在已激活的部署时,才允许将该路由加入发布流程。
@@ -107,12 +109,14 @@ graph TD
### 1. 部署包安全校验与分析
为了避免不可信的用户上传恶意压缩包攻击服务器,控制面在 `UploadDeployment` 时执行严格校验:
* **格式支持**:`zip`、`tar.gz` / `tgz`、`tar.xz` / `txz`、`tar.bz2` / `tbz2`、`tar`、`7z`。
* **大小限制**:压缩包体积由系统配置 `pages_max_package_size_mb` 控制(默认 100 MiB,范围 1~2048);展开后总体积上限为「包大小 × 4」且不低于 100 MiB。
* **大小限制**:压缩包体积由系统配置 `pages_max_package_size_mb` 控制(默认 100 MiB,范围 1~2048);展开后总体积上限为「包大小 × 4」且不低于 100 MiB(按归档**声明**的未压缩大小累计,默认不流式重读每个文件内容)。
* **数量限制**:压缩包中包含的静态文件总数不得超过 1,000 个。
* **软链接阻断**:遍历归档文件,一旦检测到任何软链接,立即抛出错误并拒绝上传,防御软链接劫持攻击。
* **路径逃逸防御**:对每个压缩文件路径进行 `Clean` 并检查是否包含 `..` 或以 `/` 开头,防御目录跨越漏洞,防止写入系统敏感路径。
* **入口文件校验**:项目指定的入口文件(例如 `index.html`,可在 `project.RootDir` 下)必须在部署包中存在,否则拒绝上传。
* **公共根目录去噪**:许多打包工具会包含一个多余的主文件夹作为公共根前缀。控制面自动探测公共根前缀并将其安全剥离。
* **整包完整性**:上传/导入时对压缩包字节计算一次 SHA-256,写入部署记录;Agent 拉包后按整包 hash 对账。包内单文件不做内容哈希。
* **可选体积实测**:`InspectOptions.VerifySizes` 可对流式统计实际字节并与声明大小比对(仍不算 hash);默认关闭以降低上传 CPU/IO。
* **历史保留**:系统配置 `pages_max_history_count`(默认 20,0 表示不限制)在每次上传成功后执行裁剪。语义为:**每个项目最多保留 N 条部署**;当前激活部署始终保留;其余名额按部署 ID 从新到旧填充;超出的非激活部署连同文件清单与存储对象一并删除。上传已成功时裁剪失败只记日志、不回滚上传;并发上传下可能短暂超过 N,后续上传的裁剪会收敛回 N。主配置版本回滚不依赖旧 Pages 包(见上节双轨关系)。
### 2. 部署包存储规划
+6 -14
View File
@@ -4,8 +4,6 @@
package pagesarchive
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"math"
@@ -25,8 +23,10 @@ type Limits struct {
// FileEntry is a regular file discovered inside a deployment package.
type FileEntry struct {
Path string
Size int64
Path string
Size int64
// Checksum is retained for API/schema compatibility and is left empty.
// Integrity is enforced via the whole-package SHA-256 on the deployment record.
Checksum string
}
@@ -45,9 +45,10 @@ type Entry struct {
IsDir bool
// IsSymlink marks symbolic links (unsupported for Pages).
IsSymlink bool
// Size is the declared uncompressed size when known; 0 means unknown.
// Size is the declared uncompressed size when known; 0 means empty or unknown.
Size uint64
// Open returns a reader for the entry body. Caller must Close it.
// May be unavailable for inspect-only tar listings (body not materialized).
Open func() (io.ReadCloser, error)
}
@@ -79,15 +80,6 @@ func copyLimited(dst io.Writer, src io.Reader, declaredSize uint64, maxBytes int
return written, err
}
func checksumReader(src io.Reader, declaredSize uint64, maxBytes int64) (string, int64, error) {
hash := sha256.New()
written, err := copyLimited(hash, src, declaredSize, maxBytes)
if err != nil {
return "", written, err
}
return hex.EncodeToString(hash.Sum(nil)), written, nil
}
func writeEntryFile(targetPath string, src io.Reader, declaredSize uint64, maxBytes int64, perm os.FileMode) (int64, error) {
if err := os.MkdirAll(filepath.Dir(targetPath), dirPerm); err != nil {
return 0, err
+29 -4
View File
@@ -4,7 +4,9 @@
package pagesarchive
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
)
@@ -31,20 +33,43 @@ func ExtractBytes(data []byte, format Format, destDir string, opts ExtractOption
return err
}
}
entries, err := listEntries(data, format)
entries, err := listEntriesAt(bytes.NewReader(data), int64(len(data)), format, true)
if err != nil {
return err
}
return extractEntries(entries, destDir, opts)
}
// ExtractFile reads path and extracts it into destDir.
// ExtractFile opens path and extracts it into destDir without buffering the
// whole archive as an intermediate []byte for zip/7z (ReaderAt). Tar-family
// formats still materialize member bodies so random Open works for extract.
func ExtractFile(filePath string, format Format, destDir string, opts ExtractOptions) error {
data, err := os.ReadFile(filePath) //nolint:gosec // controlled path
file, err := os.Open(filePath) //nolint:gosec // controlled path
if err != nil {
return err
}
return ExtractBytes(data, format, destDir, opts)
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return err
}
if format == "" {
head := make([]byte, 512)
n, readErr := file.ReadAt(head, 0)
if readErr != nil && readErr != io.EOF {
return readErr
}
format, err = DetectFormat(filePath, head[:n])
if err != nil {
return err
}
}
entries, err := listEntriesAt(file, info.Size(), format, true)
if err != nil {
return err
}
return extractEntries(entries, destDir, opts)
}
func extractEntries(entries []Entry, destDir string, opts ExtractOptions) error {
+68 -11
View File
@@ -4,7 +4,10 @@
package pagesarchive
import (
"bytes"
"fmt"
"io"
"math"
"os"
"path"
"strings"
@@ -18,15 +21,38 @@ type InspectOptions struct {
EntryFile string
// Limits bounds files and sizes.
Limits Limits
// VerifySizes, when true, streams each regular file and compares the actual
// byte count against the archive-declared size (no content hashing).
// Default false: trust zip central directory / tar header sizes.
VerifySizes bool
}
// InspectFile opens path and inspects it as a Pages deployment package.
// InspectFile opens path and inspects it as a Pages deployment package without
// loading the whole archive into memory. File inventory uses declared sizes;
// per-file content hashes are not computed.
func InspectFile(filePath string, format Format, opts InspectOptions) (*Manifest, error) {
data, err := os.ReadFile(filePath) //nolint:gosec // filePath is a controlled temp upload path
file, err := os.Open(filePath) //nolint:gosec // filePath is a controlled temp upload path
if err != nil {
return nil, err
}
return InspectBytes(data, format, opts)
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil {
return nil, err
}
if format == "" {
head := make([]byte, 512)
n, readErr := file.ReadAt(head, 0)
if readErr != nil && readErr != io.EOF {
return nil, readErr
}
format, err = DetectFormat(filePath, head[:n])
if err != nil {
return nil, err
}
}
return inspectFromReaderAt(file, info.Size(), format, opts)
}
// InspectBytes inspects an in-memory deployment package.
@@ -38,7 +64,13 @@ func InspectBytes(data []byte, format Format, opts InspectOptions) (*Manifest, e
return nil, err
}
}
entries, err := listEntries(data, format)
return inspectFromReaderAt(bytes.NewReader(data), int64(len(data)), format, opts)
}
func inspectFromReaderAt(ra io.ReaderAt, size int64, format Format, opts InspectOptions) (*Manifest, error) {
// Default: zip/7z use central directory only; tar streams headers and discards bodies.
// VerifySizes needs openable tar bodies, so materialize only when requested.
entries, err := listEntriesAt(ra, size, format, opts.VerifySizes)
if err != nil {
return nil, err
}
@@ -65,7 +97,7 @@ func buildManifest(entries []Entry, opts InspectOptions) (*Manifest, error) {
return nil, fmt.Errorf("pages file too large: %s", normalizedPath)
}
fileEntry, err := inspectRegularFile(entry, normalizedPath, limits)
fileEntry, err := inspectRegularFile(entry, normalizedPath, limits, opts.VerifySizes)
if err != nil {
return nil, err
}
@@ -130,19 +162,44 @@ func prepareEntryPath(entry Entry, commonPrefix string) (string, bool, error) {
return normalizedPath, false, nil
}
func inspectRegularFile(entry Entry, normalizedPath string, limits Limits) (FileEntry, error) {
func inspectRegularFile(entry Entry, normalizedPath string, limits Limits, verifySizes bool) (FileEntry, error) {
if entry.Size > uint64(math.MaxInt64) {
return FileEntry{}, fmt.Errorf("%s: pages file size out of bounds", normalizedPath)
}
//nolint:gosec // bounded to MaxInt64 above
declaredSize := int64(entry.Size)
if !verifySizes {
return FileEntry{
Path: normalizedPath,
Size: declaredSize,
Checksum: "",
}, nil
}
if entry.Open == nil {
return FileEntry{}, fmt.Errorf("%s: cannot verify size without entry open", normalizedPath)
}
src, err := entry.Open()
if err != nil {
return FileEntry{}, fmt.Errorf("%s: %w", normalizedPath, err)
}
checksum, fileSize, checksumErr := checksumReader(src, entry.Size, limits.MaxFileBytes)
actual, measureErr := measureReader(src, entry.Size, limits.MaxFileBytes)
_ = src.Close()
if checksumErr != nil {
return FileEntry{}, fmt.Errorf("%s: %w", normalizedPath, checksumErr)
if measureErr != nil {
return FileEntry{}, fmt.Errorf("%s: %w", normalizedPath, measureErr)
}
if declaredSize > 0 && actual != declaredSize {
return FileEntry{}, fmt.Errorf("%s: declared size %d does not match actual %d", normalizedPath, declaredSize, actual)
}
return FileEntry{
Path: normalizedPath,
Size: fileSize,
Checksum: checksum,
Size: actual,
Checksum: "",
}, nil
}
// measureReader counts bytes without hashing, enforcing maxBytes when positive.
func measureReader(src io.Reader, declaredSize uint64, maxBytes int64) (int64, error) {
return copyLimited(io.Discard, src, declaredSize, maxBytes)
}
+80 -25
View File
@@ -53,31 +53,55 @@ func (z sevenZipArchiveFile) Open() (io.ReadCloser, error) {
return z.file.Open()
}
func listEntries(data []byte, format Format) ([]Entry, error) {
// listEntriesAt lists archive members from a random-access source.
// When materializeBodies is true, tar-family streams buffer regular-file bodies so Entry.Open works.
// When false (inspect path), tar bodies are discarded after reading headers; zip/7z only use central directory metadata.
func listEntriesAt(ra io.ReaderAt, size int64, format Format, materializeBodies bool) ([]Entry, error) {
if size < 0 {
return nil, fmt.Errorf("invalid pages package size")
}
switch format {
case FormatZip:
return listZipEntries(data)
return listZipEntriesAt(ra, size)
case FormatTar:
return listTarEntries(bytes.NewReader(data))
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTar, materializeBodies)
case FormatTarGz:
gzReader, err := gzip.NewReader(bytes.NewReader(data))
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarGz, materializeBodies)
case FormatTarXz:
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarXz, materializeBodies)
case FormatTarBz2:
return listTarFamily(io.NewSectionReader(ra, 0, size), FormatTarBz2, materializeBodies)
case FormatSevenZip:
return listSevenZipEntriesAt(ra, size)
default:
return nil, fmt.Errorf("unsupported pages package format: %s", format)
}
}
func listTarFamily(r io.Reader, format Format, materializeBodies bool) ([]Entry, error) {
switch format {
case FormatTar:
if materializeBodies {
return listTarEntries(r, true)
}
return listTarEntries(r, false)
case FormatTarGz:
gzReader, err := gzip.NewReader(r)
if err != nil {
return nil, fmt.Errorf("open gzip pages package: %w", err)
}
defer func() { _ = gzReader.Close() }()
return listTarEntries(gzReader)
return listTarEntries(gzReader, materializeBodies)
case FormatTarXz:
xzReader, err := xz.NewReader(bytes.NewReader(data))
xzReader, err := xz.NewReader(r)
if err != nil {
return nil, fmt.Errorf("open xz pages package: %w", err)
}
return listTarEntries(xzReader)
return listTarEntries(xzReader, materializeBodies)
case FormatTarBz2:
return listTarEntries(bzip2.NewReader(bytes.NewReader(data)))
case FormatSevenZip:
return listSevenZipEntries(data)
return listTarEntries(bzip2.NewReader(r), materializeBodies)
default:
return nil, fmt.Errorf("unsupported pages package format: %s", format)
return nil, fmt.Errorf("unsupported tar family format: %s", format)
}
}
@@ -96,8 +120,8 @@ func entriesFromArchiveFiles(files []archiveFile) []Entry {
return entries
}
func listZipEntries(data []byte) ([]Entry, error) {
reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
func listZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
reader, err := zip.NewReader(ra, size)
if err != nil {
return nil, fmt.Errorf("open zip pages package: %w", err)
}
@@ -108,8 +132,8 @@ func listZipEntries(data []byte) ([]Entry, error) {
return entriesFromArchiveFiles(files), nil
}
func listSevenZipEntries(data []byte) ([]Entry, error) {
reader, err := sevenzip.NewReader(bytes.NewReader(data), int64(len(data)))
func listSevenZipEntriesAt(ra io.ReaderAt, size int64) ([]Entry, error) {
reader, err := sevenzip.NewReader(ra, size)
if err != nil {
return nil, fmt.Errorf("open 7z pages package: %w", err)
}
@@ -120,9 +144,8 @@ func listSevenZipEntries(data []byte) ([]Entry, error) {
return entriesFromArchiveFiles(files), nil
}
func listTarEntries(r io.Reader) ([]Entry, error) {
func listTarEntries(r io.Reader, materializeBodies bool) ([]Entry, error) {
tarReader := tar.NewReader(r)
// Tar is sequential: materialize regular file bodies so entries can be opened later.
type materialised struct {
header *tar.Header
body []byte
@@ -136,7 +159,7 @@ func listTarEntries(r io.Reader) ([]Entry, error) {
if err != nil {
return nil, fmt.Errorf("read tar pages package: %w", err)
}
item, skip, err := materialiseTarHeader(tarReader, header)
item, skip, err := readTarHeader(tarReader, header, materializeBodies)
if err != nil {
return nil, err
}
@@ -148,12 +171,12 @@ func listTarEntries(r io.Reader) ([]Entry, error) {
entries := make([]Entry, 0, len(items))
for _, item := range items {
entries = append(entries, tarEntryFromMaterialised(item.header, item.body))
entries = append(entries, tarEntryFromHeader(item.header, item.body, materializeBodies))
}
return entries, nil
}
func materialiseTarHeader(tarReader *tar.Reader, header *tar.Header) (item struct {
func readTarHeader(tarReader *tar.Reader, header *tar.Header, materializeBodies bool) (item struct {
header *tar.Header
body []byte
}, skip bool, err error) {
@@ -164,6 +187,15 @@ func materialiseTarHeader(tarReader *tar.Reader, header *tar.Header) (item struc
body []byte
}{header: header}, false, nil
case tar.TypeReg, tar.TypeRegA: //nolint:staticcheck // TypeRegA still appears in older archives
if !materializeBodies {
if err := discardTarBody(tarReader, header); err != nil {
return item, false, err
}
return struct {
header *tar.Header
body []byte
}{header: header}, false, nil
}
body, readErr := readTarBody(tarReader, header)
if readErr != nil {
return item, false, readErr
@@ -182,6 +214,20 @@ func materialiseTarHeader(tarReader *tar.Reader, header *tar.Header) (item struc
}
}
func discardTarBody(tarReader *tar.Reader, header *tar.Header) error {
if header.Size <= 0 {
_, err := io.Copy(io.Discard, tarReader)
if err != nil {
return fmt.Errorf("discard tar entry %s: %w", header.Name, err)
}
return nil
}
if _, err := io.CopyN(io.Discard, tarReader, header.Size); err != nil {
return fmt.Errorf("discard tar entry %s: %w", header.Name, err)
}
return nil
}
func readTarBody(tarReader *tar.Reader, header *tar.Header) ([]byte, error) {
if header.Size > 0 {
body := make([]byte, header.Size)
@@ -197,9 +243,9 @@ func readTarBody(tarReader *tar.Reader, header *tar.Header) ([]byte, error) {
return body, nil
}
func tarEntryFromMaterialised(header *tar.Header, body []byte) Entry {
func tarEntryFromHeader(header *tar.Header, body []byte, materializeBodies bool) Entry {
size := header.Size
if int64(len(body)) > size {
if materializeBodies && int64(len(body)) > size {
size = int64(len(body))
}
entry := Entry{
@@ -207,14 +253,23 @@ func tarEntryFromMaterialised(header *tar.Header, body []byte) Entry {
IsDir: header.Typeflag == tar.TypeDir,
IsSymlink: header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeLink,
Size: uint64(size), //nolint:gosec // non-negative sizes
Open: func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(body)), nil
},
}
if entry.IsDir || entry.IsSymlink {
entry.Open = func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(nil)), nil
}
return entry
}
if materializeBodies {
bodyCopy := body
entry.Open = func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(bodyCopy)), nil
}
return entry
}
// Inspect path: body not retained; Open is unavailable.
entry.Open = func() (io.ReadCloser, error) {
return nil, fmt.Errorf("tar entry body not materialized: %s", header.Name)
}
return entry
}
+46
View File
@@ -52,9 +52,12 @@ func TestInspectAndExtractZip(t *testing.T) {
paths := make(map[string]struct{}, len(manifest.Files))
for _, file := range manifest.Files {
paths[file.Path] = struct{}{}
assert.Empty(t, file.Checksum, "per-file checksum should not be computed")
assert.Positive(t, file.Size)
}
assert.Contains(t, paths, "index.html")
assert.Contains(t, paths, "app.js")
assert.Equal(t, int64(len("<html>ok</html>")+len("console.log(1)")), manifest.TotalSize)
dest := t.TempDir()
require.NoError(t, ExtractBytes(data, FormatZip, dest, ExtractOptions{
@@ -67,6 +70,49 @@ func TestInspectAndExtractZip(t *testing.T) {
assert.Equal(t, "<html>ok</html>", string(body))
}
func TestInspectFileUsesDeclaredSizesWithoutHash(t *testing.T) {
data := testZip(t, map[string]string{
"index.html": "<html>disk</html>",
"asset.css": "body{}",
})
path := filepath.Join(t.TempDir(), "site.zip")
require.NoError(t, os.WriteFile(path, data, 0o600))
manifest, err := InspectFile(path, FormatZip, InspectOptions{
EntryFile: "index.html",
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.NoError(t, err)
require.Equal(t, 2, manifest.FileCount)
for _, file := range manifest.Files {
assert.Empty(t, file.Checksum)
}
dest := t.TempDir()
require.NoError(t, ExtractFile(path, FormatZip, dest, ExtractOptions{
EnforceLimits: true,
Limits: Limits{MaxFiles: 100, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
}))
body, err := os.ReadFile(filepath.Join(dest, "index.html")) //nolint:gosec
require.NoError(t, err)
assert.Equal(t, "<html>disk</html>", string(body))
}
func TestInspectVerifySizesOptional(t *testing.T) {
data := testZip(t, map[string]string{
"index.html": "verify-me",
})
manifest, err := InspectBytes(data, FormatZip, InspectOptions{
EntryFile: "index.html",
VerifySizes: true,
Limits: Limits{MaxFiles: 10, MaxFileBytes: 1 << 20, MaxTotalBytes: 1 << 20},
})
require.NoError(t, err)
require.Len(t, manifest.Files, 1)
assert.Equal(t, int64(len("verify-me")), manifest.Files[0].Size)
assert.Empty(t, manifest.Files[0].Checksum)
}
func TestExtractTrustedSkipsSizeLimits(t *testing.T) {
// Content larger than a tiny limit would fail if limits were enforced.
large := strings.Repeat("x", 64)