From 38b0516937db89b7c7037fb135887e4415017177 Mon Sep 17 00:00:00 2001 From: deqiying Date: Sun, 19 Jul 2026 17:36:51 +0800 Subject: [PATCH] =?UTF-8?q?feat(pages):=20=E6=94=AF=E6=8C=81=20Remote=20?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E6=BA=90=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增部署源配置与运行态模型、安全下载、租约续期、原子激活和失败补偿。 接入内部任务与脱敏前端交互，并阻止数据库 Trace 和日志展开敏感查询参数。 --- docs/docs.go | 541 ++++++++++++- docs/swagger.json | 541 ++++++++++++- docs/swagger.yaml | 336 +++++++- .../components/deployment-upload-dialog.tsx | 268 +++---- .../(main)/pages/components/pages-utils.ts | 4 + .../components/deployment-files-panel.tsx | 86 +++ .../detail/components/deployment-history.tsx | 300 ++++++++ .../detail/components/pages-source-card.tsx | 393 ++++++++++ .../detail/components/pages-source-dialog.tsx | 376 +++++++++ .../app/(main)/pages/detail/page-client.tsx | 505 ------------- frontend/app/(main)/pages/detail/page.tsx | 307 +++++++- frontend/lib/services/index.ts | 3 + frontend/lib/services/openflare/index.ts | 11 + .../lib/services/openflare/pages.service.ts | 43 +- frontend/lib/services/openflare/types.ts | 81 +- .../tests/openflare/pages-service.test.ts | 105 ++- .../tests/openflare/pages-source-ui.test.tsx | 227 ++++++ internal/apps/admin/task/routers.go | 18 +- internal/apps/admin/task/routers_test.go | 133 ++++ internal/apps/openflare/pages/download_url.go | 266 +------ .../apps/openflare/pages/download_url_test.go | 17 +- internal/apps/openflare/pages/errs.go | 16 + internal/apps/openflare/pages/helpers.go | 25 +- internal/apps/openflare/pages/logics.go | 347 +++++++-- internal/apps/openflare/pages/logics_test.go | 2 + internal/apps/openflare/pages/routers.go | 219 +++++- .../openflare/pages/routers_source_test.go | 224 ++++++ internal/apps/openflare/pages/source.go | 495 ++++++++++++ .../apps/openflare/pages/source_provider.go | 549 ++++++++++++++ .../openflare/pages/source_provider_test.go | 459 +++++++++++ .../apps/openflare/pages/source_runtime.go | 309 ++++++++ .../openflare/pages/source_runtime_test.go | 350 +++++++++ internal/apps/openflare/pages/source_sync.go | 715 ++++++++++++++++++ .../apps/openflare/pages/source_sync_test.go | 604 +++++++++++++++ internal/apps/openflare/pages/source_tasks.go | 275 +++++++ .../apps/openflare/pages/source_tasks_test.go | 166 ++++ internal/apps/openflare/pages/source_test.go | 386 ++++++++++ .../202607190001_add_pages_source_runtime.sql | 82 ++ .../202607190001_add_pages_source_runtime.sql | 151 ++++ internal/db/migrator/migrator_test.go | 6 +- .../migrator/pages_source_migration_test.go | 322 ++++++++ internal/db/postgres.go | 37 +- internal/db/postgres_logger.go | 5 + internal/db/postgres_logger_test.go | 76 ++ internal/db/postgres_tracing_test.go | 69 ++ internal/model/openflare_pages.go | 43 +- internal/model/openflare_pages_source.go | 59 ++ internal/model/openflare_pages_source_test.go | 80 ++ .../router/v1/openflare/register_pages.go | 5 + internal/task/executor.go | 6 +- internal/task/executor_test.go | 51 ++ internal/task/handlers/register.go | 5 + internal/task/meta.go | 12 +- internal/task/meta_test.go | 25 + internal/task/permanent_error.go | 35 + internal/task/permanent_error_test.go | 27 + pkg/httppool/httppool.go | 72 +- pkg/httppool/httppool_test.go | 63 ++ 58 files changed, 9858 insertions(+), 1075 deletions(-) create mode 100644 frontend/app/(main)/pages/detail/components/deployment-files-panel.tsx create mode 100644 frontend/app/(main)/pages/detail/components/deployment-history.tsx create mode 100644 frontend/app/(main)/pages/detail/components/pages-source-card.tsx create mode 100644 frontend/app/(main)/pages/detail/components/pages-source-dialog.tsx delete mode 100644 frontend/app/(main)/pages/detail/page-client.tsx create mode 100644 frontend/tests/openflare/pages-source-ui.test.tsx create mode 100644 internal/apps/openflare/pages/routers_source_test.go create mode 100644 internal/apps/openflare/pages/source.go create mode 100644 internal/apps/openflare/pages/source_provider.go create mode 100644 internal/apps/openflare/pages/source_provider_test.go create mode 100644 internal/apps/openflare/pages/source_runtime.go create mode 100644 internal/apps/openflare/pages/source_runtime_test.go create mode 100644 internal/apps/openflare/pages/source_sync.go create mode 100644 internal/apps/openflare/pages/source_sync_test.go create mode 100644 internal/apps/openflare/pages/source_tasks.go create mode 100644 internal/apps/openflare/pages/source_tasks_test.go create mode 100644 internal/apps/openflare/pages/source_test.go create mode 100644 internal/db/migrator/goose/postgres/202607190001_add_pages_source_runtime.sql create mode 100644 internal/db/migrator/goose/sqlite/202607190001_add_pages_source_runtime.sql create mode 100644 internal/db/migrator/pages_source_migration_test.go create mode 100644 internal/db/postgres_tracing_test.go create mode 100644 internal/model/openflare_pages_source.go create mode 100644 internal/model/openflare_pages_source_test.go create mode 100644 internal/task/permanent_error.go create mode 100644 internal/task/permanent_error_test.go diff --git a/docs/docs.go b/docs/docs.go index df744b22..6947b49a 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -8584,7 +8584,7 @@ const docTemplate = `{ "SessionCookie": [] } ], - "description": "从用户提供的 HTTP(S) 链接下载部署包并创建部署记录；服务端使用浏览器伪装请求头拉取，允许内网地址与不安全 TLS 证书，需要管理员权限", + "description": "已弃用的一次性 URL 导入；使用 trusted_internal 策略兼容内网与自签名证书，不创建持久部署源", "consumes": [ "application/json" ], @@ -8595,6 +8595,7 @@ const docTemplate = `{ "openflare-pages" ], "summary": "从 URL 导入 Pages 部署包", + "deprecated": true, "parameters": [ { "type": "integer", @@ -8801,6 +8802,391 @@ const docTemplate = `{ } } }, + "/api/v1/d/pages/{id}/source": { + "get": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "返回脱敏后的项目部署源配置与运行状态，需要管理员权限", + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "获取 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "部署源", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceView" + } + } + } + ] + } + }, + "400": { + "description": "参数错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "项目或部署源不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/check": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "Remote URL 来源不支持检查更新；该端点为后续 GitHub Release 来源保留统一动作契约", + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "检查 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "任务回执", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceActionReceipt" + } + } + } + ] + } + }, + "400": { + "description": "当前来源不支持检查", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "部署源不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "409": { + "description": "来源任务正在执行", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/delete": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "幂等删除持久部署源；已有部署历史与当前生产部署保持不变", + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "删除 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "手动来源视图", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceView" + } + } + } + ] + } + }, + "400": { + "description": "参数错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "项目不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/sync": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "异步下载、校验并原子激活来源部署包；空请求体与空 JSON 对象均有效", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "同步并发布 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "同步参数", + "name": "request", + "in": "body", + "schema": { + "$ref": "#/definitions/pages.SourceSyncInput" + } + } + ], + "responses": { + "200": { + "description": "任务回执", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceActionReceipt" + } + } + } + ] + } + }, + "400": { + "description": "参数或来源类型无效", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "部署源不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "409": { + "description": "来源任务正在执行", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/update": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "Phase 1 支持 Remote URL 来源；完整地址仅写入，不会在响应中返回", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "更新 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "部署源配置", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/pages.SourceUpdateInput" + } + } + ], + "responses": { + "200": { + "description": "更新结果", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceUpdateResult" + } + } + } + ] + } + }, + "400": { + "description": "配置无效", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "项目不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, "/api/v1/d/pages/{id}/update": { "post": { "security": [ @@ -17329,12 +17715,21 @@ const docTemplate = `{ "project_id": { "type": "integer" }, + "source_label": { + "type": "string" + }, + "source_type": { + "type": "string" + }, "status": { "type": "string" }, "total_size": { "type": "integer" }, + "trigger_type": { + "type": "string" + }, "upload_id": { "type": "string", "example": "0" @@ -17382,6 +17777,150 @@ const docTemplate = `{ } } }, + "pages.SourceActionReceipt": { + "type": "object", + "properties": { + "action": { + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "task_id": { + "type": "string" + } + } + }, + "pages.SourceRevisionView": { + "type": "object", + "properties": { + "asset_name": { + "type": "string" + }, + "label": { + "type": "string" + }, + "revision": { + "type": "string" + } + } + }, + "pages.SourceSyncInput": { + "type": "object", + "properties": { + "confirmed_revision": { + "type": "string" + } + } + }, + "pages.SourceUpdateInput": { + "type": "object", + "properties": { + "asset_name": { + "type": "string" + }, + "auto_update_enabled": { + "type": "boolean" + }, + "check_interval_minutes": { + "type": "integer" + }, + "release_selector": { + "type": "string" + }, + "release_tag": { + "type": "string" + }, + "remote_network_policy": { + "type": "string" + }, + "remote_url": { + "type": "string" + }, + "remote_url_set": { + "type": "boolean" + }, + "repository_url": { + "type": "string" + }, + "source_type": { + "type": "string" + } + } + }, + "pages.SourceUpdateResult": { + "type": "object", + "properties": { + "check_task": { + "$ref": "#/definitions/pages.SourceActionReceipt" + }, + "source": { + "$ref": "#/definitions/pages.SourceView" + }, + "warning": { + "type": "string" + } + } + }, + "pages.SourceView": { + "type": "object", + "properties": { + "asset_name": { + "type": "string" + }, + "auto_update_enabled": { + "type": "boolean" + }, + "check_interval_minutes": { + "type": "integer" + }, + "display_url": { + "type": "string" + }, + "github_repository": { + "type": "string" + }, + "has_remote_url": { + "type": "boolean" + }, + "last_applied": { + "$ref": "#/definitions/pages.SourceRevisionView" + }, + "last_checked_at": { + "type": "string" + }, + "last_error": { + "type": "string" + }, + "last_seen": { + "$ref": "#/definitions/pages.SourceRevisionView" + }, + "last_synced_at": { + "type": "string" + }, + "next_check_at": { + "type": "string" + }, + "release_selector": { + "type": "string" + }, + "release_tag": { + "type": "string" + }, + "remote_network_policy": { + "type": "string" + }, + "source_type": { + "type": "string" + }, + "sync_status": { + "type": "string" + }, + "update_available": { + "type": "boolean" + } + } + }, "pages.UploadFromURLInput": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index efdab2a3..5c81cc5e 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -8577,7 +8577,7 @@ "SessionCookie": [] } ], - "description": "从用户提供的 HTTP(S) 链接下载部署包并创建部署记录；服务端使用浏览器伪装请求头拉取，允许内网地址与不安全 TLS 证书，需要管理员权限", + "description": "已弃用的一次性 URL 导入；使用 trusted_internal 策略兼容内网与自签名证书，不创建持久部署源", "consumes": [ "application/json" ], @@ -8588,6 +8588,7 @@ "openflare-pages" ], "summary": "从 URL 导入 Pages 部署包", + "deprecated": true, "parameters": [ { "type": "integer", @@ -8794,6 +8795,391 @@ } } }, + "/api/v1/d/pages/{id}/source": { + "get": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "返回脱敏后的项目部署源配置与运行状态，需要管理员权限", + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "获取 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "部署源", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceView" + } + } + } + ] + } + }, + "400": { + "description": "参数错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "项目或部署源不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/check": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "Remote URL 来源不支持检查更新；该端点为后续 GitHub Release 来源保留统一动作契约", + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "检查 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "任务回执", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceActionReceipt" + } + } + } + ] + } + }, + "400": { + "description": "当前来源不支持检查", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "部署源不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "409": { + "description": "来源任务正在执行", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/delete": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "幂等删除持久部署源；已有部署历史与当前生产部署保持不变", + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "删除 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "手动来源视图", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceView" + } + } + } + ] + } + }, + "400": { + "description": "参数错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "项目不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/sync": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "异步下载、校验并原子激活来源部署包；空请求体与空 JSON 对象均有效", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "同步并发布 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "同步参数", + "name": "request", + "in": "body", + "schema": { + "$ref": "#/definitions/pages.SourceSyncInput" + } + } + ], + "responses": { + "200": { + "description": "任务回执", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceActionReceipt" + } + } + } + ] + } + }, + "400": { + "description": "参数或来源类型无效", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "部署源不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "409": { + "description": "来源任务正在执行", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, + "/api/v1/d/pages/{id}/source/update": { + "post": { + "security": [ + { + "SessionCookie": [] + } + ], + "description": "Phase 1 支持 Remote URL 来源；完整地址仅写入，不会在响应中返回", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "openflare-pages" + ], + "summary": "更新 Pages 部署源", + "parameters": [ + { + "type": "integer", + "description": "项目 ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "部署源配置", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/pages.SourceUpdateInput" + } + } + ], + "responses": { + "200": { + "description": "更新结果", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/response.Any" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/pages.SourceUpdateResult" + } + } + } + ] + } + }, + "400": { + "description": "配置无效", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "401": { + "description": "未登录", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "404": { + "description": "项目不存在", + "schema": { + "$ref": "#/definitions/response.Any" + } + }, + "500": { + "description": "内部错误", + "schema": { + "$ref": "#/definitions/response.Any" + } + } + } + } + }, "/api/v1/d/pages/{id}/update": { "post": { "security": [ @@ -17322,12 +17708,21 @@ "project_id": { "type": "integer" }, + "source_label": { + "type": "string" + }, + "source_type": { + "type": "string" + }, "status": { "type": "string" }, "total_size": { "type": "integer" }, + "trigger_type": { + "type": "string" + }, "upload_id": { "type": "string", "example": "0" @@ -17375,6 +17770,150 @@ } } }, + "pages.SourceActionReceipt": { + "type": "object", + "properties": { + "action": { + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "task_id": { + "type": "string" + } + } + }, + "pages.SourceRevisionView": { + "type": "object", + "properties": { + "asset_name": { + "type": "string" + }, + "label": { + "type": "string" + }, + "revision": { + "type": "string" + } + } + }, + "pages.SourceSyncInput": { + "type": "object", + "properties": { + "confirmed_revision": { + "type": "string" + } + } + }, + "pages.SourceUpdateInput": { + "type": "object", + "properties": { + "asset_name": { + "type": "string" + }, + "auto_update_enabled": { + "type": "boolean" + }, + "check_interval_minutes": { + "type": "integer" + }, + "release_selector": { + "type": "string" + }, + "release_tag": { + "type": "string" + }, + "remote_network_policy": { + "type": "string" + }, + "remote_url": { + "type": "string" + }, + "remote_url_set": { + "type": "boolean" + }, + "repository_url": { + "type": "string" + }, + "source_type": { + "type": "string" + } + } + }, + "pages.SourceUpdateResult": { + "type": "object", + "properties": { + "check_task": { + "$ref": "#/definitions/pages.SourceActionReceipt" + }, + "source": { + "$ref": "#/definitions/pages.SourceView" + }, + "warning": { + "type": "string" + } + } + }, + "pages.SourceView": { + "type": "object", + "properties": { + "asset_name": { + "type": "string" + }, + "auto_update_enabled": { + "type": "boolean" + }, + "check_interval_minutes": { + "type": "integer" + }, + "display_url": { + "type": "string" + }, + "github_repository": { + "type": "string" + }, + "has_remote_url": { + "type": "boolean" + }, + "last_applied": { + "$ref": "#/definitions/pages.SourceRevisionView" + }, + "last_checked_at": { + "type": "string" + }, + "last_error": { + "type": "string" + }, + "last_seen": { + "$ref": "#/definitions/pages.SourceRevisionView" + }, + "last_synced_at": { + "type": "string" + }, + "next_check_at": { + "type": "string" + }, + "release_selector": { + "type": "string" + }, + "release_tag": { + "type": "string" + }, + "remote_network_policy": { + "type": "string" + }, + "source_type": { + "type": "string" + }, + "sync_status": { + "type": "string" + }, + "update_available": { + "type": "boolean" + } + } + }, "pages.UploadFromURLInput": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index a52e0600..b0e074d9 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2571,10 +2571,16 @@ definitions: type: integer project_id: type: integer + source_label: + type: string + source_type: + type: string status: type: string total_size: type: integer + trigger_type: + type: string upload_id: example: "0" type: string @@ -2606,6 +2612,100 @@ definitions: spa_fallback_path: type: string type: object + pages.SourceActionReceipt: + properties: + action: + type: string + execution_id: + type: string + task_id: + type: string + type: object + pages.SourceRevisionView: + properties: + asset_name: + type: string + label: + type: string + revision: + type: string + type: object + pages.SourceSyncInput: + properties: + confirmed_revision: + type: string + type: object + pages.SourceUpdateInput: + properties: + asset_name: + type: string + auto_update_enabled: + type: boolean + check_interval_minutes: + type: integer + release_selector: + type: string + release_tag: + type: string + remote_network_policy: + type: string + remote_url: + type: string + remote_url_set: + type: boolean + repository_url: + type: string + source_type: + type: string + type: object + pages.SourceUpdateResult: + properties: + check_task: + $ref: '#/definitions/pages.SourceActionReceipt' + source: + $ref: '#/definitions/pages.SourceView' + warning: + type: string + type: object + pages.SourceView: + properties: + asset_name: + type: string + auto_update_enabled: + type: boolean + check_interval_minutes: + type: integer + display_url: + type: string + github_repository: + type: string + has_remote_url: + type: boolean + last_applied: + $ref: '#/definitions/pages.SourceRevisionView' + last_checked_at: + type: string + last_error: + type: string + last_seen: + $ref: '#/definitions/pages.SourceRevisionView' + last_synced_at: + type: string + next_check_at: + type: string + release_selector: + type: string + release_tag: + type: string + remote_network_policy: + type: string + source_type: + type: string + sync_status: + type: string + update_available: + type: boolean + type: object pages.UploadFromURLInput: properties: url: @@ -9277,7 +9377,8 @@ paths: post: consumes: - application/json - description: 从用户提供的 HTTP(S) 链接下载部署包并创建部署记录；服务端使用浏览器伪装请求头拉取，允许内网地址与不安全 TLS 证书，需要管理员权限 + deprecated: true + description: 已弃用的一次性 URL 导入；使用 trusted_internal 策略兼容内网与自签名证书，不创建持久部署源 parameters: - description: 项目 ID in: path @@ -9323,6 +9424,239 @@ paths: summary: 从 URL 导入 Pages 部署包 tags: - openflare-pages + /api/v1/d/pages/{id}/source: + get: + description: 返回脱敏后的项目部署源配置与运行状态，需要管理员权限 + parameters: + - description: 项目 ID + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: 部署源 + schema: + allOf: + - $ref: '#/definitions/response.Any' + - properties: + data: + $ref: '#/definitions/pages.SourceView' + type: object + "400": + description: 参数错误 + schema: + $ref: '#/definitions/response.Any' + "401": + description: 未登录 + schema: + $ref: '#/definitions/response.Any' + "404": + description: 项目或部署源不存在 + schema: + $ref: '#/definitions/response.Any' + "500": + description: 内部错误 + schema: + $ref: '#/definitions/response.Any' + security: + - SessionCookie: [] + summary: 获取 Pages 部署源 + tags: + - openflare-pages + /api/v1/d/pages/{id}/source/check: + post: + description: Remote URL 来源不支持检查更新；该端点为后续 GitHub Release 来源保留统一动作契约 + parameters: + - description: 项目 ID + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: 任务回执 + schema: + allOf: + - $ref: '#/definitions/response.Any' + - properties: + data: + $ref: '#/definitions/pages.SourceActionReceipt' + type: object + "400": + description: 当前来源不支持检查 + schema: + $ref: '#/definitions/response.Any' + "401": + description: 未登录 + schema: + $ref: '#/definitions/response.Any' + "404": + description: 部署源不存在 + schema: + $ref: '#/definitions/response.Any' + "409": + description: 来源任务正在执行 + schema: + $ref: '#/definitions/response.Any' + "500": + description: 内部错误 + schema: + $ref: '#/definitions/response.Any' + security: + - SessionCookie: [] + summary: 检查 Pages 部署源 + tags: + - openflare-pages + /api/v1/d/pages/{id}/source/delete: + post: + description: 幂等删除持久部署源；已有部署历史与当前生产部署保持不变 + parameters: + - description: 项目 ID + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: 手动来源视图 + schema: + allOf: + - $ref: '#/definitions/response.Any' + - properties: + data: + $ref: '#/definitions/pages.SourceView' + type: object + "400": + description: 参数错误 + schema: + $ref: '#/definitions/response.Any' + "401": + description: 未登录 + schema: + $ref: '#/definitions/response.Any' + "404": + description: 项目不存在 + schema: + $ref: '#/definitions/response.Any' + "500": + description: 内部错误 + schema: + $ref: '#/definitions/response.Any' + security: + - SessionCookie: [] + summary: 删除 Pages 部署源 + tags: + - openflare-pages + /api/v1/d/pages/{id}/source/sync: + post: + consumes: + - application/json + description: 异步下载、校验并原子激活来源部署包；空请求体与空 JSON 对象均有效 + parameters: + - description: 项目 ID + in: path + name: id + required: true + type: integer + - description: 同步参数 + in: body + name: request + schema: + $ref: '#/definitions/pages.SourceSyncInput' + produces: + - application/json + responses: + "200": + description: 任务回执 + schema: + allOf: + - $ref: '#/definitions/response.Any' + - properties: + data: + $ref: '#/definitions/pages.SourceActionReceipt' + type: object + "400": + description: 参数或来源类型无效 + schema: + $ref: '#/definitions/response.Any' + "401": + description: 未登录 + schema: + $ref: '#/definitions/response.Any' + "404": + description: 部署源不存在 + schema: + $ref: '#/definitions/response.Any' + "409": + description: 来源任务正在执行 + schema: + $ref: '#/definitions/response.Any' + "500": + description: 内部错误 + schema: + $ref: '#/definitions/response.Any' + security: + - SessionCookie: [] + summary: 同步并发布 Pages 部署源 + tags: + - openflare-pages + /api/v1/d/pages/{id}/source/update: + post: + consumes: + - application/json + description: Phase 1 支持 Remote URL 来源；完整地址仅写入，不会在响应中返回 + parameters: + - description: 项目 ID + in: path + name: id + required: true + type: integer + - description: 部署源配置 + in: body + name: request + required: true + schema: + $ref: '#/definitions/pages.SourceUpdateInput' + produces: + - application/json + responses: + "200": + description: 更新结果 + schema: + allOf: + - $ref: '#/definitions/response.Any' + - properties: + data: + $ref: '#/definitions/pages.SourceUpdateResult' + type: object + "400": + description: 配置无效 + schema: + $ref: '#/definitions/response.Any' + "401": + description: 未登录 + schema: + $ref: '#/definitions/response.Any' + "404": + description: 项目不存在 + schema: + $ref: '#/definitions/response.Any' + "500": + description: 内部错误 + schema: + $ref: '#/definitions/response.Any' + security: + - SessionCookie: [] + summary: 更新 Pages 部署源 + tags: + - openflare-pages /api/v1/d/pages/{id}/update: post: consumes: diff --git a/frontend/app/(main)/pages/components/deployment-upload-dialog.tsx b/frontend/app/(main)/pages/components/deployment-upload-dialog.tsx index d47b0e83..29c17a94 100644 --- a/frontend/app/(main)/pages/components/deployment-upload-dialog.tsx +++ b/frontend/app/(main)/pages/components/deployment-upload-dialog.tsx @@ -2,7 +2,7 @@ import { useRef, useState } from 'react'; import { useMutation, useQueryClient } from '@tanstack/react-query'; -import { Link2, Loader2, UploadCloud } from 'lucide-react'; +import { UploadCloud } from 'lucide-react'; import { toast } from 'sonner'; import { Button } from '@/components/ui/button'; @@ -14,10 +14,14 @@ import { DialogHeader, DialogTitle, } from '@/components/ui/dialog'; -import { Input } from '@/components/ui/input'; -import { Label } from '@/components/ui/label'; +import { + Field, + FieldDescription, + FieldGroup, + FieldLabel, +} from '@/components/ui/field'; import { Progress } from '@/components/ui/progress'; -import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import { Spinner } from '@/components/ui/spinner'; import { PagesService } from '@/lib/services/openflare'; import { cn } from '@/lib/utils'; @@ -46,52 +50,51 @@ const PAGES_PACKAGE_EXTENSIONS = [ function isSupportedPagesPackage(fileName: string) { const lower = fileName.toLowerCase(); - return PAGES_PACKAGE_EXTENSIONS.some((ext) => lower.endsWith(ext)); + return PAGES_PACKAGE_EXTENSIONS.some((extension) => + lower.endsWith(extension), + ); +} + +export function pagesEntryPath(rootDir: string, entryFile: string) { + const root = rootDir.trim().replace(/^\/+|\/+$/g, ''); + const entry = entryFile.trim().replace(/^\/+/, ''); + return root ? `${root}/${entry}` : entry; } interface DeploymentUploadDialogProps { open: boolean; onOpenChange: (open: boolean) => void; projectId: number; + rootDir: string; + entryFile: string; } export function DeploymentUploadDialog({ open, onOpenChange, projectId, + rootDir, + entryFile, }: DeploymentUploadDialogProps) { const queryClient = useQueryClient(); const fileInputRef = useRef(null); - const [mode, setMode] = useState<'file' | 'url'>('file'); const [file, setFile] = useState(null); - const [packageURL, setPackageURL] = useState(''); const [isDragActive, setIsDragActive] = useState(false); const [uploadProgress, setUploadProgress] = useState(null); const resetForm = () => { setFile(null); - setPackageURL(''); setIsDragActive(false); setUploadProgress(null); if (fileInputRef.current) fileInputRef.current.value = ''; }; - const handleClose = (nextOpen: boolean) => { + const handleOpenChange = (nextOpen: boolean) => { if (!nextOpen) resetForm(); onOpenChange(nextOpen); }; - const invalidate = async () => { - await Promise.all([ - queryClient.invalidateQueries({ - queryKey: deploymentsQueryKey(projectId), - }), - queryClient.invalidateQueries({ queryKey: projectQueryKey(projectId) }), - queryClient.invalidateQueries({ queryKey: projectsQueryKey }), - ]); - }; - - const uploadFileMutation = useMutation({ + const uploadMutation = useMutation({ mutationFn: () => { if (!file) throw new Error('请选择部署包'); return PagesService.uploadDeployment(projectId, { @@ -101,8 +104,14 @@ export function DeploymentUploadDialog({ }, onSuccess: async () => { toast.success('部署包上传成功'); - await invalidate(); - handleClose(false); + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: deploymentsQueryKey(projectId), + }), + queryClient.invalidateQueries({ queryKey: projectQueryKey(projectId) }), + queryClient.invalidateQueries({ queryKey: projectsQueryKey }), + ]); + handleOpenChange(false); }, onError: (error) => { toast.error(error instanceof Error ? error.message : '上传失败'); @@ -110,27 +119,6 @@ export function DeploymentUploadDialog({ }, }); - const uploadURLMutation = useMutation({ - mutationFn: () => { - const url = packageURL.trim(); - if (!url) throw new Error('请填写部署包下载链接'); - if (!/^https?:\/\//i.test(url)) { - throw new Error('链接必须以 http:// 或 https:// 开头'); - } - return PagesService.uploadDeploymentFromURL(projectId, { url }); - }, - onSuccess: async () => { - toast.success('已从链接下载并创建部署'); - await invalidate(); - handleClose(false); - }, - onError: (error) => { - toast.error(error instanceof Error ? error.message : '从链接导入失败'); - }, - }); - - const isPending = uploadFileMutation.isPending || uploadURLMutation.isPending; - const handleFileSelect = (selected: File | null) => { if (!selected) return; if (!isSupportedPagesPackage(selected.name)) { @@ -140,150 +128,112 @@ export function DeploymentUploadDialog({ setFile(selected); }; - const handleSubmit = () => { - if (mode === 'file') { - uploadFileMutation.mutate(); - return; - } - uploadURLMutation.mutate(); - }; - - const canSubmit = - mode === 'file' ? Boolean(file) : packageURL.trim().length > 0; - return ( - + 上传部署包 - 支持本地上传或从 URL 下载静态资源压缩包（zip / tar.gz / tar.xz / - tar.bz2 / tar / 7z），创建部署后可在列表中激活。 + 上传不可变的静态资源压缩包，完成后可在部署历史中激活。 - setMode(value as 'file' | 'url')} - className='w-full' - > - - 本地上传 - 从 URL 下载 - - - -
+ + 本地部署包 + - handleFileSelect(e.target.files?.[0] ?? null)} - /> -
- + + + 拖拽部署包到此处，或点击选择文件 + + + zip、tar.gz、tar.xz、tar.bz2、tar、7z + + + + handleFileSelect(event.target.files?.[0] ?? null) + } + /> {file ? ( -
-

{file.name}

-

- {formatBytes(file.size)} -

-
- ) : null} + + 已选择 {file.name}（{formatBytes(file.size)}） + + ) : ( + 请选择一个受支持的压缩包。 + )} + - {uploadProgress !== null ? ( -
-
- - {uploadProgress >= 100 ? '服务端处理中' : '上传进度'} - - - {uploadProgress >= 100 ? '请稍候' : `${uploadProgress}%`} - -
- = 100 ? 100 : uploadProgress} - /> -
- ) : null} - - - -
- -
- - setPackageURL(e.target.value)} - disabled={isPending} - /> -
-

- 服务端将使用浏览器环境请求头从该链接下载压缩包，支持内网地址与自签证书 - HTTPS。 -

+ + 部署入口 +
+ {pagesEntryPath(rootDir, entryFile)}
- - + + 入口来自项目设置；部署包上传不会覆盖该配置。 + + -
- - -
+ {uploadProgress !== null ? ( + +
+ + {uploadProgress >= 100 ? '服务端处理中' : '上传进度'} + + + {uploadProgress >= 100 ? '请稍候' : `${uploadProgress}%`} + +
+ + + ) : null} + - - diff --git a/frontend/app/(main)/pages/components/pages-utils.ts b/frontend/app/(main)/pages/components/pages-utils.ts index 50581b84..f43b3fac 100644 --- a/frontend/app/(main)/pages/components/pages-utils.ts +++ b/frontend/app/(main)/pages/components/pages-utils.ts @@ -17,6 +17,10 @@ export function deploymentsQueryKey(projectId: number) { return ['openflare', 'pages', 'deployments', projectId] as const; } +export function sourceQueryKey(projectId: number) { + return ['openflare', 'pages', 'source', projectId] as const; +} + export function deploymentFilesQueryKey( projectId: number, deploymentId: number, diff --git a/frontend/app/(main)/pages/detail/components/deployment-files-panel.tsx b/frontend/app/(main)/pages/detail/components/deployment-files-panel.tsx new file mode 100644 index 00000000..7b0b4ad9 --- /dev/null +++ b/frontend/app/(main)/pages/detail/components/deployment-files-panel.tsx @@ -0,0 +1,86 @@ +'use client'; + +import { useQuery } from '@tanstack/react-query'; + +import { EmptyInline } from '@/components/layout/empty'; +import { ErrorInline } from '@/components/layout/error'; +import { Skeleton } from '@/components/ui/skeleton'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { PagesService } from '@/lib/services/openflare'; + +import { + deploymentFilesQueryKey, + formatBytes, +} from '../../components/pages-utils'; + +interface DeploymentFilesPanelProps { + projectId: number; + deploymentId: number; +} + +export function DeploymentFilesPanel({ + projectId, + deploymentId, +}: DeploymentFilesPanelProps) { + const filesQuery = useQuery({ + queryKey: deploymentFilesQueryKey(projectId, deploymentId), + queryFn: () => PagesService.listDeploymentFiles(deploymentId), + }); + + if (filesQuery.isLoading) { + return ( +
+ + +
+ ); + } + + if (filesQuery.isError) { + return ( +
+ void filesQuery.refetch()} + /> +
+ ); + } + + const files = filesQuery.data ?? []; + if (files.length === 0) { + return ; + } + + return ( + + + + 路径 + 大小 + + + + {files.map((file) => ( + + {file.path} + + {formatBytes(file.size)} + + + ))} + +
+ ); +} diff --git a/frontend/app/(main)/pages/detail/components/deployment-history.tsx b/frontend/app/(main)/pages/detail/components/deployment-history.tsx new file mode 100644 index 00000000..bd046500 --- /dev/null +++ b/frontend/app/(main)/pages/detail/components/deployment-history.tsx @@ -0,0 +1,300 @@ +'use client'; + +import { useMemo, useState } from 'react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { ChevronDown, ChevronRight } from 'lucide-react'; +import { toast } from 'sonner'; + +import { EmptyStateWithBorder } from '@/components/layout/empty'; +import { ErrorInline } from '@/components/layout/error'; +import { LoadingStateWithBorder } from '@/components/layout/loading'; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { Spinner } from '@/components/ui/spinner'; +import { type PagesDeployment, PagesService } from '@/lib/services/openflare'; +import { formatDateTime } from '@/lib/utils'; + +import { + deploymentFilesQueryKey, + deploymentsQueryKey, + formatBytes, + projectQueryKey, + projectsQueryKey, + sourceQueryKey, +} from '../../components/pages-utils'; +import { DeploymentFilesPanel } from './deployment-files-panel'; + +const SOURCE_LABELS: Record = { + manual_upload: '本地上传', + manual_url: 'URL 导入', + remote_url: 'Remote URL', + github_release: 'GitHub Release', +}; + +const TRIGGER_LABELS: Record = { + manual_upload: '手动上传', + manual_url: '手动导入', + manual_sync: '手动同步', + scheduled_auto_update: '定时更新', +}; + +interface DeploymentHistoryProps { + projectId: number; + activeDeploymentId?: number | null; +} + +type PendingAction = { + type: 'activate' | 'delete'; + deployment: PagesDeployment; +}; + +function deploymentSnapshot(deployment: PagesDeployment) { + return [ + SOURCE_LABELS[deployment.source_type], + deployment.source_label, + TRIGGER_LABELS[deployment.trigger_type], + ] + .filter(Boolean) + .join(' · '); +} + +export function DeploymentHistory({ + projectId, + activeDeploymentId, +}: DeploymentHistoryProps) { + const queryClient = useQueryClient(); + const [expandedDeploymentId, setExpandedDeploymentId] = useState< + number | null + >(null); + const [pendingAction, setPendingAction] = useState( + null, + ); + + const deploymentsQuery = useQuery({ + queryKey: deploymentsQueryKey(projectId), + queryFn: () => PagesService.listDeployments(projectId), + }); + + const deployments = useMemo(() => { + const records = [...(deploymentsQuery.data ?? [])]; + return records.sort((left, right) => { + const leftActive = + left.id === activeDeploymentId || left.status === 'active'; + const rightActive = + right.id === activeDeploymentId || right.status === 'active'; + if (leftActive !== rightActive) return leftActive ? -1 : 1; + return right.deployment_number - left.deployment_number; + }); + }, [activeDeploymentId, deploymentsQuery.data]); + + const invalidateDeploymentState = async () => { + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: deploymentsQueryKey(projectId), + }), + queryClient.invalidateQueries({ queryKey: projectQueryKey(projectId) }), + queryClient.invalidateQueries({ queryKey: sourceQueryKey(projectId) }), + queryClient.invalidateQueries({ queryKey: projectsQueryKey }), + ]); + }; + + const activateMutation = useMutation({ + mutationFn: (deploymentId: number) => + PagesService.activateDeployment(projectId, deploymentId), + onSuccess: async () => { + toast.success('部署已激活'); + await invalidateDeploymentState(); + setPendingAction(null); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : '激活失败'); + }, + }); + + const deleteMutation = useMutation({ + mutationFn: (deploymentId: number) => + PagesService.deleteDeployment(projectId, deploymentId), + onSuccess: async (_, deploymentId) => { + toast.success('部署已删除'); + queryClient.removeQueries({ + queryKey: deploymentFilesQueryKey(projectId, deploymentId), + }); + await invalidateDeploymentState(); + setPendingAction(null); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : '删除失败'); + }, + }); + + const actionPending = activateMutation.isPending || deleteMutation.isPending; + + return ( + <> + + + 部署历史 + + 部署记录不可变，来源信息是创建部署时的安全快照。 + + + + {deploymentsQuery.isLoading ? ( + + ) : deploymentsQuery.isError ? ( +
+ void deploymentsQuery.refetch()} + /> +
+ ) : deployments.length === 0 ? ( + + ) : ( + deployments.map((deployment) => { + const active = + deployment.id === activeDeploymentId || + deployment.status === 'active'; + const expanded = expandedDeploymentId === deployment.id; + + return ( +
+
+
+ +
+
+ + 部署 #{deployment.deployment_number} + + + {active ? '当前生产部署' : '历史部署'} + + + {deploymentSnapshot(deployment)} + +
+

+ {deployment.checksum.slice(0, 16)} ·{' '} + {deployment.file_count} 个文件 ·{' '} + {formatBytes(deployment.total_size)} +

+

+ 创建于 {formatDateTime(deployment.created_at)} +

+
+
+
+ + +
+
+ {expanded ? ( + + ) : null} +
+ ); + }) + )} + + + + { + if (!open && !actionPending) setPendingAction(null); + }} + > + + + + {pendingAction?.type === 'activate' ? '激活历史部署' : '删除部署'} + + + {pendingAction?.type === 'activate' + ? '激活其它历史部署会终止当前来源任务；若已开启自动更新，将同时关闭自动更新。' + : `确认删除部署 #${pendingAction?.deployment.deployment_number} 吗？此操作不可恢复。`} + + + + 取消 + { + event.preventDefault(); + if (!pendingAction) return; + if (pendingAction.type === 'activate') { + activateMutation.mutate(pendingAction.deployment.id); + } else { + deleteMutation.mutate(pendingAction.deployment.id); + } + }} + > + {actionPending ? : null} + 确认 + + + + + + ); +} diff --git a/frontend/app/(main)/pages/detail/components/pages-source-card.tsx b/frontend/app/(main)/pages/detail/components/pages-source-card.tsx new file mode 100644 index 00000000..eced9522 --- /dev/null +++ b/frontend/app/(main)/pages/detail/components/pages-source-card.tsx @@ -0,0 +1,393 @@ +'use client'; + +import { useEffect, useRef, useState } from 'react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { Download, Pencil, RefreshCw, RotateCcw } from 'lucide-react'; +import { toast } from 'sonner'; + +import { ErrorInline } from '@/components/layout/error'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { + Card, + CardAction, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { Skeleton } from '@/components/ui/skeleton'; +import { Spinner } from '@/components/ui/spinner'; +import { AdminTaskService } from '@/lib/services/admin'; +import { + type PagesSource, + type PagesSourceActionReceipt, + type PagesSourceStatus, + PagesService, +} from '@/lib/services/openflare'; +import { formatDateTime } from '@/lib/utils'; + +import { + deploymentsQueryKey, + projectQueryKey, + projectsQueryKey, + sourceQueryKey, +} from '../../components/pages-utils'; +import { PagesSourceDialog } from './pages-source-dialog'; + +const ACTION_POLL_INTERVAL = 2_000; +const ACTION_MAX_WAIT = 16 * 60 * 1_000; + +const SOURCE_STATUS: Record< + PagesSourceStatus, + { + label: string; + variant: 'default' | 'secondary' | 'destructive' | 'outline'; + } +> = { + idle: { label: '空闲', variant: 'outline' }, + checking: { label: '检查中', variant: 'secondary' }, + update_available: { label: '有可用更新', variant: 'default' }, + syncing: { label: '同步中', variant: 'secondary' }, + failed: { label: '最近同步失败', variant: 'destructive' }, + attention: { label: '需要确认', variant: 'destructive' }, +}; + +interface ActiveAction { + receipt: PagesSourceActionReceipt; + startedAt: number; +} + +function revisionSummary(source: PagesSource) { + if (source.source_type === 'manual' || !source.last_applied) + return '尚未应用'; + return `${source.last_applied.label} · ${source.last_applied.revision.slice(0, 12)}`; +} + +export function PagesSourceCard({ projectId }: { projectId: number }) { + const queryClient = useQueryClient(); + const handledExecutionID = useRef(null); + const sourcePollingStartedAt = useRef(null); + const [dialogOpen, setDialogOpen] = useState(false); + const [dialogMode, setDialogMode] = useState<'manual' | 'remote_url'>( + 'manual', + ); + const [activeAction, setActiveAction] = useState(null); + const [actionTimedOut, setActionTimedOut] = useState(false); + + const sourceQuery = useQuery({ + queryKey: sourceQueryKey(projectId), + queryFn: () => PagesService.getSource(projectId), + refetchInterval: (query) => { + const source = query.state.data; + if ( + source && + source.source_type !== 'manual' && + (source.sync_status === 'checking' || source.sync_status === 'syncing') + ) { + sourcePollingStartedAt.current ??= Date.now(); + return Date.now() - sourcePollingStartedAt.current < ACTION_MAX_WAIT + ? ACTION_POLL_INTERVAL + : false; + } + sourcePollingStartedAt.current = null; + return false; + }, + }); + + const executionQuery = useQuery({ + queryKey: [ + 'admin', + 'task-execution', + activeAction?.receipt.execution_id ?? '', + ], + queryFn: () => + AdminTaskService.getTaskExecution(activeAction!.receipt.execution_id), + enabled: Boolean(activeAction) && !actionTimedOut, + refetchInterval: (query) => { + if (actionTimedOut) return false; + const status = query.state.data?.status; + return status === 'pending' || status === 'running' + ? ACTION_POLL_INTERVAL + : false; + }, + }); + + useEffect(() => { + if (!activeAction || actionTimedOut) return; + const elapsed = Date.now() - activeAction.startedAt; + const remaining = Math.max(0, ACTION_MAX_WAIT - elapsed); + const timeout = window.setTimeout(() => setActionTimedOut(true), remaining); + return () => window.clearTimeout(timeout); + }, [actionTimedOut, activeAction]); + + useEffect(() => { + const execution = executionQuery.data; + if (!execution || !['succeeded', 'failed'].includes(execution.status)) { + return; + } + if (handledExecutionID.current === execution.id) return; + handledExecutionID.current = execution.id; + + void Promise.all([ + queryClient.invalidateQueries({ queryKey: sourceQueryKey(projectId) }), + queryClient.invalidateQueries({ queryKey: projectQueryKey(projectId) }), + queryClient.invalidateQueries({ + queryKey: deploymentsQueryKey(projectId), + }), + queryClient.invalidateQueries({ + queryKey: ['openflare', 'pages', 'deployment-files', projectId], + }), + queryClient.invalidateQueries({ queryKey: projectsQueryKey }), + ]); + + if (execution.status === 'succeeded') { + toast.success('部署源同步并发布完成'); + } else { + toast.error(execution.error_message || '部署源同步失败'); + } + setActiveAction(null); + setActionTimedOut(false); + }, [executionQuery.data, projectId, queryClient]); + + const syncMutation = useMutation({ + mutationFn: () => PagesService.syncSource(projectId, {}), + onSuccess: async (receipt) => { + handledExecutionID.current = null; + setActiveAction({ receipt, startedAt: Date.now() }); + setActionTimedOut(false); + await queryClient.invalidateQueries({ + queryKey: sourceQueryKey(projectId), + }); + toast.success('同步任务已提交'); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : '同步任务提交失败'); + }, + }); + + const source = sourceQuery.data; + const executionBusy = + activeAction !== null && + (executionQuery.data?.status === undefined || + executionQuery.data.status === 'pending' || + executionQuery.data.status === 'running'); + const sourceBusy = + source?.source_type !== 'manual' && + (source?.sync_status === 'checking' || source?.sync_status === 'syncing'); + const actionsDisabled = syncMutation.isPending || executionBusy || sourceBusy; + + const openSourceDialog = (mode: 'manual' | 'remote_url') => { + setDialogMode(mode); + setDialogOpen(true); + }; + + if (sourceQuery.isLoading) { + return ( + + + 部署源 + 加载来源配置... + + + + + + + ); + } + + if (sourceQuery.isError || !source) { + return ( + + + 部署源 + 来源配置与部署历史相互独立。 + + + void sourceQuery.refetch()} + /> + + + ); + } + + const status = + source.source_type === 'manual' + ? null + : SOURCE_STATUS[source.sync_status ?? 'idle']; + + return ( + <> + + + 部署源 + + 来源配置负责发现内容，发布结果记录在独立的部署历史中。 + + + {status ? ( + {status.label} + ) : ( + 手动部署 + )} + + + + + {source.source_type === 'manual' ? ( +
+

本地部署包

+

+ 当前没有持久化远端来源。上传部署包后，再从部署历史显式激活。 +

+
+ ) : source.source_type === 'remote_url' ? ( +
+
+ 脱敏地址 + {source.display_url} +
+
+ 网络策略 + + {source.remote_network_policy === 'trusted_internal' + ? '受信内网模式' + : '公网安全模式'} + +
+
+ 最近同步 + + {source.last_synced_at + ? formatDateTime(source.last_synced_at) + : '尚未同步'} + +
+
+ + 已应用 revision + + + {revisionSummary(source)} + +
+ {source.last_error ? ( +
+ +
+ ) : null} +
+ ) : ( +
+ 当前版本暂不提供该来源类型的编辑界面。 +
+ )} + + {executionQuery.isError ? ( + void executionQuery.refetch()} + /> + ) : null} + {actionTimedOut ? ( +
+ + 自动等待已停止，任务可能仍在后台运行。 + + +
+ ) : null} + + + + {source.source_type === 'manual' ? ( + + ) : source.source_type === 'remote_url' ? ( + <> + + + + + ) : null} + + + + + + + ); +} diff --git a/frontend/app/(main)/pages/detail/components/pages-source-dialog.tsx b/frontend/app/(main)/pages/detail/components/pages-source-dialog.tsx new file mode 100644 index 00000000..c3bdb0a1 --- /dev/null +++ b/frontend/app/(main)/pages/detail/components/pages-source-dialog.tsx @@ -0,0 +1,376 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { toast } from 'sonner'; + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog'; +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { + Field, + FieldDescription, + FieldGroup, + FieldLabel, + FieldTitle, +} from '@/components/ui/field'; +import { Input } from '@/components/ui/input'; +import { Spinner } from '@/components/ui/spinner'; +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group'; +import { + type PagesRemoteNetworkPolicy, + type PagesSource, + PagesService, +} from '@/lib/services/openflare'; + +import { + deploymentsQueryKey, + projectQueryKey, + projectsQueryKey, + sourceQueryKey, +} from '../../components/pages-utils'; + +type SourceMode = 'manual' | 'remote_url'; +type Confirmation = 'trusted_internal' | 'manual' | null; + +interface PagesSourceDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + projectId: number; + source: PagesSource; + initialMode?: SourceMode; +} + +export function PagesSourceDialog({ + open, + onOpenChange, + projectId, + source, + initialMode, +}: PagesSourceDialogProps) { + const queryClient = useQueryClient(); + const [mode, setMode] = useState('manual'); + const [networkPolicy, setNetworkPolicy] = + useState('public'); + const [replaceURL, setReplaceURL] = useState(false); + const [remoteURL, setRemoteURL] = useState(''); + const [urlError, setURLError] = useState(''); + const [confirmation, setConfirmation] = useState(null); + + useEffect(() => { + if (!open) return; + const nextMode = + initialMode ?? + (source.source_type === 'remote_url' ? 'remote_url' : 'manual'); + setMode(nextMode); + setNetworkPolicy( + source.source_type === 'remote_url' + ? source.remote_network_policy + : 'public', + ); + setReplaceURL(source.source_type !== 'remote_url'); + setRemoteURL(''); + setURLError(''); + setConfirmation(null); + }, [initialMode, open, source]); + + const invalidateSourceState = async () => { + await Promise.all([ + queryClient.invalidateQueries({ queryKey: sourceQueryKey(projectId) }), + queryClient.invalidateQueries({ queryKey: projectQueryKey(projectId) }), + queryClient.invalidateQueries({ + queryKey: deploymentsQueryKey(projectId), + }), + queryClient.invalidateQueries({ queryKey: projectsQueryKey }), + ]); + }; + + const updateMutation = useMutation({ + mutationFn: () => + PagesService.updateSource(projectId, { + source_type: 'remote_url', + remote_url_set: replaceURL, + remote_url: replaceURL ? remoteURL.trim() : '', + remote_network_policy: networkPolicy, + }), + onSuccess: async (result) => { + queryClient.setQueryData(sourceQueryKey(projectId), result.source); + await invalidateSourceState(); + toast.success('部署源已更新'); + if (result.warning) toast.warning(result.warning); + setConfirmation(null); + onOpenChange(false); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : '部署源更新失败'); + }, + }); + + const deleteMutation = useMutation({ + mutationFn: () => PagesService.deleteSource(projectId), + onSuccess: async (manualSource) => { + queryClient.setQueryData(sourceQueryKey(projectId), manualSource); + await invalidateSourceState(); + toast.success('已切换回手动部署'); + setConfirmation(null); + onOpenChange(false); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : '切换失败'); + }, + }); + + const isPending = updateMutation.isPending || deleteMutation.isPending; + + const submitRemote = () => { + if (replaceURL) { + const value = remoteURL.trim(); + if (!value) { + setURLError('请输入 Remote URL'); + return; + } + try { + const parsed = new URL(value); + if (!['http:', 'https:'].includes(parsed.protocol)) throw new Error(); + } catch { + setURLError('请输入有效的 HTTP(S) URL'); + return; + } + } + setURLError(''); + if (networkPolicy === 'trusted_internal') { + setConfirmation('trusted_internal'); + return; + } + updateMutation.mutate(); + }; + + const handleSubmit = () => { + if (mode === 'manual') { + if (source.source_type === 'manual') { + onOpenChange(false); + } else { + setConfirmation('manual'); + } + return; + } + submitRemote(); + }; + + return ( + <> + { + if (!isPending) onOpenChange(nextOpen); + }} + > + + + 部署源设置 + + 手动部署与 Remote URL + 各自保持独立配置；后续仓库构建来源会作为新的来源类型接入。 + + + + + + 来源类型 + { + if (value === 'manual' || value === 'remote_url') { + setMode(value); + } + }} + > + + 手动部署 + + + Remote URL + + + + 手动部署由管理员上传本地包；Remote URL 通过显式同步下载并发布。 + + + + {mode === 'manual' ? ( + + 手动部署 +
+ 保留现有部署与当前生产版本，后续通过“上传部署包”创建新部署。 +
+ + ) : ( + <> + + Remote URL + {source.source_type === 'remote_url' && !replaceURL ? ( +
+ + {source.display_url} + + +
+ ) : ( + { + setRemoteURL(event.target.value); + setURLError(''); + }} + /> + )} + + {urlError || + (replaceURL + ? '保存后不会回显原始地址或 query token。' + : '界面只显示脱敏地址；留空表示保留当前地址。')} + + {source.source_type === 'remote_url' && replaceURL ? ( + + ) : null} + + + + 网络策略 + { + if (value === 'public' || value === 'trusted_internal') { + setNetworkPolicy(value); + } + }} + > + + 公网安全模式 + + + 受信内网模式 + + + + {networkPolicy === 'public' + ? '阻止内网地址、代理与不安全 TLS。' + : '允许访问内网地址与自签名证书，仅用于可信来源。'} + + + + )} + + + + + + + + + + { + if (!nextOpen && !isPending) setConfirmation(null); + }} + > + + + + {confirmation === 'manual' + ? '切换回手动部署' + : '启用受信内网模式'} + + + {confirmation === 'manual' + ? '当前来源配置将被删除，但已有部署与当前生产版本会保留。' + : '该模式允许访问私有网络并接受自签名证书。请确认此地址属于可信内部来源，且不会被非可信用户控制。'} + + + + 取消 + { + event.preventDefault(); + if (confirmation === 'manual') { + deleteMutation.mutate(); + } else if (confirmation === 'trusted_internal') { + updateMutation.mutate(); + } + }} + > + {isPending ? : null} + 确认 + + + + + + ); +} diff --git a/frontend/app/(main)/pages/detail/page-client.tsx b/frontend/app/(main)/pages/detail/page-client.tsx deleted file mode 100644 index a1115b68..00000000 --- a/frontend/app/(main)/pages/detail/page-client.tsx +++ /dev/null @@ -1,505 +0,0 @@ -'use client'; - -import Link from 'next/link'; -import { useMemo, useState } from 'react'; -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; -import { useSearchParams } from 'next/navigation'; -import { - ArrowLeft, - ChevronDown, - ChevronRight, - FileText, - Loader2, - Trash2, - Upload, -} from 'lucide-react'; -import { toast } from 'sonner'; - -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from '@/components/ui/alert-dialog'; -import { EmptyStateWithBorder } from '@/components/layout/empty'; -import { ErrorInline } from '@/components/layout/error'; -import { LoadingStateWithBorder } from '@/components/layout/loading'; -import { Badge } from '@/components/ui/badge'; -import { Button } from '@/components/ui/button'; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from '@/components/ui/table'; -import { type PagesDeployment, PagesService } from '@/lib/services/openflare'; -import { formatDateTime } from '@/lib/utils'; - -import { DeploymentUploadDialog } from '../components/deployment-upload-dialog'; -import { ProjectEditorDialog } from '../components/project-editor-dialog'; -import { - deploymentFilesQueryKey, - deploymentsQueryKey, - formatBytes, - projectQueryKey, - projectsQueryKey, -} from '../components/pages-utils'; - -function DeploymentFilesPanel({ - projectId, - deployment, -}: { - projectId: number; - deployment: PagesDeployment; -}) { - const filesQuery = useQuery({ - queryKey: deploymentFilesQueryKey(projectId, deployment.id), - queryFn: () => PagesService.listDeploymentFiles(deployment.id), - }); - - if (filesQuery.isLoading) { - return ( -

加载文件清单...

- ); - } - - if (filesQuery.isError) { - return ( -

- {filesQuery.error instanceof Error - ? filesQuery.error.message - : '加载失败'} -

- ); - } - - const files = filesQuery.data ?? []; - if (files.length === 0) { - return ( -

暂无文件记录

- ); - } - - return ( -
- - - - 路径 - 大小 - - - - {files.map((file) => ( - - {file.path} - - {formatBytes(file.size)} - - - ))} - -
-
- ); -} - -export function PagesDetailPageClient() { - const searchParams = useSearchParams(); - const queryClient = useQueryClient(); - const projectId = searchParams.get('id')?.trim() ?? ''; - const parsedProjectId = Number(projectId); - - const [editorOpen, setEditorOpen] = useState(false); - const [uploadOpen, setUploadOpen] = useState(false); - const [expandedDeploymentId, setExpandedDeploymentId] = useState< - number | null - >(null); - const [deleteProjectOpen, setDeleteProjectOpen] = useState(false); - const [pendingDeploymentAction, setPendingDeploymentAction] = useState<{ - type: 'activate' | 'delete'; - deployment: PagesDeployment; - } | null>(null); - - const enabled = projectId !== '' && Number.isFinite(parsedProjectId); - - const projectQuery = useQuery({ - queryKey: projectQueryKey(projectId), - queryFn: () => PagesService.getProject(parsedProjectId), - enabled, - }); - - const deploymentsQuery = useQuery({ - queryKey: deploymentsQueryKey(parsedProjectId), - queryFn: () => PagesService.listDeployments(parsedProjectId), - enabled, - }); - - const activateMutation = useMutation({ - mutationFn: (deploymentId: number) => - PagesService.activateDeployment(parsedProjectId, deploymentId), - onSuccess: async () => { - toast.success('部署已激活'); - await Promise.all([ - queryClient.invalidateQueries({ - queryKey: deploymentsQueryKey(parsedProjectId), - }), - queryClient.invalidateQueries({ queryKey: projectQueryKey(projectId) }), - queryClient.invalidateQueries({ queryKey: projectsQueryKey }), - ]); - setPendingDeploymentAction(null); - }, - onError: (error) => { - toast.error(error instanceof Error ? error.message : '激活失败'); - }, - }); - - const deleteDeploymentMutation = useMutation({ - mutationFn: (deploymentId: number) => - PagesService.deleteDeployment(parsedProjectId, deploymentId), - onSuccess: async () => { - toast.success('部署已删除'); - await Promise.all([ - queryClient.invalidateQueries({ - queryKey: deploymentsQueryKey(parsedProjectId), - }), - queryClient.invalidateQueries({ queryKey: projectQueryKey(projectId) }), - queryClient.invalidateQueries({ queryKey: projectsQueryKey }), - ]); - setPendingDeploymentAction(null); - }, - onError: (error) => { - toast.error(error instanceof Error ? error.message : '删除失败'); - }, - }); - - const deleteProjectMutation = useMutation({ - mutationFn: () => PagesService.deleteProject(parsedProjectId), - onSuccess: async () => { - toast.success('项目已删除'); - await queryClient.invalidateQueries({ queryKey: projectsQueryKey }); - window.location.href = '/pages'; - }, - onError: (error) => { - toast.error(error instanceof Error ? error.message : '删除失败'); - }, - }); - - const project = projectQuery.data; - const deployments = useMemo( - () => deploymentsQuery.data ?? [], - [deploymentsQuery.data], - ); - - if (!enabled) { - return ( -
- -
- ); - } - - if (projectQuery.isLoading) { - return ( -
- -
- ); - } - - if (projectQuery.isError) { - return ( -
- void projectQuery.refetch()} - /> -
- ); - } - - if (!project) { - return ( -
- - -
- ); - } - - return ( -
-
-
- -
- -

- {project.name} -

-
-

- {project.slug} · {project.deployment_count} 个部署 -

-
-
- - - -
-
- -
-
-

- 状态 -

- - {project.enabled ? '已启用' : '已停用'} - -
-
-

- 当前激活 -

-

- {project.active_deployment - ? `#${project.active_deployment.deployment_number}` - : '暂无'} -

-
-
-

- 更新时间 -

-

{formatDateTime(project.updated_at)}

-
-
- -
-
-

部署历史

-

- 部署不可变；激活后发布配置，Agent 才会拉取并切换静态资源。 -

-
- - {deploymentsQuery.isLoading ? ( - - ) : deployments.length === 0 ? ( - - ) : ( -
- {deployments.map((deployment) => { - const expanded = expandedDeploymentId === deployment.id; - return ( -
-
-
- -
-
-

- #{deployment.deployment_number} -

- {deployment.status === 'active' ? ( - - 已激活 - - ) : null} -
-

- {deployment.checksum.slice(0, 16)} ·{' '} - {deployment.file_count} files ·{' '} - {formatBytes(deployment.total_size)} -

-

- 创建于 {formatDateTime(deployment.created_at)} -

-
-
-
- - -
-
- {expanded ? ( - - ) : null} -
- ); - })} -
- )} -
- - - - - - - - 删除 Pages 项目 - - 确认删除项目 {project.name} 吗？此操作不可恢复。 - - - - 取消 - deleteProjectMutation.mutate()} - > - {deleteProjectMutation.isPending ? ( - - ) : ( - '确认删除' - )} - - - - - - { - if (!open) setPendingDeploymentAction(null); - }} - > - - - - {pendingDeploymentAction?.type === 'activate' - ? '激活部署' - : '删除部署'} - - - {pendingDeploymentAction?.type === 'activate' - ? `确认激活部署 #${pendingDeploymentAction.deployment.deployment_number} 吗？` - : `确认删除部署 #${pendingDeploymentAction?.deployment.deployment_number} 吗？`} - - - - 取消 - { - if (!pendingDeploymentAction) return; - if (pendingDeploymentAction.type === 'activate') { - activateMutation.mutate( - pendingDeploymentAction.deployment.id, - ); - } else { - deleteDeploymentMutation.mutate( - pendingDeploymentAction.deployment.id, - ); - } - }} - > - 确认 - - - - -
- ); -} diff --git a/frontend/app/(main)/pages/detail/page.tsx b/frontend/app/(main)/pages/detail/page.tsx index a24e6fca..e653471b 100644 --- a/frontend/app/(main)/pages/detail/page.tsx +++ b/frontend/app/(main)/pages/detail/page.tsx @@ -1,23 +1,318 @@ -import { Suspense } from 'react'; +'use client'; +import Link from 'next/link'; +import { Suspense, useState } from 'react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { useRouter, useSearchParams } from 'next/navigation'; +import { ArrowLeft, FileText, Pencil, Trash2, Upload } from 'lucide-react'; +import { toast } from 'sonner'; + +import { EmptyStateWithBorder } from '@/components/layout/empty'; +import { ErrorInline } from '@/components/layout/error'; +import { LoadingStateWithBorder } from '@/components/layout/loading'; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { + Card, + CardAction, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; import { Skeleton } from '@/components/ui/skeleton'; +import { Spinner } from '@/components/ui/spinner'; +import { PagesService } from '@/lib/services/openflare'; +import { formatDateTime } from '@/lib/utils'; -import { PagesDetailPageClient } from './page-client'; +import { + DeploymentUploadDialog, + pagesEntryPath, +} from '../components/deployment-upload-dialog'; +import { ProjectEditorDialog } from '../components/project-editor-dialog'; +import { + projectQueryKey, + projectsQueryKey, + sourceQueryKey, +} from '../components/pages-utils'; +import { DeploymentHistory } from './components/deployment-history'; +import { PagesSourceCard } from './components/pages-source-card'; function PagesDetailPageFallback() { return ( -
- - +
+ + +
+ + +
); } +function PagesDetailRoute() { + const searchParams = useSearchParams(); + const router = useRouter(); + const queryClient = useQueryClient(); + const [editorOpen, setEditorOpen] = useState(false); + const [uploadOpen, setUploadOpen] = useState(false); + const [deleteProjectOpen, setDeleteProjectOpen] = useState(false); + + const rawProjectId = searchParams.get('id')?.trim() ?? ''; + const projectId = Number(rawProjectId); + const validProjectId = + rawProjectId !== '' && Number.isInteger(projectId) && projectId > 0; + + const projectQuery = useQuery({ + queryKey: projectQueryKey(projectId), + queryFn: () => PagesService.getProject(projectId), + enabled: validProjectId, + }); + + const deleteProjectMutation = useMutation({ + mutationFn: () => PagesService.deleteProject(projectId), + onSuccess: async () => { + toast.success('项目已删除'); + await queryClient.invalidateQueries({ queryKey: projectsQueryKey }); + router.push('/pages'); + }, + onError: (error) => { + toast.error(error instanceof Error ? error.message : '删除失败'); + }, + }); + + if (!validProjectId) { + return ( +
+ +
+ ); + } + + if (projectQuery.isLoading) { + return ( +
+ +
+ ); + } + + if (projectQuery.isError) { + return ( +
+
+ void projectQuery.refetch()} + /> +
+
+ ); + } + + const project = projectQuery.data; + if (!project) { + return ( +
+ + +
+ ); + } + + const activeDeployment = project.active_deployment; + const entryPath = pagesEntryPath(project.root_dir ?? '', project.entry_file); + + return ( +
+
+ +
+
+
+ +

+ {project.name} +

+
+

+ {project.slug} · {project.deployment_count} 个部署 +

+
+
+ + + +
+
+
+ +
+ + + 当前生产部署 + + Agent 当前应拉取并提供服务的不可变部署。 + + + + {activeDeployment ? '生产中' : '未发布'} + + + + + {activeDeployment ? ( + <> +

+ 部署 #{activeDeployment.deployment_number} +

+

+ {activeDeployment.checksum.slice(0, 20)} +

+

+ 激活于{' '} + {activeDeployment.activated_at + ? formatDateTime(activeDeployment.activated_at) + : '未知时间'} +

+ + ) : ( +

+ 尚无生产部署。上传或同步来源后，从部署历史激活一个版本。 +

+ )} + + + + + + 站点入口 + + 解包校验、发布快照与 Agent 切换共同使用此路径。 + + + + {project.enabled ? '项目已启用' : '项目已停用'} + + + + + + {entryPath} + +

+ 项目更新于 {formatDateTime(project.updated_at)} +

+ + +
+ + + + + { + setEditorOpen(nextOpen); + if (!nextOpen) { + void queryClient.invalidateQueries({ + queryKey: sourceQueryKey(projectId), + }); + } + }} + project={project} + /> + + + + + + 删除 Pages 项目 + + 确认删除项目 {project.name} 吗？此操作不可恢复。 + + + + + 取消 + + { + event.preventDefault(); + deleteProjectMutation.mutate(); + }} + > + {deleteProjectMutation.isPending ? ( + + ) : null} + 确认删除 + + + + +
+ ); +} + export default function PagesDetailPage() { return ( }> - + ); } diff --git a/frontend/lib/services/index.ts b/frontend/lib/services/index.ts index 6184f565..69e79fa1 100644 --- a/frontend/lib/services/index.ts +++ b/frontend/lib/services/index.ts @@ -251,6 +251,9 @@ export type { DnsAccountItem, PagesProject, PagesDeployment, + PagesSource, + PagesSourceActionReceipt, + PagesSourceStatus, OriginItem, OriginDetail, AccessLogList, diff --git a/frontend/lib/services/openflare/index.ts b/frontend/lib/services/openflare/index.ts index 4ef01046..7bca6843 100644 --- a/frontend/lib/services/openflare/index.ts +++ b/frontend/lib/services/openflare/index.ts @@ -109,8 +109,19 @@ export type { OriginMutationPayload, PagesDeployment, PagesDeploymentFile, + PagesGitHubReleaseSource, + PagesManualSource, PagesProject, PagesProjectPayload, + PagesRemoteNetworkPolicy, + PagesRemoteSourceUpdatePayload, + PagesRemoteURLSource, + PagesSource, + PagesSourceActionPayload, + PagesSourceActionReceipt, + PagesSourceRevision, + PagesSourceStatus, + PagesSourceUpdateResult, AcmeAccountItem, DnsAccountItem, DnsAccountMutationPayload, diff --git a/frontend/lib/services/openflare/pages.service.ts b/frontend/lib/services/openflare/pages.service.ts index e220a168..e45b0543 100644 --- a/frontend/lib/services/openflare/pages.service.ts +++ b/frontend/lib/services/openflare/pages.service.ts @@ -12,6 +12,11 @@ import type { PagesDeploymentUploadPayload, PagesProject, PagesProjectPayload, + PagesRemoteSourceUpdatePayload, + PagesSource, + PagesSourceActionPayload, + PagesSourceActionReceipt, + PagesSourceUpdateResult, } from './types'; export class PagesService extends OpenFlareBaseService { @@ -40,6 +45,41 @@ export class PagesService extends OpenFlareBaseService { return this.post(`/${id}/delete`); } + static getSource(projectId: number): Promise { + return this.get(`/${projectId}/source`); + } + + static updateSource( + projectId: number, + payload: PagesRemoteSourceUpdatePayload, + ): Promise { + return this.post( + `/${projectId}/source/update`, + payload, + ); + } + + static deleteSource(projectId: number): Promise { + return this.post(`/${projectId}/source/delete`); + } + + static checkSource(projectId: number): Promise { + return this.post( + `/${projectId}/source/check`, + {}, + ); + } + + static syncSource( + projectId: number, + payload: PagesSourceActionPayload = {}, + ): Promise { + return this.post( + `/${projectId}/source/sync`, + payload, + ); + } + static listDeployments(projectId: number): Promise { return this.get(`/${projectId}/deployments`); } @@ -58,8 +98,6 @@ export class PagesService extends OpenFlareBaseService { ): Promise { const formData = new FormData(); formData.append('package', payload.file); - formData.append('root_dir', payload.rootDir ?? ''); - formData.append('entry_file', payload.entryFile ?? 'index.html'); return this.postFormData( `/${projectId}/deployments/upload`, @@ -75,6 +113,7 @@ export class PagesService extends OpenFlareBaseService { return this.post( `/${projectId}/deployments/upload-from-url`, payload, + { timeout: apiConfig.uploadTimeout } as InternalAxiosRequestConfig, ); } diff --git a/frontend/lib/services/openflare/types.ts b/frontend/lib/services/openflare/types.ts index feede7b6..0b9be41c 100644 --- a/frontend/lib/services/openflare/types.ts +++ b/frontend/lib/services/openflare/types.ts @@ -394,9 +394,11 @@ export interface PagesDeployment { status: 'uploaded' | 'active'; file_count: number; total_size: number; - root_dir?: string; - entry_file: string; created_by: string; + source_type: 'manual_upload' | 'manual_url' | 'remote_url' | 'github_release'; + source_label: string; + trigger_type: + 'manual_upload' | 'manual_url' | 'manual_sync' | 'scheduled_auto_update'; created_at: string; activated_at?: string | null; } @@ -448,8 +450,6 @@ export interface PagesProjectPayload { export interface PagesDeploymentUploadPayload { file: File; - rootDir?: string; - entryFile?: string; onProgress?: (percent: number) => void; } @@ -457,6 +457,79 @@ export interface PagesDeploymentUploadFromURLPayload { url: string; } +export type PagesSourceStatus = + 'idle' | 'checking' | 'update_available' | 'syncing' | 'failed' | 'attention'; + +export type PagesRemoteNetworkPolicy = 'public' | 'trusted_internal'; + +export interface PagesSourceRevision { + revision: string; + label: string; + asset_name?: string; +} + +interface PagesSourceRuntimeView { + sync_status?: PagesSourceStatus; + update_available?: boolean; + last_seen?: PagesSourceRevision; + last_applied?: PagesSourceRevision; + last_checked_at?: string | null; + last_synced_at?: string | null; + next_check_at?: string | null; + last_error?: string; +} + +export interface PagesManualSource { + source_type: 'manual'; +} + +export interface PagesRemoteURLSource extends PagesSourceRuntimeView { + source_type: 'remote_url'; + has_remote_url: boolean; + display_url: string; + remote_network_policy: PagesRemoteNetworkPolicy; +} + +export interface PagesGitHubReleaseSource extends PagesSourceRuntimeView { + source_type: 'github_release'; + github_repository: string; + release_selector: 'latest' | 'tag'; + release_tag: string; + asset_name: string; + auto_update_enabled: boolean; + check_interval_minutes: number; +} + +/** + * 部署源使用判别联合，后续仓库构建来源只需增加独立 variant， + * 不需要向 Remote 或 GitHub Release 填入构建字段。 + */ +export type PagesSource = + PagesManualSource | PagesRemoteURLSource | PagesGitHubReleaseSource; + +export interface PagesRemoteSourceUpdatePayload { + source_type: 'remote_url'; + remote_url_set: boolean; + remote_url: string; + remote_network_policy: PagesRemoteNetworkPolicy; +} + +export interface PagesSourceActionPayload { + confirmed_revision?: string; +} + +export interface PagesSourceActionReceipt { + task_id: string; + execution_id: string; + action: 'check' | 'sync'; +} + +export interface PagesSourceUpdateResult { + source: PagesSource; + check_task: PagesSourceActionReceipt | null; + warning: string; +} + // ==================== Origins ==================== export interface OriginItem { diff --git a/frontend/tests/openflare/pages-service.test.ts b/frontend/tests/openflare/pages-service.test.ts index c38303be..516e5596 100644 --- a/frontend/tests/openflare/pages-service.test.ts +++ b/frontend/tests/openflare/pages-service.test.ts @@ -2,27 +2,34 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { AxiosResponse } from 'axios'; import apiClient from '@/lib/services/core/api-client'; +import { apiConfig } from '@/lib/services/core/config'; import { PagesService } from '@/lib/services/openflare/pages.service'; vi.mock('@/lib/services/core/api-client', () => ({ default: { get: vi.fn(), + post: vi.fn(), }, })); +function response(data: T) { + return { + data: { error_msg: '', data }, + status: 200, + statusText: 'OK', + headers: {}, + config: { headers: {} }, + } as AxiosResponse; +} + describe('PagesService', () => { beforeEach(() => { vi.mocked(apiClient.get).mockReset(); + vi.mocked(apiClient.post).mockReset(); }); it('requests deployment files using the backend deployment route', async () => { - vi.mocked(apiClient.get).mockResolvedValue({ - data: { error_msg: '', data: [] }, - status: 200, - statusText: 'OK', - headers: {}, - config: { headers: {} }, - } as AxiosResponse); + vi.mocked(apiClient.get).mockResolvedValue(response([])); await PagesService.listDeploymentFiles(7); @@ -31,4 +38,88 @@ describe('PagesService', () => { expect.objectContaining({ params: undefined }), ); }); + + it('connects all source endpoints with explicit action payloads', async () => { + vi.mocked(apiClient.get).mockResolvedValue( + response({ source_type: 'manual' }), + ); + vi.mocked(apiClient.post).mockResolvedValue( + response({ source_type: 'manual' }), + ); + + await PagesService.getSource(12); + await PagesService.updateSource(12, { + source_type: 'remote_url', + remote_url_set: true, + remote_url: 'https://example.com/site.zip?token=secret', + remote_network_policy: 'public', + }); + await PagesService.deleteSource(12); + await PagesService.checkSource(12); + await PagesService.syncSource(12); + + expect(apiClient.get).toHaveBeenCalledWith( + '/api/v1/d/pages/12/source', + expect.objectContaining({ params: undefined }), + ); + expect(apiClient.post).toHaveBeenNthCalledWith( + 1, + '/api/v1/d/pages/12/source/update', + expect.objectContaining({ + source_type: 'remote_url', + remote_url_set: true, + }), + undefined, + ); + expect(apiClient.post).toHaveBeenNthCalledWith( + 2, + '/api/v1/d/pages/12/source/delete', + undefined, + undefined, + ); + expect(apiClient.post).toHaveBeenNthCalledWith( + 3, + '/api/v1/d/pages/12/source/check', + {}, + undefined, + ); + expect(apiClient.post).toHaveBeenNthCalledWith( + 4, + '/api/v1/d/pages/12/source/sync', + {}, + undefined, + ); + }); + + it('uploads only the package multipart field', async () => { + vi.mocked(apiClient.post).mockResolvedValue(response({})); + const file = new File(['site'], 'site.zip', { + type: 'application/zip', + }); + + await PagesService.uploadDeployment(8, { file }); + + const formData = vi.mocked(apiClient.post).mock.calls[0]?.[1]; + expect(formData).toBeInstanceOf(FormData); + expect(Array.from((formData as FormData).keys())).toEqual(['package']); + expect(apiClient.post).toHaveBeenCalledWith( + '/api/v1/d/pages/8/deployments/upload', + formData, + expect.objectContaining({ timeout: apiConfig.uploadTimeout }), + ); + }); + + it('keeps the compatibility URL import on the long upload timeout', async () => { + vi.mocked(apiClient.post).mockResolvedValue(response({})); + + await PagesService.uploadDeploymentFromURL(8, { + url: 'https://example.com/site.zip', + }); + + expect(apiClient.post).toHaveBeenCalledWith( + '/api/v1/d/pages/8/deployments/upload-from-url', + { url: 'https://example.com/site.zip' }, + expect.objectContaining({ timeout: apiConfig.uploadTimeout }), + ); + }); }); diff --git a/frontend/tests/openflare/pages-source-ui.test.tsx b/frontend/tests/openflare/pages-source-ui.test.tsx new file mode 100644 index 00000000..b9773638 --- /dev/null +++ b/frontend/tests/openflare/pages-source-ui.test.tsx @@ -0,0 +1,227 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { DeploymentUploadDialog } from '@/app/(main)/pages/components/deployment-upload-dialog'; +import { DeploymentHistory } from '@/app/(main)/pages/detail/components/deployment-history'; +import { PagesSourceCard } from '@/app/(main)/pages/detail/components/pages-source-card'; +import { AdminTaskService } from '@/lib/services/admin'; +import { + type PagesRemoteURLSource, + PagesService, +} from '@/lib/services/openflare'; + +vi.mock('@/lib/services/openflare', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + PagesService: { + getSource: vi.fn(), + updateSource: vi.fn(), + deleteSource: vi.fn(), + syncSource: vi.fn(), + listDeployments: vi.fn(), + listDeploymentFiles: vi.fn(), + activateDeployment: vi.fn(), + deleteDeployment: vi.fn(), + uploadDeployment: vi.fn(), + }, + }; +}); + +vi.mock('@/lib/services/admin', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + AdminTaskService: { + getTaskExecution: vi.fn(), + }, + }; +}); + +function renderWithQuery(ui: React.ReactNode) { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false, gcTime: 0 }, + mutations: { retry: false }, + }, + }); + return render( + {ui}, + ); +} + +const remoteSource: PagesRemoteURLSource = { + source_type: 'remote_url', + has_remote_url: true, + display_url: 'https://artifacts.example.com/site.zip?***', + remote_network_policy: 'public', + sync_status: 'idle', + last_applied: { + revision: 'a'.repeat(64), + label: 'site.zip', + }, + last_synced_at: '2026-07-19T10:00:00Z', + last_error: '', +}; + +describe('Pages source UI', () => { + beforeEach(() => { + vi.mocked(PagesService.getSource).mockReset(); + vi.mocked(PagesService.updateSource).mockReset(); + vi.mocked(PagesService.deleteSource).mockReset(); + vi.mocked(PagesService.syncSource).mockReset(); + vi.mocked(PagesService.listDeployments).mockReset(); + vi.mocked(PagesService.listDeploymentFiles).mockReset(); + vi.mocked(PagesService.activateDeployment).mockReset(); + vi.mocked(PagesService.deleteDeployment).mockReset(); + vi.mocked(PagesService.uploadDeployment).mockReset(); + vi.mocked(AdminTaskService.getTaskExecution).mockReset(); + }); + + it('keeps Phase 1 manual source focused on upload and Remote URL', async () => { + vi.mocked(PagesService.getSource).mockResolvedValue({ + source_type: 'manual', + }); + + renderWithQuery(); + + expect(await screen.findByText('本地部署包')).toBeVisible(); + expect( + screen.getByRole('button', { name: '配置 Remote URL' }), + ).toBeVisible(); + expect(screen.queryByText('检查更新')).not.toBeInTheDocument(); + expect(screen.queryByText('自动更新')).not.toBeInTheDocument(); + }); + + it('never reuses the masked URL as an editable value', async () => { + const user = userEvent.setup(); + vi.mocked(PagesService.getSource).mockResolvedValue(remoteSource); + vi.mocked(PagesService.updateSource).mockResolvedValue({ + source: remoteSource, + check_task: null, + warning: '', + }); + + renderWithQuery(); + + expect( + await screen.findByText('https://artifacts.example.com/site.zip?***'), + ).toBeVisible(); + expect(screen.queryByText(/token=secret/)).not.toBeInTheDocument(); + + await user.click(screen.getByRole('button', { name: '编辑来源' })); + await user.click(screen.getByRole('button', { name: '更换地址' })); + + const input = screen.getByPlaceholderText( + 'https://artifacts.example.com/site.zip?token=...', + ); + expect(input).toHaveValue(''); + await user.type(input, 'https://new.example.com/site.zip?token=new'); + await user.click(screen.getByRole('button', { name: '保存 Remote 来源' })); + + await waitFor(() => { + expect(PagesService.updateSource).toHaveBeenCalledWith(9, { + source_type: 'remote_url', + remote_url_set: true, + remote_url: 'https://new.example.com/site.zip?token=new', + remote_network_policy: 'public', + }); + }); + }); + + it('requires a second confirmation for trusted internal networking', async () => { + const user = userEvent.setup(); + vi.mocked(PagesService.getSource).mockResolvedValue(remoteSource); + vi.mocked(PagesService.updateSource).mockResolvedValue({ + source: { ...remoteSource, remote_network_policy: 'trusted_internal' }, + check_task: null, + warning: '', + }); + + renderWithQuery(); + + await user.click(await screen.findByRole('button', { name: '编辑来源' })); + await user.click(screen.getByRole('radio', { name: '受信内网模式' })); + await user.click(screen.getByRole('button', { name: '保存 Remote 来源' })); + + expect(await screen.findByText('启用受信内网模式')).toBeVisible(); + expect(PagesService.updateSource).not.toHaveBeenCalled(); + + await user.click(screen.getByRole('button', { name: '确认' })); + await waitFor(() => { + expect(PagesService.updateSource).toHaveBeenCalledWith(9, { + source_type: 'remote_url', + remote_url_set: false, + remote_url: '', + remote_network_policy: 'trusted_internal', + }); + }); + }); + + it('polls the existing task execution detail after sync dispatch', async () => { + const user = userEvent.setup(); + vi.mocked(PagesService.getSource).mockResolvedValue(remoteSource); + vi.mocked(PagesService.syncSource).mockResolvedValue({ + task_id: 'manual_of_pages_source_action_1', + execution_id: '42', + action: 'sync', + }); + vi.mocked(AdminTaskService.getTaskExecution).mockResolvedValue({ + id: '42', + task_id: 'manual_of_pages_source_action_1', + task_type: 'of_pages_source_action', + task_name: 'Pages 来源动作', + status: 'succeeded', + retryable: false, + max_retry: 0, + retry_count: 0, + log: '', + error_message: '', + result: '', + duration: 1, + payload: '', + triggered_by: 'admin:1', + created_at: '2026-07-19T10:00:00Z', + updated_at: '2026-07-19T10:00:01Z', + }); + + renderWithQuery(); + + await user.click(await screen.findByRole('button', { name: '同步并发布' })); + + await waitFor(() => { + expect(PagesService.syncSource).toHaveBeenCalledWith(9, {}); + expect(AdminTaskService.getTaskExecution).toHaveBeenCalledWith('42'); + }); + }); + + it('shows the actual project entry and no one-off URL upload tab', () => { + renderWithQuery( + , + ); + + expect(screen.getByText('dist/site/home.html')).toBeVisible(); + expect(screen.queryByText('从 URL 下载')).not.toBeInTheDocument(); + expect(screen.queryByText('部署包下载链接')).not.toBeInTheDocument(); + }); + + it('renders a deployment query failure instead of an empty history', async () => { + vi.mocked(PagesService.listDeployments).mockRejectedValue( + new Error('部署历史暂时不可用'), + ); + + renderWithQuery(); + + expect(await screen.findByText('部署历史暂时不可用')).toBeVisible(); + expect(screen.queryByText('暂无部署')).not.toBeInTheDocument(); + }); +}); diff --git a/internal/apps/admin/task/routers.go b/internal/apps/admin/task/routers.go index ed969a7d..594f1eb2 100644 --- a/internal/apps/admin/task/routers.go +++ b/internal/apps/admin/task/routers.go @@ -66,7 +66,7 @@ func DispatchTask(c *gin.Context) { return } - meta := task.GetTaskMeta(req.TaskType) + meta := getAdminTaskMeta(req.TaskType) if meta == nil { response.AbortBadRequest(c, InvalidTaskType) return @@ -256,7 +256,7 @@ func CreateSchedule(c *gin.Context) { } // 校验关联的异步任务类型 - meta := task.GetTaskMeta(req.TaskType) + meta := getAdminTaskMeta(req.TaskType) if meta == nil { response.AbortBadRequest(c, InvalidTaskType) return @@ -338,6 +338,10 @@ func UpdateSchedule(c *gin.Context) { response.AbortNotFound(c, ScheduleNotFound) return } + if existingMeta := task.GetTaskMeta(schedule.TaskType); existingMeta != nil && existingMeta.InternalOnly { + response.AbortBadRequest(c, InvalidTaskType) + return + } // 校验 Cron 表达式 if _, err := cron.ParseStandard(req.Cron); err != nil { @@ -346,7 +350,7 @@ func UpdateSchedule(c *gin.Context) { } // 校验关联的异步任务类型 - meta := task.GetTaskMeta(req.TaskType) + meta := getAdminTaskMeta(req.TaskType) if meta == nil { response.AbortBadRequest(c, InvalidTaskType) return @@ -382,6 +386,14 @@ func UpdateSchedule(c *gin.Context) { c.JSON(http.StatusOK, response.OK(schedule)) } +func getAdminTaskMeta(taskType string) *task.TaskMeta { + meta := task.GetTaskMeta(taskType) + if meta == nil || meta.InternalOnly { + return nil + } + return meta +} + // DeleteSchedule 删除定时任务 // @Summary 删除定时任务 // @Description 删除指定的定时任务配置，并触发调度器热加载，需要管理员权限 diff --git a/internal/apps/admin/task/routers_test.go b/internal/apps/admin/task/routers_test.go index db63ab8b..694861e7 100644 --- a/internal/apps/admin/task/routers_test.go +++ b/internal/apps/admin/task/routers_test.go @@ -29,6 +29,20 @@ import ( "github.com/Rain-kl/Wavelet/internal/common/response" ) +const ( + testInternalOnlyTaskType = "test_internal_only_admin" + testInternalOnlyAsynqTask = "test:internal_only_admin" +) + +func registerInternalOnlyTaskMeta() { + task.RegisterTaskMeta(task.TaskMeta{ + Type: testInternalOnlyTaskType, + AsynqTask: testInternalOnlyAsynqTask, + Name: "内部测试任务", + InternalOnly: true, + }) +} + func setupTaskTestEnvironment(t *testing.T) func() { _, mr, cleanup := testhelper.SetupTestEnvironment(t) bootstrap.RegisterTasks() @@ -61,12 +75,15 @@ func setupTestRouter(authUser *model.User) *gin.Engine { adminGroup.GET("/tasks/executions", ListTaskExecutions) adminGroup.GET("/tasks/executions/:id", GetTaskExecution) adminGroup.POST("/tasks/executions/:id/retry", RetryTask) + adminGroup.POST("/tasks/schedules", CreateSchedule) + adminGroup.PUT("/tasks/schedules/:id", UpdateSchedule) return r } func TestListTaskTypes(t *testing.T) { cleanup := setupTaskTestEnvironment(t) defer cleanup() + registerInternalOnlyTaskMeta() adminUser := &model.User{ID: 1001, Username: "admin", IsAdmin: true} router := setupTestRouter(adminUser) @@ -93,6 +110,9 @@ func TestListTaskTypes(t *testing.T) { foundCleanup := false foundWarmImageCache := false for _, m := range taskMetas { + if m.Type == testInternalOnlyTaskType { + t.Errorf("internal-only task type %s must not be listed", testInternalOnlyTaskType) + } if m.Type == uploadtask.TaskTypeSystemCleanup { foundCleanup = true } @@ -108,6 +128,119 @@ func TestListTaskTypes(t *testing.T) { } } +func TestInternalOnlyTaskAdminBoundaries(t *testing.T) { + cleanup := setupTaskTestEnvironment(t) + defer cleanup() + registerInternalOnlyTaskMeta() + + adminUser := &model.User{ID: 1001, Username: "admin", IsAdmin: true} + router := setupTestRouter(adminUser) + ctx := context.Background() + + t.Run("dispatch rejects internal-only task", func(t *testing.T) { + body, err := json.Marshal(DispatchTaskRequest{TaskType: testInternalOnlyTaskType}) + require.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/tasks/dispatch", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + var resp response.Any + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Equal(t, InvalidTaskType, resp.ErrorMsg) + }) + + t.Run("create schedule rejects internal-only task", func(t *testing.T) { + isActive := true + body, err := json.Marshal(CreateScheduleRequest{ + Name: "内部任务排程", + TaskType: testInternalOnlyTaskType, + Cron: "0 * * * *", + IsActive: &isActive, + }) + require.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/tasks/schedules", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + var resp response.Any + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Equal(t, InvalidTaskType, resp.ErrorMsg) + }) + + t.Run("update cannot convert existing internal schedule to public task", func(t *testing.T) { + schedule := &model.Schedule{ + Name: "系统内部排程", + TaskType: testInternalOnlyTaskType, + Cron: "0 * * * *", + IsActive: true, + } + require.NoError(t, model.CreateSchedule(ctx, schedule)) + isActive := false + body, err := json.Marshal(UpdateScheduleRequest{ + Name: "尝试修改内部排程", + TaskType: uploadtask.TaskTypeSystemCleanup, + Cron: "5 * * * *", + IsActive: &isActive, + }) + require.NoError(t, err) + req := httptest.NewRequest( + http.MethodPut, + fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID), + bytes.NewReader(body), + ) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + unchanged, err := model.GetScheduleByID(ctx, schedule.ID) + require.NoError(t, err) + assert.Equal(t, "系统内部排程", unchanged.Name) + assert.Equal(t, testInternalOnlyTaskType, unchanged.TaskType) + assert.True(t, unchanged.IsActive) + }) + + t.Run("update public schedule rejects internal-only target task", func(t *testing.T) { + schedule := &model.Schedule{ + Name: "公开排程", + TaskType: uploadtask.TaskTypeSystemCleanup, + Cron: "0 * * * *", + IsActive: true, + } + require.NoError(t, model.CreateSchedule(ctx, schedule)) + isActive := true + body, err := json.Marshal(UpdateScheduleRequest{ + Name: "尝试切入内部任务", + TaskType: testInternalOnlyTaskType, + Cron: "10 * * * *", + IsActive: &isActive, + }) + require.NoError(t, err) + req := httptest.NewRequest( + http.MethodPut, + fmt.Sprintf("/api/v1/admin/tasks/schedules/%d", schedule.ID), + bytes.NewReader(body), + ) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + unchanged, err := model.GetScheduleByID(ctx, schedule.ID) + require.NoError(t, err) + assert.Equal(t, "公开排程", unchanged.Name) + assert.Equal(t, uploadtask.TaskTypeSystemCleanup, unchanged.TaskType) + }) +} + func TestDispatchTask(t *testing.T) { cleanup := setupTaskTestEnvironment(t) defer cleanup() diff --git a/internal/apps/openflare/pages/download_url.go b/internal/apps/openflare/pages/download_url.go index 5959d7e7..09e0a93f 100644 --- a/internal/apps/openflare/pages/download_url.go +++ b/internal/apps/openflare/pages/download_url.go @@ -5,211 +5,38 @@ package pages import ( "context" - "crypto/sha256" - "crypto/tls" - "encoding/hex" "errors" - "fmt" - "io" - "mime" - "net" - "net/http" "net/url" - "os" - "path" - "path/filepath" "strings" - "time" "github.com/Rain-kl/Wavelet/pkg/pagesarchive" ) -const ( - pagesURLDownloadTimeout = 10 * time.Minute - pagesURLMaxRedirects = 5 - pagesMagicSniffBytes = 512 - pagesURLDialTimeout = 30 * time.Second - pagesURLTLSHandshake = 15 * time.Second - pagesBrowserUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" - pagesBrowserAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7" - pagesBrowserAcceptLang = "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7" - pagesBrowserSecCHUA = `"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"` - pagesBrowserSecCHUAMobile = "?0" - pagesBrowserSecCHUAPlat = `"Windows"` -) - -// downloadPagesPackageFromURL fetches a remote archive with browser-like headers -// and writes it to a temp file. Allows private/LAN hosts and insecure TLS certs -// (self-signed / internal CA) so operators can pull from internal artifact stores. -func downloadPagesPackageFromURL(ctx context.Context, rawURL string, maxPackageBytes int64) (tempPath string, checksum string, size int64, format pagesarchive.Format, fileName string, err error) { - parsed, err := parseAndValidatePagesDownloadURL(rawURL) - if err != nil { +// downloadPagesPackageFromURL is the deprecated one-shot URL adapter. It uses +// the same bounded downloader as persisted sources, with the legacy trusted +// network policy that permits operator-managed internal artifact services. +func downloadPagesPackageFromURL( + ctx context.Context, + rawURL string, + maxPackageBytes int64, +) (tempPath string, checksum string, size int64, format pagesarchive.Format, fileName string, err error) { + if _, err := parseAndValidatePagesDownloadURL(rawURL); err != nil { return "", "", 0, "", "", err } - - resp, err := doBrowserDownload(ctx, parsed) + candidate, err := FetchRemoteSource(ctx, RemoteSourceRequest{ + URL: strings.TrimSpace(rawURL), + NetworkPolicy: RemoteNetworkPolicyTrustedInternal, + MaxPackageBytes: maxPackageBytes, + }) if err != nil { - return "", "", 0, "", "", err - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return "", "", 0, "", "", fmt.Errorf("%s: HTTP %d", errPagesPackageURLDownloadFailed, resp.StatusCode) - } - if resp.ContentLength > 0 && resp.ContentLength > maxPackageBytes { - return "", "", 0, "", "", errors.New(errPagesPackageURLTooLarge) - } - - fileName = fileNameFromDownload(resp, parsed) - format, _ = pagesarchive.DetectFormatFromName(fileName) - - tempPath, checksum, size, err = writeLimitedPackageTemp(resp.Body, format, maxPackageBytes) - if err != nil { - return "", "", 0, "", "", err - } - format, fileName, err = ensurePackageFormat(tempPath, format, fileName) - if err != nil { - _ = os.Remove(tempPath) - return "", "", 0, "", "", err - } - return tempPath, checksum, size, format, fileName, nil -} - -func newPagesURLDownloadClient() *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: pagesURLDialTimeout, - KeepAlive: pagesURLDialTimeout, - }).DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 32, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: pagesURLTLSHandshake, - ExpectContinueTimeout: time.Second, - // Allow self-signed / internal certificates for artifact hosts. - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // intentional for internal/dev artifact URLs - } - client := &http.Client{ - Timeout: pagesURLDownloadTimeout, - Transport: transport, - } - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - if len(via) >= pagesURLMaxRedirects { - return errors.New(errPagesPackageURLDownloadFailed) + if strings.Contains(err.Error(), errPagesSourceRemoteURLInvalid) { + return "", "", 0, "", "", errors.New(errPagesPackageURLInvalid) } - if err := validatePagesDownloadURLValue(req.URL); err != nil { - return err - } - applyBrowserDownloadHeaders(req, via[0].URL.String()) - return nil - } - return client -} - -func doBrowserDownload(ctx context.Context, parsed *url.URL) (*http.Response, error) { - client := newPagesURLDownloadClient() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) - if err != nil { - return nil, errors.New(errPagesPackageURLInvalid) - } - applyBrowserDownloadHeaders(req, "") - resp, err := client.Do(req) //nolint:gosec // scheme validated; private hosts and insecure TLS intentionally allowed - if err != nil { - return nil, fmt.Errorf("%s: %w", errPagesPackageURLDownloadFailed, err) - } - return resp, nil -} - -func writeLimitedPackageTemp(body io.Reader, format pagesarchive.Format, maxPackageBytes int64) (tempPath, checksum string, size int64, err error) { - temp, err := os.CreateTemp("", "openflare-pages-url-*."+safeTempSuffixOrBin(format)) - if err != nil { - return "", "", 0, err - } - tempPath = temp.Name() - defer func() { - _ = temp.Close() - if err != nil { - _ = os.Remove(tempPath) - } - }() - - hash := sha256.New() - written, copyErr := io.Copy(io.MultiWriter(temp, hash), io.LimitReader(body, maxPackageBytes+1)) - if copyErr != nil { - err = fmt.Errorf("%s: %w", errPagesPackageURLDownloadFailed, copyErr) - return "", "", 0, err - } - if written > maxPackageBytes { - err = errors.New(errPagesPackageURLTooLarge) - return "", "", 0, err - } - if written == 0 { - err = errors.New(errPagesPackageEmpty) - return "", "", 0, err - } - return tempPath, hex.EncodeToString(hash.Sum(nil)), written, nil -} - -func ensurePackageFormat(tempPath string, format pagesarchive.Format, fileName string) (pagesarchive.Format, string, error) { - if format != "" { - return format, fileName, nil - } - detected, ok := sniffPackageFormat(tempPath) - if !ok { - return "", fileName, errors.New(errPagesPackageUnsupported) - } - if !strings.Contains(strings.ToLower(fileName), ".") { - fileName = fileName + "." + pagesarchive.Extension(detected) - } - return detected, fileName, nil -} - -func sniffPackageFormat(tempPath string) (pagesarchive.Format, bool) { - file, err := os.Open(tempPath) //nolint:gosec // temp path created by us - if err != nil { - return "", false - } - defer func() { _ = file.Close() }() - head := make([]byte, pagesMagicSniffBytes) - n, _ := io.ReadFull(file, head) - if n <= 0 { - return "", false - } - return pagesarchive.DetectFormatFromBytes(head[:n]) -} - -func safeTempSuffixOrBin(format pagesarchive.Format) string { - if format == "" { - return "bin" - } - return safeTempSuffix(format) -} - -func applyBrowserDownloadHeaders(req *http.Request, referer string) { - if req == nil { - return - } - req.Header.Set("User-Agent", pagesBrowserUserAgent) - req.Header.Set("Accept", pagesBrowserAccept) - req.Header.Set("Accept-Language", pagesBrowserAcceptLang) - req.Header.Set("Cache-Control", "no-cache") - req.Header.Set("Pragma", "no-cache") - req.Header.Set("Upgrade-Insecure-Requests", "1") - req.Header.Set("Sec-Fetch-Dest", "document") - req.Header.Set("Sec-Fetch-Mode", "navigate") - req.Header.Set("Sec-Fetch-Site", "none") - req.Header.Set("Sec-Fetch-User", "?1") - req.Header.Set("Sec-Ch-Ua", pagesBrowserSecCHUA) - req.Header.Set("Sec-Ch-Ua-Mobile", pagesBrowserSecCHUAMobile) - req.Header.Set("Sec-Ch-Ua-Platform", pagesBrowserSecCHUAPlat) - if referer != "" { - req.Header.Set("Referer", referer) - req.Header.Set("Sec-Fetch-Site", "cross-site") - return - } - if req.URL != nil { - req.Header.Set("Referer", req.URL.Scheme+"://"+req.URL.Host+"/") + return "", "", 0, "", "", err } + // Ownership transfers to the existing one-shot caller, which removes the + // temporary file after the candidate deployment has been created. + return candidate.TempPath, candidate.Checksum, candidate.PackageSize, candidate.Format, candidate.SafeLabel, nil } func parseAndValidatePagesDownloadURL(raw string) (*url.URL, error) { @@ -218,57 +45,12 @@ func parseAndValidatePagesDownloadURL(raw string) (*url.URL, error) { return nil, errors.New(errPagesPackageURLRequired) } parsed, err := url.Parse(value) - if err != nil { + if err != nil || parsed.User != nil || parsed.Fragment != "" || parsed.Opaque != "" { return nil, errors.New(errPagesPackageURLInvalid) } - if err := validatePagesDownloadURLValue(parsed); err != nil { - return nil, err + scheme := strings.ToLower(strings.TrimSpace(parsed.Scheme)) + if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(parsed.Hostname()) == "" { + return nil, errors.New(errPagesPackageURLInvalid) } return parsed, nil } - -func validatePagesDownloadURLValue(parsed *url.URL) error { - if parsed == nil { - return errors.New(errPagesPackageURLInvalid) - } - scheme := strings.ToLower(strings.TrimSpace(parsed.Scheme)) - if scheme != "http" && scheme != "https" { - return errors.New(errPagesPackageURLInvalid) - } - if strings.TrimSpace(parsed.Hostname()) == "" { - return errors.New(errPagesPackageURLInvalid) - } - return nil -} - -func fileNameFromDownload(resp *http.Response, parsed *url.URL) string { - if name := fileNameFromContentDisposition(resp); name != "" { - return name - } - if parsed != nil { - base := path.Base(parsed.Path) - if base != "" && base != "." && base != "/" { - return base - } - } - return "package.bin" -} - -func fileNameFromContentDisposition(resp *http.Response) string { - if resp == nil { - return "" - } - cd := resp.Header.Get("Content-Disposition") - if cd == "" { - return "" - } - _, params, err := mime.ParseMediaType(cd) - if err != nil { - return "" - } - name := strings.TrimSpace(params["filename"]) - if name == "" { - return "" - } - return path.Base(filepath.ToSlash(name)) -} diff --git a/internal/apps/openflare/pages/download_url_test.go b/internal/apps/openflare/pages/download_url_test.go index 918cf962..fc46e65f 100644 --- a/internal/apps/openflare/pages/download_url_test.go +++ b/internal/apps/openflare/pages/download_url_test.go @@ -10,7 +10,6 @@ import ( "net/http" "net/http/httptest" "os" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -48,10 +47,10 @@ func TestDownloadPagesPackageFromURLAllowsPrivateHost(t *testing.T) { require.NoError(t, zw.Close()) zipBytes := body.Bytes() - var sawBrowserUA bool + var sawProviderUA bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") { - sawBrowserUA = true + if r.Header.Get("User-Agent") == remoteSourceUserAgent { + sawProviderUA = true } w.Header().Set("Content-Disposition", `attachment; filename="remote-site.zip"`) w.Header().Set("Content-Type", "application/zip") @@ -59,12 +58,6 @@ func TestDownloadPagesPackageFromURLAllowsPrivateHost(t *testing.T) { })) t.Cleanup(server.Close) - req, err := http.NewRequest(http.MethodGet, server.URL+"/pkg.zip", nil) - require.NoError(t, err) - applyBrowserDownloadHeaders(req, "") - assert.Contains(t, req.Header.Get("User-Agent"), "Mozilla") - assert.Contains(t, req.Header.Get("Sec-Fetch-Mode"), "navigate") - tempPath, checksum, size, format, fileName, err := downloadPagesPackageFromURL( context.Background(), server.URL+"/pkg.zip", @@ -72,11 +65,11 @@ func TestDownloadPagesPackageFromURLAllowsPrivateHost(t *testing.T) { ) require.NoError(t, err) t.Cleanup(func() { _ = os.Remove(tempPath) }) - assert.True(t, sawBrowserUA) + assert.True(t, sawProviderUA) assert.NotEmpty(t, checksum) assert.Positive(t, size) assert.Equal(t, "zip", string(format)) - assert.Equal(t, "remote-site.zip", fileName) + assert.Equal(t, "pkg.zip", fileName) } func TestUploadDeploymentFromURLPrivateHost(t *testing.T) { diff --git a/internal/apps/openflare/pages/errs.go b/internal/apps/openflare/pages/errs.go index 602825a1..15b2c270 100644 --- a/internal/apps/openflare/pages/errs.go +++ b/internal/apps/openflare/pages/errs.go @@ -36,4 +36,20 @@ const ( errPagesInvalidSnapshotFormat = "配置快照格式无效" errPagesActorMissing = "无法识别当前用户" errPagesEntryFileMissing = "当前激活部署中不存在指定入口文件" + errPagesSourceNotFound = "pages 部署源不存在" + errPagesSourceTypeRequired = "请选择 pages 部署源类型" + errPagesSourceTypeUnsupported = "当前阶段仅支持远程地址部署源" + errPagesSourceRemoteFields = "远程地址来源不能包含 GitHub 或自动更新配置" + errPagesSourceRemoteURLRequired = "请提供远程部署包地址" + errPagesSourceRemoteURLMode = "remote_url_set 与 remote_url 参数不匹配" + errPagesSourceRemoteURLInvalid = "远程部署包地址无效，仅支持不含用户信息和片段的 http/https 地址" + errPagesSourceNetworkPolicy = "远程地址网络策略仅支持 public 或 trusted_internal" + errPagesSourceCheckUnsupported = "远程地址来源不支持检查更新，请使用立即同步" + errPagesSourceActionBusy = "pages 部署源任务正在执行" + errPagesSourceActionInvalid = "pages 部署源任务参数无效" + errPagesSourceActionStale = "pages 部署源配置已变化，本次任务已跳过" + errPagesSourceLeaseLost = "pages 部署源任务执行权已失效" + errPagesSourceSyncFailed = "pages 部署源同步失败" + errPagesSourceTaskDispatchFailed = "pages 部署源任务入队失败" + errPagesSourceInternal = "pages 部署源操作失败，请稍后重试" ) diff --git a/internal/apps/openflare/pages/helpers.go b/internal/apps/openflare/pages/helpers.go index 90db9996..8d1b2f69 100644 --- a/internal/apps/openflare/pages/helpers.go +++ b/internal/apps/openflare/pages/helpers.go @@ -38,6 +38,7 @@ const ( pagesIngestMarkerKey = "pages_ingest_marker" pagesIngestMarkerV2 = "pages_deployment_v2" pagesProjectIDMetadataKey = "pages_project_id" + pagesSourceIDMetadataKey = "pages_source_id" pagesMaxPathLength = 512 bytesPerMiB = 1024 * 1024 pagesExtractedSizeMultiplier = 4 @@ -263,10 +264,29 @@ func ingestPagesDeploymentPackage( projectID uint, fileName string, format pagesarchive.Format, +) (upload.IngestResult, error) { + return ingestPagesDeploymentPackageWithSource(ctx, localPath, checksum, projectID, 0, fileName, format) +} + +func ingestPagesDeploymentPackageWithSource( + ctx context.Context, + localPath string, + checksum string, + projectID uint, + sourceID uint, + fileName string, + format pagesarchive.Format, ) (upload.IngestResult, error) { systemUser := repository.GetSystemUser(ctx) accessMode := 0 extension := pagesarchive.NormalizeNameExtension(fileName, format) + extra := map[string]any{ + pagesIngestMarkerKey: pagesIngestMarkerV2, + pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10), + } + if sourceID != 0 { + extra[pagesSourceIDMetadataKey] = strconv.FormatUint(uint64(sourceID), 10) + } return upload.IngestFromLocalPath(ctx, localPath, upload.IngestRequest{ UserID: systemUser.ID, FileName: fileName, @@ -278,10 +298,7 @@ func ingestPagesDeploymentPackage( SkipExtensionCheck: true, Policy: upload.PolicyDedupNewRecord, Metadata: model.UploadMetadata{ - Extra: map[string]any{ - pagesIngestMarkerKey: pagesIngestMarkerV2, - pagesProjectIDMetadataKey: strconv.FormatUint(uint64(projectID), 10), - }, + Extra: extra, }, }) } diff --git a/internal/apps/openflare/pages/logics.go b/internal/apps/openflare/pages/logics.go index 622564bc..3a0f0a49 100644 --- a/internal/apps/openflare/pages/logics.go +++ b/internal/apps/openflare/pages/logics.go @@ -60,6 +60,9 @@ type DeploymentView struct { FileCount int `json:"file_count"` TotalSize int64 `json:"total_size"` CreatedBy string `json:"created_by"` + SourceType string `json:"source_type"` + SourceLabel string `json:"source_label"` + TriggerType string `json:"trigger_type"` CreatedAt time.Time `json:"created_at"` ActivatedAt *time.Time `json:"activated_at"` } @@ -151,13 +154,14 @@ func UpdateProject(ctx context.Context, id uint, input Input) (*View, error) { if err != nil { return err } - if (existing.RootDir != project.RootDir || existing.EntryFile != project.EntryFile) && + contentConfigChanged := existing.RootDir != project.RootDir || existing.EntryFile != project.EntryFile + if contentConfigChanged && existing.ActiveDeploymentID != nil && *existing.ActiveDeploymentID != 0 { if err := ensureDeploymentEntry(tx, *existing.ActiveDeploymentID, project.RootDir, project.EntryFile); err != nil { return err } } - return tx.Model(&existing).Updates(map[string]any{ + updates := map[string]any{ "name": project.Name, "slug": project.Slug, "description": project.Description, @@ -170,7 +174,23 @@ func UpdateProject(ctx context.Context, id uint, input Input) (*View, error) { "api_proxy_rewrite": project.APIProxyRewrite, "root_dir": project.RootDir, "entry_file": project.EntryFile, - }).Error + } + if contentConfigChanged { + updates["content_config_version"] = existing.ContentConfigVersion + 1 + var source model.PagesProjectSource + sourceErr := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("project_id = ?", existing.ID). + First(&source).Error + if sourceErr != nil && !errors.Is(sourceErr, gorm.ErrRecordNotFound) { + return sourceErr + } + if sourceErr == nil { + if err := fenceAndNormalizeRuntime(tx, source.ID); err != nil { + return err + } + } + } + return tx.Model(&existing).Updates(updates).Error }) if err != nil { if isUniqueConstraintError(err) { @@ -221,6 +241,28 @@ func DeleteProject(ctx context.Context, id uint) error { return errors.New(errPagesDeleteReferenced) } } + var source model.PagesProjectSource + sourceErr := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("project_id = ?", project.ID). + First(&source).Error + if sourceErr != nil && !errors.Is(sourceErr, gorm.ErrRecordNotFound) { + return sourceErr + } + if sourceErr == nil { + var runtime model.PagesProjectSourceRuntime + runtimeErr := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("source_id = ?", source.ID). + First(&runtime).Error + if runtimeErr != nil && !errors.Is(runtimeErr, gorm.ErrRecordNotFound) { + return runtimeErr + } + if err := tx.Where("source_id = ?", source.ID).Delete(&model.PagesProjectSourceRuntime{}).Error; err != nil { + return err + } + if err := tx.Delete(&source).Error; err != nil { + return err + } + } if err := tx.Where("project_id = ?", project.ID).Find(&deployments).Error; err != nil { return err } @@ -233,7 +275,7 @@ func DeleteProject(ctx context.Context, id uint) error { if err := tx.Where("project_id = ?", project.ID).Delete(&model.PagesDeployment{}).Error; err != nil { return err } - if err := tx.Delete(project).Error; err != nil { + if err := tx.Delete(&lockedProject).Error; err != nil { return err } return nil @@ -301,7 +343,18 @@ func UploadDeployment(ctx context.Context, projectID uint, fileHeader *multipart return nil, err } defer func() { _ = os.Remove(tempPath) }() - return createDeploymentFromTempPackage(ctx, project, tempPath, checksum, format, fileHeader.Filename, createdBy, limits) + return createDeploymentFromTempPackage( + ctx, + project, + tempPath, + checksum, + format, + fileHeader.Filename, + createdBy, + "manual_upload", + "manual_upload", + limits, + ) } // UploadFromURLInput is the request body for downloading a deployment package from a remote URL. @@ -321,7 +374,18 @@ func UploadDeploymentFromURL(ctx context.Context, projectID uint, rawURL string, return nil, err } defer func() { _ = os.Remove(tempPath) }() - return createDeploymentFromTempPackage(ctx, project, tempPath, checksum, format, fileName, createdBy, limits) + return createDeploymentFromTempPackage( + ctx, + project, + tempPath, + checksum, + format, + fileName, + createdBy, + "manual_url", + "manual_url", + limits, + ) } func createDeploymentFromTempPackage( @@ -332,6 +396,8 @@ func createDeploymentFromTempPackage( format pagesarchive.Format, fileName string, createdBy string, + sourceType string, + triggerType string, limits pagesLimits, ) (*DeploymentView, error) { if project == nil { @@ -377,6 +443,15 @@ func createDeploymentFromTempPackage( if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&lockedProject, project.ID).Error; err != nil { return err } + var uploadRecord model.Upload + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("id = ?", ingestResult.Upload.ID). + First(&uploadRecord).Error; err != nil { + return errors.New(errPagesPackageUploadMissing) + } + if uploadRecord.Status != model.UploadStatusUsed || uploadRecord.Type != upload.ReservedPagesDeploymentType { + return errors.New(errPagesPackageUploadMissing) + } var maxNumber int if err := tx.Model(&model.PagesDeployment{}). Where("project_id = ?", project.ID). @@ -393,6 +468,9 @@ func createDeploymentFromTempPackage( FileCount: manifest.FileCount, TotalSize: manifest.TotalSize, CreatedBy: strings.TrimSpace(createdBy), + SourceType: sourceType, + SourceLabel: safeRemoteSourceLabel(fileName), + TriggerType: triggerType, } if err := tx.Create(deployment).Error; err != nil { return err @@ -582,72 +660,27 @@ func selectDeploymentsToPrune( // ActivateDeployment 激活 Pages 部署。 func ActivateDeployment(ctx context.Context, projectID uint, deploymentID uint) (*View, error) { - project, err := model.GetPagesProjectByID(ctx, projectID) + return ActivateDeploymentAs(ctx, projectID, deploymentID, "system:pages-manual-activation") +} + +// ActivateDeploymentAs activates a historical deployment and fences any +// configured source when the active deployment actually changes. +func ActivateDeploymentAs(ctx context.Context, projectID uint, deploymentID uint, actor string) (*View, error) { + if err := ensureActivationDeploymentUpload(ctx, projectID, deploymentID); err != nil { + return nil, err + } + audit, err := activateDeploymentTransaction(ctx, projectID, deploymentID, time.Now()) if err != nil { return nil, err } - deployment, err := model.GetPagesDeploymentByID(ctx, deploymentID) - if err != nil { - return nil, err - } - if deployment.ProjectID != project.ID { - return nil, errors.New(errPagesDeploymentMismatch) - } - if deployment.UploadID == 0 { - if err = ensureDeploymentUploadRecord(ctx, deployment); err != nil { - return nil, err - } - } - now := time.Now() - if err = db.DB(ctx).Transaction(func(tx *gorm.DB) error { - var project model.PagesProject - if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil { - return err - } - var deployment model.PagesDeployment - if err := tx.First(&deployment, deploymentID).Error; err != nil { - return err - } - if deployment.ProjectID != project.ID { - return errors.New(errPagesDeploymentMismatch) - } - rootDir, err := validateAndNormalizePagesRootDir(project.RootDir) - if err != nil { - return err - } - entryFile, err := validateAndNormalizePagesEntryFile(project.EntryFile) - if err != nil { - return err - } - if err := ensureDeploymentEntry(tx, deployment.ID, rootDir, entryFile); err != nil { - return err - } - var uploadRecord model.Upload - if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). - Where("id = ?", deployment.UploadID). - First(&uploadRecord).Error; err != nil { - return errors.New(errPagesPackageUploadMissing) - } - if uploadRecord.Status != model.UploadStatusUsed || uploadRecord.Type != upload.ReservedPagesDeploymentType { - return errors.New(errPagesPackageUploadMissing) - } - if err := tx.Model(&model.PagesDeployment{}). - Where("project_id = ?", project.ID). - Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil { - return err - } - if err := tx.Model(&deployment).Updates(map[string]any{ - "status": model.PagesDeploymentStatusActive, - "activated_at": &now, - }).Error; err != nil { - return err - } - return tx.Model(&project).Updates(map[string]any{ - "active_deployment_id": deployment.ID, - }).Error - }); err != nil { - return nil, err + if audit.Noop { + return GetProject(ctx, projectID) } + logger.InfoF(ctx, + "[Pages] manual activation: actor=%s project_id=%d old_deployment_id=%d new_deployment_id=%d source_type=%s source_identity=%s auto_disabled=%t", + strings.TrimSpace(actor), projectID, audit.OldDeploymentID, deploymentID, + audit.SourceType, audit.SourceIdentity, audit.AutoDisabled, + ) limits := resolvePagesLimits(ctx) if pruneErr := pruneProjectDeploymentHistory(ctx, projectID, limits.HistoryCount, 0); pruneErr != nil { logger.ErrorF(ctx, @@ -658,6 +691,179 @@ func ActivateDeployment(ctx context.Context, projectID uint, deploymentID uint) return GetProject(ctx, projectID) } +type deploymentActivationAudit struct { + OldDeploymentID uint + SourceType string + SourceIdentity string + AutoDisabled bool + Noop bool +} + +type deploymentActivationSource struct { + Source *model.PagesProjectSource + Runtime *model.PagesProjectSourceRuntime +} + +func ensureActivationDeploymentUpload(ctx context.Context, projectID uint, deploymentID uint) error { + deployment, err := model.GetPagesDeploymentByID(ctx, deploymentID) + if err != nil { + return err + } + if deployment.ProjectID != projectID { + return errors.New(errPagesDeploymentMismatch) + } + if deployment.UploadID != 0 { + return nil + } + return ensureDeploymentUploadRecord(ctx, deployment) +} + +func activateDeploymentTransaction( + ctx context.Context, + projectID uint, + deploymentID uint, + now time.Time, +) (deploymentActivationAudit, error) { + audit := deploymentActivationAudit{} + err := db.DB(ctx).Transaction(func(tx *gorm.DB) error { + var project model.PagesProject + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil { + return err + } + if project.ActiveDeploymentID != nil { + audit.OldDeploymentID = *project.ActiveDeploymentID + } + if audit.OldDeploymentID == deploymentID { + audit.Noop = true + return nil + } + + sourceState, err := lockDeploymentActivationSource(tx, project.ID) + if err != nil { + return err + } + deployment, err := loadDeploymentActivationTarget(tx, &project, deploymentID) + if err != nil { + return err + } + if err := fenceDeploymentActivationSource(tx, sourceState, deployment, &audit); err != nil { + return err + } + return switchActiveDeploymentTx(tx, &project, deployment, now) + }) + return audit, err +} + +func lockDeploymentActivationSource(tx *gorm.DB, projectID uint) (*deploymentActivationSource, error) { + var source model.PagesProjectSource + err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("project_id = ?", projectID). + First(&source).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + var runtime model.PagesProjectSourceRuntime + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("source_id = ?", source.ID). + First(&runtime).Error; err != nil { + return nil, err + } + return &deploymentActivationSource{Source: &source, Runtime: &runtime}, nil +} + +func loadDeploymentActivationTarget( + tx *gorm.DB, + project *model.PagesProject, + deploymentID uint, +) (*model.PagesDeployment, error) { + var deployment model.PagesDeployment + if err := tx.First(&deployment, deploymentID).Error; err != nil { + return nil, err + } + if deployment.ProjectID != project.ID { + return nil, errors.New(errPagesDeploymentMismatch) + } + rootDir, err := validateAndNormalizePagesRootDir(project.RootDir) + if err != nil { + return nil, err + } + entryFile, err := validateAndNormalizePagesEntryFile(project.EntryFile) + if err != nil { + return nil, err + } + if err := ensureDeploymentEntry(tx, deployment.ID, rootDir, entryFile); err != nil { + return nil, err + } + var uploadRecord model.Upload + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("id = ?", deployment.UploadID). + First(&uploadRecord).Error; err != nil { + return nil, errors.New(errPagesPackageUploadMissing) + } + if uploadRecord.Status != model.UploadStatusUsed || uploadRecord.Type != upload.ReservedPagesDeploymentType { + return nil, errors.New(errPagesPackageUploadMissing) + } + return &deployment, nil +} + +func fenceDeploymentActivationSource( + tx *gorm.DB, + state *deploymentActivationSource, + deployment *model.PagesDeployment, + audit *deploymentActivationAudit, +) error { + if state == nil { + return nil + } + audit.SourceType = state.Source.SourceType + audit.SourceIdentity = state.Source.SourceIdentity + audit.AutoDisabled = state.Source.AutoUpdateEnabled + if err := tx.Model(state.Source).Updates(map[string]any{ + "config_version": state.Source.ConfigVersion + 1, + "auto_update_enabled": false, + }).Error; err != nil { + return err + } + if deployment.SourceIdentity != nil && *deployment.SourceIdentity == state.Source.SourceIdentity && + deployment.SourceRevision != nil { + state.Runtime.LastAppliedRevision = *deployment.SourceRevision + state.Runtime.LastAppliedDetail = deployment.SourceMeta + } else { + state.Runtime.LastAppliedRevision = "" + state.Runtime.LastAppliedDetail = "" + } + return tx.Model(state.Runtime).Updates(map[string]any{ + "last_applied_revision": state.Runtime.LastAppliedRevision, + "last_applied_detail": state.Runtime.LastAppliedDetail, + sourceRuntimeColumnSyncStatus: normalizedSourceRuntimeStatus(state.Runtime), + sourceRuntimeColumnLeaseToken: "", + sourceRuntimeColumnLeaseExpiresAt: nil, + }).Error +} + +func switchActiveDeploymentTx( + tx *gorm.DB, + project *model.PagesProject, + deployment *model.PagesDeployment, + now time.Time, +) error { + if err := tx.Model(&model.PagesDeployment{}). + Where("project_id = ?", project.ID). + Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil { + return err + } + if err := tx.Model(deployment).Updates(map[string]any{ + "status": model.PagesDeploymentStatusActive, + "activated_at": &now, + }).Error; err != nil { + return err + } + return tx.Model(project).Update("active_deployment_id", deployment.ID).Error +} + // GetDeploymentPackageHash returns the upload SHA-256 hash of the deployment package. // Prefer GetProjectLatestPackageHash for Agent latest-pointer pulls. func GetDeploymentPackageHash(ctx context.Context, deploymentID uint) (string, error) { @@ -1104,7 +1310,7 @@ func buildProject(existing *model.PagesProject, input Input) (*model.PagesProjec return nil, errors.New(errPagesAPIProxyPassRequired) } parsedURL, err := url.Parse(apiProxyPass) - if err != nil || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") || parsedURL.Host == "" { + if err != nil || (parsedURL.Scheme != remoteSourceSchemeHTTP && parsedURL.Scheme != remoteSourceSchemeHTTPS) || parsedURL.Host == "" { return nil, errors.New(errPagesAPIProxyPassInvalid) } } @@ -1177,6 +1383,9 @@ func buildDeploymentView(deployment *model.PagesDeployment) DeploymentView { FileCount: deployment.FileCount, TotalSize: deployment.TotalSize, CreatedBy: deployment.CreatedBy, + SourceType: deployment.SourceType, + SourceLabel: deployment.SourceLabel, + TriggerType: deployment.TriggerType, CreatedAt: deployment.CreatedAt, ActivatedAt: deployment.ActivatedAt, } diff --git a/internal/apps/openflare/pages/logics_test.go b/internal/apps/openflare/pages/logics_test.go index 230f3352..346aba55 100644 --- a/internal/apps/openflare/pages/logics_test.go +++ b/internal/apps/openflare/pages/logics_test.go @@ -43,6 +43,8 @@ func setupPagesTestDB(t *testing.T) func() { &model.PagesProject{}, &model.PagesDeployment{}, &model.PagesDeploymentFile{}, + &model.PagesProjectSource{}, + &model.PagesProjectSourceRuntime{}, &model.ConfigVersion{}, &model.SystemConfig{}, )) diff --git a/internal/apps/openflare/pages/routers.go b/internal/apps/openflare/pages/routers.go index 6767f478..4ad4c862 100644 --- a/internal/apps/openflare/pages/routers.go +++ b/internal/apps/openflare/pages/routers.go @@ -4,7 +4,10 @@ package pages import ( + "encoding/json" + "errors" "fmt" + "io" "net/http" "strconv" @@ -12,7 +15,9 @@ import ( "github.com/Rain-kl/Wavelet/internal/apps/openflare/apiutil" "github.com/Rain-kl/Wavelet/internal/common/response" "github.com/Rain-kl/Wavelet/internal/model" + "github.com/Rain-kl/Wavelet/pkg/logger" "github.com/gin-gonic/gin" + "gorm.io/gorm" ) func handleLogicError(c *gin.Context, err error) bool { @@ -22,6 +27,53 @@ func handleLogicError(c *gin.Context, err error) bool { return apiutil.AbortNotFoundIfMissing(c, err, errPagesProjectNotFound) } +func handleSourceLogicError(c *gin.Context, err error) bool { + if err == nil { + return false + } + if errors.Is(err, gorm.ErrRecordNotFound) || err.Error() == errPagesSourceNotFound { + response.AbortNotFound(c, errPagesSourceNotFound) + return true + } + switch err.Error() { + case errPagesSourceActionBusy: + response.AbortConflict(c, errPagesSourceActionBusy) + case errPagesSourceTypeRequired, + errPagesSourceTypeUnsupported, + errPagesSourceRemoteFields, + errPagesSourceRemoteURLRequired, + errPagesSourceRemoteURLMode, + errPagesSourceRemoteURLInvalid, + errPagesSourceNetworkPolicy, + errPagesSourceCheckUnsupported, + errPagesSourceActionInvalid: + response.AbortBadRequest(c, err.Error()) + case errPagesSourceTaskDispatchFailed: + response.AbortInternal(c, errPagesSourceInternal) + default: + logger.ErrorF(c.Request.Context(), "[PagesSource] API operation failed: error=%v", err) + response.AbortInternal(c, errPagesSourceInternal) + } + return true +} + +func decodeStrictJSON(c *gin.Context, target any, allowEmpty bool) bool { + decoder := json.NewDecoder(c.Request.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + if allowEmpty && errors.Is(err, io.EOF) { + return true + } + response.AbortBadRequest(c, errPagesSourceActionInvalid) + return false + } + if err := ensureJSONEOF(decoder); err != nil { + response.AbortBadRequest(c, errPagesSourceActionInvalid) + return false + } + return true +} + func deploymentIDParam(c *gin.Context) (uint, bool) { raw := c.Param("deployment_id") if raw == "" { @@ -174,6 +226,164 @@ func DeleteProjectHandler(c *gin.Context) { c.JSON(http.StatusOK, response.OKNil()) } +// GetSourceHandler 获取 Pages 项目的部署源。 +// @Summary 获取 Pages 部署源 +// @Description 返回脱敏后的项目部署源配置与运行状态，需要管理员权限 +// @Tags openflare-pages +// @Produce json +// @Security SessionCookie +// @Param id path int true "项目 ID" +// @Success 200 {object} response.Any{data=pages.SourceView} "部署源" +// @Failure 400 {object} response.Any "参数错误" +// @Failure 401 {object} response.Any "未登录" +// @Failure 404 {object} response.Any "项目或部署源不存在" +// @Failure 500 {object} response.Any "内部错误" +// @Router /api/v1/d/pages/{id}/source [get] +func GetSourceHandler(c *gin.Context) { + projectID, ok := apiutil.IDParam(c) + if !ok { + return + } + source, err := GetSource(c.Request.Context(), projectID) + if handleSourceLogicError(c, err) { + return + } + c.JSON(http.StatusOK, response.OK(source)) +} + +// UpdateSourceHandler 创建或更新 Pages 项目部署源。 +// @Summary 更新 Pages 部署源 +// @Description Phase 1 支持 Remote URL 来源；完整地址仅写入，不会在响应中返回 +// @Tags openflare-pages +// @Accept json +// @Produce json +// @Security SessionCookie +// @Param id path int true "项目 ID" +// @Param request body pages.SourceUpdateInput true "部署源配置" +// @Success 200 {object} response.Any{data=pages.SourceUpdateResult} "更新结果" +// @Failure 400 {object} response.Any "配置无效" +// @Failure 401 {object} response.Any "未登录" +// @Failure 404 {object} response.Any "项目不存在" +// @Failure 500 {object} response.Any "内部错误" +// @Router /api/v1/d/pages/{id}/source/update [post] +func UpdateSourceHandler(c *gin.Context) { + projectID, ok := apiutil.IDParam(c) + if !ok { + return + } + var input SourceUpdateInput + if !decodeStrictJSON(c, &input, false) { + return + } + result, err := UpdateSource(c.Request.Context(), projectID, input) + if handleSourceLogicError(c, err) { + return + } + c.JSON(http.StatusOK, response.OK(result)) +} + +// DeleteSourceHandler 将 Pages 项目切换回手动部署模式。 +// @Summary 删除 Pages 部署源 +// @Description 幂等删除持久部署源；已有部署历史与当前生产部署保持不变 +// @Tags openflare-pages +// @Produce json +// @Security SessionCookie +// @Param id path int true "项目 ID" +// @Success 200 {object} response.Any{data=pages.SourceView} "手动来源视图" +// @Failure 400 {object} response.Any "参数错误" +// @Failure 401 {object} response.Any "未登录" +// @Failure 404 {object} response.Any "项目不存在" +// @Failure 500 {object} response.Any "内部错误" +// @Router /api/v1/d/pages/{id}/source/delete [post] +func DeleteSourceHandler(c *gin.Context) { + projectID, ok := apiutil.IDParam(c) + if !ok { + return + } + source, err := DeleteSource(c.Request.Context(), projectID) + if handleSourceLogicError(c, err) { + return + } + c.JSON(http.StatusOK, response.OK(source)) +} + +// CheckSourceHandler 请求检查 Pages 部署源。 +// @Summary 检查 Pages 部署源 +// @Description Remote URL 来源不支持检查更新；该端点为后续 GitHub Release 来源保留统一动作契约 +// @Tags openflare-pages +// @Produce json +// @Security SessionCookie +// @Param id path int true "项目 ID" +// @Success 200 {object} response.Any{data=pages.SourceActionReceipt} "任务回执" +// @Failure 400 {object} response.Any "当前来源不支持检查" +// @Failure 401 {object} response.Any "未登录" +// @Failure 404 {object} response.Any "部署源不存在" +// @Failure 409 {object} response.Any "来源任务正在执行" +// @Failure 500 {object} response.Any "内部错误" +// @Router /api/v1/d/pages/{id}/source/check [post] +func CheckSourceHandler(c *gin.Context) { + projectID, ok := apiutil.IDParam(c) + if !ok { + return + } + actor, ok := currentPagesActor(c) + if !ok { + return + } + receipt, err := DispatchSourceAction(c.Request.Context(), projectID, sourceActionCheck, actor, "") + if handleSourceLogicError(c, err) { + return + } + c.JSON(http.StatusOK, response.OK(receipt)) +} + +// SourceSyncInput is the optional source sync action payload. +type SourceSyncInput struct { + ConfirmedRevision string `json:"confirmed_revision"` +} + +// SyncSourceHandler 请求同步并发布 Pages 部署源。 +// @Summary 同步并发布 Pages 部署源 +// @Description 异步下载、校验并原子激活来源部署包；空请求体与空 JSON 对象均有效 +// @Tags openflare-pages +// @Accept json +// @Produce json +// @Security SessionCookie +// @Param id path int true "项目 ID" +// @Param request body pages.SourceSyncInput false "同步参数" +// @Success 200 {object} response.Any{data=pages.SourceActionReceipt} "任务回执" +// @Failure 400 {object} response.Any "参数或来源类型无效" +// @Failure 401 {object} response.Any "未登录" +// @Failure 404 {object} response.Any "部署源不存在" +// @Failure 409 {object} response.Any "来源任务正在执行" +// @Failure 500 {object} response.Any "内部错误" +// @Router /api/v1/d/pages/{id}/source/sync [post] +func SyncSourceHandler(c *gin.Context) { + projectID, ok := apiutil.IDParam(c) + if !ok { + return + } + var input SourceSyncInput + if !decodeStrictJSON(c, &input, true) { + return + } + actor, ok := currentPagesActor(c) + if !ok { + return + } + receipt, err := DispatchSourceAction( + c.Request.Context(), + projectID, + sourceActionSync, + actor, + input.ConfirmedRevision, + ) + if handleSourceLogicError(c, err) { + return + } + c.JSON(http.StatusOK, response.OK(receipt)) +} + // ListDeploymentsHandler 列出项目的全部部署。 // @Summary 列出 Pages 部署 // @Description 返回指定项目的全部部署记录，需要管理员权限 @@ -239,7 +449,8 @@ func UploadDeploymentHandler(c *gin.Context) { // UploadDeploymentFromURLHandler 从 URL 下载并创建 Pages 部署。 // @Summary 从 URL 导入 Pages 部署包 -// @Description 从用户提供的 HTTP(S) 链接下载部署包并创建部署记录；服务端使用浏览器伪装请求头拉取，允许内网地址与不安全 TLS 证书，需要管理员权限 +// @Description 已弃用的一次性 URL 导入；使用 trusted_internal 策略兼容内网与自签名证书，不创建持久部署源 +// @Deprecated // @Tags openflare-pages // @Accept json // @Produce json @@ -298,7 +509,11 @@ func ActivateDeploymentHandler(c *gin.Context) { if !ok { return } - project, err := ActivateDeployment(c.Request.Context(), projectID, deploymentID) + actor, ok := currentPagesActor(c) + if !ok { + return + } + project, err := ActivateDeploymentAs(c.Request.Context(), projectID, deploymentID, actor) if handleLogicError(c, err) { return } diff --git a/internal/apps/openflare/pages/routers_source_test.go b/internal/apps/openflare/pages/routers_source_test.go new file mode 100644 index 00000000..6bd96346 --- /dev/null +++ b/internal/apps/openflare/pages/routers_source_test.go @@ -0,0 +1,224 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Rain-kl/Wavelet/internal/apps/oauth" + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" + "github.com/Rain-kl/Wavelet/internal/task" + "github.com/Rain-kl/Wavelet/internal/testhelper" + "github.com/alicebob/miniredis/v2" + "github.com/gin-gonic/gin" + "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" +) + +type sourceHandlerEnvelope struct { + ErrorMsg string `json:"error_msg"` + Data json.RawMessage `json:"data"` +} + +func newPagesSourceTestRouter(userID uint64) *gin.Engine { + router := testhelper.NewTestGinEngine(func(ctx *gin.Context) { + oauth.SetToContext(ctx, oauth.UserObjKey, &model.User{ID: userID}) + ctx.Next() + }) + router.GET("/api/v1/d/pages/:id/source", GetSourceHandler) + router.POST("/api/v1/d/pages/:id/source/update", UpdateSourceHandler) + router.POST("/api/v1/d/pages/:id/source/delete", DeleteSourceHandler) + router.POST("/api/v1/d/pages/:id/source/check", CheckSourceHandler) + router.POST("/api/v1/d/pages/:id/source/sync", SyncSourceHandler) + return router +} + +func performPagesSourceRequest( + t *testing.T, + router http.Handler, + method string, + path string, + body []byte, +) (int, sourceHandlerEnvelope) { + t.Helper() + request := httptest.NewRequest(method, path, bytes.NewReader(body)) + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + var envelope sourceHandlerEnvelope + if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil { + t.Fatalf("json.Unmarshal(%s %s response %q) error = %v, want nil", method, path, recorder.Body.String(), err) + } + return recorder.Code, envelope +} + +func setupPagesSourceDispatchTest(t *testing.T) { + t.Helper() + miniRedis, err := miniredis.Run() + if err != nil { + t.Fatalf("miniredis.Run() error = %v, want nil", err) + } + redisClient := redis.NewClient(&redis.Options{Addr: miniRedis.Addr()}) + asynqClient := asynq.NewClient(asynq.RedisClientOpt{Addr: miniRedis.Addr()}) + previousRedis := db.Redis + previousAsynqClient := task.AsynqClient + db.Redis = redisClient + task.AsynqClient = asynqClient + task.RegisterTaskMeta(PagesSourceActionMeta) + t.Cleanup(func() { + _ = asynqClient.Close() + _ = redisClient.Close() + miniRedis.Close() + task.AsynqClient = previousAsynqClient + db.Redis = previousRedis + }) +} + +func TestPagesSourceHandlersReturnStableActionErrors(t *testing.T) { + ctx := setupPagesSourceTest(t) + router := newPagesSourceTestRouter(42) + + manualProject := mustCreatePagesSourceProject(t, ctx, "handler-no-source") + code, envelope := performPagesSourceRequest( + t, + router, + http.MethodPost, + fmt.Sprintf("/api/v1/d/pages/%d/source/sync", manualProject.ID), + nil, + ) + if got, want := code, http.StatusNotFound; got != want { + t.Errorf("POST source/sync without source status = %d, want %d", got, want) + } + if got, want := envelope.ErrorMsg, errPagesSourceNotFound; got != want { + t.Errorf("POST source/sync without source error = %q, want %q", got, want) + } + + remoteProject := mustCreatePagesSourceProject(t, ctx, "handler-check") + _, _ = mustConfigureRemoteSource( + t, + ctx, + remoteProject.ID, + "https://example.com/site.zip?token=handler-secret", + RemoteNetworkPolicyPublic, + ) + code, envelope = performPagesSourceRequest( + t, + router, + http.MethodPost, + fmt.Sprintf("/api/v1/d/pages/%d/source/check", remoteProject.ID), + nil, + ) + if got, want := code, http.StatusBadRequest; got != want { + t.Errorf("POST remote source/check status = %d, want %d", got, want) + } + if got, want := envelope.ErrorMsg, errPagesSourceCheckUnsupported; got != want { + t.Errorf("POST remote source/check error = %q, want %q", got, want) + } + if strings.Contains(string(envelope.Data), "handler-secret") || strings.Contains(envelope.ErrorMsg, "handler-secret") { + t.Errorf("POST remote source/check response = %+v, want no URL secret", envelope) + } + + busyProject := mustCreatePagesSourceProject(t, ctx, "handler-busy") + busySource, _ := mustConfigureRemoteSource( + t, + ctx, + busyProject.ID, + "https://example.com/site.zip", + RemoteNetworkPolicyPublic, + ) + future := time.Now().Add(time.Minute) + if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ?", busySource.ID). + Updates(map[string]any{ + "sync_status": pagesSourceStatusSyncing, + "lease_token": "busy-owner", + "lease_expires_at": &future, + }).Error; err != nil { + t.Fatalf("seed busy source runtime error = %v, want nil", err) + } + code, envelope = performPagesSourceRequest( + t, + router, + http.MethodPost, + fmt.Sprintf("/api/v1/d/pages/%d/source/sync", busyProject.ID), + []byte(`{}`), + ) + if got, want := code, http.StatusConflict; got != want { + t.Errorf("POST busy source/sync status = %d, want %d", got, want) + } + if got, want := envelope.ErrorMsg, errPagesSourceActionBusy; got != want { + t.Errorf("POST busy source/sync error = %q, want %q", got, want) + } +} + +func TestSyncSourceHandlerAcceptsEmptyBodyAndEmptyObject(t *testing.T) { + ctx := setupPagesSourceTest(t) + setupPagesSourceDispatchTest(t) + router := newPagesSourceTestRouter(77) + project := mustCreatePagesSourceProject(t, ctx, "handler-empty-sync") + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip?token=dispatch-secret", + RemoteNetworkPolicyPublic, + ) + path := fmt.Sprintf("/api/v1/d/pages/%d/source/sync", project.ID) + + for _, test := range []struct { + name string + body []byte + }{ + {name: "empty body", body: nil}, + {name: "empty object", body: []byte(`{}`)}, + } { + t.Run(test.name, func(t *testing.T) { + code, envelope := performPagesSourceRequest(t, router, http.MethodPost, path, test.body) + if got, want := code, http.StatusOK; got != want { + t.Fatalf("POST source/sync (%s) status = %d, want %d; error=%q", test.name, got, want, envelope.ErrorMsg) + } + if envelope.ErrorMsg != "" { + t.Errorf("POST source/sync (%s) error = %q, want empty", test.name, envelope.ErrorMsg) + } + var receipt SourceActionReceipt + if err := json.Unmarshal(envelope.Data, &receipt); err != nil { + t.Fatalf("json.Unmarshal(source/sync %s receipt) error = %v, want nil", test.name, err) + } + if receipt.TaskID == "" || receipt.ExecutionID == "" || receipt.Action != sourceActionSync { + t.Errorf("POST source/sync (%s) receipt = %+v, want task/execution IDs and action %q", test.name, receipt, sourceActionSync) + } + }) + } + + var executions []model.TaskExecution + if err := db.DB(ctx).Where("task_type = ?", PagesSourceActionTask).Order("id asc").Find(&executions).Error; err != nil { + t.Fatalf("list Pages source task executions error = %v, want nil", err) + } + if got, want := len(executions), 2; got != want { + t.Fatalf("Pages source task execution count = %d, want %d", got, want) + } + for _, execution := range executions { + if strings.Contains(execution.Payload, "dispatch-secret") || strings.Contains(execution.Payload, "http") { + t.Errorf("task execution %q payload = %s, want no Remote URL secret", execution.TaskID, execution.Payload) + } + var payload SourceActionPayload + if err := json.Unmarshal([]byte(execution.Payload), &payload); err != nil { + t.Errorf("json.Unmarshal(task execution %q payload) error = %v, want nil", execution.TaskID, err) + continue + } + if payload.SourceID != source.ID || payload.ConfigVersion != source.ConfigVersion || payload.Actor != "user:77" { + t.Errorf("task execution %q payload = %+v, want source=%d config=%d actor=user:77", execution.TaskID, payload, source.ID, source.ConfigVersion) + } + } +} diff --git a/internal/apps/openflare/pages/source.go b/internal/apps/openflare/pages/source.go new file mode 100644 index 00000000..9bc46217 --- /dev/null +++ b/internal/apps/openflare/pages/source.go @@ -0,0 +1,495 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net" + "net/url" + "path" + "strings" + "time" + + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const ( + // PagesSourceTypeManual represents projects without a persisted source row. + PagesSourceTypeManual = "manual" + // PagesSourceTypeRemoteURL represents a persisted artifact URL. + PagesSourceTypeRemoteURL = "remote_url" + // PagesSourceTypeGitHubRelease is reserved for Phase 2. + PagesSourceTypeGitHubRelease = "github_release" + + pagesSourceStatusIdle = "idle" + pagesSourceStatusChecking = "checking" + pagesSourceStatusUpdateAvailable = "update_available" + pagesSourceStatusSyncing = "syncing" + pagesSourceStatusFailed = "failed" + pagesSourceStatusAttention = "attention" + + defaultRemoteAssetLabel = "pages-package" +) + +// SourceUpdateInput is the discriminated source configuration payload. +// GitHub fields are accepted by the decoder so mode-incompatible values can be +// rejected deterministically; GitHub itself is enabled in Phase 2. +type SourceUpdateInput struct { + SourceType string `json:"source_type"` + RemoteURLSet bool `json:"remote_url_set"` + RemoteURL string `json:"remote_url"` + RemoteNetworkPolicy string `json:"remote_network_policy"` + RepositoryURL string `json:"repository_url"` + ReleaseSelector string `json:"release_selector"` + ReleaseTag string `json:"release_tag"` + AssetName string `json:"asset_name"` + AutoUpdateEnabled bool `json:"auto_update_enabled"` + CheckIntervalMinutes int `json:"check_interval_minutes"` +} + +// SourceRevisionView is a credential-free source cursor. +type SourceRevisionView struct { + Revision string `json:"revision"` + Label string `json:"label"` + AssetName string `json:"asset_name,omitempty"` +} + +// SourceView is the safe discriminated source view returned to the console. +type SourceView struct { + SourceType string `json:"source_type"` + HasRemoteURL bool `json:"has_remote_url,omitempty"` + DisplayURL string `json:"display_url,omitempty"` + RemoteNetworkPolicy string `json:"remote_network_policy,omitempty"` + GitHubRepository string `json:"github_repository,omitempty"` + ReleaseSelector string `json:"release_selector,omitempty"` + ReleaseTag string `json:"release_tag,omitempty"` + AssetName string `json:"asset_name,omitempty"` + AutoUpdateEnabled bool `json:"auto_update_enabled,omitempty"` + CheckIntervalMinutes int `json:"check_interval_minutes,omitempty"` + SyncStatus string `json:"sync_status,omitempty"` + UpdateAvailable bool `json:"update_available,omitempty"` + LastSeen *SourceRevisionView `json:"last_seen,omitempty"` + LastApplied *SourceRevisionView `json:"last_applied,omitempty"` + LastCheckedAt *time.Time `json:"last_checked_at,omitempty"` + LastSyncedAt *time.Time `json:"last_synced_at,omitempty"` + NextCheckAt *time.Time `json:"next_check_at,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +// SourceActionReceipt identifies the internal task execution created by an action API. +type SourceActionReceipt struct { + TaskID string `json:"task_id"` + ExecutionID string `json:"execution_id"` + Action string `json:"action"` +} + +// SourceUpdateResult is returned after persisting source configuration. +type SourceUpdateResult struct { + Source *SourceView `json:"source"` + CheckTask *SourceActionReceipt `json:"check_task"` + Warning string `json:"warning"` +} + +type sourceDetail struct { + Provider string `json:"provider"` + Label string `json:"label"` + AssetName string `json:"asset_name,omitempty"` + ReleaseID string `json:"release_id,omitempty"` +} + +type remoteSourceConfig struct { + URL string + Policy string + Identity string +} + +// GetSource returns the current persisted source or a manual discriminator. +func GetSource(ctx context.Context, projectID uint) (*SourceView, error) { + if _, err := model.GetPagesProjectByID(ctx, projectID); err != nil { + return nil, err + } + source, runtime, err := loadSourceByProject(ctx, projectID) + if errors.Is(err, gorm.ErrRecordNotFound) { + return &SourceView{SourceType: PagesSourceTypeManual}, nil + } + if err != nil { + return nil, err + } + return buildSourceView(source, runtime) +} + +// UpdateSource creates or updates a Remote URL source and its 1:1 runtime row. +func UpdateSource(ctx context.Context, projectID uint, input SourceUpdateInput) (*SourceUpdateResult, error) { + if err := validateRemoteSourceInput(input); err != nil { + return nil, err + } + + err := db.DB(ctx).Transaction(func(tx *gorm.DB) error { + return updateRemoteSourceTx(tx, projectID, input) + }) + if err != nil { + return nil, err + } + + view, err := GetSource(ctx, projectID) + if err != nil { + return nil, err + } + return &SourceUpdateResult{Source: view, Warning: ""}, nil +} + +func updateRemoteSourceTx(tx *gorm.DB, projectID uint, input SourceUpdateInput) error { + var project model.PagesProject + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil { + return err + } + existing, hasExisting, err := loadProjectSourceForUpdate(tx, projectID) + if err != nil { + return err + } + config, err := buildRemoteSourceConfig(existing, hasExisting, input) + if err != nil { + return err + } + if !hasExisting { + return createRemoteSourceTx(tx, projectID, config) + } + return updateExistingRemoteSourceTx(tx, existing, config) +} + +func loadProjectSourceForUpdate(tx *gorm.DB, projectID uint) (*model.PagesProjectSource, bool, error) { + var source model.PagesProjectSource + err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("project_id = ?", projectID). + First(&source).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return &source, false, nil + } + if err != nil { + return nil, false, err + } + return &source, true, nil +} + +func buildRemoteSourceConfig( + existing *model.PagesProjectSource, + hasExisting bool, + input SourceUpdateInput, +) (remoteSourceConfig, error) { + remoteURL, err := resolveUpdatedRemoteURL(existing, hasExisting, input) + if err != nil { + return remoteSourceConfig{}, err + } + parsedURL, err := parseRemoteSourceURL(remoteURL) + if err != nil { + return remoteSourceConfig{}, err + } + policy := strings.TrimSpace(input.RemoteNetworkPolicy) + if policy == "" { + policy = RemoteNetworkPolicyPublic + } + return remoteSourceConfig{ + URL: remoteURL, + Policy: policy, + Identity: remoteSourceIdentity(parsedURL), + }, nil +} + +func createRemoteSourceTx(tx *gorm.DB, projectID uint, config remoteSourceConfig) error { + source := &model.PagesProjectSource{ + ProjectID: projectID, + SourceType: PagesSourceTypeRemoteURL, + RemoteURL: config.URL, + RemoteNetworkPolicy: config.Policy, + AutoUpdateEnabled: false, + CheckIntervalMinutes: 0, + ConfigVersion: 1, + SourceIdentity: config.Identity, + } + if err := tx.Create(source).Error; err != nil { + return err + } + return tx.Create(&model.PagesProjectSourceRuntime{ + SourceID: source.ID, + SyncStatus: pagesSourceStatusIdle, + }).Error +} + +func updateExistingRemoteSourceTx( + tx *gorm.DB, + existing *model.PagesProjectSource, + config remoteSourceConfig, +) error { + if !remoteSourceConfigChanged(existing, config) { + return nil + } + var runtime model.PagesProjectSourceRuntime + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("source_id = ?", existing.ID). + First(&runtime).Error; err != nil { + return err + } + identityChanged := existing.SourceIdentity != config.Identity + if err := tx.Model(existing).Updates(map[string]any{ + "source_type": PagesSourceTypeRemoteURL, + "remote_url": config.URL, + "remote_network_policy": config.Policy, + "github_repository": "", + "release_selector": "", + "release_tag": "", + "asset_name": "", + "auto_update_enabled": false, + "check_interval_minutes": 0, + "config_version": existing.ConfigVersion + 1, + "source_identity": config.Identity, + }).Error; err != nil { + return err + } + return resetRuntimeAfterSourceUpdate(tx, &runtime, identityChanged) +} + +func remoteSourceConfigChanged(existing *model.PagesProjectSource, config remoteSourceConfig) bool { + return existing.SourceType != PagesSourceTypeRemoteURL || + existing.RemoteURL != config.URL || + existing.RemoteNetworkPolicy != config.Policy || + existing.GitHubRepository != "" || + existing.ReleaseSelector != "" || + existing.ReleaseTag != "" || + existing.AssetName != "" || + existing.AutoUpdateEnabled || + existing.CheckIntervalMinutes != 0 +} + +// DeleteSource idempotently switches a project back to manual mode. +func DeleteSource(ctx context.Context, projectID uint) (*SourceView, error) { + err := db.DB(ctx).Transaction(func(tx *gorm.DB) error { + var project model.PagesProject + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, projectID).Error; err != nil { + return err + } + var source model.PagesProjectSource + err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("project_id = ?", projectID). + First(&source).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + if err != nil { + return err + } + var runtime model.PagesProjectSourceRuntime + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("source_id = ?", source.ID). + First(&runtime).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + if err := tx.Where("source_id = ?", source.ID).Delete(&model.PagesProjectSourceRuntime{}).Error; err != nil { + return err + } + return tx.Delete(&source).Error + }) + if err != nil { + return nil, err + } + return &SourceView{SourceType: PagesSourceTypeManual}, nil +} + +func validateRemoteSourceInput(input SourceUpdateInput) error { + sourceType := strings.TrimSpace(input.SourceType) + if sourceType == "" { + return errors.New(errPagesSourceTypeRequired) + } + if sourceType != PagesSourceTypeRemoteURL { + return errors.New(errPagesSourceTypeUnsupported) + } + if strings.TrimSpace(input.RepositoryURL) != "" || strings.TrimSpace(input.ReleaseSelector) != "" || + strings.TrimSpace(input.ReleaseTag) != "" || strings.TrimSpace(input.AssetName) != "" || + input.AutoUpdateEnabled || input.CheckIntervalMinutes != 0 { + return errors.New(errPagesSourceRemoteFields) + } + policy := strings.TrimSpace(input.RemoteNetworkPolicy) + if policy != "" && policy != RemoteNetworkPolicyPublic && policy != RemoteNetworkPolicyTrustedInternal { + return errors.New(errPagesSourceNetworkPolicy) + } + if !input.RemoteURLSet && strings.TrimSpace(input.RemoteURL) != "" { + return errors.New(errPagesSourceRemoteURLMode) + } + if input.RemoteURLSet && strings.TrimSpace(input.RemoteURL) == "" { + return errors.New(errPagesSourceRemoteURLRequired) + } + return nil +} + +func resolveUpdatedRemoteURL(existing *model.PagesProjectSource, hasExisting bool, input SourceUpdateInput) (string, error) { + if input.RemoteURLSet { + return strings.TrimSpace(input.RemoteURL), nil + } + if !hasExisting || existing.SourceType != PagesSourceTypeRemoteURL || strings.TrimSpace(existing.RemoteURL) == "" { + return "", errors.New(errPagesSourceRemoteURLRequired) + } + return existing.RemoteURL, nil +} + +func parseRemoteSourceURL(raw string) (*url.URL, error) { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" { + return nil, errors.New(errPagesSourceRemoteURLInvalid) + } + parsed.Scheme = strings.ToLower(parsed.Scheme) + if parsed.Scheme != remoteSourceSchemeHTTP && parsed.Scheme != remoteSourceSchemeHTTPS { + return nil, errors.New(errPagesSourceRemoteURLInvalid) + } + if strings.TrimSpace(parsed.Hostname()) == "" { + return nil, errors.New(errPagesSourceRemoteURLInvalid) + } + return parsed, nil +} + +func remoteSourceIdentity(parsed *url.URL) string { + hostname := strings.ToLower(parsed.Hostname()) + port := parsed.Port() + if (parsed.Scheme == "https" && port == "443") || (parsed.Scheme == "http" && port == "80") { + port = "" + } + host := hostname + if port != "" { + host = net.JoinHostPort(hostname, port) + } else if strings.Contains(hostname, ":") { + host = "[" + hostname + "]" + } + canonicalPath := parsed.EscapedPath() + if canonicalPath == "" { + canonicalPath = "/" + } + canonicalPath = path.Clean("/" + strings.TrimPrefix(canonicalPath, "/")) + canonical := parsed.Scheme + "://" + host + canonicalPath + sum := sha256.Sum256([]byte("remote_url|" + canonical)) + return hex.EncodeToString(sum[:]) +} + +func displayRemoteSourceURL(raw string) string { + parsed, err := parseRemoteSourceURL(raw) + if err != nil { + return "" + } + hadQuery := parsed.RawQuery != "" + parsed.RawQuery = "" + display := parsed.String() + if hadQuery { + display += "?***" + } + return display +} + +func loadSourceByProject(ctx context.Context, projectID uint) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime, error) { + var source model.PagesProjectSource + if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil { + return nil, nil, err + } + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil { + return nil, nil, err + } + return &source, &runtime, nil +} + +func buildSourceView(source *model.PagesProjectSource, runtime *model.PagesProjectSourceRuntime) (*SourceView, error) { + if source == nil || runtime == nil { + return nil, errors.New(errPagesSourceNotFound) + } + view := &SourceView{ + SourceType: source.SourceType, + SyncStatus: runtime.SyncStatus, + LastSyncedAt: runtime.LastSyncedAt, + LastError: runtime.LastError, + } + if runtime.LastAppliedRevision != "" { + view.LastApplied = revisionView(runtime.LastAppliedRevision, runtime.LastAppliedDetail) + } + switch source.SourceType { + case PagesSourceTypeRemoteURL: + view.HasRemoteURL = strings.TrimSpace(source.RemoteURL) != "" + view.DisplayURL = displayRemoteSourceURL(source.RemoteURL) + view.RemoteNetworkPolicy = source.RemoteNetworkPolicy + case PagesSourceTypeGitHubRelease: + view.LastCheckedAt = runtime.LastCheckedAt + view.NextCheckAt = runtime.NextCheckAt + view.UpdateAvailable = runtime.LastSeenRevision != "" && runtime.LastSeenRevision != runtime.LastAppliedRevision + if runtime.LastSeenRevision != "" { + view.LastSeen = revisionView(runtime.LastSeenRevision, runtime.LastSeenDetail) + } + view.GitHubRepository = source.GitHubRepository + view.ReleaseSelector = source.ReleaseSelector + view.ReleaseTag = source.ReleaseTag + view.AssetName = source.AssetName + view.AutoUpdateEnabled = source.AutoUpdateEnabled + view.CheckIntervalMinutes = source.CheckIntervalMinutes + default: + return nil, errors.New(errPagesSourceTypeUnsupported) + } + return view, nil +} + +func revisionView(revision string, detailJSON string) *SourceRevisionView { + detail := sourceDetail{} + _ = unmarshalSourceDetail(detailJSON, &detail) + label := strings.TrimSpace(detail.Label) + if label == "" { + label = defaultRemoteAssetLabel + } + return &SourceRevisionView{ + Revision: revision, + Label: label, + AssetName: detail.AssetName, + } +} + +func unmarshalSourceDetail(raw string, detail *sourceDetail) error { + if detail == nil || strings.TrimSpace(raw) == "" { + return nil + } + return json.Unmarshal([]byte(raw), detail) +} + +func resetRuntimeAfterSourceUpdate(tx *gorm.DB, runtime *model.PagesProjectSourceRuntime, identityChanged bool) error { + updates := map[string]any{ + sourceRuntimeColumnLeaseToken: "", + sourceRuntimeColumnLeaseExpiresAt: nil, + sourceRuntimeColumnLastError: "", + } + if identityChanged { + updates["etag"] = "" + updates["last_seen_revision"] = "" + updates["last_seen_detail"] = "" + updates["last_applied_revision"] = "" + updates["last_applied_detail"] = "" + updates["last_checked_at"] = nil + updates["last_synced_at"] = nil + updates["next_check_at"] = nil + updates[sourceRuntimeColumnSyncStatus] = pagesSourceStatusIdle + } else { + updates[sourceRuntimeColumnSyncStatus] = normalizedSourceRuntimeStatus(runtime) + } + return tx.Model(runtime).Updates(updates).Error +} + +func normalizedSourceRuntimeStatus(runtime *model.PagesProjectSourceRuntime) string { + if runtime == nil { + return pagesSourceStatusIdle + } + if sourceHasSameReleaseReplacement(runtime) { + return pagesSourceStatusAttention + } + if runtime.LastSeenRevision != "" && runtime.LastSeenRevision != runtime.LastAppliedRevision { + return pagesSourceStatusUpdateAvailable + } + return pagesSourceStatusIdle +} diff --git a/internal/apps/openflare/pages/source_provider.go b/internal/apps/openflare/pages/source_provider.go new file mode 100644 index 00000000..77a242a5 --- /dev/null +++ b/internal/apps/openflare/pages/source_provider.go @@ -0,0 +1,549 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/Rain-kl/Wavelet/pkg/httppool" + "github.com/Rain-kl/Wavelet/pkg/pagesarchive" +) + +const ( + // RemoteNetworkPolicyPublic only permits publicly routable targets and + // performs DNS validation again for every connection. + RemoteNetworkPolicyPublic = "public" + // RemoteNetworkPolicyTrustedInternal permits private targets and self-signed + // TLS certificates. It is an explicit administrator trust boundary. + RemoteNetworkPolicyTrustedInternal = "trusted_internal" + + remoteSourceDownloadTimeout = 10 * time.Minute + remoteSourceResponseHeaderTimeout = 30 * time.Second + remoteSourceDialTimeout = 30 * time.Second + remoteSourceDialKeepAlive = 30 * time.Second + remoteSourceMaxRedirects = 5 + remoteSourceMagicSniffBytes = 512 + remoteSourceMaxSafeLabelBytes = 255 + remoteSourceFallbackLabel = "package" + remoteSourceUserAgent = "OpenFlare Pages Source/2" + remoteSourceSchemeHTTP = "http" + remoteSourceSchemeHTTPS = "https" +) + +type remoteProviderError string + +func (providerError remoteProviderError) Error() string { + return string(providerError) +} + +const ( + errRemoteProviderInvalidPolicy remoteProviderError = "远程来源网络策略无效" + errRemoteProviderInvalidLimit remoteProviderError = "远程来源部署包大小限制无效" + errRemoteProviderBlockedAddress remoteProviderError = "远程来源 public 策略禁止访问非公网地址" + errRemoteProviderResolveFailed remoteProviderError = "远程来源地址解析失败" + errRemoteProviderRedirectLimit remoteProviderError = "远程来源重定向次数超过限制" + errRemoteProviderDownloadFailed remoteProviderError = errPagesPackageURLDownloadFailed + errRemoteProviderTooLarge remoteProviderError = errPagesPackageURLTooLarge + errRemoteProviderEmpty remoteProviderError = errPagesPackageEmpty + errRemoteProviderUnsupported remoteProviderError = errPagesPackageUnsupported + errRemoteProviderCleanupFailed remoteProviderError = "清理远程来源临时文件失败" +) + +var remoteSourceNonPublicPrefixes = []netip.Prefix{ + // IPv4 special-use, private, link-local, documentation, multicast and + // reserved ranges. A conservative deny list is intentional for SSRF safety. + netip.MustParsePrefix("0.0.0.0/8"), + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("127.0.0.0/8"), + netip.MustParsePrefix("169.254.0.0/16"), + netip.MustParsePrefix("172.16.0.0/12"), + netip.MustParsePrefix("192.0.0.0/24"), + netip.MustParsePrefix("192.0.2.0/24"), + netip.MustParsePrefix("192.88.99.0/24"), + netip.MustParsePrefix("192.168.0.0/16"), + netip.MustParsePrefix("198.18.0.0/15"), + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/24"), + netip.MustParsePrefix("224.0.0.0/4"), + netip.MustParsePrefix("240.0.0.0/4"), + // IPv6 protocol-assignment, documentation and transition ranges that are + // not acceptable as direct public artifact origins. + netip.MustParsePrefix("2001::/23"), + netip.MustParsePrefix("2001:db8::/32"), + netip.MustParsePrefix("2002::/16"), + netip.MustParsePrefix("3fff::/20"), +} + +var remoteSourcePublicIPv6Prefix = netip.MustParsePrefix("2000::/3") + +// RemoteSourceRequest describes one immutable Remote URL package fetch. +type RemoteSourceRequest struct { + URL string + NetworkPolicy string + MaxPackageBytes int64 +} + +// SourceCandidate is a constrained, immutable archive downloaded to a +// provider-owned temporary file. The caller owns the file after a successful +// fetch and must call Cleanup when processing finishes. +type SourceCandidate struct { + TempPath string + Checksum string + PackageSize int64 + Format pagesarchive.Format + SafeLabel string +} + +// Cleanup removes the candidate temporary file. It is safe to call repeatedly. +func (candidate *SourceCandidate) Cleanup() error { + if candidate == nil || candidate.TempPath == "" { + return nil + } + tempPath := candidate.TempPath + err := os.Remove(tempPath) + if err == nil || errors.Is(err, os.ErrNotExist) { + candidate.TempPath = "" + return nil + } + return errRemoteProviderCleanupFailed +} + +type remoteSourceResolver interface { + LookupNetIP(context.Context, string, string) ([]netip.Addr, error) +} + +type remoteSourceDependencies struct { + resolver remoteSourceResolver + dialContext func(context.Context, string, string) (net.Conn, error) + createTemp func(string, string) (*os.File, error) +} + +// FetchRemoteSource downloads a Remote URL package without writing deployment +// state. Errors are reduced to safe domain messages and never contain the raw +// URL, query, response headers or response body. +func FetchRemoteSource(ctx context.Context, request RemoteSourceRequest) (*SourceCandidate, error) { + dialer := &net.Dialer{ + Timeout: remoteSourceDialTimeout, + KeepAlive: remoteSourceDialKeepAlive, + } + dependencies := remoteSourceDependencies{ + resolver: net.DefaultResolver, + dialContext: dialer.DialContext, + createTemp: os.CreateTemp, + } + return fetchRemoteSource(ctx, request, dependencies) +} + +func fetchRemoteSource(ctx context.Context, request RemoteSourceRequest, dependencies remoteSourceDependencies) (*SourceCandidate, error) { + if request.MaxPackageBytes <= 0 { + return nil, errRemoteProviderInvalidLimit + } + if dependencies.dialContext == nil || dependencies.createTemp == nil { + return nil, errRemoteProviderDownloadFailed + } + policy, err := normalizeRemoteNetworkPolicy(request.NetworkPolicy) + if err != nil { + return nil, err + } + parsed, err := parseRemoteSourceURL(request.URL) + if err != nil { + return nil, err + } + if err := validateRemoteSourceTarget(ctx, parsed, policy, dependencies.resolver); err != nil { + return nil, sanitizeRemoteProviderError(ctx, err) + } + + safeLabel, namedFormat := remoteSourceLabel(parsed) + client := newRemoteSourceClient(policy, dependencies) + defer client.CloseIdleConnections() + response, err := requestRemoteSource(ctx, client, parsed) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + return nil, fmt.Errorf("%w: HTTP %d", errRemoteProviderDownloadFailed, response.StatusCode) + } + if response.ContentLength > request.MaxPackageBytes { + return nil, errRemoteProviderTooLarge + } + + tempPath, checksum, packageSize, err := streamRemoteSourcePackage( + response.Body, + request.MaxPackageBytes, + dependencies.createTemp, + ) + if err != nil { + return nil, sanitizeRemoteProviderError(ctx, err) + } + format, safeLabel, err := detectRemoteSourceFormat(tempPath, safeLabel, namedFormat) + if err != nil { + if removeErr := os.Remove(tempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return nil, errRemoteProviderCleanupFailed + } + return nil, err + } + + return &SourceCandidate{ + TempPath: tempPath, + Checksum: checksum, + PackageSize: packageSize, + Format: format, + SafeLabel: safeLabel, + }, nil +} + +func normalizeRemoteNetworkPolicy(policy string) (string, error) { + switch strings.TrimSpace(policy) { + case "", RemoteNetworkPolicyPublic: + return RemoteNetworkPolicyPublic, nil + case RemoteNetworkPolicyTrustedInternal: + return RemoteNetworkPolicyTrustedInternal, nil + default: + return "", errRemoteProviderInvalidPolicy + } +} + +func newRemoteSourceClient(policy string, dependencies remoteSourceDependencies) *http.Client { + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + dialContext := dependencies.dialContext + if policy == RemoteNetworkPolicyPublic { + dialContext = newPublicRemoteSourceDialer(dependencies.resolver, dependencies.dialContext) + } else { + // trusted_internal is an explicit administrator-selected boundary for + // private artifact services using an internal CA or self-signed cert. + tlsConfig.InsecureSkipVerify = true //nolint:gosec // required trusted_internal semantics + } + + client := &http.Client{ + Timeout: remoteSourceDownloadTimeout, + Transport: httppool.NewTransport(httppool.TransportOptions{ + Proxy: nil, + DialContext: dialContext, + TLSClientConfig: tlsConfig, + ResponseHeaderTimeout: remoteSourceResponseHeaderTimeout, + TraceFilter: remoteSourceTraceFilter, + }), + } + client.CheckRedirect = func(next *http.Request, previous []*http.Request) error { + if len(previous) > remoteSourceMaxRedirects { + return errRemoteProviderRedirectLimit + } + stripRemoteSourceRedirectHeaders(next) + if err := validateRemoteSourceTarget(next.Context(), next.URL, policy, dependencies.resolver); err != nil { + return err + } + applyRemoteSourceHeaders(next) + return nil + } + return client +} + +func requestRemoteSource(ctx context.Context, client *http.Client, parsed *url.URL) (*http.Response, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) + if err != nil { + return nil, errors.New(errPagesSourceRemoteURLInvalid) + } + applyRemoteSourceHeaders(request) + response, err := client.Do(request) //nolint:gosec // scheme and every dial target are validated above + if err == nil { + return response, nil + } + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + return nil, sanitizeRemoteProviderError(ctx, err) +} + +func applyRemoteSourceHeaders(request *http.Request) { + request.Header.Set("User-Agent", remoteSourceUserAgent) + request.Header.Set("Accept", "application/octet-stream,application/zip,application/x-tar,application/gzip,*/*;q=0.1") + // Preserve the artifact bytes exactly as stored. Automatic HTTP gzip + // decompression would change the checksum, size and archive format. + request.Header.Set("Accept-Encoding", "identity") +} + +func stripRemoteSourceRedirectHeaders(request *http.Request) { + request.Header.Del("Authorization") + request.Header.Del("Cookie") + request.Header.Del("Proxy-Authorization") + request.Header.Del("Referer") +} + +func remoteSourceTraceFilter(request *http.Request) bool { + // otelhttp records url.full. Signed query strings must never enter traces. + return request.URL == nil || request.URL.RawQuery == "" +} + +func validateRemoteSourceTarget(ctx context.Context, target *url.URL, policy string, resolver remoteSourceResolver) error { + if target == nil || target.User != nil || target.Fragment != "" || target.Opaque != "" { + return errors.New(errPagesSourceRemoteURLInvalid) + } + scheme := strings.ToLower(strings.TrimSpace(target.Scheme)) + if (scheme != remoteSourceSchemeHTTP && scheme != remoteSourceSchemeHTTPS) || strings.TrimSpace(target.Hostname()) == "" { + return errors.New(errPagesSourceRemoteURLInvalid) + } + if policy != RemoteNetworkPolicyPublic { + return nil + } + _, err := resolvePublicRemoteSourceIPs(ctx, resolver, target.Hostname()) + return err +} + +func newPublicRemoteSourceDialer( + resolver remoteSourceResolver, + directDial func(context.Context, string, string) (net.Conn, error), +) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network string, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, errRemoteProviderDownloadFailed + } + addresses, err := resolvePublicRemoteSourceIPs(ctx, resolver, host) + if err != nil { + return nil, err + } + for _, address := range addresses { + if !remoteSourceIPMatchesNetwork(address, network) { + continue + } + connection, dialErr := directDial(ctx, network, net.JoinHostPort(address.String(), port)) + if dialErr == nil { + return connection, nil + } + } + return nil, errRemoteProviderDownloadFailed + } +} + +func resolvePublicRemoteSourceIPs(ctx context.Context, resolver remoteSourceResolver, host string) ([]netip.Addr, error) { + if strings.Contains(host, "%") { + return nil, errRemoteProviderBlockedAddress + } + if literal, parseErr := netip.ParseAddr(host); parseErr == nil { + if !isPublicRemoteSourceIP(literal) { + return nil, errRemoteProviderBlockedAddress + } + return []netip.Addr{literal}, nil + } + if resolver == nil { + return nil, errRemoteProviderResolveFailed + } + addresses, err := resolver.LookupNetIP(ctx, "ip", host) + if err != nil || len(addresses) == 0 { + return nil, errRemoteProviderResolveFailed + } + for _, address := range addresses { + if !isPublicRemoteSourceIP(address) { + return nil, errRemoteProviderBlockedAddress + } + } + return addresses, nil +} + +func isPublicRemoteSourceIP(address netip.Addr) bool { + if !address.IsValid() || address.Zone() != "" { + return false + } + address = address.Unmap() + if !address.IsGlobalUnicast() { + return false + } + if address.Is6() && !remoteSourcePublicIPv6Prefix.Contains(address) { + return false + } + for _, prefix := range remoteSourceNonPublicPrefixes { + if prefix.Contains(address) { + return false + } + } + return true +} + +func remoteSourceIPMatchesNetwork(address netip.Addr, network string) bool { + switch network { + case "tcp4": + return address.Unmap().Is4() + case "tcp6": + return address.Unmap().Is6() + default: + return true + } +} + +func streamRemoteSourcePackage( + body io.Reader, + maxPackageBytes int64, + createTemp func(string, string) (*os.File, error), +) (tempPath string, checksum string, packageSize int64, err error) { + if createTemp == nil { + return "", "", 0, errRemoteProviderDownloadFailed + } + tempFile, err := createTemp("", "openflare-pages-source-*") + if err != nil { + return "", "", 0, errRemoteProviderDownloadFailed + } + createdTempPath := tempFile.Name() + tempPath = createdTempPath + defer func() { + closeErr := tempFile.Close() + if err == nil && closeErr != nil { + err = errRemoteProviderDownloadFailed + } + if err != nil { + if removeErr := os.Remove(createdTempPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + err = errRemoteProviderCleanupFailed + } + } + }() + + hasher := sha256.New() + readLimit := maxPackageBytes + if readLimit < math.MaxInt64 { + readLimit++ + } + packageSize, err = io.Copy(io.MultiWriter(tempFile, hasher), io.LimitReader(body, readLimit)) + if err != nil { + return "", "", 0, errRemoteProviderDownloadFailed + } + if packageSize > maxPackageBytes { + return "", "", 0, errRemoteProviderTooLarge + } + if packageSize == 0 { + return "", "", 0, errRemoteProviderEmpty + } + checksum = hex.EncodeToString(hasher.Sum(nil)) + return tempPath, checksum, packageSize, nil +} + +func detectRemoteSourceFormat( + tempPath string, + safeLabel string, + namedFormat pagesarchive.Format, +) (pagesarchive.Format, string, error) { + if namedFormat != "" { + return namedFormat, safeLabel, nil + } + tempFile, err := os.Open(tempPath) //nolint:gosec // path is a provider-created temporary file + if err != nil { + return "", safeLabel, errRemoteProviderDownloadFailed + } + defer func() { _ = tempFile.Close() }() + + head := make([]byte, remoteSourceMagicSniffBytes) + readBytes, readErr := io.ReadFull(tempFile, head) + if readErr != nil && !errors.Is(readErr, io.EOF) && !errors.Is(readErr, io.ErrUnexpectedEOF) { + return "", safeLabel, errRemoteProviderDownloadFailed + } + format, ok := pagesarchive.DetectFormatFromBytes(head[:readBytes]) + if !ok { + return "", safeLabel, errRemoteProviderUnsupported + } + return format, appendRemoteSourceLabelExtension(safeLabel, format), nil +} + +func remoteSourceLabel(parsed *url.URL) (string, pagesarchive.Format) { + baseName := path.Base(parsed.Path) + if baseName == "" || baseName == "." || baseName == "/" { + baseName = remoteSourceFallbackLabel + } + safeLabel := sanitizeRemoteSourceLabel(baseName) + format, _ := pagesarchive.DetectFormatFromName(safeLabel) + return limitRemoteSourceLabel(safeLabel, format), format +} + +func sanitizeRemoteSourceLabel(label string) string { + var builder strings.Builder + lastReplacement := false + for _, character := range label { + if isRemoteSourceLabelCharacter(character) { + builder.WriteRune(character) + lastReplacement = false + continue + } + if !lastReplacement { + builder.WriteByte('-') + lastReplacement = true + } + } + safeLabel := strings.TrimSpace(builder.String()) + if safeLabel == "" || strings.Trim(safeLabel, "._-") == "" { + return remoteSourceFallbackLabel + } + return safeLabel +} + +func isRemoteSourceLabelCharacter(character rune) bool { + return character >= 'a' && character <= 'z' || + character >= 'A' && character <= 'Z' || + character >= '0' && character <= '9' || + character == '.' || character == '-' || character == '_' +} + +func limitRemoteSourceLabel(label string, format pagesarchive.Format) string { + if len(label) <= remoteSourceMaxSafeLabelBytes { + return label + } + if format == "" { + return strings.TrimRight(label[:remoteSourceMaxSafeLabelBytes], ".-_") + } + extension := "." + pagesarchive.Extension(format) + prefixLength := remoteSourceMaxSafeLabelBytes - len(extension) + prefix := strings.TrimRight(label[:prefixLength], ".-_") + if prefix == "" { + prefix = remoteSourceFallbackLabel + } + return prefix + extension +} + +func appendRemoteSourceLabelExtension(label string, format pagesarchive.Format) string { + extension := "." + pagesarchive.Extension(format) + maxPrefixLength := remoteSourceMaxSafeLabelBytes - len(extension) + if len(label) > maxPrefixLength { + label = strings.TrimRight(label[:maxPrefixLength], ".-_") + } + if label == "" { + label = remoteSourceFallbackLabel + } + return label + extension +} + +func sanitizeRemoteProviderError(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("%w: %w", errRemoteProviderDownloadFailed, ctxErr) + } + for _, safeError := range []error{ + errRemoteProviderInvalidPolicy, + errRemoteProviderInvalidLimit, + errRemoteProviderBlockedAddress, + errRemoteProviderResolveFailed, + errRemoteProviderRedirectLimit, + errRemoteProviderTooLarge, + errRemoteProviderEmpty, + errRemoteProviderUnsupported, + errRemoteProviderCleanupFailed, + } { + if errors.Is(err, safeError) { + return safeError + } + } + return errRemoteProviderDownloadFailed +} diff --git a/internal/apps/openflare/pages/source_provider_test.go b/internal/apps/openflare/pages/source_provider_test.go new file mode 100644 index 00000000..c38f1850 --- /dev/null +++ b/internal/apps/openflare/pages/source_provider_test.go @@ -0,0 +1,459 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "os" + "strconv" + "strings" + "sync/atomic" + "testing" +) + +type remoteSourceResolverFunc func(context.Context, string, string) ([]netip.Addr, error) + +func (function remoteSourceResolverFunc) LookupNetIP( + ctx context.Context, + network string, + host string, +) ([]netip.Addr, error) { + return function(ctx, network, host) +} + +func TestFetchRemoteSourceTrustedInternalSelfSignedAndSafeLabel(t *testing.T) { + packageBytes := makeRemoteSourceZIP(t) + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Query().Get("token") != "source-secret" { + t.Error("signed query did not reach the artifact server") + } + if request.Header.Get("Accept-Encoding") != "identity" { + t.Error("artifact request must disable automatic HTTP decompression") + } + writer.Header().Set("Content-Disposition", `attachment; filename="redirected.tar.gz"`) + _, _ = writer.Write(packageBytes) + })) + server.Config.ErrorLog = log.New(io.Discard, "", 0) + t.Cleanup(server.Close) + + candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: server.URL + "/original/site.zip?token=source-secret", + NetworkPolicy: RemoteNetworkPolicyTrustedInternal, + MaxPackageBytes: int64(len(packageBytes) + 1), + }) + if err != nil { + t.Fatalf("FetchRemoteSource() error = %v", err) + } + if candidate.Format != "zip" { + t.Fatalf("Format = %q, want zip", candidate.Format) + } + if candidate.SafeLabel != "site.zip" { + t.Fatalf("SafeLabel = %q, want original path basename", candidate.SafeLabel) + } + if candidate.PackageSize != int64(len(packageBytes)) { + t.Fatalf("PackageSize = %d, want %d", candidate.PackageSize, len(packageBytes)) + } + wantChecksum := sha256.Sum256(packageBytes) + if candidate.Checksum != hex.EncodeToString(wantChecksum[:]) { + t.Fatalf("Checksum = %q, want SHA-256", candidate.Checksum) + } + downloaded, err := os.ReadFile(candidate.TempPath) //nolint:gosec // provider-owned test temp file + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if !bytes.Equal(downloaded, packageBytes) { + t.Fatal("downloaded package differs from response body") + } + tempPath := candidate.TempPath + if err := candidate.Cleanup(); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + if err := candidate.Cleanup(); err != nil { + t.Fatalf("second Cleanup() error = %v", err) + } + if _, err := os.Stat(tempPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary file still exists: %v", err) + } +} + +func TestFetchRemoteSourceKeepsOriginalLabelAcrossRedirect(t *testing.T) { + packageBytes := makeRemoteSourceZIP(t) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/original/site.zip" { + writer.Header().Set("Location", "/delivery/final.tar.gz?token=redirect-secret") + writer.WriteHeader(http.StatusFound) + return + } + if request.Header.Get("Referer") != "" { + t.Error("redirect must not forward a signed source URL as Referer") + } + writer.Header().Set("Content-Disposition", `attachment; filename="response.7z"`) + _, _ = writer.Write(packageBytes) + })) + t.Cleanup(server.Close) + + candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: server.URL + "/original/site.zip?token=initial-secret", + NetworkPolicy: RemoteNetworkPolicyTrustedInternal, + MaxPackageBytes: int64(len(packageBytes) + 1), + }) + if err != nil { + t.Fatalf("FetchRemoteSource() error = %v", err) + } + defer func() { _ = candidate.Cleanup() }() + if candidate.SafeLabel != "site.zip" || candidate.Format != "zip" { + t.Fatalf("candidate = label %q format %q, want original site.zip", candidate.SafeLabel, candidate.Format) + } +} + +func TestFetchRemoteSourcePublicRejectsNonPublicAddresses(t *testing.T) { + tests := []string{ + "http://127.0.0.1/site.zip?token=loopback-secret", + "http://[::1]/site.zip?token=ipv6-secret", + "http://100.64.0.1/site.zip?token=cgnat-secret", + "http://192.0.2.1/site.zip?token=documentation-secret", + } + for _, rawURL := range tests { + t.Run(rawURL, func(t *testing.T) { + _, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: rawURL, + NetworkPolicy: RemoteNetworkPolicyPublic, + MaxPackageBytes: 1024, + }) + if !errors.Is(err, errRemoteProviderBlockedAddress) { + t.Fatalf("FetchRemoteSource() error = %v, want blocked address", err) + } + assertRemoteSourceErrorRedacted(t, err, rawURL, "secret", "token=") + }) + } +} + +func TestFetchRemoteSourcePublicDialsValidatedIP(t *testing.T) { + packageBytes := makeRemoteSourceZIP(t) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write(packageBytes) + })) + t.Cleanup(server.Close) + + var dialedAddress string + dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver()) + dependencies.dialContext = func(ctx context.Context, network string, address string) (net.Conn, error) { + dialedAddress = address + dialer := &net.Dialer{} + return dialer.DialContext(ctx, network, server.Listener.Addr().String()) + } + candidate, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: "http://artifact.example/site.zip", + NetworkPolicy: RemoteNetworkPolicyPublic, + MaxPackageBytes: int64(len(packageBytes) + 1), + }, dependencies) + if err != nil { + t.Fatalf("fetchRemoteSource() error = %v", err) + } + defer func() { _ = candidate.Cleanup() }() + if dialedAddress != "93.184.216.34:80" { + t.Fatalf("direct dial address = %q, want validated IP", dialedAddress) + } +} + +func TestFetchRemoteSourcePublicRejectsPrivateRedirect(t *testing.T) { + var requestCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + requestCount.Add(1) + writer.Header().Set("Location", "http://127.0.0.1/private.zip?token=redirect-secret") + writer.WriteHeader(http.StatusFound) + })) + t.Cleanup(server.Close) + + dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver()) + rawURL := "http://artifact.example/start.zip?token=initial-secret" + _, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: rawURL, + NetworkPolicy: RemoteNetworkPolicyPublic, + MaxPackageBytes: 1024, + }, dependencies) + if !errors.Is(err, errRemoteProviderBlockedAddress) { + t.Fatalf("fetchRemoteSource() error = %v, want blocked redirect", err) + } + if requestCount.Load() != 1 { + t.Fatalf("request count = %d, private redirect must not be requested", requestCount.Load()) + } + assertRemoteSourceErrorRedacted(t, err, rawURL, "initial-secret", "redirect-secret", "token=") +} + +func TestFetchRemoteSourcePublicRejectsDNSRebinding(t *testing.T) { + var lookupCount atomic.Int32 + resolver := remoteSourceResolverFunc(func(context.Context, string, string) ([]netip.Addr, error) { + if lookupCount.Add(1) == 1 { + return []netip.Addr{netip.MustParseAddr("93.184.216.34")}, nil + } + return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil + }) + var dialCount atomic.Int32 + dependencies := remoteSourceDependencies{ + resolver: resolver, + dialContext: func(context.Context, string, string) (net.Conn, error) { + dialCount.Add(1) + return nil, errors.New("unexpected dial") + }, + createTemp: os.CreateTemp, + } + rawURL := "http://rebind.example/site.zip?signature=dns-secret" + _, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: rawURL, + NetworkPolicy: RemoteNetworkPolicyPublic, + MaxPackageBytes: 1024, + }, dependencies) + if !errors.Is(err, errRemoteProviderBlockedAddress) { + t.Fatalf("fetchRemoteSource() error = %v, want DNS rebinding rejection", err) + } + if lookupCount.Load() != 2 { + t.Fatalf("DNS lookup count = %d, want preflight plus dial validation", lookupCount.Load()) + } + if dialCount.Load() != 0 { + t.Fatalf("direct dial count = %d, rebound address must not be dialed", dialCount.Load()) + } + assertRemoteSourceErrorRedacted(t, err, rawURL, "dns-secret", "signature=") +} + +func TestFetchRemoteSourcePublicRejectsSelfSignedTLS(t *testing.T) { + packageBytes := makeRemoteSourceZIP(t) + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write(packageBytes) + })) + server.Config.ErrorLog = log.New(io.Discard, "", 0) + t.Cleanup(server.Close) + + dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver()) + rawURL := "https://artifact.example/site.zip?signature=tls-secret" + _, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: rawURL, + NetworkPolicy: RemoteNetworkPolicyPublic, + MaxPackageBytes: int64(len(packageBytes) + 1), + }, dependencies) + if !errors.Is(err, errRemoteProviderDownloadFailed) { + t.Fatalf("fetchRemoteSource() error = %v, want strict TLS failure", err) + } + assertRemoteSourceErrorRedacted(t, err, rawURL, "tls-secret", "signature=") +} + +func TestFetchRemoteSourceRejectsChunkedBodyOverLimitAndCleansTemp(t *testing.T) { + const maxPackageBytes = int64(64) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write(bytes.Repeat([]byte{'x'}, int(maxPackageBytes))) + if flusher, ok := writer.(http.Flusher); ok { + flusher.Flush() + } + _, _ = writer.Write([]byte("overflow")) + })) + t.Cleanup(server.Close) + + tempDir := t.TempDir() + dependencies := defaultRemoteSourceDependenciesForTest() + dependencies.createTemp = func(_ string, pattern string) (*os.File, error) { + return os.CreateTemp(tempDir, pattern) + } + rawURL := server.URL + "/site.zip?token=chunk-secret" + _, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: rawURL, + NetworkPolicy: RemoteNetworkPolicyTrustedInternal, + MaxPackageBytes: maxPackageBytes, + }, dependencies) + if !errors.Is(err, errRemoteProviderTooLarge) { + t.Fatalf("fetchRemoteSource() error = %v, want actual stream limit", err) + } + assertRemoteSourceTempDirEmpty(t, tempDir) + assertRemoteSourceErrorRedacted(t, err, rawURL, "chunk-secret", "token=") +} + +func TestFetchRemoteSourceRejectsContentLengthBeforeCreatingTemp(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Length", "4096") + writer.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + var createCount atomic.Int32 + dependencies := defaultRemoteSourceDependenciesForTest() + dependencies.createTemp = func(directory string, pattern string) (*os.File, error) { + createCount.Add(1) + return os.CreateTemp(directory, pattern) + } + _, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: server.URL + "/site.zip", + NetworkPolicy: RemoteNetworkPolicyTrustedInternal, + MaxPackageBytes: 1024, + }, dependencies) + if !errors.Is(err, errRemoteProviderTooLarge) { + t.Fatalf("fetchRemoteSource() error = %v, want Content-Length rejection", err) + } + if createCount.Load() != 0 { + t.Fatalf("CreateTemp called %d times before Content-Length rejection", createCount.Load()) + } +} + +func TestFetchRemoteSourceSniffsAtLeast512BytesForTar(t *testing.T) { + packageBytes := make([]byte, remoteSourceMagicSniffBytes) + copy(packageBytes[257:], []byte("ustar")) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write(packageBytes) + })) + t.Cleanup(server.Close) + + candidate, err := FetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: server.URL + "/download", + NetworkPolicy: RemoteNetworkPolicyTrustedInternal, + MaxPackageBytes: int64(len(packageBytes) + 1), + }) + if err != nil { + t.Fatalf("FetchRemoteSource() error = %v", err) + } + defer func() { _ = candidate.Cleanup() }() + if candidate.Format != "tar" { + t.Fatalf("Format = %q, want tar detected at byte 257", candidate.Format) + } + if candidate.SafeLabel != "download.tar" { + t.Fatalf("SafeLabel = %q, want download.tar", candidate.SafeLabel) + } +} + +func TestFetchRemoteSourceRedactsURLHeadersAndBodyFromErrors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("X-Artifact-Secret", "header-secret") + _, _ = writer.Write([]byte("response-body-secret")) + })) + t.Cleanup(server.Close) + + tempDir := t.TempDir() + dependencies := defaultRemoteSourceDependenciesForTest() + dependencies.createTemp = func(_ string, pattern string) (*os.File, error) { + return os.CreateTemp(tempDir, pattern) + } + rawURL := server.URL + "/download?token=query-secret" + _, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: rawURL, + NetworkPolicy: RemoteNetworkPolicyTrustedInternal, + MaxPackageBytes: 1024, + }, dependencies) + if !errors.Is(err, errRemoteProviderUnsupported) { + t.Fatalf("fetchRemoteSource() error = %v, want unsupported archive", err) + } + assertRemoteSourceTempDirEmpty(t, tempDir) + assertRemoteSourceErrorRedacted( + t, + err, + rawURL, + "query-secret", + "header-secret", + "response-body-secret", + "token=", + ) +} + +func TestFetchRemoteSourceAllowsFiveRedirectsOnly(t *testing.T) { + packageBytes := makeRemoteSourceZIP(t) + var requestCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestCount.Add(1) + redirectNumber, _ := strconv.Atoi(strings.TrimPrefix(request.URL.Path, "/")) + if redirectNumber < remoteSourceMaxRedirects+1 { + writer.Header().Set("Location", "/"+strconv.Itoa(redirectNumber+1)) + writer.WriteHeader(http.StatusFound) + return + } + _, _ = writer.Write(packageBytes) + })) + t.Cleanup(server.Close) + + dependencies := mappedRemoteSourceDependencies(server.Listener.Addr().String(), staticPublicRemoteSourceResolver()) + _, err := fetchRemoteSource(t.Context(), RemoteSourceRequest{ + URL: "http://artifact.example/0", + NetworkPolicy: RemoteNetworkPolicyPublic, + MaxPackageBytes: int64(len(packageBytes) + 1), + }, dependencies) + if !errors.Is(err, errRemoteProviderRedirectLimit) { + t.Fatalf("fetchRemoteSource() error = %v, want redirect limit", err) + } + if requestCount.Load() != remoteSourceMaxRedirects+1 { + t.Fatalf("request count = %d, want initial plus five redirects", requestCount.Load()) + } +} + +func staticPublicRemoteSourceResolver() remoteSourceResolver { + return remoteSourceResolverFunc(func(context.Context, string, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("93.184.216.34")}, nil + }) +} + +func defaultRemoteSourceDependenciesForTest() remoteSourceDependencies { + dialer := &net.Dialer{} + return remoteSourceDependencies{ + resolver: net.DefaultResolver, + dialContext: dialer.DialContext, + createTemp: os.CreateTemp, + } +} + +func mappedRemoteSourceDependencies(targetAddress string, resolver remoteSourceResolver) remoteSourceDependencies { + dialer := &net.Dialer{} + return remoteSourceDependencies{ + resolver: resolver, + dialContext: func(ctx context.Context, network string, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, targetAddress) + }, + createTemp: os.CreateTemp, + } +} + +func makeRemoteSourceZIP(t *testing.T) []byte { + t.Helper() + var buffer bytes.Buffer + archive := zip.NewWriter(&buffer) + file, err := archive.Create("index.html") + if err != nil { + t.Fatalf("zip.Create() error = %v", err) + } + if _, err := file.Write([]byte("

OpenFlare

")); err != nil { + t.Fatalf("zip entry Write() error = %v", err) + } + if err := archive.Close(); err != nil { + t.Fatalf("zip.Close() error = %v", err) + } + return buffer.Bytes() +} + +func assertRemoteSourceTempDirEmpty(t *testing.T, directory string) { + t.Helper() + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatalf("ReadDir() error = %v", err) + } + if len(entries) != 0 { + t.Fatalf("temporary directory contains %d leaked files", len(entries)) + } +} + +func assertRemoteSourceErrorRedacted(t *testing.T, err error, sensitiveValues ...string) { + t.Helper() + if err == nil { + t.Fatal("expected an error") + } + message := err.Error() + for _, sensitiveValue := range sensitiveValues { + if sensitiveValue != "" && strings.Contains(message, sensitiveValue) { + t.Fatalf("error %q contains sensitive value %q", message, sensitiveValue) + } + } +} diff --git a/internal/apps/openflare/pages/source_runtime.go b/internal/apps/openflare/pages/source_runtime.go new file mode 100644 index 00000000..0dd8fc0f --- /dev/null +++ b/internal/apps/openflare/pages/source_runtime.go @@ -0,0 +1,309 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "strings" + "time" + + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const ( + pagesSourceCheckLeaseDuration = 2 * time.Minute + pagesSourceSyncLeaseDuration = 15 * time.Minute + sourceLeaseTokenBytes = 32 + sourceRuntimeErrorMaxBytes = 512 + sourceRevisionHexLength = 64 + + sourceRuntimeColumnSyncStatus = "sync_status" + sourceRuntimeColumnLastError = "last_error" + sourceRuntimeColumnLeaseToken = "lease_token" + sourceRuntimeColumnLeaseExpiresAt = "lease_expires_at" +) + +type sourceLeaseOutcome string + +const ( + sourceLeaseAcquired sourceLeaseOutcome = "acquired" + sourceLeaseBusy sourceLeaseOutcome = "busy" + sourceLeaseStale sourceLeaseOutcome = "stale" +) + +// sourceExecutionSnapshot captures every mutable value that can affect archive +// validation or the atomic activation decision. The queued payload deliberately +// does not carry project content configuration. +type sourceExecutionSnapshot struct { + ProjectID uint + SourceID uint + SourceConfigVersion int + ContentConfigVersion int + SourceType string + SourceIdentity string + RemoteURL string + RemoteNetworkPolicy string + RootDir string + EntryFile string + LeaseToken string + LeaseExpiresAt time.Time +} + +func acquireSourceLease( + ctx context.Context, + sourceID uint, + expectedConfigVersion int, + action string, +) (*sourceExecutionSnapshot, sourceLeaseOutcome, error) { + leaseDuration, status, err := sourceLeaseParameters(action) + if err != nil { + return nil, sourceLeaseStale, err + } + token, err := newSourceLeaseToken() + if err != nil { + return nil, sourceLeaseStale, err + } + now := time.Now() + expiresAt := now.Add(leaseDuration) + + result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ?", sourceID). + Where("lease_expires_at IS NULL OR lease_expires_at <= ?", now). + Where( + "EXISTS (SELECT 1 FROM of_pages_project_sources source WHERE source.id = ? AND source.config_version = ?)", + sourceID, + expectedConfigVersion, + ). + Updates(map[string]any{ + sourceRuntimeColumnLeaseToken: token, + sourceRuntimeColumnLeaseExpiresAt: expiresAt, + sourceRuntimeColumnSyncStatus: status, + sourceRuntimeColumnLastError: "", + }) + if result.Error != nil { + return nil, sourceLeaseStale, result.Error + } + if result.RowsAffected == 0 { + outcome, inspectErr := inspectSourceLeaseMiss(ctx, sourceID, expectedConfigVersion, now) + return nil, outcome, inspectErr + } + + snapshot, err := loadSourceExecutionSnapshot(ctx, sourceID, token) + if err != nil { + if errors.Is(err, errSourceLeaseSnapshotStale) { + return nil, sourceLeaseStale, nil + } + return nil, sourceLeaseStale, err + } + return snapshot, sourceLeaseAcquired, nil +} + +var errSourceLeaseSnapshotStale = errors.New("source lease snapshot stale") + +func loadSourceExecutionSnapshot( + ctx context.Context, + sourceID uint, + token string, +) (*sourceExecutionSnapshot, error) { + var snapshot sourceExecutionSnapshot + err := db.DB(ctx).Transaction(func(tx *gorm.DB) error { + var source model.PagesProjectSource + if err := tx.Where("id = ?", sourceID).First(&source).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return errSourceLeaseSnapshotStale + } + return err + } + var project model.PagesProject + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + First(&project, source.ProjectID).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return errSourceLeaseSnapshotStale + } + return err + } + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("id = ?", sourceID). + First(&source).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return errSourceLeaseSnapshotStale + } + return err + } + var runtime model.PagesProjectSourceRuntime + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("source_id = ?", source.ID). + First(&runtime).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return errSourceLeaseSnapshotStale + } + return err + } + // 必须在 runtime 行锁拿到后重新取时间，避免锁等待跨过 + // lease expiry 时仍使用事务开始前的旧时间继续执行。 + now := time.Now() + if runtime.LeaseToken != token || runtime.LeaseExpiresAt == nil || !runtime.LeaseExpiresAt.After(now) { + return errSourceLeaseSnapshotStale + } + snapshot = sourceExecutionSnapshot{ + ProjectID: project.ID, + SourceID: source.ID, + SourceConfigVersion: source.ConfigVersion, + ContentConfigVersion: project.ContentConfigVersion, + SourceType: source.SourceType, + SourceIdentity: source.SourceIdentity, + RemoteURL: source.RemoteURL, + RemoteNetworkPolicy: source.RemoteNetworkPolicy, + RootDir: project.RootDir, + EntryFile: project.EntryFile, + LeaseToken: token, + LeaseExpiresAt: *runtime.LeaseExpiresAt, + } + return nil + }) + if err != nil { + return nil, err + } + return &snapshot, nil +} + +func inspectSourceLeaseMiss( + ctx context.Context, + sourceID uint, + expectedConfigVersion int, + now time.Time, +) (sourceLeaseOutcome, error) { + var source model.PagesProjectSource + if err := db.DB(ctx).Where("id = ?", sourceID).First(&source).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return sourceLeaseStale, nil + } + return sourceLeaseStale, err + } + if source.ConfigVersion != expectedConfigVersion { + return sourceLeaseStale, nil + } + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return sourceLeaseStale, nil + } + return sourceLeaseStale, err + } + if runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(now) { + return sourceLeaseBusy, nil + } + return sourceLeaseStale, nil +} + +func sourceLeaseParameters(action string) (time.Duration, string, error) { + switch action { + case sourceActionCheck: + return pagesSourceCheckLeaseDuration, pagesSourceStatusChecking, nil + case sourceActionSync: + return pagesSourceSyncLeaseDuration, pagesSourceStatusSyncing, nil + default: + return 0, "", errors.New(errPagesSourceActionInvalid) + } +} + +func newSourceLeaseToken() (string, error) { + value := make([]byte, sourceLeaseTokenBytes) + if _, err := rand.Read(value); err != nil { + return "", err + } + return hex.EncodeToString(value), nil +} + +func renewSourceLease(ctx context.Context, snapshot *sourceExecutionSnapshot, duration time.Duration) (bool, error) { + if snapshot == nil || snapshot.SourceID == 0 || snapshot.LeaseToken == "" || duration <= 0 { + return false, errors.New(errPagesSourceLeaseLost) + } + now := time.Now() + expiresAt := now.Add(duration) + result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?", snapshot.SourceID, snapshot.LeaseToken, now). + Updates(map[string]any{sourceRuntimeColumnLeaseExpiresAt: expiresAt}) + if result.Error != nil { + return false, result.Error + } + if result.RowsAffected == 0 { + return false, nil + } + snapshot.LeaseExpiresAt = expiresAt + return true, nil +} + +func failSourceLease(ctx context.Context, snapshot *sourceExecutionSnapshot, message string) error { + if snapshot == nil || snapshot.SourceID == 0 || snapshot.LeaseToken == "" { + return nil + } + message = safeSourceRuntimeError(message) + now := time.Now() + return db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?", snapshot.SourceID, snapshot.LeaseToken, now). + Updates(map[string]any{ + sourceRuntimeColumnSyncStatus: pagesSourceStatusFailed, + sourceRuntimeColumnLastError: message, + sourceRuntimeColumnLeaseToken: "", + sourceRuntimeColumnLeaseExpiresAt: nil, + }).Error +} + +func safeSourceRuntimeError(message string) string { + message = strings.TrimSpace(message) + if message == "" { + return errPagesSourceSyncFailed + } + if len(message) > sourceRuntimeErrorMaxBytes { + message = message[:sourceRuntimeErrorMaxBytes] + } + return message +} + +func sourceLeaseIsBusy(ctx context.Context, sourceID uint) (bool, error) { + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil { + return false, err + } + return runtime.LeaseExpiresAt != nil && runtime.LeaseExpiresAt.After(time.Now()), nil +} + +// fenceAndNormalizeRuntime invalidates in-flight work while preserving safe +// seen/applied cursors. The caller must already hold the source row lock. +func fenceAndNormalizeRuntime(tx *gorm.DB, sourceID uint) error { + var runtime model.PagesProjectSourceRuntime + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("source_id = ?", sourceID). + First(&runtime).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + return err + } + return tx.Model(&runtime).Updates(map[string]any{ + sourceRuntimeColumnLeaseToken: "", + sourceRuntimeColumnLeaseExpiresAt: nil, + sourceRuntimeColumnSyncStatus: normalizedSourceRuntimeStatus(&runtime), + }).Error +} + +func sourceHasSameReleaseReplacement(runtime *model.PagesProjectSourceRuntime) bool { + if runtime == nil || runtime.LastSeenRevision == "" || runtime.LastSeenRevision == runtime.LastAppliedRevision { + return false + } + seen := sourceDetail{} + applied := sourceDetail{} + if unmarshalSourceDetail(runtime.LastSeenDetail, &seen) != nil || + unmarshalSourceDetail(runtime.LastAppliedDetail, &applied) != nil { + return false + } + return seen.ReleaseID != "" && seen.ReleaseID == applied.ReleaseID +} diff --git a/internal/apps/openflare/pages/source_runtime_test.go b/internal/apps/openflare/pages/source_runtime_test.go new file mode 100644 index 00000000..b971c424 --- /dev/null +++ b/internal/apps/openflare/pages/source_runtime_test.go @@ -0,0 +1,350 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" +) + +func TestSourceLeaseHeartbeatRenewsAndCancelsOnOwnershipLoss(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "lease-heartbeat") + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip", + RemoteNetworkPolicyPublic, + ) + snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync) + if err != nil || outcome != sourceLeaseAcquired || snapshot == nil { + t.Fatalf("acquireSourceLease(heartbeat) = (%+v, %q, %v), want acquired", snapshot, outcome, err) + } + + workCtx, heartbeat, err := startSourceLeaseHeartbeat(ctx, snapshot, 500*time.Millisecond, 20*time.Millisecond) + if err != nil { + t.Fatalf("startSourceLeaseHeartbeat() error = %v, want nil", err) + } + t.Cleanup(func() { _ = heartbeat.stop() }) + + var initial model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&initial).Error; err != nil { + t.Fatalf("load initial heartbeat runtime error = %v, want nil", err) + } + if initial.LeaseExpiresAt == nil { + t.Fatal("initial heartbeat expiry = nil, want non-nil") + } + deadline := time.Now().Add(2 * time.Second) + for { + var renewedRuntime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&renewedRuntime).Error; err != nil { + t.Fatalf("load renewed heartbeat runtime error = %v, want nil", err) + } + if renewedRuntime.LeaseExpiresAt != nil && renewedRuntime.LeaseExpiresAt.After(*initial.LeaseExpiresAt) { + break + } + if time.Now().After(deadline) { + t.Fatal("heartbeat did not extend lease before deadline") + } + time.Sleep(10 * time.Millisecond) + } + + if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ?", source.ID). + Update("lease_token", "replacement-owner").Error; err != nil { + t.Fatalf("replace heartbeat lease owner error = %v, want nil", err) + } + select { + case <-workCtx.Done(): + case <-time.After(2 * time.Second): + t.Fatal("heartbeat work context was not canceled after ownership loss") + } + if err := heartbeat.stop(); !errors.Is(err, errSourceLeaseHeartbeatLost) { + t.Fatalf("heartbeat.stop() error = %v, want %v", err, errSourceLeaseHeartbeatLost) + } + if err := heartbeat.stop(); !errors.Is(err, errSourceLeaseHeartbeatLost) { + t.Fatalf("heartbeat.stop() second error = %v, want stable %v", err, errSourceLeaseHeartbeatLost) + } +} + +func TestAcquireSourceLeaseConcurrentOnlyOneOwner(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "lease-concurrent") + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip", + RemoteNetworkPolicyPublic, + ) + type leaseResult struct { + snapshot *sourceExecutionSnapshot + outcome sourceLeaseOutcome + err error + } + results := make(chan leaseResult, 2) + var workers sync.WaitGroup + workers.Add(2) + for range 2 { + go func() { + defer workers.Done() + snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync) + results <- leaseResult{snapshot: snapshot, outcome: outcome, err: err} + }() + } + workers.Wait() + close(results) + + acquired := 0 + busy := 0 + for result := range results { + if result.err != nil { + t.Errorf("acquireSourceLease(concurrent) error = %v, want nil", result.err) + continue + } + switch result.outcome { + case sourceLeaseAcquired: + acquired++ + if result.snapshot == nil || result.snapshot.LeaseToken == "" { + t.Errorf("acquireSourceLease(concurrent acquired) snapshot = %+v, want token-bearing snapshot", result.snapshot) + } + case sourceLeaseBusy: + busy++ + if result.snapshot != nil { + t.Errorf("acquireSourceLease(concurrent busy) snapshot = %+v, want nil", result.snapshot) + } + default: + t.Errorf("acquireSourceLease(concurrent) outcome = %q, want acquired or busy", result.outcome) + } + } + if acquired != 1 || busy != 1 { + t.Errorf("concurrent lease outcomes = acquired:%d busy:%d, want 1 and 1", acquired, busy) + } +} + +func TestAcquireSourceLeaseMutualExclusionExpiryAndTerminalOwnership(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "lease-cas") + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip", + RemoteNetworkPolicyPublic, + ) + + first, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync) + if err != nil { + t.Fatalf("acquireSourceLease(first) error = %v, want nil", err) + } + if got, want := outcome, sourceLeaseAcquired; got != want { + t.Fatalf("acquireSourceLease(first) outcome = %q, want %q", got, want) + } + if first == nil || first.LeaseToken == "" { + t.Fatalf("acquireSourceLease(first) snapshot = %+v, want token-bearing snapshot", first) + } + + second, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync) + if err != nil { + t.Fatalf("acquireSourceLease(duplicate) error = %v, want nil", err) + } + if got, want := outcome, sourceLeaseBusy; got != want { + t.Errorf("acquireSourceLease(duplicate) outcome = %q, want %q", got, want) + } + if second != nil { + t.Errorf("acquireSourceLease(duplicate) snapshot = %+v, want nil", second) + } + + past := time.Now().Add(-time.Second) + if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ?", source.ID). + Update("lease_expires_at", &past).Error; err != nil { + t.Fatalf("expire first lease error = %v, want nil", err) + } + takeover, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync) + if err != nil { + t.Fatalf("acquireSourceLease(takeover) error = %v, want nil", err) + } + if got, want := outcome, sourceLeaseAcquired; got != want { + t.Fatalf("acquireSourceLease(takeover) outcome = %q, want %q", got, want) + } + if takeover == nil { + t.Fatal("acquireSourceLease(takeover) snapshot = nil, want non-nil") + } + if takeover.LeaseToken == "" || takeover.LeaseToken == first.LeaseToken { + t.Fatalf("takeover LeaseToken = %q, want non-empty token distinct from %q", takeover.LeaseToken, first.LeaseToken) + } + + renewed, err := renewSourceLease(ctx, first, pagesSourceSyncLeaseDuration) + if err != nil { + t.Fatalf("renewSourceLease(expired owner) error = %v, want nil", err) + } + if renewed { + t.Error("renewSourceLease(expired owner) = true, want false") + } + if err := failSourceLease(ctx, first, "stale worker must not win"); err != nil { + t.Fatalf("failSourceLease(expired owner) error = %v, want nil", err) + } + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil { + t.Fatalf("load runtime after takeover error = %v, want nil", err) + } + if got, want := runtime.LeaseToken, takeover.LeaseToken; got != want { + t.Errorf("runtime LeaseToken after stale terminal write = %q, want %q", got, want) + } + if got, want := runtime.SyncStatus, pagesSourceStatusSyncing; got != want { + t.Errorf("runtime SyncStatus after stale terminal write = %q, want %q", got, want) + } + + renewed, err = renewSourceLease(ctx, takeover, pagesSourceSyncLeaseDuration) + if err != nil { + t.Fatalf("renewSourceLease(current owner) error = %v, want nil", err) + } + if !renewed { + t.Error("renewSourceLease(current owner) = false, want true") + } + if err := failSourceLease(ctx, takeover, errPagesSourceSyncFailed); err != nil { + t.Fatalf("failSourceLease(current owner) error = %v, want nil", err) + } + var failedRuntime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&failedRuntime).Error; err != nil { + t.Fatalf("load failed runtime error = %v, want nil", err) + } + if got, want := failedRuntime.SyncStatus, pagesSourceStatusFailed; got != want { + t.Errorf("failed runtime SyncStatus = %q, want %q", got, want) + } + if failedRuntime.LeaseToken != "" || failedRuntime.LeaseExpiresAt != nil { + t.Errorf("failed runtime lease = (%q, %v), want cleared", failedRuntime.LeaseToken, failedRuntime.LeaseExpiresAt) + } +} + +func TestSourceConfigAndProjectContentChangesFenceLease(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "lease-fence") + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip?token=first", + RemoteNetworkPolicyPublic, + ) + + configSnapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync) + if err != nil || outcome != sourceLeaseAcquired { + t.Fatalf("acquireSourceLease(config fence) = (%+v, %q, %v), want acquired", configSnapshot, outcome, err) + } + if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: "https://example.com/site.zip?token=second", + RemoteNetworkPolicy: RemoteNetworkPolicyPublic, + }); err != nil { + t.Fatalf("UpdateSource(config fence) error = %v, want nil", err) + } + renewed, err := renewSourceLease(ctx, configSnapshot, pagesSourceSyncLeaseDuration) + if err != nil { + t.Fatalf("renewSourceLease(after source update) error = %v, want nil", err) + } + if renewed { + t.Error("renewSourceLease(after source update) = true, want false") + } + var updatedSource model.PagesProjectSource + if err := db.DB(ctx).Where("id = ?", source.ID).First(&updatedSource).Error; err != nil { + t.Fatalf("load updated source error = %v, want nil", err) + } + if got, want := updatedSource.ConfigVersion, source.ConfigVersion+1; got != want { + t.Errorf("updated source ConfigVersion = %d, want %d", got, want) + } + if snapshot, staleOutcome, staleErr := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync); staleErr != nil || staleOutcome != sourceLeaseStale || snapshot != nil { + t.Errorf("acquireSourceLease(old config) = (%+v, %q, %v), want (nil, %q, nil)", snapshot, staleOutcome, staleErr, sourceLeaseStale) + } + + contentSnapshot, outcome, err := acquireSourceLease(ctx, source.ID, updatedSource.ConfigVersion, sourceActionSync) + if err != nil || outcome != sourceLeaseAcquired { + t.Fatalf("acquireSourceLease(content fence) = (%+v, %q, %v), want acquired", contentSnapshot, outcome, err) + } + if _, err := UpdateProject(ctx, project.ID, Input{ + Name: project.Name, + Slug: project.Slug, + Enabled: true, + RootDir: "dist", + EntryFile: "index.html", + }); err != nil { + t.Fatalf("UpdateProject(content fence) error = %v, want nil", err) + } + renewed, err = renewSourceLease(ctx, contentSnapshot, pagesSourceSyncLeaseDuration) + if err != nil { + t.Fatalf("renewSourceLease(after content update) error = %v, want nil", err) + } + if renewed { + t.Error("renewSourceLease(after content update) = true, want false") + } + storedProject, err := model.GetPagesProjectByID(ctx, project.ID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err) + } + if got, want := storedProject.ContentConfigVersion, project.ContentConfigVersion+1; got != want { + t.Errorf("ContentConfigVersion after RootDir update = %d, want %d", got, want) + } + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil { + t.Fatalf("load fenced runtime error = %v, want nil", err) + } + if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil { + t.Errorf("content-fenced runtime lease = (%q, %v), want cleared", runtime.LeaseToken, runtime.LeaseExpiresAt) + } +} + +func TestSourceRuntimeUsesOnlySixDocumentedStates(t *testing.T) { + states := []string{ + pagesSourceStatusIdle, + pagesSourceStatusChecking, + pagesSourceStatusUpdateAvailable, + pagesSourceStatusSyncing, + pagesSourceStatusFailed, + pagesSourceStatusAttention, + } + seen := make(map[string]struct{}, len(states)) + for _, state := range states { + if strings.TrimSpace(state) == "" { + t.Errorf("documented source state = %q, want non-empty", state) + } + if _, exists := seen[state]; exists { + t.Errorf("documented source state %q is duplicated", state) + } + seen[state] = struct{}{} + } + if got, want := len(seen), 6; got != want { + t.Errorf("unique source states = %d, want %d", got, want) + } + + updateRuntime := &model.PagesProjectSourceRuntime{ + LastSeenRevision: strings.Repeat("a", 64), + LastAppliedRevision: strings.Repeat("b", 64), + LastSeenDetail: `{"release_id":"new"}`, + LastAppliedDetail: `{"release_id":"old"}`, + } + if got, want := normalizedSourceRuntimeStatus(updateRuntime), pagesSourceStatusUpdateAvailable; got != want { + t.Errorf("normalizedSourceRuntimeStatus(update) = %q, want %q", got, want) + } + attentionRuntime := &model.PagesProjectSourceRuntime{ + LastSeenRevision: strings.Repeat("a", 64), + LastAppliedRevision: strings.Repeat("b", 64), + LastSeenDetail: `{"release_id":"same"}`, + LastAppliedDetail: `{"release_id":"same"}`, + } + if got, want := normalizedSourceRuntimeStatus(attentionRuntime), pagesSourceStatusAttention; got != want { + t.Errorf("normalizedSourceRuntimeStatus(attention) = %q, want %q", got, want) + } + if got, want := normalizedSourceRuntimeStatus(&model.PagesProjectSourceRuntime{}), pagesSourceStatusIdle; got != want { + t.Errorf("normalizedSourceRuntimeStatus(idle) = %q, want %q", got, want) + } +} diff --git a/internal/apps/openflare/pages/source_sync.go b/internal/apps/openflare/pages/source_sync.go new file mode 100644 index 00000000..78430cb9 --- /dev/null +++ b/internal/apps/openflare/pages/source_sync.go @@ -0,0 +1,715 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path" + "sort" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/Rain-kl/Wavelet/internal/apps/upload" + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" + "github.com/Rain-kl/Wavelet/internal/task" + "github.com/Rain-kl/Wavelet/pkg/logger" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const ( + pagesSourceTriggerManualSync = "manual_sync" + pagesSourceCreatedBySystem = "system:pages-source-sync" + pagesSourceHeartbeatInterval = pagesSourceSyncLeaseDuration / 3 + pagesSourceCleanupTimeout = 15 * time.Second +) + +var ( + errSourceFinalFence = errors.New("pages source final fence rejected") + errSourceLeaseHeartbeatLost = errors.New("pages source lease heartbeat lost") + sourceCommitNow = time.Now +) + +type sourceSyncOutcome struct { + Deployment *DeploymentView + Reused bool + Stale bool +} + +type preparedRemoteSource struct { + Candidate *SourceCandidate + Manifest *deploymentManifest + Detail sourceDetail + DetailJSON string +} + +type sourceIngestState struct { + Result upload.IngestResult + HasIngest bool + Referenced bool +} + +type sourceCommitState struct { + Project *model.PagesProject + Source *model.PagesProjectSource + Runtime *model.PagesProjectSourceRuntime + Now time.Time +} + +type sourceLeaseHeartbeat struct { + cancel context.CancelFunc + done <-chan error + stopOnce sync.Once + stopErr error +} + +func startSourceLeaseHeartbeat( + ctx context.Context, + snapshot *sourceExecutionSnapshot, + leaseDuration time.Duration, + interval time.Duration, +) (context.Context, *sourceLeaseHeartbeat, error) { + if snapshot == nil || leaseDuration <= 0 || interval <= 0 || interval >= leaseDuration { + return nil, nil, errors.New(errPagesSourceLeaseLost) + } + renewed, err := renewSourceLease(ctx, snapshot, leaseDuration) + if err != nil { + return nil, nil, err + } + if !renewed { + return nil, nil, errSourceLeaseHeartbeatLost + } + + workCtx, cancel := context.WithCancel(ctx) + done := make(chan error, 1) + heartbeat := &sourceLeaseHeartbeat{cancel: cancel, done: done} + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-workCtx.Done(): + done <- nil + return + case <-ticker.C: + renewed, renewErr := renewSourceLease(workCtx, snapshot, leaseDuration) + if renewErr != nil { + if workCtx.Err() != nil { + done <- nil + return + } + done <- renewErr + cancel() + return + } + if !renewed { + done <- errSourceLeaseHeartbeatLost + cancel() + return + } + } + } + }() + return workCtx, heartbeat, nil +} + +func (heartbeat *sourceLeaseHeartbeat) stop() error { + if heartbeat == nil { + return nil + } + heartbeat.stopOnce.Do(func() { + heartbeat.cancel() + heartbeat.stopErr = <-heartbeat.done + }) + return heartbeat.stopErr +} + +func sourceHeartbeatOutcome(err error) (*sourceSyncOutcome, error) { + if errors.Is(err, errSourceLeaseHeartbeatLost) { + return &sourceSyncOutcome{Stale: true}, nil + } + return nil, err +} + +func recordSourceLeaseFailure(ctx context.Context, snapshot *sourceExecutionSnapshot) { + cleanupCtx, cancel := sourceCleanupContext(ctx) + defer cancel() + if err := failSourceLease(cleanupCtx, snapshot, errPagesSourceSyncFailed); err != nil { + var sourceID uint + if snapshot != nil { + sourceID = snapshot.SourceID + } + logger.WarnF(cleanupCtx, "[PagesSource] record failed runtime state failed: source_id=%d error=%v", sourceID, err) + } +} + +func sourceCleanupContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), pagesSourceCleanupTimeout) +} + +func syncRemoteSource( + ctx context.Context, + snapshot *sourceExecutionSnapshot, + actor string, +) (outcome *sourceSyncOutcome, resultErr error) { + if snapshot == nil || snapshot.SourceType != PagesSourceTypeRemoteURL { + return nil, errors.New(errPagesSourceTypeUnsupported) + } + actor = strings.TrimSpace(actor) + if actor == "" { + return nil, errors.New(errPagesSourceActionInvalid) + } + defer func() { + if resultErr != nil { + recordSourceLeaseFailure(ctx, snapshot) + } + }() + + workCtx, heartbeat, err := startSourceLeaseHeartbeat( + ctx, + snapshot, + pagesSourceSyncLeaseDuration, + pagesSourceHeartbeatInterval, + ) + if err != nil { + return sourceHeartbeatOutcome(err) + } + defer func() { + _ = heartbeat.stop() + }() + + limits := resolvePagesLimits(workCtx) + prepared, err := prepareRemoteSource(workCtx, snapshot, limits) + if err != nil { + if heartbeatErr := heartbeat.stop(); heartbeatErr != nil { + return sourceHeartbeatOutcome(heartbeatErr) + } + return nil, err + } + defer func() { + if cleanupErr := prepared.Candidate.Cleanup(); cleanupErr != nil { + logger.WarnF(ctx, "[PagesSource] cleanup temporary package failed: source_id=%d error=%v", snapshot.SourceID, cleanupErr) + } + }() + + ingestState, err := resolveSourceIngest(workCtx, snapshot, prepared) + if err != nil { + if heartbeatErr := heartbeat.stop(); heartbeatErr != nil { + return sourceHeartbeatOutcome(heartbeatErr) + } + return nil, err + } + defer func() { + compensateSourceIngest(ctx, snapshot, ingestState) + }() + + if heartbeatErr := heartbeat.stop(); heartbeatErr != nil { + return sourceHeartbeatOutcome(heartbeatErr) + } + renewed, err := renewSourceLease(ctx, snapshot, pagesSourceSyncLeaseDuration) + if err != nil { + return nil, err + } + if !renewed { + return &sourceSyncOutcome{Stale: true}, nil + } + + task.AppendLog(ctx, "[activate] 正在原子切换生产部署") + deployment, reused, referenced, err := commitSourceDeployment( + ctx, + snapshot, + prepared.Candidate.Checksum, + prepared.Detail, + prepared.DetailJSON, + actor, + prepared.Manifest, + ingestState.Result, + ingestState.HasIngest, + ) + ingestState.Referenced = referenced + if errors.Is(err, errSourceFinalFence) { + return &sourceSyncOutcome{Stale: true}, nil + } + if err != nil { + return nil, err + } + ingestState.Referenced = ingestState.HasIngest && deployment.UploadID == ingestState.Result.Upload.ID + + if pruneErr := pruneProjectDeploymentHistory(ctx, snapshot.ProjectID, limits.HistoryCount, 0); pruneErr != nil { + logger.ErrorF(ctx, + "[PagesSource] strict prune failed: project_id=%d source_id=%d keep=%d error=%v", + snapshot.ProjectID, + snapshot.SourceID, + limits.HistoryCount, + pruneErr, + ) + } + view := buildDeploymentView(deployment) + return &sourceSyncOutcome{Deployment: &view, Reused: reused}, nil +} + +func prepareRemoteSource( + ctx context.Context, + snapshot *sourceExecutionSnapshot, + limits pagesLimits, +) (*preparedRemoteSource, error) { + task.AppendLog(ctx, "[download] 正在获取远程部署包") + candidate, err := FetchRemoteSource(ctx, RemoteSourceRequest{ + URL: snapshot.RemoteURL, + NetworkPolicy: snapshot.RemoteNetworkPolicy, + MaxPackageBytes: limits.PackageBytes, + }) + if err != nil { + return nil, err + } + if candidate == nil || candidate.TempPath == "" || candidate.Checksum == "" || candidate.Format == "" { + return nil, errors.New(errPagesSourceSyncFailed) + } + rootDir, err := validateAndNormalizePagesRootDir(snapshot.RootDir) + if err != nil { + cleanupFailedRemoteCandidate(ctx, snapshot, candidate) + return nil, err + } + entryFile, err := validateAndNormalizePagesEntryFile(snapshot.EntryFile) + if err != nil { + cleanupFailedRemoteCandidate(ctx, snapshot, candidate) + return nil, err + } + task.AppendLog(ctx, "[verify] 正在校验归档结构与入口文件") + manifest, err := inspectPagesPackage(candidate.TempPath, candidate.Format, rootDir, entryFile, limits) + if err != nil { + cleanupFailedRemoteCandidate(ctx, snapshot, candidate) + return nil, err + } + detail := sourceDetail{Provider: PagesSourceTypeRemoteURL, Label: safeRemoteSourceLabel(candidate.SafeLabel)} + detailJSON, err := json.Marshal(detail) + if err != nil { + cleanupFailedRemoteCandidate(ctx, snapshot, candidate) + return nil, err + } + return &preparedRemoteSource{ + Candidate: candidate, + Manifest: manifest, + Detail: detail, + DetailJSON: string(detailJSON), + }, nil +} + +func cleanupFailedRemoteCandidate( + ctx context.Context, + snapshot *sourceExecutionSnapshot, + candidate *SourceCandidate, +) { + if err := candidate.Cleanup(); err != nil { + logger.WarnF(ctx, "[PagesSource] cleanup failed preparation package: source_id=%d error=%v", snapshot.SourceID, err) + } +} + +func resolveSourceIngest( + ctx context.Context, + snapshot *sourceExecutionSnapshot, + prepared *preparedRemoteSource, +) (*sourceIngestState, error) { + _, err := findSourceDeployment( + ctx, + snapshot.ProjectID, + snapshot.SourceIdentity, + prepared.Candidate.Checksum, + ) + if err == nil { + return &sourceIngestState{}, nil + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, err + } + task.AppendLog(ctx, "[ingest] 正在保存受管部署包") + result, err := ingestPagesDeploymentPackageWithSource( + ctx, + prepared.Candidate.TempPath, + prepared.Candidate.Checksum, + snapshot.ProjectID, + snapshot.SourceID, + prepared.Detail.Label, + prepared.Candidate.Format, + ) + if err != nil { + return nil, err + } + return &sourceIngestState{Result: result, HasIngest: true}, nil +} + +func compensateSourceIngest( + ctx context.Context, + snapshot *sourceExecutionSnapshot, + state *sourceIngestState, +) { + if state == nil || !state.HasIngest || !state.Result.Created || state.Referenced { + return + } + cleanupCtx, cancel := sourceCleanupContext(ctx) + defer cancel() + task.AppendLog(cleanupCtx, "[cleanup] 正在补偿未引用的部署包记录") + if err := removePagesUploadIfUnreferenced(cleanupCtx, snapshot.ProjectID, state.Result.Upload.ID); err != nil { + logger.ErrorF(cleanupCtx, + "[PagesSource] compensate upload failed: project_id=%d source_id=%d upload_id=%d error=%v", + snapshot.ProjectID, snapshot.SourceID, state.Result.Upload.ID, err, + ) + } +} + +func findSourceDeployment( + ctx context.Context, + projectID uint, + sourceIdentity string, + revision string, +) (*model.PagesDeployment, error) { + var deployment model.PagesDeployment + err := db.DB(ctx). + Where("project_id = ? AND source_identity = ? AND source_revision = ?", projectID, sourceIdentity, revision). + First(&deployment).Error + if err != nil { + return nil, err + } + return &deployment, nil +} + +func commitSourceDeployment( + ctx context.Context, + snapshot *sourceExecutionSnapshot, + revision string, + detail sourceDetail, + detailJSON string, + actor string, + manifest *deploymentManifest, + ingestResult upload.IngestResult, + hasIngest bool, +) (*model.PagesDeployment, bool, bool, error) { + if snapshot == nil || manifest == nil { + return nil, false, false, errors.New(errPagesSourceSyncFailed) + } + var committed model.PagesDeployment + reused := false + ingestReferenced := false + err := db.DB(ctx).Transaction(func(tx *gorm.DB) error { + state, err := lockSourceCommitState(tx, snapshot) + if err != nil { + return err + } + target, targetReused, err := resolveSourceDeploymentTx( + tx, state, revision, detail, detailJSON, actor, manifest, ingestResult, hasIngest, + ) + if err != nil { + return err + } + if err := lockSourceDeploymentUploadsTx(tx, target, ingestResult, hasIngest); err != nil { + return err + } + if err := ensureDeploymentEntry(tx, target.ID, state.Project.RootDir, state.Project.EntryFile); err != nil { + return err + } + if err := refreshSourceCommitLease(state, snapshot); err != nil { + return err + } + if err := activateSourceDeploymentTx(tx, state, target, revision, detailJSON); err != nil { + return err + } + committed = *target + reused = targetReused + ingestReferenced = hasIngest && target.UploadID == ingestResult.Upload.ID + return nil + }) + if err != nil { + return nil, false, false, err + } + return &committed, reused, ingestReferenced, nil +} + +func lockSourceCommitState(tx *gorm.DB, snapshot *sourceExecutionSnapshot) (*sourceCommitState, error) { + state := &sourceCommitState{} + var project model.PagesProject + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + First(&project, snapshot.ProjectID).Error; err != nil { + return nil, sourceFenceRecordError(err) + } + if project.ContentConfigVersion != snapshot.ContentConfigVersion { + return nil, errSourceFinalFence + } + var source model.PagesProjectSource + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("id = ? AND project_id = ?", snapshot.SourceID, snapshot.ProjectID). + First(&source).Error; err != nil { + return nil, sourceFenceRecordError(err) + } + if source.ConfigVersion != snapshot.SourceConfigVersion || + source.SourceIdentity != snapshot.SourceIdentity || + source.SourceType != snapshot.SourceType { + return nil, errSourceFinalFence + } + var runtime model.PagesProjectSourceRuntime + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("source_id = ?", source.ID). + First(&runtime).Error; err != nil { + return nil, sourceFenceRecordError(err) + } + state.Project = &project + state.Source = &source + state.Runtime = &runtime + if err := refreshSourceCommitLease(state, snapshot); err != nil { + return nil, err + } + return state, nil +} + +func refreshSourceCommitLease(state *sourceCommitState, snapshot *sourceExecutionSnapshot) error { + if state == nil || state.Runtime == nil || snapshot == nil { + return errSourceFinalFence + } + now := sourceCommitNow() + if state.Runtime.LeaseToken != snapshot.LeaseToken || state.Runtime.LeaseExpiresAt == nil || + !state.Runtime.LeaseExpiresAt.After(now) { + return errSourceFinalFence + } + state.Now = now + return nil +} + +func sourceFenceRecordError(err error) error { + if errors.Is(err, gorm.ErrRecordNotFound) { + return errSourceFinalFence + } + return err +} + +func resolveSourceDeploymentTx( + tx *gorm.DB, + state *sourceCommitState, + revision string, + detail sourceDetail, + detailJSON string, + actor string, + manifest *deploymentManifest, + ingestResult upload.IngestResult, + hasIngest bool, +) (*model.PagesDeployment, bool, error) { + var target model.PagesDeployment + err := tx.Where( + "project_id = ? AND source_identity = ? AND source_revision = ?", + state.Project.ID, + state.Source.SourceIdentity, + revision, + ).First(&target).Error + if err == nil { + return &target, true, nil + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, false, err + } + if !hasIngest { + return nil, false, errSourceFinalFence + } + return createSourceDeploymentTx( + tx, state, revision, detail, detailJSON, actor, manifest, ingestResult, + ) +} + +func createSourceDeploymentTx( + tx *gorm.DB, + state *sourceCommitState, + revision string, + detail sourceDetail, + detailJSON string, + actor string, + manifest *deploymentManifest, + ingestResult upload.IngestResult, +) (*model.PagesDeployment, bool, error) { + var maxNumber int + if err := tx.Model(&model.PagesDeployment{}). + Where("project_id = ?", state.Project.ID). + Select("COALESCE(MAX(deployment_number), 0)"). + Scan(&maxNumber).Error; err != nil { + return nil, false, err + } + identity := state.Source.SourceIdentity + revisionValue := revision + target := &model.PagesDeployment{ + ProjectID: state.Project.ID, + DeploymentNumber: maxNumber + 1, + Checksum: revision, + Status: model.PagesDeploymentStatusUploaded, + UploadID: ingestResult.Upload.ID, + FileCount: manifest.FileCount, + TotalSize: manifest.TotalSize, + CreatedBy: actor, + SourceType: state.Source.SourceType, + SourceIdentity: &identity, + SourceRevision: &revisionValue, + SourceLabel: detail.Label, + SourceMeta: detailJSON, + TriggerType: pagesSourceTriggerManualSync, + } + result := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(target) + if result.Error != nil { + return nil, false, result.Error + } + if result.RowsAffected == 0 { + return reloadSourceDeploymentTx(tx, state.Project.ID, identity, revision) + } + if err := createSourceDeploymentFilesTx(tx, target.ID, manifest.Files); err != nil { + return nil, false, err + } + return target, false, nil +} + +func reloadSourceDeploymentTx( + tx *gorm.DB, + projectID uint, + identity string, + revision string, +) (*model.PagesDeployment, bool, error) { + var target model.PagesDeployment + err := tx.Where( + "project_id = ? AND source_identity = ? AND source_revision = ?", + projectID, + identity, + revision, + ).First(&target).Error + return &target, true, err +} + +func createSourceDeploymentFilesTx(tx *gorm.DB, deploymentID uint, files []model.PagesDeploymentFile) error { + if len(files) == 0 { + return nil + } + for index := range files { + files[index].DeploymentID = deploymentID + } + return tx.Create(&files).Error +} + +func lockSourceDeploymentUploadsTx( + tx *gorm.DB, + target *model.PagesDeployment, + ingestResult upload.IngestResult, + hasIngest bool, +) error { + uploadIDs := []uint64{target.UploadID} + if hasIngest && ingestResult.Upload.ID != 0 && ingestResult.Upload.ID != target.UploadID { + uploadIDs = append(uploadIDs, ingestResult.Upload.ID) + } + sort.Slice(uploadIDs, func(i, j int) bool { return uploadIDs[i] < uploadIDs[j] }) + var records []model.Upload + if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}). + Where("id IN ?", uploadIDs). + Order("id asc"). + Find(&records).Error; err != nil { + return err + } + for index := range records { + if records[index].ID != target.UploadID { + continue + } + if records[index].Status == model.UploadStatusUsed && records[index].Type == upload.ReservedPagesDeploymentType { + return nil + } + break + } + return errSourceFinalFence +} + +func activateSourceDeploymentTx( + tx *gorm.DB, + state *sourceCommitState, + target *model.PagesDeployment, + revision string, + detailJSON string, +) error { + if err := tx.Model(&model.PagesDeployment{}). + Where("project_id = ?", state.Project.ID). + Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil { + return err + } + if err := tx.Model(target).Updates(map[string]any{ + "status": model.PagesDeploymentStatusActive, + "activated_at": &state.Now, + }).Error; err != nil { + return err + } + if err := tx.Model(state.Project).Update("active_deployment_id", target.ID).Error; err != nil { + return err + } + finishedAt := sourceCommitNow() + result := tx.Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?", + state.Runtime.SourceID, + state.Runtime.LeaseToken, + finishedAt, + ). + Updates(map[string]any{ + "last_seen_revision": revision, + "last_seen_detail": detailJSON, + "last_applied_revision": revision, + "last_applied_detail": detailJSON, + sourceRuntimeColumnSyncStatus: pagesSourceStatusIdle, + sourceRuntimeColumnLastError: "", + "last_checked_at": &finishedAt, + "last_synced_at": &finishedAt, + "next_check_at": nil, + sourceRuntimeColumnLeaseToken: "", + sourceRuntimeColumnLeaseExpiresAt: nil, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return errSourceFinalFence + } + return nil +} + +func safeRemoteSourceLabel(raw string) string { + value := strings.ReplaceAll(strings.ToValidUTF8(raw, ""), "\\", "/") + value = path.Base(strings.TrimSpace(value)) + if value == "." || value == "/" { + value = "" + } + var builder strings.Builder + for _, character := range value { + if character >= 0x20 && character != 0x7f { + builder.WriteRune(character) + } + } + value = strings.TrimSpace(builder.String()) + if value == "" { + value = defaultRemoteAssetLabel + } + if len(value) > remoteSourceMaxSafeLabelBytes { + value = value[:remoteSourceMaxSafeLabelBytes] + for !utf8.ValidString(value) { + _, size := utf8.DecodeLastRuneInString(value) + value = value[:len(value)-size] + } + } + return value +} + +func sourceSyncResultDetail(outcome *sourceSyncOutcome) string { + if outcome == nil || outcome.Deployment == nil { + return "" + } + payload := map[string]any{ + "deployment_id": outcome.Deployment.ID, + "reused": outcome.Reused, + } + encoded, err := json.Marshal(payload) + if err != nil { + return fmt.Sprintf(`{"deployment_id":%d}`, outcome.Deployment.ID) + } + return string(encoded) +} diff --git a/internal/apps/openflare/pages/source_sync_test.go b/internal/apps/openflare/pages/source_sync_test.go new file mode 100644 index 00000000..870446a8 --- /dev/null +++ b/internal/apps/openflare/pages/source_sync_test.go @@ -0,0 +1,604 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Rain-kl/Wavelet/internal/apps/upload" + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" + "github.com/Rain-kl/Wavelet/pkg/pagesarchive" + "gorm.io/gorm" +) + +func setupPagesSourceSyncTest(t *testing.T) context.Context { + t.Helper() + ctx := setupPagesSourceTest(t) + _, disableStorage := setupPagesStorageMock(t) + t.Cleanup(disableStorage) + return ctx +} + +func mustAcquireRemoteSyncLease( + t *testing.T, + ctx context.Context, + source *model.PagesProjectSource, +) *sourceExecutionSnapshot { + t.Helper() + snapshot, outcome, err := acquireSourceLease(ctx, source.ID, source.ConfigVersion, sourceActionSync) + if err != nil { + t.Fatalf("acquireSourceLease(source=%d) error = %v, want nil", source.ID, err) + } + if got, want := outcome, sourceLeaseAcquired; got != want { + t.Fatalf("acquireSourceLease(source=%d) outcome = %q, want %q", source.ID, got, want) + } + if snapshot == nil { + t.Fatalf("acquireSourceLease(source=%d) snapshot = nil, want non-nil", source.ID) + } + return snapshot +} + +func mustCreateActiveManualDeployment( + t *testing.T, + ctx context.Context, + projectID uint, + content string, +) *model.PagesDeployment { + t.Helper() + view, err := UploadDeployment( + ctx, + projectID, + testPagesMultipartFile(t, "manual.zip", testPagesZip(t, map[string]string{"index.html": content})), + "user:1", + ) + if err != nil { + t.Fatalf("UploadDeployment(project=%d) error = %v, want nil", projectID, err) + } + if _, err := ActivateDeploymentAs(ctx, projectID, view.ID, "user:1"); err != nil { + t.Fatalf("ActivateDeploymentAs(project=%d, deployment=%d) error = %v, want nil", projectID, view.ID, err) + } + deployment, err := model.GetPagesDeploymentByID(ctx, view.ID) + if err != nil { + t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", view.ID, err) + } + return deployment +} + +func newPagesArchiveServer(t *testing.T, status int, body []byte, beforeWrite func() error) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + if beforeWrite != nil { + if err := beforeWrite(); err != nil { + writer.WriteHeader(http.StatusInternalServerError) + return + } + } + writer.Header().Set("Content-Type", "application/zip") + writer.Header().Set("Content-Disposition", `attachment; filename="site.zip"`) + writer.WriteHeader(status) + _, _ = writer.Write(body) + })) + t.Cleanup(server.Close) + return server +} + +func TestSyncRemoteSourceAtomicallyActivatesAndReusesChecksum(t *testing.T) { + ctx := setupPagesSourceSyncTest(t) + project := mustCreatePagesSourceProject(t, ctx, "sync-success") + packageBytes := testPagesZip(t, map[string]string{ + "index.html": "remote-v1", + "assets/app.js": "console.log('ok')", + }) + server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil) + secret := "sync-query-secret" + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + server.URL+"/site.zip?token="+secret, + RemoteNetworkPolicyTrustedInternal, + ) + + firstSnapshot := mustAcquireRemoteSyncLease(t, ctx, source) + first, err := syncRemoteSource(ctx, firstSnapshot, "user:42") + if err != nil { + t.Fatalf("syncRemoteSource(first) error = %v, want nil", err) + } + if first == nil || first.Stale || first.Reused || first.Deployment == nil { + t.Fatalf("syncRemoteSource(first) = %+v, want new active deployment", first) + } + storedProject, err := model.GetPagesProjectByID(ctx, project.ID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err) + } + if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != first.Deployment.ID { + t.Fatalf("project ActiveDeploymentID = %v, want %d", storedProject.ActiveDeploymentID, first.Deployment.ID) + } + deployment, err := model.GetPagesDeploymentByID(ctx, first.Deployment.ID) + if err != nil { + t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", first.Deployment.ID, err) + } + expectedHash := sha256.Sum256(packageBytes) + if got, want := deployment.Checksum, hex.EncodeToString(expectedHash[:]); got != want { + t.Errorf("deployment Checksum = %q, want %q", got, want) + } + if got, want := deployment.Status, model.PagesDeploymentStatusActive; got != want { + t.Errorf("deployment Status = %q, want %q", got, want) + } + if got, want := deployment.SourceType, PagesSourceTypeRemoteURL; got != want { + t.Errorf("deployment SourceType = %q, want %q", got, want) + } + if deployment.SourceIdentity == nil || *deployment.SourceIdentity != source.SourceIdentity { + t.Errorf("deployment SourceIdentity = %v, want %q", deployment.SourceIdentity, source.SourceIdentity) + } + if deployment.SourceRevision == nil || *deployment.SourceRevision != deployment.Checksum { + t.Errorf("deployment SourceRevision = %v, want %q", deployment.SourceRevision, deployment.Checksum) + } + if got, want := deployment.CreatedBy, "user:42"; got != want { + t.Errorf("deployment CreatedBy = %q, want %q", got, want) + } + if got, want := deployment.TriggerType, pagesSourceTriggerManualSync; got != want { + t.Errorf("deployment TriggerType = %q, want %q", got, want) + } + if strings.Contains(deployment.SourceMeta, secret) || strings.Contains(deployment.SourceLabel, secret) { + t.Errorf("deployment provenance = label:%q meta:%q, want no query secret", deployment.SourceLabel, deployment.SourceMeta) + } + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil { + t.Fatalf("load source runtime error = %v, want nil", err) + } + if got, want := runtime.SyncStatus, pagesSourceStatusIdle; got != want { + t.Errorf("runtime SyncStatus = %q, want %q", got, want) + } + if got, want := runtime.LastAppliedRevision, deployment.Checksum; got != want { + t.Errorf("runtime LastAppliedRevision = %q, want %q", got, want) + } + if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil { + t.Errorf("runtime lease = (%q, %v), want cleared", runtime.LeaseToken, runtime.LeaseExpiresAt) + } + var uploadRecord model.Upload + if err := db.DB(ctx).First(&uploadRecord, deployment.UploadID).Error; err != nil { + t.Fatalf("load deployment upload %d error = %v, want nil", deployment.UploadID, err) + } + if got, want := uploadRecord.Status, model.UploadStatusUsed; got != want { + t.Errorf("deployment upload Status = %q, want %q", got, want) + } + if got, want := uploadRecord.Type, upload.ReservedPagesDeploymentType; got != want { + t.Errorf("deployment upload Type = %q, want %q", got, want) + } + if got, want := fmt.Sprint(uploadRecord.Metadata.Extra[pagesSourceIDMetadataKey]), fmt.Sprint(source.ID); got != want { + t.Errorf("deployment upload pages_source_id = %q, want %q", got, want) + } + + secondSnapshot := mustAcquireRemoteSyncLease(t, ctx, source) + second, err := syncRemoteSource(ctx, secondSnapshot, "user:42") + if err != nil { + t.Fatalf("syncRemoteSource(second) error = %v, want nil", err) + } + if second == nil || second.Stale || !second.Reused || second.Deployment == nil { + t.Fatalf("syncRemoteSource(second) = %+v, want reused active deployment", second) + } + if got, want := second.Deployment.ID, first.Deployment.ID; got != want { + t.Errorf("reused deployment ID = %d, want %d", got, want) + } + var deploymentCount, uploadCount int64 + if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("project_id = ?", project.ID).Count(&deploymentCount).Error; err != nil { + t.Fatalf("count source deployments error = %v, want nil", err) + } + if err := db.DB(ctx).Model(&model.Upload{}).Count(&uploadCount).Error; err != nil { + t.Fatalf("count source uploads error = %v, want nil", err) + } + if got, want := deploymentCount, int64(1); got != want { + t.Errorf("deployment count after identical sync = %d, want %d", got, want) + } + if got, want := uploadCount, int64(1); got != want { + t.Errorf("upload count after identical sync = %d, want %d", got, want) + } +} + +func TestSyncRemoteSourceDownloadFailureKeepsOldActive(t *testing.T) { + ctx := setupPagesSourceSyncTest(t) + project := mustCreatePagesSourceProject(t, ctx, "sync-download-fail") + oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active") + server := newPagesArchiveServer(t, http.StatusBadGateway, []byte("upstream failed"), nil) + secret := "download-failure-secret" + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + server.URL+"/site.zip?token="+secret, + RemoteNetworkPolicyTrustedInternal, + ) + + _, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:2") + if err == nil { + t.Fatal("syncRemoteSource(download failure) error = nil, want non-nil") + } + if strings.Contains(err.Error(), secret) { + t.Errorf("syncRemoteSource(download failure) error = %q, want no query secret", err) + } + assertPagesSyncFailureState(t, ctx, project.ID, source.ID, oldActive.ID, 1) +} + +func TestSyncRemoteSourceArchiveFailureKeepsOldActive(t *testing.T) { + ctx := setupPagesSourceSyncTest(t) + project := mustCreatePagesSourceProject(t, ctx, "sync-archive-fail") + oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active") + server := newPagesArchiveServer(t, http.StatusOK, []byte("not-a-valid-zip"), nil) + secret := "archive-failure-secret" + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + server.URL+"/site.zip?token="+secret, + RemoteNetworkPolicyTrustedInternal, + ) + + _, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:3") + if err == nil { + t.Fatal("syncRemoteSource(archive failure) error = nil, want non-nil") + } + if strings.Contains(err.Error(), secret) { + t.Errorf("syncRemoteSource(archive failure) error = %q, want no query secret", err) + } + assertPagesSyncFailureState(t, ctx, project.ID, source.ID, oldActive.ID, 1) +} + +func TestSyncRemoteSourceFinalFenceCompensatesIngest(t *testing.T) { + ctx := setupPagesSourceSyncTest(t) + project := mustCreatePagesSourceProject(t, ctx, "sync-final-fence") + oldActive := mustCreateActiveManualDeployment(t, ctx, project.ID, "old-active") + packageBytes := testPagesZip(t, map[string]string{"index.html": "never-activate"}) + mutationResult := make(chan error, 1) + server := newPagesArchiveServer(t, http.StatusOK, packageBytes, func() error { + err := db.DB(context.Background()).Model(&model.PagesProject{}). + Where("id = ?", project.ID). + Update("content_config_version", gorm.Expr("content_config_version + 1")).Error + mutationResult <- err + return err + }) + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + server.URL+"/site.zip?token=final-fence-secret", + RemoteNetworkPolicyTrustedInternal, + ) + + outcome, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:4") + if err != nil { + t.Fatalf("syncRemoteSource(final fence) error = %v, want nil stale outcome", err) + } + select { + case mutationErr := <-mutationResult: + if mutationErr != nil { + t.Fatalf("content version mutation error = %v, want nil", mutationErr) + } + case <-time.After(2 * time.Second): + t.Fatal("content version mutation was not observed") + } + if outcome == nil || !outcome.Stale || outcome.Deployment != nil { + t.Fatalf("syncRemoteSource(final fence) = %+v, want stale outcome without deployment", outcome) + } + storedProject, err := model.GetPagesProjectByID(ctx, project.ID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err) + } + if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != oldActive.ID { + t.Errorf("ActiveDeploymentID after final fence = %v, want %d", storedProject.ActiveDeploymentID, oldActive.ID) + } + var deployments []model.PagesDeployment + if err := db.DB(ctx).Where("project_id = ?", project.ID).Find(&deployments).Error; err != nil { + t.Fatalf("list deployments after final fence error = %v, want nil", err) + } + if got, want := len(deployments), 1; got != want { + t.Errorf("deployment count after final fence = %d, want %d", got, want) + } + var uploads []model.Upload + if err := db.DB(ctx).Order("id asc").Find(&uploads).Error; err != nil { + t.Fatalf("list uploads after final fence error = %v, want nil", err) + } + var compensated *model.Upload + for index := range uploads { + if fmt.Sprint(uploads[index].Metadata.Extra[pagesSourceIDMetadataKey]) == fmt.Sprint(source.ID) { + compensated = &uploads[index] + break + } + } + if compensated == nil { + t.Fatalf("source upload after final fence = nil, want compensated upload record") + } + if got, want := compensated.Status, model.UploadStatusDeleted; got != want { + t.Errorf("compensated upload Status = %q, want %q", got, want) + } + var danglingCount int64 + if err := db.DB(ctx).Model(&model.PagesDeployment{}). + Where("upload_id = ?", compensated.ID). + Count(&danglingCount).Error; err != nil { + t.Fatalf("count compensated upload references error = %v, want nil", err) + } + if got, want := danglingCount, int64(0); got != want { + t.Errorf("deployments referencing compensated upload = %d, want %d", got, want) + } +} + +func TestCommitSourceDeploymentRechecksLeaseAfterUploadLocks(t *testing.T) { + ctx := setupPagesSourceSyncTest(t) + project := mustCreatePagesSourceProject(t, ctx, "sync-expiry-recheck") + packageBytes := testPagesZip(t, map[string]string{"index.html": "expiry-recheck"}) + server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil) + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + server.URL+"/site.zip", + RemoteNetworkPolicyTrustedInternal, + ) + first, err := syncRemoteSource(ctx, mustAcquireRemoteSyncLease(t, ctx, source), "user:5") + if err != nil || first == nil || first.Deployment == nil { + t.Fatalf("syncRemoteSource(seed) = (%+v, %v), want deployment", first, err) + } + deployment, err := model.GetPagesDeploymentByID(ctx, first.Deployment.ID) + if err != nil { + t.Fatalf("GetPagesDeploymentByID(%d) error = %v, want nil", first.Deployment.ID, err) + } + if err := db.DB(ctx).Model(&model.PagesProject{}). + Where("id = ?", project.ID). + Update("active_deployment_id", nil).Error; err != nil { + t.Fatalf("clear active deployment error = %v, want nil", err) + } + if err := db.DB(ctx).Model(&model.PagesDeployment{}). + Where("id = ?", deployment.ID). + Update("status", model.PagesDeploymentStatusUploaded).Error; err != nil { + t.Fatalf("reset deployment status error = %v, want nil", err) + } + + snapshot := mustAcquireRemoteSyncLease(t, ctx, source) + expiresAt := time.Now().Add(time.Hour) + if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ? AND lease_token = ?", source.ID, snapshot.LeaseToken). + Update("lease_expires_at", &expiresAt).Error; err != nil { + t.Fatalf("set deterministic lease expiry error = %v, want nil", err) + } + originalNow := sourceCommitNow + nowCalls := 0 + sourceCommitNow = func() time.Time { + nowCalls++ + if nowCalls == 1 { + return expiresAt.Add(-time.Second) + } + return expiresAt.Add(time.Second) + } + t.Cleanup(func() { sourceCommitNow = originalNow }) + + _, _, _, err = commitSourceDeployment( + ctx, + snapshot, + deployment.Checksum, + sourceDetail{Provider: PagesSourceTypeRemoteURL, Label: deployment.SourceLabel}, + deployment.SourceMeta, + "user:5", + &deploymentManifest{}, + upload.IngestResult{}, + false, + ) + if !errors.Is(err, errSourceFinalFence) { + t.Fatalf("commitSourceDeployment(expired after upload lock) error = %v, want %v", err, errSourceFinalFence) + } + if nowCalls != 2 { + t.Fatalf("sourceCommitNow calls = %d, want runtime-lock and post-upload checks", nowCalls) + } + storedProject, err := model.GetPagesProjectByID(ctx, project.ID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err) + } + if storedProject.ActiveDeploymentID != nil { + t.Errorf("ActiveDeploymentID after expiry recheck = %v, want nil", storedProject.ActiveDeploymentID) + } + storedDeployment, err := model.GetPagesDeploymentByID(ctx, deployment.ID) + if err != nil { + t.Fatalf("GetPagesDeploymentByID(%d) after expiry error = %v, want nil", deployment.ID, err) + } + if got, want := storedDeployment.Status, model.PagesDeploymentStatusUploaded; got != want { + t.Errorf("deployment status after expiry recheck = %q, want %q", got, want) + } +} + +func TestCompensateSourceIngestSurvivesCanceledParentContext(t *testing.T) { + ctx := setupPagesSourceSyncTest(t) + project := mustCreatePagesSourceProject(t, ctx, "sync-canceled-compensation") + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip", + RemoteNetworkPolicyPublic, + ) + packageBytes := testPagesZip(t, map[string]string{"index.html": "cancel-compensation"}) + packagePath := filepath.Join(t.TempDir(), "site.zip") + if err := os.WriteFile(packagePath, packageBytes, 0o600); err != nil { + t.Fatalf("write test package error = %v, want nil", err) + } + digest := sha256.Sum256(packageBytes) + result, err := ingestPagesDeploymentPackageWithSource( + ctx, + packagePath, + hex.EncodeToString(digest[:]), + project.ID, + source.ID, + "site.zip", + pagesarchive.FormatZip, + ) + if err != nil { + t.Fatalf("ingestPagesDeploymentPackageWithSource() error = %v, want nil", err) + } + if !result.Created { + t.Fatal("ingest result Created = false, want a compensatable record") + } + + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + compensateSourceIngest(canceledCtx, &sourceExecutionSnapshot{ + ProjectID: project.ID, + SourceID: source.ID, + }, &sourceIngestState{ + Result: result, + HasIngest: true, + }) + + var uploadRecord model.Upload + if err := db.DB(ctx).Where("id = ?", result.Upload.ID).First(&uploadRecord).Error; err != nil { + t.Fatalf("load compensated upload error = %v, want nil", err) + } + if got, want := uploadRecord.Status, model.UploadStatusDeleted; got != want { + t.Errorf("compensated upload status = %q, want %q", got, want) + } +} + +func assertPagesSyncFailureState( + t *testing.T, + ctx context.Context, + projectID uint, + sourceID uint, + oldActiveID uint, + wantDeploymentCount int64, +) { + t.Helper() + project, err := model.GetPagesProjectByID(ctx, projectID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", projectID, err) + } + if project.ActiveDeploymentID == nil || *project.ActiveDeploymentID != oldActiveID { + t.Errorf("project %d ActiveDeploymentID = %v, want %d", projectID, project.ActiveDeploymentID, oldActiveID) + } + var deploymentCount int64 + if err := db.DB(ctx).Model(&model.PagesDeployment{}). + Where("project_id = ?", projectID). + Count(&deploymentCount).Error; err != nil { + t.Fatalf("count project %d deployments error = %v, want nil", projectID, err) + } + if got, want := deploymentCount, wantDeploymentCount; got != want { + t.Errorf("project %d deployment count = %d, want %d", projectID, got, want) + } + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil { + t.Fatalf("load source %d runtime error = %v, want nil", sourceID, err) + } + if got, want := runtime.SyncStatus, pagesSourceStatusFailed; got != want { + t.Errorf("source %d runtime SyncStatus = %q, want %q", sourceID, got, want) + } + if runtime.LeaseToken != "" || runtime.LeaseExpiresAt != nil { + t.Errorf("source %d runtime lease = (%q, %v), want cleared", sourceID, runtime.LeaseToken, runtime.LeaseExpiresAt) + } +} + +func TestCommitSourceDeploymentRejectsDeletedTargetUpload(t *testing.T) { + ctx := setupPagesSourceSyncTest(t) + project := mustCreatePagesSourceProject(t, ctx, "sync-deleted-upload") + packageBytes := testPagesZip(t, map[string]string{"index.html": "content"}) + server := newPagesArchiveServer(t, http.StatusOK, packageBytes, nil) + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + server.URL+"/site.zip", + RemoteNetworkPolicyTrustedInternal, + ) + snapshot := mustAcquireRemoteSyncLease(t, ctx, source) + + // A pre-existing source deployment whose upload was removed must never be + // reactivated into a dangling active pointer. + identity := source.SourceIdentity + revision := strings.Repeat("d", 64) + uploadRecord := &model.Upload{ + ID: 987654321, + UserID: 999, + FileName: "deleted.zip", + FilePath: "deleted.zip", + FileSize: 1, + MimeType: "application/zip", + Extension: "zip", + Hash: revision, + Type: upload.ReservedPagesDeploymentType, + Status: model.UploadStatusDeleted, + AccessMode: 0, + } + if err := db.DB(ctx).Create(uploadRecord).Error; err != nil { + t.Fatalf("create deleted upload error = %v, want nil", err) + } + deployment := &model.PagesDeployment{ + ProjectID: project.ID, + DeploymentNumber: 1, + Checksum: revision, + Status: model.PagesDeploymentStatusUploaded, + UploadID: uploadRecord.ID, + FileCount: 1, + TotalSize: 1, + CreatedBy: "user:1", + SourceType: PagesSourceTypeRemoteURL, + SourceIdentity: &identity, + SourceRevision: &revision, + SourceLabel: "deleted.zip", + SourceMeta: `{"provider":"remote_url","label":"deleted.zip"}`, + TriggerType: pagesSourceTriggerManualSync, + } + if err := db.DB(ctx).Create(deployment).Error; err != nil { + t.Fatalf("create source deployment error = %v, want nil", err) + } + if err := db.DB(ctx).Create(&model.PagesDeploymentFile{ + DeploymentID: deployment.ID, + Path: "index.html", + Size: 1, + Checksum: revision, + }).Error; err != nil { + t.Fatalf("create source deployment file error = %v, want nil", err) + } + manifest := &deploymentManifest{ + FileCount: 1, + TotalSize: 1, + EntryFile: "index.html", + } + _, _, _, err := commitSourceDeployment( + ctx, + snapshot, + revision, + sourceDetail{Provider: PagesSourceTypeRemoteURL, Label: "deleted.zip"}, + `{"provider":"remote_url","label":"deleted.zip"}`, + "user:1", + manifest, + upload.IngestResult{}, + false, + ) + if !errors.Is(err, errSourceFinalFence) { + t.Errorf("commitSourceDeployment(deleted upload) error = %v, want %v", err, errSourceFinalFence) + } + storedProject, err := model.GetPagesProjectByID(ctx, project.ID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err) + } + if storedProject.ActiveDeploymentID != nil { + t.Errorf("ActiveDeploymentID after deleted upload rejection = %v, want nil", storedProject.ActiveDeploymentID) + } + var activeCount int64 + if err := db.DB(ctx).Model(&model.PagesDeployment{}). + Where("project_id = ? AND status = ?", project.ID, model.PagesDeploymentStatusActive). + Count(&activeCount).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + t.Fatalf("count active deployments error = %v, want nil", err) + } + if got, want := activeCount, int64(0); got != want { + t.Errorf("active deployment count after deleted upload rejection = %d, want %d", got, want) + } +} diff --git a/internal/apps/openflare/pages/source_tasks.go b/internal/apps/openflare/pages/source_tasks.go new file mode 100644 index 00000000..5fef7d5b --- /dev/null +++ b/internal/apps/openflare/pages/source_tasks.go @@ -0,0 +1,275 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "errors" + "io" + "strconv" + "strings" + + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" + "github.com/Rain-kl/Wavelet/internal/task" + "github.com/Rain-kl/Wavelet/pkg/logger" + "gorm.io/gorm" +) + +const ( + // PagesSourceActionTask is the private Asynq task type for source actions. + PagesSourceActionTask = "openflare:pages_source_action" + // TaskTypePagesSourceAction is the internal task meta type. + TaskTypePagesSourceAction = "of_pages_source_action" + + sourceActionCheck = "check" + sourceActionSync = "sync" +) + +var errUnexpectedJSONTrailingValue = errors.New("unexpected trailing JSON value") + +// PagesSourceActionMeta is intentionally hidden from generic Admin task APIs. +var PagesSourceActionMeta = task.TaskMeta{ + Type: TaskTypePagesSourceAction, + AsynqTask: PagesSourceActionTask, + Name: "OpenFlare Pages 部署源操作", + Description: "检查或同步 Pages 项目部署源", + SupportsTime: false, + MaxRetry: 2, + Queue: task.QueueDefault, + Retryable: false, + InternalOnly: true, +} + +// SourceActionPayload is the credential-free internal queue contract. +type SourceActionPayload struct { + SourceID uint `json:"source_id"` + ConfigVersion int `json:"config_version"` + Action string `json:"action"` + Actor string `json:"actor"` + TargetRevision string `json:"target_revision"` + ConfirmedRevision string `json:"confirmed_revision"` +} + +// SourceActionHandler executes a validated source action. +type SourceActionHandler struct{} + +// ValidatePayload rejects unknown keys and normalizes the internal contract. +func (h *SourceActionHandler) ValidatePayload(payload []byte) ([]byte, error) { + var input SourceActionPayload + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil { + return nil, errors.New(errPagesSourceActionInvalid) + } + if err := ensureJSONEOF(decoder); err != nil { + return nil, errors.New(errPagesSourceActionInvalid) + } + input.Action = strings.TrimSpace(input.Action) + input.Actor = strings.TrimSpace(input.Actor) + input.TargetRevision = strings.TrimSpace(input.TargetRevision) + input.ConfirmedRevision = strings.TrimSpace(input.ConfirmedRevision) + if input.SourceID == 0 || input.ConfigVersion <= 0 || + (input.Action != sourceActionCheck && input.Action != sourceActionSync) || + !validPagesSourceActor(input.Actor) || + !validOptionalSourceRevision(input.TargetRevision) || + !validOptionalSourceRevision(input.ConfirmedRevision) { + return nil, errors.New(errPagesSourceActionInvalid) + } + return json.Marshal(input) +} + +// Execute validates again inside the worker and performs the source action. +func (h *SourceActionHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) { + normalized, err := h.ValidatePayload(payload) + if err != nil { + return nil, task.PermanentError(errPagesSourceActionInvalid) + } + var input SourceActionPayload + if err := json.Unmarshal(normalized, &input); err != nil { + return nil, task.PermanentError(errPagesSourceActionInvalid) + } + + var source model.PagesProjectSource + if err := db.DB(ctx).Where("id = ?", input.SourceID).First(&source).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + task.AppendLog(ctx, "[resolve] 来源已不存在，本次任务跳过") + return &task.TaskResult{Message: errPagesSourceActionStale}, nil + } + logger.ErrorF(ctx, "[PagesSource] load source failed: source_id=%d error=%v", input.SourceID, err) + return nil, errors.New(errPagesSourceSyncFailed) + } + if source.ConfigVersion != input.ConfigVersion { + task.AppendLog(ctx, "[resolve] 来源配置已变化，本次任务跳过") + return &task.TaskResult{Message: errPagesSourceActionStale}, nil + } + if input.Action == sourceActionCheck && source.SourceType == PagesSourceTypeRemoteURL { + return nil, task.PermanentError(errPagesSourceCheckUnsupported) + } + if source.SourceType != PagesSourceTypeRemoteURL { + return nil, task.PermanentError(errPagesSourceTypeUnsupported) + } + if input.TargetRevision != "" || input.ConfirmedRevision != "" { + return nil, task.PermanentError(errPagesSourceActionInvalid) + } + + task.AppendLog(ctx, "[resolve] 正在获取来源执行权") + snapshot, outcome, err := acquireSourceLease(ctx, input.SourceID, input.ConfigVersion, input.Action) + if err != nil { + logger.ErrorF(ctx, "[PagesSource] acquire lease failed: source_id=%d error=%v", input.SourceID, err) + return nil, errors.New(errPagesSourceSyncFailed) + } + switch outcome { + case sourceLeaseBusy: + task.AppendLog(ctx, "[resolve] 已有来源任务正在执行，本次任务跳过") + return &task.TaskResult{Message: errPagesSourceActionBusy}, nil + case sourceLeaseStale: + task.AppendLog(ctx, "[resolve] 来源配置或执行权已变化，本次任务跳过") + return &task.TaskResult{Message: errPagesSourceActionStale}, nil + } + + result, err := syncRemoteSource(ctx, snapshot, input.Actor) + if err != nil { + logger.ErrorF(ctx, "[PagesSource] sync failed: project_id=%d source_id=%d error=%v", snapshot.ProjectID, snapshot.SourceID, err) + if isPermanentSourceSyncError(err) { + return nil, task.PermanentError(errPagesSourceSyncFailed) + } + return nil, errors.New(errPagesSourceSyncFailed) + } + if result == nil || result.Stale { + task.AppendLog(ctx, "[activate] 来源配置或执行权已变化，本次任务未切换部署") + return &task.TaskResult{Message: errPagesSourceActionStale}, nil + } + message := "Pages 部署源同步并发布成功" + if result.Reused { + message = "Pages 部署源内容未变化，已重新激活现有部署" + } + return &task.TaskResult{Message: message, Detail: sourceSyncResultDetail(result)}, nil +} + +func ensureJSONEOF(decoder *json.Decoder) error { + var trailing any + err := decoder.Decode(&trailing) + if errors.Is(err, io.EOF) { + return nil + } + if err == nil { + return errUnexpectedJSONTrailingValue + } + return err +} + +func validPagesSourceActor(actor string) bool { + if actor == pagesSourceCreatedBySystem { + return true + } + if !strings.HasPrefix(actor, "user:") { + return false + } + id, err := strconv.ParseUint(strings.TrimPrefix(actor, "user:"), 10, 64) + return err == nil && id > 0 +} + +func validOptionalSourceRevision(value string) bool { + if value == "" { + return true + } + if len(value) != sourceRevisionHexLength { + return false + } + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 32 +} + +func isPermanentSourceSyncError(err error) bool { + if err == nil { + return false + } + message := err.Error() + return strings.Contains(message, errPagesPackageUnsupported) || + strings.Contains(message, errPagesPackageInvalid) || + strings.Contains(message, errPagesPackageEmpty) || + strings.Contains(message, errPagesPackageExtractedTooLarge) || + strings.Contains(message, errPagesPackageFileTooLarge) || + strings.Contains(message, errPagesEntryFileMissing) || + strings.Contains(message, errPagesSourceRemoteURLInvalid) || + strings.Contains(message, errPagesSourceNetworkPolicy) +} + +// DispatchSourceAction performs API preflight and enqueues a credential-free action. +func DispatchSourceAction( + ctx context.Context, + projectID uint, + action string, + actor string, + confirmedRevision string, +) (*SourceActionReceipt, error) { + action = strings.TrimSpace(action) + confirmedRevision = strings.TrimSpace(confirmedRevision) + if action != sourceActionCheck && action != sourceActionSync { + return nil, errors.New(errPagesSourceActionInvalid) + } + if !validPagesSourceActor(actor) { + return nil, errors.New(errPagesSourceActionInvalid) + } + + var source model.PagesProjectSource + if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errors.New(errPagesSourceNotFound) + } + return nil, err + } + if source.SourceType != PagesSourceTypeRemoteURL { + return nil, errors.New(errPagesSourceTypeUnsupported) + } + if action == sourceActionCheck { + return nil, errors.New(errPagesSourceCheckUnsupported) + } + if confirmedRevision != "" { + return nil, errors.New(errPagesSourceActionInvalid) + } + busy, err := sourceLeaseIsBusy(ctx, source.ID) + if err != nil { + return nil, err + } + if busy { + return nil, errors.New(errPagesSourceActionBusy) + } + + handler := &SourceActionHandler{} + rawPayload, err := json.Marshal(SourceActionPayload{ + SourceID: source.ID, + ConfigVersion: source.ConfigVersion, + Action: action, + Actor: actor, + TargetRevision: "", + ConfirmedRevision: confirmedRevision, + }) + if err != nil { + return nil, errors.New(errPagesSourceActionInvalid) + } + payload, err := handler.ValidatePayload(rawPayload) + if err != nil { + return nil, err + } + taskID, err := task.DispatchTask(ctx, TaskTypePagesSourceAction, payload, "manual") + if err != nil { + logger.ErrorF(ctx, "[PagesSource] dispatch action failed: project_id=%d source_id=%d action=%s error=%v", projectID, source.ID, action, err) + return nil, errors.New(errPagesSourceTaskDispatchFailed) + } + execution, err := model.GetTaskExecutionByTaskID(ctx, taskID) + if err != nil { + logger.ErrorF(ctx, "[PagesSource] load dispatched execution failed: source_id=%d task_id=%s error=%v", source.ID, taskID, err) + return nil, errors.New(errPagesSourceTaskDispatchFailed) + } + return &SourceActionReceipt{ + TaskID: taskID, + ExecutionID: strconv.FormatUint(execution.ID, 10), + Action: action, + }, nil +} diff --git a/internal/apps/openflare/pages/source_tasks_test.go b/internal/apps/openflare/pages/source_tasks_test.go new file mode 100644 index 00000000..92458dc6 --- /dev/null +++ b/internal/apps/openflare/pages/source_tasks_test.go @@ -0,0 +1,166 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/hibiken/asynq" +) + +func TestSourceActionPayloadValidationIsStrictAndCredentialFree(t *testing.T) { + handler := &SourceActionHandler{} + valid := SourceActionPayload{ + SourceID: 7, + ConfigVersion: 3, + Action: sourceActionSync, + Actor: "user:42", + } + raw, err := json.Marshal(valid) + if err != nil { + t.Fatalf("json.Marshal(valid payload) error = %v, want nil", err) + } + normalized, err := handler.ValidatePayload(raw) + if err != nil { + t.Fatalf("ValidatePayload(valid) error = %v, want nil", err) + } + var got SourceActionPayload + if err := json.Unmarshal(normalized, &got); err != nil { + t.Fatalf("json.Unmarshal(normalized payload) error = %v, want nil", err) + } + if got != valid { + t.Errorf("ValidatePayload(valid) = %+v, want %+v", got, valid) + } + for _, forbidden := range []string{"remote_url", "content_config_version", "expected_revision", "lease_token", "etag"} { + if strings.Contains(string(normalized), forbidden) { + t.Errorf("normalized payload = %s, want no forbidden field %q", normalized, forbidden) + } + } + + invalidPayloads := []struct { + name string + raw string + }{ + { + name: "unknown remote URL field", + raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42","remote_url":"https://example.com/site.zip?token=secret"}`, + }, + { + name: "unknown content version field", + raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42","content_config_version":9}`, + }, + { + name: "empty actor", + raw: `{"source_id":7,"config_version":3,"action":"sync","actor":""}`, + }, + { + name: "untrusted system actor", + raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"system"}`, + }, + { + name: "zero user actor", + raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:0"}`, + }, + { + name: "invalid action", + raw: `{"source_id":7,"config_version":3,"action":"activate","actor":"user:42"}`, + }, + { + name: "multiple JSON values", + raw: `{"source_id":7,"config_version":3,"action":"sync","actor":"user:42"} {}`, + }, + } + for _, test := range invalidPayloads { + t.Run(test.name, func(t *testing.T) { + normalized, err := handler.ValidatePayload([]byte(test.raw)) + if err == nil { + t.Errorf("ValidatePayload(%s) = %s, nil; want non-nil error", test.raw, normalized) + } + if err != nil && strings.Contains(err.Error(), "secret") { + t.Errorf("ValidatePayload(%s) error = %q, want credential-free error", test.name, err) + } + }) + } +} + +func TestSourceActionPayloadAcceptsOnlyRealActors(t *testing.T) { + tests := []struct { + actor string + want bool + }{ + {actor: "user:1", want: true}, + {actor: "user:18446744073709551615", want: true}, + {actor: pagesSourceCreatedBySystem, want: true}, + {actor: "", want: false}, + {actor: "user:0", want: false}, + {actor: "user:-1", want: false}, + {actor: "user:not-a-number", want: false}, + {actor: "system", want: false}, + } + for _, test := range tests { + if got := validPagesSourceActor(test.actor); got != test.want { + t.Errorf("validPagesSourceActor(%q) = %t, want %t", test.actor, got, test.want) + } + } +} + +func TestRemoteCheckActionIsPermanentWithoutExposingURL(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "task-remote-check") + secret := "task-query-secret" + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip?token="+secret, + RemoteNetworkPolicyPublic, + ) + raw, err := json.Marshal(SourceActionPayload{ + SourceID: source.ID, + ConfigVersion: source.ConfigVersion, + Action: sourceActionCheck, + Actor: "user:9", + }) + if err != nil { + t.Fatalf("json.Marshal(check payload) error = %v, want nil", err) + } + + result, err := (&SourceActionHandler{}).Execute(ctx, raw) + if result != nil { + t.Errorf("SourceActionHandler.Execute(remote check) result = %+v, want nil", result) + } + if err == nil { + t.Fatal("SourceActionHandler.Execute(remote check) error = nil, want permanent error") + } + if !errors.Is(err, asynq.SkipRetry) { + t.Errorf("SourceActionHandler.Execute(remote check) error = %v, want errors.Is(asynq.SkipRetry)", err) + } + if got, want := err.Error(), errPagesSourceCheckUnsupported; got != want { + t.Errorf("SourceActionHandler.Execute(remote check) error = %q, want %q", got, want) + } + if strings.Contains(err.Error(), secret) || strings.Contains(string(raw), secret) { + t.Errorf("remote check result error/payload = %q / %s, want no URL secret", err, raw) + } +} + +func TestPagesSourceActionMetaIsInternalOnly(t *testing.T) { + if !PagesSourceActionMeta.InternalOnly { + t.Error("PagesSourceActionMeta.InternalOnly = false, want true") + } + if PagesSourceActionMeta.Type != TaskTypePagesSourceAction { + t.Errorf("PagesSourceActionMeta.Type = %q, want %q", PagesSourceActionMeta.Type, TaskTypePagesSourceAction) + } + if PagesSourceActionMeta.AsynqTask != PagesSourceActionTask { + t.Errorf("PagesSourceActionMeta.AsynqTask = %q, want %q", PagesSourceActionMeta.AsynqTask, PagesSourceActionTask) + } + if PagesSourceActionMeta.Retryable { + t.Error("PagesSourceActionMeta.Retryable = true, want false for manual retry API") + } + if PagesSourceActionMeta.MaxRetry <= 0 { + t.Errorf("PagesSourceActionMeta.MaxRetry = %d, want bounded transient retries", PagesSourceActionMeta.MaxRetry) + } +} diff --git a/internal/apps/openflare/pages/source_test.go b/internal/apps/openflare/pages/source_test.go new file mode 100644 index 00000000..61f9dabe --- /dev/null +++ b/internal/apps/openflare/pages/source_test.go @@ -0,0 +1,386 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/Rain-kl/Wavelet/internal/db" + "github.com/Rain-kl/Wavelet/internal/model" +) + +func setupPagesSourceTest(t *testing.T) context.Context { + t.Helper() + cleanup := setupPagesTestDB(t) + t.Cleanup(cleanup) + sqlDB, err := db.DB(t.Context()).DB() + if err != nil { + t.Fatalf("db.DB().DB() error = %v, want nil", err) + } + // SQLite :memory: is scoped to one connection. Keeping one connection also + // makes lease tests exercise the production CAS without creating empty + // per-connection databases. + sqlDB.SetMaxOpenConns(1) + return t.Context() +} + +func mustCreatePagesSourceProject(t *testing.T, ctx context.Context, slug string) *model.PagesProject { + t.Helper() + view, err := CreateProject(ctx, Input{ + Name: "Source " + slug, + Slug: slug, + Enabled: true, + EntryFile: "index.html", + }) + if err != nil { + t.Fatalf("CreateProject(%q) error = %v, want nil", slug, err) + } + project, err := model.GetPagesProjectByID(ctx, view.ID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", view.ID, err) + } + return project +} + +func mustConfigureRemoteSource( + t *testing.T, + ctx context.Context, + projectID uint, + remoteURL string, + policy string, +) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime) { + t.Helper() + _, err := UpdateSource(ctx, projectID, SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: remoteURL, + RemoteNetworkPolicy: policy, + }) + if err != nil { + t.Fatalf("UpdateSource(%d, %q) error = %v, want nil", projectID, remoteURL, err) + } + var source model.PagesProjectSource + if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil { + t.Fatalf("load source for project %d error = %v, want nil", projectID, err) + } + var runtime model.PagesProjectSourceRuntime + if err := db.DB(ctx).Where("source_id = ?", source.ID).First(&runtime).Error; err != nil { + t.Fatalf("load runtime for source %d error = %v, want nil", source.ID, err) + } + return &source, &runtime +} + +func TestValidateRemoteSourceInputRejectsModeIncompatibleFields(t *testing.T) { + tests := []struct { + name string + input SourceUpdateInput + }{ + { + name: "missing source type", + input: SourceUpdateInput{ + RemoteURLSet: true, + RemoteURL: "https://example.com/site.zip", + }, + }, + { + name: "github type reserved for phase two", + input: SourceUpdateInput{ + SourceType: PagesSourceTypeGitHubRelease, + RepositoryURL: "https://github.com/example/site", + }, + }, + { + name: "remote rejects repository field", + input: SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: "https://example.com/site.zip", + RepositoryURL: "https://github.com/example/site", + }, + }, + { + name: "remote rejects automatic updates", + input: SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: "https://example.com/site.zip", + AutoUpdateEnabled: true, + }, + }, + { + name: "url value requires replacement flag", + input: SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURL: "https://example.com/site.zip", + }, + }, + { + name: "invalid network policy", + input: SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: "https://example.com/site.zip", + RemoteNetworkPolicy: "private", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := validateRemoteSourceInput(test.input); err == nil { + t.Errorf("validateRemoteSourceInput(%+v) error = nil, want non-nil", test.input) + } + }) + } +} + +func TestUpdateSourceNewRemoteRequiresExplicitURL(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "remote-requires-url") + _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + }) + if err == nil { + t.Fatal("UpdateSource(new remote without URL) error = nil, want non-nil") + } + if got, want := err.Error(), errPagesSourceRemoteURLRequired; got != want { + t.Errorf("UpdateSource(new remote without URL) error = %q, want %q", got, want) + } +} + +func TestRemoteSourceCRUDPreservesSecretAndResetsRuntimeByIdentity(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "remote-crud") + firstURL := "https://Artifacts.Example.com:443/dist/site.zip?token=first-secret&expires=1" + source, runtime := mustConfigureRemoteSource(t, ctx, project.ID, firstURL, RemoteNetworkPolicyPublic) + + if got, want := source.ConfigVersion, 1; got != want { + t.Errorf("new source ConfigVersion = %d, want %d", got, want) + } + if got, want := runtime.SyncStatus, pagesSourceStatusIdle; got != want { + t.Errorf("new runtime SyncStatus = %q, want %q", got, want) + } + view, err := GetSource(ctx, project.ID) + if err != nil { + t.Fatalf("GetSource(%d) error = %v, want nil", project.ID, err) + } + if got, want := view.DisplayURL, "https://Artifacts.Example.com:443/dist/site.zip?***"; got != want { + t.Errorf("GetSource(%d).DisplayURL = %q, want %q", project.ID, got, want) + } + encodedView, err := json.Marshal(view) + if err != nil { + t.Fatalf("json.Marshal(GetSource(%d)) error = %v, want nil", project.ID, err) + } + if strings.Contains(string(encodedView), "first-secret") || strings.Contains(string(encodedView), "expires=1") { + t.Errorf("GetSource(%d) JSON = %s, want credential-free view", project.ID, encodedView) + } + if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: firstURL, + RemoteNetworkPolicy: RemoteNetworkPolicyPublic, + }); err != nil { + t.Fatalf("UpdateSource(%d, no-op) error = %v, want nil", project.ID, err) + } + var unchangedSource model.PagesProjectSource + if err := db.DB(ctx).Where("id = ?", source.ID).First(&unchangedSource).Error; err != nil { + t.Fatalf("load no-op source error = %v, want nil", err) + } + if got, want := unchangedSource.ConfigVersion, source.ConfigVersion; got != want { + t.Errorf("no-op source ConfigVersion = %d, want unchanged %d", got, want) + } + + seenRevision := strings.Repeat("a", 64) + appliedRevision := strings.Repeat("b", 64) + future := time.Now().Add(time.Hour) + if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}). + Where("source_id = ?", source.ID). + Updates(map[string]any{ + "last_seen_revision": seenRevision, + "last_seen_detail": `{"provider":"remote_url","label":"new.zip"}`, + "last_applied_revision": appliedRevision, + "last_applied_detail": `{"provider":"remote_url","label":"old.zip"}`, + "sync_status": pagesSourceStatusSyncing, + "lease_token": "in-flight", + "lease_expires_at": &future, + }).Error; err != nil { + t.Fatalf("seed source runtime error = %v, want nil", err) + } + + // Omit the secret URL while changing policy. The stored URL and cursor must + // survive, while the in-flight lease is fenced. + if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: false, + RemoteNetworkPolicy: RemoteNetworkPolicyTrustedInternal, + }); err != nil { + t.Fatalf("UpdateSource(%d, preserve URL) error = %v, want nil", project.ID, err) + } + preservedSource, preservedRuntime, err := loadSourceByProject(ctx, project.ID) + if err != nil { + t.Fatalf("loadSourceByProject(%d) error = %v, want nil", project.ID, err) + } + if got, want := preservedSource.RemoteURL, firstURL; got != want { + t.Errorf("preserved RemoteURL = %q, want %q", got, want) + } + if got, want := preservedSource.ConfigVersion, 2; got != want { + t.Errorf("preserved source ConfigVersion = %d, want %d", got, want) + } + if got, want := preservedSource.SourceIdentity, source.SourceIdentity; got != want { + t.Errorf("preserved source identity = %q, want %q", got, want) + } + if got, want := preservedRuntime.LastSeenRevision, seenRevision; got != want { + t.Errorf("preserved LastSeenRevision = %q, want %q", got, want) + } + if got, want := preservedRuntime.SyncStatus, pagesSourceStatusUpdateAvailable; got != want { + t.Errorf("preserved runtime SyncStatus = %q, want %q", got, want) + } + if preservedRuntime.LeaseToken != "" || preservedRuntime.LeaseExpiresAt != nil { + t.Errorf("preserved runtime lease = (%q, %v), want cleared", preservedRuntime.LeaseToken, preservedRuntime.LeaseExpiresAt) + } + + // Replacing only the query secret keeps the canonical identity and cursors. + queryReplacementURL := "https://artifacts.example.com/dist/site.zip?token=second-secret" + if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: queryReplacementURL, + RemoteNetworkPolicy: RemoteNetworkPolicyTrustedInternal, + }); err != nil { + t.Fatalf("UpdateSource(%d, query replacement) error = %v, want nil", project.ID, err) + } + querySource, queryRuntime, err := loadSourceByProject(ctx, project.ID) + if err != nil { + t.Fatalf("loadSourceByProject(%d) after query replacement error = %v, want nil", project.ID, err) + } + if got, want := querySource.SourceIdentity, source.SourceIdentity; got != want { + t.Errorf("query replacement identity = %q, want %q", got, want) + } + if got, want := queryRuntime.LastSeenRevision, seenRevision; got != want { + t.Errorf("query replacement LastSeenRevision = %q, want %q", got, want) + } + + // Replacing the path changes identity and clears all remote cursors. + pathReplacementURL := "https://artifacts.example.com/dist/other.zip?token=third-secret" + if _, err := UpdateSource(ctx, project.ID, SourceUpdateInput{ + SourceType: PagesSourceTypeRemoteURL, + RemoteURLSet: true, + RemoteURL: pathReplacementURL, + RemoteNetworkPolicy: RemoteNetworkPolicyTrustedInternal, + }); err != nil { + t.Fatalf("UpdateSource(%d, path replacement) error = %v, want nil", project.ID, err) + } + pathSource, pathRuntime, err := loadSourceByProject(ctx, project.ID) + if err != nil { + t.Fatalf("loadSourceByProject(%d) after path replacement error = %v, want nil", project.ID, err) + } + if pathSource.SourceIdentity == source.SourceIdentity { + t.Errorf("path replacement identity = %q, want a new identity", pathSource.SourceIdentity) + } + if pathRuntime.LastSeenRevision != "" || pathRuntime.LastAppliedRevision != "" { + t.Errorf("path replacement cursors = (%q, %q), want empty", pathRuntime.LastSeenRevision, pathRuntime.LastAppliedRevision) + } + if got, want := pathRuntime.SyncStatus, pagesSourceStatusIdle; got != want { + t.Errorf("path replacement SyncStatus = %q, want %q", got, want) + } + pathView, err := GetSource(ctx, project.ID) + if err != nil { + t.Fatalf("GetSource(%d) after path replacement error = %v, want nil", project.ID, err) + } + pathJSON, err := json.Marshal(pathView) + if err != nil { + t.Fatalf("json.Marshal(path view) error = %v, want nil", err) + } + for _, secret := range []string{"first-secret", "second-secret", "third-secret"} { + if strings.Contains(string(pathJSON), secret) { + t.Errorf("path view JSON = %s, want no secret %q", pathJSON, secret) + } + } +} + +func TestRemoteSourceIdentityIgnoresQueryAndNormalizesDefaultPort(t *testing.T) { + first, err := parseRemoteSourceURL("HTTPS://Artifacts.Example.com:443/dist/../dist/site.zip?token=one") + if err != nil { + t.Fatalf("parseRemoteSourceURL(first) error = %v, want nil", err) + } + second, err := parseRemoteSourceURL("https://artifacts.example.com/dist/site.zip?token=two") + if err != nil { + t.Fatalf("parseRemoteSourceURL(second) error = %v, want nil", err) + } + if got, want := remoteSourceIdentity(first), remoteSourceIdentity(second); got != want { + t.Errorf("remoteSourceIdentity(first) = %q, want %q", got, want) + } +} + +func TestDeleteSourceIsIdempotentAndKeepsDeploymentState(t *testing.T) { + ctx := setupPagesSourceTest(t) + project := mustCreatePagesSourceProject(t, ctx, "source-delete") + source, _ := mustConfigureRemoteSource( + t, + ctx, + project.ID, + "https://example.com/site.zip?token=delete-secret", + RemoteNetworkPolicyPublic, + ) + deployment := &model.PagesDeployment{ + ProjectID: project.ID, + DeploymentNumber: 1, + Checksum: strings.Repeat("c", 64), + Status: model.PagesDeploymentStatusActive, + CreatedBy: "user:1", + SourceType: "manual_upload", + TriggerType: "manual_upload", + } + if err := db.DB(ctx).Create(deployment).Error; err != nil { + t.Fatalf("create deployment error = %v, want nil", err) + } + if err := db.DB(ctx).Model(&model.PagesProject{}). + Where("id = ?", project.ID). + Update("active_deployment_id", deployment.ID).Error; err != nil { + t.Fatalf("set active deployment error = %v, want nil", err) + } + + for attempt := 1; attempt <= 2; attempt++ { + view, err := DeleteSource(ctx, project.ID) + if err != nil { + t.Fatalf("DeleteSource(%d), attempt %d error = %v, want nil", project.ID, attempt, err) + } + if got, want := view.SourceType, PagesSourceTypeManual; got != want { + t.Errorf("DeleteSource(%d), attempt %d SourceType = %q, want %q", project.ID, attempt, got, want) + } + } + var sourceCount, runtimeCount, deploymentCount int64 + if err := db.DB(ctx).Model(&model.PagesProjectSource{}).Where("id = ?", source.ID).Count(&sourceCount).Error; err != nil { + t.Fatalf("count source error = %v, want nil", err) + } + if err := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).Where("source_id = ?", source.ID).Count(&runtimeCount).Error; err != nil { + t.Fatalf("count runtime error = %v, want nil", err) + } + if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("id = ?", deployment.ID).Count(&deploymentCount).Error; err != nil { + t.Fatalf("count deployment error = %v, want nil", err) + } + if sourceCount != 0 || runtimeCount != 0 || deploymentCount != 1 { + t.Errorf("DeleteSource counts = source:%d runtime:%d deployment:%d, want 0, 0, 1", sourceCount, runtimeCount, deploymentCount) + } + storedProject, err := model.GetPagesProjectByID(ctx, project.ID) + if err != nil { + t.Fatalf("GetPagesProjectByID(%d) error = %v, want nil", project.ID, err) + } + if storedProject.ActiveDeploymentID == nil || *storedProject.ActiveDeploymentID != deployment.ID { + t.Errorf("active deployment = %v, want %d", storedProject.ActiveDeploymentID, deployment.ID) + } + + manual, err := GetSource(ctx, project.ID) + if err != nil { + t.Fatalf("GetSource(%d) after delete error = %v, want nil", project.ID, err) + } + if got, want := fmt.Sprint(manual.SourceType), PagesSourceTypeManual; got != want { + t.Errorf("GetSource(%d).SourceType = %q, want %q", project.ID, got, want) + } +} diff --git a/internal/db/migrator/goose/postgres/202607190001_add_pages_source_runtime.sql b/internal/db/migrator/goose/postgres/202607190001_add_pages_source_runtime.sql new file mode 100644 index 00000000..70540ab4 --- /dev/null +++ b/internal/db/migrator/goose/postgres/202607190001_add_pages_source_runtime.sql @@ -0,0 +1,82 @@ +-- +goose Up +ALTER TABLE of_pages_projects + ADD COLUMN IF NOT EXISTS content_config_version INTEGER NOT NULL DEFAULT 0; + +ALTER TABLE of_pages_deployments + ADD COLUMN IF NOT EXISTS source_type VARCHAR(32) NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS source_identity CHAR(64), + ADD COLUMN IF NOT EXISTS source_revision CHAR(64), + ADD COLUMN IF NOT EXISTS source_label VARCHAR(255) NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS source_meta TEXT NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS trigger_type VARCHAR(32) NOT NULL DEFAULT ''; + +UPDATE of_pages_deployments +SET source_type = 'manual_upload', + trigger_type = 'manual_upload'; + +CREATE TABLE IF NOT EXISTS of_pages_project_sources ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL, + source_type VARCHAR(32) NOT NULL DEFAULT '', + remote_url TEXT NOT NULL DEFAULT '', + remote_network_policy VARCHAR(32) NOT NULL DEFAULT '', + github_repository VARCHAR(255) NOT NULL DEFAULT '', + release_selector VARCHAR(16) NOT NULL DEFAULT '', + release_tag VARCHAR(255) NOT NULL DEFAULT '', + asset_name VARCHAR(255) NOT NULL DEFAULT '', + auto_update_enabled BOOLEAN NOT NULL DEFAULT FALSE, + check_interval_minutes INTEGER NOT NULL DEFAULT 0, + config_version INTEGER NOT NULL DEFAULT 0, + source_identity CHAR(64) NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_project_sources_project_id + ON of_pages_project_sources (project_id); + +CREATE TABLE IF NOT EXISTS of_pages_project_source_runtime ( + source_id BIGINT PRIMARY KEY, + etag VARCHAR(512) NOT NULL DEFAULT '', + last_seen_revision CHAR(64) NOT NULL DEFAULT '', + last_seen_detail TEXT NOT NULL DEFAULT '', + last_applied_revision CHAR(64) NOT NULL DEFAULT '', + last_applied_detail TEXT NOT NULL DEFAULT '', + sync_status VARCHAR(32) NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + last_checked_at TIMESTAMPTZ, + last_synced_at TIMESTAMPTZ, + next_check_at TIMESTAMPTZ, + lease_expires_at TIMESTAMPTZ, + lease_token VARCHAR(64) NOT NULL DEFAULT '', + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_of_pages_project_source_runtime_next_check_at + ON of_pages_project_source_runtime (next_check_at); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_project_number + ON of_pages_deployments (project_id, deployment_number); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_source_revision + ON of_pages_deployments (project_id, source_identity, source_revision) + WHERE source_identity IS NOT NULL AND source_revision IS NOT NULL; + +-- +goose Down +DROP INDEX IF EXISTS idx_of_pages_deployments_source_revision; +DROP INDEX IF EXISTS idx_of_pages_deployments_project_number; +DROP INDEX IF EXISTS idx_of_pages_project_source_runtime_next_check_at; +DROP TABLE IF EXISTS of_pages_project_source_runtime; +DROP INDEX IF EXISTS idx_of_pages_project_sources_project_id; +DROP TABLE IF EXISTS of_pages_project_sources; + +ALTER TABLE of_pages_deployments + DROP COLUMN IF EXISTS trigger_type, + DROP COLUMN IF EXISTS source_meta, + DROP COLUMN IF EXISTS source_label, + DROP COLUMN IF EXISTS source_revision, + DROP COLUMN IF EXISTS source_identity, + DROP COLUMN IF EXISTS source_type; + +ALTER TABLE of_pages_projects + DROP COLUMN IF EXISTS content_config_version; diff --git a/internal/db/migrator/goose/sqlite/202607190001_add_pages_source_runtime.sql b/internal/db/migrator/goose/sqlite/202607190001_add_pages_source_runtime.sql new file mode 100644 index 00000000..da8012ef --- /dev/null +++ b/internal/db/migrator/goose/sqlite/202607190001_add_pages_source_runtime.sql @@ -0,0 +1,151 @@ +-- +goose Up +ALTER TABLE of_pages_projects + ADD COLUMN content_config_version INTEGER NOT NULL DEFAULT 0; + +ALTER TABLE of_pages_deployments + ADD COLUMN source_type TEXT NOT NULL DEFAULT ''; +ALTER TABLE of_pages_deployments + ADD COLUMN source_identity TEXT; +ALTER TABLE of_pages_deployments + ADD COLUMN source_revision TEXT; +ALTER TABLE of_pages_deployments + ADD COLUMN source_label TEXT NOT NULL DEFAULT ''; +ALTER TABLE of_pages_deployments + ADD COLUMN source_meta TEXT NOT NULL DEFAULT ''; +ALTER TABLE of_pages_deployments + ADD COLUMN trigger_type TEXT NOT NULL DEFAULT ''; + +UPDATE of_pages_deployments +SET source_type = 'manual_upload', + trigger_type = 'manual_upload'; + +CREATE TABLE IF NOT EXISTS of_pages_project_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + source_type TEXT NOT NULL DEFAULT '', + remote_url TEXT NOT NULL DEFAULT '', + remote_network_policy TEXT NOT NULL DEFAULT '', + github_repository TEXT NOT NULL DEFAULT '', + release_selector TEXT NOT NULL DEFAULT '', + release_tag TEXT NOT NULL DEFAULT '', + asset_name TEXT NOT NULL DEFAULT '', + auto_update_enabled INTEGER NOT NULL DEFAULT 0, + check_interval_minutes INTEGER NOT NULL DEFAULT 0, + config_version INTEGER NOT NULL DEFAULT 0, + source_identity TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_project_sources_project_id + ON of_pages_project_sources (project_id); + +CREATE TABLE IF NOT EXISTS of_pages_project_source_runtime ( + source_id INTEGER PRIMARY KEY, + etag TEXT NOT NULL DEFAULT '', + last_seen_revision TEXT NOT NULL DEFAULT '', + last_seen_detail TEXT NOT NULL DEFAULT '', + last_applied_revision TEXT NOT NULL DEFAULT '', + last_applied_detail TEXT NOT NULL DEFAULT '', + sync_status TEXT NOT NULL DEFAULT '', + last_error TEXT NOT NULL DEFAULT '', + last_checked_at DATETIME, + last_synced_at DATETIME, + next_check_at DATETIME, + lease_expires_at DATETIME, + lease_token TEXT NOT NULL DEFAULT '', + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_of_pages_project_source_runtime_next_check_at + ON of_pages_project_source_runtime (next_check_at); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_project_number + ON of_pages_deployments (project_id, deployment_number); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_deployments_source_revision + ON of_pages_deployments (project_id, source_identity, source_revision) + WHERE source_identity IS NOT NULL AND source_revision IS NOT NULL; + +-- +goose Down +DROP INDEX IF EXISTS idx_of_pages_deployments_source_revision; +DROP INDEX IF EXISTS idx_of_pages_deployments_project_number; +DROP INDEX IF EXISTS idx_of_pages_project_source_runtime_next_check_at; +DROP TABLE IF EXISTS of_pages_project_source_runtime; +DROP INDEX IF EXISTS idx_of_pages_project_sources_project_id; +DROP TABLE IF EXISTS of_pages_project_sources; + +-- SQLite 的 Down 必须重建受影响表，完整移除新增列并保留原有数据与索引。 +CREATE TABLE of_pages_deployments_before_source ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + deployment_number INTEGER NOT NULL, + checksum TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'uploaded', + upload_id INTEGER NOT NULL DEFAULT 0, + artifact_path TEXT NOT NULL, + file_count INTEGER NOT NULL DEFAULT 0, + total_size INTEGER NOT NULL DEFAULT 0, + created_by TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + activated_at DATETIME +); + +INSERT INTO of_pages_deployments_before_source ( + id, project_id, deployment_number, checksum, status, upload_id, artifact_path, + file_count, total_size, created_by, created_at, activated_at +) +SELECT + id, project_id, deployment_number, checksum, status, upload_id, artifact_path, + file_count, total_size, created_by, created_at, activated_at +FROM of_pages_deployments; + +DROP TABLE of_pages_deployments; +ALTER TABLE of_pages_deployments_before_source RENAME TO of_pages_deployments; + +CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_project_id + ON of_pages_deployments (project_id); +CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_checksum + ON of_pages_deployments (checksum); +CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_status + ON of_pages_deployments (status); +CREATE INDEX IF NOT EXISTS idx_of_pages_deployments_upload_id + ON of_pages_deployments (upload_id); + +CREATE TABLE of_pages_projects_before_source ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + slug TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + enabled INTEGER NOT NULL DEFAULT 1, + spa_fallback_enabled INTEGER NOT NULL DEFAULT 0, + spa_fallback_path TEXT NOT NULL DEFAULT '/index.html', + api_proxy_enabled INTEGER NOT NULL DEFAULT 0, + api_proxy_path TEXT NOT NULL DEFAULT '', + api_proxy_pass TEXT NOT NULL DEFAULT '', + api_proxy_rewrite TEXT NOT NULL DEFAULT '', + active_deployment_id INTEGER, + root_dir TEXT NOT NULL DEFAULT '', + entry_file TEXT NOT NULL DEFAULT 'index.html', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO of_pages_projects_before_source ( + id, name, slug, description, enabled, spa_fallback_enabled, spa_fallback_path, + api_proxy_enabled, api_proxy_path, api_proxy_pass, api_proxy_rewrite, + active_deployment_id, root_dir, entry_file, created_at, updated_at +) +SELECT + id, name, slug, description, enabled, spa_fallback_enabled, spa_fallback_path, + api_proxy_enabled, api_proxy_path, api_proxy_pass, api_proxy_rewrite, + active_deployment_id, root_dir, entry_file, created_at, updated_at +FROM of_pages_projects; + +DROP TABLE of_pages_projects; +ALTER TABLE of_pages_projects_before_source RENAME TO of_pages_projects; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_of_pages_projects_slug + ON of_pages_projects (slug); +CREATE INDEX IF NOT EXISTS idx_of_pages_projects_active_deployment_id + ON of_pages_projects (active_deployment_id); diff --git a/internal/db/migrator/migrator_test.go b/internal/db/migrator/migrator_test.go index ae9a0b88..780bdcb7 100644 --- a/internal/db/migrator/migrator_test.go +++ b/internal/db/migrator/migrator_test.go @@ -18,9 +18,9 @@ import ( "gorm.io/gorm" ) -// expectedMigratedSystemConfigCount 包含初始 32 项系统配置，以及 202606220004 -// 从 of_options 迁移过来的 48 项业务配置（OpenFlare/UptimeKuma/OpenResty）。 -const expectedMigratedSystemConfigCount = 80 +// expectedMigratedSystemConfigCount 包含初始 32 项系统配置、202606220004 +// 从 of_options 迁移过来的 48 项业务配置，以及 Pages 的 2 项业务配置。 +const expectedMigratedSystemConfigCount = 82 func TestMigrateInitializesSQLiteDatabase(t *testing.T) { sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ diff --git a/internal/db/migrator/pages_source_migration_test.go b/internal/db/migrator/pages_source_migration_test.go new file mode 100644 index 00000000..be0534a8 --- /dev/null +++ b/internal/db/migrator/pages_source_migration_test.go @@ -0,0 +1,322 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package migrator + +import ( + "database/sql" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/Rain-kl/Wavelet/internal/model" + "github.com/glebarez/sqlite" + "github.com/pressly/goose/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +const ( + pagesSourcePreviousMigration = int64(202607180001) + pagesSourceMigration = int64(202607190001) + pagesMigrationProjectID = uint(900001) + pagesMigrationDeploymentID = uint(900001) +) + +func TestPagesSourceMigrationSQLiteUpDownUp(t *testing.T) { + dbPath := t.TempDir() + "/pages-source-migration.db" + gormDB, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{ + DisableForeignKeyConstraintWhenMigrating: true, + }) + require.NoError(t, err) + + sqlDB, err := gormDB.DB() + require.NoError(t, err) + sqlDB.SetMaxOpenConns(1) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + + runPagesSourceMigrationUpDownUp(t, gormDB, sqlDB, dialectSqlite, "goose/sqlite") + + var indexSQL string + require.NoError(t, gormDB.Raw( + "SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", + "idx_of_pages_deployments_source_revision", + ).Scan(&indexSQL).Error) + assert.Contains(t, strings.ToUpper(indexSQL), "WHERE SOURCE_IDENTITY IS NOT NULL AND SOURCE_REVISION IS NOT NULL") +} + +func TestPagesSourceMigrationPostgresUpDownUp(t *testing.T) { + dsn := strings.TrimSpace(os.Getenv("OPENFLARE_TEST_POSTGRES_DSN")) + if dsn == "" { + t.Skip("OPENFLARE_TEST_POSTGRES_DSN is not set") + } + + gormDB, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ + DisableForeignKeyConstraintWhenMigrating: true, + }) + require.NoError(t, err) + sqlDB, err := gormDB.DB() + require.NoError(t, err) + sqlDB.SetMaxOpenConns(1) + + schema := fmt.Sprintf("pages_source_migration_%d", time.Now().UnixNano()) + require.Regexp(t, `^[a-z0-9_]+$`, schema) + require.NoError(t, gormDB.Exec(`CREATE SCHEMA "`+schema+`"`).Error) + require.NoError(t, gormDB.Exec(`SET search_path TO "`+schema+`"`).Error) + t.Cleanup(func() { + assert.NoError(t, gormDB.Exec("SET search_path TO public").Error) + assert.NoError(t, gormDB.Exec(`DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`).Error) + assert.NoError(t, sqlDB.Close()) + }) + + runPagesSourceMigrationUpDownUp(t, gormDB, sqlDB, dialectPostgres, "goose/postgres") +} + +func runPagesSourceMigrationUpDownUp( + t *testing.T, + gormDB *gorm.DB, + sqlDB *sql.DB, + dialect string, + dir string, +) { + t.Helper() + + goose.SetBaseFS(migrationFS) + require.NoError(t, goose.SetDialect(dialect)) + require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourcePreviousMigration)) + seedPrePagesSourceMigrationData(t, gormDB) + + require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceMigration)) + assertPagesSourceMigrationUp(t, gormDB) + + require.NoError(t, goose.DownTo(sqlDB, dir, pagesSourcePreviousMigration)) + assertPagesSourceMigrationDown(t, gormDB) + + require.NoError(t, goose.UpTo(sqlDB, dir, pagesSourceMigration)) + assertPagesSourceMigrationUpAgain(t, gormDB) +} + +func seedPrePagesSourceMigrationData(t *testing.T, gormDB *gorm.DB) { + t.Helper() + + require.NoError(t, gormDB.Exec(` + INSERT INTO of_pages_projects ( + id, name, slug, description, enabled, active_deployment_id, root_dir, entry_file + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, + pagesMigrationProjectID, + "Migration Site", + "migration-site", + "keep-project-data", + true, + pagesMigrationDeploymentID, + "public", + "home.html", + ).Error) + require.NoError(t, gormDB.Exec(` + INSERT INTO of_pages_deployments ( + id, project_id, deployment_number, checksum, status, upload_id, artifact_path, + file_count, total_size, created_by + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + pagesMigrationDeploymentID, + pagesMigrationProjectID, + 1, + strings.Repeat("a", 64), + model.PagesDeploymentStatusActive, + uint64(700001), + "legacy/package.zip", + 2, + int64(128), + "user:1", + ).Error) +} + +func assertPagesSourceMigrationUp(t *testing.T, gormDB *gorm.DB) { + t.Helper() + migrator := gormDB.Migrator() + assert.True(t, migrator.HasTable(&model.PagesProjectSource{})) + assert.True(t, migrator.HasTable(&model.PagesProjectSourceRuntime{})) + assert.True(t, migrator.HasColumn(&model.PagesProject{}, "ContentConfigVersion")) + assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceType")) + assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceIdentity")) + assert.True(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceRevision")) + assert.True(t, migrator.HasIndex(&model.PagesProjectSource{}, "idx_of_pages_project_sources_project_id")) + assert.True(t, migrator.HasIndex(&model.PagesProjectSourceRuntime{}, "idx_of_pages_project_source_runtime_next_check_at")) + assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_number")) + assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_source_revision")) + + var project model.PagesProject + require.NoError(t, gormDB.First(&project, pagesMigrationProjectID).Error) + assert.Equal(t, 0, project.ContentConfigVersion) + assert.Equal(t, "keep-project-data", project.Description) + assert.Equal(t, "public", project.RootDir) + assert.Equal(t, "home.html", project.EntryFile) + + var deployment model.PagesDeployment + require.NoError(t, gormDB.First(&deployment, pagesMigrationDeploymentID).Error) + assert.Equal(t, "manual_upload", deployment.SourceType) + assert.Equal(t, "manual_upload", deployment.TriggerType) + assert.Nil(t, deployment.SourceIdentity) + assert.Nil(t, deployment.SourceRevision) + assert.Equal(t, uint64(700001), deployment.UploadID) + + sourceID := createMigrationSourceRuntime(t, gormDB) + assertPagesSourceConstraints(t, gormDB, sourceID) +} + +func createMigrationSourceRuntime(t *testing.T, gormDB *gorm.DB) uint { + t.Helper() + source := model.PagesProjectSource{ + ProjectID: pagesMigrationProjectID, + SourceType: "remote_url", + RemoteURL: "https://example.com/site.zip?token=secret", + RemoteNetworkPolicy: "public", + CheckIntervalMinutes: 0, + ConfigVersion: 1, + SourceIdentity: strings.Repeat("b", 64), + } + require.NoError(t, gormDB.Create(&source).Error) + require.NotZero(t, source.ID) + require.NoError(t, gormDB.Create(&model.PagesProjectSourceRuntime{ + SourceID: source.ID, + SyncStatus: "idle", + }).Error) + return source.ID +} + +func assertPagesSourceConstraints(t *testing.T, gormDB *gorm.DB, sourceID uint) { + t.Helper() + + duplicateSource := model.PagesProjectSource{ + ProjectID: pagesMigrationProjectID, + SourceType: "remote_url", + ConfigVersion: 1, + SourceIdentity: strings.Repeat("c", 64), + } + assert.Error(t, gormDB.Create(&duplicateSource).Error) + + for number := 2; number <= 3; number++ { + require.NoError(t, createMigrationDeployment( + gormDB, + number, + strings.Repeat(string(rune('a'+number)), 64), + nil, + nil, + )) + } + + identity := strings.Repeat("d", 64) + revision := strings.Repeat("e", 64) + require.NoError(t, createMigrationDeployment( + gormDB, + 4, + strings.Repeat("f", 64), + &identity, + &revision, + )) + assert.Error(t, createMigrationDeployment( + gormDB, + 5, + strings.Repeat("0", 64), + &identity, + &revision, + )) + assert.Error(t, createMigrationDeployment( + gormDB, + 1, + strings.Repeat("1", 64), + nil, + nil, + )) + + var runtime model.PagesProjectSourceRuntime + require.NoError(t, gormDB.First(&runtime, sourceID).Error) + assert.Equal(t, "idle", runtime.SyncStatus) +} + +func createMigrationDeployment( + gormDB *gorm.DB, + deploymentNumber int, + checksum string, + identity *string, + revision *string, +) error { + return gormDB.Create(&model.PagesDeployment{ + ProjectID: pagesMigrationProjectID, + DeploymentNumber: deploymentNumber, + Checksum: checksum, + Status: model.PagesDeploymentStatusUploaded, + UploadID: uint64(710000 + deploymentNumber), + ArtifactPath: fmt.Sprintf("legacy/%d.zip", deploymentNumber), + SourceType: "manual_upload", + SourceIdentity: identity, + SourceRevision: revision, + TriggerType: "manual_upload", + }).Error +} + +func assertPagesSourceMigrationDown(t *testing.T, gormDB *gorm.DB) { + t.Helper() + migrator := gormDB.Migrator() + assert.False(t, migrator.HasTable(&model.PagesProjectSource{})) + assert.False(t, migrator.HasTable(&model.PagesProjectSourceRuntime{})) + assert.False(t, migrator.HasColumn(&model.PagesProject{}, "ContentConfigVersion")) + assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceType")) + assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceIdentity")) + assert.False(t, migrator.HasColumn(&model.PagesDeployment{}, "SourceRevision")) + assert.False(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_number")) + assert.False(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_source_revision")) + assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_project_id")) + assert.True(t, migrator.HasIndex(&model.PagesDeployment{}, "idx_of_pages_deployments_upload_id")) + + var project struct { + Description string + RootDir string + EntryFile string + ActiveDeploymentID *uint + } + require.NoError(t, gormDB.Table("of_pages_projects").Where("id = ?", pagesMigrationProjectID).Take(&project).Error) + assert.Equal(t, "keep-project-data", project.Description) + assert.Equal(t, "public", project.RootDir) + assert.Equal(t, "home.html", project.EntryFile) + require.NotNil(t, project.ActiveDeploymentID) + assert.Equal(t, pagesMigrationDeploymentID, *project.ActiveDeploymentID) + + var deployment struct { + UploadID uint64 + ArtifactPath string + FileCount int + TotalSize int64 + } + require.NoError(t, gormDB.Table("of_pages_deployments").Where("id = ?", pagesMigrationDeploymentID).Take(&deployment).Error) + assert.Equal(t, uint64(700001), deployment.UploadID) + assert.Equal(t, "legacy/package.zip", deployment.ArtifactPath) + assert.Equal(t, 2, deployment.FileCount) + assert.Equal(t, int64(128), deployment.TotalSize) + + var count int64 + require.NoError(t, gormDB.Table("of_pages_deployments").Where("project_id = ?", pagesMigrationProjectID).Count(&count).Error) + assert.Equal(t, int64(4), count) +} + +func assertPagesSourceMigrationUpAgain(t *testing.T, gormDB *gorm.DB) { + t.Helper() + assert.True(t, gormDB.Migrator().HasTable(&model.PagesProjectSource{})) + assert.True(t, gormDB.Migrator().HasTable(&model.PagesProjectSourceRuntime{})) + assert.True(t, gormDB.Migrator().HasColumn(&model.PagesProject{}, "ContentConfigVersion")) + assert.True(t, gormDB.Migrator().HasColumn(&model.PagesDeployment{}, "SourceRevision")) + + var count int64 + require.NoError(t, gormDB.Table("of_pages_deployments"). + Where("project_id = ? AND source_type = ? AND trigger_type = ?", pagesMigrationProjectID, "manual_upload", "manual_upload"). + Count(&count).Error) + assert.Equal(t, int64(4), count) + + require.NoError(t, gormDB.Table("of_pages_project_sources").Count(&count).Error) + assert.Zero(t, count, "source config is intentionally removed by Down and is not reconstructable") +} diff --git a/internal/db/postgres.go b/internal/db/postgres.go index 7aa72320..c16cd806 100644 --- a/internal/db/postgres.go +++ b/internal/db/postgres.go @@ -57,13 +57,10 @@ func initSQLite() { // Trace 注入 if err = db.Use( - tracing.NewPlugin( - tracing.WithoutMetrics(), - tracing.WithAttributes( - attribute.String("db.instance", sqlitePath), - attribute.String("db.system", "SQLite"), - ), - ), + newGORMTracingPlugin([]attribute.KeyValue{ + attribute.String("db.instance", sqlitePath), + attribute.String("db.system", "SQLite"), + }), ); err != nil { log.Fatalf("[SQLite] init trace failed: %v\n", err) } @@ -98,15 +95,12 @@ func initPostgres() { // Trace 注入 if err = db.Use( - tracing.NewPlugin( - tracing.WithoutMetrics(), - tracing.WithAttributes( - attribute.String("db.instance", dbConfig.Database), - attribute.String("db.ip", dbConfig.Host), - attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))), - attribute.String("db.system", "PostgreSQL"), - ), - ), + newGORMTracingPlugin([]attribute.KeyValue{ + attribute.String("db.instance", dbConfig.Database), + attribute.String("db.ip", dbConfig.Host), + attribute.String("server.address", net.JoinHostPort(dbConfig.Host, strconv.Itoa(dbConfig.Port))), + attribute.String("db.system", "PostgreSQL"), + }), ); err != nil { log.Fatalf("[PostgreSQL] init trace failed: %v\n", err) } @@ -160,6 +154,17 @@ func initPostgres() { } +// newGORMTracingPlugin 构造数据库链路追踪插件。查询参数只保留占位符，避免凭据等绑定值进入 Span。 +func newGORMTracingPlugin(attrs []attribute.KeyValue, extraOptions ...tracing.Option) gorm.Plugin { + options := []tracing.Option{ + tracing.WithoutMetrics(), + tracing.WithoutQueryVariables(), + tracing.WithAttributes(attrs...), + } + options = append(options, extraOptions...) + return tracing.NewPlugin(options...) +} + // buildDSN 构建 PostgreSQL DSN func buildDSN(host string, port int, username, password string) string { cfg := config.Config.Database diff --git a/internal/db/postgres_logger.go b/internal/db/postgres_logger.go index 0bfd0f7a..2ba1433c 100644 --- a/internal/db/postgres_logger.go +++ b/internal/db/postgres_logger.go @@ -49,6 +49,11 @@ func (l *gormZapLogger) Error(ctx context.Context, fmt string, args ...interface } } +// ParamsFilter 让 GORM 的 Trace 回调只接收参数化 SQL，避免绑定值被 Dialector.Explain 展开到日志。 +func (l *gormZapLogger) ParamsFilter(_ context.Context, sql string, _ ...interface{}) (string, []interface{}) { + return sql, nil +} + func (l *gormZapLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) { elapsed := time.Since(begin) switch { diff --git a/internal/db/postgres_logger_test.go b/internal/db/postgres_logger_test.go index 20539a5b..0d16dc4a 100644 --- a/internal/db/postgres_logger_test.go +++ b/internal/db/postgres_logger_test.go @@ -4,11 +4,40 @@ package db import ( + "context" + "strings" "testing" + "time" + "github.com/glebarez/sqlite" + "gorm.io/gorm" gormLogger "gorm.io/gorm/logger" ) +type paramsFilterCaptureLogger struct { + filter *gormZapLogger + traces []string +} + +func (l *paramsFilterCaptureLogger) LogMode(gormLogger.LogLevel) gormLogger.Interface { + return l +} + +func (l *paramsFilterCaptureLogger) Info(context.Context, string, ...interface{}) {} + +func (l *paramsFilterCaptureLogger) Warn(context.Context, string, ...interface{}) {} + +func (l *paramsFilterCaptureLogger) Error(context.Context, string, ...interface{}) {} + +func (l *paramsFilterCaptureLogger) ParamsFilter(ctx context.Context, sql string, params ...interface{}) (string, []interface{}) { + return l.filter.ParamsFilter(ctx, sql, params...) +} + +func (l *paramsFilterCaptureLogger) Trace(_ context.Context, _ time.Time, fc func() (string, int64), _ error) { + sql, _ := fc() + l.traces = append(l.traces, sql) +} + func TestParseLogLevel(t *testing.T) { t.Parallel() @@ -37,3 +66,50 @@ func TestParseLogLevel(t *testing.T) { }) } } + +func TestGormZapLoggerParamsFilterDropsBoundValues(t *testing.T) { + t.Parallel() + + const ( + query = "UPDATE openflare_pages_sources SET remote_url = ? WHERE id = ?" + secret = "https://example.test/release.zip?token=super-secret" + ) + + filteredSQL, filteredParams := (&gormZapLogger{}).ParamsFilter(t.Context(), query, secret, int64(42)) + if filteredSQL != query { + t.Fatalf("ParamsFilter() sql = %q, want %q", filteredSQL, query) + } + if filteredParams != nil { + t.Fatalf("ParamsFilter() params = %#v, want nil", filteredParams) + } +} + +func TestGormZapLoggerKeepsParameterizedSQLInTrace(t *testing.T) { + t.Parallel() + + const secret = "https://example.test/release.zip?token=trace-secret" + capture := ¶msFilterCaptureLogger{filter: &gormZapLogger{}} + testDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: capture}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + if err := testDB.Exec("CREATE TABLE source_secrets (remote_url TEXT NOT NULL)").Error; err != nil { + t.Fatalf("create table: %v", err) + } + + capture.traces = nil + if err := testDB.Exec("INSERT INTO source_secrets (remote_url) VALUES (?)", secret).Error; err != nil { + t.Fatalf("insert source secret: %v", err) + } + if len(capture.traces) != 1 { + t.Fatalf("trace count = %d, want 1", len(capture.traces)) + } + + traceSQL := capture.traces[0] + if strings.Contains(traceSQL, secret) || strings.Contains(traceSQL, "trace-secret") { + t.Fatalf("trace SQL leaked bound value: %q", traceSQL) + } + if !strings.Contains(traceSQL, "VALUES (?)") { + t.Fatalf("trace SQL = %q, want parameter placeholder", traceSQL) + } +} diff --git a/internal/db/postgres_tracing_test.go b/internal/db/postgres_tracing_test.go new file mode 100644 index 00000000..6958521f --- /dev/null +++ b/internal/db/postgres_tracing_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package db + +import ( + "context" + "strings" + "testing" + + "github.com/glebarez/sqlite" + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + semconv "go.opentelemetry.io/otel/semconv/v1.30.0" + "gorm.io/gorm" + gormLogger "gorm.io/gorm/logger" + "gorm.io/plugin/opentelemetry/tracing" +) + +func TestGORMTracingPluginDoesNotRecordQueryVariables(t *testing.T) { + t.Parallel() + + const secret = "https://example.test/release.zip?token=otel-secret" + spanRecorder := tracetest.NewSpanRecorder() + tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder)) + t.Cleanup(func() { + if err := tracerProvider.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown tracer provider: %v", err) + } + }) + + testDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: gormLogger.Default.LogMode(gormLogger.Silent), + }) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + if err := testDB.Use(newGORMTracingPlugin( + []attribute.KeyValue{attribute.String("db.instance", "trace-test")}, + tracing.WithTracerProvider(tracerProvider), + )); err != nil { + t.Fatalf("register tracing plugin: %v", err) + } + if err := testDB.Exec("CREATE TABLE source_secrets (remote_url TEXT NOT NULL)").Error; err != nil { + t.Fatalf("create table: %v", err) + } + if err := testDB.Exec("INSERT INTO source_secrets (remote_url) VALUES (?)", secret).Error; err != nil { + t.Fatalf("insert source secret: %v", err) + } + + var queryText string + for _, span := range spanRecorder.Ended() { + for _, attr := range span.Attributes() { + if attr.Key == semconv.DBQueryTextKey && strings.Contains(attr.Value.AsString(), "INSERT INTO source_secrets") { + queryText = attr.Value.AsString() + } + } + } + if queryText == "" { + t.Fatal("database query text attribute not found") + } + if strings.Contains(queryText, secret) || strings.Contains(queryText, "otel-secret") { + t.Fatalf("db.query.text leaked bound value: %q", queryText) + } + if !strings.Contains(queryText, "VALUES (?)") { + t.Fatalf("db.query.text = %q, want parameter placeholder", queryText) + } +} diff --git a/internal/model/openflare_pages.go b/internal/model/openflare_pages.go index e35006be..31ffce9d 100644 --- a/internal/model/openflare_pages.go +++ b/internal/model/openflare_pages.go @@ -18,22 +18,23 @@ const ( // PagesProject OpenFlare Pages 静态托管项目。 type PagesProject struct { - ID uint `json:"id" gorm:"primaryKey;autoIncrement"` - Name string `json:"name" gorm:"size:255;not null"` - Slug string `json:"slug" gorm:"uniqueIndex;size:128;not null"` - Description string `json:"description" gorm:"type:text;not null;default:''"` - Enabled bool `json:"enabled" gorm:"not null;default:true"` - SPAFallbackEnabled bool `json:"spa_fallback_enabled" gorm:"not null;default:false"` - SPAFallbackPath string `json:"spa_fallback_path" gorm:"size:512;not null;default:'/index.html'"` - APIProxyEnabled bool `json:"api_proxy_enabled" gorm:"not null;default:false"` - APIProxyPath string `json:"api_proxy_path" gorm:"size:255;not null;default:''"` - APIProxyPass string `json:"api_proxy_pass" gorm:"size:2048;not null;default:''"` - APIProxyRewrite string `json:"api_proxy_rewrite" gorm:"size:255;not null;default:''"` - ActiveDeploymentID *uint `json:"active_deployment_id" gorm:"index"` - RootDir string `json:"root_dir" gorm:"size:512;not null;default:''"` - EntryFile string `json:"entry_file" gorm:"size:512;not null;default:'index.html'"` - CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"` - UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"` + ID uint `json:"id" gorm:"primaryKey;autoIncrement"` + Name string `json:"name" gorm:"size:255;not null"` + Slug string `json:"slug" gorm:"uniqueIndex;size:128;not null"` + Description string `json:"description" gorm:"type:text;not null;default:''"` + Enabled bool `json:"enabled" gorm:"not null;default:true"` + SPAFallbackEnabled bool `json:"spa_fallback_enabled" gorm:"not null;default:false"` + SPAFallbackPath string `json:"spa_fallback_path" gorm:"size:512;not null;default:'/index.html'"` + APIProxyEnabled bool `json:"api_proxy_enabled" gorm:"not null;default:false"` + APIProxyPath string `json:"api_proxy_path" gorm:"size:255;not null;default:''"` + APIProxyPass string `json:"api_proxy_pass" gorm:"size:2048;not null;default:''"` + APIProxyRewrite string `json:"api_proxy_rewrite" gorm:"size:255;not null;default:''"` + ActiveDeploymentID *uint `json:"active_deployment_id" gorm:"index"` + RootDir string `json:"root_dir" gorm:"size:512;not null;default:''"` + EntryFile string `json:"entry_file" gorm:"size:512;not null;default:'index.html'"` + ContentConfigVersion int `json:"-" gorm:"not null;default:0"` + CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"` + UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"` } // TableName 表名。 @@ -44,8 +45,8 @@ func (PagesProject) TableName() string { // PagesDeployment OpenFlare Pages 不可变部署记录。 type PagesDeployment struct { ID uint `json:"id" gorm:"primaryKey;autoIncrement"` - ProjectID uint `json:"project_id" gorm:"not null;index"` - DeploymentNumber int `json:"deployment_number" gorm:"not null"` + ProjectID uint `json:"project_id" gorm:"not null;index;uniqueIndex:idx_of_pages_deployments_project_number,priority:1;uniqueIndex:idx_of_pages_deployments_source_revision,priority:1,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"` + DeploymentNumber int `json:"deployment_number" gorm:"not null;uniqueIndex:idx_of_pages_deployments_project_number,priority:2"` Checksum string `json:"checksum" gorm:"size:64;not null;index"` Status string `json:"status" gorm:"size:32;not null;default:'uploaded';index"` UploadID uint64 `json:"upload_id,string" gorm:"not null;default:0;index"` @@ -53,6 +54,12 @@ type PagesDeployment struct { FileCount int `json:"file_count" gorm:"not null;default:0"` TotalSize int64 `json:"total_size" gorm:"not null;default:0"` CreatedBy string `json:"created_by" gorm:"size:64;not null;default:''"` + SourceType string `json:"source_type" gorm:"size:32;not null;default:''"` + SourceIdentity *string `json:"-" gorm:"type:char(64);uniqueIndex:idx_of_pages_deployments_source_revision,priority:2,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"` + SourceRevision *string `json:"-" gorm:"type:char(64);uniqueIndex:idx_of_pages_deployments_source_revision,priority:3,where:source_identity IS NOT NULL AND source_revision IS NOT NULL"` + SourceLabel string `json:"source_label" gorm:"size:255;not null;default:''"` + SourceMeta string `json:"-" gorm:"type:text;not null;default:''"` + TriggerType string `json:"trigger_type" gorm:"size:32;not null;default:''"` CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"` ActivatedAt *time.Time `json:"activated_at"` } diff --git a/internal/model/openflare_pages_source.go b/internal/model/openflare_pages_source.go new file mode 100644 index 00000000..6f110978 --- /dev/null +++ b/internal/model/openflare_pages_source.go @@ -0,0 +1,59 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package model + +import "time" + +// PagesProjectSource 保存 Pages 项目的持久部署源配置。 +// +// RemoteURL 可能包含签名参数，禁止直接序列化 model；对外接口必须映射到 +// pages 包内的脱敏 source view。 +type PagesProjectSource struct { + ID uint `json:"-" gorm:"primaryKey;autoIncrement"` + ProjectID uint `json:"-" gorm:"not null;uniqueIndex:idx_of_pages_project_sources_project_id"` + SourceType string `json:"-" gorm:"size:32;not null;default:''"` + RemoteURL string `json:"-" gorm:"type:text;not null;default:''"` + RemoteNetworkPolicy string `json:"-" gorm:"size:32;not null;default:''"` + GitHubRepository string `json:"-" gorm:"column:github_repository;size:255;not null;default:''"` + ReleaseSelector string `json:"-" gorm:"size:16;not null;default:''"` + ReleaseTag string `json:"-" gorm:"size:255;not null;default:''"` + AssetName string `json:"-" gorm:"size:255;not null;default:''"` + AutoUpdateEnabled bool `json:"-" gorm:"not null;default:false"` + CheckIntervalMinutes int `json:"-" gorm:"not null;default:0"` + ConfigVersion int `json:"-" gorm:"not null;default:0"` + SourceIdentity string `json:"-" gorm:"type:char(64);not null;default:''"` + CreatedAt time.Time `json:"-" gorm:"autoCreateTime"` + UpdatedAt time.Time `json:"-" gorm:"autoUpdateTime"` +} + +// TableName 返回 Pages 项目部署源配置表名。 +func (PagesProjectSource) TableName() string { + return "of_pages_project_sources" +} + +// PagesProjectSourceRuntime 保存 Pages 项目部署源的可变运行态。 +// +// Runtime 不冗余 project_id；调用方通过 SourceID 关联配置，并在最终提交时 +// 同时校验 source config version 与 project content config version。 +type PagesProjectSourceRuntime struct { + SourceID uint `json:"-" gorm:"primaryKey;autoIncrement:false"` + ETag string `json:"-" gorm:"column:etag;size:512;not null;default:''"` + LastSeenRevision string `json:"-" gorm:"type:char(64);not null;default:''"` + LastSeenDetail string `json:"-" gorm:"type:text;not null;default:''"` + LastAppliedRevision string `json:"-" gorm:"type:char(64);not null;default:''"` + LastAppliedDetail string `json:"-" gorm:"type:text;not null;default:''"` + SyncStatus string `json:"-" gorm:"size:32;not null;default:''"` + LastError string `json:"-" gorm:"type:text;not null;default:''"` + LastCheckedAt *time.Time `json:"-"` + LastSyncedAt *time.Time `json:"-"` + NextCheckAt *time.Time `json:"-" gorm:"index:idx_of_pages_project_source_runtime_next_check_at"` + LeaseExpiresAt *time.Time `json:"-"` + LeaseToken string `json:"-" gorm:"size:64;not null;default:''"` + UpdatedAt time.Time `json:"-" gorm:"autoUpdateTime"` +} + +// TableName 返回 Pages 项目部署源运行态表名。 +func (PagesProjectSourceRuntime) TableName() string { + return "of_pages_project_source_runtime" +} diff --git a/internal/model/openflare_pages_source_test.go b/internal/model/openflare_pages_source_test.go new file mode 100644 index 00000000..16b2b637 --- /dev/null +++ b/internal/model/openflare_pages_source_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package model + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestPagesSourceModelsMatchMigrationSchema(t *testing.T) { + gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + DisableForeignKeyConstraintWhenMigrating: true, + }) + require.NoError(t, err) + require.NoError(t, gormDB.AutoMigrate( + &PagesProject{}, + &PagesDeployment{}, + &PagesProjectSource{}, + &PagesProjectSourceRuntime{}, + )) + + assert.Equal(t, "of_pages_project_sources", (PagesProjectSource{}).TableName()) + assert.Equal(t, "of_pages_project_source_runtime", (PagesProjectSourceRuntime{}).TableName()) + assert.True(t, gormDB.Migrator().HasColumn(&PagesProjectSource{}, "github_repository")) + assert.False(t, gormDB.Migrator().HasColumn(&PagesProjectSource{}, "git_hub_repository")) + assert.True(t, gormDB.Migrator().HasColumn(&PagesProjectSourceRuntime{}, "etag")) + assert.False(t, gormDB.Migrator().HasColumn(&PagesProjectSourceRuntime{}, "e_tag")) + + var indexSQL string + require.NoError(t, gormDB.Raw( + "SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", + "idx_of_pages_deployments_source_revision", + ).Scan(&indexSQL).Error) + assert.Contains(t, strings.ToUpper(indexSQL), "WHERE SOURCE_IDENTITY IS NOT NULL AND SOURCE_REVISION IS NOT NULL") +} + +func TestPagesSourceModelsDoNotSerializeSecretsOrFencingState(t *testing.T) { + sourceJSON, err := json.Marshal(PagesProjectSource{ + ID: 1, + ProjectID: 2, + RemoteURL: "https://example.com/site.zip?token=secret", + ConfigVersion: 3, + SourceIdentity: strings.Repeat("a", 64), + }) + require.NoError(t, err) + assert.JSONEq(t, `{}`, string(sourceJSON)) + + runtimeJSON, err := json.Marshal(PagesProjectSourceRuntime{ + SourceID: 1, + ETag: `"secret-etag"`, + LeaseToken: "secret-lease", + }) + require.NoError(t, err) + assert.JSONEq(t, `{}`, string(runtimeJSON)) + + identity := strings.Repeat("b", 64) + revision := strings.Repeat("c", 64) + deploymentJSON, err := json.Marshal(PagesDeployment{ + SourceType: "remote_url", + SourceIdentity: &identity, + SourceRevision: &revision, + SourceLabel: "site.zip", + SourceMeta: `{"provider":"remote_url","private":"secret"}`, + TriggerType: "manual_sync", + }) + require.NoError(t, err) + assert.NotContains(t, string(deploymentJSON), identity) + assert.NotContains(t, string(deploymentJSON), revision) + assert.NotContains(t, string(deploymentJSON), "private") + assert.Contains(t, string(deploymentJSON), `"source_type":"remote_url"`) + assert.Contains(t, string(deploymentJSON), `"source_label":"site.zip"`) + assert.Contains(t, string(deploymentJSON), `"trigger_type":"manual_sync"`) +} diff --git a/internal/router/v1/openflare/register_pages.go b/internal/router/v1/openflare/register_pages.go index 9fa2fb9c..89ba3b7c 100644 --- a/internal/router/v1/openflare/register_pages.go +++ b/internal/router/v1/openflare/register_pages.go @@ -18,6 +18,11 @@ func registerPagesRoutes(apiGroup *gin.RouterGroup) { apiutil.RegisterCollection(pagesRoute, "POST", pages.CreateProjectHandler) pagesRoute.POST("/:id/update", pages.UpdateProjectHandler) pagesRoute.POST("/:id/delete", pages.DeleteProjectHandler) + pagesRoute.GET("/:id/source", pages.GetSourceHandler) + pagesRoute.POST("/:id/source/update", pages.UpdateSourceHandler) + pagesRoute.POST("/:id/source/delete", pages.DeleteSourceHandler) + pagesRoute.POST("/:id/source/check", pages.CheckSourceHandler) + pagesRoute.POST("/:id/source/sync", pages.SyncSourceHandler) pagesRoute.GET("/:id/deployments", pages.ListDeploymentsHandler) pagesRoute.POST("/:id/deployments/upload", pages.UploadDeploymentHandler) pagesRoute.POST("/:id/deployments/upload-from-url", pages.UploadDeploymentFromURLHandler) diff --git a/internal/task/executor.go b/internal/task/executor.go index e41ae65e..c41ca9d8 100644 --- a/internal/task/executor.go +++ b/internal/task/executor.go @@ -428,7 +428,7 @@ func notifyTaskCompleted(ctx context.Context, execution *model.TaskExecution, re } func shouldFlushTaskExecutionLog(ctx context.Context, execErr error) bool { - if execErr == nil { + if isTerminalTaskExecutionError(execErr) { return true } @@ -440,6 +440,10 @@ func shouldFlushTaskExecutionLog(ctx context.Context, execErr error) bool { return retryCount >= maxRetry } +func isTerminalTaskExecutionError(execErr error) bool { + return execErr == nil || errors.Is(execErr, asynq.SkipRetry) +} + func handleFailedTask(ctx context.Context, execution *model.TaskExecution, t *asynq.Task, duration time.Duration, execErr error, span trace.Span) { execution.Status = model.TaskExecutionStatusFailed execution.ErrorMessage = execErr.Error() diff --git a/internal/task/executor_test.go b/internal/task/executor_test.go index fc53b770..b0bb2f9f 100644 --- a/internal/task/executor_test.go +++ b/internal/task/executor_test.go @@ -6,10 +6,12 @@ package task import ( "context" + "errors" "fmt" "testing" "time" + "github.com/Rain-kl/Wavelet/internal/db" "github.com/Rain-kl/Wavelet/internal/model" "github.com/Rain-kl/Wavelet/internal/testhelper" "github.com/hibiken/asynq" @@ -284,6 +286,55 @@ func TestCompleteTaskExecutionFlushesLog(t *testing.T) { assert.Contains(t, found.Log, "任务执行成功") } +func TestCompleteTaskExecutionFlushesPermanentFailureLog(t *testing.T) { + cleanup := setupTest(t) + defer cleanup() + ctx := context.Background() + + execution := &model.TaskExecution{ + TaskID: "complete_permanent_flush_001", + TaskType: testTaskType, + TaskName: "测试任务", + Status: model.TaskExecutionStatusRunning, + Retryable: true, + MaxRetry: 3, + TriggeredBy: "manual", + } + err := model.CreateTaskExecution(ctx, execution) + require.NoError(t, err) + + ctx = withTaskID(ctx, execution.TaskID) + AppendLog(ctx, "永久失败前的日志") + execErr := PermanentError("来源配置无效") + + finishTime := time.Now() + completeTaskExecution( + ctx, + execution, + asynq.NewTask(testTaskType, nil), + 100*time.Millisecond, + finishTime, + nil, + execErr, + trace.SpanFromContext(ctx), + ) + + found, err := model.GetTaskExecutionByTaskID(ctx, execution.TaskID) + require.NoError(t, err) + assert.Equal(t, model.TaskExecutionStatusFailed, found.Status) + assert.Equal(t, "来源配置无效", found.ErrorMessage) + assert.Contains(t, found.Log, "永久失败前的日志") + assert.Contains(t, found.Log, "任务执行失败") + keys, err := db.Redis.Keys(ctx, "*"+execution.TaskID+"*").Result() + require.NoError(t, err) + assert.Empty(t, keys) +} + +func TestPermanentErrorIsTerminalForLogFlush(t *testing.T) { + assert.True(t, isTerminalTaskExecutionError(PermanentError("配置无效"))) + assert.False(t, isTerminalTaskExecutionError(errors.New("temporary failure"))) +} + func TestRetryTask(t *testing.T) { cleanup := setupTest(t) defer cleanup() diff --git a/internal/task/handlers/register.go b/internal/task/handlers/register.go index ade65f8c..641d9dca 100644 --- a/internal/task/handlers/register.go +++ b/internal/task/handlers/register.go @@ -8,6 +8,7 @@ package handlers import ( "github.com/Rain-kl/Wavelet/internal/apps/admin/push" "github.com/Rain-kl/Wavelet/internal/apps/openflare" + "github.com/Rain-kl/Wavelet/internal/apps/openflare/pages" "github.com/Rain-kl/Wavelet/internal/apps/openflare/tls" "github.com/Rain-kl/Wavelet/internal/apps/upload" "github.com/Rain-kl/Wavelet/internal/apps/user" @@ -51,6 +52,10 @@ func Register() { task.RegisterHandler(openflare.UptimeKumaSyncTask, &openflare.UptimeKumaSyncHandler{}) task.RegisterTaskMeta(openflare.UptimeKumaSyncMeta) + // pages source actions are only dispatched by the Pages domain API/scanner. + task.RegisterHandler(pages.PagesSourceActionTask, &pages.SourceActionHandler{}) + task.RegisterTaskMeta(pages.PagesSourceActionMeta) + // tls single renew task.RegisterHandler(tls.SSLSingleRenewTask, &tls.SSLSingleRenewHandler{}) task.RegisterTaskMeta(tls.SSLSingleRenewMeta) diff --git a/internal/task/meta.go b/internal/task/meta.go index 019fc637..888ed80c 100644 --- a/internal/task/meta.go +++ b/internal/task/meta.go @@ -31,6 +31,7 @@ type TaskMeta struct { MaxRetry int `json:"max_retry"` Queue string `json:"queue"` Retryable bool `json:"retryable"` // 是否支持手动重试 + InternalOnly bool `json:"-"` // 是否仅允许内部业务入口调度 Params []TaskParam `json:"params,omitempty"` } @@ -51,13 +52,18 @@ func RegisterTaskMeta(meta TaskMeta) { dispatchableTasks = append(dispatchableTasks, meta) } -// GetDispatchableTasks 获取所有已注册的元数据列表（返回副本以避免并发并发读写冲突） +// GetDispatchableTasks 获取允许通过通用 Admin 入口调度的元数据列表。 func GetDispatchableTasks() []TaskMeta { dispatchableTasksMutex.RLock() defer dispatchableTasksMutex.RUnlock() - metas := make([]TaskMeta, len(dispatchableTasks)) - copy(metas, dispatchableTasks) + metas := make([]TaskMeta, 0, len(dispatchableTasks)) + for _, meta := range dispatchableTasks { + if meta.InternalOnly { + continue + } + metas = append(metas, meta) + } return metas } diff --git a/internal/task/meta_test.go b/internal/task/meta_test.go index e91c1b6b..a22e05eb 100644 --- a/internal/task/meta_test.go +++ b/internal/task/meta_test.go @@ -29,3 +29,28 @@ func TestDuplicateTaskMeta(t *testing.T) { } } } + +func TestInternalOnlyTaskMetaIsHiddenFromDispatchableTasks(t *testing.T) { + const taskType = "test_internal_only_meta" + meta := task.TaskMeta{ + Type: taskType, + AsynqTask: "test:internal_only_meta", + Name: "内部测试任务", + InternalOnly: true, + } + task.RegisterTaskMeta(meta) + + registered := task.GetTaskMeta(taskType) + if registered == nil { + t.Fatal("GetTaskMeta() did not return internal-only metadata") + } + if !registered.InternalOnly { + t.Fatal("GetTaskMeta() lost InternalOnly flag") + } + + for _, dispatchable := range task.GetDispatchableTasks() { + if dispatchable.Type == taskType { + t.Fatalf("GetDispatchableTasks() exposed internal-only task %q", taskType) + } + } +} diff --git a/internal/task/permanent_error.go b/internal/task/permanent_error.go new file mode 100644 index 00000000..a0dad4f3 --- /dev/null +++ b/internal/task/permanent_error.go @@ -0,0 +1,35 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package task + +import ( + "strings" + + "github.com/hibiken/asynq" +) + +const defaultPermanentErrorMessage = "任务无法继续执行" + +type permanentTaskError struct { + message string +} + +// PermanentError marks a safe domain message as a non-retryable task failure. +// It intentionally accepts no underlying error so Error never exposes provider, +// URL, header, response-body, or other sensitive implementation details. +func PermanentError(message string) error { + message = strings.TrimSpace(message) + if message == "" { + message = defaultPermanentErrorMessage + } + return &permanentTaskError{message: message} +} + +func (e *permanentTaskError) Error() string { + return e.message +} + +func (e *permanentTaskError) Unwrap() error { + return asynq.SkipRetry +} diff --git a/internal/task/permanent_error_test.go b/internal/task/permanent_error_test.go new file mode 100644 index 00000000..f8ae9300 --- /dev/null +++ b/internal/task/permanent_error_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package task + +import ( + "errors" + "testing" + + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" +) + +func TestPermanentErrorSkipsRetryWithoutExposingAsynqMessage(t *testing.T) { + err := PermanentError(" 来源配置无效 ") + + assert.True(t, errors.Is(err, asynq.SkipRetry)) + assert.Equal(t, "来源配置无效", err.Error()) + assert.NotContains(t, err.Error(), asynq.SkipRetry.Error()) +} + +func TestPermanentErrorUsesSafeFallbackForBlankMessage(t *testing.T) { + err := PermanentError(" ") + + assert.True(t, errors.Is(err, asynq.SkipRetry)) + assert.Equal(t, defaultPermanentErrorMessage, err.Error()) +} diff --git a/pkg/httppool/httppool.go b/pkg/httppool/httppool.go index 2297dc67..ed98a9b5 100644 --- a/pkg/httppool/httppool.go +++ b/pkg/httppool/httppool.go @@ -5,9 +5,11 @@ package httppool import ( + "context" "crypto/tls" "net" "net/http" + "net/url" "sync" "time" @@ -30,28 +32,66 @@ var ( once sync.Once ) +// TransportOptions configures the request-specific parts of a pooled HTTP +// transport. Pool sizes and timeout defaults remain managed by this package. +// A nil Proxy explicitly disables proxy use. +type TransportOptions struct { + Proxy func(*http.Request) (*url.URL, error) + DialContext func(context.Context, string, string) (net.Conn, error) + TLSClientConfig *tls.Config + ResponseHeaderTimeout time.Duration + TraceFilter func(*http.Request) bool +} + +// NewTransport returns an independently configurable pooled transport wrapped +// with OTel instrumentation. The supplied TLS configuration is cloned before +// use so later caller mutations cannot change an active transport. +func NewTransport(options TransportOptions) http.RoundTripper { + dialContext := options.DialContext + if dialContext == nil { + dialContext = (&net.Dialer{ + Timeout: dialTimeout, + KeepAlive: dialKeepAlive, + }).DialContext + } + + tlsConfig := options.TLSClientConfig + if tlsConfig == nil { + tlsConfig = &tls.Config{} + } else { + tlsConfig = tlsConfig.Clone() + } + if tlsConfig.ClientSessionCache == nil { + tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(tlsSessionCacheSize) + } + + transport := &http.Transport{ + Proxy: options.Proxy, + DialContext: dialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: maxIdleConns, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + IdleConnTimeout: idleConnTimeout, + TLSHandshakeTimeout: tlsHandshakeTimeout, + ResponseHeaderTimeout: options.ResponseHeaderTimeout, + ExpectContinueTimeout: expectContinueTimeout, + TLSClientConfig: tlsConfig, + } + otelOptions := make([]otelhttp.Option, 0, 1) + if options.TraceFilter != nil { + otelOptions = append(otelOptions, otelhttp.WithFilter(options.TraceFilter)) + } + return otelhttp.NewTransport(transport, otelOptions...) +} + // DefaultTransport returns a globally shared, optimized http.RoundTripper // with OTel instrumentation. It maintains a pool of idle TCP connections // across hosts. func DefaultTransport() http.RoundTripper { once.Do(func() { - transport := &http.Transport{ + defaultTransport = NewTransport(TransportOptions{ Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: dialTimeout, - KeepAlive: dialKeepAlive, - }).DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: maxIdleConns, - MaxIdleConnsPerHost: maxIdleConnsPerHost, - IdleConnTimeout: idleConnTimeout, - TLSHandshakeTimeout: tlsHandshakeTimeout, - ExpectContinueTimeout: expectContinueTimeout, - TLSClientConfig: &tls.Config{ - ClientSessionCache: tls.NewLRUClientSessionCache(tlsSessionCacheSize), - }, - } - defaultTransport = otelhttp.NewTransport(transport) + }) }) return defaultTransport } diff --git a/pkg/httppool/httppool_test.go b/pkg/httppool/httppool_test.go index efc71871..a753bda7 100644 --- a/pkg/httppool/httppool_test.go +++ b/pkg/httppool/httppool_test.go @@ -4,6 +4,12 @@ package httppool import ( + "context" + "crypto/tls" + "io" + "net" + "net/http" + "net/http/httptest" "testing" "time" ) @@ -35,3 +41,60 @@ func TestNewClient(t *testing.T) { t.Error("NewClient() is not configured with the default transport") } } + +func TestNewTransportUsesConfiguredDirectDialer(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write([]byte("ok")) + })) + t.Cleanup(server.Close) + + var dialedAddress string + dialer := &net.Dialer{} + transport := NewTransport(TransportOptions{ + Proxy: nil, + DialContext: func(ctx context.Context, network string, address string) (net.Conn, error) { + dialedAddress = address + return dialer.DialContext(ctx, network, server.Listener.Addr().String()) + }, + }) + client := &http.Client{Transport: transport} + t.Cleanup(client.CloseIdleConnections) + + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://artifact.example/site.zip", nil) + if err != nil { + t.Fatalf("NewRequestWithContext() error = %v", err) + } + response, err := client.Do(request) + if err != nil { + t.Fatalf("client.Do() error = %v", err) + } + defer func() { _ = response.Body.Close() }() + if _, err := io.ReadAll(response.Body); err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if dialedAddress != "artifact.example:80" { + t.Fatalf("DialContext address = %q, want direct target", dialedAddress) + } +} + +func TestNewTransportClonesTLSConfig(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(server.Close) + + tlsConfig := &tls.Config{InsecureSkipVerify: true} //nolint:gosec // test-only self-signed server + client := &http.Client{Transport: NewTransport(TransportOptions{TLSClientConfig: tlsConfig})} + t.Cleanup(client.CloseIdleConnections) + tlsConfig.InsecureSkipVerify = false + + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequestWithContext() error = %v", err) + } + response, err := client.Do(request) + if err != nil { + t.Fatalf("client.Do() error = %v", err) + } + _ = response.Body.Close() +}