Compare commits

..

3 Commits

31 changed files with 2805 additions and 716 deletions
+5
View File
@@ -63,6 +63,11 @@ jobs:
- name: Build
run: go build -v ./...
- name: Test panel upgrade lifecycle and permissions
run: |
go test ./internal/panelupgrade -count=1
go test ./internal/http/handler ./tests/contract -run 'SystemUpgrade|SelectComposeAsset|DownloadReleaseAsset|UpgradeBackup|RestoreBackup' -count=1
backend-postgres-contract:
name: Go Backend PostgreSQL Contract
runs-on: ubuntu-latest
+8
View File
@@ -12,9 +12,17 @@ import (
"go-backend/internal/app"
"go-backend/internal/config"
"go-backend/internal/panelupgrade"
)
func main() {
if len(os.Args) == 3 && os.Args[1] == "panel-upgrade-helper" {
if panelupgrade.RunHelper(os.Args[2]) != nil {
log.Print("panel upgrade helper stopped; inspect durable job status")
os.Exit(1)
}
return
}
cfg := config.FromEnv()
if cfg.JWTSecret == "" {
log.Println("warning: JWT_SECRET is empty")
@@ -186,6 +186,7 @@ func (h *Handler) Register(mux *http.ServeMux) {
mux.HandleFunc("/api/v1/system/version", h.systemVersion)
mux.HandleFunc("/api/v1/system/check-updates", h.systemCheckUpdates)
mux.HandleFunc("/api/v1/system/upgrade", h.systemUpgrade)
mux.HandleFunc("/api/v1/system/upgrade/status", h.systemUpgradeStatus)
mux.HandleFunc("/api/v1/license/activate", h.licenseActivate)
mux.HandleFunc("/api/v1/backup/export", h.backupExport)
mux.HandleFunc("/api/v1/backup/import", h.backupImport)
@@ -15,6 +15,7 @@ import (
"time"
"go-backend/internal/http/response"
"go-backend/internal/panelupgrade"
)
const (
@@ -75,6 +76,7 @@ type systemUpgradeCheckData struct {
}
type systemUpgradeRunData struct {
JobID string `json:"jobId"`
Version string `json:"version"`
Channel string `json:"channel"`
ComposeAsset string `json:"composeAsset"`
@@ -118,17 +120,7 @@ func validateBackendContainerName(value string) error {
return nil
}
func validateUpgradeVersion(value string) error {
if strings.TrimSpace(value) == "" {
return fmt.Errorf("upgrade version is empty")
}
for _, r := range value {
if r < 0x20 || r == 0x7f {
return fmt.Errorf("unsafe upgrade version: contains control character")
}
}
return nil
}
func validateUpgradeVersion(value string) error { return panelupgrade.ValidateVersion(value) }
func (e *systemUpgradeExecutor) composePath() string {
return filepath.Join(e.deployDir, "docker-compose.yml")
@@ -136,6 +128,8 @@ func (e *systemUpgradeExecutor) composePath() string {
func (e *systemUpgradeExecutor) envPath() string { return filepath.Join(e.deployDir, ".env") }
func (e *systemUpgradeExecutor) capability(ctx context.Context) systemUpgradeCapabilityData {
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
reasons := make([]string, 0)
if !filepath.IsAbs(e.deployDir) {
reasons = append(reasons, "部署目录必须是绝对路径")
@@ -143,8 +137,8 @@ func (e *systemUpgradeExecutor) capability(ctx context.Context) systemUpgradeCap
if err := validateBackendContainerName(e.backendContainer); err != nil {
reasons = append(reasons, err.Error())
}
if out, err := exec.CommandContext(ctx, "docker", "--version").CombinedOutput(); err != nil {
reasons = append(reasons, fmt.Sprintf("docker CLI不可用: %v: %s", err, strings.TrimSpace(string(out))))
if err := exec.CommandContext(ctx, "docker", "--version").Run(); err != nil {
reasons = append(reasons, "docker CLI不可用")
}
if info, err := os.Stat(dockerSocketPath); err != nil {
reasons = append(reasons, "docker socket不可用: "+err.Error())
@@ -161,10 +155,10 @@ func (e *systemUpgradeExecutor) capability(ctx context.Context) systemUpgradeCap
} else if info.IsDir() {
reasons = append(reasons, "部署.env不是文件")
}
if out, err := exec.CommandContext(ctx, "docker", "compose", "version").CombinedOutput(); err != nil {
reasons = append(reasons, fmt.Sprintf("docker compose不可用: %v: %s", err, strings.TrimSpace(string(out))))
if err := exec.CommandContext(ctx, "docker", "compose", "version").Run(); err != nil {
reasons = append(reasons, "docker compose不可用")
}
if _, err := e.currentBackendImage(ctx); err != nil {
if _, err := (panelupgrade.Manager{Dir: e.deployDir, Backend: e.backendContainer}).Identity(ctx); err != nil {
reasons = append(reasons, err.Error())
}
@@ -183,144 +177,6 @@ func (e *systemUpgradeExecutor) selectComposeAsset(current []byte) string {
return "docker-compose-v4.yml"
}
func (e *systemUpgradeExecutor) helperScript() string {
return `set -eu
LOGFILE="$PANEL_DEPLOY_DIR/upgrade.log"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOGFILE"; }
cd "$PANEL_DEPLOY_DIR"
echo "" > "$LOGFILE"
log "开始面板升级"
log "工作目录: $(pwd)"
if [ ! -f docker-compose.yml ]; then
log "错误: docker-compose.yml 不存在"
exit 1
fi
if [ ! -f .env ]; then
log "错误: .env 不存在"
exit 1
fi
log "拉取新镜像..."
if ! docker compose pull backend frontend >> "$LOGFILE" 2>&1; then
log "错误: 拉取镜像失败"
exit 1
fi
log "等待旧容器释放资源..."
sleep 3
log "重启服务(force-recreate)..."
if ! docker compose up -d --force-recreate --remove-orphans backend frontend >> "$LOGFILE" 2>&1; then
log "错误: 重启服务失败"
exit 1
fi
log "升级完成"
`
}
func (e *systemUpgradeExecutor) buildHelperRunArgs(imageID, helperName string) ([]string, error) {
if err := validateBackendContainerName(e.backendContainer); err != nil {
return nil, err
}
return []string{
"run", "-d", "--rm", "--name", helperName,
"--volumes-from", e.backendContainer,
"-v", dockerSocketPath + ":" + dockerSocketPath,
"-e", panelDeployDirEnv + "=" + e.deployDir,
"--entrypoint", "/bin/sh", imageID,
"-c", e.helperScript(),
}, nil
}
func (e *systemUpgradeExecutor) updateEnvVersion(envPath, version string) error {
if err := validateUpgradeVersion(version); err != nil {
return err
}
mode, err := fileModeOrDefault(envPath, 0o600)
if err != nil {
return err
}
data, err := os.ReadFile(envPath)
if err != nil {
return err
}
lines := strings.Split(string(data), "\n")
replaced := false
for i, line := range lines {
if strings.HasPrefix(line, "FLUX_VERSION=") {
lines[i] = "FLUX_VERSION=" + version
replaced = true
}
}
if !replaced {
trimmed := strings.TrimRight(strings.Join(lines, "\n"), "\n")
if trimmed == "" {
trimmed = "FLUX_VERSION=" + version
} else {
trimmed += "\nFLUX_VERSION=" + version
}
return writeFileWithMode(envPath, []byte(trimmed+"\n"), mode)
}
content := strings.TrimRight(strings.Join(lines, "\n"), "\n") + "\n"
return writeFileWithMode(envPath, []byte(content), mode)
}
func (e *systemUpgradeExecutor) backupFile(path string) (string, error) {
mode, err := fileModeOrDefault(path, 0o600)
if err != nil {
return "", err
}
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
backupPath := path + ".upgrade.bak"
if err := writeFileWithMode(backupPath, data, mode); err != nil {
return "", err
}
return backupPath, nil
}
func (e *systemUpgradeExecutor) restoreBackup(path string) error {
backupPath := path + ".upgrade.bak"
mode, err := fileModeOrDefault(backupPath, 0o600)
if err != nil {
return err
}
data, err := os.ReadFile(backupPath)
if err != nil {
return err
}
return writeFileWithMode(path, data, mode)
}
func (e *systemUpgradeExecutor) restoreUpgradeBackups(paths ...string) error {
var errs []string
for _, path := range paths {
if err := e.restoreBackup(path); err != nil {
errs = append(errs, fmt.Sprintf("%s: %v", path, err))
}
}
if len(errs) > 0 {
return fmt.Errorf("%s", strings.Join(errs, "; "))
}
return nil
}
func (e *systemUpgradeExecutor) replaceCompose(path string, data []byte) error {
if len(bytes.TrimSpace(data)) == 0 {
return fmt.Errorf("compose asset is empty")
}
mode, err := fileModeOrDefault(path, 0o644)
if err != nil {
return err
}
return writeFileWithMode(path, data, mode)
}
func (h *Handler) buildSystemUpgradeDownloadURL(version, filename string) string {
enabled, proxyURL := h.getGithubProxyConfig()
base := fmt.Sprintf("%s/%s/releases/download/%s/%s", strings.TrimRight(systemUpgradeReleaseBaseURL, "/"), githubRepo, version, filename)
@@ -383,55 +239,6 @@ func (h *Handler) resolveSystemUpgradeLatestReleaseByChannel(channel string) (st
return "", fmt.Errorf("未找到%s版本号", releaseChannelLabel(normalizedChannel))
}
func fileModeOrDefault(path string, fallback os.FileMode) (os.FileMode, error) {
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return fallback, nil
}
return 0, err
}
return info.Mode().Perm(), nil
}
func writeFileWithMode(path string, data []byte, mode os.FileMode) error {
if err := os.WriteFile(path, data, mode); err != nil {
return err
}
return os.Chmod(path, mode)
}
func (e *systemUpgradeExecutor) currentBackendImage(ctx context.Context) (string, error) {
if err := validateBackendContainerName(e.backendContainer); err != nil {
return "", err
}
out, err := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.Image}}", e.backendContainer).CombinedOutput()
if err != nil {
return "", fmt.Errorf("inspect backend image failed: %v: %s", err, strings.TrimSpace(string(out)))
}
imageID := strings.TrimSpace(string(out))
if imageID == "" {
return "", fmt.Errorf("backend image id is empty")
}
return imageID, nil
}
func (e *systemUpgradeExecutor) startHelper(ctx context.Context, imageID, helperName string) (string, error) {
args, err := e.buildHelperRunArgs(imageID, helperName)
if err != nil {
return "", err
}
out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput()
if err != nil {
return "", fmt.Errorf("start helper failed: %v: %s", err, strings.TrimSpace(string(out)))
}
containerID := strings.TrimSpace(string(out))
if containerID == "" {
containerID = helperName
}
return containerID, nil
}
func (h *Handler) downloadReleaseAsset(version, filename string) ([]byte, error) {
url := h.buildSystemUpgradeDownloadURL(version, filename)
client := &http.Client{Timeout: 60 * time.Second}
@@ -485,16 +292,26 @@ func releasesForChannel(releases []githubRelease, channel string) []systemUpgrad
func decodeSystemUpgradeRequest(r *http.Request, req *systemUpgradeRequest) error {
defer r.Body.Close()
body, err := io.ReadAll(r.Body)
body, err := io.ReadAll(io.LimitReader(r.Body, 4097))
if err != nil {
return err
}
if len(body) > 4096 {
return fmt.Errorf("request too large")
}
if len(bytes.TrimSpace(body)) == 0 {
return nil
}
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.DisallowUnknownFields()
return decoder.Decode(req)
if err := decoder.Decode(req); err != nil {
return err
}
var extra any
if decoder.Decode(&extra) != io.EOF {
return fmt.Errorf("unexpected trailing JSON")
}
return nil
}
func systemUpgradeVersionResponse(current, channel, latest string, lookupErr error, capability systemUpgradeCapabilityData) systemUpgradeVersionData {
@@ -595,63 +412,63 @@ func (h *Handler) systemUpgrade(w http.ResponseWriter, r *http.Request) {
return
}
}
imageID, err := exec.currentBackendImage(r.Context())
if err != nil {
response.WriteJSON(w, response.Err(-2, err.Error()))
if err := validateUpgradeVersion(version); err != nil {
response.WriteJSON(w, response.ErrDefault("版本号格式无效"))
return
}
composePath := exec.composePath()
envPath := exec.envPath()
composeData, err := os.ReadFile(composePath)
composeData, err := os.ReadFile(exec.composePath())
if err != nil {
response.WriteJSON(w, response.Err(-2, "读取compose失败: "+err.Error()))
response.WriteJSON(w, response.ErrDefault("读取部署配置失败"))
return
}
composeAsset := exec.selectComposeAsset(composeData)
newCompose, err := h.downloadReleaseAsset(version, composeAsset)
if err != nil {
response.WriteJSON(w, response.Err(-2, err.Error()))
response.WriteJSON(w, response.ErrDefault("下载目标 Compose 失败"))
return
}
if _, err := exec.backupFile(composePath); err != nil {
response.WriteJSON(w, response.Err(-2, "备份compose失败: "+err.Error()))
return
}
if _, err := exec.backupFile(envPath); err != nil {
response.WriteJSON(w, response.Err(-2, "备份.env失败: "+err.Error()))
return
}
if err := exec.replaceCompose(composePath, newCompose); err != nil {
if restoreErr := exec.restoreUpgradeBackups(composePath, envPath); restoreErr != nil {
err = fmt.Errorf("%v; 回滚失败: %v", err, restoreErr)
}
response.WriteJSON(w, response.Err(-2, "替换compose失败: "+err.Error()))
return
}
if err := exec.updateEnvVersion(envPath, version); err != nil {
if restoreErr := exec.restoreUpgradeBackups(composePath, envPath); restoreErr != nil {
err = fmt.Errorf("%v; 回滚失败: %v", err, restoreErr)
}
response.WriteJSON(w, response.Err(-2, "更新版本配置失败: "+err.Error()))
return
}
helperName := fmt.Sprintf("flvx-upgrade-helper-%d", time.Now().Unix())
helperContainer, err := exec.startHelper(r.Context(), imageID, helperName)
// A detached helper must not be canceled when the browser disconnects.
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
job, imageID, err := (panelupgrade.Manager{Dir: exec.deployDir, Backend: exec.backendContainer}).Start(ctx, version, newCompose)
if err != nil {
if restoreErr := exec.restoreUpgradeBackups(composePath, envPath); restoreErr != nil {
err = fmt.Errorf("%v; 回滚失败: %v", err, restoreErr)
}
response.WriteJSON(w, response.Err(-2, err.Error()))
response.WriteJSON(w, response.ErrDefault(err.Error()))
return
}
response.WriteJSON(w, response.OK(systemUpgradeRunData{
Version: version,
Channel: channel,
ComposeAsset: composeAsset,
HelperContainer: helperContainer,
BackendImageID: imageID,
Message: systemUpgradeMessage,
}))
response.WriteJSON(w, response.OK(systemUpgradeRunData{JobID: job.ID, Version: version, Channel: channel, ComposeAsset: composeAsset, HelperContainer: job.HelperContainer, BackendImageID: imageID, Message: systemUpgradeMessage}))
}
func (h *Handler) systemUpgradeStatus(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
response.WriteJSON(w, response.ErrDefault("请求失败"))
return
}
var req struct {
JobID string `json:"jobId"`
}
defer r.Body.Close()
decoder := json.NewDecoder(io.LimitReader(r.Body, 4097))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&req); err != nil && err != io.EOF {
response.WriteJSON(w, response.ErrDefault("请求参数错误"))
return
}
var extra any
if decoder.Decode(&extra) != io.EOF {
response.WriteJSON(w, response.ErrDefault("请求参数错误"))
return
}
e := newSystemUpgradeExecutor()
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
defer cancel()
job, err := (panelupgrade.Manager{Dir: e.deployDir, Backend: e.backendContainer}).Status(ctx, req.JobID)
if err != nil {
response.WriteJSON(w, response.ErrDefault("无法读取升级任务状态"))
return
}
response.WriteJSON(w, response.OK(struct {
CurrentVersion string `json:"currentVersion"`
Job *panelupgrade.Job `json:"job"`
}{currentPanelVersion(), job}))
}
@@ -8,7 +8,6 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
@@ -128,149 +127,12 @@ func TestSelectComposeAssetFallsBackToIPv4Template(t *testing.T) {
}
}
func TestUpdateEnvVersionReplacesExistingValue(t *testing.T) {
dir := t.TempDir()
envPath := filepath.Join(dir, ".env")
if err := os.WriteFile(envPath, []byte("FLUX_VERSION=2.1.8\nJWT_SECRET=test\n"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
exec := &systemUpgradeExecutor{deployDir: dir, backendContainer: "flux-panel-backend"}
if err := exec.updateEnvVersion(envPath, "2.1.9"); err != nil {
t.Fatalf("updateEnvVersion() error = %v", err)
}
data, err := os.ReadFile(envPath)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
want := "FLUX_VERSION=2.1.9\nJWT_SECRET=test\n"
if string(data) != want {
t.Fatalf("env content = %q, want %q", string(data), want)
}
}
func TestUpdateEnvVersionAppendsMissingValue(t *testing.T) {
dir := t.TempDir()
envPath := filepath.Join(dir, ".env")
if err := os.WriteFile(envPath, []byte("JWT_SECRET=test\n"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
exec := &systemUpgradeExecutor{deployDir: dir, backendContainer: "flux-panel-backend"}
if err := exec.updateEnvVersion(envPath, "2.1.9"); err != nil {
t.Fatalf("updateEnvVersion() error = %v", err)
}
data, err := os.ReadFile(envPath)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
want := "JWT_SECRET=test\nFLUX_VERSION=2.1.9\n"
if string(data) != want {
t.Fatalf("env content = %q, want %q", string(data), want)
}
}
func TestUpdateEnvVersionRejectsUnsafeValue(t *testing.T) {
for _, version := range []string{"", "2.1.9\nJWT_SECRET=bad", "2.1.9\rbad", "2.1.9\x00bad", "2.1.9\x1fbad"} {
t.Run(version, func(t *testing.T) {
dir := t.TempDir()
envPath := filepath.Join(dir, ".env")
original := []byte("JWT_SECRET=test\n")
if err := os.WriteFile(envPath, original, 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
exec := &systemUpgradeExecutor{deployDir: dir, backendContainer: "flux-panel-backend"}
if err := exec.updateEnvVersion(envPath, version); err == nil {
t.Fatal("expected unsafe version to fail validation")
}
data, err := os.ReadFile(envPath)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != string(original) {
t.Fatalf("env content changed to %q, want %q", string(data), string(original))
}
})
}
}
func TestUpdateEnvVersionAcceptsVersionLabels(t *testing.T) {
for _, version := range []string{"2.1.9", "2.1.9-beta14", "v-test"} {
t.Run(version, func(t *testing.T) {
dir := t.TempDir()
envPath := filepath.Join(dir, ".env")
if err := os.WriteFile(envPath, []byte("JWT_SECRET=test\n"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
exec := &systemUpgradeExecutor{deployDir: dir, backendContainer: "flux-panel-backend"}
if err := exec.updateEnvVersion(envPath, version); err != nil {
t.Fatalf("updateEnvVersion() error = %v", err)
}
})
}
}
func TestUpdateEnvVersionPreservesFileMode(t *testing.T) {
dir := t.TempDir()
envPath := filepath.Join(dir, ".env")
if err := os.WriteFile(envPath, []byte("FLUX_VERSION=2.1.8\nJWT_SECRET=test\n"), 0o600); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
exec := &systemUpgradeExecutor{deployDir: dir, backendContainer: "flux-panel-backend"}
if err := exec.updateEnvVersion(envPath, "2.1.9"); err != nil {
t.Fatalf("updateEnvVersion() error = %v", err)
}
info, err := os.Stat(envPath)
if err != nil {
t.Fatalf("Stat() error = %v", err)
}
if got := info.Mode().Perm(); got != 0o600 {
t.Fatalf("env mode = %o, want 0600", got)
}
}
func TestValidateBackendContainerNameRejectsUnsafeValue(t *testing.T) {
if err := validateBackendContainerName("flux-panel-backend;rm -rf /"); err == nil {
t.Fatal("expected unsafe container name to fail validation")
}
}
func TestBuildHelperRunArgsUsesDetachedContainer(t *testing.T) {
exec := &systemUpgradeExecutor{deployDir: "/opt/flvx-panel", backendContainer: "flux-panel-backend"}
args, err := exec.buildHelperRunArgs("sha256:abc", "flvx-upgrade-helper")
if err != nil {
t.Fatalf("buildHelperRunArgs() error = %v", err)
}
want := []string{
"run", "-d", "--rm", "--name", "flvx-upgrade-helper",
"--volumes-from", "flux-panel-backend",
"-v", "/var/run/docker.sock:/var/run/docker.sock",
"-e", "PANEL_DEPLOY_DIR=/opt/flvx-panel",
"--entrypoint", "/bin/sh", "sha256:abc",
"-c", exec.helperScript(),
}
if !reflect.DeepEqual(args, want) {
t.Fatalf("buildHelperRunArgs() = %#v, want %#v", args, want)
}
}
func TestBuildHelperRunArgsRejectsUnsafeBackendContainer(t *testing.T) {
exec := &systemUpgradeExecutor{deployDir: "/opt/flvx-panel", backendContainer: "flux-panel-backend;rm -rf /"}
if _, err := exec.buildHelperRunArgs("sha256:abc", "flvx-upgrade-helper"); err == nil {
t.Fatal("expected unsafe backend container name to fail validation")
}
}
func TestSystemVersionRejectsWrongMethod(t *testing.T) {
h := &Handler{}
req := httptest.NewRequest(http.MethodGet, "/api/v1/system/version", nil)
@@ -350,64 +212,6 @@ func TestSystemUpgradeFailsFastBeforeMutatingFiles(t *testing.T) {
}
}
func TestUpgradeBackupUsesStablePathAndRestoreRestoresOriginal(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "docker-compose.yml")
if err := os.WriteFile(path, []byte("original"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
exec := &systemUpgradeExecutor{deployDir: dir, backendContainer: "flux-panel-backend"}
backupPath, err := exec.backupFile(path)
if err != nil {
t.Fatalf("backupFile() error = %v", err)
}
if backupPath != path+".upgrade.bak" {
t.Fatalf("backup path = %q, want %q", backupPath, path+".upgrade.bak")
}
if err := os.WriteFile(path, []byte("mutated"), 0o644); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
if err := exec.restoreBackup(path); err != nil {
t.Fatalf("restoreBackup() error = %v", err)
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile() error = %v", err)
}
if string(data) != "original" {
t.Fatalf("restored content = %q, want original", string(data))
}
}
func TestRestoreBackupPreservesOriginalFileMode(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, ".env")
if err := os.WriteFile(path, []byte("FLUX_VERSION=2.1.8\nJWT_SECRET=test\n"), 0o600); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
exec := &systemUpgradeExecutor{deployDir: dir, backendContainer: "flux-panel-backend"}
if _, err := exec.backupFile(path); err != nil {
t.Fatalf("backupFile() error = %v", err)
}
if err := os.Remove(path); err != nil {
t.Fatalf("Remove() error = %v", err)
}
if err := exec.restoreBackup(path); err != nil {
t.Fatalf("restoreBackup() error = %v", err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatalf("Stat() error = %v", err)
}
if got := info.Mode().Perm(); got != 0o600 {
t.Fatalf("restored mode = %o, want 0600", got)
}
}
func TestDecodeSystemUpgradeRequestRejectsTruncatedJSON(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/v1/system/check-updates", strings.NewReader(`{"channel":"stable"`))
var payload systemUpgradeRequest
@@ -0,0 +1,66 @@
# Panel upgrade safety and recovery
The helper is a subcommand in the **currently running image**, dispatched before
application/database initialization. This fix must first be installed through the
normal administrator deployment procedure on older releases lacking the helper.
It cannot retroactively repair an already-running legacy upgrade helper.
Supported deployments have one `docker-compose.yml`, the original Compose project
labels, and a writable host bind mount containing that file and `.env`. The
helper mounts that directory at its **host absolute path** and passes explicit
project, project-directory, env-file and compose-file options. Multiple Compose
files, mismatched mounts/resources, or changed database settings fail closed.
Jobs are under `<deployment>/.panel-upgrades/<jobId>/` (directory 0700, files
0600). `*.original` contains immutable per-job configuration snapshots, not a live
database backup. `events.log` contains safe phase messages only. Legacy
`.upgrade.bak` files and `upgrade.log` are never changed. Helper containers remain
for diagnosis; they are intentionally not automatically removed. Operators may
remove completed helpers after retaining the job's files.
The `latest` pointer plus filesystem lock prevents simultaneous jobs across API
processes and backend replacement. Failed validation/pull leaves live files
untouched. A failed configuration application attempts to restore both files
before any recreate. Once `restarting` is recorded, failures require recovery:
no automatic database rollback, image rollback, or container rollback is safe.
The helper has a 30 minute overall deadline, including a maximum five minute
post-recreate verification window. Docker client timeout during recreation can
leave daemon work in flight; it is always treated as uncertain, not retried.
## Operator recovery after an interrupted/uncertain job
There is intentionally no remotely callable “clear lock” endpoint. Do not delete
job directories or rewrite status JSON to force another attempt.
1. Work on the Docker host and identify the job from `latest`, `status.json` and
`plan.json`. Preserve the entire job directory, live Compose/env files, and
container inspection information in a private location. These files can
contain credentials. Never paste `.env` or raw Compose config into public logs.
2. Restore Docker daemon access if necessary. Inspect the exact helper container
named by the job. If it is running, let it finish or explicitly stop it as an
administrator. Verify it has stopped. If launch was uncertain, establish that
no matching helper exists and no Docker create/up operation remains in flight.
Do not clear the job while any of these facts are uncertain.
3. Inspect the original project's backend and frontend container IDs, images,
mounts and health. Check the actual database and application logs privately.
After any attempted recreation, assume migrations may already have occurred.
Preserve a current database snapshot before further manual deployment work.
Use the normal documented deployment procedure to bring both services to a
consistent, healthy state. Do not copy an old database over the current one or
blindly redeploy an older image. Configuration snapshots alone cannot undo a
schema migration.
4. Only after verifying no helper/daemon work remains and both services and their
database are consistent, archive the `latest` pointer under a unique filename
in `.panel-upgrades/` (for example `resolved-<jobId>.pointer`). Do this while
the API is stopped, or hold an exclusive OS flock on `.panel-upgrades/lock`,
so no other start can race this manual recovery. Keep the job directory and
archived pointer. This permits a new job without changing recovery evidence.
5. Restart the API if stopped and reload the browser settings page so it discovers
the current `latest` task instead of following the resolved task ID. No current
job is expected. A fresh start creates a new ID and new backups, never
overwriting old evidence.
A lost helper, unavailable daemon, or stale status is displayed as
`recovery_required` while retaining the active lock. This is intentionally a
conservative derived status, so a helper that is still completing work can report
its actual durable result without racing a status request.
+311
View File
@@ -0,0 +1,311 @@
package panelupgrade
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"time"
)
const helperTimeout = 30 * time.Minute
var verificationTimeout = 5 * time.Minute
type composeConfig struct {
FullServices map[string]map[string]any `json:"-"`
Services map[string]struct {
Image string `json:"image"`
ContainerName string `json:"container_name"`
Volumes []map[string]any `json:"volumes"`
Environment map[string]string `json:"environment"`
} `json:"services"`
Volumes map[string]any `json:"volumes"`
Networks map[string]any `json:"networks"`
}
func composeArgs(p Plan, dir string, args ...string) []string {
return append([]string{"compose", "--project-name", p.Project, "--project-directory", p.HostDir, "--env-file", filepath.Join(dir, ".env"), "-f", filepath.Join(dir, "docker-compose.yml")}, args...)
}
func config(ctx context.Context, p Plan, dir string) (composeConfig, error) {
var c composeConfig
b, e := docker(ctx, composeArgs(p, dir, "config", "--format", "json")...)
if e != nil {
return c, e
}
e = json.Unmarshal(b, &c)
if e == nil {
var raw struct {
Services map[string]map[string]any `json:"services"`
}
e = json.Unmarshal(b, &raw)
c.FullServices = raw.Services
}
return c, e
}
func validateConfig(old, next composeConfig, p Plan) error {
// Compare complete normalized services, not a projection that can silently drop
// host bindings, commands, security options, networks or dependency changes.
if old.FullServices == nil || next.FullServices == nil {
return errors.New("missing full Compose configuration")
}
raw, _ := json.Marshal(old.FullServices)
var expected map[string]map[string]any
if json.Unmarshal(raw, &expected) != nil {
return errors.New("invalid configuration")
}
for _, service := range []string{"backend", "frontend"} {
if expected[service] == nil || next.FullServices[service] == nil {
return errors.New("missing service")
}
expected[service]["image"] = next.FullServices[service]["image"]
}
if env, ok := expected["backend"]["environment"].(map[string]any); ok {
env["FLUX_VERSION"] = p.Job.TargetVersion
} else {
return errors.New("missing backend environment")
}
if !reflect.DeepEqual(expected, next.FullServices) {
return errors.New("unsupported service configuration change; manual upgrade required")
}
for service, image := range map[string]string{"backend": "ghcr.io/sagit-chu/flux-panel-backend:", "frontend": "ghcr.io/sagit-chu/vite-frontend:"} {
a, ok := old.Services[service]
b, present := next.Services[service]
if !ok || !present || b.Image != image+p.Job.TargetVersion || a.ContainerName != b.ContainerName || !reflect.DeepEqual(a.Volumes, b.Volumes) {
return errors.New("service image, container identity or persistent mounts do not match the supported deployment")
}
}
if next.Services["backend"].ContainerName != p.Backend || next.Services["backend"].Environment["FLUX_VERSION"] != p.Job.TargetVersion {
return errors.New("backend identity/version mismatch")
}
for service := range next.Services {
if service != "backend" && service != "frontend" && service != "postgres" {
return errors.New("unsupported service")
}
}
for key, value := range old.Services["backend"].Environment {
if key != "FLUX_VERSION" && next.Services["backend"].Environment[key] != value {
return errors.New("backend environment changed; manual upgrade required")
}
}
for key, value := range next.Services["backend"].Environment {
if key != "FLUX_VERSION" && old.Services["backend"].Environment[key] != value {
return errors.New("backend environment changed; manual upgrade required")
}
}
// Database configuration and shared resource identities are not upgrade targets.
if !reflect.DeepEqual(old.Services["postgres"], next.Services["postgres"]) || !reflect.DeepEqual(old.Volumes, next.Volumes) || !reflect.DeepEqual(old.Networks, next.Networks) {
return errors.New("database or shared resource configuration changed; manual upgrade required")
}
return nil
}
// validateLive proves the normalized source configuration still describes the running
// backend. In particular, Compose must not attach a fresh database volume by accident.
func validateLive(ctx context.Context, p Plan, c composeConfig) error {
live, err := inspect(ctx, p.Backend)
if err != nil {
return err
}
if live.Image != p.Image || !live.State.Running || live.Config.Labels["com.docker.compose.project"] != p.Project || live.Config.Labels["com.docker.compose.service"] != "backend" {
return errors.New("running backend changed")
}
if len(live.Mounts) != len(c.Services["backend"].Volumes) {
return errors.New("unexpected live mounts; manual upgrade required")
}
for _, v := range c.Services["backend"].Volumes {
typ, _ := v["type"].(string)
src, _ := v["source"].(string)
dst, _ := v["target"].(string)
if typ == "volume" {
spec, ok := c.Volumes[src].(map[string]any)
if !ok {
return errors.New("unknown volume")
}
src, _ = spec["name"].(string)
}
if src == "" || dst == "" || (typ != "volume" && typ != "bind") {
return errors.New("unsupported mount")
}
found := false
for _, mount := range live.Mounts {
actual := mount.Source
if typ == "volume" {
actual = mount.Name
}
if mount.Type == typ && mount.Destination == dst && actual == src && mount.RW != (v["read_only"] == true) {
found = true
}
}
if !found {
return errors.New("live persistent mount does not match Compose")
}
}
env := map[string]string{}
for _, v := range live.Config.Env {
parts := strings.SplitN(v, "=", 2)
if len(parts) == 2 {
env[parts[0]] = parts[1]
}
}
for _, key := range []string{"DB_TYPE", "DB_PATH", "DATABASE_URL", "PANEL_DEPLOY_DIR", "PANEL_BACKEND_CONTAINER"} {
if env[key] != c.Services["backend"].Environment[key] {
return errors.New("live database/deployment environment does not match Compose")
}
}
return nil
}
// RunHelper runs before app/database initialization and does not depend on the old backend process.
func RunHelper(planPath string) error {
b, e := os.ReadFile(planPath)
if e != nil {
return e
}
var p Plan
if e = json.Unmarshal(b, &p); e != nil {
return e
}
if !idPattern.MatchString(p.Job.ID) || ValidateVersion(p.Job.TargetVersion) != nil || !projectPattern.MatchString(p.Project) {
return errors.New("invalid plan")
}
m := Manager{Dir: p.HostDir, Backend: p.Backend}
dir := m.jobDir(p.Job.ID)
if filepath.Clean(planPath) != filepath.Join(dir, "plan.json") {
return errors.New("invalid plan location")
}
// Reject duplicate helper invocations without affecting the live job.
f, e := os.OpenFile(filepath.Join(dir, "helper.started"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if e != nil {
return e
}
f.Close()
j, e := m.read(p.Job.ID)
if e != nil {
return e
}
if !j.Active {
return errors.New("job is not active")
}
ctx, cancel := context.WithTimeout(context.Background(), helperTimeout)
defer cancel()
fail := func(msg string) error {
if e := m.write(j, "failed", msg); e != nil {
return e
}
return errors.New(msg)
}
recovery := func(msg string) error {
m.write(j, "recovery_required", msg+";请管理员核对容器、数据库和任务备份后人工恢复,禁止自动回滚或重复升级")
return errors.New(msg)
}
if e = m.write(j, "preparing", "校验部署配置和持久化资源"); e != nil {
return e
}
old, e := config(ctx, p, p.HostDir)
if e != nil {
return fail("现有 Compose 配置校验失败,线上配置未变更")
}
next, e := config(ctx, p, dir)
if e != nil {
return fail("目标 Compose 配置校验失败,线上配置未变更")
}
if validateConfig(old, next, p) != nil {
return fail("目标镜像、服务或持久化资源不匹配,需手动升级")
}
if validateLive(ctx, p, old) != nil {
return fail("运行中后端与部署配置不一致,需手动升级")
}
if e = m.write(j, "pulling", "正在拉取并检查目标镜像,线上配置尚未变更"); e != nil {
return e
}
if _, e = docker(ctx, composeArgs(p, dir, "pull", "backend", "frontend")...); e != nil {
return fail("镜像拉取失败或超时,线上配置未变更")
}
targets := map[string]string{}
for _, service := range []string{"backend", "frontend"} {
out, err := docker(ctx, "image", "inspect", "--format", "{{.Id}}", next.Services[service].Image)
if err != nil || !strings.HasPrefix(strings.TrimSpace(string(out)), "sha256:") {
return fail("目标镜像校验失败,线上配置未变更")
}
targets[service] = strings.TrimSpace(string(out))
}
if validateLive(ctx, p, old) != nil {
return fail("拉取期间运行中后端发生变化,已安全取消")
}
// Detect edits since snapshots before replacing either live file.
for _, name := range []string{"docker-compose.yml", ".env"} {
live, err := os.ReadFile(filepath.Join(p.HostDir, name))
if err != nil {
return fail("读取线上配置失败")
}
backup, err := os.ReadFile(filepath.Join(dir, name+".original"))
if err != nil || string(live) != string(backup) {
return fail("线上配置已被修改,本任务已安全取消")
}
}
if e = m.write(j, "applying", "正在应用已校验配置"); e != nil {
return e
}
for _, name := range []string{"docker-compose.yml", ".env"} {
data, err := os.ReadFile(filepath.Join(dir, name))
if err == nil {
err = atomic(filepath.Join(p.HostDir, name), data)
}
if err != nil {
for _, restore := range []string{"docker-compose.yml", ".env"} {
original, re := os.ReadFile(filepath.Join(dir, restore+".original"))
if re != nil || atomic(filepath.Join(p.HostDir, restore), original) != nil {
return recovery("配置恢复失败,尚未执行容器重建")
}
}
return fail("应用配置失败,已恢复原配置,未执行容器重建")
}
}
// Persist the point of no automatic rollback BEFORE Docker can recreate any service.
if e = m.write(j, "restarting", "正在重建面板服务"); e != nil {
return recovery("无法记录重建状态,已停止执行")
}
if _, e = docker(ctx, composeArgs(p, p.HostDir, "up", "-d", "--no-deps", "--force-recreate", "--pull", "never", "backend", "frontend")...); e != nil {
return recovery("容器重建失败或超时,可能已部分生效")
}
if e = m.write(j, "verifying", "正在验证服务版本及健康状态"); e != nil {
return e
}
deadline := time.Now().Add(verificationTimeout)
for time.Now().Before(deadline) && ctx.Err() == nil {
ready := true
for _, service := range []string{"backend", "frontend"} {
c, err := inspect(ctx, next.Services[service].ContainerName)
if err != nil || !c.State.Running || c.Image != targets[service] || c.Config.Labels["com.docker.compose.project"] != p.Project || c.Config.Labels["com.docker.compose.service"] != service {
ready = false
continue
}
if c.State.Health != nil && c.State.Health.Status != "healthy" {
ready = false
}
if service == "backend" && c.State.Health == nil {
ready = false
}
}
if ready { // Frontend has no Docker healthcheck in the stock template.
probeCtx, probeCancel := context.WithTimeout(ctx, 10*time.Second)
_, probeErr := docker(probeCtx, "exec", next.Services["frontend"].ContainerName, "wget", "-q", "-T", "5", "-O", "/dev/null", "http://127.0.0.1:80/")
probeCancel()
ready = probeErr == nil
}
if ready {
return m.write(j, "succeeded", "升级完成,目标镜像和服务健康状态已验证")
}
select {
case <-ctx.Done():
case <-time.After(3 * time.Second):
}
}
return recovery("升级后服务健康检查失败或超时")
}
+310
View File
@@ -0,0 +1,310 @@
// Package panelupgrade implements durable, conservative Docker Compose upgrades.
package panelupgrade
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"syscall"
"time"
)
var versionPattern = regexp.MustCompile(`^v?[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9][A-Za-z0-9.-]*)?$`)
var idPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
var projectPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`)
func ValidateVersion(v string) error {
if !versionPattern.MatchString(v) || len(v) > 128 {
return errors.New("invalid release version")
}
return nil
}
type Job struct {
ID string `json:"id"`
TargetVersion string `json:"targetVersion"`
Stage string `json:"stage"`
Active bool `json:"active"`
RequiresRecovery bool `json:"requiresRecovery"`
Message string `json:"message"`
StartedAt int64 `json:"startedAt"`
UpdatedAt int64 `json:"updatedAt"`
HelperContainer string `json:"helperContainer,omitempty"`
BackupDir string `json:"backupDir,omitempty"`
}
type Plan struct {
Job Job
DeployDir, HostDir, Project, Backend, Image string
}
type Manager struct{ Dir, Backend string }
func (m Manager) root() string { return filepath.Join(m.Dir, ".panel-upgrades") }
func (m Manager) jobDir(id string) string { return filepath.Join(m.root(), id) }
func atomic(path string, b []byte) error {
f, e := os.CreateTemp(filepath.Dir(path), ".write-")
if e != nil {
return e
}
n := f.Name()
defer os.Remove(n)
if e = f.Chmod(0600); e == nil {
_, e = f.Write(b)
}
if e == nil {
e = f.Sync()
}
ce := f.Close()
if e == nil {
e = ce
}
if e != nil {
return e
}
if e = os.Rename(n, path); e != nil {
return e
}
d, e := os.Open(filepath.Dir(path))
if e != nil {
return e
}
defer d.Close()
return d.Sync()
}
func save(path string, v any) error {
b, e := json.Marshal(v)
if e != nil {
return e
}
return atomic(path, b)
}
func (m Manager) lock() (func(), error) {
if e := os.MkdirAll(m.root(), 0700); e != nil {
return nil, e
}
f, e := os.OpenFile(filepath.Join(m.root(), "lock"), os.O_CREATE|os.O_RDWR, 0600)
if e != nil {
return nil, e
}
if e = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); e != nil {
f.Close()
return nil, errors.New("another upgrade operation is in progress")
}
return func() { syscall.Flock(int(f.Fd()), syscall.LOCK_UN); f.Close() }, nil
}
func docker(ctx context.Context, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "docker", args...)
cmd.WaitDelay = 2 * time.Second
b, e := cmd.Output()
if e != nil {
return nil, errors.New("Docker operation failed (details suppressed to protect credentials)")
}
return b, nil
}
type container struct {
Image string
Config struct {
Labels map[string]string
Env []string
}
State struct {
Running bool
Health *struct{ Status string }
}
Mounts []struct {
Type, Source, Destination, Name string
RW bool
}
}
func inspect(ctx context.Context, name string) (container, error) {
var c []container
b, e := docker(ctx, "inspect", name)
if e != nil {
return container{}, e
}
if json.Unmarshal(b, &c) != nil || len(c) != 1 {
return container{}, errors.New("invalid Docker inspection")
}
return c[0], nil
}
func (m Manager) Identity(ctx context.Context) (Plan, error) {
p := Plan{DeployDir: m.Dir, Backend: m.Backend}
if !filepath.IsAbs(m.Dir) || filepath.Clean(m.Dir) != m.Dir {
return p, errors.New("deployment directory must be an absolute clean path")
}
c, e := inspect(ctx, m.Backend)
if e != nil {
return p, e
}
p.Image = c.Image
p.Project = c.Config.Labels["com.docker.compose.project"]
if !c.State.Running || !projectPattern.MatchString(p.Project) || c.Config.Labels["com.docker.compose.service"] != "backend" || !strings.HasPrefix(p.Image, "sha256:") {
return p, errors.New("cannot establish original Compose identity")
}
for _, v := range c.Mounts {
if v.Destination == m.Dir && v.Type == "bind" && v.RW {
p.HostDir = v.Source
}
}
if p.HostDir == "/" || !filepath.IsAbs(p.HostDir) || strings.ContainsAny(p.HostDir, ",\n\r") || filepath.Clean(p.HostDir) != p.HostDir {
return p, errors.New("deployment directory must be a writable host bind mount")
}
for _, protected := range []string{"/app/paneld", "/usr/local/bin/docker", "/usr/local/libexec/docker/cli-plugins/docker-compose", "/var/run/docker.sock", "/bin", "/lib", "/etc", "/proc", "/sys", "/dev"} {
if p.HostDir == protected || strings.HasPrefix(protected, p.HostDir+"/") || strings.HasPrefix(p.HostDir, protected+"/") {
return p, errors.New("host deployment mount would shadow helper runtime paths")
}
}
if c.Config.Labels["com.docker.compose.project.working_dir"] != p.HostDir || c.Config.Labels["com.docker.compose.project.config_files"] != filepath.Join(p.HostDir, "docker-compose.yml") {
return p, errors.New("custom or multiple Compose paths require manual upgrade")
}
return p, nil
}
func (m Manager) read(id string) (*Job, error) {
if !idPattern.MatchString(id) {
return nil, errors.New("invalid job ID")
}
b, e := os.ReadFile(filepath.Join(m.jobDir(id), "status.json"))
if e != nil {
return nil, e
}
var j Job
e = json.Unmarshal(b, &j)
return &j, e
}
func (m Manager) Status(ctx context.Context, id string) (*Job, error) {
if id == "" {
b, e := os.ReadFile(filepath.Join(m.root(), "latest"))
if os.IsNotExist(e) {
return nil, nil
}
if e != nil {
return nil, e
}
id = string(b)
}
j, e := m.read(id)
if e != nil {
return nil, e
}
if !j.Active || time.Now().UnixMilli()-j.UpdatedAt < 90000 {
return j, nil
}
// An unavailable daemon, missing helper, or uncertain launch must never release the task lock.
c, e := inspect(ctx, j.HelperContainer)
if e == nil && c.State.Running {
return j, nil
}
j.RequiresRecovery = true
j.Message = "升级 helper 已停止或无法确认状态。请管理员检查任务备份和容器状态;在确认安全前禁止再次升级。"
// This is a derived view: do not race the helper's durable state writes.
j.Stage = "recovery_required"
return j, nil
}
func (m Manager) write(j *Job, stage, msg string) error {
j.Stage = stage
j.Message = msg
j.UpdatedAt = time.Now().UnixMilli()
j.Active = stage != "succeeded" && stage != "failed"
j.RequiresRecovery = stage == "recovery_required"
if e := save(filepath.Join(m.jobDir(j.ID), "status.json"), j); e != nil {
return e
}
f, e := os.OpenFile(filepath.Join(m.jobDir(j.ID), "events.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if e != nil {
return e
}
defer f.Close()
_, e = fmt.Fprintf(f, "%d %s %s\n", j.UpdatedAt, stage, msg)
return e
}
func (m Manager) Start(ctx context.Context, version string, compose []byte) (*Job, string, error) {
if e := ValidateVersion(version); e != nil {
return nil, "", e
}
unlock, e := m.lock()
if e != nil {
return nil, "", e
}
defer unlock()
old, e := m.Status(ctx, "")
if e != nil {
return nil, "", e
}
if old != nil && (old.Active || old.RequiresRecovery) {
return old, "", errors.New("已有面板升级任务执行中或需要人工恢复")
}
p, e := m.Identity(ctx)
if e != nil {
return nil, "", e
}
b := make([]byte, 16)
if _, e = rand.Read(b); e != nil {
return nil, "", e
}
id := hex.EncodeToString(b)
dir := m.jobDir(id)
if e = os.Mkdir(dir, 0700); e != nil {
return nil, "", e
}
j := Job{ID: id, TargetVersion: version, StartedAt: time.Now().UnixMilli(), HelperContainer: "flvx-upgrade-" + id, BackupDir: filepath.Join(p.HostDir, ".panel-upgrades", id)}
p.Job = j
if e = m.write(&j, "preparing", "正在准备独立升级任务"); e != nil {
return nil, "", e
}
if e = atomic(filepath.Join(m.root(), "latest"), []byte(id)); e != nil {
return nil, "", e
}
fail := func(err error) (*Job, string, error) {
m.write(&j, "failed", "准备失败,线上配置未变更")
return &j, p.Image, err
}
for _, name := range []string{"docker-compose.yml", ".env"} {
data, err := os.ReadFile(filepath.Join(m.Dir, name))
if err != nil {
return fail(err)
}
if err = atomic(filepath.Join(dir, name+".original"), data); err != nil {
return fail(err)
}
}
env, e := os.ReadFile(filepath.Join(dir, ".env.original"))
if e != nil {
return fail(e)
}
lines := strings.Split(string(env), "\n")
for i, line := range lines {
if strings.HasPrefix(strings.TrimSpace(line), "FLUX_VERSION=") || strings.HasPrefix(strings.TrimSpace(line), "export FLUX_VERSION=") {
lines[i] = ""
}
}
env = []byte(strings.Join(lines, "\n") + "\nFLUX_VERSION=" + version + "\n")
if e = atomic(filepath.Join(dir, ".env"), env); e != nil {
return fail(e)
}
if e = atomic(filepath.Join(dir, "docker-compose.yml"), compose); e != nil {
return fail(e)
}
p.Job = j
if e = save(filepath.Join(dir, "plan.json"), p); e != nil {
return fail(e)
}
// No --rm: an exited helper is evidence needed after a daemon/client timeout.
args := []string{"run", "-d", "--name", j.HelperContainer, "--label", "io.flvx.upgrade-job=" + id, "--mount", "type=bind,src=" + p.HostDir + ",dst=" + p.HostDir, "--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock", "--entrypoint", "/app/paneld", p.Image, "panel-upgrade-helper", filepath.Join(p.HostDir, ".panel-upgrades", id, "plan.json")}
if _, e = docker(ctx, args...); e != nil {
c, ie := inspect(ctx, j.HelperContainer)
if ie != nil || !c.State.Running {
return &j, p.Image, errors.New("helper 启动结果不确定;任务已保留,请查询升级状态,勿重复启动")
}
}
return &j, p.Image, nil
}
@@ -0,0 +1,302 @@
package panelupgrade
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func fakeDocker(t *testing.T, body string) {
t.Helper()
d := t.TempDir()
if e := os.WriteFile(filepath.Join(d, "docker"), []byte("#!/bin/sh\n"+body), 0700); e != nil {
t.Fatal(e)
}
t.Setenv("PATH", d+":"+os.Getenv("PATH"))
}
func fixture(t *testing.T) (Manager, Plan) {
t.Helper()
dir := t.TempDir()
m := Manager{Dir: dir, Backend: "flux-panel-backend"}
id := strings.Repeat("a", 32)
os.MkdirAll(m.jobDir(id), 0700)
j := Job{ID: id, TargetVersion: "2.5.0", HelperContainer: "helper", StartedAt: time.Now().UnixMilli()}
m.write(&j, "preparing", "ready")
atomic(filepath.Join(m.root(), "latest"), []byte(id))
p := Plan{Job: j, HostDir: dir, DeployDir: "/opt/flvx-panel", Project: "original-project", Backend: m.Backend, Image: "sha256:old"}
save(filepath.Join(m.jobDir(id), "plan.json"), p)
for _, n := range []string{".env", "docker-compose.yml"} {
atomic(filepath.Join(dir, n), []byte("original"))
atomic(filepath.Join(m.jobDir(id), n+".original"), []byte("original"))
atomic(filepath.Join(m.jobDir(id), n), []byte("target"))
}
return m, p
}
func TestVersionRejectsComposeAndPathInjection(t *testing.T) {
for _, v := range []string{"latest", "../v1", "1.2.3\nSECRET=x", "1.2.3${TOKEN}", "1.2.3 x", "1.2.3;id"} {
if ValidateVersion(v) == nil {
t.Fatal(v)
}
}
for _, v := range []string{"2.5.0-rc10", "v2.5.0", "2.1.9-beta6"} {
if ValidateVersion(v) != nil {
t.Fatal(v)
}
}
}
func TestPersistentLockAcrossManagers(t *testing.T) {
m, p := fixture(t)
fakeDocker(t, "exit 1\n")
_, _, e := m.Start(context.Background(), "2.5.0", nil)
if e == nil {
t.Fatal("accepted concurrent job")
}
b, _ := os.ReadFile(filepath.Join(m.root(), "latest"))
if string(b) != p.Job.ID {
t.Fatal("changed active job")
}
}
func TestUncertainHelperNeverUnlocks(t *testing.T) {
m, p := fixture(t)
j, _ := m.read(p.Job.ID)
j.UpdatedAt = 1
save(filepath.Join(m.jobDir(j.ID), "status.json"), j)
fakeDocker(t, "exit 1\n")
j, e := m.Status(context.Background(), "")
if e != nil || !j.Active || !j.RequiresRecovery || j.Stage != "recovery_required" {
t.Fatalf("%+v %v", j, e)
}
if _, _, e = m.Start(context.Background(), "2.5.0", nil); e == nil {
t.Fatal("uncertain launch unlocked")
}
}
func configs(p Plan) (string, string) {
old := composeConfig{}
json.Unmarshal([]byte(`{"services":{"backend":{"image":"old","container_name":"flux-panel-backend","environment":{"FLUX_VERSION":"old"}},"frontend":{"image":"old","container_name":"vite-frontend"}}}`), &old)
a, _ := json.Marshal(old)
next := old
next.Services = make(map[string]struct {
Image string `json:"image"`
ContainerName string `json:"container_name"`
Volumes []map[string]any `json:"volumes"`
Environment map[string]string `json:"environment"`
})
for k, v := range old.Services {
next.Services[k] = v
}
v := next.Services["backend"]
v.Image = "ghcr.io/sagit-chu/flux-panel-backend:" + p.Job.TargetVersion
v.Environment = map[string]string{"FLUX_VERSION": p.Job.TargetVersion}
next.Services["backend"] = v
v = next.Services["frontend"]
v.Image = "ghcr.io/sagit-chu/vite-frontend:" + p.Job.TargetVersion
next.Services["frontend"] = v
b, _ := json.Marshal(next)
return string(a), string(b)
}
func TestHelperFailureBoundaries(t *testing.T) {
for _, stage := range []string{"config", "pull", "apply", "up", "health", "success"} {
t.Run(stage, func(t *testing.T) {
m, p := fixture(t)
old, next := configs(p)
dir := m.jobDir(p.Job.ID)
if stage == "apply" {
os.Remove(filepath.Join(dir, ".env"))
os.Mkdir(filepath.Join(dir, ".env"), 0700)
}
if stage == "health" {
before := verificationTimeout
verificationTimeout = 0
t.Cleanup(func() { verificationTimeout = before })
}
script := `case "$*" in
*"config --format json"*)
case "$*" in *"` + p.Job.ID + `"*) echo '` + next + `';; *) echo '` + old + `';; esac
;;
*"pull backend frontend"*) `
if stage == "pull" {
script += "echo SECRET >&2; exit 1"
} else {
script += "exit 0"
}
script += `;;
"image inspect"*) echo sha256:target;;
*"up -d"*) `
if stage == "up" {
script += "exit 1"
} else {
script += "touch \"" + filepath.Join(dir, "recreated") + "\"; exit 0"
}
script += `;;
"inspect flux-panel-backend") if [ -f "` + filepath.Join(dir, "recreated") + `" ]; then IMG=sha256:target; else IMG=sha256:old; fi; echo '[{"Image":"'"$IMG"'","State":{"Running":true,"Health":{"Status":"healthy"}},"Config":{"Labels":{"com.docker.compose.project":"original-project","com.docker.compose.service":"backend"}}}]';;
"inspect vite-frontend") echo '[{"Image":"sha256:target","State":{"Running":true},"Config":{"Labels":{"com.docker.compose.project":"original-project","com.docker.compose.service":"frontend"}}}]';;
"exec vite-frontend wget"*) exit 0;;
*) exit 1;; esac
`
if stage == "config" {
script = "echo SECRET >&2;exit 1\n"
}
fakeDocker(t, script)
e := RunHelper(filepath.Join(dir, "plan.json"))
j, _ := m.read(p.Job.ID)
want := "failed"
if stage == "up" || stage == "health" {
want = "recovery_required"
}
if stage == "success" {
want = "succeeded"
if e != nil {
t.Fatal(e)
}
}
if j.Stage != want {
t.Fatalf("stage=%s error=%v", j.Stage, e)
}
for _, n := range []string{".env", "docker-compose.yml"} {
b, _ := os.ReadFile(filepath.Join(m.Dir, n))
expected := "original"
if stage == "up" || stage == "success" || stage == "health" {
expected = "target"
}
if string(b) != expected {
t.Fatalf("%s mutated unexpectedly", n)
}
b, _ = os.ReadFile(filepath.Join(dir, n+".original"))
if string(b) != "original" {
t.Fatal("backup overwritten")
}
}
b, _ := os.ReadFile(filepath.Join(dir, "events.log"))
if strings.Contains(string(b), "SECRET") {
t.Fatal("secret leaked")
}
})
}
}
func TestComposeUsesOriginalHostPathAndProject(t *testing.T) {
p := Plan{Project: "my-project", HostDir: "/srv/panel"}
s := strings.Join(composeArgs(p, "/srv/panel/.panel-upgrades/job", "pull", "backend", "frontend"), " ")
if !strings.Contains(s, "--project-name my-project --project-directory /srv/panel") || strings.Contains(s, "remove-orphans") {
t.Fatal(s)
}
}
func TestStatusWithoutJobNeedsNoDocker(t *testing.T) {
fakeDocker(t, "exit 99\n")
j, e := (Manager{Dir: t.TempDir()}).Status(context.Background(), "")
if e != nil || j != nil {
t.Fatalf("%v %v", j, e)
}
}
func TestFullConfigRejectsUnintendedChanges(t *testing.T) {
for _, field := range []string{"ports", "network_mode", "command", "entrypoint", "privileged", "networks"} {
t.Run(field, func(t *testing.T) {
_, p := fixture(t)
oldJSON, nextJSON := configs(p)
var old, next composeConfig
json.Unmarshal([]byte(oldJSON), &old)
json.Unmarshal([]byte(nextJSON), &next)
var a, b struct {
Services map[string]map[string]any `json:"services"`
}
json.Unmarshal([]byte(oldJSON), &a)
json.Unmarshal([]byte(nextJSON), &b)
old.FullServices = a.Services
next.FullServices = b.Services
old.FullServices["frontend"][field] = "private-custom-setting"
if validateConfig(old, next, p) == nil {
t.Fatal("accepted service setting removal: " + field)
}
})
}
}
func TestLiveRejectsShadowMount(t *testing.T) {
_, p := fixture(t)
fakeDocker(t, `echo '[{"Image":"sha256:old","State":{"Running":true},"Config":{"Labels":{"com.docker.compose.project":"original-project","com.docker.compose.service":"backend"}},"Mounts":[{"Type":"bind","Source":"/srv/db/gost.db","Destination":"/app/data/gost.db","RW":true}]}]'`)
oldJSON, _ := configs(p)
var old composeConfig
json.Unmarshal([]byte(oldJSON), &old)
if validateLive(context.Background(), p, old) == nil {
t.Fatal("accepted undeclared shadow mount")
}
}
func identityJSON(dir, host, project, files string) string {
c := map[string]any{"Image": "sha256:old", "State": map[string]any{"Running": true}, "Config": map[string]any{"Labels": map[string]string{"com.docker.compose.project": project, "com.docker.compose.service": "backend", "com.docker.compose.project.working_dir": host, "com.docker.compose.project.config_files": files}}, "Mounts": []map[string]any{{"Type": "bind", "Source": host, "Destination": dir, "RW": true}}}
b, _ := json.Marshal([]any{c})
return string(b)
}
func TestStartUsesHostMountAndKeepsUniqueBackups(t *testing.T) {
dir := t.TempDir()
m := Manager{Dir: dir, Backend: "backend"}
for _, n := range []string{".env", "docker-compose.yml", ".env.upgrade.bak"} {
os.WriteFile(filepath.Join(dir, n), []byte("original-secret"), 0600)
}
args := filepath.Join(t.TempDir(), "args")
fakeDocker(t, `case "$1" in inspect) echo '`+identityJSON(dir, "/srv/original-panel", "original-project", "/srv/original-panel/docker-compose.yml")+`';; run) printf '%s\n' "$@" > '`+args+`'; echo container-id;; *) exit 1;; esac`)
j, _, e := m.Start(context.Background(), "2.5.0-rc10", []byte("target"))
if e != nil {
t.Fatal(e)
}
b, _ := os.ReadFile(args)
s := string(b)
for _, expected := range []string{"type=bind,src=/srv/original-panel,dst=/srv/original-panel", "/app/paneld", "sha256:old", "panel-upgrade-helper"} {
if !strings.Contains(s, expected) {
t.Fatal(s)
}
}
if strings.Contains(s, "--rm") || strings.Contains(s, "--volumes-from") {
t.Fatal(s)
}
first := j.ID
m.write(j, "failed", "safe failure")
j, _, e = m.Start(context.Background(), "2.5.1", []byte("target2"))
if e != nil || j.ID == first {
t.Fatalf("%v %v", j, e)
}
for _, n := range []string{".env", "docker-compose.yml", ".env.upgrade.bak"} {
b, _ := os.ReadFile(filepath.Join(dir, n))
if string(b) != "original-secret" {
t.Fatal("live or legacy overwritten")
}
}
b, _ = os.ReadFile(filepath.Join(m.jobDir(first), ".env.original"))
if string(b) != "original-secret" {
t.Fatal("old job backup changed")
}
}
func TestIdentityFailsClosed(t *testing.T) {
for _, host := range []string{"/", "/app", "/usr", "/var/run", "/proc", "relative", "/srv/bad,host"} {
t.Run(host, func(t *testing.T) {
dir := t.TempDir()
fakeDocker(t, "echo '"+identityJSON(dir, host, "project", filepath.Join(host, "docker-compose.yml"))+"'\n")
if _, e := (Manager{Dir: dir, Backend: "backend"}).Identity(context.Background()); e == nil {
t.Fatal(host)
}
})
}
}
func TestUncertainRunIsNotRetriedOrUnlocked(t *testing.T) {
dir := t.TempDir()
m := Manager{Dir: dir, Backend: "backend"}
for _, n := range []string{".env", "docker-compose.yml"} {
os.WriteFile(filepath.Join(dir, n), []byte("original"), 0600)
}
calls := filepath.Join(t.TempDir(), "runs")
fakeDocker(t, `case "$*" in "inspect backend") echo '`+identityJSON(dir, "/srv/panel", "project", "/srv/panel/docker-compose.yml")+`';; run*) echo run >> '`+calls+`'; exit 1;; *) exit 1;; esac`)
j, _, e := m.Start(context.Background(), "2.5.0", []byte("target"))
if e == nil || j == nil || !j.Active {
t.Fatalf("%v %v", j, e)
}
_, _, e = m.Start(context.Background(), "2.5.0", nil)
if e == nil {
t.Fatal("retry accepted")
}
b, _ := os.ReadFile(calls)
if string(b) != "run\n" {
t.Fatal("retried uncertain docker run")
}
}
@@ -0,0 +1,73 @@
package contract_test
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"go-backend/internal/auth"
"go-backend/internal/http/response"
)
func TestSystemUpgradeStatusRequiresAdminAndReturnsLocalState(t *testing.T) {
secret := "upgrade-status-contract-secret"
dir := t.TempDir()
t.Setenv("PANEL_DEPLOY_DIR", dir)
t.Setenv("FLUX_VERSION", "3.0.1-rc10")
// A status read with no job must not require Docker or contact GitHub.
t.Setenv("PATH", t.TempDir())
router, r := setupContractRouter(t, secret)
seedContractUser(t, r, 2, "normal_user", 1, 1)
adminToken, err := auth.GenerateToken(1, "admin_user", 0, secret)
if err != nil {
t.Fatal(err)
}
userToken, err := auth.GenerateToken(2, "normal_user", 1, secret)
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name, token string
want int
}{{"anonymous", "", 401}, {"normal user", userToken, 403}, {"admin", adminToken, 0}} {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/v1/system/upgrade/status", strings.NewReader(`{}`))
req.Header.Set("Authorization", tc.token)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
var out response.R
if err := json.NewDecoder(rr.Body).Decode(&out); err != nil {
t.Fatal(err)
}
if out.Code != tc.want {
t.Fatalf("expected code %d, got %d: %s", tc.want, out.Code, out.Msg)
}
if tc.want == 0 {
data, ok := out.Data.(map[string]interface{})
if !ok || data["currentVersion"] != "3.0.1-rc10" || data["job"] != nil {
t.Fatalf("unexpected idle status: %#v", out.Data)
}
}
})
}
entries, err := os.ReadDir(dir)
if err != nil || len(entries) != 0 {
t.Fatalf("idle status should not mutate deployment: entries=%v err=%v", entries, err)
}
for _, body := range []string{`{"jobId":"../../.env"}`, `{"jobId":`, `{"jobId":123}`} {
req := httptest.NewRequest(http.MethodPost, "/api/v1/system/upgrade/status", strings.NewReader(body))
req.Header.Set("Authorization", adminToken)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
var out response.R
if err := json.NewDecoder(rr.Body).Decode(&out); err != nil {
t.Fatal(err)
}
if out.Code == 0 || out.Data != nil || strings.Contains(out.Msg, dir) || strings.Contains(out.Msg, ".env") {
t.Fatalf("invalid request must fail without leaking filesystem detail: %#v", out)
}
}
}
+139
View File
@@ -0,0 +1,139 @@
package socket
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
type agentServiceManager struct{ kind, control, runner string }
// Alpine must never select a systemctl compatibility shim. Elsewhere prefer
// an active systemd instance, matching the installer, then OpenRC.
func detectAgentServiceManager(alpine, systemd bool, lookup func(string) (string, error)) (agentServiceManager, error) {
if !alpine && systemd {
control, err := lookup("systemctl")
if err != nil {
return agentServiceManager{}, fmt.Errorf("systemctl 不可用: %w", err)
}
runner, err := lookup("systemd-run")
if err != nil {
return agentServiceManager{}, fmt.Errorf("systemd-run 不可用: %w", err)
}
return agentServiceManager{kind: "systemd", control: control, runner: runner}, nil
}
control, err := lookup("rc-service")
if err != nil {
return agentServiceManager{}, fmt.Errorf("未检测到受支持的服务管理器 (systemd/OpenRC): %w", err)
}
return agentServiceManager{kind: "openrc", control: control}, nil
}
func (w *WebSocketReporter) agentRestartLauncher(cleanLegacy bool) (func(string, string) error, error) {
// Existing per-reporter seam is also used by checksum-only tests.
if w.startUpgradeRestart != nil {
return w.startUpgradeRestart, nil
}
detect := w.detectAgentManager
if detect == nil {
detect = func() (agentServiceManager, error) {
_, alpineErr := os.Stat("/etc/alpine-release")
_, systemdErr := os.Stat("/run/systemd/system")
return detectAgentServiceManager(alpineErr == nil, systemdErr == nil, exec.LookPath)
}
}
manager, err := detect()
if err != nil {
return nil, err
}
if manager.kind == "openrc" {
if err := checkAgentDetach(); err != nil {
return nil, err
}
}
return func(staged, binary string) error {
script := buildManagedAgentRestartScript(manager, staged, binary, cleanLegacy)
if manager.kind == "systemd" {
// Wait only for submission, never for helper completion. A separate unit is
// essential: systemctl stop kills every process in the agent's cgroup.
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, manager.runner, "--quiet", "/bin/sh", "-c", script).CombinedOutput()
if err != nil {
return fmt.Errorf("提交重启任务失败: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
return startDetachedAgentHelper(script)
}, nil
}
func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" }
func buildManagedAgentRestartScript(manager agentServiceManager, staged, binary string, cleanLegacy bool) string {
control := shellQuote(manager.control)
stop, start := control+" stop flux_agent", control+" start flux_agent"
if manager.kind == "openrc" {
stop, start = control+" flux_agent stop", control+" flux_agent start"
}
legacy := ""
if manager.kind == "systemd" && cleanLegacy {
legacy = `legacy_service=''; for service_file in /etc/systemd/system/gost.service /lib/systemd/system/gost.service /usr/lib/systemd/system/gost.service; do if [ -f "$service_file" ] && grep -Fq "WorkingDirectory=/etc/gost" "$service_file" && (grep -Fq "ExecStart=/etc/gost/gost" "$service_file" || (grep -Fq "ExecStart=/usr/local/bin/gost" "$service_file" && [ -f /etc/gost/config.json ] && [ -f /etc/gost/gost.json ])); then legacy_service="$service_file"; break; fi; done; if [ -n "$legacy_service" ]; then (` + control + ` stop gost 2>/dev/null || true); (` + control + ` disable gost 2>/dev/null || true); rm -f /usr/local/bin/gost /etc/gost/gost "$legacy_service"; (` + control + ` daemon-reload 2>/dev/null || true); fi; `
}
// The caller owns staging until launch. Afterwards the helper always removes
// it, including stop/replace failures. Never truncate the running executable.
return "trap " + shellQuote("rm -f -- "+shellQuote(staged)) + " EXIT; sleep 1; " + stop + " || exit $?; " + legacy +
"if [ ! -d " + shellQuote(binary) + " ] && mv -f -- " + shellQuote(staged) + " " + shellQuote(binary) + "; then " + start + "; else status=$?; " + start + "; exit \"$status\"; fi"
}
// Copy to a unique file on the destination filesystem, then let the caller
// atomically rename. Failure never truncates a previous usable backup.
func stageAgentCopy(source, destination string) (path string, err error) {
in, err := os.Open(source)
if err != nil {
return "", err
}
defer in.Close()
info, err := in.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", fmt.Errorf("不是普通文件: %s", source)
}
out, err := os.CreateTemp(filepath.Dir(destination), ".flux-agent-*")
if err != nil {
return "", err
}
path = out.Name()
defer func() {
out.Close()
if err != nil {
os.Remove(path)
}
}()
if _, err = io.Copy(out, in); err != nil {
return path, err
}
if err = out.Chmod(0755); err != nil {
return path, err
}
if err = out.Close(); err != nil {
return path, err
}
return path, nil
}
func backupAgentBinary(binary string) error {
staged, err := stageAgentCopy(binary, binary+".old")
if err != nil {
return err
}
defer os.Remove(staged)
return os.Rename(staged, binary+".old")
}
+20
View File
@@ -0,0 +1,20 @@
package socket
import (
"os/exec"
"syscall"
)
func checkAgentDetach() error { return nil }
func startDetachedAgentHelper(script string) error {
cmd := exec.Command("/bin/sh", "-c", script)
// The installed OpenRC service stops the agent PID. A separate session also
// survives a process-group stop. No inherited stdio or parent-death signal.
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
if err := cmd.Start(); err != nil {
return err
}
go func() { _ = cmd.Wait() }()
return nil
}
@@ -0,0 +1,132 @@
package socket
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
)
// This subprocess stands in for the running agent. The rc-service double kills
// its entire process group, not merely a fake PID, while the real helper runs.
func TestAgentDetachedParent(t *testing.T) {
dir := os.Getenv("FLVX_TEST_DETACH_ROOT")
if dir == "" {
t.Skip("subprocess helper")
}
binary := filepath.Join(dir, "agent")
control := filepath.Join(dir, "rc-service")
writeAgentFixture(t, control, "#!/bin/sh\nif [ \"$2\" = stop ]; then kill -TERM -"+strconv.Itoa(os.Getpid())+"; else echo started > "+shellQuote(filepath.Join(dir, "started"))+"; fi\n")
reporter := &WebSocketReporter{upgradeBinaryPath: binary, detectAgentManager: func() (agentServiceManager, error) { return agentServiceManager{kind: "openrc", control: control}, nil }}
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
if !response.Success {
t.Fatal(response.Message)
}
time.Sleep(15 * time.Second)
t.Fatal("original agent was not stopped")
}
func TestOpenRCHelperSurvivesAgentProcessGroupStop(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, binary+".old", "backup")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestAgentDetachedParent$", "-test.v")
cmd.Env = append(os.Environ(), "FLVX_TEST_DETACH_ROOT="+dir)
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
output, err := cmd.CombinedOutput()
if err == nil || ctx.Err() != nil {
t.Fatalf("parent termination = %v, timeout=%v, %s", err, ctx.Err(), output)
}
exit, ok := err.(*exec.ExitError)
if !ok || exit.Sys().(syscall.WaitStatus).Signal() != syscall.SIGTERM {
t.Fatalf("parent did not receive SIGTERM: %v %s", err, output)
}
waitAgentFixture(t, filepath.Join(dir, "started"), "started\n")
if readAgentFixture(t, binary) != "backup" || readAgentFixture(t, binary+".old") != "backup" {
t.Fatal("rollback failed after parent termination")
}
fmt.Println("detached OpenRC helper survived original agent process-group SIGTERM")
}
// Run only in an explicitly opted-in disposable Alpine container. The docker
// invocation mounts only the test binary, never a host /bin directory. Removing
// the container's /bin/sh symlink forces the real detached cmd.Start to fail.
func TestOpenRCCommandsCleanStageOnHelperExecFailure(t *testing.T) {
if os.Getenv("FLVX_TEST_DISPOSABLE_ALPINE_SHELL_FAILURE") != "1" {
t.Skip("requires explicit disposable Alpine container opt-in")
}
for _, path := range []string{"/etc/alpine-release", "/.dockerenv"} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("refusing shell mutation outside disposable Alpine: %s: %v", path, err)
}
}
info, err := os.Lstat("/bin/sh")
if err != nil || info.Mode()&os.ModeSymlink == 0 {
t.Fatalf("refusing to rename a non-symlink /bin/sh: %v", err)
}
saved := "/bin/.flvx-test-sh-" + strconv.Itoa(os.Getpid())
if _, err := os.Lstat(saved); !os.IsNotExist(err) {
t.Fatalf("refusing to overwrite shell backup: %v", err)
}
if err := os.Rename("/bin/sh", saved); err != nil {
t.Fatal(err)
}
defer func() {
if err := os.Rename(saved, "/bin/sh"); err != nil {
t.Errorf("restore container shell: %v", err)
}
}()
for _, operation := range []string{"UpgradeAgent", "RollbackAgent"} {
t.Run(operation, func(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, binary+".old", "prior backup")
writeAgentFixture(t, filepath.Join(dir, "rc-service"), "#!/bin/sh\nexit 0\n")
t.Setenv("PATH", dir)
// No manager or launcher injection: Alpine detection and exec.LookPath are
// production code too. Only the installation path points to an isolated dir.
reporter := &WebSocketReporter{upgradeBinaryPath: binary}
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/checksum" {
fmt.Fprint(w, abcSHA256)
} else {
fmt.Fprint(w, "abc")
}
}))
defer fixture.Close()
response := routeAgentOperation(t, reporter, operation, map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
if response.Success || !strings.Contains(response.Message, "/bin/sh") || !strings.Contains(response.Message, "no such file") {
t.Fatalf("expected real helper exec failure, got %+v", response)
}
if readAgentFixture(t, binary) != "current" {
t.Fatal("live binary changed")
}
backup := "prior backup"
if operation == "UpgradeAgent" {
backup = "current"
}
if readAgentFixture(t, binary+".old") != backup {
t.Fatal("unexpected backup contents")
}
matches, err := filepath.Glob(filepath.Join(dir, ".flux-agent-*"))
if err != nil || len(matches) != 0 {
t.Fatalf("rollback staging residue: %v %v", matches, err)
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Fatalf("upgrade staging residue: %v", err)
}
})
}
}
+8
View File
@@ -0,0 +1,8 @@
//go:build !linux
package socket
import "fmt"
func checkAgentDetach() error { return fmt.Errorf("OpenRC Agent 重启仅支持 Linux") }
func startDetachedAgentHelper(string) error { return checkAgentDetach() }
+329
View File
@@ -0,0 +1,329 @@
package socket
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
)
func routeAgentOperation(t *testing.T, reporter *WebSocketReporter, operation string, data interface{}) CommandResponse {
t.Helper()
peer := make(chan *websocket.Conn, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
if err == nil {
peer <- c
}
}))
defer server.Close()
c, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer c.Close()
reader := <-peer
defer reader.Close()
reporter.ctx = context.Background()
reporter.conn = c
reporter.connected = true
reporter.routeCommand(CommandMessage{Type: operation, RequestId: "restart-request", Data: data})
reader.SetReadDeadline(time.Now().Add(3 * time.Second))
for {
var response CommandResponse
if err := reader.ReadJSON(&response); err != nil {
t.Fatal(err)
}
if response.Type == operation+"Response" {
if response.RequestId != "restart-request" {
t.Fatalf("request id lost: %s", response.RequestId)
}
return response
}
}
}
func TestRollbackAgentStagesBackupBeforeRestart(t *testing.T) {
binary := filepath.Join(t.TempDir(), "flux_agent")
for path, body := range map[string]string{binary: "current", binary + ".old": "backup"} {
if err := os.WriteFile(path, []byte(body), 0755); err != nil {
t.Fatal(err)
}
}
called := false
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(staged, dest string) error {
called = true
body, err := os.ReadFile(staged)
if err != nil || string(body) != "backup" || staged == binary+".old" || dest != binary {
t.Fatalf("bad rollback staging: %s %s %q %v", staged, dest, body, err)
}
return os.Rename(staged, dest)
}}
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
if !response.Success || !called {
t.Fatalf("rollback failed: called=%v %s", called, response.Message)
}
body, err := os.ReadFile(binary + ".old")
if err != nil || string(body) != "backup" {
t.Fatalf("backup consumed: %q %v", body, err)
}
}
func TestAgentServiceManagerDetection(t *testing.T) {
for _, tc := range []struct {
name string
alpine, systemd bool
tools string
want string
}{
{"alpine shim", true, true, "rc-service systemctl systemd-run", "openrc"},
{"alpine missing openrc", true, true, "systemctl systemd-run", ""},
{"systemd", false, true, "rc-service systemctl systemd-run", "systemd"},
{"missing runner", false, true, "systemctl", ""},
{"openrc fallback", false, false, "rc-service", "openrc"},
{"unmanaged", false, false, "systemctl systemd-run", ""},
} {
t.Run(tc.name, func(t *testing.T) {
manager, err := detectAgentServiceManager(tc.alpine, tc.systemd, func(name string) (string, error) {
if strings.Contains(" "+tc.tools+" ", " "+name+" ") {
return name, nil
}
return "", os.ErrNotExist
})
if manager.kind != tc.want || (err != nil) != (tc.want == "") {
t.Fatalf("manager=%+v err=%v", manager, err)
}
})
}
}
func writeAgentFixture(t *testing.T, path, body string) {
t.Helper()
if err := os.WriteFile(path, []byte(body), 0755); err != nil {
t.Fatal(err)
}
}
func readAgentFixture(t *testing.T, path string) string {
t.Helper()
body, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(body)
}
func waitAgentFixture(t *testing.T, path, want string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
body, _ := os.ReadFile(path)
if string(body) == want {
return
}
time.Sleep(20 * time.Millisecond)
}
body, _ := os.ReadFile(path)
t.Fatalf("%s = %q, want %q", path, body, want)
}
func TestAgentCommandsManagerLaunch(t *testing.T) {
for _, operation := range []string{"UpgradeAgent", "RollbackAgent"} {
for _, mode := range []string{"systemd", "openrc", "missing", "rejected"} {
t.Run(operation+"/"+mode, func(t *testing.T) {
if mode == "openrc" && runtime.GOOS != "linux" {
t.Skip("detached launcher is Linux-only")
}
dir := t.TempDir()
binary := filepath.Join(dir, "flux 'agent")
log := filepath.Join(dir, "events")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, binary+".old", "backup")
control := filepath.Join(dir, "manager")
writeAgentFixture(t, control, "#!/bin/sh\nprintf '%s\\n' \"$*\" >> "+shellQuote(log)+"\n")
runner := filepath.Join(dir, "runner")
writeAgentFixture(t, runner, "#!/bin/sh\nshift\n\"$@\" >/dev/null 2>&1 &\n")
if mode == "rejected" {
writeAgentFixture(t, runner, "#!/bin/sh\necho rejected >&2\nexit 9\n")
}
reporter := &WebSocketReporter{upgradeBinaryPath: binary, detectAgentManager: func() (agentServiceManager, error) {
return detectAgentServiceManager(mode == "openrc", mode != "missing", func(name string) (string, error) {
if mode == "missing" {
return "", os.ErrNotExist
}
if name == "systemd-run" {
return runner, nil
}
return control, nil
})
}}
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/checksum" {
fmt.Fprint(w, abcSHA256)
} else {
fmt.Fprint(w, "abc")
}
}))
defer fixture.Close()
response := routeAgentOperation(t, reporter, operation, map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
if mode == "missing" || mode == "rejected" {
if response.Success {
t.Fatal("accepted failed manager launch")
}
if readAgentFixture(t, binary) != "current" {
t.Fatal("live binary mutated")
}
if mode == "missing" && readAgentFixture(t, binary+".old") != "backup" {
t.Fatal("backup mutated before manager validation")
}
matches, _ := filepath.Glob(filepath.Join(dir, ".flux-agent-*"))
if len(matches) != 0 {
t.Fatalf("staging residue %v", matches)
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Fatal("upgrade residue")
}
return
}
if !response.Success {
t.Fatal(response.Message)
}
want, backup := "abc", "current"
if operation == "RollbackAgent" {
want, backup = "backup", "backup"
}
waitAgentFixture(t, binary, want)
events := "stop flux_agent\nstart flux_agent\n"
if mode == "openrc" {
events = "flux_agent stop\nflux_agent start\n"
}
waitAgentFixture(t, log, events)
if readAgentFixture(t, binary+".old") != backup {
t.Fatal("wrong backup")
}
})
}
}
}
func TestAgentRestartScriptFailureOrdering(t *testing.T) {
for _, mode := range []string{"stop", "replace"} {
t.Run(mode, func(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
staged := filepath.Join(dir, "staged")
log := filepath.Join(dir, "events")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, staged, "replacement")
control := filepath.Join(dir, "rc-service")
script := "#!/bin/sh\necho \"$*\" >> " + shellQuote(log) + "\n"
if mode == "stop" {
script += "[ \"$2\" != stop ]\n"
}
writeAgentFixture(t, control, script)
if mode == "replace" {
writeAgentFixture(t, filepath.Join(dir, "mv"), "#!/bin/sh\nexit 23\n")
t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
}
cmd := exec.Command("/bin/sh", "-c", buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: control}, staged, binary, true))
if err := cmd.Run(); err == nil {
t.Fatal("expected failed stop/replacement")
}
if readAgentFixture(t, binary) != "current" {
t.Fatal("changed binary after failure")
}
want := "flux_agent stop\n"
if mode == "replace" {
want += "flux_agent start\n"
}
if readAgentFixture(t, log) != want {
t.Fatalf("wrong sequence %q", readAgentFixture(t, log))
}
if _, err := os.Stat(staged); !os.IsNotExist(err) {
t.Fatal("staging residue")
}
})
}
}
func TestUpgradeAgentBackupFailurePreservesExistingBackup(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
writeAgentFixture(t, binary+".old", "prior")
called := false
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(string, string) error { called = true; return nil }}
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/checksum" {
fmt.Fprint(w, abcSHA256)
} else {
fmt.Fprint(w, "abc")
}
}))
defer fixture.Close()
response := routeAgentOperation(t, reporter, "UpgradeAgent", map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
if response.Success || called {
t.Fatal("upgrade accepted without backup")
}
if readAgentFixture(t, binary+".old") != "prior" {
t.Fatal("backup lost")
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Fatal("staging residue")
}
}
func TestRollbackAgentRejectsDirectoryDestination(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
if err := os.Mkdir(binary, 0755); err != nil {
t.Fatal(err)
}
writeAgentFixture(t, binary+".old", "backup")
called := false
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(string, string) error { called = true; return nil }}
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
if response.Success || called {
t.Fatal("accepted directory destination")
}
if readAgentFixture(t, binary+".old") != "backup" {
t.Fatal("changed backup")
}
entries, err := os.ReadDir(binary)
if err != nil || len(entries) != 0 {
t.Fatalf("destination changed: %v %v", entries, err)
}
}
func TestAgentRestartScriptRejectsDirectoryDestination(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
staged := filepath.Join(dir, "stage")
if err := os.Mkdir(binary, 0755); err != nil {
t.Fatal(err)
}
writeAgentFixture(t, staged, "replacement")
script := buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: "/usr/bin/true"}, staged, binary, false)
if err := exec.Command("/bin/sh", "-c", script).Run(); err == nil {
t.Fatal("accepted directory destination")
}
entries, err := os.ReadDir(binary)
if err != nil || len(entries) != 0 {
t.Fatalf("binary moved inside directory: %v %v", entries, err)
}
}
func TestOpenRCRestartNeverCleansSystemdLegacyService(t *testing.T) {
script := buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: "rc-service"}, "staged", "binary", true)
for _, forbidden := range []string{"systemctl", "gost", "daemon-reload", "/etc/systemd"} {
if strings.Contains(script, forbidden) {
t.Fatalf("OpenRC helper contains %s: %s", forbidden, script)
}
}
}
+198
View File
@@ -0,0 +1,198 @@
package socket
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
)
// All writes are confined to TempDir; the restart callback never starts a process.
func exerciseUpgrade(t *testing.T, checksum string, serveChecksum http.HandlerFunc, valid bool) {
t.Helper()
dir := t.TempDir()
binary := filepath.Join(dir, "flux_agent")
for path, body := range map[string]string{binary: "old binary", binary + ".old": "prior backup"} {
if err := os.WriteFile(path, []byte(body), 0700); err != nil {
t.Fatal(err)
}
}
downloads := 0
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/binary-"+runtime.GOARCH {
downloads++
fmt.Fprint(w, "abc")
return
}
if r.URL.Path != "/checksum-"+runtime.GOARCH {
t.Errorf("unexpected checksum path: %s", r.URL.Path)
http.NotFound(w, r)
return
}
serveChecksum(w, r)
}))
defer fixture.Close()
if checksum == "fixture" {
checksum = fixture.URL + "/checksum-{ARCH}"
}
peer := make(chan *websocket.Conn, 1)
ws := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
if err != nil {
return
}
peer <- c
}))
defer ws.Close()
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(ws.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
reader := <-peer
defer reader.Close()
restarted := false
reporter := &WebSocketReporter{ctx: context.Background(), conn: conn, connected: true, upgradeBinaryPath: binary,
startUpgradeRestart: func(staged, destination string) error {
restarted = true
if destination != binary {
t.Errorf("wrong restart destination: %s", destination)
}
b, e := os.ReadFile(staged)
if e != nil || string(b) != "abc" {
t.Errorf("staged binary = %q, %v", b, e)
}
return nil
},
}
reporter.routeCommand(CommandMessage{Type: "UpgradeAgent", RequestId: "upgrade-test-request", Data: map[string]any{"downloadUrl": fixture.URL + "/binary-{ARCH}", "checksumUrl": checksum}})
reader.SetReadDeadline(time.Now().Add(2 * time.Second))
verified, installing := false, false
for {
var response CommandResponse
if err := reader.ReadJSON(&response); err != nil {
t.Fatal(err)
}
if response.Type == "UpgradeAgentResponse" {
if response.RequestId != "upgrade-test-request" {
t.Errorf("response request ID = %q", response.RequestId)
}
if response.Success != valid {
t.Errorf("success = %v, want %v: %s", response.Success, valid, response.Message)
}
break
}
if response.Type == "UpgradeProgress" {
data := response.Data.(map[string]interface{})
if data["stage"] == "verifying" && data["percent"] == float64(100) {
verified = true
}
if data["stage"] == "installing" {
installing = true
}
}
}
if restarted != valid || verified != valid || installing != valid {
t.Errorf("restart=%v verified=%v installing=%v, want %v", restarted, verified, installing, valid)
}
b, e := os.ReadFile(binary)
if e != nil || string(b) != "old binary" {
t.Errorf("old binary changed: %q %v", b, e)
}
if !valid {
b, e = os.ReadFile(binary + ".old")
if e != nil || string(b) != "prior backup" {
t.Errorf("backup changed: %q %v", b, e)
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Errorf("staged binary remains: %v", err)
}
}
if strings.TrimSpace(checksum) == "" && downloads != 0 {
t.Errorf("downloaded without checksum URL")
}
}
func TestUpgradeAgentRequiresChecksumURL(t *testing.T) {
exerciseUpgrade(t, "", func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected checksum request") }, false)
}
func TestUpgradeAgentRejectsChecksumHTTPFailure(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "unavailable", http.StatusNotFound) }, false)
}
const abcSHA256 = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
func TestUpgradeAgentRejectsAmbiguousChecksum(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, abcSHA256+" gost-amd64\n"+abcSHA256+" gost-arm64\n")
}, false)
}
func TestUpgradeAgentRejectsOversizedChecksum(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, abcSHA256+" "+strings.Repeat("a", 4096)) }, false)
}
func TestUpgradeAgentChecksumRecords(t *testing.T) {
for _, tc := range []struct {
name, body string
valid bool
}{
{"bare", abcSHA256, true},
{"GNU text", abcSHA256 + " gost-arm64\n", true},
{"GNU binary", abcSHA256 + " *gost-arm64\n", true},
{"uppercase CRLF surrounding whitespace", " \t" + strings.ToUpper(abcSHA256) + " gost-arm64\r\n", true},
{"bare surrounding whitespace", "\r\n\t" + abcSHA256 + " \r\n", true},
{"empty", "", false},
{"whitespace", " \r\n\t", false},
{"truncated", abcSHA256[:63], false},
{"nonhex", "z" + abcSHA256[1:], false},
{"mismatch", strings.Repeat("0", 64), false},
{"missing GNU marker", abcSHA256 + " gost-arm64", false},
{"invalid GNU marker", abcSHA256 + " ?gost-arm64", false},
{"missing filename", abcSHA256 + " *", false},
{"appended garbage", abcSHA256 + "garbage", false},
{"embedded CR", abcSHA256 + " gost\rarm64", false},
{"NUL", abcSHA256 + " gost\x00arm64", false},
{"two bare digests", abcSHA256 + "\n" + abcSHA256, false},
} {
t.Run(tc.name, func(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, tc.body) }, tc.valid)
})
}
}
func TestUpgradeAgentRejectsChecksumTransportFailure(t *testing.T) {
exerciseUpgrade(t, "://invalid", nil, false)
}
func TestUpgradeAgentRejectsChecksumReadFailure(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
// A valid-looking record is incomplete relative to the declared body length.
w.Header().Set("Content-Length", "999")
fmt.Fprint(w, abcSHA256)
}, false)
}
func TestUpgradeAgentRejectsBlankChecksumURL(t *testing.T) {
exerciseUpgrade(t, " \t\r\n", nil, false)
}
func TestUpgradeAgentRejectsChecksumConnectionFailure(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
conn, _, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
conn.Close()
}, false)
}
+95 -68
View File
@@ -14,7 +14,6 @@ import (
"net/http"
"net/url"
"os"
"os/exec"
"runtime"
"runtime/debug"
"strconv"
@@ -158,6 +157,11 @@ const (
)
type WebSocketReporter struct {
// Per-reporter upgrade dependencies keep staging and process launch isolated.
upgradeBinaryPath string
startUpgradeRestart func(string, string) error
detectAgentManager func() (agentServiceManager, error)
url string
addr string // 保存服务器地址
secret string // 保存密钥
@@ -1377,6 +1381,23 @@ func (w *WebSocketReporter) sendUpgradeProgress(stage string, percent int, messa
w.sendResponse(response)
}
// parseUpgradeChecksum accepts one bare SHA256 or one GNU sha256sum record.
// A manifest cannot safely identify the requested binary by its first record.
func parseUpgradeChecksum(body []byte) (string, error) {
record := strings.TrimSpace(string(body))
if len(record) < 64 || strings.ContainsAny(record, "\r\n\x00") {
return "", fmt.Errorf("校验文件格式无效: 需要单条 SHA256 记录")
}
digest := record[:64]
if _, err := hex.DecodeString(digest); err != nil {
return "", fmt.Errorf("校验文件格式无效: SHA256 必须为 64 位十六进制")
}
if len(record) != 64 && (len(record) < 67 || record[64] != ' ' || (record[65] != ' ' && record[65] != '*') || strings.TrimSpace(record[66:]) == "") {
return "", fmt.Errorf("校验文件格式无效: 需要单条 GNU SHA256 记录")
}
return digest, nil
}
func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
jsonData, err := json.Marshal(data)
if err != nil {
@@ -1394,6 +1415,10 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
return fmt.Errorf("下载地址不能为空")
}
if strings.TrimSpace(req.ChecksumURL) == "" {
return fmt.Errorf("校验文件地址不能为空")
}
// 替换架构占位符
downloadURL := strings.ReplaceAll(req.DownloadURL, "{ARCH}", runtime.GOARCH)
checksumURL := strings.ReplaceAll(req.ChecksumURL, "{ARCH}", runtime.GOARCH)
@@ -1402,9 +1427,11 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
fmt.Printf("📦 开始下载升级包: %s\n", downloadURL)
// 下载新版本二进制
const binaryPath = "/etc/flux_agent/flux_agent"
binaryPath := w.upgradeBinaryPath
if binaryPath == "" {
binaryPath = "/etc/flux_agent/flux_agent"
}
tmpPath := binaryPath + ".new"
backupPath := binaryPath + ".old"
resp, err := http.Get(downloadURL)
if err != nil {
@@ -1421,6 +1448,14 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
return fmt.Errorf("创建临时文件失败: %v", err)
}
// The restart process owns the staged file only after launch succeeds.
restartStarted := false
defer func() {
if !restartStarted {
_ = os.Remove(tmpPath)
}
}()
// 带进度的下载
totalSize := resp.ContentLength
var downloaded int64
@@ -1433,7 +1468,6 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
if n > 0 {
if _, wErr := outFile.Write(buf[:n]); wErr != nil {
outFile.Close()
os.Remove(tmpPath)
return fmt.Errorf("写入升级包失败: %v", wErr)
}
hasher.Write(buf[:n])
@@ -1451,104 +1485,97 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
break
}
outFile.Close()
os.Remove(tmpPath)
return fmt.Errorf("读取升级包失败: %v", readErr)
}
}
outFile.Close()
if err := outFile.Close(); err != nil {
return fmt.Errorf("关闭升级包失败: %w", err)
}
if downloaded == 0 {
os.Remove(tmpPath)
return fmt.Errorf("下载的升级包为空")
}
w.sendUpgradeProgress("downloading", 100, fmt.Sprintf("下载完成 (%d bytes)", downloaded))
// Checksum 校验
if checksumURL != "" {
w.sendUpgradeProgress("verifying", 0, "校验文件完整性...")
checksumResp, err := http.Get(checksumURL)
if err == nil {
defer checksumResp.Body.Close()
if checksumResp.StatusCode == http.StatusOK {
checksumBody, err := io.ReadAll(checksumResp.Body)
if err == nil {
// 格式: "<hash> <filename>" 或 "<hash>"
expectedHash := strings.TrimSpace(strings.Split(string(checksumBody), " ")[0])
actualHash := hex.EncodeToString(hasher.Sum(nil))
if !strings.EqualFold(expectedHash, actualHash) {
os.Remove(tmpPath)
return fmt.Errorf("校验失败: 期望 %s, 实际 %s", expectedHash, actualHash)
}
fmt.Printf("✅ Checksum 校验通过: %s\n", actualHash)
}
}
}
w.sendUpgradeProgress("verifying", 100, "校验通过")
// Every checksum failure must abort before installation.
w.sendUpgradeProgress("verifying", 0, "校验文件完整性...")
checksumResp, err := http.Get(checksumURL)
if err != nil {
return fmt.Errorf("下载校验文件失败: %w", err)
}
defer checksumResp.Body.Close()
if checksumResp.StatusCode != http.StatusOK {
return fmt.Errorf("下载校验文件失败, HTTP状态码: %d", checksumResp.StatusCode)
}
const maxChecksumBytes = 4096
checksumBody, err := io.ReadAll(io.LimitReader(checksumResp.Body, maxChecksumBytes+1))
if err != nil {
return fmt.Errorf("读取校验文件失败: %w", err)
}
if len(checksumBody) > maxChecksumBytes {
return fmt.Errorf("校验文件过大: 最大 %d bytes", maxChecksumBytes)
}
expectedHash, err := parseUpgradeChecksum(checksumBody)
if err != nil {
return err
}
actualHash := hex.EncodeToString(hasher.Sum(nil))
if !strings.EqualFold(expectedHash, actualHash) {
return fmt.Errorf("校验失败: 期望 %s, 实际 %s", expectedHash, actualHash)
}
w.sendUpgradeProgress("verifying", 100, "校验通过")
if err := os.Chmod(tmpPath, 0755); err != nil {
os.Remove(tmpPath)
return fmt.Errorf("设置执行权限失败: %v", err)
}
// 备份旧版本
w.sendUpgradeProgress("installing", 50, "备份旧版本...")
if _, err := os.Stat(binaryPath); err == nil {
// 复制旧文件作为备份(不用 rename,因为可能正在运行)
oldData, err := os.ReadFile(binaryPath)
if err == nil {
_ = os.WriteFile(backupPath, oldData, 0755)
fmt.Println("📦 旧版本已备份到", backupPath)
}
// Resolve the manager before mutating an existing rollback backup.
startRestart, err := w.agentRestartLauncher(true)
if err != nil {
return err
}
w.sendUpgradeProgress("installing", 50, "备份旧版本...")
if err := backupAgentBinary(binaryPath); err != nil {
return fmt.Errorf("备份旧版本失败: %w", err)
}
w.sendUpgradeProgress("installing", 80, "准备重启...")
fmt.Printf("✅ 升级包下载完成 (%d bytes), 准备重启...\n", downloaded)
// 执行重启脚本
// 使用 systemd-run 在独立的 transient unit 中运行重启脚本,
// 避免 systemctl stop 杀死 flux_agent cgroup 内所有进程(包括此脚本自身)导致 mv 未执行。
script := buildAgentRestartScript(tmpPath, binaryPath)
cmd := exec.Command("systemd-run", "--quiet", "/bin/sh", "-c", script)
if err := cmd.Start(); err != nil {
os.Remove(tmpPath)
if err := startRestart(tmpPath, binaryPath); err != nil {
return fmt.Errorf("启动重启脚本失败: %v", err)
}
restartStarted = true
w.sendUpgradeProgress("installing", 100, "重启中...")
fmt.Println("🔄 重启脚本已启动, Agent 将在 1 秒后重启...")
return nil
}
func (w *WebSocketReporter) handleRollbackAgent(data interface{}) error {
const binaryPath = "/etc/flux_agent/flux_agent"
backupPath := binaryPath + ".old"
// 检查备份文件是否存在
if _, err := os.Stat(backupPath); os.IsNotExist(err) {
return fmt.Errorf("没有可用的备份文件,无法回退")
binaryPath := w.upgradeBinaryPath
if binaryPath == "" {
binaryPath = "/etc/flux_agent/flux_agent"
}
fmt.Println("🔄 开始回退到旧版本...")
// 执行回退脚本(同升级逻辑,使用 systemd-run 避免 cgroup 问题)
script := fmt.Sprintf("sleep 1 && systemctl stop flux_agent && cp %s %s && systemctl start flux_agent", backupPath, binaryPath)
cmd := exec.Command("systemd-run", "--quiet", "/bin/sh", "-c", script)
if err := cmd.Start(); err != nil {
return fmt.Errorf("启动回退脚本失败: %v", err)
startRestart, err := w.agentRestartLauncher(false)
if err != nil {
return err
}
if info, err := os.Stat(binaryPath); err != nil || !info.Mode().IsRegular() {
return fmt.Errorf("回退目标不是可用的普通文件: %s", binaryPath)
}
staged, err := stageAgentCopy(binaryPath+".old", binaryPath)
if err != nil {
return fmt.Errorf("准备回退备份失败: %w", err)
}
if err := startRestart(staged, binaryPath); err != nil {
_ = os.Remove(staged)
return fmt.Errorf("启动回退脚本失败: %w", err)
}
fmt.Println("🔄 回退脚本已启动, Agent 将在 1 秒后重启...")
return nil
}
func buildAgentRestartScript(tmpPath, binaryPath string) string {
return fmt.Sprintf(
"sleep 1 && systemctl stop flux_agent && legacy_service='' && for service_file in /etc/systemd/system/gost.service /lib/systemd/system/gost.service /usr/lib/systemd/system/gost.service; do if [ -f \"$service_file\" ] && grep -Fq \"WorkingDirectory=/etc/gost\" \"$service_file\" && (grep -Fq \"ExecStart=/etc/gost/gost\" \"$service_file\" || (grep -Fq \"ExecStart=/usr/local/bin/gost\" \"$service_file\" && [ -f /etc/gost/config.json ] && [ -f /etc/gost/gost.json ])); then legacy_service=\"$service_file\"; break; fi; done && if [ -n \"$legacy_service\" ]; then (systemctl stop gost 2>/dev/null || true) && (systemctl disable gost 2>/dev/null || true) && rm -f /usr/local/bin/gost /etc/gost/gost \"$legacy_service\" && (systemctl daemon-reload 2>/dev/null || true); fi && mv %s %s && systemctl start flux_agent",
tmpPath,
binaryPath,
)
return buildManagedAgentRestartScript(agentServiceManager{kind: "systemd", control: "systemctl"}, tmpPath, binaryPath, true)
}
// updateLocalConfigJSON 将 http/tls/socks 写入工作目录下的 config.json
+5 -5
View File
@@ -186,16 +186,16 @@ func TestFormatWebSocketDialErrorIncludesHTTPStatus(t *testing.T) {
func TestAgentUpgradeRestartScriptStopsLegacyGostService(t *testing.T) {
script := buildAgentRestartScript("/tmp/flux_agent.new", "/etc/flux_agent/flux_agent")
if !strings.Contains(script, "systemctl stop flux_agent") {
if !strings.Contains(script, "'systemctl' stop flux_agent") {
t.Fatalf("expected script to stop flux_agent, got %s", script)
}
if !strings.Contains(script, "mv /tmp/flux_agent.new /etc/flux_agent/flux_agent") {
if !strings.Contains(script, "mv -f -- '/tmp/flux_agent.new' '/etc/flux_agent/flux_agent'") {
t.Fatalf("expected script to replace the flux_agent binary, got %s", script)
}
if !strings.Contains(script, "systemctl stop gost") {
if !strings.Contains(script, "'systemctl' stop gost") {
t.Fatalf("expected script to stop the legacy gost service, got %s", script)
}
if !strings.Contains(script, "systemctl disable gost") {
if !strings.Contains(script, "'systemctl' disable gost") {
t.Fatalf("expected script to disable the legacy gost service, got %s", script)
}
if !strings.Contains(script, "rm -f /usr/local/bin/gost") {
@@ -204,7 +204,7 @@ func TestAgentUpgradeRestartScriptStopsLegacyGostService(t *testing.T) {
if !strings.Contains(script, "WorkingDirectory=/etc/gost") {
t.Fatalf("expected script to scope cleanup to the legacy FLVX gost service definition, got %s", script)
}
if !strings.Contains(script, "systemctl start flux_agent") {
if !strings.Contains(script, "'systemctl' start flux_agent") {
t.Fatalf("expected script to restart flux_agent, got %s", script)
}
if strings.Contains(script, "systemctl stop flux_agent && systemctl stop gost 2>/dev/null || true") {
@@ -0,0 +1,27 @@
# Agent upgrade checksum fail-closed
## Scope
Fix the panel-issued UpgradeAgent flow only. Do not change Alpine support, installer scripts, release artifacts, production state, or publish changes.
## Observed defect
`handleUpgradeAgent` treats checksum fetch errors, non-200 responses and body read errors as success and emits `校验通过`. An absent checksum URL bypasses verification entirely. The panel already supplies a checksum URL for individual and batch upgrades, and the release workflow generates GNU sha256sum files.
## Intended behavior
- Require a nonblank checksum URL before downloading an upgrade.
- Abort before chmod, backup, or restart on fetch/status/read/format/hash failure; remove the staged binary.
- Only emit verification success after a complete, valid SHA256 record matches the downloaded bytes.
- Accept one bare 64-hex SHA256 or a single GNU sha256sum record (text or binary marker), case-insensitive hex and surrounding whitespace/CRLF. Reject empty, truncated, nonhex, multiline and oversized payloads. Do not treat an ambiguous multi-file manifest as a checksum.
- Preserve architecture substitution and current installation flow. This is integrity checking, not a signature or an independent guarantee against a compromised download source.
## Verification plan
User-confirmed seam: actual UpgradeAgent command handling, loopback HTTP fixtures, isolated binary paths and an intercepted restart action. Assert error/response and progress, untouched old executable/backup, absence of staged bytes and no restart on every invalid checksum condition. Valid records should reach the intercepted restart with exact downloaded bytes.
Use red-green slices, then race tests, module checks, build and independent scope review. No push, PR, merge, tag or deployment is authorized for this task.
## Review scope
User confirmed review against rc8 commit `fadc2b6c122df99140e0c9cbd481a1214d17d175`, covering all changes for this task and this plan. Run independent Standards and Spec reviews after the implementation is ready.
@@ -0,0 +1,29 @@
# Agent panel upgrade and rollback on OpenRC
## Scope
Repair the agent UpgradeAgent/RollbackAgent paths on Alpine/OpenRC without regressing systemd. Preserve the pending checksum fail-closed repair. Do not change live nodes, install components, publish or deploy.
## Evidence
In a disposable cached Alpine 3.22.5 linux/amd64 container, the actual handlers fail launching `systemd-run` after successful download and verification, and also when rolling back an existing backup. Installer scripts already support OpenRC, but their CI does not exercise these command handlers.
## Safety requirements
- Prefer OpenRC on Alpine, including when a systemctl shim exists; refuse missing required manager tools.
- Keep systemd's independent transient unit behavior to survive stopping the agent cgroup.
- Launch an independent OpenRC restart helper that survives termination of the original agent, without installing extra utilities.
- Use the selected manager for stop/start; do not run systemd legacy-service cleanup on OpenRC.
- Retain validated upgrade staging and backup behavior, and do not consume the rollback backup.
- Reject setup/launch errors before claiming restart was launched. Report asynchronous-completion limits honestly.
- Preserve existing command response, request ID and progress contracts.
## Proposed tests and review
User-confirmed boundaries: real UpgradeAgent/RollbackAgent command routing, isolated files, systemd/OpenRC tools as test doubles, no-manager rejection and launch/stop/replacement failures. Use cached Alpine to execute the helper with rc-service doubles, not a real OpenRC installation. Explicitly distinguish this from service-manager integration testing.
User-confirmed review baseline is fadc2b6, covering both pending fixes and plans. No commits, push, PR, merge, release or deployment.
## Publication authorization
After the local implementation, verification and independent reviews were complete, the user authorized submitting a PR, merging if checks pass, and publishing the next tag. Earlier no-publication limits above describe the implementation phase. Deployment to live nodes remains outside scope.
+59
View File
@@ -0,0 +1,59 @@
# Panel self-upgrade reliability plan
## Scope
Fix the panel's Docker Compose self-upgrade flow. The report is from a third-party deployment; this work uses repository analysis and isolated tests, not access to that deployment. Base: `a784350` (`3.0.1-rc10`). No publishing or deployment is included.
## Confirmed problems
- The UI stops at helper launch. It cannot distinguish a running, successful, or failed upgrade.
- The helper resolves the Compose project and relative bind mounts from `/opt/flvx-panel`, which need not match the original host deployment.
- The HTTP mutex ends before the helper. Repeated requests can change the same configuration and overwrite logs and fixed backups.
- Active configuration is replaced before images are pulled; helper failures do not restore it.
- Successful `compose up` is not an explicit verification of both target images and frontend availability.
These are code defects and failure risks, not proof of the reported deployment's particular failure.
## Implementation plan
1. Identify and validate the existing Compose project, host deployment mount, backend service, and configuration paths before a task can mutate deployment files. Fail safely on unsupported or ambiguous deployment layouts.
2. Add a durable task with a unique ID, per-task log and immutable backups. Keep the task lock across the HTTP request, helper execution, and backend replacement. Reconcile stopped helpers conservatively; uncertainty must not permit overlapping upgrades.
3. Run a dedicated helper command in the current backend image. Validate a staged Compose configuration and pull images before replacing active files. Use atomic file replacement. Restore configuration only when failure is known to precede service replacement. A partial or uncertain service replacement requires explicit recovery rather than an unsafe automatic database/container rollback.
4. Record progress and terminal results locally, with a bounded execution deadline. Verify target services before success. Expose only safe progress messages through an admin status endpoint, never `.env` or arbitrary command output.
5. Add frontend progress, reconnect handling, stale-response protection, reload recovery, and repeat-click prevention. A launch acknowledgment is not completion.
6. Run focused fault-injection tests, the full backend checks, frontend lint/type/build checks, and UI lifecycle exercises. Document real Docker and PostgreSQL tests that cannot run locally.
## API contract
- Existing `POST /api/v1/system/upgrade`: preserve existing response fields and add `jobId`.
- New `POST /api/v1/system/upgrade/status`: accept optional `jobId`; return `currentVersion` and nullable `job`.
- A job includes `id`, `targetVersion`, `stage`, `active`, `requiresRecovery`, `message`, `startedAt`, `updatedAt`, and optional helper/backup references.
- Stages: `preparing`, `pulling`, `applying`, `restarting`, `verifying`, `succeeded`, `failed`, `recovery_required`.
- Status reads do not call GitHub. Timestamps use Unix milliseconds. A recovery-required task must not silently permit another upgrade.
## Required regression coverage
- Original project names and host paths that differ from the in-container path; invalid/missing labels and mounts; IPv4/IPv6 templates.
- Validation and pull failure without changing active config; application failure with safe restoration; partial recreation and verification failures with preserved recovery evidence.
- Duplicate requests, interrupted starts, stopped helpers, backend restart, task timeout, and preservation of older backups and logs.
- Frontend reload, transient disconnect, success, failure, recovery required, repeated clicks, unmount, and stale responses.
- No sensitive environment contents in status responses or browser-visible error messages.
## Rollout boundary
The launcher and helper are supplied by the version currently running, not by the target Compose download. Existing rc10 installations do not acquire this fix until the corrected backend image is actually installed. The first transition to the fixed release should use the established host-side upgrade process with the original deployment directory and Compose project. Do not recommend repeated retries of the old web updater or a blind rollback after partial service replacement.
## Verification record
Implementation completed in `fix/panel-upgrade-jobs`, based on `a784350`.
- Backend: full `go test ./...`, `go test -race ./internal/panelupgrade`, `go vet ./...`, native build, and CGO-disabled Linux amd64/arm64 builds passed with Go 1.26.10.
- Added an admin/non-admin/anonymous status contract, idle read with no Docker dependency, malformed job-ID rejection, and focused upgrade tests in CI.
- Fake-Docker tests cover configuration/pull/recreate/verification failures, partial configuration restoration, immutable task backups, uncertain launch, cross-manager blocking, original project/path, full service-setting preservation, unexpected database mounts, and safe public messages.
- Frontend: changed-file ESLint (zero warnings), TypeScript and production build passed with Node 24.22.1. Nine temporary lifecycle replays passed, including two-admin refresh, repeated submissions, unknown transport outcomes, stale replies, unmount, and bounded polling.
- Mocked rendered checks passed at 1280×900 and 390×844: confirmation, pulling, an injected 503 disconnect, reconnect, verified completion, reload discovery, and recovery blocking. The pre-existing Switch `classNames` React warning remains; the deliberate 503 was expected. No new framework error overlay or mobile horizontal overflow was found.
- Independent review identified and closed full-service configuration preservation, shadow database mounts, and terminal refresh discovery issues. `git diff --check` passed.
Limitations: no real Docker daemon upgrade was run; local daemon access was unavailable to this task. Fake Docker commands do not prove real Compose/Engine behavior. The PostgreSQL-specific contract was explicitly skipped because `FLVX_POSTGRES_TEST_DSN` was not set. No third-party deployment, SSH, publishing, release, or production restart was performed during implementation.
The conservative operator recovery procedure is in `go-backend/internal/panelupgrade/README.md`. Full service-setting changes, unsupported deployment layouts, or a possible partial recreation require administrator handling instead of silently changing live settings or automatically rolling back a database. Per-task logs contain safe phase summaries, not raw secret-bearing Docker output.
@@ -0,0 +1,32 @@
# Agent checksum upgrade validation
## Scope and baseline
Reviewed local changes against `fadc2b6c122df99140e0c9cbd481a1214d17d175` (rc8), as confirmed by the user. Tests use the actual UpgradeAgent command router and WebSocket response path, loopback HTTP, temporary executable/backup paths, and an intercepted restart action. No real installer or service process is launched.
## Reproduction and fix
Four red-to-green slices reproduced missing URL, checksum HTTP 404, ambiguous multiple records, and oversized checksum acceptance. Initially the command returned success, emitted verification/install progress, and reached the restart substitute. The corrected flow returns an unsuccessful UpgradeAgentResponse before installation and removes staged bytes.
Additional cases cover blank URL, invalid request URL, connection interruption, truncated HTTP body despite a valid-looking hash, empty/whitespace checksum, short/nonhex/mismatched hash, invalid GNU marker, absent filename, garbage, NUL and embedded newline. Valid fixtures cover bare digest, GNU text and binary records, uppercase hex, CRLF and surrounding whitespace. Both URL architecture placeholders and response request IDs are checked. Failed cases preserve old executable and backup; valid cases reach the intercepted restart with exact `abc` bytes and the independent known SHA256 fixture.
## Checks
- Initial socket baseline: passed.
- UpgradeAgent targeted tests and race tests: passed.
- Socket package race tests, three runs: passed (2.009s after cleanup).
- go-gost/x: `go test ./...` and `go vet ./...`: passed.
- go-gost: `go test ./...`, `go vet ./...`, and build to a temporary output: passed.
- `git diff --check`: passed.
- All listed module checks and the build were rerun after cleanup and passed.
- Go toolchain: 1.26.10 darwin/arm64.
## Independent review
Standards: no hard violations or blockers. One optional duplicate cleanup observation was addressed by consolidating staging removal in the defer.
Spec: no actionable deviations found. Missing checksum, fetch/status/read/format/hash errors stop before success and installation; supported SHA256 formats and architecture substitution are retained. No unrelated installer, Alpine, release or deployment changes.
## Limits
This verifies the agent command and integrity gate, not actual systemd installation or live-node deployment. The fix must be installed on an agent before that agent gains the new behavior; updating the panel alone cannot retrofit old agents. Hash comparison is not a signature and does not protect against a compromised source supplying both artifact and matching checksum. No push, PR, merge, tag or deployment was performed. HTTP request timeout and installer/platform redesign remain outside this repair.
@@ -0,0 +1,43 @@
# OpenRC agent upgrade and rollback validation
## Baseline and scope
User-approved review baseline: fadc2b6c122df99140e0c9cbd481a1214d17d175. Includes the pending checksum fail-closed repair (071) and manager-aware upgrade/rollback (072). No publication, deployment, or new software installation.
## Reproduction
A disposable existing Alpine 3.22.5 linux/amd64 image on Mac mini Docker reproduced both failures: actual upgrade and rollback handlers could not find systemd-run. The upgrade used a valid checksum. Source inspection confirmed install.sh supports OpenRC but the two agent handlers did not. Existing installer CI did not test those command handlers.
## Regression coverage
Actual UpgradeAgent/RollbackAgent routing and WebSocket responses cover manager selection, Alpine preference over systemctl shims, missing manager, rejected submission, backup preservation, atomic rollback staging, request IDs and all checksum cases from 071. Script execution checks stop/replace/start order, stop failure, replacement failure recovery, directory targets and no systemd legacy cleanup on OpenRC.
Linux-only integration executes the real detached helper. An rc-service double sends SIGTERM to the original test-agent's entire process group. The test verifies that the original process receives SIGTERM while the helper survives, replaces the binary with the rollback copy, preserves the backup and invokes the start command.
## Environment and checks
- Mac mini: Go 1.26.10 darwin/arm64.
- Cached Alpine 3.22.5 linux/amd64 under Docker emulation; no package installation. Tests use loopback HTTP and service-command doubles, with no host service mounts.
- Full socket tests and focused race tests: passed.
- Alpine manager/upgrade/rollback/checksum/helper-survival suite: passed.
- go-gost/x full `go test ./...`, `go vet ./...`: passed.
- go-gost `go test ./...`, `go vet ./...`, build to temporary output: passed.
- `git diff --check`: passed.
## Limitations
Real OpenRC/systemd service integration was not run. The detached process test models the installed OpenRC PID/process-group stopping behavior, not custom cgroup policies. Successful command response acknowledges helper launch or systemd submission, not completed asynchronous service restart. No automatic downgrade is attempted if starting the replaced executable fails. Live nodes were not changed. HTTP request timeouts, release/deployment, and retrofitting already installed agents are outside this local repair.
## Independent review
Standards: no documented violations or blockers. Optional one-line legacy restart-script wrapper cleanup was deferred to preserve the existing focused legacy-service regression interface.
Spec: no blocking implementation deviation. Reviewer requested explicit OpenRC process-launch-failure coverage; follow-up validation is tracked below.
Follow-up: explicit OpenRC exec-failure coverage passed for both routed commands. A Linux-only, explicitly opted-in disposable Alpine test temporarily hides only the container's /bin/sh symlink and restores it afterward. It uses production Alpine detection, executable lookup and cmd.Start; both commands reject ENOENT, preserve the live executable and expected backup, and remove staging. No runtime test abstraction was added. The test requires Alpine and Docker markers and is skipped by default.
Reproduction of the opt-in case (only a disposable container; never mount host /bin): build the socket test binary with `CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o /tmp/flvx-072-socket.test ./socket` from go-gost/x, then run `docker run --rm --pull=never --platform linux/amd64 --network none -e FLVX_TEST_DISPOSABLE_ALPINE_SHELL_FAILURE=1 -v /tmp/flvx-072-socket.test:/probe/socket.test:ro alpine:3.22 /probe/socket.test -test.run '^TestOpenRCCommandsCleanStageOnHelperExecFailure$' -test.v`.
Final host full socket race suite passed in 8.844s. Final follow-up changed only Linux test coverage; Mac socket/vet and diff checks passed again.
Independent Spec re-review confirmed the launch-failure gap is closed, with no remaining actionable spec gaps.
+8
View File
@@ -44,6 +44,7 @@ import type {
StorageSummaryApiData,
SystemUpgradeCheckApiData,
SystemUpgradeRunApiData,
SystemUpgradeStatusApiData,
SystemUpgradeVersionApiData,
} from "./types";
@@ -317,6 +318,13 @@ export const checkSystemUpgrade = (channel: ReleaseChannel = "stable") =>
channel,
});
export const getSystemUpgradeStatus = (jobId?: string) =>
Network.post<SystemUpgradeStatusApiData>(
"/system/upgrade/status",
jobId ? { jobId } : {},
{ timeout: 10000 },
);
export const runSystemUpgrade = (
version?: string,
channel: ReleaseChannel = "stable",
+15 -2
View File
@@ -46,6 +46,7 @@ interface ApiResponse<T = unknown> {
code: number;
msg: string;
data: T;
transportError?: boolean;
}
interface RequestOptions {
@@ -82,7 +83,12 @@ const Network = {
return new Promise(function (resolve) {
// 如果baseURL是默认值且是WebView环境,说明没有设置面板地址
if (baseURL === "") {
resolve({ code: -1, msg: " - 请先设置面板地址", data: null as T });
resolve({
code: -1,
msg: " - 请先设置面板地址",
data: null as T,
transportError: true,
});
return;
}
@@ -121,6 +127,7 @@ const Network = {
resolve({
code: -1,
transportError: true,
msg: errorMessage,
data: null as T,
});
@@ -136,7 +143,12 @@ const Network = {
return new Promise(function (resolve) {
// 如果baseURL是默认值且是WebView环境,说明没有设置面板地址
if (baseURL === "") {
resolve({ code: -1, msg: " - 请先设置面板地址", data: null as T });
resolve({
code: -1,
msg: " - 请先设置面板地址",
data: null as T,
transportError: true,
});
return;
}
@@ -175,6 +187,7 @@ const Network = {
resolve({
code: -1,
transportError: true,
msg: errorMessage,
data: null as T,
});
+29
View File
@@ -550,7 +550,36 @@ export interface SystemUpgradeCheckApiData extends SystemUpgradeVersionApiData {
releases: SystemUpgradeReleaseApiItem[];
}
export type SystemUpgradeStage =
| "preparing"
| "pulling"
| "applying"
| "restarting"
| "verifying"
| "succeeded"
| "failed"
| "recovery_required";
export interface SystemUpgradeJob {
id: string;
targetVersion: string;
stage: SystemUpgradeStage;
active: boolean;
requiresRecovery: boolean;
message: string;
startedAt: number;
updatedAt: number;
helperContainer?: string;
backupDir?: string;
}
export interface SystemUpgradeStatusApiData {
currentVersion: string;
job: SystemUpgradeJob | null;
}
export interface SystemUpgradeRunApiData {
jobId: string;
version: string;
channel: "stable" | "dev";
composeAsset: string;
@@ -0,0 +1,37 @@
import { useEffect, useRef, useState } from "react";
import { getSystemUpgradeStatus, runSystemUpgrade } from "@/api";
import {
createUpgradeMonitor,
initialUpgradeState,
upgradeIsBlocked,
} from "@/utils/system-upgrade-monitor";
export function useSystemUpgrade(enabled: boolean) {
const [state, setState] = useState(initialUpgradeState);
const monitor = useRef<ReturnType<typeof createUpgradeMonitor> | null>(null);
useEffect(() => {
if (!enabled) return;
const instance = createUpgradeMonitor(
{ status: getSystemUpgradeStatus, run: runSystemUpgrade },
setState,
);
monitor.current = instance;
instance.refresh();
return () => {
instance.dispose();
monitor.current = null;
};
}, [enabled]);
return {
...state,
blocked: upgradeIsBlocked(state),
refresh: () => monitor.current?.refresh(),
submit: (version?: string, channel?: "stable" | "dev") =>
monitor.current?.submit(version, channel) ?? Promise.resolve(false),
};
}
+125 -28
View File
@@ -1,6 +1,5 @@
import type {
SystemUpgradeCheckApiData,
SystemUpgradeRunApiData,
SystemUpgradeReleaseApiItem,
SystemUpgradeVersionApiData,
} from "@/api/types";
@@ -10,6 +9,8 @@ import { useNavigate } from "react-router-dom";
import { AnimatePresence, motion } from "framer-motion";
import toast from "react-hot-toast";
import { useSystemUpgrade } from "@/hooks/use-system-upgrade";
import { upgradeMessage } from "@/utils/system-upgrade-monitor";
import { Button } from "@/shadcn-bridge/heroui/button";
import { Card, CardBody, CardHeader } from "@/shadcn-bridge/heroui/card";
import { Input } from "@/shadcn-bridge/heroui/input";
@@ -36,7 +37,6 @@ import {
getStorageSummary,
getSystemUpgradeVersion,
checkSystemUpgrade,
runSystemUpgrade,
type AnnouncementData,
} from "@/api";
import { BackIcon, SettingsIcon } from "@/components/icons";
@@ -336,7 +336,9 @@ export default function ConfigPage() {
const [systemUpgradeInfo, setSystemUpgradeInfo] =
useState<SystemUpgradeVersionApiData | null>(null);
const [systemUpgradeChecking, setSystemUpgradeChecking] = useState(false);
const [systemUpgradeExecuting, setSystemUpgradeExecuting] = useState(false);
const upgrade = useSystemUpgrade(isAdmin());
const systemUpgradeExecuting =
upgrade.submitting || Boolean(upgrade.job?.active);
const [systemUpgradeLoading, setSystemUpgradeLoading] = useState(true);
const [systemUpgradeModalOpen, setSystemUpgradeModalOpen] = useState(false);
const [systemUpgradeReleases, setSystemUpgradeReleases] = useState<
@@ -363,17 +365,38 @@ export default function ConfigPage() {
const canTriggerSystemUpgrade = Boolean(
!systemUpgradeLoading &&
!systemUpgradeChecking &&
!systemUpgradeExecuting &&
systemUpgradeInfo?.capability.capable !== false,
!upgrade.blocked &&
systemUpgradeInfo?.capability.capable === true,
);
const canOpenSystemUpgradeModal = Boolean(
systemUpgradeInfo?.capability.capable &&
systemUpgradeHasConfirmedUpdate &&
!systemUpgradeLoading &&
!systemUpgradeChecking &&
!systemUpgradeExecuting,
!upgrade.blocked,
);
useEffect(() => {
if (upgrade.job?.stage !== "succeeded" || upgrade.uncertain) return;
setSystemUpgradeReleases([]);
setSystemUpgradeCheckedChannel(null);
setSystemUpgradeSelectedVersion("");
setSystemUpgradeInfo((previous) =>
previous
? {
...previous,
currentVersion: upgrade.currentVersion,
hasUpdate: false,
}
: previous,
);
}, [
upgrade.job?.id,
upgrade.job?.stage,
upgrade.currentVersion,
upgrade.uncertain,
]);
const canGoBack =
typeof window !== "undefined" &&
typeof window.history.state?.idx === "number" &&
@@ -572,6 +595,7 @@ export default function ConfigPage() {
};
const handleOpenSystemUpgradeModal = async () => {
if (upgrade.blocked) return;
if (!canOpenSystemUpgradeModal) {
const checked = await handleCheckSystemUpgrade();
@@ -583,26 +607,15 @@ export default function ConfigPage() {
};
const handleConfirmSystemUpgrade = async () => {
setSystemUpgradeExecuting(true);
try {
const response = await runSystemUpgrade(
systemUpgradeSelectedVersion || undefined,
updateChannel,
);
if (!canTriggerSystemUpgrade) return;
const submitted = await upgrade.submit(
systemUpgradeSelectedVersion || undefined,
updateChannel,
);
if (response.code === 0 && response.data) {
const data = response.data as SystemUpgradeRunApiData;
setSystemUpgradeModalOpen(false);
setSystemUpgradeSelectedVersion("");
toast.success(data.message || "升级已触发,请稍后刷新页面");
} else {
toast.error(response.msg || "面板升级失败");
}
} catch {
toast.error("面板升级失败,请重试");
} finally {
setSystemUpgradeExecuting(false);
if (submitted) {
setSystemUpgradeModalOpen(false);
setSystemUpgradeSelectedVersion("");
}
};
@@ -1647,6 +1660,87 @@ export default function ConfigPage() {
</p>
</div>
<div
aria-live="polite"
className="space-y-2 rounded-lg border border-divider bg-background p-4 text-sm"
>
<p className="font-medium">
{upgrade.submitting
? "正在提交升级任务..."
: upgrade.job
? {
preparing: "正在准备升级",
pulling: "正在拉取镜像",
applying: "正在应用升级",
restarting: "正在重启服务",
verifying: "正在验证运行版本",
succeeded: "升级完成,运行版本已验证",
failed: "升级失败",
recovery_required: "升级中断,需要人工恢复",
}[upgrade.job.stage] || "升级状态待确认"
: upgrade.checking
? "正在恢复升级状态..."
: upgrade.uncertain
? "升级状态待确认"
: "暂无升级任务"}
</p>
{upgrade.job && (
<>
<p className="break-words text-default-600">
目标版本:{upgrade.job.targetVersion}
</p>
{upgrade.job.message && (
<p className="break-words text-default-600">
{upgradeMessage(upgrade.job.message)}
</p>
)}
{upgrade.job.active && (
<p className="text-default-500">
升级期间请勿重复操作。后端重启可能短暂断开连接,本页面会自动重新连接并验证结果。
</p>
)}
{upgrade.job.stage === "succeeded" && !upgrade.uncertain && (
<p className="text-success">
后端已验证运行版本 {upgrade.currentVersion}
。请刷新页面以加载新版前端。
</p>
)}
{(upgrade.job.requiresRecovery ||
upgrade.job.stage === "recovery_required") && (
<p className="text-danger">
已阻止再次升级。请联系服务器管理员检查升级辅助容器日志和部署目录中的备份,确认服务与数据库状态后按部署文档恢复;不要删除备份或反复触发升级。
</p>
)}
{upgrade.job.stage === "failed" && (
<p className="text-danger">
请先排查失败原因,再检查更新并重试。
</p>
)}
</>
)}
{upgrade.uncertain && !upgrade.checking && (
<p className="text-warning">
暂时无法确认升级结果,可能正在重启或连接已中断。这不代表升级失败,请勿重复提交。
</p>
)}
{upgrade.paused && (
<p className="text-warning">
自动查询已暂停,任务可能仍在运行。请检查服务连接后点击“重新查询状态”;不会重新发起升级。
</p>
)}
{upgrade.notice && (
<p className="text-danger">{upgrade.notice}</p>
)}
<Button
isDisabled={upgrade.checking || upgrade.submitting}
size="sm"
variant="flat"
onPress={upgrade.refresh}
>
重新查询状态
</Button>
</div>
{systemUpgradeLoading ? (
<div className="flex items-center gap-2 rounded-lg border border-divider bg-background px-4 py-3 text-sm text-default-500">
<Spinner size="sm" />
@@ -1658,7 +1752,9 @@ export default function ConfigPage() {
<div>
<p className="text-xs text-default-500">当前版本</p>
<p className="mt-1 font-medium">
{systemUpgradeInfo?.currentVersion || "未获取到版本信息"}
{upgrade.currentVersion ||
systemUpgradeInfo?.currentVersion ||
"未获取到版本信息"}
</p>
</div>
<div>
@@ -1776,6 +1872,7 @@ export default function ConfigPage() {
<div className="flex flex-col gap-3 pt-1 sm:flex-row sm:justify-end">
<Button
isDisabled={upgrade.blocked}
isLoading={systemUpgradeChecking}
variant="flat"
onPress={handleCheckSystemUpgrade}
@@ -2078,8 +2175,8 @@ export default function ConfigPage() {
</Button>
<Button
color="primary"
isDisabled={systemUpgradeExecuting}
isLoading={systemUpgradeExecuting}
isDisabled={!canTriggerSystemUpgrade}
isLoading={upgrade.submitting}
onPress={handleConfirmSystemUpgrade}
>
确认升级
+113 -57
View File
@@ -1,6 +1,7 @@
import type { MonitorNodeApiItem } from "@/api/types";
import type { RealtimeNodeMetric } from "@/pages/node/monitor-view";
import { useCallback, useEffect, useMemo, useState } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import toast from "react-hot-toast";
import {
RefreshCw,
@@ -44,73 +45,117 @@ export default function MonitorPage() {
const [viewMode, setViewMode] = useState<"list" | "grid">("list");
const [activeTab, setActiveTab] = useState<MonitorTab>("nodes");
const [realtimeNodeStatus, setRealtimeNodeStatus] = useState<
Record<number, "online" | "offline">
>({});
const realtimeNodeStatusRef = useRef(new Map<number, { status: number }>());
const nodesRequestIdRef = useRef(0);
const [realtimeNodeMetrics, setRealtimeNodeMetrics] = useState<
Record<number, any>
Record<number, RealtimeNodeMetric>
>({});
const handleRealtimeMessage = useCallback((message: any) => {
const nodeId = Number(message?.id ?? 0);
if (!nodeId || Number.isNaN(nodeId)) return;
const type = String(message?.type ?? "");
const payload = message?.data;
if (type === "status") {
const status = Number(payload);
setRealtimeNodeStatus((prev) => ({
...prev,
[nodeId]: status === 1 ? "online" : "offline",
}));
return;
}
if (type === "metric") {
let raw = payload;
if (typeof raw === "string") {
try {
raw = JSON.parse(raw);
} catch {
return;
}
}
if (!raw || typeof raw !== "object") return;
const metric = raw as any;
setRealtimeNodeMetrics((prev) => ({
...prev,
[nodeId]: {
cpuUsage: Number(metric.cpuUsage ?? metric.cpu_usage ?? 0),
netInSpeed: Number(metric.netInSpeed ?? metric.net_in_speed ?? 0),
netOutSpeed: Number(metric.netOutSpeed ?? metric.net_out_speed ?? 0),
tcpConns: Number(metric.tcpConns ?? metric.tcp_conns ?? 0),
udpConns: Number(metric.udpConns ?? metric.udp_conns ?? 0),
},
}));
}
const updateNodeStatus = useCallback((nodeId: number, status: number) => {
realtimeNodeStatusRef.current.set(nodeId, { status });
setNodes((prev) =>
prev.map((node) =>
Number(node.id) === nodeId ? { ...node, status } : node,
),
);
}, []);
useNodeRealtime({
const handleRealtimeMessage = useCallback(
(message: any) => {
const nodeId = Number(message?.id ?? 0);
if (!nodeId || Number.isNaN(nodeId)) return;
const type = String(message?.type ?? "");
const payload = message?.data;
if (type === "status") {
updateNodeStatus(nodeId, Number(payload) === 1 ? 1 : 0);
return;
}
if (type === "info") {
updateNodeStatus(nodeId, 1);
return;
}
if (type === "metric") {
let raw = payload;
if (typeof raw === "string") {
try {
raw = JSON.parse(raw);
} catch {
return;
}
}
if (!raw || typeof raw !== "object") return;
const metric = raw as Record<string, unknown>;
const receivedAt = Date.now();
const incomingUptime = Number(metric.uptime ?? 0);
setRealtimeNodeMetrics((prev) => ({
...prev,
[nodeId]: {
receivedAt,
cpuUsage: Number(metric.cpuUsage ?? metric.cpu_usage ?? 0),
memoryUsage: Number(metric.memoryUsage ?? metric.memory_usage ?? 0),
diskUsage: Number(metric.diskUsage ?? metric.disk_usage ?? 0),
netInBytes: Number(metric.netInBytes ?? metric.bytes_received ?? 0),
netOutBytes: Number(
metric.netOutBytes ?? metric.bytes_transmitted ?? 0,
),
netInSpeed: Number(metric.netInSpeed ?? metric.net_in_speed ?? 0),
netOutSpeed: Number(
metric.netOutSpeed ?? metric.net_out_speed ?? 0,
),
load1: Number(metric.load1 ?? 0),
load5: Number(metric.load5 ?? 0),
load15: Number(metric.load15 ?? 0),
tcpConns: Number(metric.tcpConns ?? metric.tcp_conns ?? 0),
udpConns: Number(metric.udpConns ?? metric.udp_conns ?? 0),
uptime: incomingUptime || prev[nodeId]?.uptime || 0,
},
}));
updateNodeStatus(nodeId, 1);
}
},
[updateNodeStatus],
);
const { wsConnected, wsConnecting } = useNodeRealtime({
onMessage: handleRealtimeMessage,
enabled: true,
});
const loadNodes = useCallback(async (options?: { silent?: boolean }) => {
const silent = options?.silent ?? false;
const requestId = ++nodesRequestIdRef.current;
const statusAtRequest = new Map(realtimeNodeStatusRef.current);
if (!silent) setNodesLoading(true);
try {
const response = await getMonitorNodes();
// A newer refresh or poll owns the page state, even if it finishes first.
if (requestId !== nodesRequestIdRef.current) return;
if (response.code === 0 && Array.isArray(response.data)) {
setNodesError(null);
setNodes(response.data);
// A fresh snapshot repairs missed WS events. Events received while the
// request was in flight are newer than that snapshot and must win.
setNodes(
response.data.map((node) => {
const nodeId = Number(node.id);
const latest = realtimeNodeStatusRef.current.get(nodeId);
return latest && latest !== statusAtRequest.get(nodeId)
? { ...node, status: latest.status }
: node;
}),
);
return;
}
@@ -124,14 +169,21 @@ export default function MonitorPage() {
if (!silent) toast.error(response.msg || "加载节点失败");
} catch {
if (!silent) toast.error("加载节点失败");
if (requestId === nodesRequestIdRef.current && !silent) {
toast.error("加载节点失败");
}
} finally {
if (!silent) setNodesLoading(false);
// A silent poll may supersede a foreground refresh that showed loading.
if (requestId === nodesRequestIdRef.current) setNodesLoading(false);
}
}, []);
useEffect(() => {
void loadNodes();
return () => {
nodesRequestIdRef.current += 1;
};
}, [loadNodes]);
useEffect(() => {
@@ -148,14 +200,12 @@ export default function MonitorPage() {
.map((n) => ({
id: Number(n.id),
name: String(n.name ?? ""),
connectionStatus:
realtimeNodeStatus[Number(n.id)] ||
(n.status === 1 ? "online" : "offline"),
connectionStatus: n.status === 1 ? "online" : "offline",
version: n.version,
}));
return new Map<number, MonitorNode>(list.map((n) => [n.id, n]));
}, [nodes, realtimeNodeStatus]);
}, [nodes]);
const aggregateMetrics = useMemo(() => {
let totalCpu = 0;
@@ -326,7 +376,13 @@ export default function MonitorPage() {
</div>
{activeTab === "nodes" ? (
<MonitorView nodeMap={nodeMap} viewMode={viewMode} />
<MonitorView
nodeMap={nodeMap}
realtimeNodeMetrics={realtimeNodeMetrics}
viewMode={viewMode}
wsConnected={wsConnected}
wsConnecting={wsConnecting}
/>
) : (
<TunnelMonitorView viewMode={viewMode} />
)}
+12 -109
View File
@@ -78,17 +78,19 @@ import {
} from "@/shadcn-bridge/heroui/dropdown";
import { Card, CardBody, CardHeader } from "@/shadcn-bridge/heroui/card";
import { Progress } from "@/shadcn-bridge/heroui/progress";
import { useNodeRealtime } from "@/pages/node/use-node-realtime";
interface MonitorViewProps {
nodeMap: Map<
number,
{ id: number; name: string; connectionStatus: string; version?: string }
>;
realtimeNodeMetrics: Record<number, RealtimeNodeMetric>;
wsConnected: boolean;
wsConnecting: boolean;
viewMode?: "list" | "grid";
}
type RealtimeNodeMetric = {
export type RealtimeNodeMetric = {
receivedAt: number;
cpuUsage: number;
memoryUsage: number;
@@ -532,7 +534,13 @@ const DEFAULT_SERVICE_MONITOR_LIMITS: ServiceMonitorLimitsApiData = {
maxTimeoutSec: 60,
};
export function MonitorView({ nodeMap, viewMode = "grid" }: MonitorViewProps) {
export function MonitorView({
nodeMap,
realtimeNodeMetrics,
wsConnected,
wsConnecting,
viewMode = "grid",
}: MonitorViewProps) {
const [detailNodeId, setDetailNodeId] = useState<number | null>(null);
const [selectedNodeId, setSelectedNodeId] = useState<number | null>(null);
const [metrics, setMetrics] = useState<NodeMetricApiItem[]>([]);
@@ -582,27 +590,7 @@ export function MonitorView({ nodeMap, viewMode = "grid" }: MonitorViewProps) {
const [resultsLimit, setResultsLimit] = useState(50);
const [resultsLoading, setResultsLoading] = useState(false);
const [realtimeNodeStatus, setRealtimeNodeStatus] = useState<
Record<number, "online" | "offline">
>({});
const [realtimeNodeMetrics, setRealtimeNodeMetrics] = useState<
Record<number, RealtimeNodeMetric>
>({});
const nodes = useMemo(() => {
return Array.from(nodeMap.values()).map((n) => {
const status = realtimeNodeStatus[n.id];
if (!status) {
return n;
}
return {
...n,
connectionStatus: status,
};
});
}, [nodeMap, realtimeNodeStatus]);
const nodes = useMemo(() => Array.from(nodeMap.values()), [nodeMap]);
const onlineNodes = nodes.filter((n) => n.connectionStatus === "online");
const preferredNodeId = onlineNodes[0]?.id ?? nodes[0]?.id ?? 0;
@@ -610,91 +598,6 @@ export function MonitorView({ nodeMap, viewMode = "grid" }: MonitorViewProps) {
const resolvedServiceMonitorLimits =
serviceMonitorLimits ?? DEFAULT_SERVICE_MONITOR_LIMITS;
const handleRealtimeMessage = useCallback((message: any) => {
const nodeId = Number(message?.id ?? 0);
if (!nodeId || Number.isNaN(nodeId)) {
return;
}
const type = String(message?.type ?? "");
const payload = message?.data;
if (type === "status") {
const status = Number(payload);
setRealtimeNodeStatus((prev) => ({
...prev,
[nodeId]: status === 1 ? "online" : "offline",
}));
return;
}
if (type === "metric") {
let raw = payload;
if (typeof raw === "string") {
try {
raw = JSON.parse(raw);
} catch {
return;
}
}
if (!raw || typeof raw !== "object") {
return;
}
const metric = raw as Record<string, unknown>;
const receivedAt = Date.now();
const incomingUptime = Number(metric.uptime ?? 0);
setRealtimeNodeMetrics((prev) => {
const normalized: RealtimeNodeMetric = {
receivedAt,
cpuUsage: Number(metric.cpuUsage ?? metric.cpu_usage ?? 0),
memoryUsage: Number(metric.memoryUsage ?? metric.memory_usage ?? 0),
diskUsage: Number(metric.diskUsage ?? metric.disk_usage ?? 0),
netInBytes: Number(metric.netInBytes ?? metric.bytes_received ?? 0),
netOutBytes: Number(
metric.netOutBytes ?? metric.bytes_transmitted ?? 0,
),
netInSpeed: Number(metric.netInSpeed ?? metric.net_in_speed ?? 0),
netOutSpeed: Number(metric.netOutSpeed ?? metric.net_out_speed ?? 0),
load1: Number(metric.load1 ?? 0),
load5: Number(metric.load5 ?? 0),
load15: Number(metric.load15 ?? 0),
tcpConns: Number(metric.tcpConns ?? metric.tcp_conns ?? 0),
udpConns: Number(metric.udpConns ?? metric.udp_conns ?? 0),
uptime: incomingUptime || prev[nodeId]?.uptime || 0,
};
return {
...prev,
[nodeId]: normalized,
};
});
setRealtimeNodeStatus((prev) => ({
...prev,
[nodeId]: "online",
}));
return;
}
if (type === "info") {
setRealtimeNodeStatus((prev) => ({
...prev,
[nodeId]: "online",
}));
}
}, []);
const { wsConnected, wsConnecting } = useNodeRealtime({
onMessage: handleRealtimeMessage,
enabled: !accessDenied,
});
useEffect(() => {
if (!selectedNodeId && preferredNodeId > 0) {
setSelectedNodeId(preferredNodeId);
@@ -0,0 +1,206 @@
import type {
SystemUpgradeJob,
SystemUpgradeRunApiData,
SystemUpgradeStatusApiData,
} from "@/api/types";
type Response<T> = {
code: number;
data: T;
msg?: string;
transportError?: boolean;
};
export interface UpgradeMonitorState {
job: SystemUpgradeJob | null;
currentVersion: string;
checking: boolean;
submitting: boolean;
uncertain: boolean;
paused: boolean;
notice: string;
}
export const initialUpgradeState: UpgradeMonitorState = {
job: null,
currentVersion: "",
checking: true,
submitting: false,
uncertain: true,
paused: false,
notice: "",
};
export const upgradeIsBlocked = (state: UpgradeMonitorState) =>
state.checking ||
state.submitting ||
state.uncertain ||
Boolean(
state.job?.active ||
state.job?.requiresRecovery ||
state.job?.stage === "recovery_required",
);
// The server supplies sanitized public summaries, never helper logs. Render as
// text only; remove control characters and cap length as defense in depth.
export const upgradeMessage = (message: string) =>
message.replace(/[\u0000-\u001f\u007f]/g, " ").slice(0, 500);
// Kept independent of React so lifecycle/disconnect races can be replayed
// without adding frontend test infrastructure.
export function createUpgradeMonitor(
api: {
status: (id?: string) => Promise<Response<SystemUpgradeStatusApiData>>;
run: (
version?: string,
channel?: "stable" | "dev",
) => Promise<Response<SystemUpgradeRunApiData>>;
},
publish: (state: UpgradeMonitorState) => void,
clock = {
now: () => Date.now(),
schedule: (fn: () => void, ms: number) => setTimeout(fn, ms),
cancel: (timer: ReturnType<typeof setTimeout>) => clearTimeout(timer),
},
) {
let state = { ...initialUpgradeState };
let timer: ReturnType<typeof setTimeout> | undefined;
let generation = 0;
let disposed = false;
let busy = false;
let deadline = 0;
let failures = 0;
let expectedJob: string | undefined;
let awaitingSubmission = false;
let previousJob: string | undefined;
const emit = (patch: Partial<UpgradeMonitorState>) => {
state = { ...state, ...patch };
if (!disposed) publish(state);
};
const cancelTimer = () => {
if (timer !== undefined) clock.cancel(timer);
timer = undefined;
};
const poll = async () => {
if (disposed || busy) return;
busy = true;
const request = generation;
try {
const response = await api.status(expectedJob);
if (disposed || request !== generation) return;
if (response.code !== 0 || !response.data) throw new Error("unavailable");
const { job, currentVersion } = response.data;
// An accepted or interrupted POST must not be mistaken for an idle
// server, nor may an older completed job certify the new request.
if (
(expectedJob && job?.id !== expectedJob) ||
(awaitingSubmission && (!job || job.id === previousJob))
) {
throw new Error("awaiting job");
}
failures = 0;
awaitingSubmission = false;
expectedJob = job?.id;
emit({
job,
currentVersion,
checking: false,
uncertain: false,
paused: false,
});
} catch {
if (disposed || request !== generation) return;
failures += 1;
// Disconnection is not proof of failure. Keep the last known job and
// continue blocking submissions until authoritative status is available.
emit({ checking: false, uncertain: true });
} finally {
busy = false;
if (
!disposed &&
request === generation &&
(state.uncertain || state.job?.active)
) {
if (clock.now() >= deadline) {
emit({ paused: true });
} else {
timer = clock.schedule(
() => {
void poll();
},
Math.min(15000, 2000 * 2 ** Math.min(failures, 3)),
);
}
}
}
};
return {
refresh() {
if (disposed || busy || state.submitting) return;
cancelTimer();
if (
!state.uncertain &&
!state.job?.active &&
!state.job?.requiresRecovery
) {
expectedJob = undefined;
}
deadline = clock.now() + 15 * 60 * 1000;
failures = 0;
emit({ checking: true, paused: false });
void poll();
},
async submit(version?: string, channel?: "stable" | "dev") {
if (disposed || busy || upgradeIsBlocked(state)) return false;
cancelTimer();
const request = ++generation;
previousJob = state.job?.id;
expectedJob = undefined;
awaitingSubmission = true;
emit({
job: null,
submitting: true,
uncertain: true,
paused: false,
notice: "",
});
try {
const response = await api.run(version, channel);
if (disposed || request !== generation) return false;
if (response.code === 0 && response.data?.jobId) {
expectedJob = response.data.jobId;
awaitingSubmission = false;
} else if (response.code !== 0 && !response.transportError) {
// Definite server rejection: rediscover the authoritative job,
// including concurrent submissions from other admin sessions.
awaitingSubmission = false;
emit({
notice: upgradeMessage(
response.msg || "升级请求被拒绝,请检查当前任务状态。",
),
});
}
} catch {
// The POST may already have started the helper. Reconcile, don't retry.
} finally {
if (!disposed && request === generation) {
emit({ submitting: false });
deadline = clock.now() + 15 * 60 * 1000;
void poll();
}
}
return !disposed && request === generation;
},
dispose() {
disposed = true;
generation += 1;
cancelTimer();
},
};
}