fix(agent): verify upgrades and support OpenRC lifecycle (#570)

This commit is contained in:
sagit
2026-11-09 08:59:25 +08:00
committed by GitHub
parent fadc2b6c12
commit 743bf6f278
12 changed files with 1057 additions and 73 deletions
+139
View File
@@ -0,0 +1,139 @@
package socket
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
type agentServiceManager struct{ kind, control, runner string }
// Alpine must never select a systemctl compatibility shim. Elsewhere prefer
// an active systemd instance, matching the installer, then OpenRC.
func detectAgentServiceManager(alpine, systemd bool, lookup func(string) (string, error)) (agentServiceManager, error) {
if !alpine && systemd {
control, err := lookup("systemctl")
if err != nil {
return agentServiceManager{}, fmt.Errorf("systemctl 不可用: %w", err)
}
runner, err := lookup("systemd-run")
if err != nil {
return agentServiceManager{}, fmt.Errorf("systemd-run 不可用: %w", err)
}
return agentServiceManager{kind: "systemd", control: control, runner: runner}, nil
}
control, err := lookup("rc-service")
if err != nil {
return agentServiceManager{}, fmt.Errorf("未检测到受支持的服务管理器 (systemd/OpenRC): %w", err)
}
return agentServiceManager{kind: "openrc", control: control}, nil
}
func (w *WebSocketReporter) agentRestartLauncher(cleanLegacy bool) (func(string, string) error, error) {
// Existing per-reporter seam is also used by checksum-only tests.
if w.startUpgradeRestart != nil {
return w.startUpgradeRestart, nil
}
detect := w.detectAgentManager
if detect == nil {
detect = func() (agentServiceManager, error) {
_, alpineErr := os.Stat("/etc/alpine-release")
_, systemdErr := os.Stat("/run/systemd/system")
return detectAgentServiceManager(alpineErr == nil, systemdErr == nil, exec.LookPath)
}
}
manager, err := detect()
if err != nil {
return nil, err
}
if manager.kind == "openrc" {
if err := checkAgentDetach(); err != nil {
return nil, err
}
}
return func(staged, binary string) error {
script := buildManagedAgentRestartScript(manager, staged, binary, cleanLegacy)
if manager.kind == "systemd" {
// Wait only for submission, never for helper completion. A separate unit is
// essential: systemctl stop kills every process in the agent's cgroup.
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, manager.runner, "--quiet", "/bin/sh", "-c", script).CombinedOutput()
if err != nil {
return fmt.Errorf("提交重启任务失败: %w: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
return startDetachedAgentHelper(script)
}, nil
}
func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" }
func buildManagedAgentRestartScript(manager agentServiceManager, staged, binary string, cleanLegacy bool) string {
control := shellQuote(manager.control)
stop, start := control+" stop flux_agent", control+" start flux_agent"
if manager.kind == "openrc" {
stop, start = control+" flux_agent stop", control+" flux_agent start"
}
legacy := ""
if manager.kind == "systemd" && cleanLegacy {
legacy = `legacy_service=''; for service_file in /etc/systemd/system/gost.service /lib/systemd/system/gost.service /usr/lib/systemd/system/gost.service; do if [ -f "$service_file" ] && grep -Fq "WorkingDirectory=/etc/gost" "$service_file" && (grep -Fq "ExecStart=/etc/gost/gost" "$service_file" || (grep -Fq "ExecStart=/usr/local/bin/gost" "$service_file" && [ -f /etc/gost/config.json ] && [ -f /etc/gost/gost.json ])); then legacy_service="$service_file"; break; fi; done; if [ -n "$legacy_service" ]; then (` + control + ` stop gost 2>/dev/null || true); (` + control + ` disable gost 2>/dev/null || true); rm -f /usr/local/bin/gost /etc/gost/gost "$legacy_service"; (` + control + ` daemon-reload 2>/dev/null || true); fi; `
}
// The caller owns staging until launch. Afterwards the helper always removes
// it, including stop/replace failures. Never truncate the running executable.
return "trap " + shellQuote("rm -f -- "+shellQuote(staged)) + " EXIT; sleep 1; " + stop + " || exit $?; " + legacy +
"if [ ! -d " + shellQuote(binary) + " ] && mv -f -- " + shellQuote(staged) + " " + shellQuote(binary) + "; then " + start + "; else status=$?; " + start + "; exit \"$status\"; fi"
}
// Copy to a unique file on the destination filesystem, then let the caller
// atomically rename. Failure never truncates a previous usable backup.
func stageAgentCopy(source, destination string) (path string, err error) {
in, err := os.Open(source)
if err != nil {
return "", err
}
defer in.Close()
info, err := in.Stat()
if err != nil {
return "", err
}
if !info.Mode().IsRegular() {
return "", fmt.Errorf("不是普通文件: %s", source)
}
out, err := os.CreateTemp(filepath.Dir(destination), ".flux-agent-*")
if err != nil {
return "", err
}
path = out.Name()
defer func() {
out.Close()
if err != nil {
os.Remove(path)
}
}()
if _, err = io.Copy(out, in); err != nil {
return path, err
}
if err = out.Chmod(0755); err != nil {
return path, err
}
if err = out.Close(); err != nil {
return path, err
}
return path, nil
}
func backupAgentBinary(binary string) error {
staged, err := stageAgentCopy(binary, binary+".old")
if err != nil {
return err
}
defer os.Remove(staged)
return os.Rename(staged, binary+".old")
}
+20
View File
@@ -0,0 +1,20 @@
package socket
import (
"os/exec"
"syscall"
)
func checkAgentDetach() error { return nil }
func startDetachedAgentHelper(script string) error {
cmd := exec.Command("/bin/sh", "-c", script)
// The installed OpenRC service stops the agent PID. A separate session also
// survives a process-group stop. No inherited stdio or parent-death signal.
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
if err := cmd.Start(); err != nil {
return err
}
go func() { _ = cmd.Wait() }()
return nil
}
@@ -0,0 +1,132 @@
package socket
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
)
// This subprocess stands in for the running agent. The rc-service double kills
// its entire process group, not merely a fake PID, while the real helper runs.
func TestAgentDetachedParent(t *testing.T) {
dir := os.Getenv("FLVX_TEST_DETACH_ROOT")
if dir == "" {
t.Skip("subprocess helper")
}
binary := filepath.Join(dir, "agent")
control := filepath.Join(dir, "rc-service")
writeAgentFixture(t, control, "#!/bin/sh\nif [ \"$2\" = stop ]; then kill -TERM -"+strconv.Itoa(os.Getpid())+"; else echo started > "+shellQuote(filepath.Join(dir, "started"))+"; fi\n")
reporter := &WebSocketReporter{upgradeBinaryPath: binary, detectAgentManager: func() (agentServiceManager, error) { return agentServiceManager{kind: "openrc", control: control}, nil }}
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
if !response.Success {
t.Fatal(response.Message)
}
time.Sleep(15 * time.Second)
t.Fatal("original agent was not stopped")
}
func TestOpenRCHelperSurvivesAgentProcessGroupStop(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, binary+".old", "backup")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestAgentDetachedParent$", "-test.v")
cmd.Env = append(os.Environ(), "FLVX_TEST_DETACH_ROOT="+dir)
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
output, err := cmd.CombinedOutput()
if err == nil || ctx.Err() != nil {
t.Fatalf("parent termination = %v, timeout=%v, %s", err, ctx.Err(), output)
}
exit, ok := err.(*exec.ExitError)
if !ok || exit.Sys().(syscall.WaitStatus).Signal() != syscall.SIGTERM {
t.Fatalf("parent did not receive SIGTERM: %v %s", err, output)
}
waitAgentFixture(t, filepath.Join(dir, "started"), "started\n")
if readAgentFixture(t, binary) != "backup" || readAgentFixture(t, binary+".old") != "backup" {
t.Fatal("rollback failed after parent termination")
}
fmt.Println("detached OpenRC helper survived original agent process-group SIGTERM")
}
// Run only in an explicitly opted-in disposable Alpine container. The docker
// invocation mounts only the test binary, never a host /bin directory. Removing
// the container's /bin/sh symlink forces the real detached cmd.Start to fail.
func TestOpenRCCommandsCleanStageOnHelperExecFailure(t *testing.T) {
if os.Getenv("FLVX_TEST_DISPOSABLE_ALPINE_SHELL_FAILURE") != "1" {
t.Skip("requires explicit disposable Alpine container opt-in")
}
for _, path := range []string{"/etc/alpine-release", "/.dockerenv"} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("refusing shell mutation outside disposable Alpine: %s: %v", path, err)
}
}
info, err := os.Lstat("/bin/sh")
if err != nil || info.Mode()&os.ModeSymlink == 0 {
t.Fatalf("refusing to rename a non-symlink /bin/sh: %v", err)
}
saved := "/bin/.flvx-test-sh-" + strconv.Itoa(os.Getpid())
if _, err := os.Lstat(saved); !os.IsNotExist(err) {
t.Fatalf("refusing to overwrite shell backup: %v", err)
}
if err := os.Rename("/bin/sh", saved); err != nil {
t.Fatal(err)
}
defer func() {
if err := os.Rename(saved, "/bin/sh"); err != nil {
t.Errorf("restore container shell: %v", err)
}
}()
for _, operation := range []string{"UpgradeAgent", "RollbackAgent"} {
t.Run(operation, func(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, binary+".old", "prior backup")
writeAgentFixture(t, filepath.Join(dir, "rc-service"), "#!/bin/sh\nexit 0\n")
t.Setenv("PATH", dir)
// No manager or launcher injection: Alpine detection and exec.LookPath are
// production code too. Only the installation path points to an isolated dir.
reporter := &WebSocketReporter{upgradeBinaryPath: binary}
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/checksum" {
fmt.Fprint(w, abcSHA256)
} else {
fmt.Fprint(w, "abc")
}
}))
defer fixture.Close()
response := routeAgentOperation(t, reporter, operation, map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
if response.Success || !strings.Contains(response.Message, "/bin/sh") || !strings.Contains(response.Message, "no such file") {
t.Fatalf("expected real helper exec failure, got %+v", response)
}
if readAgentFixture(t, binary) != "current" {
t.Fatal("live binary changed")
}
backup := "prior backup"
if operation == "UpgradeAgent" {
backup = "current"
}
if readAgentFixture(t, binary+".old") != backup {
t.Fatal("unexpected backup contents")
}
matches, err := filepath.Glob(filepath.Join(dir, ".flux-agent-*"))
if err != nil || len(matches) != 0 {
t.Fatalf("rollback staging residue: %v %v", matches, err)
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Fatalf("upgrade staging residue: %v", err)
}
})
}
}
+8
View File
@@ -0,0 +1,8 @@
//go:build !linux
package socket
import "fmt"
func checkAgentDetach() error { return fmt.Errorf("OpenRC Agent 重启仅支持 Linux") }
func startDetachedAgentHelper(string) error { return checkAgentDetach() }
+329
View File
@@ -0,0 +1,329 @@
package socket
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
)
func routeAgentOperation(t *testing.T, reporter *WebSocketReporter, operation string, data interface{}) CommandResponse {
t.Helper()
peer := make(chan *websocket.Conn, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
if err == nil {
peer <- c
}
}))
defer server.Close()
c, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer c.Close()
reader := <-peer
defer reader.Close()
reporter.ctx = context.Background()
reporter.conn = c
reporter.connected = true
reporter.routeCommand(CommandMessage{Type: operation, RequestId: "restart-request", Data: data})
reader.SetReadDeadline(time.Now().Add(3 * time.Second))
for {
var response CommandResponse
if err := reader.ReadJSON(&response); err != nil {
t.Fatal(err)
}
if response.Type == operation+"Response" {
if response.RequestId != "restart-request" {
t.Fatalf("request id lost: %s", response.RequestId)
}
return response
}
}
}
func TestRollbackAgentStagesBackupBeforeRestart(t *testing.T) {
binary := filepath.Join(t.TempDir(), "flux_agent")
for path, body := range map[string]string{binary: "current", binary + ".old": "backup"} {
if err := os.WriteFile(path, []byte(body), 0755); err != nil {
t.Fatal(err)
}
}
called := false
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(staged, dest string) error {
called = true
body, err := os.ReadFile(staged)
if err != nil || string(body) != "backup" || staged == binary+".old" || dest != binary {
t.Fatalf("bad rollback staging: %s %s %q %v", staged, dest, body, err)
}
return os.Rename(staged, dest)
}}
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
if !response.Success || !called {
t.Fatalf("rollback failed: called=%v %s", called, response.Message)
}
body, err := os.ReadFile(binary + ".old")
if err != nil || string(body) != "backup" {
t.Fatalf("backup consumed: %q %v", body, err)
}
}
func TestAgentServiceManagerDetection(t *testing.T) {
for _, tc := range []struct {
name string
alpine, systemd bool
tools string
want string
}{
{"alpine shim", true, true, "rc-service systemctl systemd-run", "openrc"},
{"alpine missing openrc", true, true, "systemctl systemd-run", ""},
{"systemd", false, true, "rc-service systemctl systemd-run", "systemd"},
{"missing runner", false, true, "systemctl", ""},
{"openrc fallback", false, false, "rc-service", "openrc"},
{"unmanaged", false, false, "systemctl systemd-run", ""},
} {
t.Run(tc.name, func(t *testing.T) {
manager, err := detectAgentServiceManager(tc.alpine, tc.systemd, func(name string) (string, error) {
if strings.Contains(" "+tc.tools+" ", " "+name+" ") {
return name, nil
}
return "", os.ErrNotExist
})
if manager.kind != tc.want || (err != nil) != (tc.want == "") {
t.Fatalf("manager=%+v err=%v", manager, err)
}
})
}
}
func writeAgentFixture(t *testing.T, path, body string) {
t.Helper()
if err := os.WriteFile(path, []byte(body), 0755); err != nil {
t.Fatal(err)
}
}
func readAgentFixture(t *testing.T, path string) string {
t.Helper()
body, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(body)
}
func waitAgentFixture(t *testing.T, path, want string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
body, _ := os.ReadFile(path)
if string(body) == want {
return
}
time.Sleep(20 * time.Millisecond)
}
body, _ := os.ReadFile(path)
t.Fatalf("%s = %q, want %q", path, body, want)
}
func TestAgentCommandsManagerLaunch(t *testing.T) {
for _, operation := range []string{"UpgradeAgent", "RollbackAgent"} {
for _, mode := range []string{"systemd", "openrc", "missing", "rejected"} {
t.Run(operation+"/"+mode, func(t *testing.T) {
if mode == "openrc" && runtime.GOOS != "linux" {
t.Skip("detached launcher is Linux-only")
}
dir := t.TempDir()
binary := filepath.Join(dir, "flux 'agent")
log := filepath.Join(dir, "events")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, binary+".old", "backup")
control := filepath.Join(dir, "manager")
writeAgentFixture(t, control, "#!/bin/sh\nprintf '%s\\n' \"$*\" >> "+shellQuote(log)+"\n")
runner := filepath.Join(dir, "runner")
writeAgentFixture(t, runner, "#!/bin/sh\nshift\n\"$@\" >/dev/null 2>&1 &\n")
if mode == "rejected" {
writeAgentFixture(t, runner, "#!/bin/sh\necho rejected >&2\nexit 9\n")
}
reporter := &WebSocketReporter{upgradeBinaryPath: binary, detectAgentManager: func() (agentServiceManager, error) {
return detectAgentServiceManager(mode == "openrc", mode != "missing", func(name string) (string, error) {
if mode == "missing" {
return "", os.ErrNotExist
}
if name == "systemd-run" {
return runner, nil
}
return control, nil
})
}}
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/checksum" {
fmt.Fprint(w, abcSHA256)
} else {
fmt.Fprint(w, "abc")
}
}))
defer fixture.Close()
response := routeAgentOperation(t, reporter, operation, map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
if mode == "missing" || mode == "rejected" {
if response.Success {
t.Fatal("accepted failed manager launch")
}
if readAgentFixture(t, binary) != "current" {
t.Fatal("live binary mutated")
}
if mode == "missing" && readAgentFixture(t, binary+".old") != "backup" {
t.Fatal("backup mutated before manager validation")
}
matches, _ := filepath.Glob(filepath.Join(dir, ".flux-agent-*"))
if len(matches) != 0 {
t.Fatalf("staging residue %v", matches)
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Fatal("upgrade residue")
}
return
}
if !response.Success {
t.Fatal(response.Message)
}
want, backup := "abc", "current"
if operation == "RollbackAgent" {
want, backup = "backup", "backup"
}
waitAgentFixture(t, binary, want)
events := "stop flux_agent\nstart flux_agent\n"
if mode == "openrc" {
events = "flux_agent stop\nflux_agent start\n"
}
waitAgentFixture(t, log, events)
if readAgentFixture(t, binary+".old") != backup {
t.Fatal("wrong backup")
}
})
}
}
}
func TestAgentRestartScriptFailureOrdering(t *testing.T) {
for _, mode := range []string{"stop", "replace"} {
t.Run(mode, func(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
staged := filepath.Join(dir, "staged")
log := filepath.Join(dir, "events")
writeAgentFixture(t, binary, "current")
writeAgentFixture(t, staged, "replacement")
control := filepath.Join(dir, "rc-service")
script := "#!/bin/sh\necho \"$*\" >> " + shellQuote(log) + "\n"
if mode == "stop" {
script += "[ \"$2\" != stop ]\n"
}
writeAgentFixture(t, control, script)
if mode == "replace" {
writeAgentFixture(t, filepath.Join(dir, "mv"), "#!/bin/sh\nexit 23\n")
t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
}
cmd := exec.Command("/bin/sh", "-c", buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: control}, staged, binary, true))
if err := cmd.Run(); err == nil {
t.Fatal("expected failed stop/replacement")
}
if readAgentFixture(t, binary) != "current" {
t.Fatal("changed binary after failure")
}
want := "flux_agent stop\n"
if mode == "replace" {
want += "flux_agent start\n"
}
if readAgentFixture(t, log) != want {
t.Fatalf("wrong sequence %q", readAgentFixture(t, log))
}
if _, err := os.Stat(staged); !os.IsNotExist(err) {
t.Fatal("staging residue")
}
})
}
}
func TestUpgradeAgentBackupFailurePreservesExistingBackup(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
writeAgentFixture(t, binary+".old", "prior")
called := false
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(string, string) error { called = true; return nil }}
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/checksum" {
fmt.Fprint(w, abcSHA256)
} else {
fmt.Fprint(w, "abc")
}
}))
defer fixture.Close()
response := routeAgentOperation(t, reporter, "UpgradeAgent", map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
if response.Success || called {
t.Fatal("upgrade accepted without backup")
}
if readAgentFixture(t, binary+".old") != "prior" {
t.Fatal("backup lost")
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Fatal("staging residue")
}
}
func TestRollbackAgentRejectsDirectoryDestination(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
if err := os.Mkdir(binary, 0755); err != nil {
t.Fatal(err)
}
writeAgentFixture(t, binary+".old", "backup")
called := false
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(string, string) error { called = true; return nil }}
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
if response.Success || called {
t.Fatal("accepted directory destination")
}
if readAgentFixture(t, binary+".old") != "backup" {
t.Fatal("changed backup")
}
entries, err := os.ReadDir(binary)
if err != nil || len(entries) != 0 {
t.Fatalf("destination changed: %v %v", entries, err)
}
}
func TestAgentRestartScriptRejectsDirectoryDestination(t *testing.T) {
dir := t.TempDir()
binary := filepath.Join(dir, "agent")
staged := filepath.Join(dir, "stage")
if err := os.Mkdir(binary, 0755); err != nil {
t.Fatal(err)
}
writeAgentFixture(t, staged, "replacement")
script := buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: "/usr/bin/true"}, staged, binary, false)
if err := exec.Command("/bin/sh", "-c", script).Run(); err == nil {
t.Fatal("accepted directory destination")
}
entries, err := os.ReadDir(binary)
if err != nil || len(entries) != 0 {
t.Fatalf("binary moved inside directory: %v %v", entries, err)
}
}
func TestOpenRCRestartNeverCleansSystemdLegacyService(t *testing.T) {
script := buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: "rc-service"}, "staged", "binary", true)
for _, forbidden := range []string{"systemctl", "gost", "daemon-reload", "/etc/systemd"} {
if strings.Contains(script, forbidden) {
t.Fatalf("OpenRC helper contains %s: %s", forbidden, script)
}
}
}
+198
View File
@@ -0,0 +1,198 @@
package socket
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
)
// All writes are confined to TempDir; the restart callback never starts a process.
func exerciseUpgrade(t *testing.T, checksum string, serveChecksum http.HandlerFunc, valid bool) {
t.Helper()
dir := t.TempDir()
binary := filepath.Join(dir, "flux_agent")
for path, body := range map[string]string{binary: "old binary", binary + ".old": "prior backup"} {
if err := os.WriteFile(path, []byte(body), 0700); err != nil {
t.Fatal(err)
}
}
downloads := 0
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/binary-"+runtime.GOARCH {
downloads++
fmt.Fprint(w, "abc")
return
}
if r.URL.Path != "/checksum-"+runtime.GOARCH {
t.Errorf("unexpected checksum path: %s", r.URL.Path)
http.NotFound(w, r)
return
}
serveChecksum(w, r)
}))
defer fixture.Close()
if checksum == "fixture" {
checksum = fixture.URL + "/checksum-{ARCH}"
}
peer := make(chan *websocket.Conn, 1)
ws := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
if err != nil {
return
}
peer <- c
}))
defer ws.Close()
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(ws.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
reader := <-peer
defer reader.Close()
restarted := false
reporter := &WebSocketReporter{ctx: context.Background(), conn: conn, connected: true, upgradeBinaryPath: binary,
startUpgradeRestart: func(staged, destination string) error {
restarted = true
if destination != binary {
t.Errorf("wrong restart destination: %s", destination)
}
b, e := os.ReadFile(staged)
if e != nil || string(b) != "abc" {
t.Errorf("staged binary = %q, %v", b, e)
}
return nil
},
}
reporter.routeCommand(CommandMessage{Type: "UpgradeAgent", RequestId: "upgrade-test-request", Data: map[string]any{"downloadUrl": fixture.URL + "/binary-{ARCH}", "checksumUrl": checksum}})
reader.SetReadDeadline(time.Now().Add(2 * time.Second))
verified, installing := false, false
for {
var response CommandResponse
if err := reader.ReadJSON(&response); err != nil {
t.Fatal(err)
}
if response.Type == "UpgradeAgentResponse" {
if response.RequestId != "upgrade-test-request" {
t.Errorf("response request ID = %q", response.RequestId)
}
if response.Success != valid {
t.Errorf("success = %v, want %v: %s", response.Success, valid, response.Message)
}
break
}
if response.Type == "UpgradeProgress" {
data := response.Data.(map[string]interface{})
if data["stage"] == "verifying" && data["percent"] == float64(100) {
verified = true
}
if data["stage"] == "installing" {
installing = true
}
}
}
if restarted != valid || verified != valid || installing != valid {
t.Errorf("restart=%v verified=%v installing=%v, want %v", restarted, verified, installing, valid)
}
b, e := os.ReadFile(binary)
if e != nil || string(b) != "old binary" {
t.Errorf("old binary changed: %q %v", b, e)
}
if !valid {
b, e = os.ReadFile(binary + ".old")
if e != nil || string(b) != "prior backup" {
t.Errorf("backup changed: %q %v", b, e)
}
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
t.Errorf("staged binary remains: %v", err)
}
}
if strings.TrimSpace(checksum) == "" && downloads != 0 {
t.Errorf("downloaded without checksum URL")
}
}
func TestUpgradeAgentRequiresChecksumURL(t *testing.T) {
exerciseUpgrade(t, "", func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected checksum request") }, false)
}
func TestUpgradeAgentRejectsChecksumHTTPFailure(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "unavailable", http.StatusNotFound) }, false)
}
const abcSHA256 = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
func TestUpgradeAgentRejectsAmbiguousChecksum(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, abcSHA256+" gost-amd64\n"+abcSHA256+" gost-arm64\n")
}, false)
}
func TestUpgradeAgentRejectsOversizedChecksum(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, abcSHA256+" "+strings.Repeat("a", 4096)) }, false)
}
func TestUpgradeAgentChecksumRecords(t *testing.T) {
for _, tc := range []struct {
name, body string
valid bool
}{
{"bare", abcSHA256, true},
{"GNU text", abcSHA256 + " gost-arm64\n", true},
{"GNU binary", abcSHA256 + " *gost-arm64\n", true},
{"uppercase CRLF surrounding whitespace", " \t" + strings.ToUpper(abcSHA256) + " gost-arm64\r\n", true},
{"bare surrounding whitespace", "\r\n\t" + abcSHA256 + " \r\n", true},
{"empty", "", false},
{"whitespace", " \r\n\t", false},
{"truncated", abcSHA256[:63], false},
{"nonhex", "z" + abcSHA256[1:], false},
{"mismatch", strings.Repeat("0", 64), false},
{"missing GNU marker", abcSHA256 + " gost-arm64", false},
{"invalid GNU marker", abcSHA256 + " ?gost-arm64", false},
{"missing filename", abcSHA256 + " *", false},
{"appended garbage", abcSHA256 + "garbage", false},
{"embedded CR", abcSHA256 + " gost\rarm64", false},
{"NUL", abcSHA256 + " gost\x00arm64", false},
{"two bare digests", abcSHA256 + "\n" + abcSHA256, false},
} {
t.Run(tc.name, func(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, tc.body) }, tc.valid)
})
}
}
func TestUpgradeAgentRejectsChecksumTransportFailure(t *testing.T) {
exerciseUpgrade(t, "://invalid", nil, false)
}
func TestUpgradeAgentRejectsChecksumReadFailure(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
// A valid-looking record is incomplete relative to the declared body length.
w.Header().Set("Content-Length", "999")
fmt.Fprint(w, abcSHA256)
}, false)
}
func TestUpgradeAgentRejectsBlankChecksumURL(t *testing.T) {
exerciseUpgrade(t, " \t\r\n", nil, false)
}
func TestUpgradeAgentRejectsChecksumConnectionFailure(t *testing.T) {
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
conn, _, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
conn.Close()
}, false)
}
+95 -68
View File
@@ -14,7 +14,6 @@ import (
"net/http"
"net/url"
"os"
"os/exec"
"runtime"
"runtime/debug"
"strconv"
@@ -158,6 +157,11 @@ const (
)
type WebSocketReporter struct {
// Per-reporter upgrade dependencies keep staging and process launch isolated.
upgradeBinaryPath string
startUpgradeRestart func(string, string) error
detectAgentManager func() (agentServiceManager, error)
url string
addr string // 保存服务器地址
secret string // 保存密钥
@@ -1377,6 +1381,23 @@ func (w *WebSocketReporter) sendUpgradeProgress(stage string, percent int, messa
w.sendResponse(response)
}
// parseUpgradeChecksum accepts one bare SHA256 or one GNU sha256sum record.
// A manifest cannot safely identify the requested binary by its first record.
func parseUpgradeChecksum(body []byte) (string, error) {
record := strings.TrimSpace(string(body))
if len(record) < 64 || strings.ContainsAny(record, "\r\n\x00") {
return "", fmt.Errorf("校验文件格式无效: 需要单条 SHA256 记录")
}
digest := record[:64]
if _, err := hex.DecodeString(digest); err != nil {
return "", fmt.Errorf("校验文件格式无效: SHA256 必须为 64 位十六进制")
}
if len(record) != 64 && (len(record) < 67 || record[64] != ' ' || (record[65] != ' ' && record[65] != '*') || strings.TrimSpace(record[66:]) == "") {
return "", fmt.Errorf("校验文件格式无效: 需要单条 GNU SHA256 记录")
}
return digest, nil
}
func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
jsonData, err := json.Marshal(data)
if err != nil {
@@ -1394,6 +1415,10 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
return fmt.Errorf("下载地址不能为空")
}
if strings.TrimSpace(req.ChecksumURL) == "" {
return fmt.Errorf("校验文件地址不能为空")
}
// 替换架构占位符
downloadURL := strings.ReplaceAll(req.DownloadURL, "{ARCH}", runtime.GOARCH)
checksumURL := strings.ReplaceAll(req.ChecksumURL, "{ARCH}", runtime.GOARCH)
@@ -1402,9 +1427,11 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
fmt.Printf("📦 开始下载升级包: %s\n", downloadURL)
// 下载新版本二进制
const binaryPath = "/etc/flux_agent/flux_agent"
binaryPath := w.upgradeBinaryPath
if binaryPath == "" {
binaryPath = "/etc/flux_agent/flux_agent"
}
tmpPath := binaryPath + ".new"
backupPath := binaryPath + ".old"
resp, err := http.Get(downloadURL)
if err != nil {
@@ -1421,6 +1448,14 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
return fmt.Errorf("创建临时文件失败: %v", err)
}
// The restart process owns the staged file only after launch succeeds.
restartStarted := false
defer func() {
if !restartStarted {
_ = os.Remove(tmpPath)
}
}()
// 带进度的下载
totalSize := resp.ContentLength
var downloaded int64
@@ -1433,7 +1468,6 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
if n > 0 {
if _, wErr := outFile.Write(buf[:n]); wErr != nil {
outFile.Close()
os.Remove(tmpPath)
return fmt.Errorf("写入升级包失败: %v", wErr)
}
hasher.Write(buf[:n])
@@ -1451,104 +1485,97 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
break
}
outFile.Close()
os.Remove(tmpPath)
return fmt.Errorf("读取升级包失败: %v", readErr)
}
}
outFile.Close()
if err := outFile.Close(); err != nil {
return fmt.Errorf("关闭升级包失败: %w", err)
}
if downloaded == 0 {
os.Remove(tmpPath)
return fmt.Errorf("下载的升级包为空")
}
w.sendUpgradeProgress("downloading", 100, fmt.Sprintf("下载完成 (%d bytes)", downloaded))
// Checksum 校验
if checksumURL != "" {
w.sendUpgradeProgress("verifying", 0, "校验文件完整性...")
checksumResp, err := http.Get(checksumURL)
if err == nil {
defer checksumResp.Body.Close()
if checksumResp.StatusCode == http.StatusOK {
checksumBody, err := io.ReadAll(checksumResp.Body)
if err == nil {
// 格式: "<hash> <filename>" 或 "<hash>"
expectedHash := strings.TrimSpace(strings.Split(string(checksumBody), " ")[0])
actualHash := hex.EncodeToString(hasher.Sum(nil))
if !strings.EqualFold(expectedHash, actualHash) {
os.Remove(tmpPath)
return fmt.Errorf("校验失败: 期望 %s, 实际 %s", expectedHash, actualHash)
}
fmt.Printf("✅ Checksum 校验通过: %s\n", actualHash)
}
}
}
w.sendUpgradeProgress("verifying", 100, "校验通过")
// Every checksum failure must abort before installation.
w.sendUpgradeProgress("verifying", 0, "校验文件完整性...")
checksumResp, err := http.Get(checksumURL)
if err != nil {
return fmt.Errorf("下载校验文件失败: %w", err)
}
defer checksumResp.Body.Close()
if checksumResp.StatusCode != http.StatusOK {
return fmt.Errorf("下载校验文件失败, HTTP状态码: %d", checksumResp.StatusCode)
}
const maxChecksumBytes = 4096
checksumBody, err := io.ReadAll(io.LimitReader(checksumResp.Body, maxChecksumBytes+1))
if err != nil {
return fmt.Errorf("读取校验文件失败: %w", err)
}
if len(checksumBody) > maxChecksumBytes {
return fmt.Errorf("校验文件过大: 最大 %d bytes", maxChecksumBytes)
}
expectedHash, err := parseUpgradeChecksum(checksumBody)
if err != nil {
return err
}
actualHash := hex.EncodeToString(hasher.Sum(nil))
if !strings.EqualFold(expectedHash, actualHash) {
return fmt.Errorf("校验失败: 期望 %s, 实际 %s", expectedHash, actualHash)
}
w.sendUpgradeProgress("verifying", 100, "校验通过")
if err := os.Chmod(tmpPath, 0755); err != nil {
os.Remove(tmpPath)
return fmt.Errorf("设置执行权限失败: %v", err)
}
// 备份旧版本
w.sendUpgradeProgress("installing", 50, "备份旧版本...")
if _, err := os.Stat(binaryPath); err == nil {
// 复制旧文件作为备份(不用 rename,因为可能正在运行)
oldData, err := os.ReadFile(binaryPath)
if err == nil {
_ = os.WriteFile(backupPath, oldData, 0755)
fmt.Println("📦 旧版本已备份到", backupPath)
}
// Resolve the manager before mutating an existing rollback backup.
startRestart, err := w.agentRestartLauncher(true)
if err != nil {
return err
}
w.sendUpgradeProgress("installing", 50, "备份旧版本...")
if err := backupAgentBinary(binaryPath); err != nil {
return fmt.Errorf("备份旧版本失败: %w", err)
}
w.sendUpgradeProgress("installing", 80, "准备重启...")
fmt.Printf("✅ 升级包下载完成 (%d bytes), 准备重启...\n", downloaded)
// 执行重启脚本
// 使用 systemd-run 在独立的 transient unit 中运行重启脚本,
// 避免 systemctl stop 杀死 flux_agent cgroup 内所有进程(包括此脚本自身)导致 mv 未执行。
script := buildAgentRestartScript(tmpPath, binaryPath)
cmd := exec.Command("systemd-run", "--quiet", "/bin/sh", "-c", script)
if err := cmd.Start(); err != nil {
os.Remove(tmpPath)
if err := startRestart(tmpPath, binaryPath); err != nil {
return fmt.Errorf("启动重启脚本失败: %v", err)
}
restartStarted = true
w.sendUpgradeProgress("installing", 100, "重启中...")
fmt.Println("🔄 重启脚本已启动, Agent 将在 1 秒后重启...")
return nil
}
func (w *WebSocketReporter) handleRollbackAgent(data interface{}) error {
const binaryPath = "/etc/flux_agent/flux_agent"
backupPath := binaryPath + ".old"
// 检查备份文件是否存在
if _, err := os.Stat(backupPath); os.IsNotExist(err) {
return fmt.Errorf("没有可用的备份文件,无法回退")
binaryPath := w.upgradeBinaryPath
if binaryPath == "" {
binaryPath = "/etc/flux_agent/flux_agent"
}
fmt.Println("🔄 开始回退到旧版本...")
// 执行回退脚本(同升级逻辑,使用 systemd-run 避免 cgroup 问题)
script := fmt.Sprintf("sleep 1 && systemctl stop flux_agent && cp %s %s && systemctl start flux_agent", backupPath, binaryPath)
cmd := exec.Command("systemd-run", "--quiet", "/bin/sh", "-c", script)
if err := cmd.Start(); err != nil {
return fmt.Errorf("启动回退脚本失败: %v", err)
startRestart, err := w.agentRestartLauncher(false)
if err != nil {
return err
}
if info, err := os.Stat(binaryPath); err != nil || !info.Mode().IsRegular() {
return fmt.Errorf("回退目标不是可用的普通文件: %s", binaryPath)
}
staged, err := stageAgentCopy(binaryPath+".old", binaryPath)
if err != nil {
return fmt.Errorf("准备回退备份失败: %w", err)
}
if err := startRestart(staged, binaryPath); err != nil {
_ = os.Remove(staged)
return fmt.Errorf("启动回退脚本失败: %w", err)
}
fmt.Println("🔄 回退脚本已启动, Agent 将在 1 秒后重启...")
return nil
}
func buildAgentRestartScript(tmpPath, binaryPath string) string {
return fmt.Sprintf(
"sleep 1 && systemctl stop flux_agent && legacy_service='' && for service_file in /etc/systemd/system/gost.service /lib/systemd/system/gost.service /usr/lib/systemd/system/gost.service; do if [ -f \"$service_file\" ] && grep -Fq \"WorkingDirectory=/etc/gost\" \"$service_file\" && (grep -Fq \"ExecStart=/etc/gost/gost\" \"$service_file\" || (grep -Fq \"ExecStart=/usr/local/bin/gost\" \"$service_file\" && [ -f /etc/gost/config.json ] && [ -f /etc/gost/gost.json ])); then legacy_service=\"$service_file\"; break; fi; done && if [ -n \"$legacy_service\" ]; then (systemctl stop gost 2>/dev/null || true) && (systemctl disable gost 2>/dev/null || true) && rm -f /usr/local/bin/gost /etc/gost/gost \"$legacy_service\" && (systemctl daemon-reload 2>/dev/null || true); fi && mv %s %s && systemctl start flux_agent",
tmpPath,
binaryPath,
)
return buildManagedAgentRestartScript(agentServiceManager{kind: "systemd", control: "systemctl"}, tmpPath, binaryPath, true)
}
// updateLocalConfigJSON 将 http/tls/socks 写入工作目录下的 config.json
+5 -5
View File
@@ -186,16 +186,16 @@ func TestFormatWebSocketDialErrorIncludesHTTPStatus(t *testing.T) {
func TestAgentUpgradeRestartScriptStopsLegacyGostService(t *testing.T) {
script := buildAgentRestartScript("/tmp/flux_agent.new", "/etc/flux_agent/flux_agent")
if !strings.Contains(script, "systemctl stop flux_agent") {
if !strings.Contains(script, "'systemctl' stop flux_agent") {
t.Fatalf("expected script to stop flux_agent, got %s", script)
}
if !strings.Contains(script, "mv /tmp/flux_agent.new /etc/flux_agent/flux_agent") {
if !strings.Contains(script, "mv -f -- '/tmp/flux_agent.new' '/etc/flux_agent/flux_agent'") {
t.Fatalf("expected script to replace the flux_agent binary, got %s", script)
}
if !strings.Contains(script, "systemctl stop gost") {
if !strings.Contains(script, "'systemctl' stop gost") {
t.Fatalf("expected script to stop the legacy gost service, got %s", script)
}
if !strings.Contains(script, "systemctl disable gost") {
if !strings.Contains(script, "'systemctl' disable gost") {
t.Fatalf("expected script to disable the legacy gost service, got %s", script)
}
if !strings.Contains(script, "rm -f /usr/local/bin/gost") {
@@ -204,7 +204,7 @@ func TestAgentUpgradeRestartScriptStopsLegacyGostService(t *testing.T) {
if !strings.Contains(script, "WorkingDirectory=/etc/gost") {
t.Fatalf("expected script to scope cleanup to the legacy FLVX gost service definition, got %s", script)
}
if !strings.Contains(script, "systemctl start flux_agent") {
if !strings.Contains(script, "'systemctl' start flux_agent") {
t.Fatalf("expected script to restart flux_agent, got %s", script)
}
if strings.Contains(script, "systemctl stop flux_agent && systemctl stop gost 2>/dev/null || true") {