mirror of
https://github.com/Sagit-chu/flvx.git
synced 2026-11-17 01:44:23 +08:00
fix(agent): verify upgrades and support OpenRC lifecycle (#570)
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
package socket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type agentServiceManager struct{ kind, control, runner string }
|
||||
|
||||
// Alpine must never select a systemctl compatibility shim. Elsewhere prefer
|
||||
// an active systemd instance, matching the installer, then OpenRC.
|
||||
func detectAgentServiceManager(alpine, systemd bool, lookup func(string) (string, error)) (agentServiceManager, error) {
|
||||
if !alpine && systemd {
|
||||
control, err := lookup("systemctl")
|
||||
if err != nil {
|
||||
return agentServiceManager{}, fmt.Errorf("systemctl 不可用: %w", err)
|
||||
}
|
||||
runner, err := lookup("systemd-run")
|
||||
if err != nil {
|
||||
return agentServiceManager{}, fmt.Errorf("systemd-run 不可用: %w", err)
|
||||
}
|
||||
return agentServiceManager{kind: "systemd", control: control, runner: runner}, nil
|
||||
}
|
||||
control, err := lookup("rc-service")
|
||||
if err != nil {
|
||||
return agentServiceManager{}, fmt.Errorf("未检测到受支持的服务管理器 (systemd/OpenRC): %w", err)
|
||||
}
|
||||
return agentServiceManager{kind: "openrc", control: control}, nil
|
||||
}
|
||||
|
||||
func (w *WebSocketReporter) agentRestartLauncher(cleanLegacy bool) (func(string, string) error, error) {
|
||||
// Existing per-reporter seam is also used by checksum-only tests.
|
||||
if w.startUpgradeRestart != nil {
|
||||
return w.startUpgradeRestart, nil
|
||||
}
|
||||
detect := w.detectAgentManager
|
||||
if detect == nil {
|
||||
detect = func() (agentServiceManager, error) {
|
||||
_, alpineErr := os.Stat("/etc/alpine-release")
|
||||
_, systemdErr := os.Stat("/run/systemd/system")
|
||||
return detectAgentServiceManager(alpineErr == nil, systemdErr == nil, exec.LookPath)
|
||||
}
|
||||
}
|
||||
manager, err := detect()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if manager.kind == "openrc" {
|
||||
if err := checkAgentDetach(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return func(staged, binary string) error {
|
||||
script := buildManagedAgentRestartScript(manager, staged, binary, cleanLegacy)
|
||||
if manager.kind == "systemd" {
|
||||
// Wait only for submission, never for helper completion. A separate unit is
|
||||
// essential: systemctl stop kills every process in the agent's cgroup.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, manager.runner, "--quiet", "/bin/sh", "-c", script).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("提交重启任务失败: %w: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return startDetachedAgentHelper(script)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" }
|
||||
|
||||
func buildManagedAgentRestartScript(manager agentServiceManager, staged, binary string, cleanLegacy bool) string {
|
||||
control := shellQuote(manager.control)
|
||||
stop, start := control+" stop flux_agent", control+" start flux_agent"
|
||||
if manager.kind == "openrc" {
|
||||
stop, start = control+" flux_agent stop", control+" flux_agent start"
|
||||
}
|
||||
legacy := ""
|
||||
if manager.kind == "systemd" && cleanLegacy {
|
||||
legacy = `legacy_service=''; for service_file in /etc/systemd/system/gost.service /lib/systemd/system/gost.service /usr/lib/systemd/system/gost.service; do if [ -f "$service_file" ] && grep -Fq "WorkingDirectory=/etc/gost" "$service_file" && (grep -Fq "ExecStart=/etc/gost/gost" "$service_file" || (grep -Fq "ExecStart=/usr/local/bin/gost" "$service_file" && [ -f /etc/gost/config.json ] && [ -f /etc/gost/gost.json ])); then legacy_service="$service_file"; break; fi; done; if [ -n "$legacy_service" ]; then (` + control + ` stop gost 2>/dev/null || true); (` + control + ` disable gost 2>/dev/null || true); rm -f /usr/local/bin/gost /etc/gost/gost "$legacy_service"; (` + control + ` daemon-reload 2>/dev/null || true); fi; `
|
||||
}
|
||||
// The caller owns staging until launch. Afterwards the helper always removes
|
||||
// it, including stop/replace failures. Never truncate the running executable.
|
||||
return "trap " + shellQuote("rm -f -- "+shellQuote(staged)) + " EXIT; sleep 1; " + stop + " || exit $?; " + legacy +
|
||||
"if [ ! -d " + shellQuote(binary) + " ] && mv -f -- " + shellQuote(staged) + " " + shellQuote(binary) + "; then " + start + "; else status=$?; " + start + "; exit \"$status\"; fi"
|
||||
}
|
||||
|
||||
// Copy to a unique file on the destination filesystem, then let the caller
|
||||
// atomically rename. Failure never truncates a previous usable backup.
|
||||
func stageAgentCopy(source, destination string) (path string, err error) {
|
||||
in, err := os.Open(source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer in.Close()
|
||||
info, err := in.Stat()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", fmt.Errorf("不是普通文件: %s", source)
|
||||
}
|
||||
out, err := os.CreateTemp(filepath.Dir(destination), ".flux-agent-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
path = out.Name()
|
||||
defer func() {
|
||||
out.Close()
|
||||
if err != nil {
|
||||
os.Remove(path)
|
||||
}
|
||||
}()
|
||||
if _, err = io.Copy(out, in); err != nil {
|
||||
return path, err
|
||||
}
|
||||
if err = out.Chmod(0755); err != nil {
|
||||
return path, err
|
||||
}
|
||||
if err = out.Close(); err != nil {
|
||||
return path, err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func backupAgentBinary(binary string) error {
|
||||
staged, err := stageAgentCopy(binary, binary+".old")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(staged)
|
||||
return os.Rename(staged, binary+".old")
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package socket
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func checkAgentDetach() error { return nil }
|
||||
|
||||
func startDetachedAgentHelper(script string) error {
|
||||
cmd := exec.Command("/bin/sh", "-c", script)
|
||||
// The installed OpenRC service stops the agent PID. A separate session also
|
||||
// survives a process-group stop. No inherited stdio or parent-death signal.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { _ = cmd.Wait() }()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package socket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// This subprocess stands in for the running agent. The rc-service double kills
|
||||
// its entire process group, not merely a fake PID, while the real helper runs.
|
||||
func TestAgentDetachedParent(t *testing.T) {
|
||||
dir := os.Getenv("FLVX_TEST_DETACH_ROOT")
|
||||
if dir == "" {
|
||||
t.Skip("subprocess helper")
|
||||
}
|
||||
binary := filepath.Join(dir, "agent")
|
||||
control := filepath.Join(dir, "rc-service")
|
||||
writeAgentFixture(t, control, "#!/bin/sh\nif [ \"$2\" = stop ]; then kill -TERM -"+strconv.Itoa(os.Getpid())+"; else echo started > "+shellQuote(filepath.Join(dir, "started"))+"; fi\n")
|
||||
reporter := &WebSocketReporter{upgradeBinaryPath: binary, detectAgentManager: func() (agentServiceManager, error) { return agentServiceManager{kind: "openrc", control: control}, nil }}
|
||||
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
|
||||
if !response.Success {
|
||||
t.Fatal(response.Message)
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
t.Fatal("original agent was not stopped")
|
||||
}
|
||||
|
||||
func TestOpenRCHelperSurvivesAgentProcessGroupStop(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "agent")
|
||||
writeAgentFixture(t, binary, "current")
|
||||
writeAgentFixture(t, binary+".old", "backup")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestAgentDetachedParent$", "-test.v")
|
||||
cmd.Env = append(os.Environ(), "FLVX_TEST_DETACH_ROOT="+dir)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err == nil || ctx.Err() != nil {
|
||||
t.Fatalf("parent termination = %v, timeout=%v, %s", err, ctx.Err(), output)
|
||||
}
|
||||
exit, ok := err.(*exec.ExitError)
|
||||
if !ok || exit.Sys().(syscall.WaitStatus).Signal() != syscall.SIGTERM {
|
||||
t.Fatalf("parent did not receive SIGTERM: %v %s", err, output)
|
||||
}
|
||||
waitAgentFixture(t, filepath.Join(dir, "started"), "started\n")
|
||||
if readAgentFixture(t, binary) != "backup" || readAgentFixture(t, binary+".old") != "backup" {
|
||||
t.Fatal("rollback failed after parent termination")
|
||||
}
|
||||
fmt.Println("detached OpenRC helper survived original agent process-group SIGTERM")
|
||||
}
|
||||
|
||||
// Run only in an explicitly opted-in disposable Alpine container. The docker
|
||||
// invocation mounts only the test binary, never a host /bin directory. Removing
|
||||
// the container's /bin/sh symlink forces the real detached cmd.Start to fail.
|
||||
func TestOpenRCCommandsCleanStageOnHelperExecFailure(t *testing.T) {
|
||||
if os.Getenv("FLVX_TEST_DISPOSABLE_ALPINE_SHELL_FAILURE") != "1" {
|
||||
t.Skip("requires explicit disposable Alpine container opt-in")
|
||||
}
|
||||
for _, path := range []string{"/etc/alpine-release", "/.dockerenv"} {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("refusing shell mutation outside disposable Alpine: %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
info, err := os.Lstat("/bin/sh")
|
||||
if err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||
t.Fatalf("refusing to rename a non-symlink /bin/sh: %v", err)
|
||||
}
|
||||
saved := "/bin/.flvx-test-sh-" + strconv.Itoa(os.Getpid())
|
||||
if _, err := os.Lstat(saved); !os.IsNotExist(err) {
|
||||
t.Fatalf("refusing to overwrite shell backup: %v", err)
|
||||
}
|
||||
if err := os.Rename("/bin/sh", saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.Rename(saved, "/bin/sh"); err != nil {
|
||||
t.Errorf("restore container shell: %v", err)
|
||||
}
|
||||
}()
|
||||
for _, operation := range []string{"UpgradeAgent", "RollbackAgent"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "agent")
|
||||
writeAgentFixture(t, binary, "current")
|
||||
writeAgentFixture(t, binary+".old", "prior backup")
|
||||
writeAgentFixture(t, filepath.Join(dir, "rc-service"), "#!/bin/sh\nexit 0\n")
|
||||
t.Setenv("PATH", dir)
|
||||
// No manager or launcher injection: Alpine detection and exec.LookPath are
|
||||
// production code too. Only the installation path points to an isolated dir.
|
||||
reporter := &WebSocketReporter{upgradeBinaryPath: binary}
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/checksum" {
|
||||
fmt.Fprint(w, abcSHA256)
|
||||
} else {
|
||||
fmt.Fprint(w, "abc")
|
||||
}
|
||||
}))
|
||||
defer fixture.Close()
|
||||
response := routeAgentOperation(t, reporter, operation, map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
|
||||
if response.Success || !strings.Contains(response.Message, "/bin/sh") || !strings.Contains(response.Message, "no such file") {
|
||||
t.Fatalf("expected real helper exec failure, got %+v", response)
|
||||
}
|
||||
if readAgentFixture(t, binary) != "current" {
|
||||
t.Fatal("live binary changed")
|
||||
}
|
||||
backup := "prior backup"
|
||||
if operation == "UpgradeAgent" {
|
||||
backup = "current"
|
||||
}
|
||||
if readAgentFixture(t, binary+".old") != backup {
|
||||
t.Fatal("unexpected backup contents")
|
||||
}
|
||||
matches, err := filepath.Glob(filepath.Join(dir, ".flux-agent-*"))
|
||||
if err != nil || len(matches) != 0 {
|
||||
t.Fatalf("rollback staging residue: %v %v", matches, err)
|
||||
}
|
||||
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
|
||||
t.Fatalf("upgrade staging residue: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
//go:build !linux
|
||||
|
||||
package socket
|
||||
|
||||
import "fmt"
|
||||
|
||||
func checkAgentDetach() error { return fmt.Errorf("OpenRC Agent 重启仅支持 Linux") }
|
||||
func startDetachedAgentHelper(string) error { return checkAgentDetach() }
|
||||
@@ -0,0 +1,329 @@
|
||||
package socket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func routeAgentOperation(t *testing.T, reporter *WebSocketReporter, operation string, data interface{}) CommandResponse {
|
||||
t.Helper()
|
||||
peer := make(chan *websocket.Conn, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
|
||||
if err == nil {
|
||||
peer <- c
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
c, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
reader := <-peer
|
||||
defer reader.Close()
|
||||
reporter.ctx = context.Background()
|
||||
reporter.conn = c
|
||||
reporter.connected = true
|
||||
reporter.routeCommand(CommandMessage{Type: operation, RequestId: "restart-request", Data: data})
|
||||
reader.SetReadDeadline(time.Now().Add(3 * time.Second))
|
||||
for {
|
||||
var response CommandResponse
|
||||
if err := reader.ReadJSON(&response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Type == operation+"Response" {
|
||||
if response.RequestId != "restart-request" {
|
||||
t.Fatalf("request id lost: %s", response.RequestId)
|
||||
}
|
||||
return response
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackAgentStagesBackupBeforeRestart(t *testing.T) {
|
||||
binary := filepath.Join(t.TempDir(), "flux_agent")
|
||||
for path, body := range map[string]string{binary: "current", binary + ".old": "backup"} {
|
||||
if err := os.WriteFile(path, []byte(body), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
called := false
|
||||
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(staged, dest string) error {
|
||||
called = true
|
||||
body, err := os.ReadFile(staged)
|
||||
if err != nil || string(body) != "backup" || staged == binary+".old" || dest != binary {
|
||||
t.Fatalf("bad rollback staging: %s %s %q %v", staged, dest, body, err)
|
||||
}
|
||||
return os.Rename(staged, dest)
|
||||
}}
|
||||
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
|
||||
if !response.Success || !called {
|
||||
t.Fatalf("rollback failed: called=%v %s", called, response.Message)
|
||||
}
|
||||
body, err := os.ReadFile(binary + ".old")
|
||||
if err != nil || string(body) != "backup" {
|
||||
t.Fatalf("backup consumed: %q %v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentServiceManagerDetection(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
alpine, systemd bool
|
||||
tools string
|
||||
want string
|
||||
}{
|
||||
{"alpine shim", true, true, "rc-service systemctl systemd-run", "openrc"},
|
||||
{"alpine missing openrc", true, true, "systemctl systemd-run", ""},
|
||||
{"systemd", false, true, "rc-service systemctl systemd-run", "systemd"},
|
||||
{"missing runner", false, true, "systemctl", ""},
|
||||
{"openrc fallback", false, false, "rc-service", "openrc"},
|
||||
{"unmanaged", false, false, "systemctl systemd-run", ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
manager, err := detectAgentServiceManager(tc.alpine, tc.systemd, func(name string) (string, error) {
|
||||
if strings.Contains(" "+tc.tools+" ", " "+name+" ") {
|
||||
return name, nil
|
||||
}
|
||||
return "", os.ErrNotExist
|
||||
})
|
||||
if manager.kind != tc.want || (err != nil) != (tc.want == "") {
|
||||
t.Fatalf("manager=%+v err=%v", manager, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeAgentFixture(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func readAgentFixture(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
func waitAgentFixture(t *testing.T, path, want string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
body, _ := os.ReadFile(path)
|
||||
if string(body) == want {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
body, _ := os.ReadFile(path)
|
||||
t.Fatalf("%s = %q, want %q", path, body, want)
|
||||
}
|
||||
|
||||
func TestAgentCommandsManagerLaunch(t *testing.T) {
|
||||
for _, operation := range []string{"UpgradeAgent", "RollbackAgent"} {
|
||||
for _, mode := range []string{"systemd", "openrc", "missing", "rejected"} {
|
||||
t.Run(operation+"/"+mode, func(t *testing.T) {
|
||||
if mode == "openrc" && runtime.GOOS != "linux" {
|
||||
t.Skip("detached launcher is Linux-only")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "flux 'agent")
|
||||
log := filepath.Join(dir, "events")
|
||||
writeAgentFixture(t, binary, "current")
|
||||
writeAgentFixture(t, binary+".old", "backup")
|
||||
control := filepath.Join(dir, "manager")
|
||||
writeAgentFixture(t, control, "#!/bin/sh\nprintf '%s\\n' \"$*\" >> "+shellQuote(log)+"\n")
|
||||
runner := filepath.Join(dir, "runner")
|
||||
writeAgentFixture(t, runner, "#!/bin/sh\nshift\n\"$@\" >/dev/null 2>&1 &\n")
|
||||
if mode == "rejected" {
|
||||
writeAgentFixture(t, runner, "#!/bin/sh\necho rejected >&2\nexit 9\n")
|
||||
}
|
||||
reporter := &WebSocketReporter{upgradeBinaryPath: binary, detectAgentManager: func() (agentServiceManager, error) {
|
||||
return detectAgentServiceManager(mode == "openrc", mode != "missing", func(name string) (string, error) {
|
||||
if mode == "missing" {
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
if name == "systemd-run" {
|
||||
return runner, nil
|
||||
}
|
||||
return control, nil
|
||||
})
|
||||
}}
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/checksum" {
|
||||
fmt.Fprint(w, abcSHA256)
|
||||
} else {
|
||||
fmt.Fprint(w, "abc")
|
||||
}
|
||||
}))
|
||||
defer fixture.Close()
|
||||
response := routeAgentOperation(t, reporter, operation, map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
|
||||
if mode == "missing" || mode == "rejected" {
|
||||
if response.Success {
|
||||
t.Fatal("accepted failed manager launch")
|
||||
}
|
||||
if readAgentFixture(t, binary) != "current" {
|
||||
t.Fatal("live binary mutated")
|
||||
}
|
||||
if mode == "missing" && readAgentFixture(t, binary+".old") != "backup" {
|
||||
t.Fatal("backup mutated before manager validation")
|
||||
}
|
||||
matches, _ := filepath.Glob(filepath.Join(dir, ".flux-agent-*"))
|
||||
if len(matches) != 0 {
|
||||
t.Fatalf("staging residue %v", matches)
|
||||
}
|
||||
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
|
||||
t.Fatal("upgrade residue")
|
||||
}
|
||||
return
|
||||
}
|
||||
if !response.Success {
|
||||
t.Fatal(response.Message)
|
||||
}
|
||||
want, backup := "abc", "current"
|
||||
if operation == "RollbackAgent" {
|
||||
want, backup = "backup", "backup"
|
||||
}
|
||||
waitAgentFixture(t, binary, want)
|
||||
events := "stop flux_agent\nstart flux_agent\n"
|
||||
if mode == "openrc" {
|
||||
events = "flux_agent stop\nflux_agent start\n"
|
||||
}
|
||||
waitAgentFixture(t, log, events)
|
||||
if readAgentFixture(t, binary+".old") != backup {
|
||||
t.Fatal("wrong backup")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentRestartScriptFailureOrdering(t *testing.T) {
|
||||
for _, mode := range []string{"stop", "replace"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "agent")
|
||||
staged := filepath.Join(dir, "staged")
|
||||
log := filepath.Join(dir, "events")
|
||||
writeAgentFixture(t, binary, "current")
|
||||
writeAgentFixture(t, staged, "replacement")
|
||||
control := filepath.Join(dir, "rc-service")
|
||||
script := "#!/bin/sh\necho \"$*\" >> " + shellQuote(log) + "\n"
|
||||
if mode == "stop" {
|
||||
script += "[ \"$2\" != stop ]\n"
|
||||
}
|
||||
writeAgentFixture(t, control, script)
|
||||
if mode == "replace" {
|
||||
writeAgentFixture(t, filepath.Join(dir, "mv"), "#!/bin/sh\nexit 23\n")
|
||||
t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
|
||||
}
|
||||
cmd := exec.Command("/bin/sh", "-c", buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: control}, staged, binary, true))
|
||||
if err := cmd.Run(); err == nil {
|
||||
t.Fatal("expected failed stop/replacement")
|
||||
}
|
||||
if readAgentFixture(t, binary) != "current" {
|
||||
t.Fatal("changed binary after failure")
|
||||
}
|
||||
want := "flux_agent stop\n"
|
||||
if mode == "replace" {
|
||||
want += "flux_agent start\n"
|
||||
}
|
||||
if readAgentFixture(t, log) != want {
|
||||
t.Fatalf("wrong sequence %q", readAgentFixture(t, log))
|
||||
}
|
||||
if _, err := os.Stat(staged); !os.IsNotExist(err) {
|
||||
t.Fatal("staging residue")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradeAgentBackupFailurePreservesExistingBackup(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "agent")
|
||||
writeAgentFixture(t, binary+".old", "prior")
|
||||
called := false
|
||||
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(string, string) error { called = true; return nil }}
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/checksum" {
|
||||
fmt.Fprint(w, abcSHA256)
|
||||
} else {
|
||||
fmt.Fprint(w, "abc")
|
||||
}
|
||||
}))
|
||||
defer fixture.Close()
|
||||
response := routeAgentOperation(t, reporter, "UpgradeAgent", map[string]string{"downloadUrl": fixture.URL + "/binary", "checksumUrl": fixture.URL + "/checksum"})
|
||||
if response.Success || called {
|
||||
t.Fatal("upgrade accepted without backup")
|
||||
}
|
||||
if readAgentFixture(t, binary+".old") != "prior" {
|
||||
t.Fatal("backup lost")
|
||||
}
|
||||
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
|
||||
t.Fatal("staging residue")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackAgentRejectsDirectoryDestination(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "agent")
|
||||
if err := os.Mkdir(binary, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeAgentFixture(t, binary+".old", "backup")
|
||||
called := false
|
||||
reporter := &WebSocketReporter{upgradeBinaryPath: binary, startUpgradeRestart: func(string, string) error { called = true; return nil }}
|
||||
response := routeAgentOperation(t, reporter, "RollbackAgent", nil)
|
||||
if response.Success || called {
|
||||
t.Fatal("accepted directory destination")
|
||||
}
|
||||
if readAgentFixture(t, binary+".old") != "backup" {
|
||||
t.Fatal("changed backup")
|
||||
}
|
||||
entries, err := os.ReadDir(binary)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("destination changed: %v %v", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentRestartScriptRejectsDirectoryDestination(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "agent")
|
||||
staged := filepath.Join(dir, "stage")
|
||||
if err := os.Mkdir(binary, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeAgentFixture(t, staged, "replacement")
|
||||
script := buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: "/usr/bin/true"}, staged, binary, false)
|
||||
if err := exec.Command("/bin/sh", "-c", script).Run(); err == nil {
|
||||
t.Fatal("accepted directory destination")
|
||||
}
|
||||
entries, err := os.ReadDir(binary)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("binary moved inside directory: %v %v", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenRCRestartNeverCleansSystemdLegacyService(t *testing.T) {
|
||||
script := buildManagedAgentRestartScript(agentServiceManager{kind: "openrc", control: "rc-service"}, "staged", "binary", true)
|
||||
for _, forbidden := range []string{"systemctl", "gost", "daemon-reload", "/etc/systemd"} {
|
||||
if strings.Contains(script, forbidden) {
|
||||
t.Fatalf("OpenRC helper contains %s: %s", forbidden, script)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package socket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// All writes are confined to TempDir; the restart callback never starts a process.
|
||||
func exerciseUpgrade(t *testing.T, checksum string, serveChecksum http.HandlerFunc, valid bool) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
binary := filepath.Join(dir, "flux_agent")
|
||||
for path, body := range map[string]string{binary: "old binary", binary + ".old": "prior backup"} {
|
||||
if err := os.WriteFile(path, []byte(body), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
downloads := 0
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/binary-"+runtime.GOARCH {
|
||||
downloads++
|
||||
fmt.Fprint(w, "abc")
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/checksum-"+runtime.GOARCH {
|
||||
t.Errorf("unexpected checksum path: %s", r.URL.Path)
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
serveChecksum(w, r)
|
||||
}))
|
||||
defer fixture.Close()
|
||||
if checksum == "fixture" {
|
||||
checksum = fixture.URL + "/checksum-{ARCH}"
|
||||
}
|
||||
peer := make(chan *websocket.Conn, 1)
|
||||
ws := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
peer <- c
|
||||
}))
|
||||
defer ws.Close()
|
||||
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(ws.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
reader := <-peer
|
||||
defer reader.Close()
|
||||
restarted := false
|
||||
reporter := &WebSocketReporter{ctx: context.Background(), conn: conn, connected: true, upgradeBinaryPath: binary,
|
||||
startUpgradeRestart: func(staged, destination string) error {
|
||||
restarted = true
|
||||
if destination != binary {
|
||||
t.Errorf("wrong restart destination: %s", destination)
|
||||
}
|
||||
b, e := os.ReadFile(staged)
|
||||
if e != nil || string(b) != "abc" {
|
||||
t.Errorf("staged binary = %q, %v", b, e)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
reporter.routeCommand(CommandMessage{Type: "UpgradeAgent", RequestId: "upgrade-test-request", Data: map[string]any{"downloadUrl": fixture.URL + "/binary-{ARCH}", "checksumUrl": checksum}})
|
||||
reader.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
verified, installing := false, false
|
||||
for {
|
||||
var response CommandResponse
|
||||
if err := reader.ReadJSON(&response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Type == "UpgradeAgentResponse" {
|
||||
if response.RequestId != "upgrade-test-request" {
|
||||
t.Errorf("response request ID = %q", response.RequestId)
|
||||
}
|
||||
if response.Success != valid {
|
||||
t.Errorf("success = %v, want %v: %s", response.Success, valid, response.Message)
|
||||
}
|
||||
break
|
||||
}
|
||||
if response.Type == "UpgradeProgress" {
|
||||
data := response.Data.(map[string]interface{})
|
||||
if data["stage"] == "verifying" && data["percent"] == float64(100) {
|
||||
verified = true
|
||||
}
|
||||
if data["stage"] == "installing" {
|
||||
installing = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if restarted != valid || verified != valid || installing != valid {
|
||||
t.Errorf("restart=%v verified=%v installing=%v, want %v", restarted, verified, installing, valid)
|
||||
}
|
||||
b, e := os.ReadFile(binary)
|
||||
if e != nil || string(b) != "old binary" {
|
||||
t.Errorf("old binary changed: %q %v", b, e)
|
||||
}
|
||||
if !valid {
|
||||
b, e = os.ReadFile(binary + ".old")
|
||||
if e != nil || string(b) != "prior backup" {
|
||||
t.Errorf("backup changed: %q %v", b, e)
|
||||
}
|
||||
if _, err := os.Stat(binary + ".new"); !os.IsNotExist(err) {
|
||||
t.Errorf("staged binary remains: %v", err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(checksum) == "" && downloads != 0 {
|
||||
t.Errorf("downloaded without checksum URL")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradeAgentRequiresChecksumURL(t *testing.T) {
|
||||
exerciseUpgrade(t, "", func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected checksum request") }, false)
|
||||
}
|
||||
|
||||
func TestUpgradeAgentRejectsChecksumHTTPFailure(t *testing.T) {
|
||||
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "unavailable", http.StatusNotFound) }, false)
|
||||
}
|
||||
|
||||
const abcSHA256 = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
||||
|
||||
func TestUpgradeAgentRejectsAmbiguousChecksum(t *testing.T) {
|
||||
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, abcSHA256+" gost-amd64\n"+abcSHA256+" gost-arm64\n")
|
||||
}, false)
|
||||
}
|
||||
|
||||
func TestUpgradeAgentRejectsOversizedChecksum(t *testing.T) {
|
||||
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, abcSHA256+" "+strings.Repeat("a", 4096)) }, false)
|
||||
}
|
||||
|
||||
func TestUpgradeAgentChecksumRecords(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, body string
|
||||
valid bool
|
||||
}{
|
||||
{"bare", abcSHA256, true},
|
||||
{"GNU text", abcSHA256 + " gost-arm64\n", true},
|
||||
{"GNU binary", abcSHA256 + " *gost-arm64\n", true},
|
||||
{"uppercase CRLF surrounding whitespace", " \t" + strings.ToUpper(abcSHA256) + " gost-arm64\r\n", true},
|
||||
{"bare surrounding whitespace", "\r\n\t" + abcSHA256 + " \r\n", true},
|
||||
{"empty", "", false},
|
||||
{"whitespace", " \r\n\t", false},
|
||||
{"truncated", abcSHA256[:63], false},
|
||||
{"nonhex", "z" + abcSHA256[1:], false},
|
||||
{"mismatch", strings.Repeat("0", 64), false},
|
||||
{"missing GNU marker", abcSHA256 + " gost-arm64", false},
|
||||
{"invalid GNU marker", abcSHA256 + " ?gost-arm64", false},
|
||||
{"missing filename", abcSHA256 + " *", false},
|
||||
{"appended garbage", abcSHA256 + "garbage", false},
|
||||
{"embedded CR", abcSHA256 + " gost\rarm64", false},
|
||||
{"NUL", abcSHA256 + " gost\x00arm64", false},
|
||||
{"two bare digests", abcSHA256 + "\n" + abcSHA256, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, tc.body) }, tc.valid)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradeAgentRejectsChecksumTransportFailure(t *testing.T) {
|
||||
exerciseUpgrade(t, "://invalid", nil, false)
|
||||
}
|
||||
|
||||
func TestUpgradeAgentRejectsChecksumReadFailure(t *testing.T) {
|
||||
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
|
||||
// A valid-looking record is incomplete relative to the declared body length.
|
||||
w.Header().Set("Content-Length", "999")
|
||||
fmt.Fprint(w, abcSHA256)
|
||||
}, false)
|
||||
}
|
||||
|
||||
func TestUpgradeAgentRejectsBlankChecksumURL(t *testing.T) {
|
||||
exerciseUpgrade(t, " \t\r\n", nil, false)
|
||||
}
|
||||
|
||||
func TestUpgradeAgentRejectsChecksumConnectionFailure(t *testing.T) {
|
||||
exerciseUpgrade(t, "fixture", func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
conn.Close()
|
||||
}, false)
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
@@ -158,6 +157,11 @@ const (
|
||||
)
|
||||
|
||||
type WebSocketReporter struct {
|
||||
// Per-reporter upgrade dependencies keep staging and process launch isolated.
|
||||
upgradeBinaryPath string
|
||||
startUpgradeRestart func(string, string) error
|
||||
detectAgentManager func() (agentServiceManager, error)
|
||||
|
||||
url string
|
||||
addr string // 保存服务器地址
|
||||
secret string // 保存密钥
|
||||
@@ -1377,6 +1381,23 @@ func (w *WebSocketReporter) sendUpgradeProgress(stage string, percent int, messa
|
||||
w.sendResponse(response)
|
||||
}
|
||||
|
||||
// parseUpgradeChecksum accepts one bare SHA256 or one GNU sha256sum record.
|
||||
// A manifest cannot safely identify the requested binary by its first record.
|
||||
func parseUpgradeChecksum(body []byte) (string, error) {
|
||||
record := strings.TrimSpace(string(body))
|
||||
if len(record) < 64 || strings.ContainsAny(record, "\r\n\x00") {
|
||||
return "", fmt.Errorf("校验文件格式无效: 需要单条 SHA256 记录")
|
||||
}
|
||||
digest := record[:64]
|
||||
if _, err := hex.DecodeString(digest); err != nil {
|
||||
return "", fmt.Errorf("校验文件格式无效: SHA256 必须为 64 位十六进制")
|
||||
}
|
||||
if len(record) != 64 && (len(record) < 67 || record[64] != ' ' || (record[65] != ' ' && record[65] != '*') || strings.TrimSpace(record[66:]) == "") {
|
||||
return "", fmt.Errorf("校验文件格式无效: 需要单条 GNU SHA256 记录")
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
@@ -1394,6 +1415,10 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
|
||||
return fmt.Errorf("下载地址不能为空")
|
||||
}
|
||||
|
||||
if strings.TrimSpace(req.ChecksumURL) == "" {
|
||||
return fmt.Errorf("校验文件地址不能为空")
|
||||
}
|
||||
|
||||
// 替换架构占位符
|
||||
downloadURL := strings.ReplaceAll(req.DownloadURL, "{ARCH}", runtime.GOARCH)
|
||||
checksumURL := strings.ReplaceAll(req.ChecksumURL, "{ARCH}", runtime.GOARCH)
|
||||
@@ -1402,9 +1427,11 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
|
||||
fmt.Printf("📦 开始下载升级包: %s\n", downloadURL)
|
||||
|
||||
// 下载新版本二进制
|
||||
const binaryPath = "/etc/flux_agent/flux_agent"
|
||||
binaryPath := w.upgradeBinaryPath
|
||||
if binaryPath == "" {
|
||||
binaryPath = "/etc/flux_agent/flux_agent"
|
||||
}
|
||||
tmpPath := binaryPath + ".new"
|
||||
backupPath := binaryPath + ".old"
|
||||
|
||||
resp, err := http.Get(downloadURL)
|
||||
if err != nil {
|
||||
@@ -1421,6 +1448,14 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
|
||||
return fmt.Errorf("创建临时文件失败: %v", err)
|
||||
}
|
||||
|
||||
// The restart process owns the staged file only after launch succeeds.
|
||||
restartStarted := false
|
||||
defer func() {
|
||||
if !restartStarted {
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
|
||||
// 带进度的下载
|
||||
totalSize := resp.ContentLength
|
||||
var downloaded int64
|
||||
@@ -1433,7 +1468,6 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
|
||||
if n > 0 {
|
||||
if _, wErr := outFile.Write(buf[:n]); wErr != nil {
|
||||
outFile.Close()
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("写入升级包失败: %v", wErr)
|
||||
}
|
||||
hasher.Write(buf[:n])
|
||||
@@ -1451,104 +1485,97 @@ func (w *WebSocketReporter) handleUpgradeAgent(data interface{}) error {
|
||||
break
|
||||
}
|
||||
outFile.Close()
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("读取升级包失败: %v", readErr)
|
||||
}
|
||||
}
|
||||
outFile.Close()
|
||||
if err := outFile.Close(); err != nil {
|
||||
return fmt.Errorf("关闭升级包失败: %w", err)
|
||||
}
|
||||
|
||||
if downloaded == 0 {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("下载的升级包为空")
|
||||
}
|
||||
|
||||
w.sendUpgradeProgress("downloading", 100, fmt.Sprintf("下载完成 (%d bytes)", downloaded))
|
||||
|
||||
// Checksum 校验
|
||||
if checksumURL != "" {
|
||||
w.sendUpgradeProgress("verifying", 0, "校验文件完整性...")
|
||||
checksumResp, err := http.Get(checksumURL)
|
||||
if err == nil {
|
||||
defer checksumResp.Body.Close()
|
||||
if checksumResp.StatusCode == http.StatusOK {
|
||||
checksumBody, err := io.ReadAll(checksumResp.Body)
|
||||
if err == nil {
|
||||
// 格式: "<hash> <filename>" 或 "<hash>"
|
||||
expectedHash := strings.TrimSpace(strings.Split(string(checksumBody), " ")[0])
|
||||
actualHash := hex.EncodeToString(hasher.Sum(nil))
|
||||
if !strings.EqualFold(expectedHash, actualHash) {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("校验失败: 期望 %s, 实际 %s", expectedHash, actualHash)
|
||||
}
|
||||
fmt.Printf("✅ Checksum 校验通过: %s\n", actualHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
w.sendUpgradeProgress("verifying", 100, "校验通过")
|
||||
// Every checksum failure must abort before installation.
|
||||
w.sendUpgradeProgress("verifying", 0, "校验文件完整性...")
|
||||
checksumResp, err := http.Get(checksumURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("下载校验文件失败: %w", err)
|
||||
}
|
||||
defer checksumResp.Body.Close()
|
||||
if checksumResp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("下载校验文件失败, HTTP状态码: %d", checksumResp.StatusCode)
|
||||
}
|
||||
const maxChecksumBytes = 4096
|
||||
checksumBody, err := io.ReadAll(io.LimitReader(checksumResp.Body, maxChecksumBytes+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("读取校验文件失败: %w", err)
|
||||
}
|
||||
if len(checksumBody) > maxChecksumBytes {
|
||||
return fmt.Errorf("校验文件过大: 最大 %d bytes", maxChecksumBytes)
|
||||
}
|
||||
expectedHash, err := parseUpgradeChecksum(checksumBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actualHash := hex.EncodeToString(hasher.Sum(nil))
|
||||
if !strings.EqualFold(expectedHash, actualHash) {
|
||||
return fmt.Errorf("校验失败: 期望 %s, 实际 %s", expectedHash, actualHash)
|
||||
}
|
||||
w.sendUpgradeProgress("verifying", 100, "校验通过")
|
||||
|
||||
if err := os.Chmod(tmpPath, 0755); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("设置执行权限失败: %v", err)
|
||||
}
|
||||
|
||||
// 备份旧版本
|
||||
w.sendUpgradeProgress("installing", 50, "备份旧版本...")
|
||||
if _, err := os.Stat(binaryPath); err == nil {
|
||||
// 复制旧文件作为备份(不用 rename,因为可能正在运行)
|
||||
oldData, err := os.ReadFile(binaryPath)
|
||||
if err == nil {
|
||||
_ = os.WriteFile(backupPath, oldData, 0755)
|
||||
fmt.Println("📦 旧版本已备份到", backupPath)
|
||||
}
|
||||
// Resolve the manager before mutating an existing rollback backup.
|
||||
startRestart, err := w.agentRestartLauncher(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.sendUpgradeProgress("installing", 50, "备份旧版本...")
|
||||
if err := backupAgentBinary(binaryPath); err != nil {
|
||||
return fmt.Errorf("备份旧版本失败: %w", err)
|
||||
}
|
||||
|
||||
w.sendUpgradeProgress("installing", 80, "准备重启...")
|
||||
fmt.Printf("✅ 升级包下载完成 (%d bytes), 准备重启...\n", downloaded)
|
||||
|
||||
// 执行重启脚本
|
||||
// 使用 systemd-run 在独立的 transient unit 中运行重启脚本,
|
||||
// 避免 systemctl stop 杀死 flux_agent cgroup 内所有进程(包括此脚本自身)导致 mv 未执行。
|
||||
script := buildAgentRestartScript(tmpPath, binaryPath)
|
||||
cmd := exec.Command("systemd-run", "--quiet", "/bin/sh", "-c", script)
|
||||
if err := cmd.Start(); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
if err := startRestart(tmpPath, binaryPath); err != nil {
|
||||
return fmt.Errorf("启动重启脚本失败: %v", err)
|
||||
}
|
||||
|
||||
restartStarted = true
|
||||
w.sendUpgradeProgress("installing", 100, "重启中...")
|
||||
fmt.Println("🔄 重启脚本已启动, Agent 将在 1 秒后重启...")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *WebSocketReporter) handleRollbackAgent(data interface{}) error {
|
||||
const binaryPath = "/etc/flux_agent/flux_agent"
|
||||
backupPath := binaryPath + ".old"
|
||||
|
||||
// 检查备份文件是否存在
|
||||
if _, err := os.Stat(backupPath); os.IsNotExist(err) {
|
||||
return fmt.Errorf("没有可用的备份文件,无法回退")
|
||||
binaryPath := w.upgradeBinaryPath
|
||||
if binaryPath == "" {
|
||||
binaryPath = "/etc/flux_agent/flux_agent"
|
||||
}
|
||||
|
||||
fmt.Println("🔄 开始回退到旧版本...")
|
||||
|
||||
// 执行回退脚本(同升级逻辑,使用 systemd-run 避免 cgroup 问题)
|
||||
script := fmt.Sprintf("sleep 1 && systemctl stop flux_agent && cp %s %s && systemctl start flux_agent", backupPath, binaryPath)
|
||||
cmd := exec.Command("systemd-run", "--quiet", "/bin/sh", "-c", script)
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("启动回退脚本失败: %v", err)
|
||||
startRestart, err := w.agentRestartLauncher(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info, err := os.Stat(binaryPath); err != nil || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("回退目标不是可用的普通文件: %s", binaryPath)
|
||||
}
|
||||
staged, err := stageAgentCopy(binaryPath+".old", binaryPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("准备回退备份失败: %w", err)
|
||||
}
|
||||
if err := startRestart(staged, binaryPath); err != nil {
|
||||
_ = os.Remove(staged)
|
||||
return fmt.Errorf("启动回退脚本失败: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("🔄 回退脚本已启动, Agent 将在 1 秒后重启...")
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildAgentRestartScript(tmpPath, binaryPath string) string {
|
||||
return fmt.Sprintf(
|
||||
"sleep 1 && systemctl stop flux_agent && legacy_service='' && for service_file in /etc/systemd/system/gost.service /lib/systemd/system/gost.service /usr/lib/systemd/system/gost.service; do if [ -f \"$service_file\" ] && grep -Fq \"WorkingDirectory=/etc/gost\" \"$service_file\" && (grep -Fq \"ExecStart=/etc/gost/gost\" \"$service_file\" || (grep -Fq \"ExecStart=/usr/local/bin/gost\" \"$service_file\" && [ -f /etc/gost/config.json ] && [ -f /etc/gost/gost.json ])); then legacy_service=\"$service_file\"; break; fi; done && if [ -n \"$legacy_service\" ]; then (systemctl stop gost 2>/dev/null || true) && (systemctl disable gost 2>/dev/null || true) && rm -f /usr/local/bin/gost /etc/gost/gost \"$legacy_service\" && (systemctl daemon-reload 2>/dev/null || true); fi && mv %s %s && systemctl start flux_agent",
|
||||
tmpPath,
|
||||
binaryPath,
|
||||
)
|
||||
return buildManagedAgentRestartScript(agentServiceManager{kind: "systemd", control: "systemctl"}, tmpPath, binaryPath, true)
|
||||
}
|
||||
|
||||
// updateLocalConfigJSON 将 http/tls/socks 写入工作目录下的 config.json
|
||||
|
||||
@@ -186,16 +186,16 @@ func TestFormatWebSocketDialErrorIncludesHTTPStatus(t *testing.T) {
|
||||
func TestAgentUpgradeRestartScriptStopsLegacyGostService(t *testing.T) {
|
||||
script := buildAgentRestartScript("/tmp/flux_agent.new", "/etc/flux_agent/flux_agent")
|
||||
|
||||
if !strings.Contains(script, "systemctl stop flux_agent") {
|
||||
if !strings.Contains(script, "'systemctl' stop flux_agent") {
|
||||
t.Fatalf("expected script to stop flux_agent, got %s", script)
|
||||
}
|
||||
if !strings.Contains(script, "mv /tmp/flux_agent.new /etc/flux_agent/flux_agent") {
|
||||
if !strings.Contains(script, "mv -f -- '/tmp/flux_agent.new' '/etc/flux_agent/flux_agent'") {
|
||||
t.Fatalf("expected script to replace the flux_agent binary, got %s", script)
|
||||
}
|
||||
if !strings.Contains(script, "systemctl stop gost") {
|
||||
if !strings.Contains(script, "'systemctl' stop gost") {
|
||||
t.Fatalf("expected script to stop the legacy gost service, got %s", script)
|
||||
}
|
||||
if !strings.Contains(script, "systemctl disable gost") {
|
||||
if !strings.Contains(script, "'systemctl' disable gost") {
|
||||
t.Fatalf("expected script to disable the legacy gost service, got %s", script)
|
||||
}
|
||||
if !strings.Contains(script, "rm -f /usr/local/bin/gost") {
|
||||
@@ -204,7 +204,7 @@ func TestAgentUpgradeRestartScriptStopsLegacyGostService(t *testing.T) {
|
||||
if !strings.Contains(script, "WorkingDirectory=/etc/gost") {
|
||||
t.Fatalf("expected script to scope cleanup to the legacy FLVX gost service definition, got %s", script)
|
||||
}
|
||||
if !strings.Contains(script, "systemctl start flux_agent") {
|
||||
if !strings.Contains(script, "'systemctl' start flux_agent") {
|
||||
t.Fatalf("expected script to restart flux_agent, got %s", script)
|
||||
}
|
||||
if strings.Contains(script, "systemctl stop flux_agent && systemctl stop gost 2>/dev/null || true") {
|
||||
|
||||
Reference in New Issue
Block a user