mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-08 16:46:37 +08:00
[功能] 添加对应用结果的警告支持,优化配置激活和回滚逻辑
This commit is contained in:
@@ -138,6 +138,13 @@ func (e *DockerExecutor) Reload(ctx context.Context) error {
|
||||
}
|
||||
output, err = e.Runner.Run(ctx, e.DockerBinary, "exec", e.ContainerName, dockerRuntimeCommand, "-s", "reload")
|
||||
if err != nil {
|
||||
if e.shouldRecreateAfterReloadFailure(string(output)) {
|
||||
slog.Warn("docker openresty reload failed due to missing mounted files, recreating container", "container", e.ContainerName)
|
||||
if recreateErr := e.EnsureRuntime(ctx, true); recreateErr != nil {
|
||||
return fmt.Errorf("docker exec %s reload failed: %w: %s; recreate failed: %v", dockerRuntimeCommand, err, string(output), recreateErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("docker exec %s reload failed: %w: %s", dockerRuntimeCommand, err, string(output))
|
||||
}
|
||||
return nil
|
||||
@@ -216,6 +223,9 @@ func (e *DockerExecutor) runContainer(ctx context.Context) error {
|
||||
if runErr != nil {
|
||||
return fmt.Errorf("docker run openresty failed: %w: %s", runErr, string(runOutput))
|
||||
}
|
||||
if err := e.CheckHealth(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Info("docker openresty container started", "container", e.ContainerName)
|
||||
return nil
|
||||
}
|
||||
@@ -236,6 +246,28 @@ func (e *DockerExecutor) validateMountSources() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *DockerExecutor) shouldRecreateAfterReloadFailure(output string) bool {
|
||||
text := strings.ToLower(strings.TrimSpace(output))
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
if !strings.Contains(text, "no such file") && !strings.Contains(text, "cannot load certificate") {
|
||||
return false
|
||||
}
|
||||
paths := []string{
|
||||
strings.ToLower(e.NginxCertDir),
|
||||
strings.ToLower(e.NginxLuaDir),
|
||||
strings.ToLower(DockerMainConfigPath),
|
||||
strings.ToLower("/etc/nginx/conf.d"),
|
||||
}
|
||||
for _, path := range paths {
|
||||
if strings.TrimSpace(path) != "" && strings.Contains(text, path) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *DockerExecutor) containerNotRunningError(ctx context.Context) error {
|
||||
inspectSummary := ""
|
||||
inspectOutput, inspectErr := e.Runner.Run(ctx, e.DockerBinary, "inspect", "-f", "status={{.State.Status}} exit_code={{.State.ExitCode}} error={{printf \"%q\" .State.Error}} oom_killed={{.State.OOMKilled}} finished_at={{.State.FinishedAt}}", e.ContainerName)
|
||||
@@ -326,43 +358,89 @@ type Manager struct {
|
||||
Executor Executor
|
||||
}
|
||||
|
||||
func (m *Manager) Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) error {
|
||||
type ApplyStatus string
|
||||
|
||||
const (
|
||||
ApplyStatusSuccess ApplyStatus = "success"
|
||||
ApplyStatusWarning ApplyStatus = "warning"
|
||||
ApplyStatusFatal ApplyStatus = "fatal"
|
||||
)
|
||||
|
||||
type ApplyOutcome struct {
|
||||
Status ApplyStatus
|
||||
Message string
|
||||
}
|
||||
|
||||
func (m *Manager) Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) ApplyOutcome {
|
||||
slog.Info("openresty apply started", "main_config", m.MainConfigPath, "route_config", m.RouteConfigPath, "cert_files", len(supportFiles))
|
||||
backup, err := m.backup()
|
||||
if err != nil {
|
||||
return fatalApplyOutcome(fmt.Errorf("backup openresty config failed: %w", err))
|
||||
}
|
||||
if err = m.writeTargetFiles(mainConfig, routeConfig, supportFiles); err != nil {
|
||||
return m.rollbackAfterFailedApply(ctx, backup, fmt.Errorf("write openresty config failed: %w", err))
|
||||
}
|
||||
if err = m.activateConfig(ctx); err != nil {
|
||||
return m.rollbackAfterFailedApply(ctx, backup, fmt.Errorf("activate openresty runtime failed: %w", err))
|
||||
}
|
||||
slog.Info("openresty apply completed successfully", "main_config", m.MainConfigPath, "route_config", m.RouteConfigPath)
|
||||
return ApplyOutcome{Status: ApplyStatusSuccess}
|
||||
}
|
||||
|
||||
func (m *Manager) writeTargetFiles(mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) error {
|
||||
if err := m.EnsureLuaAssets(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.EnsureLuaAssets(); err != nil {
|
||||
slog.Error("writing lua assets failed, restoring backup", "error", err)
|
||||
_ = m.restore(backup)
|
||||
return err
|
||||
}
|
||||
if err = m.writeCertFiles(supportFiles); err != nil {
|
||||
slog.Error("writing cert files failed, restoring backup", "error", err)
|
||||
_ = m.restore(backup)
|
||||
if err := m.writeCertFiles(supportFiles); err != nil {
|
||||
return err
|
||||
}
|
||||
renderedMainConfig := m.renderMainConfig(mainConfig)
|
||||
if err = os.WriteFile(m.MainConfigPath, []byte(renderedMainConfig), 0o644); err != nil {
|
||||
slog.Error("writing openresty main config failed, restoring backup", "error", err)
|
||||
_ = m.restore(backup)
|
||||
if err := os.WriteFile(m.MainConfigPath, []byte(renderedMainConfig), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
renderedRouteConfig := m.renderRouteConfig(routeConfig)
|
||||
if err = os.WriteFile(m.RouteConfigPath, []byte(renderedRouteConfig), 0o644); err != nil {
|
||||
slog.Error("writing openresty route config failed, restoring backup", "error", err)
|
||||
_ = m.restore(backup)
|
||||
if err := os.WriteFile(m.RouteConfigPath, []byte(renderedRouteConfig), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.Executor.Reload(ctx); err != nil {
|
||||
slog.Error("openresty reload failed after config write, restoring backup", "error", err)
|
||||
_ = m.restore(backup)
|
||||
return err
|
||||
}
|
||||
slog.Info("openresty apply completed successfully", "main_config", m.MainConfigPath, "route_config", m.RouteConfigPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) activateConfig(ctx context.Context) error {
|
||||
if m.Executor == nil {
|
||||
return errors.New("executor 未配置")
|
||||
}
|
||||
if _, ok := m.Executor.(*DockerExecutor); ok {
|
||||
return m.Executor.EnsureRuntime(ctx, true)
|
||||
}
|
||||
return m.Executor.Reload(ctx)
|
||||
}
|
||||
|
||||
func (m *Manager) rollbackAfterFailedApply(ctx context.Context, backup *backupState, applyErr error) ApplyOutcome {
|
||||
slog.Warn("openresty apply failed, restoring previous config", "error", applyErr)
|
||||
if err := m.restore(backup); err != nil {
|
||||
return fatalApplyOutcome(fmt.Errorf("restore openresty backup failed after apply error %v: %w", applyErr, err))
|
||||
}
|
||||
if err := m.activateConfig(ctx); err != nil {
|
||||
return fatalApplyOutcome(fmt.Errorf("apply failed: %v; rollback recovery failed: %w", applyErr, err))
|
||||
}
|
||||
message := fmt.Sprintf("apply failed, rolled back to previous config: %v", applyErr)
|
||||
slog.Warn("openresty apply rolled back successfully", "message", message)
|
||||
return ApplyOutcome{
|
||||
Status: ApplyStatusWarning,
|
||||
Message: message,
|
||||
}
|
||||
}
|
||||
|
||||
func fatalApplyOutcome(err error) ApplyOutcome {
|
||||
if err == nil {
|
||||
return ApplyOutcome{Status: ApplyStatusFatal}
|
||||
}
|
||||
return ApplyOutcome{
|
||||
Status: ApplyStatusFatal,
|
||||
Message: strings.TrimSpace(err.Error()),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) EnsureLuaAssets() error {
|
||||
if strings.TrimSpace(m.LuaDir) == "" {
|
||||
return nil
|
||||
|
||||
@@ -28,6 +28,11 @@ type fakeExecutor struct {
|
||||
reloadErr error
|
||||
}
|
||||
|
||||
type scriptedExecutor struct {
|
||||
reloadErrors []error
|
||||
reloadCalls int
|
||||
}
|
||||
|
||||
func (r *fakeRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
r.calls = append(r.calls, runCall{name: name, args: append([]string{}, args...)})
|
||||
if r.runFn != nil {
|
||||
@@ -56,6 +61,31 @@ func (e *fakeExecutor) Restart(ctx context.Context) error {
|
||||
return e.reloadErr
|
||||
}
|
||||
|
||||
func (e *scriptedExecutor) Test(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *scriptedExecutor) Reload(ctx context.Context) error {
|
||||
index := e.reloadCalls
|
||||
e.reloadCalls++
|
||||
if index >= len(e.reloadErrors) {
|
||||
return nil
|
||||
}
|
||||
return e.reloadErrors[index]
|
||||
}
|
||||
|
||||
func (e *scriptedExecutor) EnsureRuntime(ctx context.Context, recreate bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *scriptedExecutor) CheckHealth(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *scriptedExecutor) Restart(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPathExecutorCommands(t *testing.T) {
|
||||
runner := &fakeRunner{}
|
||||
executor := &PathExecutor{
|
||||
@@ -209,10 +239,15 @@ func TestDockerExecutorStartsContainerWhenMissing(t *testing.T) {
|
||||
|
||||
func TestDockerExecutorStartsStoppedContainer(t *testing.T) {
|
||||
mainConfigPath, routeConfigDir, certDir, luaDir := prepareDockerMountSources(t)
|
||||
inspectCalls := 0
|
||||
runner := &fakeRunner{
|
||||
runFn: func(name string, args ...string) ([]byte, error) {
|
||||
if len(args) >= 2 && args[0] == "inspect" {
|
||||
return []byte("false"), nil
|
||||
inspectCalls++
|
||||
if inspectCalls < 3 {
|
||||
return []byte("false"), nil
|
||||
}
|
||||
return []byte("true"), nil
|
||||
}
|
||||
return []byte("ok"), nil
|
||||
},
|
||||
@@ -234,8 +269,8 @@ func TestDockerExecutorStartsStoppedContainer(t *testing.T) {
|
||||
t.Fatalf("Reload failed: %v", err)
|
||||
}
|
||||
|
||||
if len(runner.calls) != 4 {
|
||||
t.Fatalf("expected 4 calls, got %d", len(runner.calls))
|
||||
if len(runner.calls) != 5 {
|
||||
t.Fatalf("expected 5 calls, got %d", len(runner.calls))
|
||||
}
|
||||
if runner.calls[0].args[0] != "inspect" {
|
||||
t.Fatalf("expected docker inspect on first call, got %#v", runner.calls[0])
|
||||
@@ -249,6 +284,9 @@ func TestDockerExecutorStartsStoppedContainer(t *testing.T) {
|
||||
if runner.calls[3].args[0] != "run" {
|
||||
t.Fatalf("expected docker run on fourth call, got %#v", runner.calls[3])
|
||||
}
|
||||
if runner.calls[4].args[0] != "inspect" {
|
||||
t.Fatalf("expected docker inspect after run, got %#v", runner.calls[4])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerExecutorReloadsRunningContainerInPlace(t *testing.T) {
|
||||
@@ -287,9 +325,55 @@ func TestDockerExecutorReloadsRunningContainerInPlace(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerExecutorReloadRecreatesContainerWhenMountedCertMissing(t *testing.T) {
|
||||
mainConfigPath, routeConfigDir, certDir, luaDir := prepareDockerMountSources(t)
|
||||
runner := &fakeRunner{
|
||||
runFn: func(name string, args ...string) ([]byte, error) {
|
||||
if len(args) >= 1 && args[0] == "inspect" {
|
||||
return []byte("true"), nil
|
||||
}
|
||||
if len(args) >= 2 && args[0] == "exec" {
|
||||
return []byte(`nginx: [emerg] cannot load certificate "/etc/nginx/openflare-certs/1.crt": BIO_new_file() failed (SSL: error:80000002:system library::No such file or directory)`), errors.New("exit status 1")
|
||||
}
|
||||
return []byte("ok"), nil
|
||||
},
|
||||
}
|
||||
executor := &DockerExecutor{
|
||||
DockerBinary: "docker",
|
||||
ContainerName: "openflare-openresty",
|
||||
Image: "openresty/openresty:alpine",
|
||||
MainConfigPath: mainConfigPath,
|
||||
RouteConfigDir: routeConfigDir,
|
||||
CertDir: certDir,
|
||||
NginxCertDir: "/etc/nginx/openflare-certs",
|
||||
LuaDir: luaDir,
|
||||
NginxLuaDir: "/etc/nginx/openflare-lua",
|
||||
OpenrestyObservabilityPort: 18081,
|
||||
Runner: runner,
|
||||
}
|
||||
|
||||
if err := executor.Reload(context.Background()); err != nil {
|
||||
t.Fatalf("Reload failed: %v", err)
|
||||
}
|
||||
|
||||
if len(runner.calls) != 6 {
|
||||
t.Fatalf("expected 6 calls, got %d", len(runner.calls))
|
||||
}
|
||||
if runner.calls[2].args[0] != "inspect" || runner.calls[3].args[0] != "rm" || runner.calls[4].args[0] != "run" || runner.calls[5].args[0] != "inspect" {
|
||||
t.Fatalf("expected recreate after reload failure, got %#v", runner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerExecutorRunContainerMountsManagedFiles(t *testing.T) {
|
||||
mainConfigPath, routeConfigDir, certDir, luaDir := prepareDockerMountSources(t)
|
||||
runner := &fakeRunner{}
|
||||
runner := &fakeRunner{
|
||||
runFn: func(name string, args ...string) ([]byte, error) {
|
||||
if len(args) >= 1 && args[0] == "inspect" {
|
||||
return []byte("true"), nil
|
||||
}
|
||||
return []byte("ok"), nil
|
||||
},
|
||||
}
|
||||
executor := &DockerExecutor{
|
||||
DockerBinary: "docker",
|
||||
ContainerName: "openflare-openresty",
|
||||
@@ -308,8 +392,8 @@ func TestDockerExecutorRunContainerMountsManagedFiles(t *testing.T) {
|
||||
t.Fatalf("runContainer failed: %v", err)
|
||||
}
|
||||
|
||||
if len(runner.calls) != 1 {
|
||||
t.Fatalf("expected one docker run call, got %d", len(runner.calls))
|
||||
if len(runner.calls) != 2 {
|
||||
t.Fatalf("expected docker run plus health check, got %d calls", len(runner.calls))
|
||||
}
|
||||
|
||||
expectedArgs := []string{
|
||||
@@ -327,6 +411,9 @@ func TestDockerExecutorRunContainerMountsManagedFiles(t *testing.T) {
|
||||
if !reflect.DeepEqual(runner.calls[0].args, expectedArgs) {
|
||||
t.Fatalf("unexpected docker run args: %#v", runner.calls[0].args)
|
||||
}
|
||||
if !reflect.DeepEqual(runner.calls[1].args, []string{"inspect", "-f", "{{.State.Running}}", "openflare-openresty"}) {
|
||||
t.Fatalf("unexpected docker health check args: %#v", runner.calls[1].args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerExecutorRecreatesContainerOnStartup(t *testing.T) {
|
||||
@@ -356,8 +443,8 @@ func TestDockerExecutorRecreatesContainerOnStartup(t *testing.T) {
|
||||
if err := executor.EnsureRuntime(context.Background(), true); err != nil {
|
||||
t.Fatalf("EnsureRuntime failed: %v", err)
|
||||
}
|
||||
if len(runner.calls) != 3 {
|
||||
t.Fatalf("expected 3 calls, got %d", len(runner.calls))
|
||||
if len(runner.calls) != 4 {
|
||||
t.Fatalf("expected 4 calls, got %d", len(runner.calls))
|
||||
}
|
||||
if runner.calls[1].args[0] != "rm" {
|
||||
t.Fatalf("expected docker rm on second call, got %#v", runner.calls[1])
|
||||
@@ -365,6 +452,9 @@ func TestDockerExecutorRecreatesContainerOnStartup(t *testing.T) {
|
||||
if runner.calls[2].args[0] != "run" {
|
||||
t.Fatalf("expected docker run on third call, got %#v", runner.calls[2])
|
||||
}
|
||||
if runner.calls[3].args[0] != "inspect" {
|
||||
t.Fatalf("expected docker inspect after run, got %#v", runner.calls[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerExecutorRunContainerRejectsMissingMainConfigFile(t *testing.T) {
|
||||
@@ -497,21 +587,21 @@ func TestManagerApplyAndChecksumIncludeMainConfig(t *testing.T) {
|
||||
Executor: &fakeExecutor{},
|
||||
}
|
||||
|
||||
err := manager.Apply(
|
||||
outcome := manager.Apply(
|
||||
context.Background(),
|
||||
"include __OPENFLARE_ROUTE_CONFIG__;\naccess_log __OPENFLARE_ACCESS_LOG__ openflare_json;\n",
|
||||
"ssl_certificate __OPENFLARE_CERT_DIR__/1.crt;\n",
|
||||
[]protocol.SupportFile{{Path: "1.crt", Content: "cert"}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Apply failed: %v", err)
|
||||
if outcome.Status != ApplyStatusSuccess {
|
||||
t.Fatalf("Apply failed: %#v", outcome)
|
||||
}
|
||||
|
||||
mainData, err := os.ReadFile(mainPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read main config: %v", err)
|
||||
}
|
||||
expectedMain := "include " + routePath + ";\naccess_log " + filepath.Join(filepath.Dir(routePath), "openflare_access.log") + " openflare_json;\n"
|
||||
expectedMain := "include " + routePath + ";\naccess_log " + filepath.ToSlash(filepath.Join(filepath.Dir(routePath), "openflare_access.log")) + " openflare_json;\n"
|
||||
if string(mainData) != expectedMain {
|
||||
t.Fatalf("unexpected main config: %s", string(mainData))
|
||||
}
|
||||
@@ -553,8 +643,8 @@ func TestManagerApplyUsesRuntimeRouteConfigPath(t *testing.T) {
|
||||
Executor: &fakeExecutor{},
|
||||
}
|
||||
|
||||
if err := manager.Apply(context.Background(), "include __OPENFLARE_ROUTE_CONFIG__;\naccess_log __OPENFLARE_ACCESS_LOG__ openflare_json;\n", "server { listen 80; }\n", nil); err != nil {
|
||||
t.Fatalf("Apply failed: %v", err)
|
||||
if outcome := manager.Apply(context.Background(), "include __OPENFLARE_ROUTE_CONFIG__;\naccess_log __OPENFLARE_ACCESS_LOG__ openflare_json;\n", "server { listen 80; }\n", nil); outcome.Status != ApplyStatusSuccess {
|
||||
t.Fatalf("Apply failed: %#v", outcome)
|
||||
}
|
||||
|
||||
mainData, err := os.ReadFile(mainPath)
|
||||
@@ -631,12 +721,12 @@ func TestManagerApplyWritesSupportFilesAndReplacesPlaceholder(t *testing.T) {
|
||||
Executor: &fakeExecutor{},
|
||||
}
|
||||
|
||||
err := manager.Apply(context.Background(), "include __OPENFLARE_ROUTE_CONFIG__;\n__OPENFLARE_RESOLVER_DIRECTIVE__server { listen __OPENFLARE_OBSERVABILITY_LISTEN__; }", "ssl_certificate __OPENFLARE_CERT_DIR__/1.crt;", []protocol.SupportFile{
|
||||
outcome := manager.Apply(context.Background(), "include __OPENFLARE_ROUTE_CONFIG__;\n__OPENFLARE_RESOLVER_DIRECTIVE__server { listen __OPENFLARE_OBSERVABILITY_LISTEN__; }", "ssl_certificate __OPENFLARE_CERT_DIR__/1.crt;", []protocol.SupportFile{
|
||||
{Path: "1.crt", Content: "cert-data"},
|
||||
{Path: "1.key", Content: "key-data"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Apply failed: %v", err)
|
||||
if outcome.Status != ApplyStatusSuccess {
|
||||
t.Fatalf("Apply failed: %#v", outcome)
|
||||
}
|
||||
|
||||
routeData, err := os.ReadFile(manager.RouteConfigPath)
|
||||
@@ -667,7 +757,7 @@ func TestManagerApplyWritesSupportFilesAndReplacesPlaceholder(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("expected managed lua file to exist, stat err = %v", err)
|
||||
}
|
||||
if luaInfo.Mode().Perm() != 0o644 {
|
||||
if runtime.GOOS != "windows" && luaInfo.Mode().Perm() != 0o644 {
|
||||
t.Fatalf("unexpected lua mode: %o", luaInfo.Mode().Perm())
|
||||
}
|
||||
}
|
||||
@@ -802,11 +892,11 @@ func TestManagerRollbackRestoresCertFiles(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
err := manager.Apply(context.Background(), "new-main", "new-route", []protocol.SupportFile{
|
||||
outcome := manager.Apply(context.Background(), "new-main", "new-route", []protocol.SupportFile{
|
||||
{Path: "1.crt", Content: "new-cert"},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected Apply to fail")
|
||||
if outcome.Status != ApplyStatusFatal {
|
||||
t.Fatalf("expected fatal apply outcome, got %#v", outcome)
|
||||
}
|
||||
|
||||
mainData, err := os.ReadFile(mainPath)
|
||||
@@ -832,6 +922,51 @@ func TestManagerRollbackRestoresCertFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerApplyReturnsWarningWhenRollbackRecoversRuntime(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
routePath := filepath.Join(tempDir, "routes.conf")
|
||||
mainPath := filepath.Join(tempDir, "nginx.conf")
|
||||
certDir := filepath.Join(tempDir, "certs")
|
||||
if err := os.MkdirAll(certDir, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll failed: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(mainPath, []byte("old-main"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile failed: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(routePath, []byte("old-route"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile failed: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(certDir, "1.crt"), []byte("old-cert"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile failed: %v", err)
|
||||
}
|
||||
manager := &Manager{
|
||||
MainConfigPath: mainPath,
|
||||
RouteConfigPath: routePath,
|
||||
CertDir: certDir,
|
||||
NginxCertDir: "/etc/nginx/openflare-certs",
|
||||
LuaDir: filepath.Join(tempDir, "lua"),
|
||||
NginxLuaDir: "/etc/nginx/openflare-lua",
|
||||
Executor: &scriptedExecutor{
|
||||
reloadErrors: []error{errors.New("target config failed"), nil},
|
||||
},
|
||||
}
|
||||
|
||||
outcome := manager.Apply(context.Background(), "new-main", "new-route", []protocol.SupportFile{
|
||||
{Path: "1.crt", Content: "new-cert"},
|
||||
})
|
||||
if outcome.Status != ApplyStatusWarning {
|
||||
t.Fatalf("expected warning apply outcome, got %#v", outcome)
|
||||
}
|
||||
|
||||
mainData, err := os.ReadFile(mainPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read main config: %v", err)
|
||||
}
|
||||
if string(mainData) != "old-main" {
|
||||
t.Fatalf("expected main rollback, got %s", string(mainData))
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerCertFileTargetPathRejectsEscapes(t *testing.T) {
|
||||
manager := &Manager{CertDir: filepath.Join(t.TempDir(), "certs")}
|
||||
if err := os.MkdirAll(manager.CertDir, 0o755); err != nil {
|
||||
@@ -883,11 +1018,11 @@ func TestManagerApplyRejectsCertFilePathTraversal(t *testing.T) {
|
||||
Executor: &fakeExecutor{},
|
||||
}
|
||||
|
||||
err := manager.Apply(context.Background(), "main", "route", []protocol.SupportFile{
|
||||
outcome := manager.Apply(context.Background(), "main", "route", []protocol.SupportFile{
|
||||
{Path: "../escape.crt", Content: "bad"},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected Apply to reject traversal path")
|
||||
if outcome.Status != ApplyStatusWarning {
|
||||
t.Fatalf("expected warning apply outcome, got %#v", outcome)
|
||||
}
|
||||
|
||||
if _, statErr := os.Stat(filepath.Join(tempDir, "escape.crt")); !os.IsNotExist(statErr) {
|
||||
|
||||
@@ -4,15 +4,18 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"openflare-agent/internal/nginx"
|
||||
"openflare-agent/internal/protocol"
|
||||
"openflare-agent/internal/state"
|
||||
)
|
||||
|
||||
const (
|
||||
ApplyResultSuccess = "success"
|
||||
ApplyResultWarning = "warning"
|
||||
ApplyResultFailed = "failed"
|
||||
)
|
||||
|
||||
@@ -22,7 +25,7 @@ type ConfigClient interface {
|
||||
}
|
||||
|
||||
type NginxManager interface {
|
||||
Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) error
|
||||
Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) nginx.ApplyOutcome
|
||||
EnsureRuntime(ctx context.Context, recreate bool) error
|
||||
CurrentChecksum() (string, error)
|
||||
}
|
||||
@@ -154,55 +157,79 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
|
||||
mainConfigChecksum := checksumString(config.MainConfig)
|
||||
routeConfigChecksum := checksumString(routeConfig)
|
||||
slog.Info("applying new openresty config", "mode", mode, "from_version", snapshot.CurrentVersion, "to_version", config.Version, "old_checksum", currentChecksum, "new_checksum", config.Checksum)
|
||||
if err := s.nginxManager.Apply(ctx, config.MainConfig, routeConfig, config.SupportFiles); err != nil {
|
||||
slog.Error("apply openresty config failed", "mode", mode, "version", config.Version, "error", err)
|
||||
snapshot.LastError = err.Error()
|
||||
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
|
||||
snapshot.OpenrestyMessage = err.Error()
|
||||
_ = s.stateStore.Save(snapshot)
|
||||
reportErr := s.client.ReportApplyLog(ctx, protocol.ApplyLogPayload{
|
||||
NodeID: snapshot.NodeID,
|
||||
Version: config.Version,
|
||||
Result: ApplyResultFailed,
|
||||
Message: err.Error(),
|
||||
Checksum: config.Checksum,
|
||||
MainConfigChecksum: mainConfigChecksum,
|
||||
RouteConfigChecksum: routeConfigChecksum,
|
||||
SupportFileCount: len(config.SupportFiles),
|
||||
})
|
||||
if reportErr != nil {
|
||||
slog.Error("report failed apply log failed", "version", config.Version, "error", reportErr)
|
||||
return reportErr
|
||||
outcome := s.nginxManager.Apply(ctx, config.MainConfig, routeConfig, config.SupportFiles)
|
||||
message := strings.TrimSpace(outcome.Message)
|
||||
if outcome.Status == "" {
|
||||
outcome.Status = nginx.ApplyStatusFatal
|
||||
if message == "" {
|
||||
message = "openresty apply returned empty outcome"
|
||||
}
|
||||
slog.Warn("failed apply log reported", "version", config.Version)
|
||||
return err
|
||||
}
|
||||
slog.Info("openresty config applied successfully", "mode", mode, "version", config.Version)
|
||||
snapshot.CurrentVersion = config.Version
|
||||
snapshot.CurrentChecksum = config.Checksum
|
||||
snapshot.LastError = ""
|
||||
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
|
||||
snapshot.OpenrestyMessage = ""
|
||||
|
||||
reportResult := ApplyResultFailed
|
||||
switch outcome.Status {
|
||||
case nginx.ApplyStatusSuccess:
|
||||
slog.Info("openresty config applied successfully", "mode", mode, "version", config.Version)
|
||||
snapshot.CurrentVersion = config.Version
|
||||
snapshot.CurrentChecksum = config.Checksum
|
||||
snapshot.LastError = ""
|
||||
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
|
||||
snapshot.OpenrestyMessage = ""
|
||||
reportResult = ApplyResultSuccess
|
||||
if message == "" {
|
||||
message = "apply success"
|
||||
}
|
||||
case nginx.ApplyStatusWarning:
|
||||
if message == "" {
|
||||
message = "apply rolled back to previous config"
|
||||
}
|
||||
slog.Warn("openresty config apply rolled back", "mode", mode, "version", config.Version, "message", message)
|
||||
snapshot.LastError = message
|
||||
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
|
||||
snapshot.OpenrestyMessage = message
|
||||
reportResult = ApplyResultWarning
|
||||
default:
|
||||
if message == "" {
|
||||
message = "openresty apply failed"
|
||||
}
|
||||
slog.Error("apply openresty config failed", "mode", mode, "version", config.Version, "message", message)
|
||||
snapshot.LastError = message
|
||||
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
|
||||
snapshot.OpenrestyMessage = message
|
||||
}
|
||||
|
||||
if err := s.stateStore.Save(snapshot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.client.ReportApplyLog(ctx, protocol.ApplyLogPayload{
|
||||
NodeID: snapshot.NodeID,
|
||||
Version: config.Version,
|
||||
Result: ApplyResultSuccess,
|
||||
Message: "apply success",
|
||||
Result: reportResult,
|
||||
Message: message,
|
||||
Checksum: config.Checksum,
|
||||
MainConfigChecksum: mainConfigChecksum,
|
||||
RouteConfigChecksum: routeConfigChecksum,
|
||||
SupportFileCount: len(config.SupportFiles),
|
||||
}); err != nil {
|
||||
slog.Error("report successful apply log failed", "version", config.Version, "error", err)
|
||||
slog.Error("report apply log failed", "version", config.Version, "result", reportResult, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Debug("successful apply log reported", "version", config.Version)
|
||||
if reportResult == ApplyResultFailed {
|
||||
slog.Warn("failed apply log reported", "version", config.Version)
|
||||
return outcomeError(config.Version, message)
|
||||
}
|
||||
slog.Debug("apply log reported", "version", config.Version, "result", reportResult)
|
||||
return nil
|
||||
}
|
||||
|
||||
func outcomeError(version string, message string) error {
|
||||
trimmed := strings.TrimSpace(message)
|
||||
if trimmed == "" {
|
||||
trimmed = "openresty apply failed"
|
||||
}
|
||||
return fmt.Errorf("apply version %s failed: %s", version, trimmed)
|
||||
}
|
||||
|
||||
func checksumString(content string) string {
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
return hex.EncodeToString(sum[:])
|
||||
|
||||
@@ -24,7 +24,7 @@ type fakeClient struct {
|
||||
}
|
||||
|
||||
type fakeManager struct {
|
||||
applyErr error
|
||||
applyOutcome nginx.ApplyOutcome
|
||||
currentChecksum string
|
||||
currentChecksumErr error
|
||||
ensureErr error
|
||||
@@ -64,11 +64,14 @@ func (f *fakeClient) ReportApplyLog(ctx context.Context, payload protocol.ApplyL
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *fakeManager) Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) error {
|
||||
func (m *fakeManager) Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) nginx.ApplyOutcome {
|
||||
m.applyMainContents = append(m.applyMainContents, mainConfig)
|
||||
m.applyRouteContents = append(m.applyRouteContents, routeConfig)
|
||||
m.applyFiles = append(m.applyFiles, append([]protocol.SupportFile(nil), supportFiles...))
|
||||
return m.applyErr
|
||||
if m.applyOutcome.Status == "" {
|
||||
return nginx.ApplyOutcome{Status: nginx.ApplyStatusSuccess}
|
||||
}
|
||||
return m.applyOutcome
|
||||
}
|
||||
|
||||
func (m *fakeManager) EnsureRuntime(ctx context.Context, recreate bool) error {
|
||||
@@ -166,17 +169,7 @@ func TestSyncOnceRollbackOnNginxFailure(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
tempDir := t.TempDir()
|
||||
mainPath := filepath.Join(tempDir, "nginx.conf")
|
||||
routePath := filepath.Join(tempDir, "routes.conf")
|
||||
if err := os.WriteFile(mainPath, []byte("worker_processes auto;"), 0o644); err != nil {
|
||||
t.Fatalf("failed to seed main file: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(routePath, []byte("server { listen 80; }"), 0o644); err != nil {
|
||||
t.Fatalf("failed to seed route file: %v", err)
|
||||
}
|
||||
|
||||
stateStore := state.NewStore(filepath.Join(tempDir, "state.json"))
|
||||
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
||||
nodeID, err := stateStore.EnsureNodeID()
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureNodeID failed: %v", err)
|
||||
@@ -189,11 +182,10 @@ func TestSyncOnceRollbackOnNginxFailure(t *testing.T) {
|
||||
t.Fatalf("failed to seed state: %v", err)
|
||||
}
|
||||
|
||||
service := New(client, &nginx.Manager{
|
||||
MainConfigPath: mainPath,
|
||||
RouteConfigPath: routePath,
|
||||
Executor: &fakeExecutor{
|
||||
testErr: context.DeadlineExceeded,
|
||||
service := New(client, &fakeManager{
|
||||
applyOutcome: nginx.ApplyOutcome{
|
||||
Status: nginx.ApplyStatusFatal,
|
||||
Message: "openresty failed after rollback",
|
||||
},
|
||||
}, stateStore)
|
||||
|
||||
@@ -202,22 +194,7 @@ func TestSyncOnceRollbackOnNginxFailure(t *testing.T) {
|
||||
Checksum: client.config.Checksum,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected SyncOnce to fail when nginx test fails")
|
||||
}
|
||||
|
||||
data, readErr := os.ReadFile(routePath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("failed to read route file after rollback: %v", readErr)
|
||||
}
|
||||
if string(data) != "server { listen 80; }" {
|
||||
t.Fatal("expected original route config to be restored after rollback")
|
||||
}
|
||||
mainData, readErr := os.ReadFile(mainPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("failed to read main file after rollback: %v", readErr)
|
||||
}
|
||||
if string(mainData) != "worker_processes auto;" {
|
||||
t.Fatal("expected original main config to be restored after rollback")
|
||||
t.Fatal("expected SyncOnce to fail when apply outcome is fatal")
|
||||
}
|
||||
snapshot, loadErr := stateStore.Load()
|
||||
if loadErr != nil {
|
||||
@@ -226,6 +203,9 @@ func TestSyncOnceRollbackOnNginxFailure(t *testing.T) {
|
||||
if snapshot.CurrentVersion != "20260309-001" {
|
||||
t.Fatal("expected failed sync not to overwrite current version")
|
||||
}
|
||||
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusUnhealthy {
|
||||
t.Fatalf("expected unhealthy openresty status, got %q", snapshot.OpenrestyStatus)
|
||||
}
|
||||
if len(client.reports) != 1 || client.reports[0].Result != ApplyResultFailed {
|
||||
t.Fatal("expected failed apply report to be sent")
|
||||
}
|
||||
@@ -240,6 +220,64 @@ func TestSyncOnceRollbackOnNginxFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncOnceReportsWarningWhenRollbackKeepsOpenrestyHealthy(t *testing.T) {
|
||||
client := &fakeClient{
|
||||
config: protocol.ActiveConfigResponse{
|
||||
Version: "20260309-002",
|
||||
Checksum: "checksum-2",
|
||||
MainConfig: "worker_processes 2;",
|
||||
RouteConfig: "server { listen 81; }",
|
||||
RenderedConfig: "server { listen 81; }",
|
||||
SupportFiles: []protocol.SupportFile{{Path: "1.crt", Content: "cert"}},
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
},
|
||||
}
|
||||
|
||||
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
||||
nodeID, err := stateStore.EnsureNodeID()
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureNodeID failed: %v", err)
|
||||
}
|
||||
if err = stateStore.Save(&state.Snapshot{
|
||||
NodeID: nodeID,
|
||||
CurrentVersion: "20260309-001",
|
||||
CurrentChecksum: "checksum-1",
|
||||
}); err != nil {
|
||||
t.Fatalf("failed to seed state: %v", err)
|
||||
}
|
||||
|
||||
service := New(client, &fakeManager{
|
||||
applyOutcome: nginx.ApplyOutcome{
|
||||
Status: nginx.ApplyStatusWarning,
|
||||
Message: "apply failed, rolled back to previous config",
|
||||
},
|
||||
}, stateStore)
|
||||
|
||||
if err = service.SyncOnce(context.Background(), &protocol.ActiveConfigMeta{
|
||||
Version: client.config.Version,
|
||||
Checksum: client.config.Checksum,
|
||||
}); err != nil {
|
||||
t.Fatalf("expected warning outcome to keep sync successful, got %v", err)
|
||||
}
|
||||
|
||||
snapshot, err := stateStore.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load state: %v", err)
|
||||
}
|
||||
if snapshot.CurrentVersion != "20260309-001" || snapshot.CurrentChecksum != "checksum-1" {
|
||||
t.Fatal("expected warning apply to keep previous version state")
|
||||
}
|
||||
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
|
||||
t.Fatalf("expected healthy openresty after rollback, got %q", snapshot.OpenrestyStatus)
|
||||
}
|
||||
if snapshot.LastError == "" {
|
||||
t.Fatal("expected rollback warning to be recorded")
|
||||
}
|
||||
if len(client.reports) != 1 || client.reports[0].Result != ApplyResultWarning {
|
||||
t.Fatal("expected warning apply report to be sent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncOnStartupRecreatesRuntimeWhenChecksumMatches(t *testing.T) {
|
||||
client := &fakeClient{
|
||||
config: protocol.ActiveConfigResponse{
|
||||
|
||||
Reference in New Issue
Block a user