[新增] 实现安全兜底配置功能,允许在无历史配置时启动 OpenResty 并返回 503 状态

This commit is contained in:
ryan
2026-05-29 10:07:49 +08:00
parent 79ed8fd6ab
commit 32762fdf3c
9 changed files with 270 additions and 5 deletions
+1 -1
View File
@@ -6,7 +6,7 @@
</div>
<p align="center">
<p align="center
<a href="https://raw.githubusercontent.com/Rain-kl/OpenFlare/main/LICENSE">
<img src="https://img.shields.io/github/license/Rain-kl/OpenFlare?color=brightgreen" alt="license">
</a>
+1
View File
@@ -17,6 +17,7 @@ services:
environment:
OPENFLARE_SERVER_URL: "http://host.docker.internal:3000"
OPENFLARE_AGENT_TOKEN: "373956188ddead1df6dd7c86cd330b73"
LOG_LEVEL: "debug"
extra_hosts:
- "host.docker.internal:host-gateway"
+2 -1
View File
@@ -239,7 +239,8 @@ Agent 必须满足:
* 写入主配置、路由配置与必要证书文件。
* 写入新配置后执行 `openresty -t -c <main_config_path>`,再 reload;reload 发现运行时未启动时允许直接启动 OpenResty。
* 新配置激活失败时必须先尝试用目标配置恢复运行,再回滚到旧配置并重新拉起 OpenResty。
* 回滚后 OpenResty 恢复正常时上报警告;回滚后仍无法恢复运行时上报失败。
* 回滚后 OpenResty 恢复正常时上报警告;如果本地没有历史主配置可恢复,必须允许写入内置安全兜底配置并拉起只监听 `80` 端口、统一返回 `503` 的 OpenResty 运行态。
* 兜底运行态不得清除失败目标的阻断状态;应用记录必须能体现目标版本失败但 fallback runtime 已启动。存在历史主配置但回滚后仍无法恢复运行时上报失败。
* 某个目标 `version + checksum` 一旦应用失败并回退,Agent 必须在本地状态中阻断该目标的重复应用。
## 前端请求、状态与类型
+1 -1
View File
@@ -58,7 +58,7 @@ Agent 发现新版本后会:
5. reload;如果运行时未启动,则尝试用当前配置启动 OpenResty。
6. 上报成功、警告或失败。
如果新配置激活失败,Agent 必须尝试恢复运行;回滚成功时上报警告,回滚后仍无法恢复运行时上报失败。
如果新配置激活失败,Agent 必须尝试恢复运行;回滚成功时上报警告。若本地没有历史主配置可回滚,Agent 会写入内置安全兜底配置并尝试拉起 OpenResty:该配置只监听 `80` 端口,不包含任何用户路由,统一返回 `503 Service Unavailable` 与 `OpenFlare: No Valid Configuration`。兜底启动成功时仍阻断失败目标版本并上报警告;存在历史主配置但回滚后仍无法恢复运行时上报失败。
某个目标 `version + checksum` 一旦应用失败并回退,Agent 会在本地状态中阻断该目标重复应用。只有远端激活版本或 checksum 发生变化,才允许再次尝试。
+2
View File
@@ -143,6 +143,8 @@ journalctl -u openflare-agent -f
注意:某个目标 `version + checksum` 一旦应用失败并回退,Agent 会在本地状态中阻断该目标重复应用。修正配置后需要重新发布生成新的 checksum,或激活旧版本回滚。
如果这是 Agent 首次应用配置,且本地没有历史 `nginx.conf` 可回滚,失败目标仍会被阻断,但 Agent 会尝试进入安全兜底运行态。此时应用记录和 Agent 日志会包含 `fallback runtime started`,OpenResty 只监听 `80` 端口并统一返回 `503` 与 `OpenFlare: No Valid Configuration`。修正配置并重新发布新版本后,Agent 会覆盖兜底配置并恢复正常代理。
## OpenResty 应用失败
常见原因:
+72 -1
View File
@@ -155,6 +155,25 @@ const (
ApplyStatusFatal ApplyStatus = "fatal"
)
const safeDefaultFallbackMainConfig = `# This file is generated by OpenFlare safe default fallback.
worker_processes auto;
pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
default_type text/plain;
server {
listen 80 default_server;
server_name _;
return 503 "OpenFlare: No Valid Configuration\n";
}
}
`
type ApplyOutcome struct {
Status ApplyStatus
Message string
@@ -219,7 +238,18 @@ func (m *Manager) rollbackAfterFailedApply(ctx context.Context, backup *backupSt
return fatalApplyOutcome(fmt.Errorf("restore openresty backup failed after apply error %v: %w", applyErr, err))
}
if err := m.activateConfig(ctx); err != nil {
return fatalApplyOutcome(fmt.Errorf("apply failed: %v; rollback recovery failed: %w", applyErr, err))
if backup != nil && backup.MainExisted {
return fatalApplyOutcome(fmt.Errorf("apply failed: %v; rollback recovery failed: %w", applyErr, err))
}
if fallbackErr := m.EnsureSafeFallbackRuntime(ctx, fmt.Sprintf("apply failed: %v; rollback recovery failed: %v", applyErr, err)); fallbackErr != nil {
return fatalApplyOutcome(fmt.Errorf("apply failed: %v; rollback recovery failed: %w; fallback recovery failed: %v", applyErr, err, fallbackErr))
}
message := fmt.Sprintf("apply failed, but fallback runtime started: %v; rollback recovery failed: %v", applyErr, err)
slog.Warn("openresty apply recovered with safe default fallback", "message", message)
return ApplyOutcome{
Status: ApplyStatusWarning,
Message: message,
}
}
message := fmt.Sprintf("apply failed, rolled back to previous config: %v", applyErr)
slog.Warn("openresty apply rolled back successfully", "message", message)
@@ -276,6 +306,25 @@ func (m *Manager) EnsureRuntime(ctx context.Context, recreate bool) error {
return m.Executor.EnsureRuntime(ctx, recreate)
}
func (m *Manager) EnsureSafeFallbackRuntime(ctx context.Context, reason string) error {
if m.Executor == nil {
return errors.New("executor 未配置")
}
trimmedReason := strings.TrimSpace(reason)
if trimmedReason == "" {
trimmedReason = "no valid local openresty config is available"
}
slog.Warn("starting openresty safe default fallback runtime", "reason", trimmedReason)
if err := m.writeSafeDefaultFallbackFiles(); err != nil {
return fmt.Errorf("write safe default fallback config failed: %w", err)
}
if err := m.activateConfig(ctx); err != nil {
return fmt.Errorf("activate safe default fallback runtime failed: %w", err)
}
slog.Warn("openresty safe default fallback runtime started", "main_config", m.MainConfigPath, "route_config", m.RouteConfigPath)
return nil
}
func (m *Manager) CheckHealth(ctx context.Context) error {
if m.Executor == nil {
return errors.New("executor 未配置")
@@ -675,6 +724,28 @@ func (m *Manager) restorePowConfig(state *backupState) error {
return os.WriteFile(configPath, []byte(state.PowConfig.Content), 0o644)
}
func (m *Manager) writeSafeDefaultFallbackFiles() error {
if strings.TrimSpace(m.MainConfigPath) == "" {
return errors.New("main config path 不能为空")
}
if strings.TrimSpace(m.RouteConfigPath) == "" {
return errors.New("route config path 不能为空")
}
if err := os.MkdirAll(filepath.Dir(m.MainConfigPath), 0o755); err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(m.RouteConfigPath), 0o755); err != nil {
return err
}
if err := os.WriteFile(m.RouteConfigPath, nil, 0o644); err != nil {
return err
}
if err := os.WriteFile(m.MainConfigPath, []byte(safeDefaultFallbackMainConfig), 0o644); err != nil {
return err
}
return nil
}
func removeLegacyPowConfig(path string) error {
if strings.TrimSpace(path) == "" {
return nil
+50 -1
View File
@@ -29,6 +29,8 @@ type fakeExecutor struct {
}
type scriptedExecutor struct {
testErrors []error
testCalls int
reloadErrors []error
reloadCalls int
}
@@ -62,7 +64,12 @@ func (e *fakeExecutor) Restart(ctx context.Context) error {
}
func (e *scriptedExecutor) Test(ctx context.Context) error {
return nil
index := e.testCalls
e.testCalls++
if index >= len(e.testErrors) {
return nil
}
return e.testErrors[index]
}
func (e *scriptedExecutor) Reload(ctx context.Context) error {
@@ -733,6 +740,48 @@ func TestManagerApplyReturnsWarningWhenRollbackRecoversRuntime(t *testing.T) {
}
}
func TestManagerApplyStartsSafeFallbackWhenNoRollbackConfigExists(t *testing.T) {
tempDir := t.TempDir()
routePath := filepath.Join(tempDir, "routes.conf")
mainPath := filepath.Join(tempDir, "nginx.conf")
executor := &scriptedExecutor{
testErrors: []error{errors.New("target config failed"), errors.New("rollback config missing"), nil},
}
manager := &Manager{
MainConfigPath: mainPath,
RouteConfigPath: routePath,
Executor: executor,
}
outcome := manager.Apply(context.Background(), "bad-main", "bad-route", nil)
if outcome.Status != ApplyStatusWarning {
t.Fatalf("expected warning apply outcome, got %#v", outcome)
}
if !strings.Contains(outcome.Message, "fallback runtime started") {
t.Fatalf("expected fallback message, got %q", outcome.Message)
}
if executor.testCalls != 3 {
t.Fatalf("expected target, rollback, and fallback tests, got %d", executor.testCalls)
}
mainData, err := os.ReadFile(mainPath)
if err != nil {
t.Fatalf("failed to read main config: %v", err)
}
if !strings.Contains(string(mainData), "OpenFlare: No Valid Configuration") {
t.Fatalf("expected safe fallback main config, got %s", string(mainData))
}
if !strings.Contains(string(mainData), "listen 80 default_server") {
t.Fatalf("expected fallback to listen on port 80, got %s", string(mainData))
}
routeData, err := os.ReadFile(routePath)
if err != nil {
t.Fatalf("failed to read route config: %v", err)
}
if len(routeData) != 0 {
t.Fatalf("expected fallback route config to be empty, got %q", string(routeData))
}
}
func TestManagerCertFileTargetPathRejectsEscapes(t *testing.T) {
manager := &Manager{CertDir: filepath.Join(t.TempDir(), "certs")}
if err := os.MkdirAll(manager.CertDir, 0o755); err != nil {
+20
View File
@@ -27,6 +27,7 @@ type ConfigClient interface {
type NginxManager interface {
Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) nginx.ApplyOutcome
EnsureRuntime(ctx context.Context, recreate bool) error
EnsureSafeFallbackRuntime(ctx context.Context, reason string) error
CurrentChecksum() (string, error)
}
@@ -264,10 +265,29 @@ func outcomeError(version string, message string) error {
func (s *Service) ensureRuntimeForCurrentConfig(ctx context.Context, mode string, snapshot *state.Snapshot, currentChecksum string) error {
if strings.TrimSpace(currentChecksum) == "" {
slog.Warn("blocked config cannot be retried and no local checksum is available for runtime recovery", "mode", mode, "blocked_version", snapshot.BlockedVersion)
reason := fmt.Sprintf("blocked config %s has no valid local config available for runtime recovery", strings.TrimSpace(snapshot.BlockedVersion))
if err := s.nginxManager.EnsureSafeFallbackRuntime(ctx, reason); err != nil {
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = err.Error()
_ = s.stateStore.Save(snapshot)
return err
}
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = "safe default fallback runtime started"
return nil
}
slog.Info("ensuring runtime with current local config while active target remains blocked", "mode", mode, "current_version", snapshot.CurrentVersion, "current_checksum", currentChecksum, "blocked_version", snapshot.BlockedVersion)
if err := s.nginxManager.EnsureRuntime(ctx, true); err != nil {
if strings.TrimSpace(snapshot.CurrentChecksum) == "" {
reason := fmt.Sprintf("blocked config %s has no historical config and current local config cannot start: %v", strings.TrimSpace(snapshot.BlockedVersion), err)
if fallbackErr := s.nginxManager.EnsureSafeFallbackRuntime(ctx, reason); fallbackErr == nil {
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = "safe default fallback runtime started"
return nil
} else {
err = fmt.Errorf("%v; fallback recovery failed: %w", err, fallbackErr)
}
}
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = err.Error()
_ = s.stateStore.Save(snapshot)
@@ -28,7 +28,9 @@ type fakeManager struct {
currentChecksum string
currentChecksumErr error
ensureErr error
fallbackErr error
ensureCalls []bool
fallbackReasons []string
applyMainContents []string
applyRouteContents []string
applyFiles [][]protocol.SupportFile
@@ -79,6 +81,11 @@ func (m *fakeManager) EnsureRuntime(ctx context.Context, recreate bool) error {
return m.ensureErr
}
func (m *fakeManager) EnsureSafeFallbackRuntime(ctx context.Context, reason string) error {
m.fallbackReasons = append(m.fallbackReasons, reason)
return m.fallbackErr
}
func (m *fakeManager) CurrentChecksum() (string, error) {
return m.currentChecksum, m.currentChecksumErr
}
@@ -480,6 +487,120 @@ func TestSyncOnStartupKeepsBlockedVersionSuppressedUntilNewTargetArrives(t *test
}
}
func TestSyncOnStartupStartsFallbackWhenBlockedVersionHasNoLocalConfig(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-007",
Checksum: "checksum-7",
MainConfig: "worker_processes 7;",
RouteConfig: "server { listen 87; }",
RenderedConfig: "server { listen 87; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, but fallback runtime started",
OpenrestyStatus: protocol.OpenrestyStatusUnhealthy,
OpenrestyMessage: "apply failed, but fallback runtime started",
LastError: "apply failed, but fallback runtime started",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{}
service := New(client, manager, stateStore)
if err = service.SyncOnStartup(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-007",
Checksum: "checksum-7",
}); err != nil {
t.Fatalf("expected blocked startup target to start fallback, got %v", err)
}
if len(manager.fallbackReasons) != 1 {
t.Fatalf("expected fallback runtime to be started once, got %d", len(manager.fallbackReasons))
}
if client.fetchCalls != 0 {
t.Fatalf("expected blocked startup target to skip fetch, got %d", client.fetchCalls)
}
if len(client.reports) != 0 {
t.Fatal("expected blocked startup target to skip duplicate apply report")
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.BlockedVersion != "20260309-007" || snapshot.BlockedChecksum != "checksum-7" {
t.Fatalf("expected blocked target to remain recorded, got %+v", snapshot)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected fallback startup recovery to mark openresty healthy, got %q", snapshot.OpenrestyStatus)
}
if snapshot.OpenrestyMessage != "safe default fallback runtime started" {
t.Fatalf("expected fallback status message, got %q", snapshot.OpenrestyMessage)
}
}
func TestSyncOnStartupStartsFallbackWhenResidualConfigCannotRecover(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-007",
Checksum: "checksum-7",
MainConfig: "worker_processes 7;",
RouteConfig: "server { listen 87; }",
RenderedConfig: "server { listen 87; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, but fallback runtime started",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{
currentChecksum: "residual-checksum",
ensureErr: context.DeadlineExceeded,
}
service := New(client, manager, stateStore)
if err = service.SyncOnStartup(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-007",
Checksum: "checksum-7",
}); err != nil {
t.Fatalf("expected residual config failure to start fallback, got %v", err)
}
if len(manager.ensureCalls) != 1 {
t.Fatalf("expected residual config to be tested once, got %d", len(manager.ensureCalls))
}
if len(manager.fallbackReasons) != 1 {
t.Fatalf("expected fallback runtime to be started once, got %d", len(manager.fallbackReasons))
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected fallback startup recovery to mark openresty healthy, got %q", snapshot.OpenrestyStatus)
}
if snapshot.BlockedVersion != "20260309-007" || snapshot.BlockedChecksum != "checksum-7" {
t.Fatalf("expected blocked target to remain recorded, got %+v", snapshot)
}
}
func TestSyncOnceClearsBlockedTargetWhenNewVersionArrives(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{