[新增] 实现安全兜底配置功能,允许在无历史配置时启动 OpenResty 并返回 503 状态

This commit is contained in:
ryan
2026-05-29 10:07:49 +08:00
parent 79ed8fd6ab
commit 32762fdf3c
9 changed files with 270 additions and 5 deletions
+72 -1
View File
@@ -155,6 +155,25 @@ const (
ApplyStatusFatal ApplyStatus = "fatal"
)
const safeDefaultFallbackMainConfig = `# This file is generated by OpenFlare safe default fallback.
worker_processes auto;
pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
default_type text/plain;
server {
listen 80 default_server;
server_name _;
return 503 "OpenFlare: No Valid Configuration\n";
}
}
`
type ApplyOutcome struct {
Status ApplyStatus
Message string
@@ -219,7 +238,18 @@ func (m *Manager) rollbackAfterFailedApply(ctx context.Context, backup *backupSt
return fatalApplyOutcome(fmt.Errorf("restore openresty backup failed after apply error %v: %w", applyErr, err))
}
if err := m.activateConfig(ctx); err != nil {
return fatalApplyOutcome(fmt.Errorf("apply failed: %v; rollback recovery failed: %w", applyErr, err))
if backup != nil && backup.MainExisted {
return fatalApplyOutcome(fmt.Errorf("apply failed: %v; rollback recovery failed: %w", applyErr, err))
}
if fallbackErr := m.EnsureSafeFallbackRuntime(ctx, fmt.Sprintf("apply failed: %v; rollback recovery failed: %v", applyErr, err)); fallbackErr != nil {
return fatalApplyOutcome(fmt.Errorf("apply failed: %v; rollback recovery failed: %w; fallback recovery failed: %v", applyErr, err, fallbackErr))
}
message := fmt.Sprintf("apply failed, but fallback runtime started: %v; rollback recovery failed: %v", applyErr, err)
slog.Warn("openresty apply recovered with safe default fallback", "message", message)
return ApplyOutcome{
Status: ApplyStatusWarning,
Message: message,
}
}
message := fmt.Sprintf("apply failed, rolled back to previous config: %v", applyErr)
slog.Warn("openresty apply rolled back successfully", "message", message)
@@ -276,6 +306,25 @@ func (m *Manager) EnsureRuntime(ctx context.Context, recreate bool) error {
return m.Executor.EnsureRuntime(ctx, recreate)
}
func (m *Manager) EnsureSafeFallbackRuntime(ctx context.Context, reason string) error {
if m.Executor == nil {
return errors.New("executor 未配置")
}
trimmedReason := strings.TrimSpace(reason)
if trimmedReason == "" {
trimmedReason = "no valid local openresty config is available"
}
slog.Warn("starting openresty safe default fallback runtime", "reason", trimmedReason)
if err := m.writeSafeDefaultFallbackFiles(); err != nil {
return fmt.Errorf("write safe default fallback config failed: %w", err)
}
if err := m.activateConfig(ctx); err != nil {
return fmt.Errorf("activate safe default fallback runtime failed: %w", err)
}
slog.Warn("openresty safe default fallback runtime started", "main_config", m.MainConfigPath, "route_config", m.RouteConfigPath)
return nil
}
func (m *Manager) CheckHealth(ctx context.Context) error {
if m.Executor == nil {
return errors.New("executor 未配置")
@@ -675,6 +724,28 @@ func (m *Manager) restorePowConfig(state *backupState) error {
return os.WriteFile(configPath, []byte(state.PowConfig.Content), 0o644)
}
func (m *Manager) writeSafeDefaultFallbackFiles() error {
if strings.TrimSpace(m.MainConfigPath) == "" {
return errors.New("main config path 不能为空")
}
if strings.TrimSpace(m.RouteConfigPath) == "" {
return errors.New("route config path 不能为空")
}
if err := os.MkdirAll(filepath.Dir(m.MainConfigPath), 0o755); err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(m.RouteConfigPath), 0o755); err != nil {
return err
}
if err := os.WriteFile(m.RouteConfigPath, nil, 0o644); err != nil {
return err
}
if err := os.WriteFile(m.MainConfigPath, []byte(safeDefaultFallbackMainConfig), 0o644); err != nil {
return err
}
return nil
}
func removeLegacyPowConfig(path string) error {
if strings.TrimSpace(path) == "" {
return nil
+50 -1
View File
@@ -29,6 +29,8 @@ type fakeExecutor struct {
}
type scriptedExecutor struct {
testErrors []error
testCalls int
reloadErrors []error
reloadCalls int
}
@@ -62,7 +64,12 @@ func (e *fakeExecutor) Restart(ctx context.Context) error {
}
func (e *scriptedExecutor) Test(ctx context.Context) error {
return nil
index := e.testCalls
e.testCalls++
if index >= len(e.testErrors) {
return nil
}
return e.testErrors[index]
}
func (e *scriptedExecutor) Reload(ctx context.Context) error {
@@ -733,6 +740,48 @@ func TestManagerApplyReturnsWarningWhenRollbackRecoversRuntime(t *testing.T) {
}
}
func TestManagerApplyStartsSafeFallbackWhenNoRollbackConfigExists(t *testing.T) {
tempDir := t.TempDir()
routePath := filepath.Join(tempDir, "routes.conf")
mainPath := filepath.Join(tempDir, "nginx.conf")
executor := &scriptedExecutor{
testErrors: []error{errors.New("target config failed"), errors.New("rollback config missing"), nil},
}
manager := &Manager{
MainConfigPath: mainPath,
RouteConfigPath: routePath,
Executor: executor,
}
outcome := manager.Apply(context.Background(), "bad-main", "bad-route", nil)
if outcome.Status != ApplyStatusWarning {
t.Fatalf("expected warning apply outcome, got %#v", outcome)
}
if !strings.Contains(outcome.Message, "fallback runtime started") {
t.Fatalf("expected fallback message, got %q", outcome.Message)
}
if executor.testCalls != 3 {
t.Fatalf("expected target, rollback, and fallback tests, got %d", executor.testCalls)
}
mainData, err := os.ReadFile(mainPath)
if err != nil {
t.Fatalf("failed to read main config: %v", err)
}
if !strings.Contains(string(mainData), "OpenFlare: No Valid Configuration") {
t.Fatalf("expected safe fallback main config, got %s", string(mainData))
}
if !strings.Contains(string(mainData), "listen 80 default_server") {
t.Fatalf("expected fallback to listen on port 80, got %s", string(mainData))
}
routeData, err := os.ReadFile(routePath)
if err != nil {
t.Fatalf("failed to read route config: %v", err)
}
if len(routeData) != 0 {
t.Fatalf("expected fallback route config to be empty, got %q", string(routeData))
}
}
func TestManagerCertFileTargetPathRejectsEscapes(t *testing.T) {
manager := &Manager{CertDir: filepath.Join(t.TempDir(), "certs")}
if err := os.MkdirAll(manager.CertDir, 0o755); err != nil {
+20
View File
@@ -27,6 +27,7 @@ type ConfigClient interface {
type NginxManager interface {
Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) nginx.ApplyOutcome
EnsureRuntime(ctx context.Context, recreate bool) error
EnsureSafeFallbackRuntime(ctx context.Context, reason string) error
CurrentChecksum() (string, error)
}
@@ -264,10 +265,29 @@ func outcomeError(version string, message string) error {
func (s *Service) ensureRuntimeForCurrentConfig(ctx context.Context, mode string, snapshot *state.Snapshot, currentChecksum string) error {
if strings.TrimSpace(currentChecksum) == "" {
slog.Warn("blocked config cannot be retried and no local checksum is available for runtime recovery", "mode", mode, "blocked_version", snapshot.BlockedVersion)
reason := fmt.Sprintf("blocked config %s has no valid local config available for runtime recovery", strings.TrimSpace(snapshot.BlockedVersion))
if err := s.nginxManager.EnsureSafeFallbackRuntime(ctx, reason); err != nil {
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = err.Error()
_ = s.stateStore.Save(snapshot)
return err
}
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = "safe default fallback runtime started"
return nil
}
slog.Info("ensuring runtime with current local config while active target remains blocked", "mode", mode, "current_version", snapshot.CurrentVersion, "current_checksum", currentChecksum, "blocked_version", snapshot.BlockedVersion)
if err := s.nginxManager.EnsureRuntime(ctx, true); err != nil {
if strings.TrimSpace(snapshot.CurrentChecksum) == "" {
reason := fmt.Sprintf("blocked config %s has no historical config and current local config cannot start: %v", strings.TrimSpace(snapshot.BlockedVersion), err)
if fallbackErr := s.nginxManager.EnsureSafeFallbackRuntime(ctx, reason); fallbackErr == nil {
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = "safe default fallback runtime started"
return nil
} else {
err = fmt.Errorf("%v; fallback recovery failed: %w", err, fallbackErr)
}
}
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = err.Error()
_ = s.stateStore.Save(snapshot)
@@ -28,7 +28,9 @@ type fakeManager struct {
currentChecksum string
currentChecksumErr error
ensureErr error
fallbackErr error
ensureCalls []bool
fallbackReasons []string
applyMainContents []string
applyRouteContents []string
applyFiles [][]protocol.SupportFile
@@ -79,6 +81,11 @@ func (m *fakeManager) EnsureRuntime(ctx context.Context, recreate bool) error {
return m.ensureErr
}
func (m *fakeManager) EnsureSafeFallbackRuntime(ctx context.Context, reason string) error {
m.fallbackReasons = append(m.fallbackReasons, reason)
return m.fallbackErr
}
func (m *fakeManager) CurrentChecksum() (string, error) {
return m.currentChecksum, m.currentChecksumErr
}
@@ -480,6 +487,120 @@ func TestSyncOnStartupKeepsBlockedVersionSuppressedUntilNewTargetArrives(t *test
}
}
func TestSyncOnStartupStartsFallbackWhenBlockedVersionHasNoLocalConfig(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-007",
Checksum: "checksum-7",
MainConfig: "worker_processes 7;",
RouteConfig: "server { listen 87; }",
RenderedConfig: "server { listen 87; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, but fallback runtime started",
OpenrestyStatus: protocol.OpenrestyStatusUnhealthy,
OpenrestyMessage: "apply failed, but fallback runtime started",
LastError: "apply failed, but fallback runtime started",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{}
service := New(client, manager, stateStore)
if err = service.SyncOnStartup(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-007",
Checksum: "checksum-7",
}); err != nil {
t.Fatalf("expected blocked startup target to start fallback, got %v", err)
}
if len(manager.fallbackReasons) != 1 {
t.Fatalf("expected fallback runtime to be started once, got %d", len(manager.fallbackReasons))
}
if client.fetchCalls != 0 {
t.Fatalf("expected blocked startup target to skip fetch, got %d", client.fetchCalls)
}
if len(client.reports) != 0 {
t.Fatal("expected blocked startup target to skip duplicate apply report")
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.BlockedVersion != "20260309-007" || snapshot.BlockedChecksum != "checksum-7" {
t.Fatalf("expected blocked target to remain recorded, got %+v", snapshot)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected fallback startup recovery to mark openresty healthy, got %q", snapshot.OpenrestyStatus)
}
if snapshot.OpenrestyMessage != "safe default fallback runtime started" {
t.Fatalf("expected fallback status message, got %q", snapshot.OpenrestyMessage)
}
}
func TestSyncOnStartupStartsFallbackWhenResidualConfigCannotRecover(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-007",
Checksum: "checksum-7",
MainConfig: "worker_processes 7;",
RouteConfig: "server { listen 87; }",
RenderedConfig: "server { listen 87; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, but fallback runtime started",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{
currentChecksum: "residual-checksum",
ensureErr: context.DeadlineExceeded,
}
service := New(client, manager, stateStore)
if err = service.SyncOnStartup(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-007",
Checksum: "checksum-7",
}); err != nil {
t.Fatalf("expected residual config failure to start fallback, got %v", err)
}
if len(manager.ensureCalls) != 1 {
t.Fatalf("expected residual config to be tested once, got %d", len(manager.ensureCalls))
}
if len(manager.fallbackReasons) != 1 {
t.Fatalf("expected fallback runtime to be started once, got %d", len(manager.fallbackReasons))
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected fallback startup recovery to mark openresty healthy, got %q", snapshot.OpenrestyStatus)
}
if snapshot.BlockedVersion != "20260309-007" || snapshot.BlockedChecksum != "checksum-7" {
t.Fatalf("expected blocked target to remain recorded, got %+v", snapshot)
}
}
func TestSyncOnceClearsBlockedTargetWhenNewVersionArrives(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{