[新增] 实现安全兜底配置功能,允许在无历史配置时启动 OpenResty 并返回 503 状态

This commit is contained in:
ryan
2026-05-29 10:07:49 +08:00
parent 79ed8fd6ab
commit 32762fdf3c
9 changed files with 270 additions and 5 deletions
+20
View File
@@ -27,6 +27,7 @@ type ConfigClient interface {
type NginxManager interface {
Apply(ctx context.Context, mainConfig string, routeConfig string, supportFiles []protocol.SupportFile) nginx.ApplyOutcome
EnsureRuntime(ctx context.Context, recreate bool) error
EnsureSafeFallbackRuntime(ctx context.Context, reason string) error
CurrentChecksum() (string, error)
}
@@ -264,10 +265,29 @@ func outcomeError(version string, message string) error {
func (s *Service) ensureRuntimeForCurrentConfig(ctx context.Context, mode string, snapshot *state.Snapshot, currentChecksum string) error {
if strings.TrimSpace(currentChecksum) == "" {
slog.Warn("blocked config cannot be retried and no local checksum is available for runtime recovery", "mode", mode, "blocked_version", snapshot.BlockedVersion)
reason := fmt.Sprintf("blocked config %s has no valid local config available for runtime recovery", strings.TrimSpace(snapshot.BlockedVersion))
if err := s.nginxManager.EnsureSafeFallbackRuntime(ctx, reason); err != nil {
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = err.Error()
_ = s.stateStore.Save(snapshot)
return err
}
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = "safe default fallback runtime started"
return nil
}
slog.Info("ensuring runtime with current local config while active target remains blocked", "mode", mode, "current_version", snapshot.CurrentVersion, "current_checksum", currentChecksum, "blocked_version", snapshot.BlockedVersion)
if err := s.nginxManager.EnsureRuntime(ctx, true); err != nil {
if strings.TrimSpace(snapshot.CurrentChecksum) == "" {
reason := fmt.Sprintf("blocked config %s has no historical config and current local config cannot start: %v", strings.TrimSpace(snapshot.BlockedVersion), err)
if fallbackErr := s.nginxManager.EnsureSafeFallbackRuntime(ctx, reason); fallbackErr == nil {
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = "safe default fallback runtime started"
return nil
} else {
err = fmt.Errorf("%v; fallback recovery failed: %w", err, fallbackErr)
}
}
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = err.Error()
_ = s.stateStore.Save(snapshot)
@@ -28,7 +28,9 @@ type fakeManager struct {
currentChecksum string
currentChecksumErr error
ensureErr error
fallbackErr error
ensureCalls []bool
fallbackReasons []string
applyMainContents []string
applyRouteContents []string
applyFiles [][]protocol.SupportFile
@@ -79,6 +81,11 @@ func (m *fakeManager) EnsureRuntime(ctx context.Context, recreate bool) error {
return m.ensureErr
}
func (m *fakeManager) EnsureSafeFallbackRuntime(ctx context.Context, reason string) error {
m.fallbackReasons = append(m.fallbackReasons, reason)
return m.fallbackErr
}
func (m *fakeManager) CurrentChecksum() (string, error) {
return m.currentChecksum, m.currentChecksumErr
}
@@ -480,6 +487,120 @@ func TestSyncOnStartupKeepsBlockedVersionSuppressedUntilNewTargetArrives(t *test
}
}
func TestSyncOnStartupStartsFallbackWhenBlockedVersionHasNoLocalConfig(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-007",
Checksum: "checksum-7",
MainConfig: "worker_processes 7;",
RouteConfig: "server { listen 87; }",
RenderedConfig: "server { listen 87; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, but fallback runtime started",
OpenrestyStatus: protocol.OpenrestyStatusUnhealthy,
OpenrestyMessage: "apply failed, but fallback runtime started",
LastError: "apply failed, but fallback runtime started",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{}
service := New(client, manager, stateStore)
if err = service.SyncOnStartup(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-007",
Checksum: "checksum-7",
}); err != nil {
t.Fatalf("expected blocked startup target to start fallback, got %v", err)
}
if len(manager.fallbackReasons) != 1 {
t.Fatalf("expected fallback runtime to be started once, got %d", len(manager.fallbackReasons))
}
if client.fetchCalls != 0 {
t.Fatalf("expected blocked startup target to skip fetch, got %d", client.fetchCalls)
}
if len(client.reports) != 0 {
t.Fatal("expected blocked startup target to skip duplicate apply report")
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.BlockedVersion != "20260309-007" || snapshot.BlockedChecksum != "checksum-7" {
t.Fatalf("expected blocked target to remain recorded, got %+v", snapshot)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected fallback startup recovery to mark openresty healthy, got %q", snapshot.OpenrestyStatus)
}
if snapshot.OpenrestyMessage != "safe default fallback runtime started" {
t.Fatalf("expected fallback status message, got %q", snapshot.OpenrestyMessage)
}
}
func TestSyncOnStartupStartsFallbackWhenResidualConfigCannotRecover(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-007",
Checksum: "checksum-7",
MainConfig: "worker_processes 7;",
RouteConfig: "server { listen 87; }",
RenderedConfig: "server { listen 87; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, but fallback runtime started",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{
currentChecksum: "residual-checksum",
ensureErr: context.DeadlineExceeded,
}
service := New(client, manager, stateStore)
if err = service.SyncOnStartup(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-007",
Checksum: "checksum-7",
}); err != nil {
t.Fatalf("expected residual config failure to start fallback, got %v", err)
}
if len(manager.ensureCalls) != 1 {
t.Fatalf("expected residual config to be tested once, got %d", len(manager.ensureCalls))
}
if len(manager.fallbackReasons) != 1 {
t.Fatalf("expected fallback runtime to be started once, got %d", len(manager.fallbackReasons))
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected fallback startup recovery to mark openresty healthy, got %q", snapshot.OpenrestyStatus)
}
if snapshot.BlockedVersion != "20260309-007" || snapshot.BlockedChecksum != "checksum-7" {
t.Fatalf("expected blocked target to remain recorded, got %+v", snapshot)
}
}
func TestSyncOnceClearsBlockedTargetWhenNewVersionArrives(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{