[功能] 添加节点健康事件清理功能,优化节点观测数据管理

This commit is contained in:
ryan
2026-03-18 23:11:48 +08:00
parent 4945b8b44f
commit 782304012c
16 changed files with 3473 additions and 433 deletions
+1
View File
@@ -173,6 +173,7 @@ Origin
* 主配置、路由配置、证书与 Lua 资源写入
* 执行 `openresty -t` / `openresty -s reload`
* 失败回滚
* 对已失败并回退的目标版本做本地熔断,直到控制面出现新的激活版本
* 节点观测采集与结果上报
### 7.3 `openflare_server/web`
+1
View File
@@ -192,6 +192,7 @@ Agent 必须满足:
* 写入新配置后以运行态恢复为目标执行激活,Docker 模式优先重建容器并确认容器保持运行
* 新配置激活失败时必须先尝试用目标配置恢复运行,再回滚到旧配置并重新拉起 OpenResty
* 回滚后 OpenResty 恢复正常时上报警告;回滚后仍无法恢复运行时上报失败
* 某个目标 `version + checksum` 一旦应用失败并回退,Agent 必须在本地状态中阻断该目标的重复应用;只有远端激活版本或 checksum 发生变化时,才允许再次尝试
## 6. 测试与交付要求
+3
View File
@@ -13,6 +13,9 @@ type Snapshot struct {
NodeID string `json:"node_id"`
CurrentVersion string `json:"current_version"`
CurrentChecksum string `json:"current_checksum"`
BlockedVersion string `json:"blocked_version"`
BlockedChecksum string `json:"blocked_checksum"`
BlockedReason string `json:"blocked_reason"`
LastError string `json:"last_error"`
OpenrestyStatus string `json:"openresty_status"`
OpenrestyMessage string `json:"openresty_message"`
+83 -2
View File
@@ -105,13 +105,27 @@ func (s *Service) sync(ctx context.Context, startup bool, target *protocol.Activ
}
snapshot.CurrentVersion = target.Version
snapshot.CurrentChecksum = target.Checksum
clearBlockedTarget(snapshot)
snapshot.LastError = ""
slog.Debug("sync finished without changes", "mode", mode, "version", target.Version)
return s.stateStore.Save(snapshot)
}
if isBlockedTarget(snapshot, target.Version, target.Checksum) {
slog.Warn("skipping blocked config version after previous failed apply", "mode", mode, "version", target.Version, "checksum", target.Checksum)
if startup {
if err = s.ensureRuntimeForCurrentConfig(ctx, mode, snapshot, currentChecksum); err != nil {
return err
}
return s.stateStore.Save(snapshot)
}
return nil
}
if hasBlockedTarget(snapshot) {
clearBlockedTarget(snapshot)
}
if snapshot.CurrentVersion == target.Version && snapshot.CurrentChecksum == target.Checksum && !startup {
slog.Debug("skipping config fetch because state already records target version/checksum", "version", target.Version, "checksum", target.Checksum)
return nil
return s.stateStore.Save(snapshot)
}
config, err := s.client.GetActiveConfig(ctx)
@@ -139,6 +153,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
}
snapshot.CurrentVersion = config.Version
snapshot.CurrentChecksum = config.Checksum
clearBlockedTarget(snapshot)
snapshot.LastError = ""
slog.Debug("sync finished without changes", "mode", mode, "version", config.Version)
return s.stateStore.Save(snapshot)
@@ -146,9 +161,22 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
if target != nil && (target.Version != config.Version || target.Checksum != config.Checksum) {
slog.Warn("active config changed between heartbeat and fetch", "heartbeat_version", target.Version, "heartbeat_checksum", target.Checksum, "fetched_version", config.Version, "fetched_checksum", config.Checksum)
}
if isBlockedTarget(snapshot, config.Version, config.Checksum) {
slog.Warn("skipping blocked config after fetch because the same version previously failed", "mode", mode, "version", config.Version, "checksum", config.Checksum)
if startup {
if err := s.ensureRuntimeForCurrentConfig(ctx, mode, snapshot, currentChecksum); err != nil {
return err
}
return s.stateStore.Save(snapshot)
}
return nil
}
if hasBlockedTarget(snapshot) {
clearBlockedTarget(snapshot)
}
if snapshot.CurrentVersion == config.Version && snapshot.CurrentChecksum == config.Checksum && !startup {
slog.Debug("skipping apply because state already records target version/checksum", "version", config.Version, "checksum", config.Checksum)
return nil
return s.stateStore.Save(snapshot)
}
routeConfig := config.RouteConfig
if routeConfig == "" {
@@ -172,6 +200,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
slog.Info("openresty config applied successfully", "mode", mode, "version", config.Version)
snapshot.CurrentVersion = config.Version
snapshot.CurrentChecksum = config.Checksum
clearBlockedTarget(snapshot)
snapshot.LastError = ""
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = ""
@@ -184,6 +213,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
message = "apply rolled back to previous config"
}
slog.Warn("openresty config apply rolled back", "mode", mode, "version", config.Version, "message", message)
markBlockedTarget(snapshot, config.Version, config.Checksum, message)
snapshot.LastError = message
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
snapshot.OpenrestyMessage = message
@@ -193,6 +223,7 @@ func (s *Service) applyIfNeeded(ctx context.Context, mode string, startup bool,
message = "openresty apply failed"
}
slog.Error("apply openresty config failed", "mode", mode, "version", config.Version, "message", message)
markBlockedTarget(snapshot, config.Version, config.Checksum, message)
snapshot.LastError = message
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = message
@@ -230,6 +261,56 @@ func outcomeError(version string, message string) error {
return fmt.Errorf("apply version %s failed: %s", version, trimmed)
}
func (s *Service) ensureRuntimeForCurrentConfig(ctx context.Context, mode string, snapshot *state.Snapshot, currentChecksum string) error {
if strings.TrimSpace(currentChecksum) == "" {
slog.Warn("blocked config cannot be retried and no local checksum is available for runtime recovery", "mode", mode, "blocked_version", snapshot.BlockedVersion)
return nil
}
slog.Info("ensuring runtime with current local config while active target remains blocked", "mode", mode, "current_version", snapshot.CurrentVersion, "current_checksum", currentChecksum, "blocked_version", snapshot.BlockedVersion)
if err := s.nginxManager.EnsureRuntime(ctx, true); err != nil {
snapshot.OpenrestyStatus = protocol.OpenrestyStatusUnhealthy
snapshot.OpenrestyMessage = err.Error()
_ = s.stateStore.Save(snapshot)
return err
}
snapshot.OpenrestyStatus = protocol.OpenrestyStatusHealthy
if strings.TrimSpace(snapshot.OpenrestyMessage) == strings.TrimSpace(snapshot.BlockedReason) {
snapshot.OpenrestyMessage = ""
}
return nil
}
func markBlockedTarget(snapshot *state.Snapshot, version string, checksum string, reason string) {
if snapshot == nil {
return
}
snapshot.BlockedVersion = strings.TrimSpace(version)
snapshot.BlockedChecksum = strings.TrimSpace(checksum)
snapshot.BlockedReason = strings.TrimSpace(reason)
}
func clearBlockedTarget(snapshot *state.Snapshot) {
if snapshot == nil {
return
}
snapshot.BlockedVersion = ""
snapshot.BlockedChecksum = ""
snapshot.BlockedReason = ""
}
func hasBlockedTarget(snapshot *state.Snapshot) bool {
return snapshot != nil && (strings.TrimSpace(snapshot.BlockedVersion) != "" || strings.TrimSpace(snapshot.BlockedChecksum) != "")
}
func isBlockedTarget(snapshot *state.Snapshot, version string, checksum string) bool {
if snapshot == nil {
return false
}
return strings.TrimSpace(snapshot.BlockedVersion) == strings.TrimSpace(version) &&
strings.TrimSpace(snapshot.BlockedChecksum) == strings.TrimSpace(checksum) &&
(strings.TrimSpace(version) != "" || strings.TrimSpace(checksum) != "")
}
func checksumString(content string) string {
sum := sha256.Sum256([]byte(content))
return hex.EncodeToString(sum[:])
@@ -203,6 +203,9 @@ func TestSyncOnceRollbackOnNginxFailure(t *testing.T) {
if snapshot.CurrentVersion != "20260309-001" {
t.Fatal("expected failed sync not to overwrite current version")
}
if snapshot.BlockedVersion != "20260309-002" || snapshot.BlockedChecksum != "checksum-2" {
t.Fatalf("expected failed target version to be blocked, got %+v", snapshot)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusUnhealthy {
t.Fatalf("expected unhealthy openresty status, got %q", snapshot.OpenrestyStatus)
}
@@ -267,6 +270,9 @@ func TestSyncOnceReportsWarningWhenRollbackKeepsOpenrestyHealthy(t *testing.T) {
if snapshot.CurrentVersion != "20260309-001" || snapshot.CurrentChecksum != "checksum-1" {
t.Fatal("expected warning apply to keep previous version state")
}
if snapshot.BlockedVersion != "20260309-002" || snapshot.BlockedChecksum != "checksum-2" {
t.Fatalf("expected rolled-back target version to be blocked, got %+v", snapshot)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected healthy openresty after rollback, got %q", snapshot.OpenrestyStatus)
}
@@ -368,6 +374,165 @@ func TestSyncOnStartupRecordsRuntimeFailure(t *testing.T) {
}
}
func TestSyncOnceSkipsPreviouslyBlockedVersion(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-006",
Checksum: "checksum-6",
MainConfig: "worker_processes 6;",
RouteConfig: "server { listen 86; }",
RenderedConfig: "server { listen 86; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
CurrentVersion: "20260309-005",
CurrentChecksum: "checksum-5",
BlockedVersion: "20260309-006",
BlockedChecksum: "checksum-6",
BlockedReason: "apply failed, rolled back to previous config",
LastError: "apply failed, rolled back to previous config",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{currentChecksum: "checksum-5"}
service := New(client, manager, stateStore)
if err = service.SyncOnce(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-006",
Checksum: "checksum-6",
}); err != nil {
t.Fatalf("expected blocked version to be skipped, got %v", err)
}
if client.fetchCalls != 0 {
t.Fatalf("expected blocked version to skip fetch, got %d", client.fetchCalls)
}
if len(manager.applyMainContents) != 0 {
t.Fatal("expected blocked version to skip apply")
}
if len(client.reports) != 0 {
t.Fatal("expected blocked version to skip reporting duplicate apply result")
}
}
func TestSyncOnStartupKeepsBlockedVersionSuppressedUntilNewTargetArrives(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-007",
Checksum: "checksum-7",
MainConfig: "worker_processes 7;",
RouteConfig: "server { listen 87; }",
RenderedConfig: "server { listen 87; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
CurrentVersion: "20260309-005",
CurrentChecksum: "checksum-5",
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, rolled back to previous config",
OpenrestyStatus: protocol.OpenrestyStatusUnhealthy,
OpenrestyMessage: "apply failed, rolled back to previous config",
LastError: "apply failed, rolled back to previous config",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{currentChecksum: "checksum-5"}
service := New(client, manager, stateStore)
if err = service.SyncOnStartup(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-007",
Checksum: "checksum-7",
}); err != nil {
t.Fatalf("expected blocked startup target to be skipped, got %v", err)
}
if len(manager.ensureCalls) != 1 || !manager.ensureCalls[0] {
t.Fatal("expected startup skip to ensure runtime with current local config")
}
if client.fetchCalls != 0 {
t.Fatalf("expected blocked startup target to skip fetch, got %d", client.fetchCalls)
}
if len(client.reports) != 0 {
t.Fatal("expected blocked startup target to skip duplicate apply report")
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.BlockedVersion != "20260309-007" || snapshot.BlockedChecksum != "checksum-7" {
t.Fatalf("expected blocked target to remain recorded, got %+v", snapshot)
}
if snapshot.OpenrestyStatus != protocol.OpenrestyStatusHealthy {
t.Fatalf("expected startup runtime recovery to mark openresty healthy, got %q", snapshot.OpenrestyStatus)
}
}
func TestSyncOnceClearsBlockedTargetWhenNewVersionArrives(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
Version: "20260309-008",
Checksum: "checksum-8",
MainConfig: "worker_processes 8;",
RouteConfig: "server { listen 88; }",
RenderedConfig: "server { listen 88; }",
CreatedAt: time.Now().Format(time.RFC3339),
},
}
stateStore := state.NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID, err := stateStore.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
if err = stateStore.Save(&state.Snapshot{
NodeID: nodeID,
CurrentVersion: "20260309-005",
CurrentChecksum: "checksum-5",
BlockedVersion: "20260309-007",
BlockedChecksum: "checksum-7",
BlockedReason: "apply failed, rolled back to previous config",
}); err != nil {
t.Fatalf("failed to seed state: %v", err)
}
manager := &fakeManager{}
service := New(client, manager, stateStore)
if err = service.SyncOnce(context.Background(), &protocol.ActiveConfigMeta{
Version: "20260309-008",
Checksum: "checksum-8",
}); err != nil {
t.Fatalf("expected new target version to be applied, got %v", err)
}
if client.fetchCalls != 1 {
t.Fatalf("expected new target to trigger fetch, got %d", client.fetchCalls)
}
if len(manager.applyMainContents) != 1 {
t.Fatal("expected new target to trigger apply")
}
snapshot, err := stateStore.Load()
if err != nil {
t.Fatalf("failed to load state: %v", err)
}
if snapshot.BlockedVersion != "" || snapshot.BlockedChecksum != "" {
t.Fatalf("expected blocked target to be cleared after new version succeeds, got %+v", snapshot)
}
if snapshot.CurrentVersion != "20260309-008" || snapshot.CurrentChecksum != "checksum-8" {
t.Fatalf("expected current version to move to new target, got %+v", snapshot)
}
}
func TestSyncOnceSkipsFetchWhenHeartbeatChecksumMatches(t *testing.T) {
client := &fakeClient{
config: protocol.ActiveConfigResponse{
+24
View File
@@ -247,3 +247,27 @@ func GetNodeObservability(c *gin.Context) {
}
respondSuccess(c, view)
}
// CleanupNodeHealthEvents godoc
// @Summary Cleanup node health events
// @Tags Nodes
// @Produce json
// @Security BearerAuth
// @Param id path int true "Node ID"
// @Success 200 {object} map[string]interface{}
// @Failure 400 {object} map[string]interface{}
// @Router /api/nodes/{id}/observability/cleanup [post]
func CleanupNodeHealthEvents(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil || id == 0 {
respondBadRequest(c, "")
return
}
result, err := service.CleanupNodeHealthEvents(uint(id))
if err != nil {
respondFailure(c, err.Error())
return
}
respondSuccess(c, result)
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+692 -70
View File
@@ -1,5 +1,12 @@
basePath: /
definitions:
controller.geoIPLookupRequest:
properties:
ip:
type: string
provider:
type: string
type: object
model.Option:
properties:
key:
@@ -7,12 +14,94 @@ definitions:
value:
type: string
type: object
service.AgentBufferedObservabilityRecord:
properties:
access_logs:
items:
$ref: '#/definitions/service.AgentNodeAccessLog'
type: array
snapshot:
$ref: '#/definitions/service.AgentNodeMetricSnapshot'
traffic_report:
$ref: '#/definitions/service.AgentNodeTrafficReport'
window_started_at_unix:
type: integer
type: object
service.AgentNodeAccessLog:
properties:
host:
type: string
logged_at_unix:
type: integer
path:
type: string
remote_addr:
type: string
status_code:
type: integer
type: object
service.AgentNodeHealthEvent:
properties:
event_type:
type: string
message:
type: string
metadata:
additionalProperties:
type: string
type: object
severity:
type: string
triggered_at_unix:
type: integer
type: object
service.AgentNodeMetricSnapshot:
properties:
captured_at_unix:
type: integer
cpu_usage_percent:
type: number
disk_read_bytes:
type: integer
disk_write_bytes:
type: integer
memory_total_bytes:
type: integer
memory_used_bytes:
type: integer
network_rx_bytes:
type: integer
network_tx_bytes:
type: integer
openresty_connections:
type: integer
openresty_rx_bytes:
type: integer
openresty_tx_bytes:
type: integer
storage_total_bytes:
type: integer
storage_used_bytes:
type: integer
type: object
service.AgentNodePayload:
properties:
access_logs:
items:
$ref: '#/definitions/service.AgentNodeAccessLog'
type: array
agent_version:
type: string
buffered_observability:
items:
$ref: '#/definitions/service.AgentBufferedObservabilityRecord'
type: array
current_version:
type: string
health_events:
items:
$ref: '#/definitions/service.AgentNodeHealthEvent'
type: array
ip:
type: string
last_error:
@@ -27,15 +116,82 @@ definitions:
type: string
openresty_status:
type: string
profile:
$ref: '#/definitions/service.AgentNodeSystemProfile'
snapshot:
$ref: '#/definitions/service.AgentNodeMetricSnapshot'
traffic_report:
$ref: '#/definitions/service.AgentNodeTrafficReport'
type: object
service.AgentNodeSystemProfile:
properties:
architecture:
type: string
cpu_cores:
type: integer
cpu_model:
type: string
hostname:
type: string
kernel_version:
type: string
os_name:
type: string
os_version:
type: string
reported_at_unix:
type: integer
total_disk_bytes:
type: integer
total_memory_bytes:
type: integer
uptime_seconds:
type: integer
type: object
service.AgentNodeTrafficReport:
properties:
error_count:
type: integer
request_count:
type: integer
source_countries:
additionalProperties:
format: int64
type: integer
type: object
status_codes:
additionalProperties:
format: int64
type: integer
type: object
top_domains:
additionalProperties:
format: int64
type: integer
type: object
unique_visitor_count:
type: integer
window_ended_at_unix:
type: integer
window_started_at_unix:
type: integer
type: object
service.ApplyLogPayload:
properties:
checksum:
type: string
main_config_checksum:
type: string
message:
type: string
node_id:
type: string
result:
type: string
route_config_checksum:
type: string
support_file_count:
type: integer
version:
type: string
type: object
@@ -54,6 +210,16 @@ definitions:
properties:
auto_update_enabled:
type: boolean
geo_latitude:
type: number
geo_longitude:
type: number
geo_manual_override:
type: boolean
geo_name:
type: string
ip:
type: string
name:
type: string
type: object
@@ -66,6 +232,14 @@ definitions:
type: object
service.ProxyRouteInput:
properties:
cache_enabled:
type: boolean
cache_policy:
type: string
cache_rules:
items:
type: string
type: array
cert_id:
type: integer
custom_headers:
@@ -78,14 +252,18 @@ definitions:
type: boolean
enabled:
type: boolean
origin_url:
type: string
origin_host:
type: string
origin_url:
type: string
redirect_http:
type: boolean
remark:
type: string
upstreams:
items:
type: string
type: array
type: object
service.TLSCertificateInput:
properties:
@@ -104,6 +282,204 @@ info:
title: OpenFlare Server API
version: "3.0"
paths:
/api/access-logs/:
get:
parameters:
- description: Node ID
in: query
name: node_id
type: string
- description: Remote address
in: query
name: remote_addr
type: string
- description: Host
in: query
name: host
type: string
- description: Path
in: query
name: path
type: string
- description: Page index
in: query
name: p
type: integer
- description: Page size
in: query
name: page_size
type: integer
- description: Sort by
in: query
name: sort_by
type: string
- description: Sort order
in: query
name: sort_order
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: List access logs
tags:
- AccessLogs
/api/access-logs/cleanup:
post:
consumes:
- application/json
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Cleanup access logs by retention days
tags:
- AccessLogs
/api/access-logs/folds:
get:
parameters:
- description: Node ID
in: query
name: node_id
type: string
- description: Remote address
in: query
name: remote_addr
type: string
- description: Host
in: query
name: host
type: string
- description: Path
in: query
name: path
type: string
- description: Page index
in: query
name: p
type: integer
- description: Page size
in: query
name: page_size
type: integer
- description: Sort by
in: query
name: sort_by
type: string
- description: Sort order
in: query
name: sort_order
type: string
- description: Fold minutes
in: query
name: fold_minutes
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: List folded access logs
tags:
- AccessLogs
/api/access-logs/ip-summary:
get:
parameters:
- description: Node ID
in: query
name: node_id
type: string
- description: Remote address
in: query
name: remote_addr
type: string
- description: Host
in: query
name: host
type: string
- description: Page index
in: query
name: p
type: integer
- description: Page size
in: query
name: page_size
type: integer
- description: Sort by
in: query
name: sort_by
type: string
- description: Sort order
in: query
name: sort_order
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: List access log IP summaries
tags:
- AccessLogs
/api/access-logs/ip-summary/trend:
get:
parameters:
- description: Node ID
in: query
name: node_id
type: string
- description: Remote address
in: query
name: remote_addr
required: true
type: string
- description: Host
in: query
name: host
type: string
- description: Hours
in: query
name: hours
type: integer
- description: Bucket minutes
in: query
name: bucket_minutes
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Get access log IP trend
tags:
- AccessLogs
/api/agent/apply-logs:
post:
consumes:
@@ -226,6 +602,23 @@ paths:
summary: List apply logs
tags:
- ApplyLogs
/api/apply-logs/cleanup:
post:
consumes:
- application/json
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Cleanup apply logs
tags:
- ApplyLogs
/api/config-versions/:
get:
produces:
@@ -241,8 +634,34 @@ paths:
summary: List config versions
tags:
- ConfigVersions
/api/config-versions/{id}:
get:
parameters:
- description: Version ID
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Get config version detail
tags:
- ConfigVersions
/api/config-versions/{id}/activate:
put:
post:
parameters:
- description: Version ID
in: path
@@ -327,6 +746,26 @@ paths:
summary: Publish a new config version
tags:
- ConfigVersions
/api/dashboard/overview:
get:
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Get dashboard overview
tags:
- Dashboard
/api/managed-domains/:
get:
produces:
@@ -370,8 +809,8 @@ paths:
summary: Create managed domain
tags:
- ManagedDomains
/api/managed-domains/{id}:
delete:
/api/managed-domains/{id}/delete:
post:
parameters:
- description: Managed domain ID
in: path
@@ -396,7 +835,8 @@ paths:
summary: Delete managed domain
tags:
- ManagedDomains
put:
/api/managed-domains/{id}/update:
post:
consumes:
- application/json
parameters:
@@ -493,65 +933,6 @@ paths:
summary: Create node
tags:
- Nodes
/api/nodes/{id}:
delete:
parameters:
- description: Node ID
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Delete node
tags:
- Nodes
put:
consumes:
- application/json
parameters:
- description: Node ID
in: path
name: id
required: true
type: integer
- description: Node payload
in: body
name: payload
required: true
schema:
$ref: '#/definitions/service.NodeInput'
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Update node
tags:
- Nodes
/api/nodes/{id}/agent-release:
get:
parameters:
@@ -608,6 +989,92 @@ paths:
summary: Request agent self-update on node
tags:
- Nodes
/api/nodes/{id}/delete:
post:
parameters:
- description: Node ID
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Delete node
tags:
- Nodes
/api/nodes/{id}/observability:
get:
parameters:
- description: Node ID
in: path
name: id
required: true
type: integer
- description: Lookback window in hours
in: query
name: hours
type: integer
- description: Max records per section
in: query
name: limit
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Get node observability details
tags:
- Nodes
/api/nodes/{id}/observability/cleanup:
post:
parameters:
- description: Node ID
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Cleanup node health events
tags:
- Nodes
/api/nodes/{id}/openresty-restart:
post:
parameters:
@@ -634,6 +1101,40 @@ paths:
summary: Request openresty restart on node
tags:
- Nodes
/api/nodes/{id}/update:
post:
consumes:
- application/json
parameters:
- description: Node ID
in: path
name: id
required: true
type: integer
- description: Node payload
in: body
name: payload
required: true
schema:
$ref: '#/definitions/service.NodeInput'
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Update node
tags:
- Nodes
/api/nodes/bootstrap-token:
get:
produces:
@@ -677,7 +1178,35 @@ paths:
summary: List editable options
tags:
- Options
put:
/api/option/geoip/lookup:
post:
consumes:
- application/json
parameters:
- description: GeoIP lookup payload
in: body
name: payload
required: true
schema:
$ref: '#/definitions/controller.geoIPLookupRequest'
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
summary: Test GeoIP lookup
tags:
- Options
/api/option/update:
post:
consumes:
- application/json
parameters:
@@ -746,8 +1275,8 @@ paths:
summary: Create proxy route
tags:
- ProxyRoutes
/api/proxy-routes/{id}:
delete:
/api/proxy-routes/{id}/delete:
post:
parameters:
- description: Route ID
in: path
@@ -772,7 +1301,8 @@ paths:
summary: Delete proxy route
tags:
- ProxyRoutes
put:
/api/proxy-routes/{id}/update:
post:
consumes:
- application/json
parameters:
@@ -862,7 +1392,59 @@ paths:
tags:
- TLSCertificates
/api/tls-certificates/{id}:
delete:
get:
parameters:
- description: Certificate ID
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Get TLS certificate detail
tags:
- TLSCertificates
/api/tls-certificates/{id}/content:
get:
parameters:
- description: Certificate ID
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Get TLS certificate PEM content
tags:
- TLSCertificates
/api/tls-certificates/{id}/delete:
post:
parameters:
- description: Certificate ID
in: path
@@ -887,6 +1469,40 @@ paths:
summary: Delete TLS certificate
tags:
- TLSCertificates
/api/tls-certificates/{id}/update:
post:
consumes:
- application/json
parameters:
- description: Certificate ID
in: path
name: id
required: true
type: integer
- description: TLS certificate payload
in: body
name: payload
required: true
schema:
$ref: '#/definitions/service.TLSCertificateInput'
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties: true
type: object
"400":
description: Bad Request
schema:
additionalProperties: true
type: object
security:
- BearerAuth: []
summary: Update TLS certificate from PEM
tags:
- TLSCertificates
/api/tls-certificates/import-file:
post:
consumes:
@@ -942,6 +1558,12 @@ paths:
summary: Get latest GitHub release
tags:
- Update
/api/update/logs/ws:
get:
responses: {}
summary: Stream server upgrade logs over websocket
tags:
- Update
/api/update/manual-upgrade:
post:
consumes:
@@ -40,3 +40,8 @@ func ListActiveNodeHealthEvents() (events []*NodeHealthEvent, err error) {
err = DB.Where("status = ?", "active").Order("last_triggered_at desc").Find(&events).Error
return events, err
}
func DeleteNodeHealthEvents(nodeID string) (deleted int64, err error) {
result := DB.Where("node_id = ?", nodeID).Delete(&NodeHealthEvent{})
return result.RowsAffected, result.Error
}
+1
View File
@@ -126,6 +126,7 @@ func SetApiRouter(router *gin.Engine) {
nodeRoute.POST("/", controller.CreateNode)
nodeRoute.GET("/:id/agent-release", controller.GetNodeAgentRelease)
nodeRoute.GET("/:id/observability", controller.GetNodeObservability)
nodeRoute.POST("/:id/observability/cleanup", controller.CleanupNodeHealthEvents)
nodeRoute.POST("/:id/agent-update", controller.RequestNodeAgentUpdate)
nodeRoute.POST("/:id/openresty-restart", controller.RequestNodeOpenrestyRestart)
nodeRoute.POST("/:id/update", controller.UpdateNode)
@@ -281,12 +281,41 @@ func TestPhase2AgentLifecycle(t *testing.T) {
t.Fatal("expected node list to expose openresty message")
}
if err := model.DB.Create(&model.NodeHealthEvent{
NodeID: createdNode.NodeID,
EventType: "openresty_down",
Severity: service.NodeHealthSeverityCritical,
Status: service.NodeHealthEventStatusActive,
Message: "docker run openresty failed: bind 80 already allocated",
FirstTriggeredAt: time.Now().Add(-2 * time.Minute),
LastTriggeredAt: time.Now().Add(-time.Minute),
ReportedAt: time.Now().Add(-time.Minute),
}).Error; err != nil {
t.Fatalf("failed to insert node health event: %v", err)
}
observabilityResp := performJSONRequest(t, engine, adminToken, http.MethodGet, "/api/nodes/"+toString(createdNode.ID)+"/observability?hours=24&limit=20", nil)
var observability service.NodeObservabilityView
decodeResponseData(t, observabilityResp, &observability)
if observability.NodeID != createdNode.NodeID {
t.Fatalf("expected observability response for node %s, got %s", createdNode.NodeID, observability.NodeID)
}
if len(observability.HealthEvents) != 1 {
t.Fatalf("expected observability response to include health events, got %+v", observability.HealthEvents)
}
cleanupHealthResp := performJSONRequest(t, engine, adminToken, http.MethodPost, "/api/nodes/"+toString(createdNode.ID)+"/observability/cleanup", nil)
var cleanupHealthResult service.NodeHealthEventCleanupResult
decodeResponseData(t, cleanupHealthResp, &cleanupHealthResult)
if cleanupHealthResult.NodeID != createdNode.NodeID || cleanupHealthResult.DeletedCount != 1 {
t.Fatalf("unexpected node health cleanup result: %+v", cleanupHealthResult)
}
observabilityAfterCleanupResp := performJSONRequest(t, engine, adminToken, http.MethodGet, "/api/nodes/"+toString(createdNode.ID)+"/observability?hours=24&limit=20", nil)
decodeResponseData(t, observabilityAfterCleanupResp, &observability)
if len(observability.HealthEvents) != 0 {
t.Fatalf("expected health events to be cleaned up, got %+v", observability.HealthEvents)
}
restartResp := performJSONRequest(t, engine, adminToken, http.MethodPost, "/api/nodes/"+toString(createdNode.ID)+"/openresty-restart", nil)
decodeResponseData(t, restartResp, &createdNode)
@@ -42,6 +42,11 @@ type NodeObservabilityTrends struct {
DiskIO24h []DiskIOTrendPoint `json:"disk_io_24h"`
}
type NodeHealthEventCleanupResult struct {
NodeID string `json:"node_id"`
DeletedCount int64 `json:"deleted_count"`
}
func GetNodeObservability(id uint, query NodeObservabilityQuery) (*NodeObservabilityView, error) {
now := time.Now()
node, err := model.GetNodeByID(id)
@@ -105,6 +110,21 @@ func GetNodeObservability(id uint, query NodeObservabilityQuery) (*NodeObservabi
}, nil
}
func CleanupNodeHealthEvents(id uint) (*NodeHealthEventCleanupResult, error) {
node, err := model.GetNodeByID(id)
if err != nil {
return nil, err
}
deletedCount, err := model.DeleteNodeHealthEvents(node.NodeID)
if err != nil {
return nil, err
}
return &NodeHealthEventCleanupResult{
NodeID: node.NodeID,
DeletedCount: deletedCount,
}, nil
}
func latestMetricSnapshot(snapshots []*model.NodeMetricSnapshot) *model.NodeMetricSnapshot {
for _, snapshot := range snapshots {
if snapshot != nil {
@@ -1201,6 +1201,66 @@ func TestGetNodeObservabilityAllowsMissingProfile(t *testing.T) {
}
}
func TestCleanupNodeHealthEvents(t *testing.T) {
setupServiceTestDB(t)
node := &model.Node{
NodeID: "node-health-cleanup",
Name: "health-cleanup-edge",
IP: "10.0.0.72",
AgentToken: "token-health-cleanup",
AgentVersion: "v0.6.0",
NginxVersion: "1.27.1.2",
Status: NodeStatusOnline,
}
if err := node.Insert(); err != nil {
t.Fatalf("failed to insert node: %v", err)
}
resolvedAt := time.Now().Add(-4 * time.Minute)
if err := model.DB.Create(&model.NodeHealthEvent{
NodeID: node.NodeID,
EventType: "sync_error",
Severity: NodeHealthSeverityWarning,
Status: NodeHealthEventStatusActive,
Message: "checksum mismatch",
FirstTriggeredAt: time.Now().Add(-2 * time.Minute),
LastTriggeredAt: time.Now().Add(-time.Minute),
ReportedAt: time.Now().Add(-time.Minute),
}).Error; err != nil {
t.Fatalf("failed to insert first node health event: %v", err)
}
if err := model.DB.Create(&model.NodeHealthEvent{
NodeID: node.NodeID,
EventType: "openresty_down",
Severity: NodeHealthSeverityCritical,
Status: NodeHealthEventStatusResolved,
Message: "openresty exited unexpectedly",
FirstTriggeredAt: time.Now().Add(-10 * time.Minute),
LastTriggeredAt: time.Now().Add(-5 * time.Minute),
ReportedAt: time.Now().Add(-5 * time.Minute),
ResolvedAt: &resolvedAt,
}).Error; err != nil {
t.Fatalf("failed to insert second node health event: %v", err)
}
result, err := CleanupNodeHealthEvents(node.ID)
if err != nil {
t.Fatalf("CleanupNodeHealthEvents failed: %v", err)
}
if result.NodeID != node.NodeID || result.DeletedCount != 2 {
t.Fatalf("unexpected cleanup result: %+v", result)
}
events, err := model.ListNodeHealthEvents(node.NodeID, false, 10)
if err != nil {
t.Fatalf("failed to list node health events after cleanup: %v", err)
}
if len(events) != 0 {
t.Fatalf("expected node health events to be removed, got %+v", events)
}
}
func TestGetDashboardOverview(t *testing.T) {
setupServiceTestDB(t)
@@ -85,3 +85,12 @@ export function getNodeObservability(
`/nodes/${id}/observability${query ? `?${query}` : ''}`,
);
}
export function cleanupNodeHealthEvents(id: number) {
return apiRequest<{ node_id: string; deleted_count: number }>(
`/nodes/${id}/observability/cleanup`,
{
method: 'POST',
},
);
}
@@ -20,6 +20,7 @@ import { ConfigVersionSnapshotModal } from '@/features/config-versions/component
import type { ConfigVersionSummary } from '@/features/config-versions/types';
import { getApplyLogs } from '@/features/apply-logs/api/apply-logs';
import {
cleanupNodeHealthEvents,
deleteNode,
getNodeAgentRelease,
getNodeObservability,
@@ -250,6 +251,8 @@ export function NodeDetailPage({ nodeId }: { nodeId: string }) {
const [healthEventFilter, setHealthEventFilter] =
useState<HealthEventFilter>('all');
const [activeTab, setActiveTab] = useState<NodeDetailTab>('dashboard');
const [isHealthEventCleanupModalOpen, setHealthEventCleanupModalOpen] =
useState(false);
const nodesQuery = useQuery({
queryKey: nodesQueryKey,
@@ -373,6 +376,27 @@ export function NodeDetailPage({ nodeId }: { nodeId: string }) {
},
});
const cleanupHealthEventsMutation = useMutation({
mutationFn: () => cleanupNodeHealthEvents(Number(nodeId)),
onSuccess: async (result) => {
setFeedback({
tone: 'success',
message:
result.deleted_count > 0
? `已清理 ${result.deleted_count} 条健康事件日志。`
: '当前没有可清理的健康事件日志。',
});
setHealthEventCleanupModalOpen(false);
await Promise.all([
queryClient.invalidateQueries({ queryKey: ['node-observability', nodeId] }),
queryClient.invalidateQueries({ queryKey: ['dashboard', 'overview'] }),
]);
},
onError: (error) => {
setFeedback({ tone: 'danger', message: getErrorMessage(error) });
},
});
const handleDelete = () => {
if (!node) {
return;
@@ -1215,6 +1239,20 @@ export function NodeDetailPage({ nodeId }: { nodeId: string }) {
<AppCard
title="健康事件时间线"
description="保留活动与已恢复事件,帮助判断问题是持续中、间歇性还是已经恢复。"
action={
<DangerButton
type="button"
disabled={
cleanupHealthEventsMutation.isPending ||
!observability?.health_events.length
}
onClick={() => setHealthEventCleanupModalOpen(true)}
>
{cleanupHealthEventsMutation.isPending
? '清理中...'
: '清理日志'}
</DangerButton>
}
>
{observability?.health_events.length ? (
<div className="space-y-4">
@@ -1703,6 +1741,49 @@ export function NodeDetailPage({ nodeId }: { nodeId: string }) {
}}
/>
<AppModal
isOpen={isHealthEventCleanupModalOpen}
onClose={() => setHealthEventCleanupModalOpen(false)}
title="清理健康事件日志"
description={
node
? `确认清理节点“${node.name}”的健康事件时间线吗?已清理的历史记录将立即从当前页面移除,后续只有新的节点上报才会再次出现。`
: '确认清理当前节点的健康事件时间线吗?'
}
footer={
<div className="flex flex-wrap justify-end gap-3">
<SecondaryButton
type="button"
onClick={() => setHealthEventCleanupModalOpen(false)}
>
取消
</SecondaryButton>
<DangerButton
type="button"
disabled={cleanupHealthEventsMutation.isPending}
onClick={() => {
setFeedback(null);
cleanupHealthEventsMutation.mutate();
}}
>
{cleanupHealthEventsMutation.isPending ? '清理中...' : '确认清理'}
</DangerButton>
</div>
}
>
{cleanupHealthEventsMutation.isError ? (
<ErrorState
title="健康事件清理失败"
description={getErrorMessage(cleanupHealthEventsMutation.error)}
/>
) : (
<div className="space-y-3 text-sm text-[var(--foreground-secondary)]">
<p>该操作会删除当前节点已记录的全部健康事件,包括活动中与已恢复事件。</p>
<p>这不会影响节点后续继续上报新的健康事件,但现有时间线与相关摘要会立即刷新。</p>
</div>
)}
</AppModal>
<AppModal
isOpen={isAgentUpdateModalOpen}
onClose={() => setIsAgentUpdateModalOpen(false)}