tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-05 23:26:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add observability buffer for heartbeat data

Browse Source 
- Introduced `ObservabilityBufferStore` to manage buffered observability records.
- Implemented methods for upserting, replaying, and acknowledging records in the buffer.
- Enhanced `AgentNodePayload` and `NodePayload` to include buffered observability data.
- Updated tests to cover new functionality for observability buffering.
- Modified existing services to persist and handle buffered observability data during heartbeats.
- Added configuration options for observability buffer path and replay minutes.
- Updated documentation to reflect new observability features and configurations.
This commit is contained in:
ryan
2026-03-14 17:57:50 +08:00
parent 7628397785
commit e25b41fd75
18 changed files with 798 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
atsf_server/service/agent.go
+14 -13
View File
@@ -24,19 +24,20 @@ const (
)
type AgentNodePayload struct {
NodeID string `json:"node_id"`
Name string `json:"name"`
IP string `json:"ip"`
AgentVersion string `json:"agent_version"`
NginxVersion string `json:"nginx_version"`
CurrentVersion string `json:"current_version"`
LastError string `json:"last_error"`
OpenrestyStatus string `json:"openresty_status"`
OpenrestyMessage string `json:"openresty_message"`
Profile *AgentNodeSystemProfile `json:"profile,omitempty"`
Snapshot *AgentNodeMetricSnapshot `json:"snapshot,omitempty"`
TrafficReport *AgentNodeTrafficReport `json:"traffic_report,omitempty"`
HealthEvents []AgentNodeHealthEvent `json:"health_events"`
NodeID string `json:"node_id"`
Name string `json:"name"`
IP string `json:"ip"`
AgentVersion string `json:"agent_version"`
NginxVersion string `json:"nginx_version"`
CurrentVersion string `json:"current_version"`
LastError string `json:"last_error"`
OpenrestyStatus string `json:"openresty_status"`
OpenrestyMessage string `json:"openresty_message"`
Profile *AgentNodeSystemProfile `json:"profile,omitempty"`
Snapshot *AgentNodeMetricSnapshot `json:"snapshot,omitempty"`
TrafficReport *AgentNodeTrafficReport `json:"traffic_report,omitempty"`
BufferedObservability []AgentBufferedObservabilityRecord `json:"buffered_observability,omitempty"`
HealthEvents []AgentNodeHealthEvent `json:"health_events"`
}
type ApplyLogPayload struct {
atsf_server/service/node_update_test.go
+185 -2
View File
@@ -666,6 +666,135 @@ func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) {
}
}
func TestHeartbeatNodePersistsBufferedObservabilityPayload(t *testing.T) {
setupServiceTestDB(t)
node := &model.Node{
NodeID: "node-observe-buffered",
Name: "observe-buffered-edge",
IP: "10.0.0.32",
AgentToken: "token-observe-buffered",
AgentVersion: "v0.6.0",
NginxVersion: "1.27.1.2",
Status: NodeStatusOnline,
}
if err := node.Insert(); err != nil {
t.Fatalf("failed to seed node: %v", err)
}
now := time.Now().UTC()
_, err := HeartbeatNode(node, AgentNodePayload{
NodeID: node.NodeID,
Name: node.Name,
IP: node.IP,
AgentVersion: node.AgentVersion,
NginxVersion: node.NginxVersion,
Snapshot: &AgentNodeMetricSnapshot{
CapturedAtUnix: now.Unix(),
CPUUsagePercent: 25,
MemoryUsedBytes: 2 * 1024 * 1024 * 1024,
MemoryTotalBytes: 8 * 1024 * 1024 * 1024,
},
TrafficReport: &AgentNodeTrafficReport{
WindowStartedAtUnix: now.Add(-time.Minute).Unix(),
WindowEndedAtUnix: now.Unix(),
RequestCount: 20,
ErrorCount: 1,
UniqueVisitorCount: 10,
StatusCodes: map[string]int64{"200": 19, "500": 1},
TopDomains: map[string]int64{"edge.example.com": 20},
SourceCountries: map[string]int64{"CN": 12},
},
BufferedObservability: []AgentBufferedObservabilityRecord{
{
WindowStartedAtUnix: now.Add(-2 * time.Minute).Unix(),
Snapshot: &AgentNodeMetricSnapshot{
CapturedAtUnix: now.Add(-2 * time.Minute).Unix(),
CPUUsagePercent: 30,
MemoryUsedBytes: 3 * 1024 * 1024 * 1024,
MemoryTotalBytes: 8 * 1024 * 1024 * 1024,
},
TrafficReport: &AgentNodeTrafficReport{
WindowStartedAtUnix: now.Add(-2 * time.Minute).Unix(),
WindowEndedAtUnix: now.Add(-time.Minute).Unix(),
RequestCount: 40,
ErrorCount: 2,
UniqueVisitorCount: 18,
StatusCodes: map[string]int64{"200": 38, "500": 2},
TopDomains: map[string]int64{"edge.example.com": 40},
SourceCountries: map[string]int64{"CN": 20},
},
},
},
})
if err != nil {
t.Fatalf("expected heartbeat to succeed: %v", err)
}
snapshots, err := model.ListNodeMetricSnapshots(node.NodeID, time.Time{}, 10)
if err != nil {
t.Fatalf("expected node snapshots query to succeed: %v", err)
}
if len(snapshots) != 2 {
t.Fatalf("expected current and buffered snapshots, got %+v", snapshots)
}
reports, err := model.ListNodeRequestReports(node.NodeID, time.Time{}, 10)
if err != nil {
t.Fatalf("expected node request reports query to succeed: %v", err)
}
if len(reports) != 2 {
t.Fatalf("expected current and buffered reports, got %+v", reports)
}
_, err = HeartbeatNode(node, AgentNodePayload{
NodeID: node.NodeID,
Name: node.Name,
IP: node.IP,
AgentVersion: node.AgentVersion,
NginxVersion: node.NginxVersion,
BufferedObservability: []AgentBufferedObservabilityRecord{
{
WindowStartedAtUnix: now.Add(-2 * time.Minute).Unix(),
Snapshot: &AgentNodeMetricSnapshot{
CapturedAtUnix: now.Add(-2 * time.Minute).Unix(),
CPUUsagePercent: 30,
MemoryUsedBytes: 3 * 1024 * 1024 * 1024,
MemoryTotalBytes: 8 * 1024 * 1024 * 1024,
},
TrafficReport: &AgentNodeTrafficReport{
WindowStartedAtUnix: now.Add(-2 * time.Minute).Unix(),
WindowEndedAtUnix: now.Add(-time.Minute).Unix(),
RequestCount: 40,
ErrorCount: 2,
UniqueVisitorCount: 18,
StatusCodes: map[string]int64{"200": 38, "500": 2},
TopDomains: map[string]int64{"edge.example.com": 40},
SourceCountries: map[string]int64{"CN": 20},
},
},
},
})
if err != nil {
t.Fatalf("expected heartbeat replay dedupe to succeed: %v", err)
}
snapshots, err = model.ListNodeMetricSnapshots(node.NodeID, time.Time{}, 10)
if err != nil {
t.Fatalf("expected node snapshots query to succeed after replay: %v", err)
}
if len(snapshots) != 2 {
t.Fatalf("expected replay dedupe to keep snapshot count stable, got %+v", snapshots)
}
reports, err = model.ListNodeRequestReports(node.NodeID, time.Time{}, 10)
if err != nil {
t.Fatalf("expected node request reports query to succeed after replay: %v", err)
}
if len(reports) != 2 {
t.Fatalf("expected replay dedupe to keep report count stable, got %+v", reports)
}
}
func TestHeartbeatNodeResolvesMissingHealthEvents(t *testing.T) {
setupServiceTestDB(t)
@@ -754,6 +883,23 @@ func TestGetNodeObservability(t *testing.T) {
}); err != nil {
t.Fatalf("failed to insert node system profile: %v", err)
}
if err := (&model.NodeMetricSnapshot{
NodeID: node.NodeID,
CapturedAt: time.Now().Add(-time.Hour),
CPUUsagePercent: 60,
MemoryUsedBytes: 14 * 1024 * 1024 * 1024,
MemoryTotalBytes: 16 * 1024 * 1024 * 1024,
StorageUsedBytes: 90 * 1024 * 1024 * 1024,
StorageTotalBytes: 100 * 1024 * 1024 * 1024,
DiskReadBytes: 0,
DiskWriteBytes: 0,
NetworkRxBytes: 2048,
NetworkTxBytes: 4096,
OpenrestyRxBytes: 8192,
OpenrestyTxBytes: 16384,
}).Insert(); err != nil {
t.Fatalf("failed to insert node metric baseline snapshot: %v", err)
}
if err := (&model.NodeMetricSnapshot{
NodeID: node.NodeID,
CapturedAt: time.Now(),
@@ -807,8 +953,11 @@ func TestGetNodeObservability(t *testing.T) {
if view.Profile == nil || view.Profile.OSName != "Ubuntu" {
t.Fatalf("unexpected profile: %+v", view.Profile)
}
if len(view.MetricSnapshots) != 1 {
t.Fatalf("expected 1 metric snapshot, got %d", len(view.MetricSnapshots))
if len(view.MetricSnapshots) != 2 {
t.Fatalf("expected 2 metric snapshots, got %d", len(view.MetricSnapshots))
}
if view.MetricSnapshots[0].DiskWriteBytes != 2048 {
t.Fatalf("expected latest metric snapshot to stay intact, got %+v", view.MetricSnapshots[0])
}
if len(view.TrafficReports) != 1 || view.TrafficReports[0].RequestCount != 123 {
t.Fatalf("unexpected traffic reports: %+v", view.TrafficReports)
@@ -923,6 +1072,23 @@ func TestGetDashboardOverview(t *testing.T) {
}
}
if err := (&model.NodeMetricSnapshot{
NodeID: "node-dashboard-a",
CapturedAt: now.Add(-time.Hour),
CPUUsagePercent: 40,
MemoryUsedBytes: 4 * 1024 * 1024 * 1024,
MemoryTotalBytes: 8 * 1024 * 1024 * 1024,
StorageUsedBytes: 48 * 1024 * 1024 * 1024,
StorageTotalBytes: 100 * 1024 * 1024 * 1024,
DiskReadBytes: 0,
DiskWriteBytes: 0,
NetworkRxBytes: 100,
NetworkTxBytes: 150,
OpenrestyRxBytes: 300,
OpenrestyTxBytes: 450,
}).Insert(); err != nil {
t.Fatalf("failed to insert node a baseline metric snapshot: %v", err)
}
if err := (&model.NodeMetricSnapshot{
NodeID: "node-dashboard-a",
CapturedAt: now,
@@ -940,6 +1106,23 @@ func TestGetDashboardOverview(t *testing.T) {
}).Insert(); err != nil {
t.Fatalf("failed to insert node a metric snapshot: %v", err)
}
if err := (&model.NodeMetricSnapshot{
NodeID: "node-dashboard-b",
CapturedAt: now.Add(-time.Hour),
CPUUsagePercent: 88,
MemoryUsedBytes: 14 * 1024 * 1024 * 1024,
MemoryTotalBytes: 16 * 1024 * 1024 * 1024,
StorageUsedBytes: 93 * 1024 * 1024 * 1024,
StorageTotalBytes: 100 * 1024 * 1024 * 1024,
DiskReadBytes: 0,
DiskWriteBytes: 0,
NetworkRxBytes: 200,
NetworkTxBytes: 300,
OpenrestyRxBytes: 500,
OpenrestyTxBytes: 700,
}).Insert(); err != nil {
t.Fatalf("failed to insert node b baseline metric snapshot: %v", err)
}
if err := (&model.NodeMetricSnapshot{
NodeID: "node-dashboard-b",
CapturedAt: now,
atsf_server/service/observability.go
+24 -3
View File
@@ -60,6 +60,12 @@ type AgentNodeTrafficReport struct {
SourceCountries map[string]int64 `json:"source_countries"`
}
type AgentBufferedObservabilityRecord struct {
WindowStartedAtUnix int64 `json:"window_started_at_unix"`
Snapshot *AgentNodeMetricSnapshot `json:"snapshot,omitempty"`
TrafficReport *AgentNodeTrafficReport `json:"traffic_report,omitempty"`
}
type AgentNodeHealthEvent struct {
EventType string `json:"event_type"`
Severity string `json:"severity"`
@@ -72,7 +78,7 @@ func persistHeartbeatObservability(nodeID string, payload AgentNodePayload, repo
if strings.TrimSpace(nodeID) == "" {
return
}
if payload.Profile == nil && payload.Snapshot == nil && payload.TrafficReport == nil && payload.HealthEvents == nil {
if payload.Profile == nil && payload.Snapshot == nil && payload.TrafficReport == nil && len(payload.BufferedObservability) == 0 && payload.HealthEvents == nil {
return
}
@@ -80,6 +86,9 @@ func persistHeartbeatObservability(nodeID string, payload AgentNodePayload, repo
if err := persistNodeSystemProfile(tx, nodeID, payload.Profile, reportedAt); err != nil {
return err
}
if err := persistBufferedObservability(tx, nodeID, payload.BufferedObservability, reportedAt); err != nil {
return err
}
if err := persistNodeMetricSnapshot(tx, nodeID, payload.Snapshot, reportedAt); err != nil {
return err
}
@@ -97,6 +106,18 @@ func persistHeartbeatObservability(nodeID string, payload AgentNodePayload, repo
}
}
func persistBufferedObservability(tx *gorm.DB, nodeID string, records []AgentBufferedObservabilityRecord, reportedAt time.Time) error {
for _, record := range records {
if err := persistNodeMetricSnapshot(tx, nodeID, record.Snapshot, reportedAt); err != nil {
return err
}
if err := persistNodeTrafficReport(tx, nodeID, record.TrafficReport, reportedAt); err != nil {
return err
}
}
return nil
}
func persistNodeSystemProfile(tx *gorm.DB, nodeID string, profile *AgentNodeSystemProfile, reportedAt time.Time) error {
if profile == nil {
return nil
@@ -140,7 +161,7 @@ func persistNodeMetricSnapshot(tx *gorm.DB, nodeID string, snapshot *AgentNodeMe
OpenrestyConnections: snapshot.OpenrestyConnections,
RawJSON: marshalJSON(snapshot),
}
return tx.Create(record).Error
return tx.Where("node_id = ? AND captured_at = ?", nodeID, record.CapturedAt).Assign(record).FirstOrCreate(record).Error
}
func persistNodeTrafficReport(tx *gorm.DB, nodeID string, report *AgentNodeTrafficReport, reportedAt time.Time) error {
@@ -162,7 +183,7 @@ func persistNodeTrafficReport(tx *gorm.DB, nodeID string, report *AgentNodeTraff
SourceCountriesJSON: marshalJSON(report.SourceCountries),
RawJSON: marshalJSON(report),
}
return tx.Create(record).Error
return tx.Where("node_id = ? AND window_started_at = ? AND window_ended_at = ?", nodeID, record.WindowStartedAt, record.WindowEndedAt).Assign(record).FirstOrCreate(record).Error
}
func reconcileNodeHealthEvents(tx *gorm.DB, nodeID string, events []AgentNodeHealthEvent, reportedAt time.Time) error {
atsf_server/service/observability_trends.go
+43 -2
View File
@@ -2,6 +2,7 @@ package service
import (
"atsflare/model"
"sort"
"time"
)
@@ -148,13 +149,53 @@ func buildDiskIOTrendPoints(now time.Time, snapshots []*model.NodeMetricSnapshot
accumulators[index].nodes = make(map[string]struct{})
}
sort.Slice(snapshots, func(i int, j int) bool {
if snapshots[i].CapturedAt.Equal(snapshots[j].CapturedAt) {
return snapshots[i].NodeID < snapshots[j].NodeID
}
return snapshots[i].CapturedAt.Before(snapshots[j].CapturedAt)
})
type diskCounterState struct {
read int64
write int64
seen bool
}
previousByNode := make(map[string]diskCounterState, len(snapshots))
for _, snapshot := range snapshots {
nodeKey := snapshot.NodeID
if nodeKey == "" {
nodeKey = "__unknown__"
}
previous := previousByNode[nodeKey]
previousByNode[nodeKey] = diskCounterState{
read: snapshot.DiskReadBytes,
write: snapshot.DiskWriteBytes,
seen: true,
}
if !previous.seen {
continue
}
index, ok := trendBucketIndex(snapshot.CapturedAt, start)
if !ok {
continue
}
points[index].DiskReadBytes += snapshot.DiskReadBytes
points[index].DiskWriteBytes += snapshot.DiskWriteBytes
readDelta := snapshot.DiskReadBytes - previous.read
writeDelta := snapshot.DiskWriteBytes - previous.write
if readDelta < 0 {
readDelta = 0
}
if writeDelta < 0 {
writeDelta = 0
}
points[index].DiskReadBytes += readDelta
points[index].DiskWriteBytes += writeDelta
if snapshot.NodeID != "" {
accumulators[index].nodes[snapshot.NodeID] = struct{}{}
}
atsf_server/service/observability_trends_test.go
+32
View File
@@ -0,0 +1,32 @@
package service
import (
"atsflare/model"
"testing"
"time"
)
func TestBuildDiskIOTrendPointsUsesCounterDelta(t *testing.T) {
now := time.Date(2026, 3, 14, 18, 30, 0, 0, time.UTC)
start := trendWindowStart(now)
points := buildDiskIOTrendPoints(now, []*model.NodeMetricSnapshot{
{
NodeID: "node-a",
CapturedAt: start.Add(22 * time.Hour),
DiskReadBytes: 100,
DiskWriteBytes: 200,
},
{
NodeID: "node-a",
CapturedAt: start.Add(23 * time.Hour),
DiskReadBytes: 250,
DiskWriteBytes: 260,
},
})
last := points[len(points)-1]
if last.DiskReadBytes != 150 || last.DiskWriteBytes != 60 {
t.Fatalf("expected disk io trend to use counter delta, got %+v", last)
}
}
atsf_server/web/features/dashboard/components/dashboard-overview.tsx
+9 -2
View File
@@ -43,6 +43,13 @@ function formatBytes(value?: number | null) {
return `${current.toFixed(current >= 100 || index === 0 ? 0 : 1)} ${units[index]}`;
}
function formatBytesPerSecond(value?: number | null, windowSeconds = 1) {
if (!value || value <= 0 || windowSeconds <= 0) {
return '—';
}
return `${formatBytes(value / windowSeconds)}/s`;
}
function getErrorMessage(error: unknown) {
return error instanceof Error ? error.message : '请求失败，请稍后重试。';
}
@@ -281,7 +288,7 @@ export function DashboardOverview() {
values: overview.trends.network_24h.map(
(point) => point.openresty_rx_bytes,
),
valueFormatter: formatBytes,
valueFormatter: (value) => formatBytesPerSecond(value, 3600),
},
{
label: 'OpenResty 出站',
@@ -289,7 +296,7 @@ export function DashboardOverview() {
values: overview.trends.network_24h.map(
(point) => point.openresty_tx_bytes,
),
valueFormatter: formatBytes,
valueFormatter: (value) => formatBytesPerSecond(value, 3600),
},
]}
/>
atsf_server/web/features/nodes/components/node-detail-page.tsx
+17 -4
View File
@@ -89,6 +89,13 @@ function formatBytes(value?: number | null) {
return `${current.toFixed(current >= 100 || index === 0 ? 0 : 1)} ${units[index]}`;
}
function formatBytesPerSecond(value?: number | null, windowSeconds = 1) {
if (!value || value <= 0 || windowSeconds <= 0) {
return '—';
}
return `${formatBytes(value / windowSeconds)}/s`;
}
function formatPercent(value?: number | null) {
if (value === undefined || value === null || Number.isNaN(value)) {
return '—';
@@ -929,11 +936,17 @@ export function NodeDetailPage({ nodeId }: { nodeId: string }) {
<div className="mt-3 space-y-2 text-sm text-[var(--foreground-secondary)]">
<p>
入站：
{formatBytes(latestMetricSnapshot.openresty_rx_bytes)}
{formatBytesPerSecond(
latestMetricSnapshot.openresty_rx_bytes,
60,
)}
</p>
<p>
出站：
{formatBytes(latestMetricSnapshot.openresty_tx_bytes)}
{formatBytesPerSecond(
latestMetricSnapshot.openresty_tx_bytes,
60,
)}
</p>
</div>
</div>
@@ -1087,7 +1100,7 @@ export function NodeDetailPage({ nodeId }: { nodeId: string }) {
observability?.trends.network_24h.map(
(point) => point.openresty_rx_bytes,
) ?? [],
valueFormatter: formatBytes,
valueFormatter: (value) => formatBytesPerSecond(value, 3600),
},
{
label: 'OpenResty 出站',
@@ -1096,7 +1109,7 @@ export function NodeDetailPage({ nodeId }: { nodeId: string }) {
observability?.trends.network_24h.map(
(point) => point.openresty_tx_bytes,
) ?? [],
valueFormatter: formatBytes,
valueFormatter: (value) => formatBytesPerSecond(value, 3600),
},
]}
/>