feat: add observability buffer for heartbeat data

- Introduced `ObservabilityBufferStore` to manage buffered observability records.
- Implemented methods for upserting, replaying, and acknowledging records in the buffer.
- Enhanced `AgentNodePayload` and `NodePayload` to include buffered observability data.
- Updated tests to cover new functionality for observability buffering.
- Modified existing services to persist and handle buffered observability data during heartbeats.
- Added configuration options for observability buffer path and replay minutes.
- Updated documentation to reflect new observability features and configurations.

atsf_agent/internal/config/config.go

@@ -12,12 +12,14 @@ import (
 )
 
 const (
-	defaultDockerMainConfigRelativePath  = "etc/nginx/nginx.conf"
-	defaultDockerRouteConfigRelativePath = "etc/nginx/conf.d/atsflare_routes.conf"
-	defaultSupportDirRelativePath        = "etc/nginx/support"
-	defaultDockerStateRelativePath       = "var/lib/atsflare/agent-state.json"
-	defaultDockerOpenRestySupportDir     = "/etc/nginx/atsflare-support"
-	defaultOpenRestyObservabilityPort    = 18081
+	defaultDockerMainConfigRelativePath    = "etc/nginx/nginx.conf"
+	defaultDockerRouteConfigRelativePath   = "etc/nginx/conf.d/atsflare_routes.conf"
+	defaultSupportDirRelativePath          = "etc/nginx/support"
+	defaultDockerStateRelativePath         = "var/lib/atsflare/agent-state.json"
+	defaultObservabilityBufferRelativePath = "var/lib/atsflare/observability-buffer.json"
+	defaultDockerOpenRestySupportDir       = "/etc/nginx/atsflare-support"
+	defaultOpenRestyObservabilityPort      = 18081
+	defaultObservabilityReplayMinutes      = 15
 )
 
 type Config struct {
@@ -38,6 +40,8 @@ type Config struct {
 	SupportDir                 string              `json:"support_dir"`
 	OpenrestySupportDir        string              `json:"openresty_support_dir"`
 	OpenrestyObservabilityPort int                 `json:"openresty_observability_port"`
+	ObservabilityBufferPath    string              `json:"observability_buffer_path"`
+	ObservabilityReplayMinutes int                 `json:"observability_replay_minutes"`
 	StatePath                  string              `json:"state_path"`
 	HeartbeatInterval          MillisecondDuration `json:"heartbeat_interval"`
 	RequestTimeout             MillisecondDuration `json:"request_timeout"`
@@ -62,6 +66,8 @@ type configFile struct {
 	LegacyCertDir              string              `json:"cert_dir"`
 	LegacyOpenrestyCertDir     string              `json:"openresty_cert_dir"`
 	OpenrestyObservabilityPort int                 `json:"openresty_observability_port"`
+	ObservabilityBufferPath    string              `json:"observability_buffer_path"`
+	ObservabilityReplayMinutes int                 `json:"observability_replay_minutes"`
 	StatePath                  string              `json:"state_path"`
 	HeartbeatInterval          MillisecondDuration `json:"heartbeat_interval"`
 	RequestTimeout             MillisecondDuration `json:"request_timeout"`
@@ -92,6 +98,8 @@ func Load(path string) (*Config, error) {
 		SupportDir:                 firstNonEmpty(file.SupportDir, file.LegacyCertDir),
 		OpenrestySupportDir:        firstNonEmpty(file.OpenrestySupportDir, file.LegacyOpenrestyCertDir),
 		OpenrestyObservabilityPort: file.OpenrestyObservabilityPort,
+		ObservabilityBufferPath:    file.ObservabilityBufferPath,
+		ObservabilityReplayMinutes: file.ObservabilityReplayMinutes,
 		StatePath:                  file.StatePath,
 		HeartbeatInterval:          file.HeartbeatInterval,
 		RequestTimeout:             file.RequestTimeout,
@@ -153,6 +161,12 @@ func applyDefaults(cfg *Config, baseDir string) {
 	if cfg.OpenrestyObservabilityPort <= 0 {
 		cfg.OpenrestyObservabilityPort = defaultOpenRestyObservabilityPort
 	}
+	if cfg.ObservabilityBufferPath == "" {
+		cfg.ObservabilityBufferPath = joinManagedPath(cfg.DataDir, defaultObservabilityBufferRelativePath)
+	}
+	if cfg.ObservabilityReplayMinutes <= 0 {
+		cfg.ObservabilityReplayMinutes = defaultObservabilityReplayMinutes
+	}
 	if cfg.HeartbeatInterval <= 0 {
 		cfg.HeartbeatInterval = MillisecondDuration(10 * time.Second)
 	}
@@ -184,6 +198,9 @@ func normalizeManagedPaths(cfg *Config) {
 	if usesSlashPath(cfg.StatePath) {
 		cfg.StatePath = filepath.ToSlash(cfg.StatePath)
 	}
+	if usesSlashPath(cfg.ObservabilityBufferPath) {
+		cfg.ObservabilityBufferPath = filepath.ToSlash(cfg.ObservabilityBufferPath)
+	}
 }
 
 func usesSlashPath(path string) bool {
@@ -213,6 +230,9 @@ func validate(cfg *Config) error {
 	if cfg.OpenrestyObservabilityPort <= 0 || cfg.OpenrestyObservabilityPort > 65535 {
 		return errors.New("openresty_observability_port 必须在 1-65535 之间")
 	}
+	if cfg.ObservabilityReplayMinutes <= 0 {
+		return errors.New("observability_replay_minutes 必须大于 0")
+	}
 	return nil
 }
 

atsf_agent/internal/config/config_test.go

@@ -54,9 +54,15 @@ func TestLoadDockerModeUsesManagedPaths(t *testing.T) {
 	if cfg.StatePath != filepath.Join(dir, "data", defaultDockerStateRelativePath) {
 		t.Fatalf("unexpected state path: %s", cfg.StatePath)
 	}
+	if cfg.ObservabilityBufferPath != filepath.Join(dir, "data", defaultObservabilityBufferRelativePath) {
+		t.Fatalf("unexpected observability buffer path: %s", cfg.ObservabilityBufferPath)
+	}
 	if cfg.OpenrestyObservabilityPort != defaultOpenRestyObservabilityPort {
 		t.Fatalf("unexpected openresty observability port: %d", cfg.OpenrestyObservabilityPort)
 	}
+	if cfg.ObservabilityReplayMinutes != defaultObservabilityReplayMinutes {
+		t.Fatalf("unexpected observability replay minutes: %d", cfg.ObservabilityReplayMinutes)
+	}
 }
 
 func TestLoadPathModeKeepsExplicitPaths(t *testing.T) {
@@ -94,6 +100,9 @@ func TestLoadPathModeKeepsExplicitPaths(t *testing.T) {
 	if cfg.StatePath != "/tmp/agent-state.json" {
 		t.Fatalf("unexpected state path: %s", cfg.StatePath)
 	}
+	if cfg.ObservabilityBufferPath != filepath.Join(dir, "data", defaultObservabilityBufferRelativePath) {
+		t.Fatalf("unexpected observability buffer path: %s", cfg.ObservabilityBufferPath)
+	}
 	if cfg.OpenrestySupportDir != cfg.SupportDir {
 		t.Fatalf("expected path mode openresty support dir to equal support dir, got %s / %s", cfg.OpenrestySupportDir, cfg.SupportDir)
 	}
@@ -134,6 +143,9 @@ func TestLoadUsesCustomDataDirForGeneratedFiles(t *testing.T) {
 	if cfg.StatePath != "/srv/atsflare/"+defaultDockerStateRelativePath {
 		t.Fatalf("unexpected state path: %s", cfg.StatePath)
 	}
+	if cfg.ObservabilityBufferPath != "/srv/atsflare/"+defaultObservabilityBufferRelativePath {
+		t.Fatalf("unexpected observability buffer path: %s", cfg.ObservabilityBufferPath)
+	}
 	if cfg.SupportDir != "/srv/atsflare/"+defaultSupportDirRelativePath {
 		t.Fatalf("unexpected support dir: %s", cfg.SupportDir)
 	}
@@ -213,6 +225,9 @@ func TestSavePersistsMillisecondsAndOmitsRuntimeVersions(t *testing.T) {
 	if decoded["openresty_observability_port"] != float64(defaultOpenRestyObservabilityPort) {
 		t.Fatalf("unexpected observability port: %#v", decoded["openresty_observability_port"])
 	}
+	if decoded["observability_replay_minutes"] != float64(defaultObservabilityReplayMinutes) {
+		t.Fatalf("unexpected observability replay minutes: %#v", decoded["observability_replay_minutes"])
+	}
 	if _, ok := decoded["nginx_path"]; ok {
 		t.Fatal("legacy nginx_path should not be persisted")
 	}