[优化] go 引用调整

This commit is contained in:
ryan
2026-06-06 10:26:20 +08:00
parent ee1110b752
commit 3cfefb4367
552 changed files with 1642 additions and 2185 deletions
@@ -0,0 +1,229 @@
package state
import (
"encoding/json"
"os"
"path/filepath"
"sort"
"strconv"
"sync"
"github.com/rain-kl/openflare/openflare-agent/internal/protocol"
)
const observabilityBufferWindowSeconds = 60
type ObservabilityBufferRecord struct {
WindowStartedAtUnix int64 `json:"window_started_at_unix"`
Snapshot *protocol.NodeMetricSnapshot `json:"snapshot,omitempty"`
OpenrestyObservation *protocol.NodeOpenrestyObservation `json:"openresty_observation,omitempty"`
TrafficReport *protocol.NodeTrafficReport `json:"traffic_report,omitempty"`
AccessLogs []protocol.NodeAccessLog `json:"access_logs,omitempty"`
QueuedAtUnix int64 `json:"queued_at_unix"`
}
type ObservabilityBufferStore struct {
path string
mu sync.Mutex
}
func NewObservabilityBufferStore(path string) *ObservabilityBufferStore {
return &ObservabilityBufferStore{path: filepath.Clean(path)}
}
func (s *ObservabilityBufferStore) Upsert(record ObservabilityBufferRecord, retainAfterUnix int64) error {
if s == nil || record.WindowStartedAtUnix <= 0 || (record.Snapshot == nil && record.OpenrestyObservation == nil && record.TrafficReport == nil && len(record.AccessLogs) == 0) {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
records, err := s.loadUnlocked()
if err != nil {
return err
}
records = pruneObservabilityBufferRecords(records, retainAfterUnix)
replaced := false
for index := range records {
if records[index].WindowStartedAtUnix != record.WindowStartedAtUnix {
continue
}
records[index] = mergeObservabilityBufferRecord(records[index], record)
replaced = true
break
}
if !replaced {
records = append(records, record)
}
sort.Slice(records, func(i int, j int) bool {
return records[i].WindowStartedAtUnix < records[j].WindowStartedAtUnix
})
return s.saveUnlocked(records)
}
func mergeObservabilityBufferRecord(existing ObservabilityBufferRecord, incoming ObservabilityBufferRecord) ObservabilityBufferRecord {
merged := existing
if incoming.Snapshot != nil {
merged.Snapshot = incoming.Snapshot
}
if incoming.OpenrestyObservation != nil {
merged.OpenrestyObservation = incoming.OpenrestyObservation
}
if incoming.TrafficReport != nil {
merged.TrafficReport = incoming.TrafficReport
}
merged.AccessLogs = mergeAccessLogs(existing.AccessLogs, incoming.AccessLogs)
if incoming.QueuedAtUnix > 0 {
merged.QueuedAtUnix = incoming.QueuedAtUnix
}
return merged
}
func mergeAccessLogs(existing []protocol.NodeAccessLog, incoming []protocol.NodeAccessLog) []protocol.NodeAccessLog {
if len(existing) == 0 && len(incoming) == 0 {
return nil
}
merged := make([]protocol.NodeAccessLog, 0, len(existing)+len(incoming))
seen := make(map[string]struct{}, len(existing)+len(incoming))
appendIfNeeded := func(items []protocol.NodeAccessLog) {
for _, item := range items {
key := accessLogKey(item)
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
merged = append(merged, item)
}
}
appendIfNeeded(existing)
appendIfNeeded(incoming)
sort.Slice(merged, func(i int, j int) bool {
if merged[i].LoggedAtUnix == merged[j].LoggedAtUnix {
return accessLogKey(merged[i]) < accessLogKey(merged[j])
}
return merged[i].LoggedAtUnix < merged[j].LoggedAtUnix
})
return merged
}
func accessLogKey(item protocol.NodeAccessLog) string {
return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + strconv.Itoa(item.StatusCode)
}
func (s *ObservabilityBufferStore) Replayable(currentWindowStartedAtUnix int64, retainAfterUnix int64) ([]ObservabilityBufferRecord, error) {
if s == nil {
return nil, nil
}
s.mu.Lock()
defer s.mu.Unlock()
records, err := s.loadUnlocked()
if err != nil {
return nil, err
}
records = pruneObservabilityBufferRecords(records, retainAfterUnix)
if err = s.saveUnlocked(records); err != nil {
return nil, err
}
result := make([]ObservabilityBufferRecord, 0, len(records))
for _, record := range records {
if currentWindowStartedAtUnix > 0 && record.WindowStartedAtUnix >= currentWindowStartedAtUnix {
continue
}
result = append(result, record)
}
return result, nil
}
func (s *ObservabilityBufferStore) Ack(windowStartedAtUnix []int64, retainAfterUnix int64) error {
if s == nil || len(windowStartedAtUnix) == 0 {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
records, err := s.loadUnlocked()
if err != nil {
return err
}
acked := make(map[int64]struct{}, len(windowStartedAtUnix))
for _, value := range windowStartedAtUnix {
if value > 0 {
acked[value] = struct{}{}
}
}
filtered := make([]ObservabilityBufferRecord, 0, len(records))
for _, record := range records {
if _, ok := acked[record.WindowStartedAtUnix]; ok {
continue
}
filtered = append(filtered, record)
}
filtered = pruneObservabilityBufferRecords(filtered, retainAfterUnix)
return s.saveUnlocked(filtered)
}
func (s *ObservabilityBufferStore) loadUnlocked() ([]ObservabilityBufferRecord, error) {
data, err := os.ReadFile(s.path)
if err != nil {
if os.IsNotExist(err) {
return []ObservabilityBufferRecord{}, nil
}
return nil, err
}
if len(data) == 0 {
return []ObservabilityBufferRecord{}, nil
}
var records []ObservabilityBufferRecord
if err = json.Unmarshal(data, &records); err != nil {
return nil, err
}
return records, nil
}
func (s *ObservabilityBufferStore) saveUnlocked(records []ObservabilityBufferRecord) error {
if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
return err
}
data, err := json.MarshalIndent(records, "", " ")
if err != nil {
return err
}
return os.WriteFile(s.path, data, 0o644)
}
func ObservabilityWindowStartedAt(snapshot *protocol.NodeMetricSnapshot, openresty *protocol.NodeOpenrestyObservation, traffic *protocol.NodeTrafficReport) int64 {
if traffic != nil && traffic.WindowStartedAtUnix > 0 {
return traffic.WindowStartedAtUnix - (traffic.WindowStartedAtUnix % observabilityBufferWindowSeconds)
}
if openresty != nil && openresty.CapturedAtUnix > 0 {
return openresty.CapturedAtUnix - (openresty.CapturedAtUnix % observabilityBufferWindowSeconds)
}
if snapshot == nil || snapshot.CapturedAtUnix <= 0 {
return 0
}
return snapshot.CapturedAtUnix - (snapshot.CapturedAtUnix % observabilityBufferWindowSeconds)
}
func pruneObservabilityBufferRecords(records []ObservabilityBufferRecord, retainAfterUnix int64) []ObservabilityBufferRecord {
if len(records) == 0 {
return []ObservabilityBufferRecord{}
}
filtered := make([]ObservabilityBufferRecord, 0, len(records))
for _, record := range records {
if record.WindowStartedAtUnix <= 0 {
continue
}
if retainAfterUnix > 0 && record.WindowStartedAtUnix < retainAfterUnix {
continue
}
filtered = append(filtered, record)
}
sort.Slice(filtered, func(i int, j int) bool {
return filtered[i].WindowStartedAtUnix < filtered[j].WindowStartedAtUnix
})
return filtered
}
@@ -0,0 +1,98 @@
package state
import (
"path/filepath"
"testing"
"github.com/rain-kl/openflare/openflare-agent/internal/protocol"
)
func TestObservabilityBufferStoreUpsertReplayAndAck(t *testing.T) {
store := NewObservabilityBufferStore(filepath.Join(t.TempDir(), "observability-buffer.json"))
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
Snapshot: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403205},
TrafficReport: &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403200, WindowEndedAtUnix: 1710403260, RequestCount: 5},
QueuedAtUnix: 1710403205,
}, 1710403000); err != nil {
t.Fatalf("first upsert failed: %v", err)
}
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
Snapshot: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403255},
TrafficReport: &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403200, WindowEndedAtUnix: 1710403260, RequestCount: 12},
QueuedAtUnix: 1710403255,
}, 1710403000); err != nil {
t.Fatalf("second upsert failed: %v", err)
}
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403260,
Snapshot: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403265},
TrafficReport: &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403260, WindowEndedAtUnix: 1710403320, RequestCount: 2},
QueuedAtUnix: 1710403265,
}, 1710403000); err != nil {
t.Fatalf("third upsert failed: %v", err)
}
records, err := store.Replayable(1710403260, 1710403000)
if err != nil {
t.Fatalf("Replayable failed: %v", err)
}
if len(records) != 1 {
t.Fatalf("expected one replayable record before current window, got %d", len(records))
}
if records[0].TrafficReport == nil || records[0].TrafficReport.RequestCount != 12 {
t.Fatalf("expected replayable record to keep latest upsert, got %+v", records[0])
}
if err = store.Ack([]int64{1710403200}, 1710403000); err != nil {
t.Fatalf("Ack failed: %v", err)
}
records, err = store.Replayable(0, 1710403000)
if err != nil {
t.Fatalf("Replayable after ack failed: %v", err)
}
if len(records) != 1 || records[0].WindowStartedAtUnix != 1710403260 {
t.Fatalf("unexpected records after ack: %+v", records)
}
}
func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) {
store := NewObservabilityBufferStore(filepath.Join(t.TempDir(), "observability-buffer.json"))
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
AccessLogs: []protocol.NodeAccessLog{
{LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200},
},
}, 1710403000); err != nil {
t.Fatalf("first upsert failed: %v", err)
}
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
AccessLogs: []protocol.NodeAccessLog{
{LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200},
{LoggedAtUnix: 1710403205, RemoteAddr: "10.0.0.2", Host: "app.example.com", Path: "/b", StatusCode: 502},
},
}, 1710403000); err != nil {
t.Fatalf("second upsert failed: %v", err)
}
records, err := store.Replayable(0, 1710403000)
if err != nil {
t.Fatalf("Replayable failed: %v", err)
}
if len(records) != 1 || len(records[0].AccessLogs) != 2 {
t.Fatalf("expected merged access logs, got %+v", records)
}
}
func TestObservabilityWindowStartedAt(t *testing.T) {
if value := ObservabilityWindowStartedAt(nil, nil, &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403200}); value != 1710403200 {
t.Fatalf("unexpected traffic window start: %d", value)
}
if value := ObservabilityWindowStartedAt(&protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403259}, nil, nil); value != 1710403200 {
t.Fatalf("unexpected snapshot-derived window start: %d", value)
}
}
+106
View File
@@ -0,0 +1,106 @@
package state
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"sync"
)
type Snapshot struct {
NodeID string `json:"node_id"`
CurrentVersion string `json:"current_version"`
CurrentChecksum string `json:"current_checksum"`
BlockedVersion string `json:"blocked_version"`
BlockedChecksum string `json:"blocked_checksum"`
BlockedReason string `json:"blocked_reason"`
LastError string `json:"last_error"`
OpenrestyStatus string `json:"openresty_status"`
OpenrestyMessage string `json:"openresty_message"`
LastProfileFingerprint string `json:"last_profile_fingerprint"`
LastCPUStatTotal uint64 `json:"last_cpu_stat_total"`
LastCPUStatIdle uint64 `json:"last_cpu_stat_idle"`
LastMetricAtUnix int64 `json:"last_metric_at_unix"`
AccessLogOffset int64 `json:"access_log_offset"`
}
type Store struct {
path string
mu sync.Mutex
}
func NewStore(path string) *Store {
return &Store{path: filepath.Clean(path)}
}
func (s *Store) Load() (*Snapshot, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.loadUnlocked()
}
func (s *Store) EnsureNodeID() (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
snapshot, err := s.loadUnlocked()
if err != nil {
return "", err
}
if snapshot.NodeID != "" {
return snapshot.NodeID, nil
}
snapshot.NodeID, err = newNodeID()
if err != nil {
return "", err
}
if err = s.saveUnlocked(snapshot); err != nil {
return "", err
}
return snapshot.NodeID, nil
}
func (s *Store) Save(snapshot *Snapshot) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.saveUnlocked(snapshot)
}
func (s *Store) loadUnlocked() (*Snapshot, error) {
data, err := os.ReadFile(s.path)
if err != nil {
if os.IsNotExist(err) {
return &Snapshot{}, nil
}
return nil, err
}
snapshot := &Snapshot{}
if len(data) == 0 {
return snapshot, nil
}
if err = json.Unmarshal(data, snapshot); err != nil {
return nil, err
}
return snapshot, nil
}
func (s *Store) saveUnlocked(snapshot *Snapshot) error {
if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
return err
}
data, err := json.MarshalIndent(snapshot, "", " ")
if err != nil {
return err
}
return os.WriteFile(s.path, data, 0o644)
}
func newNodeID() (string, error) {
buf := make([]byte, 8)
if _, err := rand.Read(buf); err != nil {
return "", err
}
return "node-" + hex.EncodeToString(buf), nil
}
@@ -0,0 +1,150 @@
package state
import (
"fmt"
"os"
"path/filepath"
"sync"
"testing"
)
func TestEnsureNodeIDPersists(t *testing.T) {
store := NewStore(filepath.Join(t.TempDir(), "state.json"))
nodeID1, err := store.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID failed: %v", err)
}
nodeID2, err := store.EnsureNodeID()
if err != nil {
t.Fatalf("EnsureNodeID second call failed: %v", err)
}
if nodeID1 == "" || nodeID1 != nodeID2 {
t.Fatal("expected node id to persist across calls")
}
}
func TestStore_Load_NonExistentFile(t *testing.T) {
// Loading from a non-existent path should succeed and return an empty Snapshot
tempFile := filepath.Join(t.TempDir(), "nonexistent.json")
store := NewStore(tempFile)
snap, err := store.Load()
if err != nil {
t.Fatalf("expected Load to succeed for non-existent file, got err: %v", err)
}
if snap == nil {
t.Fatal("expected non-nil snapshot")
}
if snap.NodeID != "" || snap.CurrentVersion != "" {
t.Errorf("expected empty snapshot, got: %+v", snap)
}
}
func TestStore_Load_EmptyFile(t *testing.T) {
// Loading from an empty file should succeed and return an empty Snapshot
tempFile := filepath.Join(t.TempDir(), "empty.json")
if err := os.WriteFile(tempFile, []byte(""), 0644); err != nil {
t.Fatalf("failed to create empty file: %v", err)
}
store := NewStore(tempFile)
snap, err := store.Load()
if err != nil {
t.Fatalf("expected Load to succeed for empty file, got err: %v", err)
}
if snap == nil {
t.Fatal("expected non-nil snapshot")
}
if snap.NodeID != "" {
t.Errorf("expected empty snapshot, got: %+v", snap)
}
}
func TestStore_Load_InvalidJSON(t *testing.T) {
// Loading from a corrupted file with invalid JSON should fail with parsing error
tempFile := filepath.Join(t.TempDir(), "corrupted.json")
if err := os.WriteFile(tempFile, []byte("{invalid-json"), 0644); err != nil {
t.Fatalf("failed to create corrupted file: %v", err)
}
store := NewStore(tempFile)
_, err := store.Load()
if err == nil {
t.Fatal("expected Load to fail for corrupted JSON file")
}
}
func TestStore_SaveAndLoad(t *testing.T) {
tempFile := filepath.Join(t.TempDir(), "state.json")
store := NewStore(tempFile)
original := &Snapshot{
NodeID: "node-test-123",
CurrentVersion: "20260531-001",
CurrentChecksum: "chk-active-xyz",
BlockedVersion: "20260531-002",
BlockedChecksum: "chk-blocked-abc",
BlockedReason: "invalid upstream domain name",
LastError: "configuration reload timeout",
OpenrestyStatus: "unhealthy",
}
if err := store.Save(original); err != nil {
t.Fatalf("expected Save to succeed, got: %v", err)
}
loaded, err := store.Load()
if err != nil {
t.Fatalf("expected Load to succeed, got: %v", err)
}
if loaded.NodeID != original.NodeID ||
loaded.CurrentVersion != original.CurrentVersion ||
loaded.CurrentChecksum != original.CurrentChecksum ||
loaded.BlockedVersion != original.BlockedVersion ||
loaded.BlockedChecksum != original.BlockedChecksum ||
loaded.BlockedReason != original.BlockedReason ||
loaded.LastError != original.LastError ||
loaded.OpenrestyStatus != original.OpenrestyStatus {
t.Errorf("loaded snapshot does not match original: %+v vs %+v", loaded, original)
}
}
func TestStore_ConcurrencySafety(t *testing.T) {
tempFile := filepath.Join(t.TempDir(), "state.json")
store := NewStore(tempFile)
var wg sync.WaitGroup
workers := 20
iterations := 50
// Run concurrent writers and readers
for i := 0; i < workers; i++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
for j := 0; j < iterations; j++ {
// Concurrently save
snap := &Snapshot{
NodeID: fmt.Sprintf("node-%d", workerID),
CurrentVersion: fmt.Sprintf("v-%d", j),
}
if err := store.Save(snap); err != nil {
t.Errorf("Save failed under concurrency: %v", err)
}
// Concurrently load
if _, err := store.Load(); err != nil {
t.Errorf("Load failed under concurrency: %v", err)
}
// Concurrently ensure ID
if _, err := store.EnsureNodeID(); err != nil {
t.Errorf("EnsureNodeID failed under concurrency: %v", err)
}
}
}(i)
}
wg.Wait()
}