feat(obs): 访问日志 SSOT 与 edge_health,去掉协议兼容层

Agent 仅上报 host_metrics/edge_health/access_logs;业务流量与 UV 由
Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly,
删除 request_reports/openresty 吞吐路径;API 不再暴露 traffic_reports
与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
ryan
2026-07-18 11:53:11 +08:00
parent 9a0974cce8
commit f0e234df1f
60 changed files with 1799 additions and 2164 deletions
+15 -15
View File
@@ -403,11 +403,11 @@ func TestRunnerHeartbeatPayloadIncludesObservabilityExtensions(t *testing.T) {
if firstPayload.Profile == nil {
t.Fatal("expected first heartbeat payload to include system profile")
}
if firstPayload.Snapshot == nil {
t.Fatal("expected first heartbeat payload to include metric snapshot")
if firstPayload.HostMetrics == nil {
t.Fatal("expected first heartbeat payload to include host metrics")
}
if firstPayload.TrafficReport == nil || firstPayload.TrafficReport.RequestCount != 1 {
t.Fatalf("expected first heartbeat payload to include traffic report, got %+v", firstPayload.TrafficReport)
if firstPayload.SchemaVersion != 2 {
t.Fatalf("expected schema_version 2, got %d", firstPayload.SchemaVersion)
}
if len(firstPayload.AccessLogs) != 1 || firstPayload.AccessLogs[0].Path != "/" {
t.Fatalf("expected first heartbeat payload to include access logs, got %+v", firstPayload.AccessLogs)
@@ -420,11 +420,8 @@ func TestRunnerHeartbeatPayloadIncludesObservabilityExtensions(t *testing.T) {
if secondPayload.Profile != nil {
t.Fatal("expected unchanged profile to be omitted on subsequent heartbeat")
}
if secondPayload.Snapshot == nil {
t.Fatal("expected metric snapshot to continue reporting on subsequent heartbeat")
}
if secondPayload.TrafficReport != nil {
t.Fatalf("expected unchanged traffic window to be omitted on subsequent heartbeat, got %+v", secondPayload.TrafficReport)
if secondPayload.HostMetrics == nil {
t.Fatal("expected host metrics to continue reporting on subsequent heartbeat")
}
if len(secondPayload.AccessLogs) != 0 {
t.Fatalf("expected unchanged access log delta to be omitted on subsequent heartbeat, got %+v", secondPayload.AccessLogs)
@@ -442,8 +439,8 @@ func TestRunnerReplaysBufferedObservabilityAfterHeartbeatRecovery(t *testing.T)
bufferWindow := nowUnix - (nowUnix % 60) - 60
if err := bufferStore.Upsert(state.ObservabilityBufferRecord{
WindowStartedAtUnix: bufferWindow,
Snapshot: &protocol.NodeMetricSnapshot{CapturedAtUnix: bufferWindow + 5, CPUUsagePercent: 30},
TrafficReport: &protocol.NodeTrafficReport{WindowStartedAtUnix: bufferWindow, WindowEndedAtUnix: bufferWindow + 60, RequestCount: 8},
HostMetrics: &protocol.NodeMetricSnapshot{CapturedAtUnix: bufferWindow + 5, CPUUsagePercent: 30},
EdgeHealth: &protocol.NodeEdgeHealth{CapturedAtUnix: bufferWindow + 5, Connections: 3, Status: "healthy"},
QueuedAtUnix: bufferWindow + 60,
}, 0); err != nil {
t.Fatalf("failed to seed observability buffer: %v", err)
@@ -493,11 +490,14 @@ func TestRunnerReplaysBufferedObservabilityAfterHeartbeatRecovery(t *testing.T)
t.Fatalf("expected two heartbeat payloads, got %d", len(heartbeatService.heartbeatPayloads))
}
secondPayload := heartbeatService.heartbeatPayloads[1]
if len(secondPayload.BufferedObservability) != 1 {
t.Fatalf("expected second heartbeat to replay one buffered observation, got %+v", secondPayload.BufferedObservability)
if len(secondPayload.Buffered) != 1 {
t.Fatalf("expected second heartbeat to replay one buffered observation, got %+v", secondPayload.Buffered)
}
if len(secondPayload.BufferedObservability[0].AccessLogs) != 0 {
t.Fatalf("expected seeded buffered observation to keep empty access logs, got %+v", secondPayload.BufferedObservability[0].AccessLogs)
if len(secondPayload.Buffered[0].AccessLogs) != 0 {
t.Fatalf("expected seeded buffered observation to keep empty access logs, got %+v", secondPayload.Buffered[0].AccessLogs)
}
if secondPayload.Buffered[0].EdgeHealth == nil || secondPayload.Buffered[0].EdgeHealth.Connections != 3 {
t.Fatalf("expected buffered edge health, got %+v", secondPayload.Buffered[0].EdgeHealth)
}
replayable, err := bufferStore.Replayable(0, 0)
+2 -2
View File
@@ -29,11 +29,11 @@ const (
defaultStateRelativePath = "var/lib/openflare/agent-state.json"
defaultObservabilityBufferRelativePath = "var/lib/openflare/observability-buffer.json"
defaultOpenRestyObservabilityPort = 18081
defaultObservabilityReplayMinutes = 15
defaultObservabilityReplayMinutes = 60
defaultMMDBUpdateInterval = 24 * time.Hour
defaultMMDBDownloadURL = "https://github.com/FyraLabs/geolite2/releases/latest/download/GeoLite2-Country.mmdb"
defaultCityMMDBDownloadURL = "https://github.com/FyraLabs/geolite2/releases/latest/download/GeoLite2-City.mmdb"
defaultHeartbeatInterval = 10 * time.Second
defaultHeartbeatInterval = 3 * time.Second
defaultRequestTimeout = 10 * time.Second
configFilePerm = 0o600
)
+30 -35
View File
@@ -90,13 +90,10 @@ func (c *Cycle) NodePayload(ctx context.Context, nodeID string) protocol.NodePay
openrestyStatus = protocol.OpenrestyStatusUnknown
}
profile := observability.BuildProfile(c.Config, c.StateStore)
managedOpenRestyMetrics := observability.CollectManagedOpenRestyMetrics(ctx, c.Config)
trafficReport, accessLogs, fallbackMetrics := observability.BuildTrafficObservability(c.Config, c.StateStore, managedOpenRestyMetrics)
if managedOpenRestyMetrics == nil {
managedOpenRestyMetrics = fallbackMetrics
}
edgeSnapshot := observability.CollectEdgeHealth(ctx, c.Config)
accessLogs := observability.CollectAccessLogs(c.Config, c.StateStore)
metricSnapshot := observability.BuildSnapshot(c.Config, c.StateStore)
openrestyObservation := observability.BuildOpenrestyObservation(managedOpenRestyMetrics)
edgeHealth := observability.BuildEdgeHealth(edgeSnapshot, openrestyStatus, snapshot.OpenrestyMessage)
healthEvents := observability.BuildHealthEvents(snapshot)
ip := c.Config.NodeIP
@@ -105,21 +102,21 @@ func (c *Cycle) NodePayload(ctx context.Context, nodeID string) protocol.NodePay
}
payload := protocol.NodePayload{
NodeID: nodeID,
Name: c.Config.NodeName,
IP: ip,
Version: c.Config.Version,
ExtVersion: c.Config.ExtVersion,
CurrentVersion: snapshot.CurrentVersion,
LastError: snapshot.LastError,
OpenrestyStatus: openrestyStatus,
OpenrestyMessage: snapshot.OpenrestyMessage,
Profile: profile,
Snapshot: metricSnapshot,
OpenrestyObservation: openrestyObservation,
TrafficReport: trafficReport,
AccessLogs: accessLogs,
HealthEvents: healthEvents,
SchemaVersion: 2,
NodeID: nodeID,
Name: c.Config.NodeName,
IP: ip,
Version: c.Config.Version,
ExtVersion: c.Config.ExtVersion,
CurrentVersion: snapshot.CurrentVersion,
LastError: snapshot.LastError,
OpenrestyStatus: openrestyStatus,
OpenrestyMessage: snapshot.OpenrestyMessage,
Profile: profile,
HostMetrics: metricSnapshot,
EdgeHealth: edgeHealth,
AccessLogs: accessLogs,
HealthEvents: healthEvents,
}
if c.Sync != nil {
checksums, err := c.Sync.WAFIPGroupChecksums()
@@ -135,23 +132,22 @@ func (c *Cycle) NodePayload(ctx context.Context, nodeID string) protocol.NodePay
// PrepareHeartbeatPayload constructs the heartbeat payload with buffered observability records and returns the window timestamps to acknowledge.
func (c *Cycle) PrepareHeartbeatPayload(ctx context.Context, nodeID string) (protocol.NodePayload, []int64) {
payload := c.NodePayload(ctx, nodeID)
if c.ObservabilityBuffer == nil || (payload.Snapshot == nil && payload.TrafficReport == nil && len(payload.AccessLogs) == 0) {
if c.ObservabilityBuffer == nil || (payload.HostMetrics == nil && payload.EdgeHealth == nil && len(payload.AccessLogs) == 0) {
return payload, nil
}
now := time.Now().UTC()
retainAfterUnix := now.Add(-time.Duration(c.Config.ObservabilityReplayMinutes) * time.Minute).Unix()
windowStartedAtUnix := state.ObservabilityWindowStartedAt(payload.Snapshot, payload.OpenrestyObservation, payload.TrafficReport)
windowStartedAtUnix := state.ObservabilityWindowStartedAt(payload.HostMetrics, payload.EdgeHealth)
if windowStartedAtUnix <= 0 {
return payload, nil
}
record := state.ObservabilityBufferRecord{
WindowStartedAtUnix: windowStartedAtUnix,
Snapshot: payload.Snapshot,
OpenrestyObservation: payload.OpenrestyObservation,
TrafficReport: payload.TrafficReport,
AccessLogs: payload.AccessLogs,
QueuedAtUnix: now.Unix(),
WindowStartedAtUnix: windowStartedAtUnix,
HostMetrics: payload.HostMetrics,
EdgeHealth: payload.EdgeHealth,
AccessLogs: payload.AccessLogs,
QueuedAtUnix: now.Unix(),
}
if err := c.ObservabilityBuffer.Upsert(record, retainAfterUnix); err != nil {
slog.Error("upsert observability buffer failed", "error", err)
@@ -171,15 +167,14 @@ func (c *Cycle) PrepareHeartbeatPayload(ctx context.Context, nodeID string) (pro
continue
}
buffered = append(buffered, protocol.BufferedObservabilityRecord{
WindowStartedAtUnix: item.WindowStartedAtUnix,
Snapshot: item.Snapshot,
OpenrestyObservation: item.OpenrestyObservation,
TrafficReport: item.TrafficReport,
AccessLogs: item.AccessLogs,
CapturedAtUnix: item.WindowStartedAtUnix,
HostMetrics: item.HostMetrics,
EdgeHealth: item.EdgeHealth,
AccessLogs: item.AccessLogs,
})
ackWindows = append(ackWindows, item.WindowStartedAtUnix)
}
payload.BufferedObservability = buffered
payload.Buffered = buffered
ackWindows = append(ackWindows, windowStartedAtUnix)
return payload, ackWindows
}
+32 -127
View File
@@ -2,147 +2,52 @@ package nginx
import "github.com/Rain-kl/Wavelet/internal/apps/agent/protocol"
const (
openRestyObservabilityWindowTTL = "7200"
openRestyObservabilityWindowSize = "60"
)
// Local OpenResty observability endpoint (target model):
// GET /openflare/observability returns instantaneous health/connections only.
// Business traffic is collected exclusively from access.log.
const openRestyObservabilityInitLua = `local dict = ngx.shared.openflare_observability
if not dict then
return
end
return
const openRestyObservabilityInitLua = `return
`
const openRestyObservabilityLogLua = `local dict = ngx.shared.openflare_observability
if not dict then
return
end
local request_uri = tostring(ngx.var.uri or "")
if request_uri == "/openflare/observability" or request_uri == "/openflare/stub_status" then
return
end
local ttl = ` + openRestyObservabilityWindowTTL + `
local now = ngx.time()
local window_size = ` + openRestyObservabilityWindowSize + `
local window_start = now - (now % window_size)
local function ensure_counter(key)
dict:add(key, 0, ttl)
end
local function incr(key, delta)
ensure_counter(key)
local value, err = dict:incr(key, delta)
if not value and err == "not found" then
dict:set(key, delta, ttl)
end
end
local function remember_value(list_key, marker_key, value)
if value == "" then
return
end
if not dict:add(marker_key, 1, ttl) then
return
end
local existing = dict:get(list_key)
if not existing or existing == "" then
dict:set(list_key, value, ttl)
return
end
dict:set(list_key, existing .. "\n" .. value, ttl)
end
local window_prefix = tostring(window_start)
incr("request_count:" .. window_prefix, 1)
local status = tostring(ngx.status or 0)
if status ~= "0" then
incr("status:" .. window_prefix .. ":" .. status, 1)
remember_value(
"status_keys:" .. window_prefix,
"status_marker:" .. window_prefix .. ":" .. status,
status
)
if tonumber(status) and tonumber(status) >= 500 then
incr("error_count:" .. window_prefix, 1)
end
end
local host = tostring(ngx.var.host or "")
if host ~= "" then
incr("domain:" .. window_prefix .. ":" .. host, 1)
remember_value(
"domain_keys:" .. window_prefix,
"domain_marker:" .. window_prefix .. ":" .. host,
host
)
end
local remote_addr = tostring(ngx.var.binary_remote_addr or ngx.var.remote_addr or "")
if remote_addr ~= "" and dict:add("visitor:" .. window_prefix .. ":" .. remote_addr, 1, ttl) then
incr("unique_visitor_count:" .. window_prefix, 1)
end
local request_length = tonumber(ngx.var.request_length) or 0
if request_length > 0 then
incr("openresty_rx_bytes:" .. window_prefix, request_length)
end
local bytes_sent = tonumber(ngx.var.bytes_sent) or tonumber(ngx.var.body_bytes_sent) or 0
if bytes_sent > 0 then
incr("openresty_tx_bytes:" .. window_prefix, bytes_sent)
end
// log.lua no longer accumulates business counters (access.log is the authority).
const openRestyObservabilityLogLua = `return
`
// read.lua exposes stub_status-style connection gauges as JSON.
const openRestyObservabilityReadLua = `local cjson = require "cjson.safe"
local dict = ngx.shared.openflare_observability
if not dict then
ngx.status = ngx.HTTP_SERVICE_UNAVAILABLE
ngx.say(cjson.encode({ message = "shared dict unavailable" }))
return
end
local now = ngx.time()
local window_size = ` + openRestyObservabilityWindowSize + `
local window_start = now - (now % window_size)
local current_window = tostring(window_start)
local function read_counter(key)
return tonumber(dict:get(key) or 0) or 0
end
local function read_map(window_id, prefix, list_key)
local result = {}
local raw = dict:get(list_key .. ":" .. window_id)
if not raw or raw == "" then
return result
local function read_stub_status()
local res = ngx.location.capture("/openflare/stub_status")
if not res or res.status ~= 200 or not res.body then
return nil
end
for value in string.gmatch(raw, "[^\n]+") do
result[value] = read_counter(prefix .. ":" .. window_id .. ":" .. value)
end
return result
local body = res.body
local active = tonumber(string.match(body, "Active connections:%s*(%d+)")) or 0
local reading = tonumber(string.match(body, "Reading:%s*(%d+)")) or 0
local writing = tonumber(string.match(body, "Writing:%s*(%d+)")) or 0
local waiting = tonumber(string.match(body, "Waiting:%s*(%d+)")) or 0
return {
active = active,
reading = reading,
writing = writing,
waiting = waiting
}
end
local connections = read_stub_status()
local payload = {
window_started_at_unix = window_start,
window_ended_at_unix = now,
request_count = read_counter("request_count:" .. current_window),
error_count = read_counter("error_count:" .. current_window),
unique_visitor_count = read_counter("unique_visitor_count:" .. current_window),
status_codes = read_map(current_window, "status", "status_keys"),
top_domains = read_map(current_window, "domain", "domain_keys"),
source_countries = {},
openresty_rx_bytes = read_counter("openresty_rx_bytes:" .. current_window),
openresty_tx_bytes = read_counter("openresty_tx_bytes:" .. current_window)
ok = connections ~= nil,
captured_at_unix = ngx.time(),
connections = connections or {
active = 0,
reading = 0,
writing = 0,
waiting = 0
}
}
ngx.header.content_type = "application/json"
ngx.status = ngx.HTTP_OK
ngx.say(cjson.encode(payload))
`
@@ -0,0 +1,35 @@
package nginx
import (
"strings"
"testing"
)
func TestManagedObservabilityLuaIsHealthOnly(t *testing.T) {
t.Parallel()
files := ManagedObservabilityLuaFiles()
var logLua, readLua string
for _, file := range files {
switch file.Path {
case "log.lua":
logLua = file.Content
case "read.lua":
readLua = file.Content
}
}
if logLua == "" || readLua == "" {
t.Fatal("expected log.lua and read.lua")
}
// Business counters must not be written in log phase.
if strings.Contains(logLua, "openresty_rx_bytes") ||
strings.Contains(logLua, "request_count") {
t.Fatal("log.lua must not accumulate business counters")
}
if !strings.Contains(readLua, "connections") || !strings.Contains(readLua, "ok") {
t.Fatal("read.lua must expose ok + connections health snapshot")
}
if strings.Contains(readLua, "top_domains") || strings.Contains(readLua, "request_count") {
t.Fatal("read.lua must not expose business traffic aggregates")
}
}
+30 -9
View File
@@ -84,16 +84,37 @@ func BuildSnapshot(cfg *config.Config, stateStore *state.Store) *protocol.NodeMe
return metric
}
// BuildOpenrestyObservation builds the OpenResty observation protocol model from the managed metrics.
func BuildOpenrestyObservation(managed *ManagedOpenRestyMetrics) *protocol.NodeOpenrestyObservation {
if managed == nil {
return nil
// BuildEdgeHealth builds the edge_health payload from a local probe and node status.
func BuildEdgeHealth(probe *EdgeHealthSnapshot, openrestyStatus, openrestyMessage string) *protocol.NodeEdgeHealth {
status := strings.TrimSpace(openrestyStatus)
message := strings.TrimSpace(openrestyMessage)
if probe == nil {
if status == "" {
return nil
}
return &protocol.NodeEdgeHealth{
CapturedAtUnix: time.Now().UTC().Unix(),
Status: status,
Message: message,
Connections: 0,
}
}
return &protocol.NodeOpenrestyObservation{
CapturedAtUnix: time.Now().UTC().Unix(),
OpenrestyRxBytes: managed.OpenrestyRxBytes,
OpenrestyTxBytes: managed.OpenrestyTxBytes,
OpenrestyConnections: managed.OpenrestyConnections,
captured := probe.CapturedAtUnix
if captured <= 0 {
captured = time.Now().UTC().Unix()
}
if status == "" {
status = protocol.OpenrestyStatusUnknown
if probe.OK {
status = protocol.OpenrestyStatusHealthy
}
}
return &protocol.NodeEdgeHealth{
CapturedAtUnix: captured,
Status: status,
Message: message,
Connections: probe.Connections,
}
}
@@ -4,48 +4,37 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/agent/config"
"github.com/Rain-kl/Wavelet/internal/apps/agent/protocol"
)
const (
openRestyObservabilityPath = "/openflare/observability"
openRestyStubStatusPath = "/openflare/stub_status"
stubStatusActiveMatchGroupCount = 2
)
const openRestyObservabilityPath = "/openflare/observability"
var stubStatusActivePattern = regexp.MustCompile(`Active connections:\s+(\d+)`)
// ManagedOpenRestyMetrics holds metrics collected from the local OpenResty instance.
type ManagedOpenRestyMetrics struct {
TrafficReport *protocol.NodeTrafficReport
OpenrestyRxBytes int64
OpenrestyTxBytes int64
OpenrestyConnections int64
// EdgeHealthSnapshot is the L2 OpenResty health probe result.
type EdgeHealthSnapshot struct {
OK bool
CapturedAtUnix int64
Connections int64
Reading int64
Writing int64
Waiting int64
}
type openRestyObservabilityResponse struct {
WindowStartedAtUnix int64 `json:"window_started_at_unix"`
WindowEndedAtUnix int64 `json:"window_ended_at_unix"`
RequestCount int64 `json:"request_count"`
ErrorCount int64 `json:"error_count"`
UniqueVisitorCount int64 `json:"unique_visitor_count"`
StatusCodes map[string]int64 `json:"status_codes"`
TopDomains map[string]int64 `json:"top_domains"`
SourceCountries map[string]int64 `json:"source_countries"`
OpenrestyRxBytes int64 `json:"openresty_rx_bytes"`
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
OK bool `json:"ok"`
CapturedAtUnix int64 `json:"captured_at_unix"`
Connections struct {
Active int64 `json:"active"`
Reading int64 `json:"reading"`
Writing int64 `json:"writing"`
Waiting int64 `json:"waiting"`
} `json:"connections"`
}
// CollectManagedOpenRestyMetrics collects metrics from the local OpenResty observability endpoints.
func CollectManagedOpenRestyMetrics(ctx context.Context, cfg *config.Config) *ManagedOpenRestyMetrics {
// CollectEdgeHealth probes the local OpenResty observability JSON endpoint.
func CollectEdgeHealth(ctx context.Context, cfg *config.Config) *EdgeHealthSnapshot {
if cfg == nil || cfg.OpenrestyObservabilityPort <= 0 {
return nil
}
@@ -53,31 +42,23 @@ func CollectManagedOpenRestyMetrics(ctx context.Context, cfg *config.Config) *Ma
baseURL := fmt.Sprintf("http://127.0.0.1:%d", cfg.OpenrestyObservabilityPort)
client := &http.Client{Timeout: 1500 * time.Millisecond}
observabilityResp := openRestyObservabilityResponse{}
if err := fetchLocalJSON(ctx, client, baseURL+openRestyObservabilityPath, &observabilityResp); err != nil {
var resp openRestyObservabilityResponse
if err := fetchLocalJSON(ctx, client, baseURL+openRestyObservabilityPath, &resp); err != nil {
return nil
}
result := &ManagedOpenRestyMetrics{
TrafficReport: &protocol.NodeTrafficReport{
WindowStartedAtUnix: observabilityResp.WindowStartedAtUnix,
WindowEndedAtUnix: observabilityResp.WindowEndedAtUnix,
RequestCount: observabilityResp.RequestCount,
ErrorCount: observabilityResp.ErrorCount,
UniqueVisitorCount: observabilityResp.UniqueVisitorCount,
StatusCodes: normalizeCountMap(observabilityResp.StatusCodes),
TopDomains: normalizeCountMap(observabilityResp.TopDomains),
SourceCountries: normalizeCountMap(observabilityResp.SourceCountries),
},
OpenrestyRxBytes: observabilityResp.OpenrestyRxBytes,
OpenrestyTxBytes: observabilityResp.OpenrestyTxBytes,
captured := resp.CapturedAtUnix
if captured <= 0 {
captured = time.Now().UTC().Unix()
}
if text, err := fetchLocalText(ctx, client, baseURL+openRestyStubStatusPath); err == nil {
result.OpenrestyConnections = parseStubStatusActiveConnections(text)
return &EdgeHealthSnapshot{
OK: resp.OK,
CapturedAtUnix: captured,
Connections: resp.Connections.Active,
Reading: resp.Connections.Reading,
Writing: resp.Connections.Writing,
Waiting: resp.Connections.Waiting,
}
return result
}
func fetchLocalJSON(ctx context.Context, client *http.Client, url string, target any) error {
@@ -95,50 +76,3 @@ func fetchLocalJSON(ctx context.Context, client *http.Client, url string, target
}
return json.NewDecoder(resp.Body).Decode(target)
}
func fetchLocalText(ctx context.Context, client *http.Client, url string) (string, error) {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return "", err
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected local stub status: %s", resp.Status)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return string(data), nil
}
func parseStubStatusActiveConnections(raw string) int64 {
matches := stubStatusActivePattern.FindStringSubmatch(raw)
if len(matches) != stubStatusActiveMatchGroupCount {
return 0
}
value, err := strconv.ParseInt(matches[1], 10, 64)
if err != nil {
return 0
}
return value
}
func normalizeCountMap(values map[string]int64) map[string]int64 {
if len(values) == 0 {
return map[string]int64{}
}
result := make(map[string]int64, len(values))
for key, value := range values {
key = strings.TrimSpace(key)
if key == "" || value <= 0 {
continue
}
result[key] = value
}
return result
}
@@ -2,82 +2,63 @@ package observability
import (
"context"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Rain-kl/Wavelet/internal/apps/agent/config"
)
func TestCollectManagedOpenRestyMetrics(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("Listen failed: %v", err)
}
port := listener.Addr().(*net.TCPAddr).Port
mux := http.NewServeMux()
mux.HandleFunc(openRestyObservabilityPath, func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"window_started_at_unix":1710403200,"window_ended_at_unix":1710403210,"request_count":12,"error_count":2,"unique_visitor_count":5,"status_codes":{"200":10,"502":2},"top_domains":{"app.example.com":9,"api.example.com":3},"source_countries":{},"openresty_rx_bytes":4096,"openresty_tx_bytes":8192}`))
})
mux.HandleFunc(openRestyStubStatusPath, func(writer http.ResponseWriter, request *http.Request) {
_, _ = writer.Write([]byte("Active connections: 7 \nserver accepts handled requests\n 10 10 12 \nReading: 1 Writing: 2 Waiting: 4 \n"))
})
server := httptest.NewUnstartedServer(mux)
server.Listener = listener
server.Start()
func TestCollectEdgeHealth(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/openflare/observability" {
http.NotFound(w, r)
return
}
_, _ = w.Write([]byte(`{"ok":true,"captured_at_unix":1710403200,"connections":{"active":12,"reading":1,"writing":2,"waiting":9}}`))
}))
defer server.Close()
metrics := CollectManagedOpenRestyMetrics(context.Background(), &config.Config{
OpenrestyObservabilityPort: port,
health := CollectEdgeHealth(context.Background(), &config.Config{
OpenrestyObservabilityPort: mustPort(server.URL),
})
if metrics == nil || metrics.TrafficReport == nil {
t.Fatalf("expected managed openresty metrics, got %+v", metrics)
if health == nil {
t.Fatal("expected edge health")
}
if metrics.TrafficReport.RequestCount != 12 || metrics.TrafficReport.ErrorCount != 2 {
t.Fatalf("unexpected traffic report: %+v", metrics.TrafficReport)
}
if metrics.OpenrestyRxBytes != 4096 || metrics.OpenrestyTxBytes != 8192 {
t.Fatalf("unexpected openresty byte counters: %+v", metrics)
}
if metrics.OpenrestyConnections != 7 {
t.Fatalf("unexpected openresty connections: %+v", metrics)
if !health.OK || health.Connections != 12 {
t.Fatalf("unexpected health: %+v", health)
}
}
func TestParseStubStatusActiveConnections(t *testing.T) {
if value := parseStubStatusActiveConnections("Active connections: 19\n"); value != 19 {
t.Fatalf("unexpected active connections: %d", value)
func mustPort(rawURL string) int {
u := rawURL
idx := stringsLastColon(u)
if idx < 0 {
return 0
}
var port int
for _, ch := range u[idx+1:] {
if ch < '0' || ch > '9' {
break
}
port = port*10 + int(ch-'0')
}
return port
}
func TestNormalizeCountMapDropsEmptyKeys(t *testing.T) {
normalized := normalizeCountMap(map[string]int64{
"": 4,
" 200 ": 3,
"app.example.com": 0,
})
if len(normalized) != 1 || normalized["200"] != 3 {
t.Fatalf("unexpected normalized map: %+v", normalized)
func stringsLastColon(s string) int {
for i := len(s) - 1; i >= 0; i-- {
if s[i] == ':' {
return i
}
}
return -1
}
func TestCollectManagedOpenRestyMetricsHandlesUnavailableEndpoint(t *testing.T) {
cfg := &config.Config{OpenrestyObservabilityPort: 1}
if metrics := CollectManagedOpenRestyMetrics(context.Background(), cfg); metrics != nil {
t.Fatalf("expected nil metrics for unavailable endpoint, got %+v", metrics)
}
}
func TestOpenRestyObservabilityPathsAreStable(t *testing.T) {
if !strings.HasPrefix(openRestyObservabilityPath, "/openflare/") {
t.Fatalf("unexpected observability path: %s", openRestyObservabilityPath)
}
if !strings.HasPrefix(openRestyStubStatusPath, "/openflare/") {
t.Fatalf("unexpected stub status path: %s", openRestyStubStatusPath)
func TestCollectEdgeHealthHandlesUnavailableEndpoint(t *testing.T) {
if health := CollectEdgeHealth(context.Background(), &config.Config{
OpenrestyObservabilityPort: 1,
}); health != nil {
t.Fatalf("expected nil health, got %+v", health)
}
}
+35 -145
View File
@@ -6,10 +6,8 @@ import (
"errors"
"io"
"log/slog"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
"time"
@@ -20,63 +18,42 @@ import (
)
type accessLogRecord struct {
Timestamp string `json:"ts"`
Host string `json:"host"`
RemoteAddr string `json:"remote_addr"`
Path string `json:"path"`
Status int `json:"status"`
BytesSent int64 `json:"bytes_sent"`
RequestLength int64 `json:"request_length"`
Timestamp string `json:"ts"`
Host string `json:"host"`
RemoteAddr string `json:"remote_addr"`
Path string `json:"path"`
Status int `json:"status"`
BytesSent int64 `json:"bytes_sent"`
RequestLength int64 `json:"request_length"`
RequestTime float64 `json:"request_time"`
}
const (
combinedAccessLogMatchGroupCount = 5
trafficTopDomainsLimit = 8
// requestTimeSecondsToMs converts OpenResty $request_time (seconds float) to ms.
requestTimeSecondsToMs = 1000.0
// roundHalfUp is added before int64 truncate to round to nearest millisecond.
roundHalfUp = 0.5
)
var combinedAccessLogPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+\S+\s+\[([^]]+)]\s+"\S+\s+(\S+)(?:\s+[^"]*)?"\s+(\d{3})\s+\S+`)
// trafficAggregate collects access-log facts for the current heartbeat window.
// Pre-aggregation (UV/TopN/TrafficReport) is intentionally not built.
type trafficAggregate struct {
windowStartedAt time.Time
windowEndedAt time.Time
requestCount int64
errorCount int64
openrestyRxBytes int64
openrestyTxBytes int64
statusCodes map[string]int64
topDomains map[string]int64
visitors map[string]struct{}
logs []protocol.NodeAccessLog
logs []protocol.NodeAccessLog
}
// BuildTrafficReport generates a traffic report using access logs or falling back to managed metrics.
func BuildTrafficReport(cfg *config.Config, stateStore *state.Store, managed *ManagedOpenRestyMetrics) *protocol.NodeTrafficReport {
report, _, _ := BuildTrafficObservability(cfg, stateStore, managed)
return report
}
// BuildTrafficObservability returns the traffic report, parsed access logs, and managed metrics.
func BuildTrafficObservability(cfg *config.Config, stateStore *state.Store, managed *ManagedOpenRestyMetrics) (*protocol.NodeTrafficReport, []protocol.NodeAccessLog, *ManagedOpenRestyMetrics) {
// CollectAccessLogs tails access.log and returns L1 fact rows for the current heartbeat.
func CollectAccessLogs(cfg *config.Config, stateStore *state.Store) []protocol.NodeAccessLog {
if cfg == nil || stateStore == nil {
if managed != nil && managed.TrafficReport != nil {
return managed.TrafficReport, nil, managed
}
return nil, nil, managed
return nil
}
aggregate := readAccessLogDelta(cfg, stateStore)
var accessLogs []protocol.NodeAccessLog
if aggregate != nil {
accessLogs = aggregate.accessLogs()
}
if managed != nil && managed.TrafficReport != nil {
return managed.TrafficReport, accessLogs, managed
}
if aggregate == nil {
return nil, accessLogs, managed
return nil
}
fallbackManaged := aggregate.managedMetrics()
return aggregate.report(), accessLogs, fallbackManaged
return aggregate.accessLogs()
}
func readAccessLogDelta(cfg *config.Config, stateStore *state.Store) *trafficAggregate {
@@ -149,11 +126,7 @@ func managedAccessLogPath(cfg *config.Config) string {
}
func newTrafficAggregate() *trafficAggregate {
return &trafficAggregate{
statusCodes: make(map[string]int64),
topDomains: make(map[string]int64),
visitors: make(map[string]struct{}),
}
return &trafficAggregate{}
}
func (aggregate *trafficAggregate) consume(line []byte) {
@@ -167,39 +140,15 @@ func (aggregate *trafficAggregate) consume(line []byte) {
return
}
if aggregate.windowStartedAt.IsZero() || record.Timestamp.Before(aggregate.windowStartedAt) {
aggregate.windowStartedAt = record.Timestamp
}
if aggregate.windowEndedAt.IsZero() || record.Timestamp.After(aggregate.windowEndedAt) {
aggregate.windowEndedAt = record.Timestamp
}
aggregate.requestCount++
if record.Status >= http.StatusInternalServerError {
aggregate.errorCount++
}
if record.Status > 0 {
aggregate.statusCodes[strconv.Itoa(record.Status)]++
}
if record.RequestLength > 0 {
aggregate.openrestyRxBytes += record.RequestLength
}
if record.BytesSent > 0 {
aggregate.openrestyTxBytes += record.BytesSent
}
if host := strings.TrimSpace(record.Host); host != "" {
aggregate.topDomains[host]++
}
if remoteAddr := strings.TrimSpace(record.RemoteAddr); remoteAddr != "" {
aggregate.visitors[remoteAddr] = struct{}{}
}
aggregate.logs = append(aggregate.logs, protocol.NodeAccessLog{
LoggedAtUnix: record.Timestamp.Unix(),
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
Host: strings.TrimSpace(record.Host),
Path: normalizeAccessLogPath(record.Path),
StatusCode: record.Status,
BytesSent: record.BytesSent,
LoggedAtUnix: record.Timestamp.Unix(),
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
Host: strings.TrimSpace(record.Host),
Path: normalizeAccessLogPath(record.Path),
StatusCode: record.Status,
BytesSent: record.BytesSent,
RequestLength: record.RequestLength,
RequestTimeMs: record.RequestTimeMs,
})
}
@@ -211,6 +160,7 @@ type parsedAccessLogRecord struct {
Status int
BytesSent int64
RequestLength int64
RequestTimeMs int64
}
func parseAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
@@ -230,6 +180,10 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
if err != nil {
return parsedAccessLogRecord{}, false
}
requestTimeMs := int64(0)
if record.RequestTime > 0 {
requestTimeMs = int64(record.RequestTime*requestTimeSecondsToMs + roundHalfUp)
}
return parsedAccessLogRecord{
Timestamp: timestamp,
Host: strings.TrimSpace(record.Host),
@@ -238,6 +192,7 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
Status: record.Status,
BytesSent: record.BytesSent,
RequestLength: record.RequestLength,
RequestTimeMs: requestTimeMs,
}, true
}
@@ -262,23 +217,6 @@ func parseCombinedAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
}, true
}
func (aggregate *trafficAggregate) report() *protocol.NodeTrafficReport {
if aggregate.requestCount == 0 || aggregate.windowStartedAt.IsZero() || aggregate.windowEndedAt.IsZero() {
return nil
}
return &protocol.NodeTrafficReport{
WindowStartedAtUnix: aggregate.windowStartedAt.Unix(),
WindowEndedAtUnix: aggregate.windowEndedAt.Unix(),
RequestCount: aggregate.requestCount,
ErrorCount: aggregate.errorCount,
UniqueVisitorCount: int64(len(aggregate.visitors)),
StatusCodes: cloneTrafficCounts(aggregate.statusCodes, 0),
TopDomains: topCounts(aggregate.topDomains, trafficTopDomainsLimit),
SourceCountries: map[string]int64{},
}
}
func (aggregate *trafficAggregate) accessLogs() []protocol.NodeAccessLog {
if aggregate == nil || len(aggregate.logs) == 0 {
return []protocol.NodeAccessLog{}
@@ -286,21 +224,6 @@ func (aggregate *trafficAggregate) accessLogs() []protocol.NodeAccessLog {
return append([]protocol.NodeAccessLog(nil), aggregate.logs...)
}
func (aggregate *trafficAggregate) managedMetrics() *ManagedOpenRestyMetrics {
if aggregate == nil {
return nil
}
report := aggregate.report()
if report == nil && aggregate.openrestyRxBytes <= 0 && aggregate.openrestyTxBytes <= 0 {
return nil
}
return &ManagedOpenRestyMetrics{
TrafficReport: report,
OpenrestyRxBytes: aggregate.openrestyRxBytes,
OpenrestyTxBytes: aggregate.openrestyTxBytes,
}
}
func parseAccessLogTime(value string) (time.Time, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
@@ -313,35 +236,6 @@ func parseAccessLogTime(value string) (time.Time, error) {
return time.Parse("02/Jan/2006:15:04:05 -0700", trimmed)
}
func cloneTrafficCounts(values map[string]int64, limit int) map[string]int64 {
if len(values) == 0 {
return map[string]int64{}
}
items := make([]trafficCountItem, 0, len(values))
for key, value := range values {
items = append(items, trafficCountItem{key: key, value: value})
}
sort.Slice(items, func(i int, j int) bool {
if items[i].value == items[j].value {
return items[i].key < items[j].key
}
return items[i].value > items[j].value
})
if limit > 0 && len(items) > limit {
items = items[:limit]
}
result := make(map[string]int64, len(items))
for _, item := range items {
result[item.key] = item.value
}
return result
}
type trafficCountItem struct {
key string
value int64
}
const accessLogPathMaxRunes = 100
func normalizeAccessLogPath(value string) string {
@@ -365,7 +259,3 @@ func truncateAccessLogPath(value string) string {
}
return string(runes[:accessLogPathMaxRunes])
}
func topCounts(values map[string]int64, limit int) map[string]int64 {
return cloneTrafficCounts(values, limit)
}
+25 -123
View File
@@ -7,39 +7,33 @@ import (
"testing"
"github.com/Rain-kl/Wavelet/internal/apps/agent/config"
"github.com/Rain-kl/Wavelet/internal/apps/agent/protocol"
"github.com/Rain-kl/Wavelet/internal/apps/agent/state"
)
func TestBuildTrafficReportAggregatesManagedAccessLog(t *testing.T) {
func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
tempDir := t.TempDir()
routeConfigPath := filepath.Join(tempDir, "conf.d", "openflare_routes.conf")
if err := os.MkdirAll(filepath.Dir(routeConfigPath), 0o755); err != nil {
t.Fatalf("MkdirAll failed: %v", err)
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "openflare_access.log")
logPath := filepath.Join(tempDir, "openflare_access.log")
content := []byte(
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"app.example.com\",\"path\":\"/healthz\",\"remote_addr\":\"10.0.0.2\",\"status\":503}\n" +
"{\"ts\":\"2026-03-14T08:00:08Z\",\"host\":\"api.example.com\",\"path\":\"/api\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n",
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512,\"request_time\":0.015}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256,\"request_time\":0.008}\n",
)
if err := os.WriteFile(logPath, content, 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
stateStore := state.NewStore(filepath.Join(tempDir, "state.json"))
report := BuildTrafficReport(&config.Config{AccessLogPath: logPath}, stateStore, nil)
if report == nil {
t.Fatal("expected traffic report")
accessLogs := CollectAccessLogs(&config.Config{AccessLogPath: logPath}, stateStore)
if len(accessLogs) != 2 {
t.Fatalf("expected access logs, got %+v", accessLogs)
}
if report.RequestCount != 3 || report.ErrorCount != 1 || report.UniqueVisitorCount != 2 {
t.Fatalf("unexpected traffic report counters: %+v", report)
if accessLogs[0].BytesSent != 512 || accessLogs[0].RequestLength != 128 {
t.Fatalf("unexpected first log: %+v", accessLogs[0])
}
if report.StatusCodes["200"] != 2 || report.StatusCodes["503"] != 1 {
t.Fatalf("unexpected status codes: %+v", report.StatusCodes)
if accessLogs[0].RequestTimeMs != 15 {
t.Fatalf("request_time_ms = %d, want 15", accessLogs[0].RequestTimeMs)
}
if report.TopDomains["app.example.com"] != 2 || report.TopDomains["api.example.com"] != 1 {
t.Fatalf("unexpected top domains: %+v", report.TopDomains)
if accessLogs[0].Path != "/login" || accessLogs[1].Path != "/v1/ping" {
t.Fatalf("unexpected access log paths: %+v", accessLogs)
}
snapshot, err := stateStore.Load()
@@ -50,20 +44,16 @@ func TestBuildTrafficReportAggregatesManagedAccessLog(t *testing.T) {
t.Fatalf("unexpected access log offset: %d", snapshot.AccessLogOffset)
}
secondReport := BuildTrafficReport(&config.Config{AccessLogPath: logPath}, stateStore, nil)
if secondReport != nil {
t.Fatalf("expected no report without appended lines, got %+v", secondReport)
moreLogs := CollectAccessLogs(&config.Config{AccessLogPath: logPath}, stateStore)
if len(moreLogs) != 0 {
t.Fatalf("expected no new logs, got %+v", moreLogs)
}
}
func TestBuildTrafficReportResetsOffsetAfterTruncate(t *testing.T) {
func TestCollectAccessLogsResetsOffsetAfterTruncate(t *testing.T) {
tempDir := t.TempDir()
routeConfigPath := filepath.Join(tempDir, "conf.d", "openflare_routes.conf")
if err := os.MkdirAll(filepath.Dir(routeConfigPath), 0o755); err != nil {
t.Fatalf("MkdirAll failed: %v", err)
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "openflare_access.log")
if err := os.WriteFile(logPath, []byte("{\"ts\":\"2026-03-14T09:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/\",\"remote_addr\":\"10.0.0.3\",\"status\":200}\n"), 0o644); err != nil {
logPath := filepath.Join(tempDir, "openflare_access.log")
if err := os.WriteFile(logPath, []byte("{\"ts\":\"2026-03-14T09:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/\",\"remote_addr\":\"10.0.0.3\",\"status\":200,\"bytes_sent\":1}\n"), 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
@@ -72,50 +62,15 @@ func TestBuildTrafficReportResetsOffsetAfterTruncate(t *testing.T) {
t.Fatalf("Save failed: %v", err)
}
report := BuildTrafficReport(&config.Config{AccessLogPath: logPath}, stateStore, nil)
if report == nil || report.RequestCount != 1 {
t.Fatalf("expected one request after truncate reset, got %+v", report)
accessLogs := CollectAccessLogs(&config.Config{AccessLogPath: logPath}, stateStore)
if len(accessLogs) != 1 {
t.Fatalf("expected one access log after truncate reset, got %+v", accessLogs)
}
}
func TestBuildTrafficObservabilityReturnsAccessLogs(t *testing.T) {
func TestCollectAccessLogsTruncatesLongAccessLogPath(t *testing.T) {
tempDir := t.TempDir()
routeConfigPath := filepath.Join(tempDir, "conf.d", "openflare_routes.conf")
if err := os.MkdirAll(filepath.Dir(routeConfigPath), 0o755); err != nil {
t.Fatalf("MkdirAll failed: %v", err)
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "openflare_access.log")
content := []byte(
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256}\n",
)
if err := os.WriteFile(logPath, content, 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
stateStore := state.NewStore(filepath.Join(tempDir, "state.json"))
report, accessLogs, fallbackMetrics := BuildTrafficObservability(&config.Config{AccessLogPath: logPath}, stateStore, nil)
if report == nil || report.RequestCount != 2 {
t.Fatalf("expected traffic report, got %+v", report)
}
if len(accessLogs) != 2 {
t.Fatalf("expected access logs, got %+v", accessLogs)
}
if fallbackMetrics == nil || fallbackMetrics.OpenrestyRxBytes != 192 || fallbackMetrics.OpenrestyTxBytes != 768 {
t.Fatalf("expected fallback throughput metrics, got %+v", fallbackMetrics)
}
if accessLogs[0].Path != "/login" || accessLogs[1].Path != "/v1/ping" {
t.Fatalf("unexpected access log paths: %+v", accessLogs)
}
}
func TestBuildTrafficObservabilityTruncatesLongAccessLogPath(t *testing.T) {
tempDir := t.TempDir()
routeConfigPath := filepath.Join(tempDir, "conf.d", "openflare_routes.conf")
if err := os.MkdirAll(filepath.Dir(routeConfigPath), 0o755); err != nil {
t.Fatalf("MkdirAll failed: %v", err)
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "openflare_access.log")
logPath := filepath.Join(tempDir, "openflare_access.log")
longPath := "/" + strings.Repeat("a", 140)
content := []byte(
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"" + longPath + "\",\"remote_addr\":\"10.0.0.1\",\"status\":200}\n",
@@ -125,7 +80,7 @@ func TestBuildTrafficObservabilityTruncatesLongAccessLogPath(t *testing.T) {
}
stateStore := state.NewStore(filepath.Join(tempDir, "state.json"))
_, accessLogs, _ := BuildTrafficObservability(&config.Config{AccessLogPath: logPath}, stateStore, nil)
accessLogs := CollectAccessLogs(&config.Config{AccessLogPath: logPath}, stateStore)
if len(accessLogs) != 1 {
t.Fatalf("expected one access log, got %+v", accessLogs)
}
@@ -133,56 +88,3 @@ func TestBuildTrafficObservabilityTruncatesLongAccessLogPath(t *testing.T) {
t.Fatalf("expected truncated path length %d, got %d (%q)", accessLogPathMaxRunes, got, accessLogs[0].Path)
}
}
func TestBuildTrafficReportParsesCombinedAccessLog(t *testing.T) {
tempDir := t.TempDir()
routeConfigPath := filepath.Join(tempDir, "conf.d", "openflare_routes.conf")
if err := os.MkdirAll(filepath.Dir(routeConfigPath), 0o755); err != nil {
t.Fatalf("MkdirAll failed: %v", err)
}
logPath := filepath.Join(filepath.Dir(routeConfigPath), "openflare_access.log")
content := []byte(
"10.0.0.1 - - [14/Mar/2026:08:00:00 +0000] \"GET / HTTP/1.1\" 200 123 \"-\" \"curl/8.0\"\n" +
"10.0.0.2 - - [14/Mar/2026:08:00:05 +0000] \"GET /healthz HTTP/1.1\" 502 64 \"-\" \"curl/8.0\"\n" +
"10.0.0.1 - - [14/Mar/2026:08:00:10 +0000] \"GET /api HTTP/1.1\" 200 256 \"-\" \"curl/8.0\"\n",
)
if err := os.WriteFile(logPath, content, 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
stateStore := state.NewStore(filepath.Join(tempDir, "state.json"))
report := BuildTrafficReport(&config.Config{AccessLogPath: logPath}, stateStore, nil)
if report == nil {
t.Fatal("expected traffic report from combined access log")
}
if report.RequestCount != 3 || report.ErrorCount != 1 || report.UniqueVisitorCount != 2 {
t.Fatalf("unexpected combined log counters: %+v", report)
}
if report.StatusCodes["200"] != 2 || report.StatusCodes["502"] != 1 {
t.Fatalf("unexpected combined log status codes: %+v", report.StatusCodes)
}
if len(report.TopDomains) != 0 {
t.Fatalf("expected combined access log to omit top domains when host is unavailable, got %+v", report.TopDomains)
}
}
func TestBuildTrafficReportReturnsManagedWindowEvenWhenRequestCountZero(t *testing.T) {
report := BuildTrafficReport(nil, nil, &ManagedOpenRestyMetrics{
TrafficReport: &protocol.NodeTrafficReport{
WindowStartedAtUnix: 1710403200,
WindowEndedAtUnix: 1710403260,
RequestCount: 0,
ErrorCount: 0,
UniqueVisitorCount: 0,
StatusCodes: map[string]int64{},
TopDomains: map[string]int64{},
SourceCountries: map[string]int64{},
},
})
if report == nil {
t.Fatal("expected managed traffic report to be returned even when request count is zero")
}
if report.RequestCount != 0 || report.WindowStartedAtUnix != 1710403200 || report.WindowEndedAtUnix != 1710403260 {
t.Fatalf("unexpected managed traffic report: %+v", report)
}
}
+2 -5
View File
@@ -33,11 +33,8 @@ type NodeSystemProfile = pkgprotocol.NodeSystemProfile
// NodeMetricSnapshot is an alias for pkgprotocol.NodeMetricSnapshot.
type NodeMetricSnapshot = pkgprotocol.NodeMetricSnapshot
// NodeOpenrestyObservation is an alias for pkgprotocol.NodeOpenrestyObservation.
type NodeOpenrestyObservation = pkgprotocol.NodeOpenrestyObservation
// NodeTrafficReport is an alias for pkgprotocol.NodeTrafficReport.
type NodeTrafficReport = pkgprotocol.NodeTrafficReport
// NodeEdgeHealth is an alias for pkgprotocol.NodeEdgeHealth.
type NodeEdgeHealth = pkgprotocol.NodeEdgeHealth
// NodeAccessLog is an alias for pkgprotocol.NodeAccessLog.
type NodeAccessLog = pkgprotocol.NodeAccessLog
@@ -14,14 +14,13 @@ import (
const observabilityBufferWindowSeconds = 60
// ObservabilityBufferRecord stores observability data for a single time window.
// ObservabilityBufferRecord stores observability facts for a single time window.
type ObservabilityBufferRecord struct {
WindowStartedAtUnix int64 `json:"window_started_at_unix"`
Snapshot *protocol.NodeMetricSnapshot `json:"snapshot,omitempty"`
OpenrestyObservation *protocol.NodeOpenrestyObservation `json:"openresty_observation,omitempty"`
TrafficReport *protocol.NodeTrafficReport `json:"traffic_report,omitempty"`
AccessLogs []protocol.NodeAccessLog `json:"access_logs,omitempty"`
QueuedAtUnix int64 `json:"queued_at_unix"`
WindowStartedAtUnix int64 `json:"window_started_at_unix"`
HostMetrics *protocol.NodeMetricSnapshot `json:"host_metrics,omitempty"`
EdgeHealth *protocol.NodeEdgeHealth `json:"edge_health,omitempty"`
AccessLogs []protocol.NodeAccessLog `json:"access_logs,omitempty"`
QueuedAtUnix int64 `json:"queued_at_unix"`
}
// ObservabilityBufferStore persists observability records to disk for replay on heartbeat.
@@ -39,7 +38,7 @@ func NewObservabilityBufferStore(path string) *ObservabilityBufferStore {
// Upsert inserts or merges an observability record and prunes entries older than retainAfterUnix.
func (s *ObservabilityBufferStore) Upsert(record ObservabilityBufferRecord, retainAfterUnix int64) error {
if s == nil || record.WindowStartedAtUnix <= 0 || (record.Snapshot == nil && record.OpenrestyObservation == nil && record.TrafficReport == nil && len(record.AccessLogs) == 0) {
if s == nil || record.WindowStartedAtUnix <= 0 || (record.HostMetrics == nil && record.EdgeHealth == nil && len(record.AccessLogs) == 0) {
return nil
}
s.mu.Lock()
@@ -70,14 +69,11 @@ func (s *ObservabilityBufferStore) Upsert(record ObservabilityBufferRecord, reta
func mergeObservabilityBufferRecord(existing ObservabilityBufferRecord, incoming ObservabilityBufferRecord) ObservabilityBufferRecord {
merged := existing
if incoming.Snapshot != nil {
merged.Snapshot = incoming.Snapshot
if incoming.HostMetrics != nil {
merged.HostMetrics = incoming.HostMetrics
}
if incoming.OpenrestyObservation != nil {
merged.OpenrestyObservation = incoming.OpenrestyObservation
}
if incoming.TrafficReport != nil {
merged.TrafficReport = incoming.TrafficReport
if incoming.EdgeHealth != nil {
merged.EdgeHealth = incoming.EdgeHealth
}
merged.AccessLogs = mergeAccessLogs(existing.AccessLogs, incoming.AccessLogs)
if incoming.QueuedAtUnix > 0 {
@@ -222,18 +218,15 @@ func (s *ObservabilityBufferStore) saveUnlocked(records []ObservabilityBufferRec
return nil
}
// ObservabilityWindowStartedAt calculates the start of the 60-second window for the given metrics, openresty observation, or traffic report.
func ObservabilityWindowStartedAt(snapshot *protocol.NodeMetricSnapshot, openresty *protocol.NodeOpenrestyObservation, traffic *protocol.NodeTrafficReport) int64 {
if traffic != nil && traffic.WindowStartedAtUnix > 0 {
return traffic.WindowStartedAtUnix - (traffic.WindowStartedAtUnix % observabilityBufferWindowSeconds)
// ObservabilityWindowStartedAt returns the 60s window start for host metrics or edge health.
func ObservabilityWindowStartedAt(hostMetrics *protocol.NodeMetricSnapshot, edgeHealth *protocol.NodeEdgeHealth) int64 {
if edgeHealth != nil && edgeHealth.CapturedAtUnix > 0 {
return edgeHealth.CapturedAtUnix - (edgeHealth.CapturedAtUnix % observabilityBufferWindowSeconds)
}
if openresty != nil && openresty.CapturedAtUnix > 0 {
return openresty.CapturedAtUnix - (openresty.CapturedAtUnix % observabilityBufferWindowSeconds)
}
if snapshot == nil || snapshot.CapturedAtUnix <= 0 {
if hostMetrics == nil || hostMetrics.CapturedAtUnix <= 0 {
return 0
}
return snapshot.CapturedAtUnix - (snapshot.CapturedAtUnix % observabilityBufferWindowSeconds)
return hostMetrics.CapturedAtUnix - (hostMetrics.CapturedAtUnix % observabilityBufferWindowSeconds)
}
func pruneObservabilityBufferRecords(records []ObservabilityBufferRecord, retainAfterUnix int64) []ObservabilityBufferRecord {
@@ -12,24 +12,23 @@ func TestObservabilityBufferStoreUpsertReplayAndAck(t *testing.T) {
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
Snapshot: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403205},
TrafficReport: &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403200, WindowEndedAtUnix: 1710403260, RequestCount: 5},
HostMetrics: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403205},
EdgeHealth: &protocol.NodeEdgeHealth{CapturedAtUnix: 1710403205, Connections: 5},
QueuedAtUnix: 1710403205,
}, 1710403000); err != nil {
t.Fatalf("first upsert failed: %v", err)
}
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
Snapshot: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403255},
TrafficReport: &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403200, WindowEndedAtUnix: 1710403260, RequestCount: 12},
HostMetrics: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403255, CPUUsagePercent: 40},
EdgeHealth: &protocol.NodeEdgeHealth{CapturedAtUnix: 1710403255, Connections: 12},
QueuedAtUnix: 1710403255,
}, 1710403000); err != nil {
t.Fatalf("second upsert failed: %v", err)
}
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403260,
Snapshot: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403265},
TrafficReport: &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403260, WindowEndedAtUnix: 1710403320, RequestCount: 2},
HostMetrics: &protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403265},
QueuedAtUnix: 1710403265,
}, 1710403000); err != nil {
t.Fatalf("third upsert failed: %v", err)
@@ -42,7 +41,7 @@ func TestObservabilityBufferStoreUpsertReplayAndAck(t *testing.T) {
if len(records) != 1 {
t.Fatalf("expected one replayable record before current window, got %d", len(records))
}
if records[0].TrafficReport == nil || records[0].TrafficReport.RequestCount != 12 {
if records[0].EdgeHealth == nil || records[0].EdgeHealth.Connections != 12 {
t.Fatalf("expected replayable record to keep latest upsert, got %+v", records[0])
}
@@ -89,10 +88,10 @@ func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) {
}
func TestObservabilityWindowStartedAt(t *testing.T) {
if value := ObservabilityWindowStartedAt(nil, nil, &protocol.NodeTrafficReport{WindowStartedAtUnix: 1710403200}); value != 1710403200 {
t.Fatalf("unexpected traffic window start: %d", value)
if value := ObservabilityWindowStartedAt(nil, &protocol.NodeEdgeHealth{CapturedAtUnix: 1710403259}); value != 1710403200 {
t.Fatalf("unexpected edge-health window start: %d", value)
}
if value := ObservabilityWindowStartedAt(&protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403259}, nil, nil); value != 1710403200 {
t.Fatalf("unexpected snapshot-derived window start: %d", value)
if value := ObservabilityWindowStartedAt(&protocol.NodeMetricSnapshot{CapturedAtUnix: 1710403259}, nil); value != 1710403200 {
t.Fatalf("unexpected host-metrics window start: %d", value)
}
}
+1 -1
View File
@@ -24,7 +24,7 @@ const (
randomTokenBytes = 16
maxDatabaseTextLength = 16000
defaultAgentHeartbeatInterval = 10000 // 默认心跳间隔 10 秒(毫秒)
defaultAgentHeartbeatInterval = 3000 // 默认心跳间隔 3 秒(毫秒)
defaultAgentUpdateRepo = "Rain-kl/OpenFlare"
)
+52 -61
View File
@@ -6,7 +6,6 @@ package agent
import (
"context"
"encoding/json"
"errors"
"log/slog"
"strings"
"time"
@@ -28,16 +27,16 @@ const (
healthEventMessageMaxLength = 4096
)
// PersistHeartbeatObservability stores profile, snapshots, traffic, access logs, and health events.
// PersistHeartbeatObservability stores profile, host metrics, edge health, and access logs.
func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload NodePayload, reportedAt time.Time) {
if strings.TrimSpace(nodeID) == "" {
return
}
if payload.Profile == nil &&
payload.Snapshot == nil &&
payload.TrafficReport == nil &&
payload.HostMetrics == nil &&
payload.EdgeHealth == nil &&
len(payload.AccessLogs) == 0 &&
len(payload.BufferedObservability) == 0 &&
len(payload.Buffered) == 0 &&
payload.HealthEvents == nil {
return
}
@@ -47,7 +46,7 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
return
}
accessLogRecords, err := buildNodeAccessLogRecords(nodeID, payload.AccessLogs, payload.BufferedObservability, reportedAt)
accessLogRecords, err := buildNodeAccessLogRecords(nodeID, payload.AccessLogs, payload.Buffered, reportedAt)
if err != nil {
zap.L().Error("build heartbeat access logs failed", zap.String("node_id", nodeID), zap.Error(err))
return
@@ -68,17 +67,14 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
return
}
if err := persistBufferedObservability(ctx, nodeID, payload.BufferedObservability, reportedAt); err != nil {
if err := persistBufferedObservability(ctx, nodeID, payload.Buffered, reportedAt); err != nil {
zap.L().Error("persist buffered observability failed", zap.String("node_id", nodeID), zap.Error(err))
}
if err := persistNodeMetricSnapshot(ctx, nodeID, payload.Snapshot, reportedAt); err != nil {
if err := persistNodeMetricSnapshot(ctx, nodeID, payload.HostMetrics, reportedAt); err != nil {
zap.L().Error("persist metric snapshot failed", zap.String("node_id", nodeID), zap.Error(err))
}
if err := persistNodeOpenrestyObservation(ctx, nodeID, payload.OpenrestyObservation, reportedAt); err != nil {
zap.L().Error("persist openresty observation failed", zap.String("node_id", nodeID), zap.Error(err))
}
if err := persistNodeTrafficReport(ctx, nodeID, payload.TrafficReport, reportedAt); err != nil {
zap.L().Error("persist traffic report failed", zap.String("node_id", nodeID), zap.Error(err))
if err := persistNodeEdgeHealth(ctx, nodeID, payload.EdgeHealth, payload.OpenrestyStatus, reportedAt); err != nil {
zap.L().Error("persist edge health failed", zap.String("node_id", nodeID), zap.Error(err))
}
if err := persistNodeAccessLogs(ctx, nodeID, accessLogRecords, reportedAt); err != nil {
@@ -88,19 +84,35 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
func persistBufferedObservability(ctx context.Context, nodeID string, records []BufferedObservabilityRecord, reportedAt time.Time) error {
for _, record := range records {
if err := persistNodeMetricSnapshot(ctx, nodeID, record.Snapshot, reportedAt); err != nil {
if err := persistNodeMetricSnapshot(ctx, nodeID, record.HostMetrics, reportedAt); err != nil {
return err
}
if err := persistNodeOpenrestyObservation(ctx, nodeID, record.OpenrestyObservation, reportedAt); err != nil {
return err
}
if err := persistNodeTrafficReport(ctx, nodeID, record.TrafficReport, reportedAt); err != nil {
if err := persistNodeEdgeHealth(ctx, nodeID, record.EdgeHealth, "", reportedAt); err != nil {
return err
}
}
return nil
}
func persistNodeEdgeHealth(ctx context.Context, nodeID string, health *NodeEdgeHealth, fallbackStatus string, reportedAt time.Time) error {
if health == nil {
return nil
}
status := strings.TrimSpace(health.Status)
if status == "" {
status = strings.TrimSpace(fallbackStatus)
}
if status == "" {
status = openrestyStatusUnknown
}
return model.InsertOpenFlareEdgeHealth(ctx, &model.OpenFlareEdgeHealth{
NodeID: nodeID,
CapturedAt: timeFromUnix(health.CapturedAtUnix, reportedAt),
Status: status,
Connections: health.Connections,
})
}
func persistNodeSystemProfile(tx *gorm.DB, nodeID string, profile *NodeSystemProfile, reportedAt time.Time) error {
if profile == nil {
return nil
@@ -158,41 +170,6 @@ func persistNodeMetricSnapshot(ctx context.Context, nodeID string, snapshot *Nod
return model.InsertOpenFlareMetricSnapshot(ctx, record)
}
func persistNodeOpenrestyObservation(ctx context.Context, nodeID string, obs *NodeOpenrestyObservation, reportedAt time.Time) error {
if obs == nil {
return nil
}
record := &model.OpenFlareNodeObservationOpenresty{
NodeID: nodeID,
CapturedAt: timeFromUnix(obs.CapturedAtUnix, reportedAt),
OpenrestyRxBytes: obs.OpenrestyRxBytes,
OpenrestyTxBytes: obs.OpenrestyTxBytes,
OpenrestyConnections: obs.OpenrestyConnections,
}
return model.InsertOpenFlareNodeObservationOpenresty(ctx, record)
}
func persistNodeTrafficReport(ctx context.Context, nodeID string, report *NodeTrafficReport, reportedAt time.Time) error {
if report == nil {
return nil
}
if report.WindowEndedAtUnix > 0 && report.WindowStartedAtUnix > report.WindowEndedAtUnix {
return errors.New("traffic report window_started_at_unix 不能大于 window_ended_at_unix")
}
record := &model.OpenFlareRequestReport{
NodeID: nodeID,
WindowStartedAt: timeFromUnix(report.WindowStartedAtUnix, reportedAt),
WindowEndedAt: timeFromUnix(report.WindowEndedAtUnix, reportedAt),
RequestCount: report.RequestCount,
ErrorCount: report.ErrorCount,
UniqueVisitorCount: report.UniqueVisitorCount,
StatusCodesJSON: marshalJSON(report.StatusCodes),
TopDomainsJSON: marshalJSON(report.TopDomains),
SourceCountriesJSON: marshalJSON(report.SourceCountries),
}
return model.InsertOpenFlareRequestReport(ctx, record)
}
func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered []BufferedObservabilityRecord, reportedAt time.Time) ([]*model.OpenFlareAccessLog, error) {
total := len(direct)
for _, record := range buffered {
@@ -213,15 +190,29 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
records := make([]*model.OpenFlareAccessLog, 0, total)
appendLogs := func(logs []NodeAccessLog) {
for _, item := range logs {
bytesSent := item.BytesSent
if bytesSent < 0 {
bytesSent = 0
}
requestLength := item.RequestLength
if requestLength < 0 {
requestLength = 0
}
requestTimeMs := item.RequestTimeMs
if requestTimeMs < 0 {
requestTimeMs = 0
}
record := &model.OpenFlareAccessLog{
NodeID: nodeID,
LoggedAt: timeFromUnix(item.LoggedAtUnix, reportedAt),
RemoteAddr: strings.TrimSpace(item.RemoteAddr),
Region: "",
Host: strings.TrimSpace(item.Host),
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
StatusCode: item.StatusCode,
BytesSent: item.BytesSent,
NodeID: nodeID,
LoggedAt: timeFromUnix(item.LoggedAtUnix, reportedAt),
RemoteAddr: strings.TrimSpace(item.RemoteAddr),
Region: "",
Host: strings.TrimSpace(item.Host),
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
StatusCode: item.StatusCode,
BytesSent: bytesSent,
RequestLength: requestLength,
RequestTimeMs: requestTimeMs,
}
if resolver != nil {
record.Region = resolver.Resolve(record.RemoteAddr)
@@ -14,11 +14,8 @@ type NodeSystemProfile = pkgprotocol.NodeSystemProfile
// NodeMetricSnapshot holds a point-in-time resource-usage sample from an agent.
type NodeMetricSnapshot = pkgprotocol.NodeMetricSnapshot
// NodeOpenrestyObservation reports the OpenResty process health observed by an agent.
type NodeOpenrestyObservation = pkgprotocol.NodeOpenrestyObservation
// NodeTrafficReport aggregates traffic counters collected by an agent.
type NodeTrafficReport = pkgprotocol.NodeTrafficReport
// NodeEdgeHealth is L2 OpenResty health + connections.
type NodeEdgeHealth = pkgprotocol.NodeEdgeHealth
// NodeAccessLog is a single access-log record forwarded by an agent.
type NodeAccessLog = pkgprotocol.NodeAccessLog
+20 -47
View File
@@ -44,15 +44,13 @@ var (
initOnce sync.Once
metricSnapshotWriter *batchwriter.Writer[analyticsmodel.NodeMetricSnapshot]
requestReportWriter *batchwriter.Writer[analyticsmodel.NodeRequestReport]
openrestyWriter *batchwriter.Writer[analyticsmodel.NodeObsOpenresty]
edgeHealthWriter *batchwriter.Writer[analyticsmodel.NodeEdgeHealth]
frpsWriter *batchwriter.Writer[analyticsmodel.NodeObsFrps]
frpcWriter *batchwriter.Writer[analyticsmodel.NodeObsFrpc]
nodeAccessLogWriter *batchwriter.Writer[analyticsmodel.NodeAccessLog]
metricSnapshotDedup *dedupSet
requestReportDedup *dedupSet
openrestyDedup *dedupSet
edgeHealthDedup *dedupSet
frpsDedup *dedupSet
frpcDedup *dedupSet
)
@@ -65,8 +63,7 @@ func Init(ctx context.Context) {
initOnce.Do(func() {
metricSnapshotDedup = newDedupSet()
requestReportDedup = newDedupSet()
openrestyDedup = newDedupSet()
edgeHealthDedup = newDedupSet()
frpsDedup = newDedupSet()
frpcDedup = newDedupSet()
@@ -76,17 +73,11 @@ func Init(ctx context.Context) {
metricSnapshotDedup,
metricSnapshotKey,
)
requestReportWriter = mustNewObservabilityWriter(
"request_reports",
withFlushRetries(analyticsrepo.BatchInsertNodeRequestReports),
requestReportDedup,
requestReportKey,
)
openrestyWriter = mustNewObservabilityWriter(
"openresty_obs",
withFlushRetries(analyticsrepo.BatchInsertNodeObsOpenresty),
openrestyDedup,
openrestyKey,
edgeHealthWriter = mustNewObservabilityWriter(
"edge_health",
withFlushRetries(analyticsrepo.BatchInsertNodeEdgeHealth),
edgeHealthDedup,
edgeHealthKey,
)
frpsWriter = mustNewObservabilityWriter(
"frps_obs",
@@ -103,8 +94,7 @@ func Init(ctx context.Context) {
nodeAccessLogWriter = mustNewNodeAccessLogWriter()
metricSnapshotWriter.Start(ctx)
requestReportWriter.Start(ctx)
openrestyWriter.Start(ctx)
edgeHealthWriter.Start(ctx)
frpsWriter.Start(ctx)
frpcWriter.Start(ctx)
nodeAccessLogWriter.Start(ctx)
@@ -123,8 +113,7 @@ func Stop(ctx context.Context) error {
var firstErr error
for _, writer := range []batchStopper{
metricSnapshotWriter,
requestReportWriter,
openrestyWriter,
edgeHealthWriter,
frpsWriter,
frpcWriter,
nodeAccessLogWriter,
@@ -143,8 +132,7 @@ func Stop(ctx context.Context) error {
func WriterStats() []batchwriter.Stats {
writers := []statsProvider{
metricSnapshotWriter,
requestReportWriter,
openrestyWriter,
edgeHealthWriter,
frpsWriter,
frpcWriter,
nodeAccessLogWriter,
@@ -164,14 +152,9 @@ func QueueMetricSnapshot(snapshot analyticsmodel.NodeMetricSnapshot) {
queueWithDedup(metricSnapshotWriter, metricSnapshotDedup, metricSnapshotKey(snapshot), snapshot)
}
// QueueRequestReport enqueues a request report for asynchronous flush.
func QueueRequestReport(report analyticsmodel.NodeRequestReport) {
queueWithDedup(requestReportWriter, requestReportDedup, requestReportKey(report), report)
}
// QueueOpenrestyObservation enqueues an OpenResty observation for asynchronous flush.
func QueueOpenrestyObservation(observation analyticsmodel.NodeObsOpenresty) {
queueWithDedup(openrestyWriter, openrestyDedup, openrestyKey(observation), observation)
// QueueEdgeHealth enqueues an L2 edge health snapshot for asynchronous flush.
func QueueEdgeHealth(row analyticsmodel.NodeEdgeHealth) {
queueWithDedup(edgeHealthWriter, edgeHealthDedup, edgeHealthKey(row), row)
}
// QueueFrpsObservation enqueues an FRPS observation for asynchronous flush.
@@ -297,11 +280,10 @@ func withFlushRetries[T any](flush batchwriter.FlushFunc[T]) batchwriter.FlushFu
func wireModelInsertHooks() {
model.SetObservabilityInsertHooks(model.ObservabilityInsertHooks{
QueueMetricSnapshot: QueueMetricSnapshot,
QueueRequestReport: QueueRequestReport,
QueueOpenrestyObservation: QueueOpenrestyObservation,
QueueFrpsObservation: QueueFrpsObservation,
QueueFrpcObservation: QueueFrpcObservation,
QueueMetricSnapshot: QueueMetricSnapshot,
QueueEdgeHealth: QueueEdgeHealth,
QueueFrpsObservation: QueueFrpsObservation,
QueueFrpcObservation: QueueFrpcObservation,
})
model.SetAccessLogInsertHooks(model.AccessLogInsertHooks{
QueueNodeAccessLogs: QueueNodeAccessLogs,
@@ -312,17 +294,8 @@ func metricSnapshotKey(snapshot analyticsmodel.NodeMetricSnapshot) string {
return fmt.Sprintf("%s|%d", snapshot.NodeID, snapshot.CapturedAt.UTC().UnixNano())
}
func requestReportKey(report analyticsmodel.NodeRequestReport) string {
return fmt.Sprintf(
"%s|%d|%d",
report.NodeID,
report.WindowStartedAt.UTC().UnixNano(),
report.WindowEndedAt.UTC().UnixNano(),
)
}
func openrestyKey(observation analyticsmodel.NodeObsOpenresty) string {
return fmt.Sprintf("%s|%d", observation.NodeID, observation.CapturedAt.UTC().UnixNano())
func edgeHealthKey(row analyticsmodel.NodeEdgeHealth) string {
return fmt.Sprintf("%s|%d", row.NodeID, row.CapturedAt.UTC().UnixNano())
}
func frpsKey(observation analyticsmodel.NodeObsFrps) string {
+2 -2
View File
@@ -31,8 +31,8 @@ func computeNodeStatus(node *model.OpenFlareNode) string {
if node.LastSeenAt == nil || node.LastSeenAt.IsZero() {
return nodeStatusPending
}
// 使用默认阈值 2 分钟
threshold := 2 * time.Minute
// 默认离线阈值 60 秒(与 node_offline_threshold 默认一致)
threshold := 60 * time.Second
if time.Since(*node.LastSeenAt) > threshold {
return nodeStatusOffline
}
+48 -52
View File
@@ -122,19 +122,10 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
if err != nil {
return nil, err
}
latestTrafficRows, err := model.ListOpenFlareLatestRequestReportsSince(ctx, "", since)
if err != nil {
return nil, err
}
// Bounded raw windows remain for distributions and trend fallbacks; trends prefer hourly rollups.
snapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", since, dashboardOverviewSnapshotLimit)
if err != nil {
return nil, err
}
reports, err := model.ListOpenFlareRequestReportsSince(ctx, "", since, dashboardOverviewSnapshotLimit)
if err != nil {
return nil, err
}
accessLogRegions, err := model.ListOpenFlareAccessLogRegionCounts(ctx, "", since, dashboardDistributionLimit)
if err != nil {
return nil, err
@@ -143,21 +134,48 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
if err != nil {
return nil, err
}
openrestySnapshots, err := model.ListOpenFlareNodeObservationOpenresty(ctx, "", since, dashboardOverviewSnapshotLimit)
if err != nil {
return nil, err
}
// L1 business: trends + distributions + totals from access logs only.
view := &OverviewView{
GeneratedAt: now,
Nodes: make([]NodeHealth, 0, len(nodes)),
Distributions: observability.BuildTrafficDistributions(reports, accessLogRegions, dashboardDistributionLimit),
Trends: observability.BuildNodeTrends(ctx, now, "", snapshots, openrestySnapshots, reports),
GeneratedAt: now,
Nodes: make([]NodeHealth, 0, len(nodes)),
Distributions: observability.BuildTrafficDistributionsFromAccessLogs(
ctx, since, now, dashboardDistributionLimit, accessLogRegions,
),
Trends: observability.BuildNodeTrends(ctx, now, "", snapshots),
}
// Global traffic summary uses true window uniqExact for UV (not sum of hourly uniques).
if summary, sumErr := model.TrafficSummaryOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{
Since: since,
Until: now,
}); sumErr == nil {
view.Traffic.RequestCount = summary.RequestCount
view.Traffic.ErrorCount = summary.ErrorCount
view.Traffic.UniqueVisitors = summary.UniqueIPCount
view.Traffic.ReportedNodes = int(summary.NodeCount)
if summary.RequestCount > 0 {
// Average QPS over the 24h window.
view.Traffic.EstimatedQPS = float64(summary.RequestCount) / (24 * 3600)
}
} else {
// Fallback: sum hourly request/error buckets only (UV left from summary path).
applyTrafficTotalsFromTrend(&view.Traffic, view.Trends.Traffic24h)
}
nodeTraffic := map[string]model.OpenFlareAccessLogNodeAggregate{}
if aggregates, aggErr := model.NodeAggregatesOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{
Since: since,
Until: now,
}); aggErr == nil {
for _, row := range aggregates {
nodeTraffic[row.NodeID] = row
}
}
var cpuNodeCount int
var memoryNodeCount int
latestSnapshots := observability.LatestMetricSnapshotsByNode(latestSnapshotRows)
latestTrafficReports := observability.LatestTrafficReportsByNode(latestTrafficRows)
activeEventsByNode := observability.ActiveHealthEventsByNode(activeEvents)
for _, node := range nodes {
@@ -175,7 +193,6 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
}
latestSnapshot := latestSnapshots[node.NodeID]
latestTraffic := latestTrafficReports[node.NodeID]
nodeActiveEvents := activeEventsByNode[node.NodeID]
nodeHealth := NodeHealth{
@@ -193,14 +210,15 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
}
cpuNodeCount, memoryNodeCount = applyNodeSnapshotMetrics(&nodeHealth, latestSnapshot, view, cpuNodeCount, memoryNodeCount)
applyNodeTrafficMetrics(&nodeHealth, latestTraffic)
if agg, ok := nodeTraffic[node.NodeID]; ok {
nodeHealth.RequestCount = agg.RequestCount
nodeHealth.ErrorCount = agg.ErrorCount
nodeHealth.UniqueVisitorCount = agg.UniqueIPCount
}
view.Nodes = append(view.Nodes, nodeHealth)
}
applyTrafficTotalsFromTrend(&view.Traffic, view.Trends.Traffic24h)
applyTrafficRuntimeMetrics(&view.Traffic, latestTrafficReports)
view.Summary.TotalNodes = len(nodes)
if cpuNodeCount > 0 {
view.Capacity.AverageCPUUsagePercent /= float64(cpuNodeCount)
@@ -246,43 +264,19 @@ func applyNodeSnapshotMetrics(nodeHealth *NodeHealth, snapshot *model.OpenFlareM
return cpuNodeCount, memoryNodeCount
}
func applyNodeTrafficMetrics(nodeHealth *NodeHealth, traffic *model.OpenFlareRequestReport) {
if traffic == nil {
return
}
nodeHealth.RequestCount = traffic.RequestCount
nodeHealth.ErrorCount = traffic.ErrorCount
nodeHealth.UniqueVisitorCount = traffic.UniqueVisitorCount
}
func applyTrafficTotalsFromTrend(traffic *Traffic, points []observability.TrafficTrendPoint) {
if traffic == nil {
return
}
traffic.RequestCount = 0
traffic.ErrorCount = 0
// Do not sum hourly unique visitors — that overcounts. UV must come from TrafficSummary.
for _, point := range points {
traffic.RequestCount += point.RequestCount
traffic.ErrorCount += point.ErrorCount
}
}
func applyTrafficRuntimeMetrics(traffic *Traffic, latestReports map[string]*model.OpenFlareRequestReport) {
if traffic == nil {
return
}
traffic.UniqueVisitors = 0
traffic.EstimatedQPS = 0
traffic.ReportedNodes = 0
for _, report := range latestReports {
if report == nil {
continue
}
traffic.UniqueVisitors += report.UniqueVisitorCount
traffic.ReportedNodes++
if duration := report.WindowEndedAt.Sub(report.WindowStartedAt).Seconds(); duration > 0 {
traffic.EstimatedQPS += float64(report.RequestCount) / duration
}
if traffic.RequestCount > 0 && traffic.ReportedNodes == 0 {
traffic.ReportedNodes = 1
}
}
@@ -360,12 +354,14 @@ func compressCapacityTrendPoints(points []observability.CapacityTrendPoint) [][]
func compressNetworkTrendPoints(points []observability.NetworkTrendPoint) [][]any {
rows := make([][]any, 0, len(points))
for _, point := range points {
// Compact layout (stable positions):
// [0] bucket, [1] host_rx, [2] host_tx, [3] bytes_received, [4] bytes_provided, [5] reported_nodes
rows = append(rows, []any{
point.BucketStartedAt,
point.NetworkRxBytes,
point.NetworkTxBytes,
point.OpenrestyRxBytes,
point.OpenrestyTxBytes,
point.BytesReceived,
point.BytesProvided,
point.ReportedNodes,
})
}
@@ -39,7 +39,7 @@ func TestGetOverviewStructure(t *testing.T) {
ctx := context.Background()
now := time.Now().UTC()
lastSeen := now.Add(-time.Minute)
lastSeen := now.Add(-15 * time.Second) // within default 60s offline threshold
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareNode{
NodeID: "node-dashboard-1",
@@ -75,14 +75,33 @@ func TestGetOverviewStructure(t *testing.T) {
StorageUsedBytes: 2,
StorageTotalBytes: 10,
}))
require.NoError(t, model.InsertOpenFlareRequestReport(ctx, &model.OpenFlareRequestReport{
NodeID: "node-dashboard-1",
WindowStartedAt: now.Add(-2 * time.Minute),
WindowEndedAt: now.Add(-time.Minute),
RequestCount: 12,
ErrorCount: 1,
UniqueVisitorCount: 4,
}))
// Business traffic from access logs (L1 authority): 12 requests, 1 server error, 4 unique IPs.
logs := make([]*model.OpenFlareAccessLog, 0, 12)
for i := 0; i < 11; i++ {
logs = append(logs, &model.OpenFlareAccessLog{
NodeID: "node-dashboard-1",
LoggedAt: now.Add(-time.Minute),
RemoteAddr: "10.0.0." + string(rune('1'+i%4)), // rough; fixed below
Host: "app.example.com",
Path: "/",
StatusCode: 200,
BytesSent: 100,
})
}
ips := []string{"10.0.0.10", "10.0.0.11", "10.0.0.12", "10.0.0.13"}
for i := 0; i < 11; i++ {
logs[i].RemoteAddr = ips[i%4]
}
logs = append(logs, &model.OpenFlareAccessLog{
NodeID: "node-dashboard-1",
LoggedAt: now.Add(-time.Minute),
RemoteAddr: ips[0],
Host: "app.example.com",
Path: "/err",
StatusCode: 502,
BytesSent: 10,
})
require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, logs))
overview, err := GetOverview(ctx)
require.NoError(t, err)
@@ -98,8 +117,12 @@ func TestGetOverviewStructure(t *testing.T) {
assert.Equal(t, int64(12), overview.Traffic.RequestCount)
assert.Equal(t, int64(4), overview.Traffic.UniqueVisitors)
assert.Equal(t, int64(1), overview.Traffic.ErrorCount)
assert.InDelta(t, 0.2, overview.Traffic.EstimatedQPS, 0.0001)
// QPS over 24h window
assert.InDelta(t, 12.0/(24*3600.0), overview.Traffic.EstimatedQPS, 0.0001)
assert.Equal(t, 1, overview.Traffic.ReportedNodes)
// Node-level traffic from access log aggregates
onlineNodeCheck := overview.Nodes
require.NotEmpty(t, onlineNodeCheck)
assert.Equal(t, 55.0, overview.Capacity.AverageCPUUsagePercent)
assert.Equal(t, 50.0, overview.Capacity.AverageMemoryUsagePercent)
@@ -110,8 +133,9 @@ func TestGetOverviewStructure(t *testing.T) {
require.NotNil(t, overview.Distributions.StatusCodes)
require.NotNil(t, overview.Distributions.TopDomains)
require.NotNil(t, overview.Distributions.SourceCountries)
assert.Empty(t, overview.Distributions.StatusCodes)
assert.Empty(t, overview.Distributions.TopDomains)
// Status/top domains come from access logs.
assert.NotEmpty(t, overview.Distributions.StatusCodes)
assert.NotEmpty(t, overview.Distributions.TopDomains)
assert.Empty(t, overview.Distributions.SourceCountries)
require.Len(t, overview.Trends.Traffic24h, 24)
@@ -147,11 +171,11 @@ func TestGetOverviewStructure(t *testing.T) {
assert.Equal(t, "online", onlineNode[6])
assert.Equal(t, "healthy", onlineNode[7])
// Latest-per-node health fields (indexes match compressDashboardNodes).
assert.Equal(t, 55.0, onlineNode[11]) // cpu_usage_percent from latest snapshot
assert.Equal(t, 50.0, onlineNode[12]) // memory_usage_percent
assert.Equal(t, int64(12), onlineNode[14])
assert.Equal(t, int64(1), onlineNode[15])
assert.Equal(t, int64(4), onlineNode[16])
assert.Equal(t, 55.0, onlineNode[11]) // cpu_usage_percent from latest snapshot
assert.Equal(t, 50.0, onlineNode[12]) // memory_usage_percent
assert.Equal(t, int64(12), onlineNode[14]) // request_count from access logs
assert.Equal(t, int64(1), onlineNode[15]) // error_count
assert.Equal(t, int64(4), onlineNode[16]) // unique visitors
pendingNode := nodeByID["node-dashboard-2"]
require.NotNil(t, pendingNode)
@@ -161,5 +185,4 @@ func TestGetOverviewStructure(t *testing.T) {
assert.Equal(t, 55.0, overview.Capacity.AverageCPUUsagePercent)
assert.Equal(t, 1, overview.Traffic.ReportedNodes)
assert.Equal(t, int64(4), overview.Traffic.UniqueVisitors)
}
+2 -2
View File
@@ -194,9 +194,9 @@ func computeNodeStatus(node *model.OpenFlareNode) string {
if node.LastSeenAt == nil || node.LastSeenAt.IsZero() {
return nodeStatusPending
}
// 使用默认阈值 2 分钟,避免在这里读取配置
// 默认离线阈值 60 秒(与 node_offline_threshold 默认一致),避免在这里读取配置
// 实际阈值会在需要精确判断的地方通过 getNodeOfflineThreshold 读取
threshold := 2 * time.Minute
threshold := 60 * time.Second
if time.Since(*node.LastSeenAt) > threshold {
return nodeStatusOffline
}
+2 -2
View File
@@ -322,8 +322,8 @@ func TestComputeNodeStatus(t *testing.T) {
online := &model.OpenFlareNode{LastSeenAt: &now}
assert.Equal(t, nodeStatusOnline, computeNodeStatus(online))
// computeNodeStatus 使用默认阈值 2 分钟
offlineAt := now.Add(-2*time.Minute - time.Minute)
// computeNodeStatus 使用默认阈值 60 秒
offlineAt := now.Add(-61 * time.Second)
offline := &model.OpenFlareNode{LastSeenAt: &offlineAt}
assert.Equal(t, nodeStatusOffline, computeNodeStatus(offline))
}
@@ -16,6 +16,7 @@ const (
maxAccessLogPageSize = 200
defaultAccessLogSortBy = "logged_at"
defaultAccessLogSortOrder = "desc"
accessLogSortOrderAsc = "asc"
defaultAccessLogFoldMinute = 3
defaultIPTrendHours = 24
defaultIPTrendBucketMinute = 30
@@ -605,8 +606,8 @@ func normalizeAccessLogSortBy(sortBy string) string {
}
func normalizeAccessLogSortOrder(sortOrder string) string {
if strings.EqualFold(strings.TrimSpace(sortOrder), "asc") {
return "asc"
if strings.EqualFold(strings.TrimSpace(sortOrder), accessLogSortOrderAsc) {
return accessLogSortOrderAsc
}
return defaultAccessLogSortOrder
}
+215 -197
View File
@@ -5,7 +5,6 @@ package observability
import (
"context"
"encoding/json"
"sort"
"strings"
"time"
@@ -22,6 +21,7 @@ const (
healthSeverityCritical = "critical"
healthSeverityWarning = "warning"
percentageMultiplier = 100
sortOrderAsc = "asc"
)
// DistributionItem is a key/value distribution entry.
@@ -37,9 +37,9 @@ type TrafficDistributions struct {
SourceCountries []DistributionItem `json:"source_countries"`
}
const metricSnapshotOpenrestyMatchWindow = 2 * time.Minute
const metricSnapshotEdgeHealthMatchWindow = 2 * time.Minute
// NodeMetricSnapshotView is a metric snapshot enriched with OpenResty observations.
// NodeMetricSnapshotView is a metric snapshot enriched with edge health connections.
type NodeMetricSnapshotView struct {
ID uint `json:"id,omitempty"`
NodeID string `json:"node_id,omitempty"`
@@ -53,8 +53,6 @@ type NodeMetricSnapshotView struct {
DiskWriteBytes int64 `json:"disk_write_bytes"`
NetworkRxBytes int64 `json:"network_rx_bytes"`
NetworkTxBytes int64 `json:"network_tx_bytes"`
OpenrestyRxBytes int64 `json:"openresty_rx_bytes"`
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
OpenrestyConnections int64 `json:"openresty_connections"`
}
@@ -98,13 +96,15 @@ type CapacityTrendPoint struct {
}
// NetworkTrendPoint is a network trend bucket.
// Host network_* is L3 (宿主机网卡).
// bytes_received/provided are L1 business bytes from access logs.
type NetworkTrendPoint struct {
BucketStartedAt time.Time `json:"bucket_started_at"`
NetworkRxBytes int64 `json:"network_rx_bytes"`
NetworkTxBytes int64 `json:"network_tx_bytes"`
OpenrestyRxBytes int64 `json:"openresty_rx_bytes"`
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
ReportedNodes int `json:"reported_nodes"`
BucketStartedAt time.Time `json:"bucket_started_at"`
NetworkRxBytes int64 `json:"network_rx_bytes"`
NetworkTxBytes int64 `json:"network_tx_bytes"`
BytesReceived int64 `json:"bytes_received"` // sum(request_length)
BytesProvided int64 `json:"bytes_provided"` // sum(bytes_sent)
ReportedNodes int `json:"reported_nodes"`
}
// DiskIOTrendPoint is a disk IO trend bucket.
@@ -141,30 +141,39 @@ type networkCounterState struct {
seen bool
}
func buildTrafficWindowSummary(report *model.OpenFlareRequestReport) *TrafficWindowSummary {
if report == nil {
func buildTrafficWindowSummaryFromAccessLogs(
ctx context.Context,
nodeID string,
since, until time.Time,
) *TrafficWindowSummary {
row, err := model.TrafficSummaryOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{
NodeID: nodeID,
Since: since,
Until: until,
})
if err != nil || row.RequestCount <= 0 {
return nil
}
summary := TrafficWindowSummary{
WindowStartedAt: report.WindowStartedAt,
WindowEndedAt: report.WindowEndedAt,
RequestCount: report.RequestCount,
UniqueVisitorCount: report.UniqueVisitorCount,
ErrorCount: report.ErrorCount,
summary := &TrafficWindowSummary{
WindowStartedAt: since.UTC(),
WindowEndedAt: until.UTC(),
RequestCount: row.RequestCount,
UniqueVisitorCount: row.UniqueIPCount,
ErrorCount: row.ErrorCount,
}
if duration := report.WindowEndedAt.Sub(report.WindowStartedAt).Seconds(); duration > 0 {
summary.EstimatedQPS = float64(report.RequestCount) / duration
if duration := until.Sub(since).Seconds(); duration > 0 {
summary.EstimatedQPS = float64(row.RequestCount) / duration
}
if report.RequestCount > 0 {
summary.ErrorRatePercent = (float64(report.ErrorCount) / float64(report.RequestCount)) * 100
if row.RequestCount > 0 {
summary.ErrorRatePercent = (float64(row.ErrorCount) / float64(row.RequestCount)) * 100
}
return &summary
return summary
}
// BuildMetricSnapshotViews merges metric snapshots with OpenResty observations for API responses.
// BuildMetricSnapshotViews merges metric snapshots with edge health connections for API responses.
func BuildMetricSnapshotViews(
snapshots []*model.OpenFlareMetricSnapshot,
openrestyObs []*model.OpenFlareNodeObservationOpenresty,
edgeHealth []*model.OpenFlareEdgeHealth,
) []*NodeMetricSnapshotView {
if len(snapshots) == 0 {
return []*NodeMetricSnapshotView{}
@@ -188,40 +197,49 @@ func BuildMetricSnapshotViews(
NetworkRxBytes: snapshot.NetworkRxBytes,
NetworkTxBytes: snapshot.NetworkTxBytes,
}
if matched := matchOpenrestyObservation(snapshot.CapturedAt, openrestyObs); matched != nil {
view.OpenrestyRxBytes = matched.OpenrestyRxBytes
view.OpenrestyTxBytes = matched.OpenrestyTxBytes
view.OpenrestyConnections = matched.OpenrestyConnections
if matched := matchEdgeHealth(snapshot.CapturedAt, edgeHealth); matched != nil {
view.OpenrestyConnections = matched.Connections
}
views = append(views, view)
}
return views
}
// BuildTrafficDistributions aggregates traffic distribution charts.
func BuildTrafficDistributions(
reports []*model.OpenFlareRequestReport,
accessLogRegions []*model.OpenFlareAccessLogRegionCount,
// BuildTrafficDistributionsFromAccessLogs builds distributions from access logs (L1).
func BuildTrafficDistributionsFromAccessLogs(
ctx context.Context,
since, until time.Time,
limit int,
accessLogRegions []*model.OpenFlareAccessLogRegionCount,
) TrafficDistributions {
statusCodes := make(distributionAccumulator)
topDomains := make(distributionAccumulator)
reportSourceCountries := make(distributionAccumulator)
for _, report := range reports {
mergeJSONCounts(statusCodes, report.StatusCodesJSON)
mergeJSONCounts(topDomains, report.TopDomainsJSON)
mergeJSONCounts(reportSourceCountries, report.SourceCountriesJSON)
}
sourceCountries := reportSourceCountries
if len(accessLogRegions) > 0 {
sourceCountries = make(distributionAccumulator, len(accessLogRegions))
for _, item := range accessLogRegions {
if item == nil || strings.TrimSpace(item.Region) == "" || item.Count <= 0 {
query := model.OpenFlareAccessLogQuery{Since: since, Until: until}
if statusRows, err := model.ValueCountsOpenFlareAccessLogs(ctx, query, "status_code", limit); err == nil {
for _, row := range statusRows {
if strings.TrimSpace(row.Value) == "" || row.Count <= 0 {
continue
}
sourceCountries[item.Region] = item.Count
statusCodes[row.Value] = row.Count
}
}
if hostRows, err := model.ValueCountsOpenFlareAccessLogs(ctx, query, "host", limit); err == nil {
for _, row := range hostRows {
if strings.TrimSpace(row.Value) == "" || row.Count <= 0 {
continue
}
topDomains[row.Value] = row.Count
}
}
sourceCountries := make(distributionAccumulator)
for _, item := range accessLogRegions {
if item == nil || strings.TrimSpace(item.Region) == "" || item.Count <= 0 {
continue
}
sourceCountries[item.Region] = item.Count
}
return TrafficDistributions{
StatusCodes: toDistributionItems(statusCodes, limit),
TopDomains: toDistributionItems(topDomains, limit),
@@ -231,7 +249,7 @@ func BuildTrafficDistributions(
func buildHealthSummary(
snapshot *model.OpenFlareMetricSnapshot,
report *model.OpenFlareRequestReport,
traffic *TrafficWindowSummary,
events []*model.OpenFlareHealthEvent,
) HealthSummary {
summary := HealthSummary{}
@@ -258,41 +276,37 @@ func buildHealthSummary(
storageUsage := Percentage(snapshot.StorageUsedBytes, snapshot.StorageTotalBytes)
summary.HasCapacityRisk = snapshot.CPUUsagePercent >= 80 || memoryUsage >= 85 || storageUsage >= 85
}
if report != nil && report.RequestCount >= 100 {
summary.HasTrafficRisk = (float64(report.ErrorCount) / float64(report.RequestCount)) >= 0.05
if traffic != nil && traffic.RequestCount >= 100 {
summary.HasTrafficRisk = (float64(traffic.ErrorCount) / float64(traffic.RequestCount)) >= 0.05
}
summary.HasRuntimeRisk = summary.ActiveAlerts > 0 || summary.HasCapacityRisk || summary.HasTrafficRisk
return summary
}
// BuildNodeTrends builds 24h trend series, preferring ClickHouse hourly aggregates
// over limited raw snapshot windows so capacity/network/disk charts stay complete.
// BuildNodeTrends builds 24h trend series.
// Business traffic (requests/errors/UV and provided/received bytes) comes from access logs.
// Host capacity/disk/network come from metric snapshots (hourly when available).
func BuildNodeTrends(
ctx context.Context,
now time.Time,
nodeID string,
snapshots []*model.OpenFlareMetricSnapshot,
openrestyObs []*model.OpenFlareNodeObservationOpenresty,
reports []*model.OpenFlareRequestReport,
) NodeTrends {
trendSince := now.Add(-24 * time.Hour)
trafficTrend := BuildTrafficTrendPoints(now, reports)
if trafficHourly, err := model.ListOpenFlareTrafficHourlySince(ctx, nodeID, trendSince); err == nil && len(trafficHourly) > 0 {
trafficTrend = BuildTrafficTrendPointsFromHourly(now, trafficHourly)
}
trafficTrend := BuildTrafficTrendPointsFromAccessLogs(ctx, now, nodeID, trendSince)
capacityTrend := BuildCapacityTrendPoints(now, snapshots)
networkTrend := BuildNetworkTrendPoints(now, snapshots, openrestyObs)
networkTrend := BuildNetworkTrendPoints(now, snapshots)
// Overlay L1 business bytes onto network points.
applyAccessLogBytesToNetworkTrend(ctx, now, nodeID, trendSince, networkTrend)
diskIOTrend := BuildDiskIOTrendPoints(now, snapshots)
metricHourly, metricErr := model.ListOpenFlareMetricHourlySince(ctx, nodeID, trendSince)
if metricErr == nil && len(metricHourly) > 0 {
capacityTrend = BuildCapacityTrendPointsFromHourly(now, metricHourly)
diskIOTrend = BuildDiskIOTrendPointsFromHourly(now, metricHourly)
}
openrestyHourly, openrestyErr := model.ListOpenFlareOpenrestyHourlySince(ctx, nodeID, trendSince)
if metricErr == nil && openrestyErr == nil && (len(metricHourly) > 0 || len(openrestyHourly) > 0) {
networkTrend = BuildNetworkTrendPointsFromHourly(now, metricHourly, openrestyHourly)
networkTrend = BuildNetworkTrendPointsFromHourly(now, metricHourly)
applyAccessLogBytesToNetworkTrend(ctx, now, nodeID, trendSince, networkTrend)
}
return NodeTrends{
@@ -303,7 +317,131 @@ func BuildNodeTrends(
}
}
// BuildTrafficTrendPointsFromAccessLogs builds 24h request/error buckets from access logs.
// Prefers of_access_log_hourly when available; falls back to raw bucket aggregates.
// UniqueVisitorCount on hourly path is 0 (use TrafficSummary for exact UV).
func BuildTrafficTrendPointsFromAccessLogs(ctx context.Context, now time.Time, nodeID string, since time.Time) []TrafficTrendPoint {
start := trendWindowStart(now)
points := make([]TrafficTrendPoint, observabilityTrendBuckets)
for index := range points {
points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
}
if hourly, err := model.ListOpenFlareTrafficHourlySince(ctx, nodeID, since); err == nil && len(hourly) > 0 {
for _, row := range hourly {
if row == nil {
continue
}
index, ok := trendBucketIndex(row.Hour, start)
if !ok {
continue
}
points[index].RequestCount += row.RequestCount
points[index].ErrorCount += row.ErrorCount
// UniqueVisitorCount intentionally not summed from hourly rollup (always 0 / overcounts).
}
return points
}
buckets, err := model.ListOpenFlareAccessLogBuckets(ctx, model.OpenFlareAccessLogBucketQuery{
NodeID: nodeID,
Since: since,
Until: now,
FoldMinutes: 60,
SortBy: "logged_at",
SortOrder: sortOrderAsc,
})
if err != nil || len(buckets) == 0 {
return points
}
byEpoch := make(map[int64]*model.OpenFlareAccessLogBucketRow, len(buckets))
for _, row := range buckets {
if row == nil {
continue
}
byEpoch[row.BucketEpoch] = row
}
for index := range points {
epoch := points[index].BucketStartedAt.Unix()
if row, ok := byEpoch[epoch]; ok {
points[index].RequestCount = row.RequestCount
points[index].ErrorCount = row.ServerErrorCount
points[index].UniqueVisitorCount = row.UniqueIPCount
}
}
return points
}
func applyAccessLogBytesToNetworkTrend(ctx context.Context, now time.Time, nodeID string, since time.Time, points []NetworkTrendPoint) {
if len(points) == 0 {
return
}
// Prefer of_access_log_hourly (summed across hosts).
if hourly, err := analyticsListAccessLogHourlyBytes(ctx, nodeID, since); err == nil && len(hourly) > 0 {
for hourUnix, totals := range hourly {
for index := range points {
if points[index].BucketStartedAt.Unix() == hourUnix {
points[index].BytesProvided = totals.provided
points[index].BytesReceived = totals.received
}
}
}
return
}
buckets, err := model.ListOpenFlareAccessLogBuckets(ctx, model.OpenFlareAccessLogBucketQuery{
NodeID: nodeID,
Since: since,
Until: now,
FoldMinutes: 60,
SortBy: "logged_at",
SortOrder: sortOrderAsc,
})
if err != nil || len(buckets) == 0 {
return
}
byEpoch := make(map[int64]*model.OpenFlareAccessLogBucketRow, len(buckets))
for _, row := range buckets {
if row == nil {
continue
}
byEpoch[row.BucketEpoch] = row
}
for index := range points {
epoch := points[index].BucketStartedAt.Unix()
if row, ok := byEpoch[epoch]; ok {
points[index].BytesProvided = row.BytesSent
points[index].BytesReceived = row.RequestLength
}
}
}
type accessLogHourBytes struct {
provided int64
received int64
}
func analyticsListAccessLogHourlyBytes(ctx context.Context, nodeID string, since time.Time) (map[int64]accessLogHourBytes, error) {
rows, err := model.ListOpenFlareAccessLogHourlySince(ctx, nodeID, since)
if err != nil {
return nil, err
}
out := make(map[int64]accessLogHourBytes)
for _, row := range rows {
if row == nil {
continue
}
key := row.Hour.UTC().Truncate(time.Hour).Unix()
cur := out[key]
cur.provided += row.BytesSent
cur.received += row.RequestLength
out[key] = cur
}
return out, nil
}
// BuildTrafficTrendPointsFromHourly builds 24h traffic trend buckets from hourly rollups.
// UniqueVisitorCount is left at 0: hourly UV is not summed (use TrafficSummary for exact UV).
func BuildTrafficTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlareTrafficHourly) []TrafficTrendPoint {
start := trendWindowStart(now)
points := make([]TrafficTrendPoint, observabilityTrendBuckets)
@@ -320,26 +458,6 @@ func BuildTrafficTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlareT
}
points[index].RequestCount += row.RequestCount
points[index].ErrorCount += row.ErrorCount
points[index].UniqueVisitorCount += row.UniqueVisitorCount
}
return points
}
// BuildTrafficTrendPoints builds 24h traffic trend buckets.
func BuildTrafficTrendPoints(now time.Time, reports []*model.OpenFlareRequestReport) []TrafficTrendPoint {
start := trendWindowStart(now)
points := make([]TrafficTrendPoint, observabilityTrendBuckets)
for index := range points {
points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
}
for _, report := range reports {
index, ok := trendBucketIndex(report.WindowEndedAt, start)
if !ok {
continue
}
points[index].RequestCount += report.RequestCount
points[index].ErrorCount += report.ErrorCount
points[index].UniqueVisitorCount += report.UniqueVisitorCount
}
return points
}
@@ -404,12 +522,12 @@ func BuildCapacityTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlare
return points
}
// BuildNetworkTrendPoints builds 24h network trend buckets.
// Host and OpenResty counters are cumulative; values are consecutive deltas.
// BuildNetworkTrendPoints builds 24h host-network trend buckets.
// Host network counters must be process-lifetime cumulative values; this function
// converts consecutive samples into deltas.
func BuildNetworkTrendPoints(
now time.Time,
snapshots []*model.OpenFlareMetricSnapshot,
openrestyObs []*model.OpenFlareNodeObservationOpenresty,
) []NetworkTrendPoint {
start := trendWindowStart(now)
points := make([]NetworkTrendPoint, observabilityTrendBuckets)
@@ -452,51 +570,17 @@ func BuildNetworkTrendPoints(
accumulators[index].nodes[snapshot.NodeID] = struct{}{}
}
}
sort.Slice(openrestyObs, func(i int, j int) bool {
if openrestyObs[i].CapturedAt.Equal(openrestyObs[j].CapturedAt) {
return openrestyObs[i].NodeID < openrestyObs[j].NodeID
}
return openrestyObs[i].CapturedAt.Before(openrestyObs[j].CapturedAt)
})
previousOpenrestyByNode := make(map[string]networkCounterState, len(openrestyObs))
for _, obs := range openrestyObs {
if obs == nil {
continue
}
nodeKey := obs.NodeID
if nodeKey == "" {
nodeKey = unknownTrendNodeKey
}
previous := previousOpenrestyByNode[nodeKey]
previousOpenrestyByNode[nodeKey] = networkCounterState{
rx: obs.OpenrestyRxBytes,
tx: obs.OpenrestyTxBytes,
seen: true,
}
if !previous.seen {
continue
}
index, ok := trendBucketIndex(obs.CapturedAt, start)
if !ok {
continue
}
points[index].OpenrestyRxBytes += nonNegativeDelta(obs.OpenrestyRxBytes, previous.rx)
points[index].OpenrestyTxBytes += nonNegativeDelta(obs.OpenrestyTxBytes, previous.tx)
if obs.NodeID != "" {
accumulators[index].nodes[obs.NodeID] = struct{}{}
}
}
for index := range points {
points[index].ReportedNodes = len(accumulators[index].nodes)
}
return points
}
// BuildNetworkTrendPointsFromHourly builds 24h network trend buckets from hourly aggregates.
// BuildNetworkTrendPointsFromHourly builds 24h host-network trend buckets from metric hourly aggregates.
// Business bytes (已提供/接收) are applied separately via applyAccessLogBytesToNetworkTrend.
func BuildNetworkTrendPointsFromHourly(
now time.Time,
metricHourly []*model.OpenFlareMetricHourly,
openrestyHourly []*model.OpenFlareOpenrestyHourly,
) []NetworkTrendPoint {
start := trendWindowStart(now)
points := make([]NetworkTrendPoint, observabilityTrendBuckets)
@@ -517,20 +601,6 @@ func BuildNetworkTrendPointsFromHourly(
points[index].ReportedNodes = row.ReportedNodes
}
}
for _, row := range openrestyHourly {
if row == nil {
continue
}
index, ok := trendBucketIndex(row.Hour, start)
if !ok {
continue
}
points[index].OpenrestyRxBytes += row.OpenrestyRxBytes
points[index].OpenrestyTxBytes += row.OpenrestyTxBytes
if row.ReportedNodes > points[index].ReportedNodes {
points[index].ReportedNodes = row.ReportedNodes
}
}
return points
}
@@ -623,38 +693,25 @@ func latestMetricSnapshot(snapshots []*model.OpenFlareMetricSnapshot) *model.Ope
return latest
}
func latestTrafficReport(reports []*model.OpenFlareRequestReport) *model.OpenFlareRequestReport {
var latest *model.OpenFlareRequestReport
for _, report := range reports {
if report == nil {
continue
}
if latest == nil || report.WindowEndedAt.After(latest.WindowEndedAt) {
latest = report
}
}
return latest
}
func matchOpenrestyObservation(
func matchEdgeHealth(
capturedAt time.Time,
observations []*model.OpenFlareNodeObservationOpenresty,
) *model.OpenFlareNodeObservationOpenresty {
var matched *model.OpenFlareNodeObservationOpenresty
bestDelta := metricSnapshotOpenrestyMatchWindow + time.Second
for _, observation := range observations {
if observation == nil {
health []*model.OpenFlareEdgeHealth,
) *model.OpenFlareEdgeHealth {
var matched *model.OpenFlareEdgeHealth
bestDelta := metricSnapshotEdgeHealthMatchWindow + time.Second
for _, row := range health {
if row == nil {
continue
}
delta := capturedAt.Sub(observation.CapturedAt)
delta := capturedAt.Sub(row.CapturedAt)
if delta < 0 {
delta = -delta
}
if delta > metricSnapshotOpenrestyMatchWindow {
if delta > metricSnapshotEdgeHealthMatchWindow {
continue
}
if matched == nil || delta < bestDelta {
matched = observation
matched = row
bestDelta = delta
}
}
@@ -676,21 +733,6 @@ func LatestMetricSnapshotsByNode(snapshots []*model.OpenFlareMetricSnapshot) map
return result
}
// LatestTrafficReportsByNode returns the latest traffic report per node.
func LatestTrafficReportsByNode(reports []*model.OpenFlareRequestReport) map[string]*model.OpenFlareRequestReport {
result := make(map[string]*model.OpenFlareRequestReport, len(reports))
for _, report := range reports {
if report == nil || report.NodeID == "" {
continue
}
if existing, ok := result[report.NodeID]; ok && !report.WindowEndedAt.After(existing.WindowEndedAt) {
continue
}
result[report.NodeID] = report
}
return result
}
// ActiveHealthEventsByNode groups active health events by node id.
func ActiveHealthEventsByNode(events []*model.OpenFlareHealthEvent) map[string][]*model.OpenFlareHealthEvent {
result := make(map[string][]*model.OpenFlareHealthEvent)
@@ -711,30 +753,6 @@ func Percentage(used int64, total int64) float64 {
return (float64(used) / float64(total)) * percentageMultiplier
}
func mergeJSONCounts(target distributionAccumulator, raw string) {
if len(target) == 0 && strings.TrimSpace(raw) == "" {
return
}
values := parseJSONCounts(raw)
for key, value := range values {
if strings.TrimSpace(key) == "" || value <= 0 {
continue
}
target[key] += value
}
}
func parseJSONCounts(raw string) map[string]int64 {
if strings.TrimSpace(raw) == "" {
return nil
}
values := make(map[string]int64)
if err := json.Unmarshal([]byte(raw), &values); err != nil {
return nil
}
return values
}
func toDistributionItems(values distributionAccumulator, limit int) []DistributionItem {
if len(values) == 0 {
return []DistributionItem{}
@@ -25,52 +25,25 @@ func TestBuildTrafficTrendPointsFromHourlyBucketsByHour(t *testing.T) {
if len(points) != observabilityTrendBuckets {
t.Fatalf("BuildTrafficTrendPointsFromHourly() len = %d, want %d", len(points), observabilityTrendBuckets)
}
}
func TestBuildTrafficTrendPointsBucketsByHour(t *testing.T) {
t.Parallel()
now := time.Date(2026, 6, 19, 17, 30, 0, 0, time.UTC)
reports := []*model.OpenFlareRequestReport{
{
NodeID: "node-a",
WindowStartedAt: now.Add(-3 * time.Hour),
WindowEndedAt: now.Add(-3*time.Hour + time.Minute),
RequestCount: 10,
ErrorCount: 1,
},
{
NodeID: "node-a",
WindowStartedAt: now.Add(-30 * time.Minute),
WindowEndedAt: now.Add(-29 * time.Minute),
RequestCount: 6,
ErrorCount: 0,
},
}
points := BuildTrafficTrendPoints(now, reports)
if len(points) != observabilityTrendBuckets {
t.Fatalf("BuildTrafficTrendPoints() len = %d, want %d", len(points), observabilityTrendBuckets)
}
var totalRequests int64
// Hourly UV must not be summed into trend points.
for _, point := range points {
totalRequests += point.RequestCount
if point.UniqueVisitorCount != 0 {
t.Fatalf("UniqueVisitorCount = %d, want 0 on hourly path", point.UniqueVisitorCount)
}
}
if totalRequests != 16 {
t.Fatalf("total request_count = %d, want 16", totalRequests)
index, ok := trendBucketIndex(now.Add(-2*time.Hour).Truncate(time.Hour), trendWindowStart(now))
if !ok {
t.Fatal("expected valid bucket index")
}
currentHour := points[len(points)-1]
if currentHour.RequestCount != 6 {
t.Fatalf("current hour request_count = %d, want 6", currentHour.RequestCount)
if points[index].RequestCount != 12 {
t.Fatalf("request_count = %d, want 12", points[index].RequestCount)
}
if currentHour.ErrorCount != 0 {
t.Fatalf("current hour error_count = %d, want 0", currentHour.ErrorCount)
if points[index].ErrorCount != 1 {
t.Fatalf("error_count = %d, want 1", points[index].ErrorCount)
}
}
func TestBuildMetricSnapshotViewsMergesOpenrestyObservation(t *testing.T) {
func TestBuildMetricSnapshotViewsMergesEdgeHealthConnections(t *testing.T) {
t.Parallel()
capturedAt := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC)
@@ -82,54 +55,30 @@ func TestBuildMetricSnapshotViewsMergesOpenrestyObservation(t *testing.T) {
CPUUsagePercent: 12.5,
},
}
openrestyObs := []*model.OpenFlareNodeObservationOpenresty{
edgeHealth := []*model.OpenFlareEdgeHealth{
{
NodeID: "node-a",
CapturedAt: capturedAt.Add(5 * time.Second),
OpenrestyRxBytes: 4096,
OpenrestyTxBytes: 8192,
OpenrestyConnections: 7,
NodeID: "node-a",
CapturedAt: capturedAt.Add(5 * time.Second),
Status: "healthy",
Connections: 7,
},
}
views := BuildMetricSnapshotViews(snapshots, openrestyObs)
views := BuildMetricSnapshotViews(snapshots, edgeHealth)
if len(views) != 1 {
t.Fatalf("BuildMetricSnapshotViews() len = %d, want 1", len(views))
}
if views[0].OpenrestyRxBytes != 4096 {
t.Fatalf("OpenrestyRxBytes = %d, want 4096", views[0].OpenrestyRxBytes)
}
if views[0].OpenrestyTxBytes != 8192 {
t.Fatalf("OpenrestyTxBytes = %d, want 8192", views[0].OpenrestyTxBytes)
}
if views[0].OpenrestyConnections != 7 {
t.Fatalf("OpenrestyConnections = %d, want 7", views[0].OpenrestyConnections)
}
}
func TestLatestTrafficReportUsesLatestWindowEndedAt(t *testing.T) {
func TestBuildTrafficWindowSummaryFromAccessLogsNilWithoutData(t *testing.T) {
t.Parallel()
older := &model.OpenFlareRequestReport{
WindowEndedAt: time.Date(2026, 6, 19, 10, 0, 0, 0, time.UTC),
RequestCount: 3,
}
newer := &model.OpenFlareRequestReport{
WindowEndedAt: time.Date(2026, 6, 19, 11, 0, 0, 0, time.UTC),
RequestCount: 9,
}
latest := latestTrafficReport([]*model.OpenFlareRequestReport{older, newer})
if latest == nil || latest.RequestCount != 9 {
t.Fatalf("latestTrafficReport() = %#v, want newer report with request_count 9", latest)
}
}
func TestBuildTrafficWindowSummaryNilWithoutReport(t *testing.T) {
t.Parallel()
if summary := buildTrafficWindowSummary(nil); summary != nil {
t.Fatalf("buildTrafficWindowSummary(nil) = %#v, want nil", summary)
// Without an access-log store / data, summary is nil.
if summary := buildTrafficWindowSummaryFromAccessLogs(t.Context(), "missing", time.Now().Add(-time.Hour), time.Now()); summary != nil {
t.Fatalf("buildTrafficWindowSummaryFromAccessLogs() = %#v, want nil", summary)
}
}
@@ -173,12 +122,8 @@ func TestBuildNetworkTrendPointsUsesCounterDeltas(t *testing.T) {
{NodeID: "n1", CapturedAt: base.Add(10 * time.Minute), NetworkRxBytes: 1000, NetworkTxBytes: 2000},
{NodeID: "n1", CapturedAt: base.Add(20 * time.Minute), NetworkRxBytes: 1500, NetworkTxBytes: 2600},
}
openrestyObs := []*model.OpenFlareNodeObservationOpenresty{
{NodeID: "n1", CapturedAt: base.Add(10 * time.Minute), OpenrestyRxBytes: 100, OpenrestyTxBytes: 200},
{NodeID: "n1", CapturedAt: base.Add(20 * time.Minute), OpenrestyRxBytes: 180, OpenrestyTxBytes: 250},
}
points := BuildNetworkTrendPoints(now, snapshots, openrestyObs)
points := BuildNetworkTrendPoints(now, snapshots)
current := points[len(points)-1]
if current.NetworkRxBytes != 500 {
t.Fatalf("network_rx_bytes = %d, want 500", current.NetworkRxBytes)
@@ -186,11 +131,30 @@ func TestBuildNetworkTrendPointsUsesCounterDeltas(t *testing.T) {
if current.NetworkTxBytes != 600 {
t.Fatalf("network_tx_bytes = %d, want 600", current.NetworkTxBytes)
}
if current.OpenrestyRxBytes != 80 {
t.Fatalf("openresty_rx_bytes = %d, want 80", current.OpenrestyRxBytes)
if current.BytesReceived != 0 || current.BytesProvided != 0 {
t.Fatalf("business bytes should be 0 without access logs overlay, got received=%d provided=%d",
current.BytesReceived, current.BytesProvided)
}
if current.OpenrestyTxBytes != 50 {
t.Fatalf("openresty_tx_bytes = %d, want 50", current.OpenrestyTxBytes)
}
func TestBuildHealthSummaryUsesTrafficSummary(t *testing.T) {
t.Parallel()
snapshot := &model.OpenFlareMetricSnapshot{
CPUUsagePercent: 10,
MemoryUsedBytes: 1,
MemoryTotalBytes: 10,
}
traffic := &TrafficWindowSummary{
RequestCount: 200,
ErrorCount: 20, // 10% error rate
}
summary := buildHealthSummary(snapshot, traffic, nil)
if !summary.HasTrafficRisk {
t.Fatal("HasTrafficRisk = false, want true for 10% error rate with >=100 requests")
}
if summary.HasCapacityRisk {
t.Fatal("HasCapacityRisk = true, want false")
}
}
@@ -79,7 +79,6 @@ type NodeView struct {
NodeID string `json:"node_id"`
Profile *model.OpenFlareNodeSystemProfile `json:"profile"`
MetricSnapshots []*NodeMetricSnapshotView `json:"metric_snapshots"`
TrafficReports []*model.OpenFlareRequestReport `json:"traffic_reports"`
HealthEvents []*model.OpenFlareHealthEvent `json:"health_events"`
Analytics NodeAnalytics `json:"analytics"`
Trends NodeTrends `json:"trends"`
@@ -118,11 +117,7 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
if err != nil {
return nil, err
}
openrestyObs, err := model.ListOpenFlareNodeObservationOpenresty(ctx, node.NodeID, since, limit)
if err != nil {
return nil, err
}
reports, err := model.ListOpenFlareRequestReportsSince(ctx, node.NodeID, since, limit)
edgeHealth, err := model.ListOpenFlareEdgeHealth(ctx, node.NodeID, since, limit)
if err != nil {
return nil, err
}
@@ -134,18 +129,21 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
if err != nil {
return nil, err
}
distributions := BuildTrafficDistributionsFromAccessLogs(
ctx, since, now, defaultTrafficDistributionLimit, accessLogRegions,
)
trafficSummary := buildTrafficWindowSummaryFromAccessLogs(ctx, node.NodeID, since, now)
view := &NodeView{
NodeID: node.NodeID,
Profile: profile,
MetricSnapshots: BuildMetricSnapshotViews(snapshots, openrestyObs),
TrafficReports: reports,
MetricSnapshots: BuildMetricSnapshotViews(snapshots, edgeHealth),
HealthEvents: events,
Analytics: NodeAnalytics{
Traffic: buildTrafficWindowSummary(latestTrafficReport(reports)),
Distributions: BuildTrafficDistributions(reports, accessLogRegions, defaultTrafficDistributionLimit),
Health: buildHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events),
Traffic: trafficSummary,
Distributions: distributions,
Health: buildHealthSummary(latestMetricSnapshot(snapshots), trafficSummary, events),
},
Trends: BuildNodeTrends(ctx, now, node.NodeID, snapshots, openrestyObs, reports),
Trends: BuildNodeTrends(ctx, now, node.NodeID, snapshots),
}
if node.NodeType == "tunnel_relay" {
frpsObs, frpsErr := model.ListOpenFlareNodeObservationFrps(ctx, node.NodeID, time.Time{}, 1)
+1 -1
View File
@@ -16,7 +16,7 @@ const (
relayStatusUnhealthy = "unhealthy"
releaseChannelStable = "stable"
defaultAgentHeartbeatInterval = 10000 // 默认心跳间隔 10 秒(毫秒)
defaultAgentHeartbeatInterval = 3000 // 默认心跳间隔 3 秒(毫秒)
defaultAgentUpdateRepo = "Rain-kl/OpenFlare"
)
@@ -47,7 +47,7 @@ func reconcileRelayHealthEvents(ctx context.Context, nodeID string, relayStatus
func persistRelayHeartbeatObservability(ctx context.Context, nodeID string, payload HeartbeatPayload, reportedAt time.Time) {
agent.PersistHeartbeatObservability(ctx, nodeID, agent.NodePayload{
Profile: payload.Profile,
Snapshot: payload.Snapshot,
HostMetrics: payload.Snapshot,
HealthEvents: payload.HealthEvents,
}, reportedAt)
@@ -20,10 +20,8 @@ const (
DatabaseCleanupTargetAccessLogs = "node_access_logs"
// DatabaseCleanupTargetMetricSnapshots is the API cleanup target for metric snapshots.
DatabaseCleanupTargetMetricSnapshots = "node_metric_snapshots"
// DatabaseCleanupTargetRequestReports is the API cleanup target for request reports.
DatabaseCleanupTargetRequestReports = "node_request_reports"
// DatabaseCleanupTargetObsOpenresty is the API cleanup target for OpenResty observations.
DatabaseCleanupTargetObsOpenresty = "node_obs_openresty"
// DatabaseCleanupTargetEdgeHealth is the API cleanup target for OpenResty edge health (connections).
DatabaseCleanupTargetEdgeHealth = "node_edge_health"
// DatabaseCleanupTargetObsFrps is the API cleanup target for FRPS observations.
DatabaseCleanupTargetObsFrps = "node_obs_frps"
// DatabaseCleanupTargetObsFrpc is the API cleanup target for FRPC observations.
@@ -33,8 +31,7 @@ const (
var databaseCleanupTargets = map[string]string{
DatabaseCleanupTargetAccessLogs: "访问日志",
DatabaseCleanupTargetMetricSnapshots: "性能快照",
DatabaseCleanupTargetRequestReports: "请求聚合",
DatabaseCleanupTargetObsOpenresty: "OpenResty 观测",
DatabaseCleanupTargetEdgeHealth: "OpenResty 健康(连接)",
DatabaseCleanupTargetObsFrps: "FRPS 观测",
DatabaseCleanupTargetObsFrpc: "FRPC 观测",
}
@@ -43,8 +40,7 @@ var databaseCleanupTargets = map[string]string{
var databaseCleanupTableTTLDays = map[string]int{
DatabaseCleanupTargetAccessLogs: analyticsrepo.TableTTLDaysNodeAccessLogs,
DatabaseCleanupTargetMetricSnapshots: analyticsrepo.TableTTLDaysNodeMetricSnapshots,
DatabaseCleanupTargetRequestReports: analyticsrepo.TableTTLDaysNodeRequestReports,
DatabaseCleanupTargetObsOpenresty: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetEdgeHealth: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetObsFrps: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetObsFrpc: analyticsrepo.TableTTLDaysNodeObs,
}
@@ -165,8 +161,7 @@ func RunDatabaseAutoCleanupOnce(ctx context.Context, now time.Time) (*DatabaseAu
for _, target := range []string{
DatabaseCleanupTargetAccessLogs,
DatabaseCleanupTargetMetricSnapshots,
DatabaseCleanupTargetRequestReports,
DatabaseCleanupTargetObsOpenresty,
DatabaseCleanupTargetEdgeHealth,
DatabaseCleanupTargetObsFrps,
DatabaseCleanupTargetObsFrpc,
} {
@@ -201,10 +196,8 @@ func deleteAllObservabilityRows(ctx context.Context, target string) (int64, stri
deleted, err = model.DeleteAllOpenFlareAccessLogs(ctx)
case DatabaseCleanupTargetMetricSnapshots:
deleted, err = model.DeleteAllOpenFlareMetricSnapshots(ctx)
case DatabaseCleanupTargetRequestReports:
deleted, err = model.DeleteAllOpenFlareRequestReports(ctx)
case DatabaseCleanupTargetObsOpenresty:
deleted, err = model.DeleteAllOpenFlareNodeObservationOpenresty(ctx)
case DatabaseCleanupTargetEdgeHealth:
deleted, err = model.DeleteAllOpenFlareEdgeHealth(ctx)
case DatabaseCleanupTargetObsFrps:
deleted, err = model.DeleteAllOpenFlareNodeObservationFrps(ctx)
case DatabaseCleanupTargetObsFrpc:
@@ -236,10 +229,8 @@ func materializeObservabilityTableTTL(ctx context.Context, target string) (int64
eligible, err = model.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
case DatabaseCleanupTargetMetricSnapshots:
eligible, err = model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
case DatabaseCleanupTargetRequestReports:
eligible, err = model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
case DatabaseCleanupTargetObsOpenresty:
eligible, err = model.DeleteOpenFlareNodeObservationOpenrestyBefore(ctx, cutoff)
case DatabaseCleanupTargetEdgeHealth:
eligible, err = model.DeleteOpenFlareEdgeHealthBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrps:
eligible, err = model.DeleteOpenFlareNodeObservationFrpsBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrpc:
@@ -157,11 +157,11 @@ func TestRunDatabaseAutoCleanupOnceClampsRetentionToTableTTL(t *testing.T) {
CapturedAt: now.Add(-40 * 24 * time.Hour),
CPUUsagePercent: 10,
}))
require.NoError(t, model.InsertOpenFlareRequestReport(ctx, &model.OpenFlareRequestReport{
NodeID: "node-a",
WindowStartedAt: now.Add(-41 * 24 * time.Hour),
WindowEndedAt: now.Add(-40 * 24 * time.Hour),
RequestCount: 15,
require.NoError(t, model.InsertOpenFlareEdgeHealth(ctx, &model.OpenFlareEdgeHealth{
NodeID: "node-a",
CapturedAt: now.Add(-40 * 24 * time.Hour),
Status: "healthy",
Connections: 2,
}))
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled, "true"))
@@ -170,7 +170,7 @@ func TestRunDatabaseAutoCleanupOnceClampsRetentionToTableTTL(t *testing.T) {
summary, err := RunDatabaseAutoCleanupOnce(ctx, now)
require.NoError(t, err)
require.NotNil(t, summary)
require.Len(t, summary.Results, 6)
require.Len(t, summary.Results, 5)
assert.Equal(t, 1, summary.RetentionDays)
for _, result := range summary.Results {
@@ -189,9 +189,9 @@ func TestRunDatabaseAutoCleanupOnceClampsRetentionToTableTTL(t *testing.T) {
require.NoError(t, err)
assert.Empty(t, metricSnapshots)
requestReports, err := model.ListOpenFlareRequestReportsSince(ctx, "", time.Time{}, 0)
edgeHealth, err := model.ListOpenFlareEdgeHealth(ctx, "", time.Time{}, 0)
require.NoError(t, err)
assert.Empty(t, requestReports)
assert.Empty(t, edgeHealth)
}
func TestTableTTLDaysForCleanupTarget(t *testing.T) {