This commit is contained in:
ryan
2026-06-19 15:13:24 +08:00
parent 0b34792709
commit 32861c5db9
376 changed files with 3648 additions and 19957 deletions
@@ -1,3 +1,4 @@
// Package observability provides system and service level observability data collection for the agent.
package observability
import (
@@ -15,6 +16,9 @@ import (
edgeobs "github.com/Rain-kl/Wavelet/internal/apps/edge/observability"
)
const nodeHealthEventInitialCapacity = 2
// BuildProfile collects the system profile and returns it only if the fingerprint has changed.
func BuildProfile(cfg *config.Config, stateStore *state.Store) *protocol.NodeSystemProfile {
profile := collectProfile(cfg)
if profile == nil {
@@ -38,6 +42,7 @@ func BuildProfile(cfg *config.Config, stateStore *state.Store) *protocol.NodeSys
return profile
}
// BuildSnapshot captures current system metrics and returns a metric snapshot.
func BuildSnapshot(cfg *config.Config, stateStore *state.Store) *protocol.NodeMetricSnapshot {
now := time.Now().UTC()
metric := &protocol.NodeMetricSnapshot{
@@ -79,6 +84,7 @@ func BuildSnapshot(cfg *config.Config, stateStore *state.Store) *protocol.NodeMe
return metric
}
// BuildOpenrestyObservation builds the OpenResty observation protocol model from the managed metrics.
func BuildOpenrestyObservation(managed *ManagedOpenRestyMetrics) *protocol.NodeOpenrestyObservation {
if managed == nil {
return nil
@@ -91,11 +97,12 @@ func BuildOpenrestyObservation(managed *ManagedOpenRestyMetrics) *protocol.NodeO
}
}
// BuildHealthEvents converts system snapshot health state into a list of health events.
func BuildHealthEvents(snapshot *state.Snapshot) []protocol.NodeHealthEvent {
if snapshot == nil {
return []protocol.NodeHealthEvent{}
}
events := make([]protocol.NodeHealthEvent, 0, 2)
events := make([]protocol.NodeHealthEvent, 0, nodeHealthEventInitialCapacity)
nowUnix := time.Now().UTC().Unix()
if strings.TrimSpace(snapshot.OpenrestyStatus) == protocol.OpenrestyStatusUnhealthy {
events = append(events, protocol.NodeHealthEvent{
@@ -147,4 +154,4 @@ func fingerprintProfile(profile *protocol.NodeSystemProfile) string {
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
}
@@ -1,6 +1,7 @@
package observability
import (
"context"
"encoding/json"
"fmt"
"io"
@@ -14,11 +15,15 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/agent/protocol"
)
const openRestyObservabilityPath = "/openflare/observability"
const openRestyStubStatusPath = "/openflare/stub_status"
const (
openRestyObservabilityPath = "/openflare/observability"
openRestyStubStatusPath = "/openflare/stub_status"
stubStatusActiveMatchGroupCount = 2
)
var stubStatusActivePattern = regexp.MustCompile(`Active connections:\s+(\d+)`)
// ManagedOpenRestyMetrics holds metrics collected from the local OpenResty instance.
type ManagedOpenRestyMetrics struct {
TrafficReport *protocol.NodeTrafficReport
OpenrestyRxBytes int64
@@ -39,7 +44,8 @@ type openRestyObservabilityResponse struct {
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
}
func CollectManagedOpenRestyMetrics(cfg *config.Config) *ManagedOpenRestyMetrics {
// CollectManagedOpenRestyMetrics collects metrics from the local OpenResty observability endpoints.
func CollectManagedOpenRestyMetrics(ctx context.Context, cfg *config.Config) *ManagedOpenRestyMetrics {
if cfg == nil || cfg.OpenrestyObservabilityPort <= 0 {
return nil
}
@@ -48,7 +54,7 @@ func CollectManagedOpenRestyMetrics(cfg *config.Config) *ManagedOpenRestyMetrics
client := &http.Client{Timeout: 1500 * time.Millisecond}
observabilityResp := openRestyObservabilityResponse{}
if err := fetchLocalJSON(client, baseURL+openRestyObservabilityPath, &observabilityResp); err != nil {
if err := fetchLocalJSON(ctx, client, baseURL+openRestyObservabilityPath, &observabilityResp); err != nil {
return nil
}
@@ -67,31 +73,39 @@ func CollectManagedOpenRestyMetrics(cfg *config.Config) *ManagedOpenRestyMetrics
OpenrestyTxBytes: observabilityResp.OpenrestyTxBytes,
}
if text, err := fetchLocalText(client, baseURL+openRestyStubStatusPath); err == nil {
if text, err := fetchLocalText(ctx, client, baseURL+openRestyStubStatusPath); err == nil {
result.OpenrestyConnections = parseStubStatusActiveConnections(text)
}
return result
}
func fetchLocalJSON(client *http.Client, url string, target any) error {
resp, err := client.Get(url)
func fetchLocalJSON(ctx context.Context, client *http.Client, url string, target any) error {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return err
}
defer resp.Body.Close()
resp, err := client.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected local observability status: %s", resp.Status)
}
return json.NewDecoder(resp.Body).Decode(target)
}
func fetchLocalText(client *http.Client, url string) (string, error) {
resp, err := client.Get(url)
func fetchLocalText(ctx context.Context, client *http.Client, url string) (string, error) {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return "", err
}
defer resp.Body.Close()
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected local stub status: %s", resp.Status)
}
@@ -104,7 +118,7 @@ func fetchLocalText(client *http.Client, url string) (string, error) {
func parseStubStatusActiveConnections(raw string) int64 {
matches := stubStatusActivePattern.FindStringSubmatch(raw)
if len(matches) != 2 {
if len(matches) != stubStatusActiveMatchGroupCount {
return 0
}
value, err := strconv.ParseInt(matches[1], 10, 64)
@@ -1,6 +1,7 @@
package observability
import (
"context"
"net"
"net/http"
"net/http/httptest"
@@ -31,7 +32,7 @@ func TestCollectManagedOpenRestyMetrics(t *testing.T) {
server.Start()
defer server.Close()
metrics := CollectManagedOpenRestyMetrics(&config.Config{
metrics := CollectManagedOpenRestyMetrics(context.Background(), &config.Config{
OpenrestyObservabilityPort: port,
})
if metrics == nil || metrics.TrafficReport == nil {
@@ -67,7 +68,7 @@ func TestNormalizeCountMapDropsEmptyKeys(t *testing.T) {
func TestCollectManagedOpenRestyMetricsHandlesUnavailableEndpoint(t *testing.T) {
cfg := &config.Config{OpenrestyObservabilityPort: 1}
if metrics := CollectManagedOpenRestyMetrics(cfg); metrics != nil {
if metrics := CollectManagedOpenRestyMetrics(context.Background(), cfg); metrics != nil {
t.Fatalf("expected nil metrics for unavailable endpoint, got %+v", metrics)
}
}
+12 -4
View File
@@ -6,6 +6,7 @@ import (
"errors"
"io"
"log/slog"
"net/http"
"os"
"regexp"
"sort"
@@ -28,6 +29,11 @@ type accessLogRecord struct {
RequestLength int64 `json:"request_length"`
}
const (
combinedAccessLogMatchGroupCount = 5
trafficTopDomainsLimit = 8
)
var combinedAccessLogPattern = regexp.MustCompile(`^(\S+)\s+\S+\s+\S+\s+\[([^]]+)]\s+"\S+\s+(\S+)(?:\s+[^"]*)?"\s+(\d{3})\s+\S+`)
type trafficAggregate struct {
@@ -43,11 +49,13 @@ type trafficAggregate struct {
logs []protocol.NodeAccessLog
}
// BuildTrafficReport generates a traffic report using access logs or falling back to managed metrics.
func BuildTrafficReport(cfg *config.Config, stateStore *state.Store, managed *ManagedOpenRestyMetrics) *protocol.NodeTrafficReport {
report, _, _ := BuildTrafficObservability(cfg, stateStore, managed)
return report
}
// BuildTrafficObservability returns the traffic report, parsed access logs, and managed metrics.
func BuildTrafficObservability(cfg *config.Config, stateStore *state.Store, managed *ManagedOpenRestyMetrics) (*protocol.NodeTrafficReport, []protocol.NodeAccessLog, *ManagedOpenRestyMetrics) {
if cfg == nil || stateStore == nil {
if managed != nil && managed.TrafficReport != nil {
@@ -78,7 +86,7 @@ func readAccessLogDelta(cfg *config.Config, stateStore *state.Store) *trafficAgg
}
logPath := managedAccessLogPath(cfg)
file, err := os.Open(logPath)
file, err := os.Open(logPath) //nolint:gosec // path is the configured managed access log location
if err != nil {
if os.IsNotExist(err) {
if snapshot.AccessLogOffset != 0 {
@@ -167,7 +175,7 @@ func (aggregate *trafficAggregate) consume(line []byte) {
}
aggregate.requestCount++
if record.Status >= 500 {
if record.Status >= http.StatusInternalServerError {
aggregate.errorCount++
}
if record.Status > 0 {
@@ -234,7 +242,7 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
func parseCombinedAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
matches := combinedAccessLogPattern.FindStringSubmatch(raw)
if len(matches) != 5 {
if len(matches) != combinedAccessLogMatchGroupCount {
return parsedAccessLogRecord{}, false
}
timestamp, err := parseAccessLogTime(matches[2])
@@ -265,7 +273,7 @@ func (aggregate *trafficAggregate) report() *protocol.NodeTrafficReport {
ErrorCount: aggregate.errorCount,
UniqueVisitorCount: int64(len(aggregate.visitors)),
StatusCodes: cloneTrafficCounts(aggregate.statusCodes, 0),
TopDomains: topCounts(aggregate.topDomains, 8),
TopDomains: topCounts(aggregate.topDomains, trafficTopDomainsLimit),
SourceCountries: map[string]int64{},
}
}