mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 23:26:38 +08:00
feat(obs): 访问日志 SSOT 与 edge_health,去掉协议兼容层
Agent 仅上报 host_metrics/edge_health/access_logs;业务流量与 UV 由 Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly, 删除 request_reports/openresty 吞吐路径;API 不再暴露 traffic_reports 与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
@@ -9,6 +9,7 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -16,6 +17,11 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/db/idgen"
|
||||
)
|
||||
|
||||
const (
|
||||
accessLogColumnStatusCode = "status_code"
|
||||
accessLogColumnHost = "host"
|
||||
)
|
||||
|
||||
type memoryAccessLogStore struct {
|
||||
mu sync.RWMutex
|
||||
records []*OpenFlareAccessLog
|
||||
@@ -123,6 +129,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
}
|
||||
item.RequestCount++
|
||||
item.BytesSent += row.BytesSent
|
||||
item.RequestLength += row.RequestLength
|
||||
switch {
|
||||
case row.StatusCode < 400:
|
||||
item.SuccessCount++
|
||||
@@ -155,6 +162,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
ClientErrorCount: result[index].ClientErrorCount,
|
||||
ServerErrorCount: result[index].ServerErrorCount,
|
||||
BytesSent: result[index].BytesSent,
|
||||
RequestLength: result[index].RequestLength,
|
||||
}
|
||||
}
|
||||
sortOpenFlareAccessLogBucketRows(bucketRows, filter.SortBy, filter.SortOrder)
|
||||
@@ -168,6 +176,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
ClientErrorCount: bucketRows[index].ClientErrorCount,
|
||||
ServerErrorCount: bucketRows[index].ServerErrorCount,
|
||||
BytesSent: bucketRows[index].BytesSent,
|
||||
RequestLength: bucketRows[index].RequestLength,
|
||||
}
|
||||
}
|
||||
if filter.PageSize > 0 {
|
||||
@@ -434,6 +443,113 @@ func (s *memoryAccessLogStore) DeleteByNodeBefore(_ context.Context, nodeID stri
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) TrafficSummary(_ context.Context, filter OpenFlareAccessLogQuery) (OpenFlareAccessLogTrafficSummary, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
ips := make(map[string]struct{})
|
||||
nodes := make(map[string]struct{})
|
||||
var summary OpenFlareAccessLogTrafficSummary
|
||||
for _, row := range rows {
|
||||
summary.RequestCount++
|
||||
summary.BytesSent += row.BytesSent
|
||||
summary.RequestLength += row.RequestLength
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
summary.ErrorCount++
|
||||
}
|
||||
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
|
||||
ips[ip] = struct{}{}
|
||||
}
|
||||
if id := strings.TrimSpace(row.NodeID); id != "" {
|
||||
nodes[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
summary.UniqueIPCount = int64(len(ips))
|
||||
summary.NodeCount = int64(len(nodes))
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) ValueCounts(_ context.Context, filter OpenFlareAccessLogQuery, column string, limit int) ([]OpenFlareAccessLogValueCount, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
col := strings.TrimSpace(strings.ToLower(column))
|
||||
if col != accessLogColumnStatusCode && col != accessLogColumnHost {
|
||||
return nil, nil
|
||||
}
|
||||
rows := s.filterRecords(filter)
|
||||
counts := make(map[string]int64)
|
||||
for _, row := range rows {
|
||||
var value string
|
||||
if col == accessLogColumnStatusCode {
|
||||
value = strconv.Itoa(row.StatusCode)
|
||||
} else {
|
||||
value = strings.TrimSpace(row.Host)
|
||||
}
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
counts[value]++
|
||||
}
|
||||
result := make([]OpenFlareAccessLogValueCount, 0, len(counts))
|
||||
for value, count := range counts {
|
||||
result = append(result, OpenFlareAccessLogValueCount{Value: value, Count: count})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].Count == result[j].Count {
|
||||
return result[i].Value < result[j].Value
|
||||
}
|
||||
return result[i].Count > result[j].Count
|
||||
})
|
||||
if limit > 0 && len(result) > limit {
|
||||
result = result[:limit]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) NodeAggregates(_ context.Context, filter OpenFlareAccessLogQuery) ([]OpenFlareAccessLogNodeAggregate, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
type acc struct {
|
||||
OpenFlareAccessLogNodeAggregate
|
||||
ips map[string]struct{}
|
||||
}
|
||||
byNode := make(map[string]*acc)
|
||||
for _, row := range rows {
|
||||
id := strings.TrimSpace(row.NodeID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
item := byNode[id]
|
||||
if item == nil {
|
||||
item = &acc{
|
||||
OpenFlareAccessLogNodeAggregate: OpenFlareAccessLogNodeAggregate{NodeID: id},
|
||||
ips: make(map[string]struct{}),
|
||||
}
|
||||
byNode[id] = item
|
||||
}
|
||||
item.RequestCount++
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
item.ErrorCount++
|
||||
}
|
||||
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
|
||||
item.ips[ip] = struct{}{}
|
||||
}
|
||||
}
|
||||
result := make([]OpenFlareAccessLogNodeAggregate, 0, len(byNode))
|
||||
for _, item := range byNode {
|
||||
item.UniqueIPCount = int64(len(item.ips))
|
||||
result = append(result, item.OpenFlareAccessLogNodeAggregate)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].RequestCount == result[j].RequestCount {
|
||||
return result[i].NodeID < result[j].NodeID
|
||||
}
|
||||
return result[i].RequestCount > result[j].RequestCount
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) filterRecords(query OpenFlareAccessLogQuery) []*OpenFlareAccessLog {
|
||||
result := make([]*OpenFlareAccessLog, 0, len(s.records))
|
||||
for _, row := range s.records {
|
||||
|
||||
Reference in New Issue
Block a user