mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-01 22:46:38 +08:00
feat(obs): 访问日志 SSOT 与 edge_health，去掉协议兼容层
Agent 仅上报 host_metrics/edge_health/access_logs；业务流量与 UV 由 Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly， 删除 request_reports/openresty 吞吐路径；API 不再暴露 traffic_reports 与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
ryan
@@ -10,21 +10,23 @@ import (
|
||||
|
||||
const (
|
||||
nodeAccessLogTableName = "of_node_access_logs"
|
||||
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, created_at"
|
||||
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, request_length, request_time_ms, created_at"
|
||||
)
|
||||
|
||||
// NodeAccessLog stores OpenFlare edge node access records in ClickHouse.
|
||||
type NodeAccessLog struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
LoggedAt time.Time `gorm:"column:logged_at"`
|
||||
RemoteAddr string `gorm:"column:remote_addr"`
|
||||
Region string `gorm:"column:region"`
|
||||
Host string `gorm:"column:host"`
|
||||
Path string `gorm:"column:path"`
|
||||
StatusCode int32 `gorm:"column:status_code"`
|
||||
BytesSent uint64 `gorm:"column:bytes_sent"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
LoggedAt time.Time `gorm:"column:logged_at"`
|
||||
RemoteAddr string `gorm:"column:remote_addr"`
|
||||
Region string `gorm:"column:region"`
|
||||
Host string `gorm:"column:host"`
|
||||
Path string `gorm:"column:path"`
|
||||
StatusCode int32 `gorm:"column:status_code"`
|
||||
BytesSent uint64 `gorm:"column:bytes_sent"`
|
||||
RequestLength uint64 `gorm:"column:request_length"`
|
||||
RequestTimeMs uint32 `gorm:"column:request_time_ms"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
|
||||
@@ -13,6 +13,7 @@ type NodeAccessLogBucketAggregate struct {
|
||||
UniqueIPCount int64 `gorm:"column:unique_ip_count"`
|
||||
UniqueHostCount int64 `gorm:"column:unique_host_count"`
|
||||
BytesSent int64 `gorm:"column:bytes_sent"`
|
||||
RequestLength int64 `gorm:"column:request_length"`
|
||||
}
|
||||
|
||||
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
|
||||
|
||||
@@ -12,11 +12,8 @@ const (
|
||||
nodeMetricSnapshotTableName = "of_node_metric_snapshots"
|
||||
nodeMetricSnapshotInsertColumns = "id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at"
|
||||
|
||||
nodeRequestReportTableName = "of_node_request_reports"
|
||||
nodeRequestReportInsertColumns = "id, node_id, window_started_at, window_ended_at, request_count, error_count, unique_visitor_count, status_codes_json, top_domains_json, source_countries_json, created_at"
|
||||
|
||||
nodeObsOpenrestyTableName = "of_node_obs_openresty"
|
||||
nodeObsOpenrestyInsertColumns = "id, node_id, captured_at, openresty_rx_bytes, openresty_tx_bytes, openresty_connections, created_at"
|
||||
nodeEdgeHealthTableName = "of_node_edge_health"
|
||||
nodeEdgeHealthInsertColumns = "id, node_id, captured_at, status, connections, created_at"
|
||||
|
||||
nodeObsFrpsTableName = "of_node_obs_frps"
|
||||
nodeObsFrpsInsertColumns = "id, node_id, captured_at, frps_connections, frps_proxy_count, frps_client_count, frps_proxies, created_at"
|
||||
@@ -57,60 +54,40 @@ func (NodeMetricSnapshot) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeMetricSnapshotTableName, nodeMetricSnapshotInsertColumns)
|
||||
}
|
||||
|
||||
// NodeRequestReport stores aggregated node request window reports in ClickHouse.
|
||||
type NodeRequestReport struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
WindowStartedAt time.Time `gorm:"column:window_started_at"`
|
||||
WindowEndedAt time.Time `gorm:"column:window_ended_at"`
|
||||
RequestCount int64 `gorm:"column:request_count"`
|
||||
ErrorCount int64 `gorm:"column:error_count"`
|
||||
UniqueVisitorCount int64 `gorm:"column:unique_visitor_count"`
|
||||
StatusCodesJSON string `gorm:"column:status_codes_json"`
|
||||
TopDomainsJSON string `gorm:"column:top_domains_json"`
|
||||
SourceCountriesJSON string `gorm:"column:source_countries_json"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
// NodeEdgeHealth stores L2 OpenResty health snapshots (connections + status).
|
||||
type NodeEdgeHealth struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
CapturedAt time.Time `gorm:"column:captured_at"`
|
||||
Status string `gorm:"column:status"`
|
||||
Connections int64 `gorm:"column:connections"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (NodeRequestReport) TableName() string {
|
||||
return nodeRequestReportTableName
|
||||
func (NodeEdgeHealth) TableName() string {
|
||||
return nodeEdgeHealthTableName
|
||||
}
|
||||
|
||||
// InsertColumns returns comma-separated column names for batch insert.
|
||||
func (NodeRequestReport) InsertColumns() string {
|
||||
return nodeRequestReportInsertColumns
|
||||
func (NodeEdgeHealth) InsertColumns() string {
|
||||
return nodeEdgeHealthInsertColumns
|
||||
}
|
||||
|
||||
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
|
||||
func (NodeRequestReport) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeRequestReportTableName, nodeRequestReportInsertColumns)
|
||||
func (NodeEdgeHealth) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeEdgeHealthTableName, nodeEdgeHealthInsertColumns)
|
||||
}
|
||||
|
||||
// NodeObsOpenresty stores OpenResty observability snapshots in ClickHouse.
|
||||
type NodeObsOpenresty struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
CapturedAt time.Time `gorm:"column:captured_at"`
|
||||
OpenrestyRxBytes int64 `gorm:"column:openresty_rx_bytes"`
|
||||
OpenrestyTxBytes int64 `gorm:"column:openresty_tx_bytes"`
|
||||
OpenrestyConnections int64 `gorm:"column:openresty_connections"`
|
||||
CreatedAt time.Time `gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (NodeObsOpenresty) TableName() string {
|
||||
return nodeObsOpenrestyTableName
|
||||
}
|
||||
|
||||
// InsertColumns returns comma-separated column names for batch insert.
|
||||
func (NodeObsOpenresty) InsertColumns() string {
|
||||
return nodeObsOpenrestyInsertColumns
|
||||
}
|
||||
|
||||
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
|
||||
func (NodeObsOpenresty) BatchInsertSQL() string {
|
||||
return fmt.Sprintf("INSERT INTO %s (%s)", nodeObsOpenrestyTableName, nodeObsOpenrestyInsertColumns)
|
||||
// AccessLogHourly is a Server-side hourly rollup of access logs.
|
||||
type AccessLogHourly struct {
|
||||
NodeID string `gorm:"column:node_id"`
|
||||
Hour time.Time `gorm:"column:hour"`
|
||||
Host string `gorm:"column:host"`
|
||||
RequestCount int64 `gorm:"column:request_count"`
|
||||
ErrorCount int64 `gorm:"column:error_count"`
|
||||
BytesSent int64 `gorm:"column:bytes_sent"`
|
||||
RequestLength int64 `gorm:"column:request_length"`
|
||||
}
|
||||
|
||||
// NodeObsFrps stores FRPS observability snapshots in ClickHouse.
|
||||
|
||||
@@ -72,6 +72,21 @@ func CountOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery
|
||||
return currentAccessLogStore().Count(ctx, query)
|
||||
}
|
||||
|
||||
// TrafficSummaryOpenFlareAccessLogs returns window-level request/error/UV/bytes summary.
|
||||
func TrafficSummaryOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery) (OpenFlareAccessLogTrafficSummary, error) {
|
||||
return currentAccessLogStore().TrafficSummary(ctx, query)
|
||||
}
|
||||
|
||||
// ValueCountsOpenFlareAccessLogs groups logs by status_code or host.
|
||||
func ValueCountsOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery, column string, limit int) ([]OpenFlareAccessLogValueCount, error) {
|
||||
return currentAccessLogStore().ValueCounts(ctx, query, column, limit)
|
||||
}
|
||||
|
||||
// NodeAggregatesOpenFlareAccessLogs returns per-node request/error/UV for the window.
|
||||
func NodeAggregatesOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery) ([]OpenFlareAccessLogNodeAggregate, error) {
|
||||
return currentAccessLogStore().NodeAggregates(ctx, query)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogRegionCounts returns region counts for access logs.
|
||||
func ListOpenFlareAccessLogRegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareAccessLogRegionCount, error) {
|
||||
return currentAccessLogStore().RegionCounts(ctx, nodeID, since, limit)
|
||||
@@ -192,6 +207,7 @@ func buildOpenFlareAccessLogBucketRows(ctx context.Context, query OpenFlareAcces
|
||||
ClientErrorCount: partial.ClientErrorCount,
|
||||
ServerErrorCount: partial.ServerErrorCount,
|
||||
BytesSent: partial.BytesSent,
|
||||
RequestLength: partial.RequestLength,
|
||||
})
|
||||
}
|
||||
return rows, nil
|
||||
|
||||
@@ -50,11 +50,38 @@ type accessLogStore interface {
|
||||
IPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error)
|
||||
CountIPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery) (int64, error)
|
||||
IPTrend(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error)
|
||||
TrafficSummary(ctx context.Context, filter OpenFlareAccessLogQuery) (OpenFlareAccessLogTrafficSummary, error)
|
||||
ValueCounts(ctx context.Context, filter OpenFlareAccessLogQuery, column string, limit int) ([]OpenFlareAccessLogValueCount, error)
|
||||
NodeAggregates(ctx context.Context, filter OpenFlareAccessLogQuery) ([]OpenFlareAccessLogNodeAggregate, error)
|
||||
DeleteAll(ctx context.Context) (int64, error)
|
||||
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogTrafficSummary is a window-level traffic summary from access logs.
|
||||
type OpenFlareAccessLogTrafficSummary struct {
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
BytesSent int64
|
||||
RequestLength int64
|
||||
NodeCount int64
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogValueCount is a dimension value count.
|
||||
type OpenFlareAccessLogValueCount struct {
|
||||
Value string
|
||||
Count int64
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type OpenFlareAccessLogNodeAggregate struct {
|
||||
NodeID string
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
}
|
||||
|
||||
var (
|
||||
accessLogStoreMu sync.RWMutex
|
||||
accessLogStoreHolder accessLogStore
|
||||
@@ -176,6 +203,50 @@ func (clickhouseAccessLogStore) DeleteByNodeBefore(ctx context.Context, nodeID s
|
||||
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) TrafficSummary(ctx context.Context, filter OpenFlareAccessLogQuery) (OpenFlareAccessLogTrafficSummary, error) {
|
||||
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
if err != nil {
|
||||
return OpenFlareAccessLogTrafficSummary{}, err
|
||||
}
|
||||
return OpenFlareAccessLogTrafficSummary{
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueIPCount: row.UniqueIPCount,
|
||||
BytesSent: row.BytesSent,
|
||||
RequestLength: row.RequestLength,
|
||||
NodeCount: row.NodeCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) ValueCounts(ctx context.Context, filter OpenFlareAccessLogQuery, column string, limit int) ([]OpenFlareAccessLogValueCount, error) {
|
||||
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]OpenFlareAccessLogValueCount, len(rows))
|
||||
for i, row := range rows {
|
||||
result[i] = OpenFlareAccessLogValueCount{Value: row.Value, Count: row.Count}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) NodeAggregates(ctx context.Context, filter OpenFlareAccessLogQuery) ([]OpenFlareAccessLogNodeAggregate, error) {
|
||||
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]OpenFlareAccessLogNodeAggregate, len(rows))
|
||||
for i, row := range rows {
|
||||
result[i] = OpenFlareAccessLogNodeAggregate{
|
||||
NodeID: row.NodeID,
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueIPCount: row.UniqueIPCount,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func toNodeAccessLogFilter(query OpenFlareAccessLogQuery) analyticsrepo.NodeAccessLogFilter {
|
||||
return analyticsrepo.NodeAccessLogFilter{
|
||||
NodeID: query.NodeID,
|
||||
@@ -197,17 +268,27 @@ func toAnalyticsNodeAccessLog(record *OpenFlareAccessLog) analyticsmodel.NodeAcc
|
||||
if record.BytesSent > 0 {
|
||||
bytesSent = uint64(record.BytesSent)
|
||||
}
|
||||
var requestLength uint64
|
||||
if record.RequestLength > 0 {
|
||||
requestLength = uint64(record.RequestLength)
|
||||
}
|
||||
var requestTimeMs uint32
|
||||
if record.RequestTimeMs > 0 && record.RequestTimeMs <= int64(math.MaxUint32) {
|
||||
requestTimeMs = uint32(record.RequestTimeMs)
|
||||
}
|
||||
return analyticsmodel.NodeAccessLog{
|
||||
ID: record.ID,
|
||||
NodeID: record.NodeID,
|
||||
LoggedAt: record.LoggedAt,
|
||||
RemoteAddr: record.RemoteAddr,
|
||||
Region: record.Region,
|
||||
Host: record.Host,
|
||||
Path: record.Path,
|
||||
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
|
||||
BytesSent: bytesSent,
|
||||
CreatedAt: record.CreatedAt,
|
||||
ID: record.ID,
|
||||
NodeID: record.NodeID,
|
||||
LoggedAt: record.LoggedAt,
|
||||
RemoteAddr: record.RemoteAddr,
|
||||
Region: record.Region,
|
||||
Host: record.Host,
|
||||
Path: record.Path,
|
||||
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
|
||||
BytesSent: bytesSent,
|
||||
RequestLength: requestLength,
|
||||
RequestTimeMs: requestTimeMs,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,17 +301,25 @@ func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*OpenFla
|
||||
} else {
|
||||
bytesSent = math.MaxInt64
|
||||
}
|
||||
var requestLength int64
|
||||
if row.RequestLength <= math.MaxInt64 {
|
||||
requestLength = int64(row.RequestLength)
|
||||
} else {
|
||||
requestLength = math.MaxInt64
|
||||
}
|
||||
result[index] = &OpenFlareAccessLog{
|
||||
ID: row.ID,
|
||||
NodeID: row.NodeID,
|
||||
LoggedAt: row.LoggedAt,
|
||||
RemoteAddr: row.RemoteAddr,
|
||||
Region: row.Region,
|
||||
Host: row.Host,
|
||||
Path: row.Path,
|
||||
StatusCode: int(row.StatusCode),
|
||||
BytesSent: bytesSent,
|
||||
CreatedAt: row.CreatedAt,
|
||||
ID: row.ID,
|
||||
NodeID: row.NodeID,
|
||||
LoggedAt: row.LoggedAt,
|
||||
RemoteAddr: row.RemoteAddr,
|
||||
Region: row.Region,
|
||||
Host: row.Host,
|
||||
Path: row.Path,
|
||||
StatusCode: int(row.StatusCode),
|
||||
BytesSent: bytesSent,
|
||||
RequestLength: requestLength,
|
||||
RequestTimeMs: int64(row.RequestTimeMs),
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -16,6 +17,11 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/db/idgen"
|
||||
)
|
||||
|
||||
const (
|
||||
accessLogColumnStatusCode = "status_code"
|
||||
accessLogColumnHost = "host"
|
||||
)
|
||||
|
||||
type memoryAccessLogStore struct {
|
||||
mu sync.RWMutex
|
||||
records []*OpenFlareAccessLog
|
||||
@@ -123,6 +129,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
}
|
||||
item.RequestCount++
|
||||
item.BytesSent += row.BytesSent
|
||||
item.RequestLength += row.RequestLength
|
||||
switch {
|
||||
case row.StatusCode < 400:
|
||||
item.SuccessCount++
|
||||
@@ -155,6 +162,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
ClientErrorCount: result[index].ClientErrorCount,
|
||||
ServerErrorCount: result[index].ServerErrorCount,
|
||||
BytesSent: result[index].BytesSent,
|
||||
RequestLength: result[index].RequestLength,
|
||||
}
|
||||
}
|
||||
sortOpenFlareAccessLogBucketRows(bucketRows, filter.SortBy, filter.SortOrder)
|
||||
@@ -168,6 +176,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
ClientErrorCount: bucketRows[index].ClientErrorCount,
|
||||
ServerErrorCount: bucketRows[index].ServerErrorCount,
|
||||
BytesSent: bucketRows[index].BytesSent,
|
||||
RequestLength: bucketRows[index].RequestLength,
|
||||
}
|
||||
}
|
||||
if filter.PageSize > 0 {
|
||||
@@ -434,6 +443,113 @@ func (s *memoryAccessLogStore) DeleteByNodeBefore(_ context.Context, nodeID stri
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) TrafficSummary(_ context.Context, filter OpenFlareAccessLogQuery) (OpenFlareAccessLogTrafficSummary, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
ips := make(map[string]struct{})
|
||||
nodes := make(map[string]struct{})
|
||||
var summary OpenFlareAccessLogTrafficSummary
|
||||
for _, row := range rows {
|
||||
summary.RequestCount++
|
||||
summary.BytesSent += row.BytesSent
|
||||
summary.RequestLength += row.RequestLength
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
summary.ErrorCount++
|
||||
}
|
||||
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
|
||||
ips[ip] = struct{}{}
|
||||
}
|
||||
if id := strings.TrimSpace(row.NodeID); id != "" {
|
||||
nodes[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
summary.UniqueIPCount = int64(len(ips))
|
||||
summary.NodeCount = int64(len(nodes))
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) ValueCounts(_ context.Context, filter OpenFlareAccessLogQuery, column string, limit int) ([]OpenFlareAccessLogValueCount, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
col := strings.TrimSpace(strings.ToLower(column))
|
||||
if col != accessLogColumnStatusCode && col != accessLogColumnHost {
|
||||
return nil, nil
|
||||
}
|
||||
rows := s.filterRecords(filter)
|
||||
counts := make(map[string]int64)
|
||||
for _, row := range rows {
|
||||
var value string
|
||||
if col == accessLogColumnStatusCode {
|
||||
value = strconv.Itoa(row.StatusCode)
|
||||
} else {
|
||||
value = strings.TrimSpace(row.Host)
|
||||
}
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
counts[value]++
|
||||
}
|
||||
result := make([]OpenFlareAccessLogValueCount, 0, len(counts))
|
||||
for value, count := range counts {
|
||||
result = append(result, OpenFlareAccessLogValueCount{Value: value, Count: count})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].Count == result[j].Count {
|
||||
return result[i].Value < result[j].Value
|
||||
}
|
||||
return result[i].Count > result[j].Count
|
||||
})
|
||||
if limit > 0 && len(result) > limit {
|
||||
result = result[:limit]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) NodeAggregates(_ context.Context, filter OpenFlareAccessLogQuery) ([]OpenFlareAccessLogNodeAggregate, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
type acc struct {
|
||||
OpenFlareAccessLogNodeAggregate
|
||||
ips map[string]struct{}
|
||||
}
|
||||
byNode := make(map[string]*acc)
|
||||
for _, row := range rows {
|
||||
id := strings.TrimSpace(row.NodeID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
item := byNode[id]
|
||||
if item == nil {
|
||||
item = &acc{
|
||||
OpenFlareAccessLogNodeAggregate: OpenFlareAccessLogNodeAggregate{NodeID: id},
|
||||
ips: make(map[string]struct{}),
|
||||
}
|
||||
byNode[id] = item
|
||||
}
|
||||
item.RequestCount++
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
item.ErrorCount++
|
||||
}
|
||||
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
|
||||
item.ips[ip] = struct{}{}
|
||||
}
|
||||
}
|
||||
result := make([]OpenFlareAccessLogNodeAggregate, 0, len(byNode))
|
||||
for _, item := range byNode {
|
||||
item.UniqueIPCount = int64(len(item.ips))
|
||||
result = append(result, item.OpenFlareAccessLogNodeAggregate)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].RequestCount == result[j].RequestCount {
|
||||
return result[i].NodeID < result[j].NodeID
|
||||
}
|
||||
return result[i].RequestCount > result[j].RequestCount
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) filterRecords(query OpenFlareAccessLogQuery) []*OpenFlareAccessLog {
|
||||
result := make([]*OpenFlareAccessLog, 0, len(s.records))
|
||||
for _, row := range s.records {
|
||||
|
||||
@@ -37,40 +37,21 @@ func (OpenFlareMetricSnapshot) TableName() string {
|
||||
return "of_node_metric_snapshots"
|
||||
}
|
||||
|
||||
// OpenFlareRequestReport stores aggregated traffic windows per node in ClickHouse (database: openflare, table: of_node_request_reports).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareRequestReport struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
WindowStartedAt time.Time `json:"window_started_at" gorm:"index"`
|
||||
WindowEndedAt time.Time `json:"window_ended_at" gorm:"index"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
UniqueVisitorCount int64 `json:"unique_visitor_count"`
|
||||
StatusCodesJSON string `json:"status_codes_json" gorm:"type:text"`
|
||||
TopDomainsJSON string `json:"top_domains_json" gorm:"type:text"`
|
||||
SourceCountriesJSON string `json:"source_countries_json" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareRequestReport) TableName() string {
|
||||
return "of_node_request_reports"
|
||||
}
|
||||
|
||||
// OpenFlareAccessLog stores a single access log row in ClickHouse (database: openflare, table: of_node_access_logs).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareAccessLog struct {
|
||||
ID uint64 `json:"id,string" gorm:"column:id"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
LoggedAt time.Time `json:"logged_at" gorm:"index"`
|
||||
RemoteAddr string `json:"remote_addr" gorm:"index;size:128"`
|
||||
Region string `json:"region" gorm:"size:128"`
|
||||
Host string `json:"host" gorm:"index;size:255"`
|
||||
Path string `json:"path" gorm:"size:2048"`
|
||||
StatusCode int `json:"status_code" gorm:"index"`
|
||||
BytesSent int64 `json:"bytes_sent" gorm:"column:bytes_sent;not null;default:0"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
ID uint64 `json:"id,string" gorm:"column:id"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
LoggedAt time.Time `json:"logged_at" gorm:"index"`
|
||||
RemoteAddr string `json:"remote_addr" gorm:"index;size:128"`
|
||||
Region string `json:"region" gorm:"size:128"`
|
||||
Host string `json:"host" gorm:"index;size:255"`
|
||||
Path string `json:"path" gorm:"size:2048"`
|
||||
StatusCode int `json:"status_code" gorm:"index"`
|
||||
BytesSent int64 `json:"bytes_sent" gorm:"column:bytes_sent;not null;default:0"`
|
||||
RequestLength int64 `json:"request_length" gorm:"column:request_length;not null;default:0"`
|
||||
RequestTimeMs int64 `json:"request_time_ms" gorm:"column:request_time_ms;not null;default:0"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
@@ -130,21 +111,19 @@ func (OpenFlareNodeSystemProfile) TableName() string {
|
||||
return "of_node_system_profiles"
|
||||
}
|
||||
|
||||
// OpenFlareNodeObservationOpenresty stores openresty network observations in ClickHouse (database: openflare, table: of_node_obs_openresty).
|
||||
// ClickHouse DDL is managed by goose; reads/writes go through internal/repository/analytics.
|
||||
type OpenFlareNodeObservationOpenresty struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
|
||||
CapturedAt time.Time `json:"captured_at" gorm:"index"`
|
||||
OpenrestyRxBytes int64 `json:"openresty_rx_bytes"`
|
||||
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
|
||||
OpenrestyConnections int64 `json:"openresty_connections"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
// OpenFlareEdgeHealth is L2 OpenResty health (of_node_edge_health).
|
||||
type OpenFlareEdgeHealth struct {
|
||||
ID uint `json:"id"`
|
||||
NodeID string `json:"node_id"`
|
||||
CapturedAt time.Time `json:"captured_at"`
|
||||
Status string `json:"status"`
|
||||
Connections int64 `json:"connections"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// TableName returns the GORM table name.
|
||||
func (OpenFlareNodeObservationOpenresty) TableName() string {
|
||||
return "of_node_obs_openresty"
|
||||
// TableName returns the ClickHouse table name.
|
||||
func (OpenFlareEdgeHealth) TableName() string {
|
||||
return "of_node_edge_health"
|
||||
}
|
||||
|
||||
// OpenFlareNodeObservationFrpc stores tunnel client frpc observations in ClickHouse (database: openflare, table: of_node_obs_frpc).
|
||||
@@ -223,6 +202,7 @@ type OpenFlareAccessLogBucketRow struct {
|
||||
ClientErrorCount int64 `json:"client_error_count"`
|
||||
ServerErrorCount int64 `json:"server_error_count"`
|
||||
BytesSent int64 `json:"bytes_sent"`
|
||||
RequestLength int64 `json:"request_length"`
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogBucketIPQuery filters folded IP summary queries (v1 stub).
|
||||
@@ -314,14 +294,9 @@ func InsertOpenFlareMetricSnapshot(ctx context.Context, record *OpenFlareMetricS
|
||||
return currentObservabilityStore().InsertMetricSnapshot(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareRequestReport inserts a request report into ClickHouse.
|
||||
func InsertOpenFlareRequestReport(ctx context.Context, record *OpenFlareRequestReport) error {
|
||||
return currentObservabilityStore().InsertRequestReport(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareNodeObservationOpenresty inserts an OpenResty observation into ClickHouse.
|
||||
func InsertOpenFlareNodeObservationOpenresty(ctx context.Context, record *OpenFlareNodeObservationOpenresty) error {
|
||||
return currentObservabilityStore().InsertNodeObservationOpenresty(ctx, record)
|
||||
// InsertOpenFlareEdgeHealth inserts an L2 edge health snapshot into ClickHouse.
|
||||
func InsertOpenFlareEdgeHealth(ctx context.Context, record *OpenFlareEdgeHealth) error {
|
||||
return currentObservabilityStore().InsertEdgeHealth(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareNodeObservationFrps inserts an FRPS observation into ClickHouse.
|
||||
@@ -357,28 +332,6 @@ func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string,
|
||||
return openFlareLatestMetricSnapshots(all), nil
|
||||
}
|
||||
|
||||
// ListOpenFlareRequestReportsSince returns request reports since the given time.
|
||||
func ListOpenFlareRequestReportsSince(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareRequestReport, error) {
|
||||
return currentObservabilityStore().ListRequestReports(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareLatestRequestReportsSince returns the latest request report per node.
|
||||
// Prefer ClickHouse LIMIT 1 BY; on CH unavailability fall back to store list + reduce.
|
||||
func ListOpenFlareLatestRequestReportsSince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareRequestReport, error) {
|
||||
rows, err := analyticsrepo.ListLatestNodeRequestReports(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
if err == nil {
|
||||
return fromAnalyticsNodeRequestReports(rows), nil
|
||||
}
|
||||
all, listErr := ListOpenFlareRequestReportsSince(ctx, nodeID, since, 0)
|
||||
if listErr != nil {
|
||||
return nil, err
|
||||
}
|
||||
return openFlareLatestRequestReports(all), nil
|
||||
}
|
||||
|
||||
func openFlareLatestMetricSnapshots(snapshots []*OpenFlareMetricSnapshot) []*OpenFlareMetricSnapshot {
|
||||
latestByNode := make(map[string]*OpenFlareMetricSnapshot, len(snapshots))
|
||||
for _, snapshot := range snapshots {
|
||||
@@ -397,24 +350,6 @@ func openFlareLatestMetricSnapshots(snapshots []*OpenFlareMetricSnapshot) []*Ope
|
||||
return result
|
||||
}
|
||||
|
||||
func openFlareLatestRequestReports(reports []*OpenFlareRequestReport) []*OpenFlareRequestReport {
|
||||
latestByNode := make(map[string]*OpenFlareRequestReport, len(reports))
|
||||
for _, report := range reports {
|
||||
if report == nil || report.NodeID == "" {
|
||||
continue
|
||||
}
|
||||
if existing, ok := latestByNode[report.NodeID]; ok && !report.WindowEndedAt.After(existing.WindowEndedAt) {
|
||||
continue
|
||||
}
|
||||
latestByNode[report.NodeID] = report
|
||||
}
|
||||
result := make([]*OpenFlareRequestReport, 0, len(latestByNode))
|
||||
for _, report := range latestByNode {
|
||||
result = append(result, report)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// OpenFlareTrafficHourly is an hourly traffic rollup row.
|
||||
type OpenFlareTrafficHourly struct {
|
||||
NodeID string `json:"node_id"`
|
||||
@@ -425,6 +360,7 @@ type OpenFlareTrafficHourly struct {
|
||||
}
|
||||
|
||||
// ListOpenFlareTrafficHourlySince returns hourly traffic rollup rows since the given time.
|
||||
// Source: of_access_log_hourly (M5).
|
||||
func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareTrafficHourly, error) {
|
||||
rows, err := analyticsrepo.ListNodeTrafficHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
@@ -446,6 +382,41 @@ func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since t
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogHourly is a per-node/host hourly access log rollup.
|
||||
type OpenFlareAccessLogHourly struct {
|
||||
NodeID string `json:"node_id"`
|
||||
Hour time.Time `json:"hour"`
|
||||
Host string `json:"host"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
BytesSent int64 `json:"bytes_sent"`
|
||||
RequestLength int64 `json:"request_length"`
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogHourlySince returns of_access_log_hourly rows since the given time.
|
||||
func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareAccessLogHourly, error) {
|
||||
rows, err := analyticsrepo.ListAccessLogHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*OpenFlareAccessLogHourly, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &OpenFlareAccessLogHourly{
|
||||
NodeID: row.NodeID,
|
||||
Hour: row.Hour,
|
||||
Host: row.Host,
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
BytesSent: row.BytesSent,
|
||||
RequestLength: row.RequestLength,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// OpenFlareMetricHourly is an hourly metric snapshot aggregation row.
|
||||
type OpenFlareMetricHourly struct {
|
||||
Hour time.Time `json:"hour"`
|
||||
@@ -458,14 +429,6 @@ type OpenFlareMetricHourly struct {
|
||||
ReportedNodes int `json:"reported_nodes"`
|
||||
}
|
||||
|
||||
// OpenFlareOpenrestyHourly is an hourly OpenResty observation aggregation row.
|
||||
type OpenFlareOpenrestyHourly struct {
|
||||
Hour time.Time `json:"hour"`
|
||||
OpenrestyRxBytes int64 `json:"openresty_rx_bytes"`
|
||||
OpenrestyTxBytes int64 `json:"openresty_tx_bytes"`
|
||||
ReportedNodes int `json:"reported_nodes"`
|
||||
}
|
||||
|
||||
// ListOpenFlareMetricHourlySince returns hourly metric aggregates since the given time.
|
||||
func ListOpenFlareMetricHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareMetricHourly, error) {
|
||||
rows, err := analyticsrepo.ListNodeMetricHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
@@ -491,27 +454,6 @@ func ListOpenFlareMetricHourlySince(ctx context.Context, nodeID string, since ti
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareOpenrestyHourlySince returns hourly OpenResty aggregates since the given time.
|
||||
func ListOpenFlareOpenrestyHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareOpenrestyHourly, error) {
|
||||
rows, err := analyticsrepo.ListNodeOpenrestyHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*OpenFlareOpenrestyHourly, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &OpenFlareOpenrestyHourly{
|
||||
Hour: row.Hour,
|
||||
OpenrestyRxBytes: row.OpenrestyRxBytes,
|
||||
OpenrestyTxBytes: row.OpenrestyTxBytes,
|
||||
ReportedNodes: row.ReportedNodes,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareActiveHealthEvents returns active health events across all nodes.
|
||||
func ListOpenFlareActiveHealthEvents(ctx context.Context) ([]*OpenFlareHealthEvent, error) {
|
||||
conn := db.DB(ctx)
|
||||
@@ -561,24 +503,14 @@ func DeleteAllOpenFlareMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllMetricSnapshots(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareRequestReportsBefore deletes request reports ending before cutoff.
|
||||
func DeleteOpenFlareRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteRequestReportsBefore(ctx, cutoff)
|
||||
// DeleteOpenFlareEdgeHealthBefore deletes edge health rows captured before cutoff.
|
||||
func DeleteOpenFlareEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteEdgeHealthBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareRequestReports deletes all request reports.
|
||||
func DeleteAllOpenFlareRequestReports(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllRequestReports(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationOpenrestyBefore deletes OpenResty observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteNodeObservationOpenrestyBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationOpenresty deletes all OpenResty observations.
|
||||
func DeleteAllOpenFlareNodeObservationOpenresty(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllNodeObservationOpenresty(ctx)
|
||||
// DeleteAllOpenFlareEdgeHealth deletes all edge health snapshots.
|
||||
func DeleteAllOpenFlareEdgeHealth(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllEdgeHealth(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
|
||||
@@ -633,9 +565,9 @@ func GetOpenFlareNodeSystemProfile(ctx context.Context, nodeID string) (*OpenFla
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareNodeObservationOpenresty returns openresty observations.
|
||||
func ListOpenFlareNodeObservationOpenresty(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareNodeObservationOpenresty, error) {
|
||||
return currentObservabilityStore().ListNodeObservationOpenresty(ctx, nodeID, since, limit)
|
||||
// ListOpenFlareEdgeHealth returns L2 edge health snapshots.
|
||||
func ListOpenFlareEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareEdgeHealth, error) {
|
||||
return currentObservabilityStore().ListEdgeHealth(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareNodeObservationFrpc returns frpc observations.
|
||||
|
||||
@@ -6,6 +6,7 @@ package model
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -16,11 +17,10 @@ import (
|
||||
// ObservabilityInsertHooks queues observability rows for async ClickHouse write.
|
||||
// Wired from openflare/chwriter.Init so model never imports the apps layer.
|
||||
type ObservabilityInsertHooks struct {
|
||||
QueueMetricSnapshot func(analyticsmodel.NodeMetricSnapshot)
|
||||
QueueRequestReport func(analyticsmodel.NodeRequestReport)
|
||||
QueueOpenrestyObservation func(analyticsmodel.NodeObsOpenresty)
|
||||
QueueFrpsObservation func(analyticsmodel.NodeObsFrps)
|
||||
QueueFrpcObservation func(analyticsmodel.NodeObsFrpc)
|
||||
QueueMetricSnapshot func(analyticsmodel.NodeMetricSnapshot)
|
||||
QueueEdgeHealth func(analyticsmodel.NodeEdgeHealth)
|
||||
QueueFrpsObservation func(analyticsmodel.NodeObsFrps)
|
||||
QueueFrpcObservation func(analyticsmodel.NodeObsFrpc)
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -47,15 +47,10 @@ type observabilityStore interface {
|
||||
DeleteAllMetricSnapshots(ctx context.Context) (int64, error)
|
||||
DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
|
||||
InsertRequestReport(ctx context.Context, record *OpenFlareRequestReport) error
|
||||
ListRequestReports(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareRequestReport, error)
|
||||
DeleteAllRequestReports(ctx context.Context) (int64, error)
|
||||
DeleteRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
|
||||
InsertNodeObservationOpenresty(ctx context.Context, record *OpenFlareNodeObservationOpenresty) error
|
||||
ListNodeObservationOpenresty(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareNodeObservationOpenresty, error)
|
||||
DeleteAllNodeObservationOpenresty(ctx context.Context) (int64, error)
|
||||
DeleteNodeObservationOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
InsertEdgeHealth(ctx context.Context, record *OpenFlareEdgeHealth) error
|
||||
ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareEdgeHealth, error)
|
||||
DeleteAllEdgeHealth(ctx context.Context) (int64, error)
|
||||
DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
|
||||
InsertNodeObservationFrps(ctx context.Context, record *OpenFlareNodeObservationFrps) error
|
||||
ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareNodeObservationFrps, error)
|
||||
@@ -128,56 +123,40 @@ func (clickhouseObservabilityStore) DeleteMetricSnapshotsBefore(ctx context.Cont
|
||||
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertRequestReport(_ context.Context, record *OpenFlareRequestReport) error {
|
||||
const edgeHealthStatusUnknown = "unknown"
|
||||
|
||||
func normalizeEdgeHealthStatus(status string) string {
|
||||
status = strings.TrimSpace(status)
|
||||
if status == "" {
|
||||
return edgeHealthStatusUnknown
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertEdgeHealth(_ context.Context, record *OpenFlareEdgeHealth) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if hook := currentObservabilityInsertHooks().QueueRequestReport; hook != nil {
|
||||
hook(toAnalyticsNodeRequestReport(record))
|
||||
if hook := currentObservabilityInsertHooks().QueueEdgeHealth; hook != nil {
|
||||
hook(toAnalyticsNodeEdgeHealth(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) ListRequestReports(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareRequestReport, error) {
|
||||
rows, err := analyticsrepo.ListNodeRequestReports(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
func (clickhouseObservabilityStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareEdgeHealth, error) {
|
||||
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeRequestReports(rows), nil
|
||||
return fromAnalyticsNodeEdgeHealth(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteAllRequestReports(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeRequestReports(ctx)
|
||||
func (clickhouseObservabilityStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeRequestReportsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertNodeObservationOpenresty(_ context.Context, record *OpenFlareNodeObservationOpenresty) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if hook := currentObservabilityInsertHooks().QueueOpenrestyObservation; hook != nil {
|
||||
hook(toAnalyticsNodeObsOpenresty(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) ListNodeObservationOpenresty(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareNodeObservationOpenresty, error) {
|
||||
rows, err := analyticsrepo.ListNodeObsOpenresty(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeObsOpenresty(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteAllNodeObservationOpenresty(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeObsOpenresty(ctx)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteNodeObservationOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeObsOpenrestyBefore(ctx, cutoff)
|
||||
func (clickhouseObservabilityStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertNodeObservationFrps(_ context.Context, record *OpenFlareNodeObservationFrps) error {
|
||||
@@ -280,65 +259,27 @@ func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot)
|
||||
return result
|
||||
}
|
||||
|
||||
func toAnalyticsNodeRequestReport(record *OpenFlareRequestReport) analyticsmodel.NodeRequestReport {
|
||||
return analyticsmodel.NodeRequestReport{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
WindowStartedAt: record.WindowStartedAt,
|
||||
WindowEndedAt: record.WindowEndedAt,
|
||||
RequestCount: record.RequestCount,
|
||||
ErrorCount: record.ErrorCount,
|
||||
UniqueVisitorCount: record.UniqueVisitorCount,
|
||||
StatusCodesJSON: record.StatusCodesJSON,
|
||||
TopDomainsJSON: record.TopDomainsJSON,
|
||||
SourceCountriesJSON: record.SourceCountriesJSON,
|
||||
CreatedAt: record.CreatedAt,
|
||||
func toAnalyticsNodeEdgeHealth(record *OpenFlareEdgeHealth) analyticsmodel.NodeEdgeHealth {
|
||||
return analyticsmodel.NodeEdgeHealth{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
CapturedAt: record.CapturedAt,
|
||||
Status: normalizeEdgeHealthStatus(record.Status),
|
||||
Connections: record.Connections,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func fromAnalyticsNodeRequestReports(rows []analyticsmodel.NodeRequestReport) []*OpenFlareRequestReport {
|
||||
result := make([]*OpenFlareRequestReport, len(rows))
|
||||
func fromAnalyticsNodeEdgeHealth(rows []analyticsmodel.NodeEdgeHealth) []*OpenFlareEdgeHealth {
|
||||
result := make([]*OpenFlareEdgeHealth, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &OpenFlareRequestReport{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
WindowStartedAt: row.WindowStartedAt,
|
||||
WindowEndedAt: row.WindowEndedAt,
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueVisitorCount: row.UniqueVisitorCount,
|
||||
StatusCodesJSON: row.StatusCodesJSON,
|
||||
TopDomainsJSON: row.TopDomainsJSON,
|
||||
SourceCountriesJSON: row.SourceCountriesJSON,
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toAnalyticsNodeObsOpenresty(record *OpenFlareNodeObservationOpenresty) analyticsmodel.NodeObsOpenresty {
|
||||
return analyticsmodel.NodeObsOpenresty{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
CapturedAt: record.CapturedAt,
|
||||
OpenrestyRxBytes: record.OpenrestyRxBytes,
|
||||
OpenrestyTxBytes: record.OpenrestyTxBytes,
|
||||
OpenrestyConnections: record.OpenrestyConnections,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func fromAnalyticsNodeObsOpenresty(rows []analyticsmodel.NodeObsOpenresty) []*OpenFlareNodeObservationOpenresty {
|
||||
result := make([]*OpenFlareNodeObservationOpenresty, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &OpenFlareNodeObservationOpenresty{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
OpenrestyRxBytes: row.OpenrestyRxBytes,
|
||||
OpenrestyTxBytes: row.OpenrestyTxBytes,
|
||||
OpenrestyConnections: row.OpenrestyConnections,
|
||||
CreatedAt: row.CreatedAt,
|
||||
result[index] = &OpenFlareEdgeHealth{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
Status: normalizeEdgeHealthStatus(row.Status),
|
||||
Connections: row.Connections,
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
|
||||
@@ -16,8 +16,7 @@ import (
|
||||
type memoryObservabilityStore struct {
|
||||
mu sync.RWMutex
|
||||
metricSnapshots []*OpenFlareMetricSnapshot
|
||||
requestReports []*OpenFlareRequestReport
|
||||
openrestyObs []*OpenFlareNodeObservationOpenresty
|
||||
edgeHealth []*OpenFlareEdgeHealth
|
||||
frpsObs []*OpenFlareNodeObservationFrps
|
||||
frpcObs []*OpenFlareNodeObservationFrpc
|
||||
}
|
||||
@@ -69,93 +68,46 @@ func (s *memoryObservabilityStore) DeleteMetricSnapshotsBefore(_ context.Context
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) InsertRequestReport(_ context.Context, record *OpenFlareRequestReport) error {
|
||||
func (s *memoryObservabilityStore) InsertEdgeHealth(_ context.Context, record *OpenFlareEdgeHealth) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
copyRecord := cloneOpenFlareRequestReport(record)
|
||||
if memoryRequestReportExists(s.requestReports, copyRecord.NodeID, copyRecord.WindowStartedAt, copyRecord.WindowEndedAt) {
|
||||
return nil
|
||||
}
|
||||
s.requestReports = append(s.requestReports, copyRecord)
|
||||
s.edgeHealth = append(s.edgeHealth, cloneOpenFlareEdgeHealth(record))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) ListRequestReports(_ context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareRequestReport, error) {
|
||||
func (s *memoryObservabilityStore) ListEdgeHealth(_ context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareEdgeHealth, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := memoryFilterRequestReports(s.requestReports, nodeID, since)
|
||||
sortOpenFlareRequestReports(rows)
|
||||
rows := memoryFilterEdgeHealth(s.edgeHealth, nodeID, since)
|
||||
sortOpenFlareEdgeHealth(rows)
|
||||
return memoryLimitObservabilityRows(rows, limit), nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteAllRequestReports(_ context.Context) (int64, error) {
|
||||
func (s *memoryObservabilityStore) DeleteAllEdgeHealth(_ context.Context) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := int64(len(s.requestReports))
|
||||
s.requestReports = nil
|
||||
count := int64(len(s.edgeHealth))
|
||||
s.edgeHealth = nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteRequestReportsBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
func (s *memoryObservabilityStore) DeleteEdgeHealthBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cutoff = cutoff.UTC()
|
||||
remaining := make([]*OpenFlareRequestReport, 0, len(s.requestReports))
|
||||
remaining := make([]*OpenFlareEdgeHealth, 0, len(s.edgeHealth))
|
||||
var deleted int64
|
||||
for _, row := range s.requestReports {
|
||||
if row.WindowEndedAt.Before(cutoff) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.requestReports = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) InsertNodeObservationOpenresty(_ context.Context, record *OpenFlareNodeObservationOpenresty) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.openrestyObs = append(s.openrestyObs, cloneOpenFlareNodeObservationOpenresty(record))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) ListNodeObservationOpenresty(_ context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareNodeObservationOpenresty, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := memoryFilterOpenrestyObservations(s.openrestyObs, nodeID, since)
|
||||
sortOpenFlareNodeObservationOpenresty(rows)
|
||||
return memoryLimitObservabilityRows(rows, limit), nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteAllNodeObservationOpenresty(_ context.Context) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := int64(len(s.openrestyObs))
|
||||
s.openrestyObs = nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteNodeObservationOpenrestyBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cutoff = cutoff.UTC()
|
||||
remaining := make([]*OpenFlareNodeObservationOpenresty, 0, len(s.openrestyObs))
|
||||
var deleted int64
|
||||
for _, row := range s.openrestyObs {
|
||||
for _, row := range s.edgeHealth {
|
||||
if row.CapturedAt.Before(cutoff) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.openrestyObs = remaining
|
||||
s.edgeHealth = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
@@ -259,22 +211,8 @@ func memoryFilterMetricSnapshots(rows []*OpenFlareMetricSnapshot, nodeID string,
|
||||
return result
|
||||
}
|
||||
|
||||
func memoryFilterRequestReports(rows []*OpenFlareRequestReport, nodeID string, since time.Time) []*OpenFlareRequestReport {
|
||||
result := make([]*OpenFlareRequestReport, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
|
||||
continue
|
||||
}
|
||||
if !since.IsZero() && row.WindowEndedAt.Before(since) {
|
||||
continue
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func memoryFilterOpenrestyObservations(rows []*OpenFlareNodeObservationOpenresty, nodeID string, since time.Time) []*OpenFlareNodeObservationOpenresty {
|
||||
result := make([]*OpenFlareNodeObservationOpenresty, 0, len(rows))
|
||||
func memoryFilterEdgeHealth(rows []*OpenFlareEdgeHealth, nodeID string, since time.Time) []*OpenFlareEdgeHealth {
|
||||
result := make([]*OpenFlareEdgeHealth, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
|
||||
continue
|
||||
@@ -333,19 +271,6 @@ func memoryMetricSnapshotExists(rows []*OpenFlareMetricSnapshot, nodeID string,
|
||||
return false
|
||||
}
|
||||
|
||||
func memoryRequestReportExists(rows []*OpenFlareRequestReport, nodeID string, windowStartedAt, windowEndedAt time.Time) bool {
|
||||
windowStartedAt = windowStartedAt.UTC()
|
||||
windowEndedAt = windowEndedAt.UTC()
|
||||
for _, row := range rows {
|
||||
if row.NodeID == nodeID &&
|
||||
row.WindowStartedAt.UTC().Equal(windowStartedAt) &&
|
||||
row.WindowEndedAt.UTC().Equal(windowEndedAt) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sortOpenFlareMetricSnapshots(items []*OpenFlareMetricSnapshot) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
@@ -360,21 +285,7 @@ func sortOpenFlareMetricSnapshots(items []*OpenFlareMetricSnapshot) {
|
||||
})
|
||||
}
|
||||
|
||||
func sortOpenFlareRequestReports(items []*OpenFlareRequestReport) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
if compare := openFlareAccessLogCompareInt64(left.WindowEndedAt.Unix(), right.WindowEndedAt.Unix()); compare != 0 {
|
||||
return compare > 0
|
||||
}
|
||||
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
|
||||
})
|
||||
}
|
||||
|
||||
func sortOpenFlareNodeObservationOpenresty(items []*OpenFlareNodeObservationOpenresty) {
|
||||
func sortOpenFlareEdgeHealth(items []*OpenFlareEdgeHealth) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
@@ -441,22 +352,7 @@ func cloneOpenFlareMetricSnapshot(record *OpenFlareMetricSnapshot) *OpenFlareMet
|
||||
return ©Record
|
||||
}
|
||||
|
||||
func cloneOpenFlareRequestReport(record *OpenFlareRequestReport) *OpenFlareRequestReport {
|
||||
copyRecord := *record
|
||||
if copyRecord.ID == 0 {
|
||||
copyRecord.ID = uint(idgen.NextUint64ID())
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if copyRecord.CreatedAt.IsZero() {
|
||||
copyRecord.CreatedAt = now
|
||||
}
|
||||
copyRecord.WindowStartedAt = copyRecord.WindowStartedAt.UTC()
|
||||
copyRecord.WindowEndedAt = copyRecord.WindowEndedAt.UTC()
|
||||
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
|
||||
return ©Record
|
||||
}
|
||||
|
||||
func cloneOpenFlareNodeObservationOpenresty(record *OpenFlareNodeObservationOpenresty) *OpenFlareNodeObservationOpenresty {
|
||||
func cloneOpenFlareEdgeHealth(record *OpenFlareEdgeHealth) *OpenFlareEdgeHealth {
|
||||
copyRecord := *record
|
||||
if copyRecord.ID == 0 {
|
||||
copyRecord.ID = uint(idgen.NextUint64ID())
|
||||
@@ -468,6 +364,9 @@ func cloneOpenFlareNodeObservationOpenresty(record *OpenFlareNodeObservationOpen
|
||||
if copyRecord.CapturedAt.IsZero() {
|
||||
copyRecord.CapturedAt = now
|
||||
}
|
||||
if strings.TrimSpace(copyRecord.Status) == "" {
|
||||
copyRecord.Status = edgeHealthStatusUnknown
|
||||
}
|
||||
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
|
||||
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
|
||||
return ©Record
|
||||
|
||||
Reference in New Issue
Block a user