feat(obs): 访问日志 SSOT 与 edge_health,去掉协议兼容层

Agent 仅上报 host_metrics/edge_health/access_logs;业务流量与 UV 由
Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly,
删除 request_reports/openresty 吞吐路径;API 不再暴露 traffic_reports
与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
ryan
2026-07-18 11:53:11 +08:00
parent 9a0974cce8
commit f0e234df1f
60 changed files with 1799 additions and 2164 deletions
@@ -6,6 +6,7 @@ package analytics
import (
"context"
"fmt"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
@@ -35,7 +36,7 @@ func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]anal
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, created_at
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, request_length, request_time_ms, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
@@ -68,6 +69,8 @@ func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, er
&item.Path,
&item.StatusCode,
&item.BytesSent,
&item.RequestLength,
&item.RequestTimeMs,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node access log row: %w", err)
@@ -143,3 +146,157 @@ ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
}
return result, nil
}
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
type NodeAccessLogTrafficSummary struct {
RequestCount int64
ErrorCount int64
UniqueIPCount int64
BytesSent int64
RequestLength int64
NodeCount int64
}
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
type NodeAccessLogValueCount struct {
Value string
Count int64
}
// NodeAccessLogNodeAggregate is per-node traffic over a window.
type NodeAccessLogNodeAggregate struct {
NodeID string
RequestCount int64
ErrorCount int64
UniqueIPCount int64
}
// TrafficSummaryNodeAccessLogs returns request/error/UV/bytes/node counts for the filter.
func TrafficSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (NodeAccessLogTrafficSummary, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return NodeAccessLogTrafficSummary{}, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
count() AS request_count,
countIf(status_code >= 500) AS error_count,
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips,
sum(bytes_sent) AS bytes_sent,
sum(request_length) AS request_length,
uniqExactIf(node_id, node_id != '') AS node_count
FROM %s
WHERE %s`, tableName, clause)
var requestCount, errorCount, uniqueIPs, bytesSent, requestLength, nodeCount uint64
if err := conn.QueryRow(ctx, sql, args...).Scan(
&requestCount, &errorCount, &uniqueIPs, &bytesSent, &requestLength, &nodeCount,
); err != nil {
return NodeAccessLogTrafficSummary{}, fmt.Errorf("traffic summary node access logs: %w", err)
}
return NodeAccessLogTrafficSummary{
RequestCount: safeInt64Count(requestCount),
ErrorCount: safeInt64Count(errorCount),
UniqueIPCount: safeInt64Count(uniqueIPs),
BytesSent: safeInt64Count(bytesSent),
RequestLength: safeInt64Count(requestLength),
NodeCount: safeInt64Count(nodeCount),
}, nil
}
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
// Allowed columns: status_code, host.
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
col := strings.TrimSpace(strings.ToLower(column))
switch col {
case nodeAccessLogColumnStatusCode, nodeAccessLogColumnHost:
default:
return nil, fmt.Errorf("unsupported value count column: %s", column)
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
// status_code is numeric; cast to string for a uniform Value field.
var valueExpr string
if col == nodeAccessLogColumnStatusCode {
valueExpr = "toString(" + nodeAccessLogColumnStatusCode + ")"
} else {
valueExpr = "trim(" + nodeAccessLogColumnHost + ")"
}
sql := fmt.Sprintf(`
SELECT %s AS value, count() AS count
FROM %s
WHERE %s AND %s != ''
GROUP BY value
ORDER BY count DESC, value ASC`, valueExpr, tableName, clause, valueExpr)
if limit > 0 {
sql += clickHouseLimitClause
args = append(args, limit)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("value counts node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogValueCount
for rows.Next() {
var (
value string
count uint64
)
if err := rows.Scan(&value, &count); err != nil {
return nil, fmt.Errorf("scan value count row: %w", err)
}
result = append(result, NodeAccessLogValueCount{
Value: value,
Count: safeInt64Count(count),
})
}
return result, nil
}
// NodeAggregatesNodeAccessLogs returns per-node request/error/UV aggregates.
func NodeAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogNodeAggregate, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
node_id,
count() AS request_count,
countIf(status_code >= 500) AS error_count,
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips
FROM %s
WHERE %s AND node_id != ''
GROUP BY node_id
ORDER BY request_count DESC, node_id ASC`, tableName, clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("node aggregates node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogNodeAggregate
for rows.Next() {
var (
nodeID string
requestCount, errorCount, uniqueIPs uint64
)
if err := rows.Scan(&nodeID, &requestCount, &errorCount, &uniqueIPs); err != nil {
return nil, fmt.Errorf("scan node aggregate row: %w", err)
}
result = append(result, NodeAccessLogNodeAggregate{
NodeID: nodeID,
RequestCount: safeInt64Count(requestCount),
ErrorCount: safeInt64Count(errorCount),
UniqueIPCount: safeInt64Count(uniqueIPs),
})
}
return result, nil
}