mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-03 07:06:36 +08:00
feat(obs): 访问日志 SSOT 与 edge_health,去掉协议兼容层
Agent 仅上报 host_metrics/edge_health/access_logs;业务流量与 UV 由 Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly, 删除 request_reports/openresty 吞吐路径;API 不再暴露 traffic_reports 与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
@@ -6,6 +6,7 @@ package analytics
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
@@ -35,7 +36,7 @@ func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]anal
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, created_at
|
||||
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, request_length, request_time_ms, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
|
||||
@@ -68,6 +69,8 @@ func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, er
|
||||
&item.Path,
|
||||
&item.StatusCode,
|
||||
&item.BytesSent,
|
||||
&item.RequestLength,
|
||||
&item.RequestTimeMs,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node access log row: %w", err)
|
||||
@@ -143,3 +146,157 @@ ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
|
||||
type NodeAccessLogTrafficSummary struct {
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
BytesSent int64
|
||||
RequestLength int64
|
||||
NodeCount int64
|
||||
}
|
||||
|
||||
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
|
||||
type NodeAccessLogValueCount struct {
|
||||
Value string
|
||||
Count int64
|
||||
}
|
||||
|
||||
// NodeAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type NodeAccessLogNodeAggregate struct {
|
||||
NodeID string
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
}
|
||||
|
||||
// TrafficSummaryNodeAccessLogs returns request/error/UV/bytes/node counts for the filter.
|
||||
func TrafficSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (NodeAccessLogTrafficSummary, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return NodeAccessLogTrafficSummary{}, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
count() AS request_count,
|
||||
countIf(status_code >= 500) AS error_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length,
|
||||
uniqExactIf(node_id, node_id != '') AS node_count
|
||||
FROM %s
|
||||
WHERE %s`, tableName, clause)
|
||||
var requestCount, errorCount, uniqueIPs, bytesSent, requestLength, nodeCount uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(
|
||||
&requestCount, &errorCount, &uniqueIPs, &bytesSent, &requestLength, &nodeCount,
|
||||
); err != nil {
|
||||
return NodeAccessLogTrafficSummary{}, fmt.Errorf("traffic summary node access logs: %w", err)
|
||||
}
|
||||
return NodeAccessLogTrafficSummary{
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
ErrorCount: safeInt64Count(errorCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPs),
|
||||
BytesSent: safeInt64Count(bytesSent),
|
||||
RequestLength: safeInt64Count(requestLength),
|
||||
NodeCount: safeInt64Count(nodeCount),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
|
||||
// Allowed columns: status_code, host.
|
||||
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
col := strings.TrimSpace(strings.ToLower(column))
|
||||
switch col {
|
||||
case nodeAccessLogColumnStatusCode, nodeAccessLogColumnHost:
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported value count column: %s", column)
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
// status_code is numeric; cast to string for a uniform Value field.
|
||||
var valueExpr string
|
||||
if col == nodeAccessLogColumnStatusCode {
|
||||
valueExpr = "toString(" + nodeAccessLogColumnStatusCode + ")"
|
||||
} else {
|
||||
valueExpr = "trim(" + nodeAccessLogColumnHost + ")"
|
||||
}
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT %s AS value, count() AS count
|
||||
FROM %s
|
||||
WHERE %s AND %s != ''
|
||||
GROUP BY value
|
||||
ORDER BY count DESC, value ASC`, valueExpr, tableName, clause, valueExpr)
|
||||
if limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("value counts node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []NodeAccessLogValueCount
|
||||
for rows.Next() {
|
||||
var (
|
||||
value string
|
||||
count uint64
|
||||
)
|
||||
if err := rows.Scan(&value, &count); err != nil {
|
||||
return nil, fmt.Errorf("scan value count row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogValueCount{
|
||||
Value: value,
|
||||
Count: safeInt64Count(count),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NodeAggregatesNodeAccessLogs returns per-node request/error/UV aggregates.
|
||||
func NodeAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogNodeAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
node_id,
|
||||
count() AS request_count,
|
||||
countIf(status_code >= 500) AS error_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips
|
||||
FROM %s
|
||||
WHERE %s AND node_id != ''
|
||||
GROUP BY node_id
|
||||
ORDER BY request_count DESC, node_id ASC`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("node aggregates node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []NodeAccessLogNodeAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
nodeID string
|
||||
requestCount, errorCount, uniqueIPs uint64
|
||||
)
|
||||
if err := rows.Scan(&nodeID, &requestCount, &errorCount, &uniqueIPs); err != nil {
|
||||
return nil, fmt.Errorf("scan node aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogNodeAggregate{
|
||||
NodeID: nodeID,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
ErrorCount: safeInt64Count(errorCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPs),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user