tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-03 23:06:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): support traffic bytes tracking in edge access logs and refactor analytics models

Browse Source 
- Add `bytes_sent` to `NodeAccessLog` on both Agent and Master Server.
- Create ClickHouse migration `202607120001_add_bytes_sent_to_node_access_logs.sql`.
- Refactor duplicate stats structs by centralizing them into `analyticsmodel` package with type aliases.
- Simplify access log store delegations and remove redundant mapping loops.
- Regenerate Swagger documentation.
- Update changelog index.md.
This commit is contained in:
ryan
2026-07-12 17:27:57 +08:00
parent f02fc9676a
commit 41cd23a64d
19 changed files with 246 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/repository/analytics/node_access_log.go
+11 -9
View File
@@ -35,7 +35,7 @@ func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]anal
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, created_at
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
@@ -67,6 +67,7 @@ func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, er
&item.Host,
&item.Path,
&item.StatusCode,
&item.BytesSent,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node access log row: %w", err)
@@ -78,11 +79,11 @@ func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, er
return result, nil
}
// CountNodeAccessLogs returns total records and distinct IPs matching filter.
func CountNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, int64, error) {
// CountNodeAccessLogs returns total records, distinct IPs, and total bytes sent matching filter.
func CountNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, int64, int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, 0, err
return 0, 0, 0, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
@@ -90,14 +91,15 @@ func CountNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64
countSQL := fmt.Sprintf(`
SELECT
count() AS total_records,
uniqExactIf(remote_addr, remote_addr != '') AS total_ips
uniqExactIf(remote_addr, remote_addr != '') AS total_ips,
sum(bytes_sent) AS total_bytes
FROM %s
WHERE %s`, tableName, clause)
var totalRecords, totalIPs uint64
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords, &totalIPs); err != nil {
return 0, 0, fmt.Errorf("count node access logs: %w", err)
var totalRecords, totalIPs, totalBytes uint64
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords, &totalIPs, &totalBytes); err != nil {
return 0, 0, 0, fmt.Errorf("count node access logs: %w", err)
}
return safeInt64Count(totalRecords), safeInt64Count(totalIPs), nil
return safeInt64Count(totalRecords), safeInt64Count(totalIPs), safeInt64Count(totalBytes), nil
}
// RegionCountsNodeAccessLogs returns region counts for a node since a time.
internal/repository/analytics/node_access_log_stats.go
+18 -49
View File
@@ -8,60 +8,27 @@ import (
"fmt"
"strings"
"time"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// NodeAccessLogBucketAggregate is a folded bucket aggregate row.
type NodeAccessLogBucketAggregate struct {
BucketEpoch int64
RequestCount int64
SuccessCount int64
ClientErrorCount int64
ServerErrorCount int64
UniqueIPCount int64
UniqueHostCount int64
}
type NodeAccessLogBucketAggregate = analyticsmodel.NodeAccessLogBucketAggregate
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
type NodeAccessLogWAFIPAggregate struct {
RemoteAddr string
RequestCount int64
Status404Count int64
ClientErrorCount int64
ServerErrorCount int64
IPHostCount int64
LastSeenEpoch int64
StatusCounts map[int]int64
}
type NodeAccessLogWAFIPAggregate = analyticsmodel.NodeAccessLogWAFIPAggregate
// NodeAccessLogBucketDimension is a bucket dimension value.
type NodeAccessLogBucketDimension struct {
BucketEpoch int64
Value string
}
type NodeAccessLogBucketDimension = analyticsmodel.NodeAccessLogBucketDimension
// NodeAccessLogIPAggregate is an IP aggregate row.
type NodeAccessLogIPAggregate struct {
RemoteAddr string
RequestCount int64
SuccessCount int64
ClientErrorCount int64
ServerErrorCount int64
LastSeenEpoch int64
}
type NodeAccessLogIPAggregate = analyticsmodel.NodeAccessLogIPAggregate
// NodeAccessLogIPSummary is an IP summary row.
type NodeAccessLogIPSummary struct {
RemoteAddr string
TotalRequests int64
RecentRequests int64
LastSeenEpoch int64
}
type NodeAccessLogIPSummary = analyticsmodel.NodeAccessLogIPSummary
// NodeAccessLogIPTrend is an IP trend bucket row.
type NodeAccessLogIPTrend struct {
BucketEpoch int64
RequestCount int64
}
type NodeAccessLogIPTrend = analyticsmodel.NodeAccessLogIPTrend
// BucketAggregatesNodeAccessLogs returns folded bucket aggregates with unique IP/host counts.
func BucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogBucketAggregate, error) {
@@ -80,7 +47,8 @@ SELECT
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
countIf(status_code >= 500) AS server_error_count,
uniqExactIf(remote_addr, remote_addr != '') AS unique_ip_count,
uniqExactIf(host, host != '') AS unique_host_count
uniqExactIf(host, host != '') AS unique_host_count,
sum(bytes_sent) AS bytes_sent
FROM %s
WHERE %s
GROUP BY bucket_epoch
@@ -101,10 +69,10 @@ ORDER BY %s`, bucketExpr, tableName, clause, nodeAccessLogBucketOrderClause(filt
var result []NodeAccessLogBucketAggregate
for rows.Next() {
var (
bucketEpoch int64
requestCount, successCount, clientErrorCount, serverErrorCount, uniqueIPCount, uniqueHostCount uint64
bucketEpoch int64
requestCount, successCount, clientErrorCount, serverErrorCount, uniqueIPCount, uniqueHostCount, bytesSent uint64
)
if err := rows.Scan(&bucketEpoch, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &uniqueIPCount, &uniqueHostCount); err != nil {
if err := rows.Scan(&bucketEpoch, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &uniqueIPCount, &uniqueHostCount, &bytesSent); err != nil {
return nil, fmt.Errorf("scan bucket aggregate row: %w", err)
}
result = append(result, NodeAccessLogBucketAggregate{
@@ -115,6 +83,7 @@ ORDER BY %s`, bucketExpr, tableName, clause, nodeAccessLogBucketOrderClause(filt
ServerErrorCount: safeInt64Count(serverErrorCount),
UniqueIPCount: safeInt64Count(uniqueIPCount),
UniqueHostCount: safeInt64Count(uniqueHostCount),
BytesSent: safeInt64Count(bytesSent),
})
}
return result, nil
@@ -215,8 +184,8 @@ GROUP BY remote_addr`, lastSeenExpr, tableName, queryClause)
var result []NodeAccessLogIPAggregate
for rows.Next() {
var (
remoteAddr string
lastSeenEpoch int64
remoteAddr string
lastSeenEpoch int64
requestCount, successCount, clientErrorCount, serverErrorCount uint64
)
if err := rows.Scan(&remoteAddr, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &lastSeenEpoch); err != nil {
@@ -347,8 +316,8 @@ GROUP BY remote_addr`, hostIsIPExpr, lastSeenExpr, tableName, clause)
order := make([]string, 0)
for rows.Next() {
var (
remoteAddr string
lastSeenEpoch int64
remoteAddr string
lastSeenEpoch int64
requestCount, status404Count, clientErrorCount, serverErrorCount, ipHostCount uint64
)
if err := rows.Scan(&remoteAddr, &requestCount, &status404Count, &clientErrorCount, &serverErrorCount, &ipHostCount, &lastSeenEpoch); err != nil {