tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-02 14:56:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): support traffic bytes tracking in edge access logs and refactor analytics models

Browse Source 
- Add `bytes_sent` to `NodeAccessLog` on both Agent and Master Server.
- Create ClickHouse migration `202607120001_add_bytes_sent_to_node_access_logs.sql`.
- Refactor duplicate stats structs by centralizing them into `analyticsmodel` package with type aliases.
- Simplify access log store delegations and remove redundant mapping loops.
- Regenerate Swagger documentation.
- Update changelog index.md.
This commit is contained in:
ryan
2026-07-12 17:27:57 +08:00
parent f02fc9676a
commit 41cd23a64d
19 changed files with 246 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/model/analytics/node_access_log.go
+3 -2
View File
@@ -10,7 +10,7 @@ import (
const (
nodeAccessLogTableName = "of_node_access_logs"
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, status_code, created_at"
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, created_at"
)
// NodeAccessLog stores OpenFlare edge node access records in ClickHouse.
@@ -23,6 +23,7 @@ type NodeAccessLog struct {
Host string `gorm:"column:host"`
Path string `gorm:"column:path"`
StatusCode int32 `gorm:"column:status_code"`
BytesSent uint64 `gorm:"column:bytes_sent"`
CreatedAt time.Time `gorm:"column:created_at"`
}
@@ -39,4 +40,4 @@ func (NodeAccessLog) InsertColumns() string {
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
func (NodeAccessLog) BatchInsertSQL() string {
return fmt.Sprintf("INSERT INTO %s (%s)", nodeAccessLogTableName, nodeAccessLogInsertColumns)
}
}
internal/model/analytics/node_access_log_stats.go
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
// NodeAccessLogBucketAggregate is a folded bucket aggregate row.
type NodeAccessLogBucketAggregate struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
RequestCount int64 `gorm:"column:request_count"`
SuccessCount int64 `gorm:"column:success_count"`
ClientErrorCount int64 `gorm:"column:client_error_count"`
ServerErrorCount int64 `gorm:"column:server_error_count"`
UniqueIPCount int64 `gorm:"column:unique_ip_count"`
UniqueHostCount int64 `gorm:"column:unique_host_count"`
BytesSent int64 `gorm:"column:bytes_sent"`
}
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
type NodeAccessLogWAFIPAggregate struct {
RemoteAddr string
RequestCount int64
Status404Count int64
ClientErrorCount int64
ServerErrorCount int64
IPHostCount int64
LastSeenEpoch int64
StatusCounts map[int]int64
}
// NodeAccessLogBucketDimension is a bucket dimension value.
type NodeAccessLogBucketDimension struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
Value string `gorm:"column:value"`
}
// NodeAccessLogIPAggregate is an IP aggregate row.
type NodeAccessLogIPAggregate struct {
RemoteAddr string `gorm:"column:remote_addr"`
RequestCount int64 `gorm:"column:request_count"`
SuccessCount int64 `gorm:"column:success_count"`
ClientErrorCount int64 `gorm:"column:client_error_count"`
ServerErrorCount int64 `gorm:"column:server_error_count"`
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
}
// NodeAccessLogIPSummary is an IP summary row.
type NodeAccessLogIPSummary struct {
RemoteAddr string `gorm:"column:remote_addr"`
TotalRequests int64 `gorm:"column:total_requests"`
RecentRequests int64 `gorm:"column:recent_requests"`
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
}
// NodeAccessLogIPTrend is an IP trend bucket row.
type NodeAccessLogIPTrend struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
RequestCount int64 `gorm:"column:request_count"`
}
internal/model/openflare_access_log.go
+11 -48
View File
@@ -9,6 +9,8 @@ import (
"sort"
"strings"
"time"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
const (
@@ -18,52 +20,12 @@ const (
secondsPerMinute = 60
)
type openFlareAccessLogBucketAggregateRow struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
RequestCount int64 `gorm:"column:request_count"`
SuccessCount int64 `gorm:"column:success_count"`
ClientErrorCount int64 `gorm:"column:client_error_count"`
ServerErrorCount int64 `gorm:"column:server_error_count"`
UniqueIPCount int64 `gorm:"column:unique_ip_count"`
UniqueHostCount int64 `gorm:"column:unique_host_count"`
}
type openFlareAccessLogBucketDimensionRow struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
Value string `gorm:"column:value"`
}
type openFlareAccessLogIPAggregateRow struct {
RemoteAddr string `gorm:"column:remote_addr"`
RequestCount int64 `gorm:"column:request_count"`
SuccessCount int64 `gorm:"column:success_count"`
ClientErrorCount int64 `gorm:"column:client_error_count"`
ServerErrorCount int64 `gorm:"column:server_error_count"`
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
}
type openFlareAccessLogIPSummaryRow struct {
RemoteAddr string `gorm:"column:remote_addr"`
TotalRequests int64 `gorm:"column:total_requests"`
RecentRequests int64 `gorm:"column:recent_requests"`
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
}
type openFlareAccessLogIPTrendRow struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
RequestCount int64 `gorm:"column:request_count"`
}
type openFlareAccessLogWAFIPAggregateRow struct {
RemoteAddr string
RequestCount int64
Status404Count int64
ClientErrorCount int64
ServerErrorCount int64
IPHostCount int64
LastSeenEpoch int64
StatusCounts map[int]int64
}
type openFlareAccessLogBucketAggregateRow = analyticsmodel.NodeAccessLogBucketAggregate
type openFlareAccessLogBucketDimensionRow = analyticsmodel.NodeAccessLogBucketDimension
type openFlareAccessLogIPAggregateRow = analyticsmodel.NodeAccessLogIPAggregate
type openFlareAccessLogIPSummaryRow = analyticsmodel.NodeAccessLogIPSummary
type openFlareAccessLogIPTrendRow = analyticsmodel.NodeAccessLogIPTrend
type openFlareAccessLogWAFIPAggregateRow = analyticsmodel.NodeAccessLogWAFIPAggregate
// ListOpenFlareAccessLogWAFIPAggregates returns per-IP aggregates for WAF automatic rules.
func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLogWAFIPAggregate, error) {
@@ -105,8 +67,8 @@ func ListOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery)
return currentAccessLogStore().List(ctx, query)
}
// CountOpenFlareAccessLogs counts access logs and distinct IPs matching the query.
func CountOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, error) {
// CountOpenFlareAccessLogs counts access logs, distinct IPs, and total bytes sent matching the query.
func CountOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, int64, error) {
return currentAccessLogStore().Count(ctx, query)
}
@@ -229,6 +191,7 @@ func buildOpenFlareAccessLogBucketRows(ctx context.Context, query OpenFlareAcces
SuccessCount: partial.SuccessCount,
ClientErrorCount: partial.ClientErrorCount,
ServerErrorCount: partial.ServerErrorCount,
BytesSent: partial.BytesSent,
})
}
return rows, nil
internal/model/openflare_access_log_store.go
+21 -91
View File
@@ -5,6 +5,7 @@ package model
import (
"context"
"math"
"sync"
"time"
@@ -39,7 +40,7 @@ func currentAccessLogInsertHooks() AccessLogInsertHooks {
type accessLogStore interface {
InsertBatch(ctx context.Context, records []*OpenFlareAccessLog) error
List(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLog, error)
Count(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, error)
Count(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, int64, error)
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareAccessLogRegionCount, error)
BucketAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error)
CountBuckets(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
@@ -112,7 +113,7 @@ func (clickhouseAccessLogStore) List(ctx context.Context, query OpenFlareAccessL
return fromAnalyticsNodeAccessLogs(rows), nil
}
func (clickhouseAccessLogStore) Count(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, error) {
func (clickhouseAccessLogStore) Count(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, int64, error) {
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
@@ -132,23 +133,7 @@ func (clickhouseAccessLogStore) RegionCounts(ctx context.Context, nodeID string,
}
func (clickhouseAccessLogStore) BucketAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error) {
rows, err := analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
if err != nil {
return nil, err
}
result := make([]openFlareAccessLogBucketAggregateRow, len(rows))
for index, row := range rows {
result[index] = openFlareAccessLogBucketAggregateRow{
BucketEpoch: row.BucketEpoch,
RequestCount: row.RequestCount,
SuccessCount: row.SuccessCount,
ClientErrorCount: row.ClientErrorCount,
ServerErrorCount: row.ServerErrorCount,
UniqueIPCount: row.UniqueIPCount,
UniqueHostCount: row.UniqueHostCount,
}
}
return result, nil
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
}
func (clickhouseAccessLogStore) CountBuckets(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
@@ -156,54 +141,15 @@ func (clickhouseAccessLogStore) CountBuckets(ctx context.Context, filter OpenFla
}
func (clickhouseAccessLogStore) BucketDimensions(ctx context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
rows, err := analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, bucketSeconds)
if err != nil {
return nil, err
}
result := make([]openFlareAccessLogBucketDimensionRow, len(rows))
for index, row := range rows {
result[index] = openFlareAccessLogBucketDimensionRow{
BucketEpoch: row.BucketEpoch,
Value: row.Value,
}
}
return result, nil
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, bucketSeconds)
}
func (clickhouseAccessLogStore) IPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error) {
rows, err := analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), exactRemoteAddr)
if err != nil {
return nil, err
}
result := make([]openFlareAccessLogIPAggregateRow, len(rows))
for index, row := range rows {
result[index] = openFlareAccessLogIPAggregateRow{
RemoteAddr: row.RemoteAddr,
RequestCount: row.RequestCount,
SuccessCount: row.SuccessCount,
ClientErrorCount: row.ClientErrorCount,
ServerErrorCount: row.ServerErrorCount,
LastSeenEpoch: row.LastSeenEpoch,
}
}
return result, nil
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), exactRemoteAddr)
}
func (clickhouseAccessLogStore) IPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error) {
rows, err := analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), recentSince)
if err != nil {
return nil, err
}
result := make([]openFlareAccessLogIPSummaryRow, len(rows))
for index, row := range rows {
result[index] = openFlareAccessLogIPSummaryRow{
RemoteAddr: row.RemoteAddr,
TotalRequests: row.TotalRequests,
RecentRequests: row.RecentRequests,
LastSeenEpoch: row.LastSeenEpoch,
}
}
return result, nil
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), recentSince)
}
func (clickhouseAccessLogStore) CountIPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery) (int64, error) {
@@ -211,39 +157,11 @@ func (clickhouseAccessLogStore) CountIPSummaries(ctx context.Context, filter Ope
}
func (clickhouseAccessLogStore) WAFIPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
rows, err := analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
if err != nil {
return nil, err
}
result := make([]openFlareAccessLogWAFIPAggregateRow, len(rows))
for index, row := range rows {
result[index] = openFlareAccessLogWAFIPAggregateRow{
RemoteAddr: row.RemoteAddr,
RequestCount: row.RequestCount,
Status404Count: row.Status404Count,
ClientErrorCount: row.ClientErrorCount,
ServerErrorCount: row.ServerErrorCount,
IPHostCount: row.IPHostCount,
LastSeenEpoch: row.LastSeenEpoch,
StatusCounts: row.StatusCounts,
}
}
return result, nil
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
}
func (clickhouseAccessLogStore) IPTrend(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
rows, err := analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
if err != nil {
return nil, err
}
result := make([]openFlareAccessLogIPTrendRow, len(rows))
for index, row := range rows {
result[index] = openFlareAccessLogIPTrendRow{
BucketEpoch: row.BucketEpoch,
RequestCount: row.RequestCount,
}
}
return result, nil
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
}
func (clickhouseAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
@@ -275,6 +193,10 @@ func toNodeAccessLogFilter(query OpenFlareAccessLogQuery) analyticsrepo.NodeAcce
}
func toAnalyticsNodeAccessLog(record *OpenFlareAccessLog) analyticsmodel.NodeAccessLog {
var bytesSent uint64
if record.BytesSent > 0 {
bytesSent = uint64(record.BytesSent)
}
return analyticsmodel.NodeAccessLog{
ID: record.ID,
NodeID: record.NodeID,
@@ -284,6 +206,7 @@ func toAnalyticsNodeAccessLog(record *OpenFlareAccessLog) analyticsmodel.NodeAcc
Host: record.Host,
Path: record.Path,
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
BytesSent: bytesSent,
CreatedAt: record.CreatedAt,
}
}
@@ -291,6 +214,12 @@ func toAnalyticsNodeAccessLog(record *OpenFlareAccessLog) analyticsmodel.NodeAcc
func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*OpenFlareAccessLog {
result := make([]*OpenFlareAccessLog, len(rows))
for index, row := range rows {
var bytesSent int64
if row.BytesSent <= math.MaxInt64 {
bytesSent = int64(row.BytesSent)
} else {
bytesSent = math.MaxInt64
}
result[index] = &OpenFlareAccessLog{
ID: row.ID,
NodeID: row.NodeID,
@@ -300,6 +229,7 @@ func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*OpenFla
Host: row.Host,
Path: row.Path,
StatusCode: int(row.StatusCode),
BytesSent: bytesSent,
CreatedAt: row.CreatedAt,
}
}
internal/model/openflare_access_log_store_memory.go
+7 -2
View File
@@ -55,19 +55,21 @@ func (s *memoryAccessLogStore) List(_ context.Context, query OpenFlareAccessLogQ
return cloneAccessLogSlice(rows), nil
}
func (s *memoryAccessLogStore) Count(_ context.Context, query OpenFlareAccessLogQuery) (int64, int64, error) {
func (s *memoryAccessLogStore) Count(_ context.Context, query OpenFlareAccessLogQuery) (int64, int64, int64, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(query)
ips := make(map[string]struct{})
var totalBytes int64
for _, row := range rows {
totalBytes += row.BytesSent
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
ips[remoteAddr] = struct{}{}
}
return int64(len(rows)), int64(len(ips)), nil
return int64(len(rows)), int64(len(ips)), totalBytes, nil
}
func (s *memoryAccessLogStore) RegionCounts(_ context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareAccessLogRegionCount, error) {
@@ -120,6 +122,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
aggregates[bucketEpoch] = item
}
item.RequestCount++
item.BytesSent += row.BytesSent
switch {
case row.StatusCode < 400:
item.SuccessCount++
@@ -151,6 +154,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
SuccessCount: result[index].SuccessCount,
ClientErrorCount: result[index].ClientErrorCount,
ServerErrorCount: result[index].ServerErrorCount,
BytesSent: result[index].BytesSent,
}
}
sortOpenFlareAccessLogBucketRows(bucketRows, filter.SortBy, filter.SortOrder)
@@ -163,6 +167,7 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
SuccessCount: bucketRows[index].SuccessCount,
ClientErrorCount: bucketRows[index].ClientErrorCount,
ServerErrorCount: bucketRows[index].ServerErrorCount,
BytesSent: bucketRows[index].BytesSent,
}
}
if filter.PageSize > 0 {
internal/model/openflare_access_log_test.go
+3 -3
View File
@@ -76,7 +76,7 @@ func TestCountOpenFlareAccessLogs(t *testing.T) {
query := OpenFlareAccessLogQuery{
Since: now.Add(-10 * time.Minute),
}
totalRecords, totalIPs, err := CountOpenFlareAccessLogs(ctx, query)
totalRecords, totalIPs, _, err := CountOpenFlareAccessLogs(ctx, query)
require.NoError(t, err)
assert.Equal(t, int64(5), totalRecords)
assert.Equal(t, int64(3), totalIPs)
@@ -113,7 +113,7 @@ func TestDeleteOpenFlareAccessLogsBefore(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, int64(3), deleted)
totalRecords, _, err := CountOpenFlareAccessLogs(ctx, OpenFlareAccessLogQuery{Since: now.Add(-10 * time.Minute)})
totalRecords, _, _, err := CountOpenFlareAccessLogs(ctx, OpenFlareAccessLogQuery{Since: now.Add(-10 * time.Minute)})
require.NoError(t, err)
assert.Equal(t, int64(2), totalRecords)
}
}
internal/model/openflare_observability.go
+2
View File
@@ -69,6 +69,7 @@ type OpenFlareAccessLog struct {
Host string `json:"host" gorm:"index;size:255"`
Path string `json:"path" gorm:"size:2048"`
StatusCode int `json:"status_code" gorm:"index"`
BytesSent int64 `json:"bytes_sent" gorm:"column:bytes_sent;not null;default:0"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
}
@@ -221,6 +222,7 @@ type OpenFlareAccessLogBucketRow struct {
SuccessCount int64 `json:"success_count"`
ClientErrorCount int64 `json:"client_error_count"`
ServerErrorCount int64 `json:"server_error_count"`
BytesSent int64 `json:"bytes_sent"`
}
// OpenFlareAccessLogBucketIPQuery filters folded IP summary queries (v1 stub).