tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-02 23:06:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): support traffic bytes tracking in edge access logs and refactor analytics models

Browse Source 
- Add `bytes_sent` to `NodeAccessLog` on both Agent and Master Server.
- Create ClickHouse migration `202607120001_add_bytes_sent_to_node_access_logs.sql`.
- Refactor duplicate stats structs by centralizing them into `analyticsmodel` package with type aliases.
- Simplify access log store delegations and remove redundant mapping loops.
- Regenerate Swagger documentation.
- Update changelog index.md.
This commit is contained in:
ryan
2026-07-12 17:27:57 +08:00
parent f02fc9676a
commit 41cd23a64d
19 changed files with 246 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apps/agent/observability/traffic.go
+1
View File
@@ -199,6 +199,7 @@ func (aggregate *trafficAggregate) consume(line []byte) {
Host: strings.TrimSpace(record.Host),
Path: normalizeAccessLogPath(record.Path),
StatusCode: record.Status,
BytesSent: record.BytesSent,
})
}
internal/apps/openflare/observability/access_log_logics.go
+2 -2
View File
@@ -197,7 +197,7 @@ func ListAccessLogs(ctx context.Context, input AccessLogQuery) (*AccessLogList,
if err != nil {
return nil, err
}
totalRecords, totalIPs, err := model.CountOpenFlareAccessLogs(ctx, modelQuery)
totalRecords, totalIPs, _, err := model.CountOpenFlareAccessLogs(ctx, modelQuery)
if err != nil {
return nil, err
}
@@ -260,7 +260,7 @@ func ListFoldedAccessLogs(ctx context.Context, input AccessLogQuery) (*FoldedAcc
if err != nil {
return nil, err
}
totalRecords, totalIPs, err := model.CountOpenFlareAccessLogs(ctx, modelQuery)
totalRecords, totalIPs, _, err := model.CountOpenFlareAccessLogs(ctx, modelQuery)
if err != nil {
return nil, err
}
internal/apps/openflare/zone/logics_test.go
+10 -5
View File
@@ -71,11 +71,11 @@ func TestGetStatsAggregatesZoneHosts(t *testing.T) {
now := time.Now().UTC()
require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
{NodeID: "n1", LoggedAt: now.Add(-1 * time.Hour), RemoteAddr: "1.1.1.1", Host: "api.example.com", Path: "/", StatusCode: 200},
{NodeID: "n1", LoggedAt: now.Add(-2 * time.Hour), RemoteAddr: "1.1.1.1", Host: "www.example.com", Path: "/", StatusCode: 200},
{NodeID: "n1", LoggedAt: now.Add(-3 * time.Hour), RemoteAddr: "2.2.2.2", Host: "api.example.com", Path: "/x", StatusCode: 404},
{NodeID: "n1", LoggedAt: now.Add(-3 * time.Hour), RemoteAddr: "3.3.3.3", Host: "other.com", Path: "/", StatusCode: 200},
{NodeID: "n1", LoggedAt: now.Add(-48 * time.Hour), RemoteAddr: "4.4.4.4", Host: "api.example.com", Path: "/", StatusCode: 200},
{NodeID: "n1", LoggedAt: now.Add(-1 * time.Hour), RemoteAddr: "1.1.1.1", Host: "api.example.com", Path: "/", StatusCode: 200, BytesSent: 1000},
{NodeID: "n1", LoggedAt: now.Add(-2 * time.Hour), RemoteAddr: "1.1.1.1", Host: "www.example.com", Path: "/", StatusCode: 200, BytesSent: 500},
{NodeID: "n1", LoggedAt: now.Add(-3 * time.Hour), RemoteAddr: "2.2.2.2", Host: "api.example.com", Path: "/x", StatusCode: 404, BytesSent: 200},
{NodeID: "n1", LoggedAt: now.Add(-3 * time.Hour), RemoteAddr: "3.3.3.3", Host: "other.com", Path: "/", StatusCode: 200, BytesSent: 100},
{NodeID: "n1", LoggedAt: now.Add(-48 * time.Hour), RemoteAddr: "4.4.4.4", Host: "api.example.com", Path: "/", StatusCode: 200, BytesSent: 800},
}))
stats, err := GetStats(ctx, zone.ID, "24h")
@@ -83,20 +83,25 @@ func TestGetStatsAggregatesZoneHosts(t *testing.T) {
require.Equal(t, StatsRange24h, stats.Range)
require.Equal(t, int64(3), stats.RequestCount)
require.Equal(t, int64(2), stats.UniqueVisitors)
require.Equal(t, int64(1700), stats.BytesSent)
require.Equal(t, 2, stats.DomainCount)
require.True(t, stats.Available)
require.NotEmpty(t, stats.Series)
require.Equal(t, 60, stats.BucketMinutes)
var seriesRequests int64
var seriesBytes int64
for _, point := range stats.Series {
seriesRequests += point.RequestCount
seriesBytes += point.BytesSent
}
require.Equal(t, int64(3), seriesRequests)
require.Equal(t, int64(1700), seriesBytes)
stats7d, err := GetStats(ctx, zone.ID, "7d")
require.NoError(t, err)
require.Equal(t, int64(4), stats7d.RequestCount)
require.Equal(t, int64(3), stats7d.UniqueVisitors)
require.Equal(t, int64(2500), stats7d.BytesSent)
require.NotEmpty(t, stats7d.Series)
_, err = GetStats(ctx, zone.ID, "1h")
internal/apps/openflare/zone/stats.go
+11 -12
View File
@@ -22,19 +22,19 @@ const (
// StatsRange24h represents a 24-hour time window.
StatsRange24h StatsRange = "24h"
// StatsRange7d represents a 7-day time window.
StatsRange7d StatsRange = "7d"
StatsRange7d StatsRange = "7d"
// StatsRange30d represents a 30-day time window.
StatsRange30d StatsRange = "30d"
)
const (
hoursPerDay = 24
daysPerWeek = 7
daysPerMonth = 30
minutesPerHour = 60
bucketMinutes24h = 60
bucketMinutes7d = 6 * minutesPerHour
bucketMinutes30d = 24 * minutesPerHour
hoursPerDay = 24
daysPerWeek = 7
daysPerMonth = 30
minutesPerHour = 60
bucketMinutes24h = 60
bucketMinutes7d = 6 * minutesPerHour
bucketMinutes30d = 24 * minutesPerHour
)
// StatsPoint is one bucket on a Zone traffic chart.
@@ -120,7 +120,7 @@ func GetStats(ctx context.Context, id uint, rangeRaw string) (*Stats, error) {
return result, nil
}
requestCount, uniqueVisitors, err := model.CountOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{
requestCount, uniqueVisitors, totalBytesSent, err := model.CountOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{
Hosts: hosts,
Since: since,
Until: now,
@@ -134,8 +134,7 @@ func GetStats(ctx context.Context, id uint, rangeRaw string) (*Stats, error) {
}
result.RequestCount = requestCount
result.UniqueVisitors = uniqueVisitors
// Bytes are not yet persisted on edge access logs; keep the field for UI compatibility.
result.BytesSent = 0
result.BytesSent = totalBytesSent
buckets, err := model.ListOpenFlareAccessLogBuckets(ctx, model.OpenFlareAccessLogBucketQuery{
Hosts: hosts,
@@ -166,7 +165,7 @@ func GetStats(ctx context.Context, id uint, rangeRaw string) (*Stats, error) {
if row, ok := byEpoch[epoch]; ok {
series[index].RequestCount = row.RequestCount
series[index].UniqueVisitors = row.UniqueIPCount
series[index].BytesSent = 0
series[index].BytesSent = row.BytesSent
}
}
result.Series = series