tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-05 15:26:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(access-logs): 接入 User-Agent 与设备/浏览器/状态码分布

Browse Source 
- Agent 访问日志上报 user_agent，OpenResty log_format 输出 http_user_agent
- of_node_access_logs 新增 user_agent 列并写入 ClickHouse
- 访问日志概览新增：设备类型饼图、状态码饼图、浏览器/OS/User-Agent 排行
- 日志明细列表增加 User-Agent 列
- 扩展 UA 解析工具（browser/os/device）并支持 CLI 识别
This commit is contained in:
ryan
2026-07-18 21:18:59 +08:00
parent 177578ef4e
commit e49078ac3b
28 changed files with 498 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apps/agent/observability/traffic.go
+4
View File
@@ -22,6 +22,7 @@ type accessLogRecord struct {
Host string `json:"host"`
RemoteAddr string `json:"remote_addr"`
Path string `json:"path"`
UserAgent string `json:"user_agent"`
Status int `json:"status"`
BytesSent int64 `json:"bytes_sent"`
RequestLength int64 `json:"request_length"`
@@ -145,6 +146,7 @@ func (aggregate *trafficAggregate) consume(line []byte) {
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
Host: strings.TrimSpace(record.Host),
Path: normalizeAccessLogPath(record.Path),
UserAgent: strings.TrimSpace(record.UserAgent),
StatusCode: record.Status,
BytesSent: record.BytesSent,
RequestLength: record.RequestLength,
@@ -157,6 +159,7 @@ type parsedAccessLogRecord struct {
Host string
RemoteAddr string
Path string
UserAgent string
Status int
BytesSent int64
RequestLength int64
@@ -189,6 +192,7 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
Host: strings.TrimSpace(record.Host),
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
Path: normalizeAccessLogPath(record.Path),
UserAgent: strings.TrimSpace(record.UserAgent),
Status: record.Status,
BytesSent: record.BytesSent,
RequestLength: record.RequestLength,
internal/apps/agent/observability/traffic_test.go
+5 -2
View File
@@ -14,8 +14,8 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
tempDir := t.TempDir()
logPath := filepath.Join(tempDir, "openflare_access.log")
content := []byte(
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512,\"request_time\":0.015}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256,\"request_time\":0.008}\n",
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512,\"request_time\":0.015,\"user_agent\":\"Mozilla/5.0\"}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256,\"request_time\":0.008,\"user_agent\":\"curl/8.0\"}\n",
)
if err := os.WriteFile(logPath, content, 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
@@ -35,6 +35,9 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
if accessLogs[0].Path != "/login" || accessLogs[1].Path != "/v1/ping" {
t.Fatalf("unexpected access log paths: %+v", accessLogs)
}
if accessLogs[0].UserAgent != "Mozilla/5.0" || accessLogs[1].UserAgent != "curl/8.0" {
t.Fatalf("unexpected user agents: %+v", accessLogs)
}
snapshot, err := stateStore.Load()
if err != nil {
internal/apps/agent/state/observability_buffer.go
+1 -1
View File
@@ -117,7 +117,7 @@ func mergeAccessLogs(existing []protocol.NodeAccessLog, incoming []protocol.Node
}
func accessLogKey(item protocol.NodeAccessLog) string {
return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + strconv.Itoa(item.StatusCode)
return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + item.UserAgent + "|" + strconv.Itoa(item.StatusCode)
}
// Replayable returns buffered records from windows before currentWindowStartedAtUnix.
internal/apps/openflare/agent/observability.go
+2
View File
@@ -24,6 +24,7 @@ const (
healthSeverityWarning = "warning"
healthSeverityCritical = "critical"
accessLogPathMaxLength = 100
accessLogUserAgentMaxLength = 512
healthEventMessageMaxLength = 4096
)
@@ -208,6 +209,7 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
Region: "",
Host: strings.TrimSpace(item.Host),
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
UserAgent: truncateForDatabase(strings.TrimSpace(item.UserAgent), accessLogUserAgentMaxLength),
StatusCode: item.StatusCode,
BytesSent: bytesSent,
RequestLength: requestLength,
internal/apps/openflare/observability/access_log_logics.go
+67 -13
View File
@@ -9,6 +9,7 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
const (
@@ -52,6 +53,7 @@ type AccessLogView struct {
Region string `json:"region"`
Host string `json:"host"`
Path string `json:"path"`
UserAgent string `json:"user_agent"`
StatusCode int `json:"status_code"`
}
@@ -207,13 +209,18 @@ type AccessLogOverviewTrends struct {
// AccessLogOverview is the access-log analytics overview payload.
type AccessLogOverview struct {
GeneratedAt time.Time `json:"generated_at"`
Hours int `json:"hours"`
Summary AccessLogOverviewSummary `json:"summary"`
Trends AccessLogOverviewTrends `json:"trends"`
TopPaths []DistributionItem `json:"top_paths"`
TopHosts []DistributionItem `json:"top_hosts"`
TopIPs []DistributionItem `json:"top_ips"`
GeneratedAt time.Time `json:"generated_at"`
Hours int `json:"hours"`
Summary AccessLogOverviewSummary `json:"summary"`
Trends AccessLogOverviewTrends `json:"trends"`
TopPaths []DistributionItem `json:"top_paths"`
TopHosts []DistributionItem `json:"top_hosts"`
TopIPs []DistributionItem `json:"top_ips"`
DeviceTypes []DistributionItem `json:"device_types"`
TopBrowsers []DistributionItem `json:"top_browsers"`
TopOperatingSystems []DistributionItem `json:"top_operating_systems"`
TopUserAgents []DistributionItem `json:"top_user_agents"`
StatusCodes []DistributionItem `json:"status_codes"`
}
// AccessLogCleanupInput is the cleanup request payload.
@@ -229,9 +236,10 @@ type AccessLogCleanupResult struct {
}
const (
defaultAccessLogOverviewHours = 24
maxAccessLogOverviewHours = 24 * 30
accessLogOverviewTopLimit = 10
defaultAccessLogOverviewHours = 24
maxAccessLogOverviewHours = 24 * 30
accessLogOverviewTopLimit = 10
accessLogOverviewUASampleLimit = 200
)
// GetAccessLogOverview returns summary metrics, trends, and top rankings.
@@ -254,6 +262,7 @@ func GetAccessLogOverview(ctx context.Context, input AccessLogOverviewQuery) (*A
requestPoints, visitPoints, bandwidthPoints := buildAccessLogOverviewTrends(
ctx, now, normalized.Hours, query,
)
deviceTypes, topBrowsers, topOSes, topUserAgents := buildAccessLogUADistributions(ctx, query)
return &AccessLogOverview{
GeneratedAt: now,
@@ -268,9 +277,14 @@ func GetAccessLogOverview(ctx context.Context, input AccessLogOverviewQuery) (*A
Visits: visitPoints,
Bandwidth: bandwidthPoints,
},
TopPaths: valueCountDistribution(ctx, query, "path", accessLogOverviewTopLimit),
TopHosts: valueCountDistribution(ctx, query, "host", accessLogOverviewTopLimit),
TopIPs: valueCountDistribution(ctx, query, "remote_addr", accessLogOverviewTopLimit),
TopPaths: valueCountDistribution(ctx, query, "path", accessLogOverviewTopLimit),
TopHosts: valueCountDistribution(ctx, query, "host", accessLogOverviewTopLimit),
TopIPs: valueCountDistribution(ctx, query, "remote_addr", accessLogOverviewTopLimit),
DeviceTypes: deviceTypes,
TopBrowsers: topBrowsers,
TopOperatingSystems: topOSes,
TopUserAgents: topUserAgents,
StatusCodes: valueCountDistribution(ctx, query, "status_code", accessLogOverviewTopLimit),
}, nil
}
@@ -309,6 +323,45 @@ func valueCountDistribution(
return items
}
func buildAccessLogUADistributions(
ctx context.Context,
query model.OpenFlareAccessLogQuery,
) (
deviceTypes []DistributionItem,
topBrowsers []DistributionItem,
topOSes []DistributionItem,
topUserAgents []DistributionItem,
) {
uaRows := valueCountDistribution(ctx, query, "user_agent", accessLogOverviewUASampleLimit)
if len(uaRows) == 0 {
return []DistributionItem{}, []DistributionItem{}, []DistributionItem{}, []DistributionItem{}
}
deviceAcc := make(distributionAccumulator)
browserAcc := make(distributionAccumulator)
osAcc := make(distributionAccumulator)
for _, row := range uaRows {
ua := row.Key
count := row.Value
deviceAcc[analyticsrepo.ParseDeviceType(ua)] += count
browserAcc[analyticsrepo.ParseBrowserName(ua)] += count
osAcc[analyticsrepo.ParseOSName(ua)] += count
}
topUserAgents = make([]DistributionItem, 0, accessLogOverviewTopLimit)
for _, row := range uaRows {
if len(topUserAgents) >= accessLogOverviewTopLimit {
break
}
topUserAgents = append(topUserAgents, row)
}
return toDistributionItems(deviceAcc, 0),
toDistributionItems(browserAcc, accessLogOverviewTopLimit),
toDistributionItems(osAcc, accessLogOverviewTopLimit),
topUserAgents
}
func buildAccessLogOverviewTrends(
ctx context.Context,
now time.Time,
@@ -394,6 +447,7 @@ func ListAccessLogs(ctx context.Context, input AccessLogQuery) (*AccessLogList,
Region: item.Region,
Host: item.Host,
Path: item.Path,
UserAgent: item.UserAgent,
StatusCode: item.StatusCode,
})
}