feat(access-logs): 接入 User-Agent 与设备/浏览器/状态码分布

- Agent 访问日志上报 user_agent,OpenResty log_format 输出 http_user_agent
- of_node_access_logs 新增 user_agent 列并写入 ClickHouse
- 访问日志概览新增:设备类型饼图、状态码饼图、浏览器/OS/User-Agent 排行
- 日志明细列表增加 User-Agent 列
- 扩展 UA 解析工具(browser/os/device)并支持 CLI 识别
This commit is contained in:
ryan
2026-07-18 21:18:59 +08:00
parent 177578ef4e
commit e49078ac3b
28 changed files with 498 additions and 62 deletions
@@ -28,7 +28,8 @@ func TestParseBrowserName(t *testing.T) {
{name: "edge", ua: "Mozilla/5.0 Edg/120.0.0.0", want: "Edge"},
{name: "wechat", ua: "MicroMessenger/8.0", want: "WeChat"},
{name: "postman", ua: "PostmanRuntime/7.36.0", want: "Postman"},
{name: "other", ua: "curl/8.0", want: "Other"},
{name: "cli", ua: "curl/8.0", want: "CLI"},
{name: "other", ua: "CustomClient/1.0", want: "Other"},
}
for _, tt := range tests {
+110 -20
View File
@@ -5,26 +5,116 @@ package analytics
import "strings"
const (
uaLabelUnknown = "Unknown"
uaLabelBot = "Bot"
uaLabelOther = "Other"
uaTokenBot = "bot"
uaTokenAndroid = "android"
uaTokenSpider = "spider"
uaTokenCrawler = "crawler"
)
type uaMatchRule struct {
label string
contains []string
allOf []string
noneOf []string
}
func matchUARules(uaLower string, rules []uaMatchRule, fallback string) string {
if uaLower == "" {
return uaLabelUnknown
}
for _, rule := range rules {
matched := false
for _, token := range rule.contains {
if strings.Contains(uaLower, token) {
matched = true
break
}
}
if !matched && len(rule.allOf) > 0 {
matched = true
for _, token := range rule.allOf {
if !strings.Contains(uaLower, token) {
matched = false
break
}
}
}
if !matched {
continue
}
excluded := false
for _, token := range rule.noneOf {
if strings.Contains(uaLower, token) {
excluded = true
break
}
}
if excluded {
continue
}
return rule.label
}
return fallback
}
var browserRules = []uaMatchRule{
{label: "WeChat", contains: []string{"micromessenger"}},
{label: "Postman", contains: []string{"postman"}},
{label: "CLI", contains: []string{"curl/", "wget/"}},
{label: "Edge", contains: []string{"edg/", "edgios/", "edga/"}},
{label: "Opera", contains: []string{"opr/", "opera"}},
{label: "Firefox", contains: []string{"firefox", "fxios"}},
{label: "Chrome", contains: []string{"crios", "chrome"}, noneOf: []string{"chromium"}},
{label: "Chromium", contains: []string{"chromium"}},
{label: "Safari", contains: []string{"safari"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp"}},
}
var osRules = []uaMatchRule{
{label: "Android", contains: []string{uaTokenAndroid}},
{label: "iOS", contains: []string{"iphone", "ipad", "ipod", "ios"}},
{label: "Windows", contains: []string{"windows"}},
{label: "macOS", contains: []string{"mac os x", "macintosh", "macos"}},
{label: "Chrome OS", contains: []string{"cros"}},
{label: "Linux", contains: []string{"linux"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler}},
}
var deviceRules = []uaMatchRule{
{
label: uaLabelBot,
contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp", "curl/", "wget/", "python-requests", "go-http-client", "postman"},
},
{
label: "Tablet",
contains: []string{"ipad", "tablet"},
},
{
label: "Tablet",
allOf: []string{uaTokenAndroid},
noneOf: []string{"mobile"},
},
{
label: "Mobile",
contains: []string{"mobi", "iphone", "ipod", uaTokenAndroid},
},
}
// ParseBrowserName performs lightweight User-Agent browser identification.
func ParseBrowserName(ua string) string {
uaLower := strings.ToLower(ua)
if strings.Contains(uaLower, "micromessenger") {
return "WeChat"
}
if strings.Contains(uaLower, "postman") {
return "Postman"
}
if strings.Contains(uaLower, "edg/") || strings.Contains(uaLower, "edge") {
return "Edge"
}
if strings.Contains(uaLower, "firefox") {
return "Firefox"
}
if strings.Contains(uaLower, "chrome") {
return "Chrome"
}
if strings.Contains(uaLower, "safari") {
return "Safari"
}
return "Other"
return matchUARules(strings.ToLower(ua), browserRules, uaLabelOther)
}
// ParseOSName performs lightweight User-Agent OS identification.
func ParseOSName(ua string) string {
return matchUARules(strings.ToLower(ua), osRules, uaLabelOther)
}
// ParseDeviceType performs lightweight User-Agent device type identification.
func ParseDeviceType(ua string) string {
return matchUARules(strings.ToLower(ua), deviceRules, "Desktop")
}
@@ -36,7 +36,7 @@ func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]anal
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, request_length, request_time_ms, created_at
SELECT id, node_id, logged_at, remote_addr, region, host, path, user_agent, status_code, bytes_sent, request_length, request_time_ms, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
@@ -55,6 +55,7 @@ ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.
return scanNodeAccessLogRows(rows)
}
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, error) {
var result []analyticsmodel.NodeAccessLog
for rows.Next() {
@@ -67,6 +68,7 @@ func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, er
&item.Region,
&item.Host,
&item.Path,
&item.UserAgent,
&item.StatusCode,
&item.BytesSent,
&item.RequestLength,
@@ -206,7 +208,7 @@ WHERE %s`, tableName, clause)
}
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
// Allowed columns: status_code, host, path, remote_addr.
// Allowed columns: status_code, host, path, remote_addr, user_agent.
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
conn, err := nodeAccessLogConn()
if err != nil {
@@ -261,6 +263,8 @@ func nodeAccessLogValueCountExpr(column string) (string, bool) {
return "trim(" + nodeAccessLogColumnPath + ")", true
case nodeAccessLogColumnRemoteAddr:
return "trim(" + nodeAccessLogColumnRemoteAddr + ")", true
case nodeAccessLogColumnUserAgent:
return "trim(" + nodeAccessLogColumnUserAgent + ")", true
default:
return "", false
}
@@ -20,6 +20,7 @@ const (
nodeAccessLogColumnStatusCode = "status_code"
nodeAccessLogColumnHost = "host"
nodeAccessLogColumnPath = "path"
nodeAccessLogColumnUserAgent = "user_agent"
nodeAccessLogColumnLoggedAt = "logged_at"
)
@@ -49,8 +49,9 @@ func TestBatchInsertNodeAccessLogs_UsesModelBatchSQL(t *testing.T) {
assert.True(t, mockBatch.sendCalled)
require.Len(t, mockBatch.rows, 1)
assert.Equal(t, "node-a", mockBatch.rows[0][1])
require.Len(t, mockBatch.rows[0], 12)
assert.Equal(t, uint64(2048), mockBatch.rows[0][8]) // bytes_sent
assert.Equal(t, uint64(0), mockBatch.rows[0][9]) // request_length
assert.Equal(t, uint32(0), mockBatch.rows[0][10]) // request_time_ms
require.Len(t, mockBatch.rows[0], 13)
assert.Equal(t, "", mockBatch.rows[0][7]) // user_agent
assert.Equal(t, uint64(2048), mockBatch.rows[0][9]) // bytes_sent
assert.Equal(t, uint64(0), mockBatch.rows[0][10]) // request_length
assert.Equal(t, uint32(0), mockBatch.rows[0][11]) // request_time_ms
}
@@ -46,6 +46,7 @@ func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAc
logItem.Region,
logItem.Host,
logItem.Path,
strings.TrimSpace(logItem.UserAgent),
logItem.StatusCode,
logItem.BytesSent,
logItem.RequestLength,
@@ -159,6 +159,7 @@ ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
return scanNodeObsFrpcRows(rows)
}
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
func scanNodeMetricSnapshotRows(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
var result []analyticsmodel.NodeMetricSnapshot
for rows.Next() {