mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-09 00:56:37 +08:00
feat(access-logs): 接入 User-Agent 与设备/浏览器/状态码分布
- Agent 访问日志上报 user_agent,OpenResty log_format 输出 http_user_agent - of_node_access_logs 新增 user_agent 列并写入 ClickHouse - 访问日志概览新增:设备类型饼图、状态码饼图、浏览器/OS/User-Agent 排行 - 日志明细列表增加 User-Agent 列 - 扩展 UA 解析工具(browser/os/device)并支持 CLI 识别
This commit is contained in:
@@ -24,6 +24,7 @@ sidebar: false
|
|||||||
### 变更
|
### 变更
|
||||||
|
|
||||||
- 访问日志页面重构为「概览」与「日志明细」两个标签页:概览展示请求量、访问量、带宽趋势与 Top Paths/Hosts/IPs,明细列表保留检索能力;已移除时间折叠与 IP 汇总视图。
|
- 访问日志页面重构为「概览」与「日志明细」两个标签页:概览展示请求量、访问量、带宽趋势与 Top Paths/Hosts/IPs,明细列表保留检索能力;已移除时间折叠与 IP 汇总视图。
|
||||||
|
- 边缘访问日志现支持上报并存储 User-Agent;概览新增设备类型饼图,以及浏览器、操作系统、User-Agent 排行。
|
||||||
|
|
||||||
## [v3.4.0] - 2026-07-18
|
## [v3.4.0] - 2026-07-18
|
||||||
|
|
||||||
|
|||||||
@@ -14325,6 +14325,9 @@ const docTemplate = `{
|
|||||||
},
|
},
|
||||||
"status_code": {
|
"status_code": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"user_agent": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -16232,15 +16235,33 @@ const docTemplate = `{
|
|||||||
"observability.AccessLogOverview": {
|
"observability.AccessLogOverview": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"device_types": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"generated_at": {
|
"generated_at": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"hours": {
|
"hours": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
"status_codes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"summary": {
|
"summary": {
|
||||||
"$ref": "#/definitions/observability.AccessLogOverviewSummary"
|
"$ref": "#/definitions/observability.AccessLogOverviewSummary"
|
||||||
},
|
},
|
||||||
|
"top_browsers": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"top_hosts": {
|
"top_hosts": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
@@ -16253,12 +16274,24 @@ const docTemplate = `{
|
|||||||
"$ref": "#/definitions/observability.DistributionItem"
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"top_operating_systems": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"top_paths": {
|
"top_paths": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
"$ref": "#/definitions/observability.DistributionItem"
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"top_user_agents": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"trends": {
|
"trends": {
|
||||||
"$ref": "#/definitions/observability.AccessLogOverviewTrends"
|
"$ref": "#/definitions/observability.AccessLogOverviewTrends"
|
||||||
}
|
}
|
||||||
@@ -16341,6 +16374,9 @@ const docTemplate = `{
|
|||||||
},
|
},
|
||||||
"status_code": {
|
"status_code": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"user_agent": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -14318,6 +14318,9 @@
|
|||||||
},
|
},
|
||||||
"status_code": {
|
"status_code": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"user_agent": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -16225,15 +16228,33 @@
|
|||||||
"observability.AccessLogOverview": {
|
"observability.AccessLogOverview": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"device_types": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"generated_at": {
|
"generated_at": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"hours": {
|
"hours": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
"status_codes": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"summary": {
|
"summary": {
|
||||||
"$ref": "#/definitions/observability.AccessLogOverviewSummary"
|
"$ref": "#/definitions/observability.AccessLogOverviewSummary"
|
||||||
},
|
},
|
||||||
|
"top_browsers": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"top_hosts": {
|
"top_hosts": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
@@ -16246,12 +16267,24 @@
|
|||||||
"$ref": "#/definitions/observability.DistributionItem"
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"top_operating_systems": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"top_paths": {
|
"top_paths": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
"$ref": "#/definitions/observability.DistributionItem"
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"top_user_agents": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/observability.DistributionItem"
|
||||||
|
}
|
||||||
|
},
|
||||||
"trends": {
|
"trends": {
|
||||||
"$ref": "#/definitions/observability.AccessLogOverviewTrends"
|
"$ref": "#/definitions/observability.AccessLogOverviewTrends"
|
||||||
}
|
}
|
||||||
@@ -16334,6 +16367,9 @@
|
|||||||
},
|
},
|
||||||
"status_code": {
|
"status_code": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"user_agent": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -673,6 +673,8 @@ definitions:
|
|||||||
type: integer
|
type: integer
|
||||||
status_code:
|
status_code:
|
||||||
type: integer
|
type: integer
|
||||||
|
user_agent:
|
||||||
|
type: string
|
||||||
type: object
|
type: object
|
||||||
github_com_Rain-kl_Wavelet_pkg_protocol.NodeEdgeHealth:
|
github_com_Rain-kl_Wavelet_pkg_protocol.NodeEdgeHealth:
|
||||||
properties:
|
properties:
|
||||||
@@ -1939,12 +1941,24 @@ definitions:
|
|||||||
type: object
|
type: object
|
||||||
observability.AccessLogOverview:
|
observability.AccessLogOverview:
|
||||||
properties:
|
properties:
|
||||||
|
device_types:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
|
type: array
|
||||||
generated_at:
|
generated_at:
|
||||||
type: string
|
type: string
|
||||||
hours:
|
hours:
|
||||||
type: integer
|
type: integer
|
||||||
|
status_codes:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
|
type: array
|
||||||
summary:
|
summary:
|
||||||
$ref: '#/definitions/observability.AccessLogOverviewSummary'
|
$ref: '#/definitions/observability.AccessLogOverviewSummary'
|
||||||
|
top_browsers:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
|
type: array
|
||||||
top_hosts:
|
top_hosts:
|
||||||
items:
|
items:
|
||||||
$ref: '#/definitions/observability.DistributionItem'
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
@@ -1953,10 +1967,18 @@ definitions:
|
|||||||
items:
|
items:
|
||||||
$ref: '#/definitions/observability.DistributionItem'
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
type: array
|
type: array
|
||||||
|
top_operating_systems:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
|
type: array
|
||||||
top_paths:
|
top_paths:
|
||||||
items:
|
items:
|
||||||
$ref: '#/definitions/observability.DistributionItem'
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
type: array
|
type: array
|
||||||
|
top_user_agents:
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/observability.DistributionItem'
|
||||||
|
type: array
|
||||||
trends:
|
trends:
|
||||||
$ref: '#/definitions/observability.AccessLogOverviewTrends'
|
$ref: '#/definitions/observability.AccessLogOverviewTrends'
|
||||||
type: object
|
type: object
|
||||||
@@ -2011,6 +2033,8 @@ definitions:
|
|||||||
type: string
|
type: string
|
||||||
status_code:
|
status_code:
|
||||||
type: integer
|
type: integer
|
||||||
|
user_agent:
|
||||||
|
type: string
|
||||||
type: object
|
type: object
|
||||||
observability.CapacityTrendPoint:
|
observability.CapacityTrendPoint:
|
||||||
properties:
|
properties:
|
||||||
|
|||||||
@@ -123,6 +123,7 @@ export function DetailTab({
|
|||||||
<TableHead className='text-xs'>IP</TableHead>
|
<TableHead className='text-xs'>IP</TableHead>
|
||||||
<TableHead className='text-xs'>域名</TableHead>
|
<TableHead className='text-xs'>域名</TableHead>
|
||||||
<TableHead className='text-xs'>路径</TableHead>
|
<TableHead className='text-xs'>路径</TableHead>
|
||||||
|
<TableHead className='text-xs'>User-Agent</TableHead>
|
||||||
<TableHead className='text-xs'>状态码</TableHead>
|
<TableHead className='text-xs'>状态码</TableHead>
|
||||||
</TableRow>
|
</TableRow>
|
||||||
</TableHeader>
|
</TableHeader>
|
||||||
@@ -139,9 +140,15 @@ export function DetailTab({
|
|||||||
{item.remote_addr}
|
{item.remote_addr}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell className='text-xs'>{item.host}</TableCell>
|
<TableCell className='text-xs'>{item.host}</TableCell>
|
||||||
<TableCell className='text-xs max-w-48 truncate'>
|
<TableCell className='text-xs max-w-40 truncate'>
|
||||||
{item.path}
|
{item.path}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
|
<TableCell
|
||||||
|
className='text-xs max-w-56 truncate'
|
||||||
|
title={item.user_agent || undefined}
|
||||||
|
>
|
||||||
|
{item.user_agent || '—'}
|
||||||
|
</TableCell>
|
||||||
<TableCell>
|
<TableCell>
|
||||||
<Badge variant='outline' className='text-[10px]'>
|
<Badge variant='outline' className='text-[10px]'>
|
||||||
{item.status_code}
|
{item.status_code}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
import { useMemo } from 'react';
|
import { useMemo } from 'react';
|
||||||
import type { EChartsOption } from 'echarts';
|
import type { EChartsOption } from 'echarts';
|
||||||
import ReactECharts from 'echarts-for-react';
|
import ReactECharts from 'echarts-for-react';
|
||||||
|
import { Cell, Pie, PieChart } from 'recharts';
|
||||||
|
|
||||||
import { RankChart } from '@/components/data/rank-chart';
|
import { RankChart } from '@/components/data/rank-chart';
|
||||||
import { TrendChart } from '@/components/data/trend-chart';
|
import { TrendChart } from '@/components/data/trend-chart';
|
||||||
@@ -16,8 +17,19 @@ import {
|
|||||||
CardHeader,
|
CardHeader,
|
||||||
CardTitle,
|
CardTitle,
|
||||||
} from '@/components/ui/card';
|
} from '@/components/ui/card';
|
||||||
|
import {
|
||||||
|
ChartContainer,
|
||||||
|
ChartLegend,
|
||||||
|
ChartLegendContent,
|
||||||
|
ChartTooltip,
|
||||||
|
ChartTooltipContent,
|
||||||
|
type ChartConfig,
|
||||||
|
} from '@/components/ui/chart';
|
||||||
import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group';
|
import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group';
|
||||||
import type { AccessLogOverview } from '@/lib/services/openflare';
|
import type {
|
||||||
|
AccessLogOverview,
|
||||||
|
DistributionItem,
|
||||||
|
} from '@/lib/services/openflare';
|
||||||
import { formatBytes, formatCompactNumber } from '@/lib/utils/metrics';
|
import { formatBytes, formatCompactNumber } from '@/lib/utils/metrics';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
@@ -27,6 +39,15 @@ import {
|
|||||||
type OverviewRangeHours,
|
type OverviewRangeHours,
|
||||||
} from './access-log-utils';
|
} from './access-log-utils';
|
||||||
|
|
||||||
|
const DEVICE_COLORS = [
|
||||||
|
'#38bdf8',
|
||||||
|
'#34d399',
|
||||||
|
'#f59e0b',
|
||||||
|
'#a78bfa',
|
||||||
|
'#f472b6',
|
||||||
|
'#94a3b8',
|
||||||
|
];
|
||||||
|
|
||||||
function SparklineMetricCard({
|
function SparklineMetricCard({
|
||||||
title,
|
title,
|
||||||
value,
|
value,
|
||||||
@@ -132,6 +153,13 @@ function SparklineMetricCard({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function toRankItems(items: DistributionItem[] | undefined) {
|
||||||
|
return (items ?? []).map((item) => ({
|
||||||
|
label: item.key,
|
||||||
|
value: item.value,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
function RankCard({
|
function RankCard({
|
||||||
title,
|
title,
|
||||||
description,
|
description,
|
||||||
@@ -164,6 +192,101 @@ function RankCard({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function PieDistributionCard({
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
items,
|
||||||
|
emptyMessage,
|
||||||
|
}: {
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
items: DistributionItem[];
|
||||||
|
emptyMessage: string;
|
||||||
|
}) {
|
||||||
|
const chartData = useMemo(
|
||||||
|
() =>
|
||||||
|
items.map((item, index) => ({
|
||||||
|
name: item.key,
|
||||||
|
value: item.value,
|
||||||
|
fill: DEVICE_COLORS[index % DEVICE_COLORS.length],
|
||||||
|
})),
|
||||||
|
[items],
|
||||||
|
);
|
||||||
|
|
||||||
|
const chartConfig = useMemo(() => {
|
||||||
|
const config: ChartConfig = {};
|
||||||
|
chartData.forEach((item) => {
|
||||||
|
config[item.name] = {
|
||||||
|
label: item.name,
|
||||||
|
color: item.fill,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return config;
|
||||||
|
}, [chartData]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Card className='border-dashed shadow-none'>
|
||||||
|
<CardHeader className='pb-3'>
|
||||||
|
<CardTitle className='text-sm font-semibold text-foreground'>
|
||||||
|
{title}
|
||||||
|
</CardTitle>
|
||||||
|
<CardDescription className='text-xs text-muted-foreground'>
|
||||||
|
{description}
|
||||||
|
</CardDescription>
|
||||||
|
</CardHeader>
|
||||||
|
<CardContent className='pt-0'>
|
||||||
|
{chartData.length === 0 ? (
|
||||||
|
<div className='flex h-[450px] items-center justify-center rounded-md border border-dashed bg-muted/20 text-sm text-muted-foreground'>
|
||||||
|
{emptyMessage}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<ChartContainer
|
||||||
|
config={chartConfig}
|
||||||
|
className='mx-auto h-[450px] w-full'
|
||||||
|
>
|
||||||
|
<PieChart>
|
||||||
|
<Pie
|
||||||
|
data={chartData}
|
||||||
|
dataKey='value'
|
||||||
|
nameKey='name'
|
||||||
|
cx='50%'
|
||||||
|
cy='46%'
|
||||||
|
innerRadius={72}
|
||||||
|
outerRadius={110}
|
||||||
|
paddingAngle={2}
|
||||||
|
>
|
||||||
|
{chartData.map((entry) => (
|
||||||
|
<Cell key={entry.name} fill={entry.fill} />
|
||||||
|
))}
|
||||||
|
</Pie>
|
||||||
|
<ChartTooltip
|
||||||
|
cursor={false}
|
||||||
|
content={
|
||||||
|
<ChartTooltipContent
|
||||||
|
hideLabel
|
||||||
|
formatter={(value, name) => (
|
||||||
|
<>
|
||||||
|
<span className='text-muted-foreground'>{name}</span>
|
||||||
|
<span className='ml-auto font-mono font-medium tabular-nums text-foreground'>
|
||||||
|
{formatCompactNumber(Number(value ?? 0))}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<ChartLegend
|
||||||
|
content={<ChartLegendContent nameKey='name' />}
|
||||||
|
className='flex-wrap justify-center gap-x-4 gap-y-1 pt-2 text-[11px]'
|
||||||
|
/>
|
||||||
|
</PieChart>
|
||||||
|
</ChartContainer>
|
||||||
|
)}
|
||||||
|
</CardContent>
|
||||||
|
</Card>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function OverviewRangeSwitch({
|
function OverviewRangeSwitch({
|
||||||
hours,
|
hours,
|
||||||
onHoursChange,
|
onHoursChange,
|
||||||
@@ -314,33 +437,60 @@ function OverviewContent({
|
|||||||
</CardContent>
|
</CardContent>
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
|
<div className='grid gap-6 xl:grid-cols-2'>
|
||||||
|
<PieDistributionCard
|
||||||
|
title='Requests by device type'
|
||||||
|
description='按设备类型统计请求占比。'
|
||||||
|
items={data.device_types ?? []}
|
||||||
|
emptyMessage='暂无设备类型数据'
|
||||||
|
/>
|
||||||
|
<PieDistributionCard
|
||||||
|
title='Status code'
|
||||||
|
description='按 HTTP 状态码统计请求占比。'
|
||||||
|
items={data.status_codes ?? []}
|
||||||
|
emptyMessage='暂无状态码分布数据'
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className='grid gap-6 xl:grid-cols-3'>
|
<div className='grid gap-6 xl:grid-cols-3'>
|
||||||
<RankCard
|
<RankCard
|
||||||
title='Top Paths'
|
title='Top Paths'
|
||||||
description='访问量最高的请求路径。'
|
description='访问量最高的请求路径。'
|
||||||
color='#a78bfa'
|
color='#a78bfa'
|
||||||
items={data.top_paths.map((item) => ({
|
items={toRankItems(data.top_paths)}
|
||||||
label: item.key,
|
|
||||||
value: item.value,
|
|
||||||
}))}
|
|
||||||
/>
|
/>
|
||||||
<RankCard
|
<RankCard
|
||||||
title='Top Hosts'
|
title='Top Hosts'
|
||||||
description='流量集中的访问域名。'
|
description='流量集中的访问域名。'
|
||||||
color='#34d399'
|
color='#34d399'
|
||||||
items={data.top_hosts.map((item) => ({
|
items={toRankItems(data.top_hosts)}
|
||||||
label: item.key,
|
|
||||||
value: item.value,
|
|
||||||
}))}
|
|
||||||
/>
|
/>
|
||||||
<RankCard
|
<RankCard
|
||||||
title='Top IPs'
|
title='Top IPs'
|
||||||
description='请求次数最多的来源 IP。'
|
description='请求次数最多的来源 IP。'
|
||||||
color='#38bdf8'
|
color='#38bdf8'
|
||||||
items={data.top_ips.map((item) => ({
|
items={toRankItems(data.top_ips)}
|
||||||
label: item.key,
|
/>
|
||||||
value: item.value,
|
</div>
|
||||||
}))}
|
|
||||||
|
<div className='grid gap-6 xl:grid-cols-3'>
|
||||||
|
<RankCard
|
||||||
|
title='Top browsers'
|
||||||
|
description='按浏览器聚合的请求排行。'
|
||||||
|
color='#f59e0b'
|
||||||
|
items={toRankItems(data.top_browsers)}
|
||||||
|
/>
|
||||||
|
<RankCard
|
||||||
|
title='Top Operating System'
|
||||||
|
description='按操作系统聚合的请求排行。'
|
||||||
|
color='#22c55e'
|
||||||
|
items={toRankItems(data.top_operating_systems)}
|
||||||
|
/>
|
||||||
|
<RankCard
|
||||||
|
title='Top User-Agent'
|
||||||
|
description='原始 User-Agent 请求排行。'
|
||||||
|
color='#818cf8'
|
||||||
|
items={toRankItems(data.top_user_agents)}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
|
|||||||
@@ -527,6 +527,11 @@ export interface AccessLogOverview {
|
|||||||
top_paths: DistributionItem[];
|
top_paths: DistributionItem[];
|
||||||
top_hosts: DistributionItem[];
|
top_hosts: DistributionItem[];
|
||||||
top_ips: DistributionItem[];
|
top_ips: DistributionItem[];
|
||||||
|
device_types: DistributionItem[];
|
||||||
|
top_browsers: DistributionItem[];
|
||||||
|
top_operating_systems: DistributionItem[];
|
||||||
|
top_user_agents: DistributionItem[];
|
||||||
|
status_codes: DistributionItem[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AccessLogItem {
|
export interface AccessLogItem {
|
||||||
@@ -538,6 +543,7 @@ export interface AccessLogItem {
|
|||||||
region: string;
|
region: string;
|
||||||
host: string;
|
host: string;
|
||||||
path: string;
|
path: string;
|
||||||
|
user_agent: string;
|
||||||
status_code: number;
|
status_code: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ type accessLogRecord struct {
|
|||||||
Host string `json:"host"`
|
Host string `json:"host"`
|
||||||
RemoteAddr string `json:"remote_addr"`
|
RemoteAddr string `json:"remote_addr"`
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
|
UserAgent string `json:"user_agent"`
|
||||||
Status int `json:"status"`
|
Status int `json:"status"`
|
||||||
BytesSent int64 `json:"bytes_sent"`
|
BytesSent int64 `json:"bytes_sent"`
|
||||||
RequestLength int64 `json:"request_length"`
|
RequestLength int64 `json:"request_length"`
|
||||||
@@ -145,6 +146,7 @@ func (aggregate *trafficAggregate) consume(line []byte) {
|
|||||||
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
|
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
|
||||||
Host: strings.TrimSpace(record.Host),
|
Host: strings.TrimSpace(record.Host),
|
||||||
Path: normalizeAccessLogPath(record.Path),
|
Path: normalizeAccessLogPath(record.Path),
|
||||||
|
UserAgent: strings.TrimSpace(record.UserAgent),
|
||||||
StatusCode: record.Status,
|
StatusCode: record.Status,
|
||||||
BytesSent: record.BytesSent,
|
BytesSent: record.BytesSent,
|
||||||
RequestLength: record.RequestLength,
|
RequestLength: record.RequestLength,
|
||||||
@@ -157,6 +159,7 @@ type parsedAccessLogRecord struct {
|
|||||||
Host string
|
Host string
|
||||||
RemoteAddr string
|
RemoteAddr string
|
||||||
Path string
|
Path string
|
||||||
|
UserAgent string
|
||||||
Status int
|
Status int
|
||||||
BytesSent int64
|
BytesSent int64
|
||||||
RequestLength int64
|
RequestLength int64
|
||||||
@@ -189,6 +192,7 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
|
|||||||
Host: strings.TrimSpace(record.Host),
|
Host: strings.TrimSpace(record.Host),
|
||||||
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
|
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
|
||||||
Path: normalizeAccessLogPath(record.Path),
|
Path: normalizeAccessLogPath(record.Path),
|
||||||
|
UserAgent: strings.TrimSpace(record.UserAgent),
|
||||||
Status: record.Status,
|
Status: record.Status,
|
||||||
BytesSent: record.BytesSent,
|
BytesSent: record.BytesSent,
|
||||||
RequestLength: record.RequestLength,
|
RequestLength: record.RequestLength,
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
|
|||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
logPath := filepath.Join(tempDir, "openflare_access.log")
|
logPath := filepath.Join(tempDir, "openflare_access.log")
|
||||||
content := []byte(
|
content := []byte(
|
||||||
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512,\"request_time\":0.015}\n" +
|
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512,\"request_time\":0.015,\"user_agent\":\"Mozilla/5.0\"}\n" +
|
||||||
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256,\"request_time\":0.008}\n",
|
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256,\"request_time\":0.008,\"user_agent\":\"curl/8.0\"}\n",
|
||||||
)
|
)
|
||||||
if err := os.WriteFile(logPath, content, 0o644); err != nil {
|
if err := os.WriteFile(logPath, content, 0o644); err != nil {
|
||||||
t.Fatalf("WriteFile failed: %v", err)
|
t.Fatalf("WriteFile failed: %v", err)
|
||||||
@@ -35,6 +35,9 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
|
|||||||
if accessLogs[0].Path != "/login" || accessLogs[1].Path != "/v1/ping" {
|
if accessLogs[0].Path != "/login" || accessLogs[1].Path != "/v1/ping" {
|
||||||
t.Fatalf("unexpected access log paths: %+v", accessLogs)
|
t.Fatalf("unexpected access log paths: %+v", accessLogs)
|
||||||
}
|
}
|
||||||
|
if accessLogs[0].UserAgent != "Mozilla/5.0" || accessLogs[1].UserAgent != "curl/8.0" {
|
||||||
|
t.Fatalf("unexpected user agents: %+v", accessLogs)
|
||||||
|
}
|
||||||
|
|
||||||
snapshot, err := stateStore.Load()
|
snapshot, err := stateStore.Load()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ func mergeAccessLogs(existing []protocol.NodeAccessLog, incoming []protocol.Node
|
|||||||
}
|
}
|
||||||
|
|
||||||
func accessLogKey(item protocol.NodeAccessLog) string {
|
func accessLogKey(item protocol.NodeAccessLog) string {
|
||||||
return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + strconv.Itoa(item.StatusCode)
|
return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + item.UserAgent + "|" + strconv.Itoa(item.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Replayable returns buffered records from windows before currentWindowStartedAtUnix.
|
// Replayable returns buffered records from windows before currentWindowStartedAtUnix.
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ const (
|
|||||||
healthSeverityWarning = "warning"
|
healthSeverityWarning = "warning"
|
||||||
healthSeverityCritical = "critical"
|
healthSeverityCritical = "critical"
|
||||||
accessLogPathMaxLength = 100
|
accessLogPathMaxLength = 100
|
||||||
|
accessLogUserAgentMaxLength = 512
|
||||||
healthEventMessageMaxLength = 4096
|
healthEventMessageMaxLength = 4096
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -208,6 +209,7 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
|
|||||||
Region: "",
|
Region: "",
|
||||||
Host: strings.TrimSpace(item.Host),
|
Host: strings.TrimSpace(item.Host),
|
||||||
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
|
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
|
||||||
|
UserAgent: truncateForDatabase(strings.TrimSpace(item.UserAgent), accessLogUserAgentMaxLength),
|
||||||
StatusCode: item.StatusCode,
|
StatusCode: item.StatusCode,
|
||||||
BytesSent: bytesSent,
|
BytesSent: bytesSent,
|
||||||
RequestLength: requestLength,
|
RequestLength: requestLength,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/Rain-kl/Wavelet/internal/model"
|
"github.com/Rain-kl/Wavelet/internal/model"
|
||||||
|
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -52,6 +53,7 @@ type AccessLogView struct {
|
|||||||
Region string `json:"region"`
|
Region string `json:"region"`
|
||||||
Host string `json:"host"`
|
Host string `json:"host"`
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
|
UserAgent string `json:"user_agent"`
|
||||||
StatusCode int `json:"status_code"`
|
StatusCode int `json:"status_code"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,13 +209,18 @@ type AccessLogOverviewTrends struct {
|
|||||||
|
|
||||||
// AccessLogOverview is the access-log analytics overview payload.
|
// AccessLogOverview is the access-log analytics overview payload.
|
||||||
type AccessLogOverview struct {
|
type AccessLogOverview struct {
|
||||||
GeneratedAt time.Time `json:"generated_at"`
|
GeneratedAt time.Time `json:"generated_at"`
|
||||||
Hours int `json:"hours"`
|
Hours int `json:"hours"`
|
||||||
Summary AccessLogOverviewSummary `json:"summary"`
|
Summary AccessLogOverviewSummary `json:"summary"`
|
||||||
Trends AccessLogOverviewTrends `json:"trends"`
|
Trends AccessLogOverviewTrends `json:"trends"`
|
||||||
TopPaths []DistributionItem `json:"top_paths"`
|
TopPaths []DistributionItem `json:"top_paths"`
|
||||||
TopHosts []DistributionItem `json:"top_hosts"`
|
TopHosts []DistributionItem `json:"top_hosts"`
|
||||||
TopIPs []DistributionItem `json:"top_ips"`
|
TopIPs []DistributionItem `json:"top_ips"`
|
||||||
|
DeviceTypes []DistributionItem `json:"device_types"`
|
||||||
|
TopBrowsers []DistributionItem `json:"top_browsers"`
|
||||||
|
TopOperatingSystems []DistributionItem `json:"top_operating_systems"`
|
||||||
|
TopUserAgents []DistributionItem `json:"top_user_agents"`
|
||||||
|
StatusCodes []DistributionItem `json:"status_codes"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// AccessLogCleanupInput is the cleanup request payload.
|
// AccessLogCleanupInput is the cleanup request payload.
|
||||||
@@ -229,9 +236,10 @@ type AccessLogCleanupResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultAccessLogOverviewHours = 24
|
defaultAccessLogOverviewHours = 24
|
||||||
maxAccessLogOverviewHours = 24 * 30
|
maxAccessLogOverviewHours = 24 * 30
|
||||||
accessLogOverviewTopLimit = 10
|
accessLogOverviewTopLimit = 10
|
||||||
|
accessLogOverviewUASampleLimit = 200
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetAccessLogOverview returns summary metrics, trends, and top rankings.
|
// GetAccessLogOverview returns summary metrics, trends, and top rankings.
|
||||||
@@ -254,6 +262,7 @@ func GetAccessLogOverview(ctx context.Context, input AccessLogOverviewQuery) (*A
|
|||||||
requestPoints, visitPoints, bandwidthPoints := buildAccessLogOverviewTrends(
|
requestPoints, visitPoints, bandwidthPoints := buildAccessLogOverviewTrends(
|
||||||
ctx, now, normalized.Hours, query,
|
ctx, now, normalized.Hours, query,
|
||||||
)
|
)
|
||||||
|
deviceTypes, topBrowsers, topOSes, topUserAgents := buildAccessLogUADistributions(ctx, query)
|
||||||
|
|
||||||
return &AccessLogOverview{
|
return &AccessLogOverview{
|
||||||
GeneratedAt: now,
|
GeneratedAt: now,
|
||||||
@@ -268,9 +277,14 @@ func GetAccessLogOverview(ctx context.Context, input AccessLogOverviewQuery) (*A
|
|||||||
Visits: visitPoints,
|
Visits: visitPoints,
|
||||||
Bandwidth: bandwidthPoints,
|
Bandwidth: bandwidthPoints,
|
||||||
},
|
},
|
||||||
TopPaths: valueCountDistribution(ctx, query, "path", accessLogOverviewTopLimit),
|
TopPaths: valueCountDistribution(ctx, query, "path", accessLogOverviewTopLimit),
|
||||||
TopHosts: valueCountDistribution(ctx, query, "host", accessLogOverviewTopLimit),
|
TopHosts: valueCountDistribution(ctx, query, "host", accessLogOverviewTopLimit),
|
||||||
TopIPs: valueCountDistribution(ctx, query, "remote_addr", accessLogOverviewTopLimit),
|
TopIPs: valueCountDistribution(ctx, query, "remote_addr", accessLogOverviewTopLimit),
|
||||||
|
DeviceTypes: deviceTypes,
|
||||||
|
TopBrowsers: topBrowsers,
|
||||||
|
TopOperatingSystems: topOSes,
|
||||||
|
TopUserAgents: topUserAgents,
|
||||||
|
StatusCodes: valueCountDistribution(ctx, query, "status_code", accessLogOverviewTopLimit),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -309,6 +323,45 @@ func valueCountDistribution(
|
|||||||
return items
|
return items
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildAccessLogUADistributions(
|
||||||
|
ctx context.Context,
|
||||||
|
query model.OpenFlareAccessLogQuery,
|
||||||
|
) (
|
||||||
|
deviceTypes []DistributionItem,
|
||||||
|
topBrowsers []DistributionItem,
|
||||||
|
topOSes []DistributionItem,
|
||||||
|
topUserAgents []DistributionItem,
|
||||||
|
) {
|
||||||
|
uaRows := valueCountDistribution(ctx, query, "user_agent", accessLogOverviewUASampleLimit)
|
||||||
|
if len(uaRows) == 0 {
|
||||||
|
return []DistributionItem{}, []DistributionItem{}, []DistributionItem{}, []DistributionItem{}
|
||||||
|
}
|
||||||
|
|
||||||
|
deviceAcc := make(distributionAccumulator)
|
||||||
|
browserAcc := make(distributionAccumulator)
|
||||||
|
osAcc := make(distributionAccumulator)
|
||||||
|
for _, row := range uaRows {
|
||||||
|
ua := row.Key
|
||||||
|
count := row.Value
|
||||||
|
deviceAcc[analyticsrepo.ParseDeviceType(ua)] += count
|
||||||
|
browserAcc[analyticsrepo.ParseBrowserName(ua)] += count
|
||||||
|
osAcc[analyticsrepo.ParseOSName(ua)] += count
|
||||||
|
}
|
||||||
|
|
||||||
|
topUserAgents = make([]DistributionItem, 0, accessLogOverviewTopLimit)
|
||||||
|
for _, row := range uaRows {
|
||||||
|
if len(topUserAgents) >= accessLogOverviewTopLimit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
topUserAgents = append(topUserAgents, row)
|
||||||
|
}
|
||||||
|
|
||||||
|
return toDistributionItems(deviceAcc, 0),
|
||||||
|
toDistributionItems(browserAcc, accessLogOverviewTopLimit),
|
||||||
|
toDistributionItems(osAcc, accessLogOverviewTopLimit),
|
||||||
|
topUserAgents
|
||||||
|
}
|
||||||
|
|
||||||
func buildAccessLogOverviewTrends(
|
func buildAccessLogOverviewTrends(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
now time.Time,
|
now time.Time,
|
||||||
@@ -394,6 +447,7 @@ func ListAccessLogs(ctx context.Context, input AccessLogQuery) (*AccessLogList,
|
|||||||
Region: item.Region,
|
Region: item.Region,
|
||||||
Host: item.Host,
|
Host: item.Host,
|
||||||
Path: item.Path,
|
Path: item.Path,
|
||||||
|
UserAgent: item.UserAgent,
|
||||||
StatusCode: item.StatusCode,
|
StatusCode: item.StatusCode,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
-- +goose Up
|
||||||
|
ALTER TABLE of_node_access_logs
|
||||||
|
ADD COLUMN IF NOT EXISTS user_agent String DEFAULT '';
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS user_agent;
|
||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
nodeAccessLogTableName = "of_node_access_logs"
|
nodeAccessLogTableName = "of_node_access_logs"
|
||||||
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, request_length, request_time_ms, created_at"
|
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, user_agent, status_code, bytes_sent, request_length, request_time_ms, created_at"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NodeAccessLog stores OpenFlare edge node access records in ClickHouse.
|
// NodeAccessLog stores OpenFlare edge node access records in ClickHouse.
|
||||||
@@ -22,6 +22,7 @@ type NodeAccessLog struct {
|
|||||||
Region string `gorm:"column:region"`
|
Region string `gorm:"column:region"`
|
||||||
Host string `gorm:"column:host"`
|
Host string `gorm:"column:host"`
|
||||||
Path string `gorm:"column:path"`
|
Path string `gorm:"column:path"`
|
||||||
|
UserAgent string `gorm:"column:user_agent"`
|
||||||
StatusCode int32 `gorm:"column:status_code"`
|
StatusCode int32 `gorm:"column:status_code"`
|
||||||
BytesSent uint64 `gorm:"column:bytes_sent"`
|
BytesSent uint64 `gorm:"column:bytes_sent"`
|
||||||
RequestLength uint64 `gorm:"column:request_length"`
|
RequestLength uint64 `gorm:"column:request_length"`
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ func TrafficSummaryOpenFlareAccessLogs(ctx context.Context, query OpenFlareAcces
|
|||||||
return currentAccessLogStore().TrafficSummary(ctx, query)
|
return currentAccessLogStore().TrafficSummary(ctx, query)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValueCountsOpenFlareAccessLogs groups logs by status_code, host, path, or remote_addr.
|
// ValueCountsOpenFlareAccessLogs groups logs by status_code, host, path, remote_addr, or user_agent.
|
||||||
func ValueCountsOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery, column string, limit int) ([]OpenFlareAccessLogValueCount, error) {
|
func ValueCountsOpenFlareAccessLogs(ctx context.Context, query OpenFlareAccessLogQuery, column string, limit int) ([]OpenFlareAccessLogValueCount, error) {
|
||||||
return currentAccessLogStore().ValueCounts(ctx, query, column, limit)
|
return currentAccessLogStore().ValueCounts(ctx, query, column, limit)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -284,6 +284,7 @@ func toAnalyticsNodeAccessLog(record *OpenFlareAccessLog) analyticsmodel.NodeAcc
|
|||||||
Region: record.Region,
|
Region: record.Region,
|
||||||
Host: record.Host,
|
Host: record.Host,
|
||||||
Path: record.Path,
|
Path: record.Path,
|
||||||
|
UserAgent: record.UserAgent,
|
||||||
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
|
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
|
||||||
BytesSent: bytesSent,
|
BytesSent: bytesSent,
|
||||||
RequestLength: requestLength,
|
RequestLength: requestLength,
|
||||||
@@ -315,6 +316,7 @@ func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*OpenFla
|
|||||||
Region: row.Region,
|
Region: row.Region,
|
||||||
Host: row.Host,
|
Host: row.Host,
|
||||||
Path: row.Path,
|
Path: row.Path,
|
||||||
|
UserAgent: row.UserAgent,
|
||||||
StatusCode: int(row.StatusCode),
|
StatusCode: int(row.StatusCode),
|
||||||
BytesSent: bytesSent,
|
BytesSent: bytesSent,
|
||||||
RequestLength: requestLength,
|
RequestLength: requestLength,
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ const (
|
|||||||
accessLogColumnHost = "host"
|
accessLogColumnHost = "host"
|
||||||
accessLogColumnPath = "path"
|
accessLogColumnPath = "path"
|
||||||
accessLogColumnRemoteAddr = "remote_addr"
|
accessLogColumnRemoteAddr = "remote_addr"
|
||||||
|
accessLogColumnUserAgent = "user_agent"
|
||||||
)
|
)
|
||||||
|
|
||||||
type memoryAccessLogStore struct {
|
type memoryAccessLogStore struct {
|
||||||
@@ -476,7 +477,7 @@ func (s *memoryAccessLogStore) ValueCounts(_ context.Context, filter OpenFlareAc
|
|||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
col := strings.TrimSpace(strings.ToLower(column))
|
col := strings.TrimSpace(strings.ToLower(column))
|
||||||
switch col {
|
switch col {
|
||||||
case accessLogColumnStatusCode, accessLogColumnHost, accessLogColumnPath, accessLogColumnRemoteAddr:
|
case accessLogColumnStatusCode, accessLogColumnHost, accessLogColumnPath, accessLogColumnRemoteAddr, accessLogColumnUserAgent:
|
||||||
default:
|
default:
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -493,6 +494,8 @@ func (s *memoryAccessLogStore) ValueCounts(_ context.Context, filter OpenFlareAc
|
|||||||
value = strings.TrimSpace(row.Path)
|
value = strings.TrimSpace(row.Path)
|
||||||
case accessLogColumnRemoteAddr:
|
case accessLogColumnRemoteAddr:
|
||||||
value = strings.TrimSpace(row.RemoteAddr)
|
value = strings.TrimSpace(row.RemoteAddr)
|
||||||
|
case accessLogColumnUserAgent:
|
||||||
|
value = strings.TrimSpace(row.UserAgent)
|
||||||
}
|
}
|
||||||
if value == "" {
|
if value == "" {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ type OpenFlareAccessLog struct {
|
|||||||
Region string `json:"region" gorm:"size:128"`
|
Region string `json:"region" gorm:"size:128"`
|
||||||
Host string `json:"host" gorm:"index;size:255"`
|
Host string `json:"host" gorm:"index;size:255"`
|
||||||
Path string `json:"path" gorm:"size:2048"`
|
Path string `json:"path" gorm:"size:2048"`
|
||||||
|
UserAgent string `json:"user_agent" gorm:"column:user_agent;size:512"`
|
||||||
StatusCode int `json:"status_code" gorm:"index"`
|
StatusCode int `json:"status_code" gorm:"index"`
|
||||||
BytesSent int64 `json:"bytes_sent" gorm:"column:bytes_sent;not null;default:0"`
|
BytesSent int64 `json:"bytes_sent" gorm:"column:bytes_sent;not null;default:0"`
|
||||||
RequestLength int64 `json:"request_length" gorm:"column:request_length;not null;default:0"`
|
RequestLength int64 `json:"request_length" gorm:"column:request_length;not null;default:0"`
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ http {
|
|||||||
fastcgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/fastcgi_temp;
|
fastcgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/fastcgi_temp;
|
||||||
uwsgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/uwsgi_temp;
|
uwsgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/uwsgi_temp;
|
||||||
scgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/scgi_temp;
|
scgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/scgi_temp;
|
||||||
{{OpenRestyConnectionUpgradeMap}}{{OpenRestyDefaultServerBlock}} log_format openflare_json escape=json '{"ts":"$time_iso8601","host":"$host","path":"$request_uri","remote_addr":"$remote_addr","status":$status,"request_time":$request_time,"bytes_sent":$body_bytes_sent,"request_length":$request_length}';
|
{{OpenRestyConnectionUpgradeMap}}{{OpenRestyDefaultServerBlock}} log_format openflare_json escape=json '{"ts":"$time_iso8601","host":"$host","path":"$request_uri","remote_addr":"$remote_addr","status":$status,"request_time":$request_time,"bytes_sent":$body_bytes_sent,"request_length":$request_length,"user_agent":"$http_user_agent"}';
|
||||||
access_log {{OpenRestyAccessLogPath}} openflare_json;
|
access_log {{OpenRestyAccessLogPath}} openflare_json;
|
||||||
sendfile on;
|
sendfile on;
|
||||||
tcp_nopush on;
|
tcp_nopush on;
|
||||||
|
|||||||
@@ -28,7 +28,8 @@ func TestParseBrowserName(t *testing.T) {
|
|||||||
{name: "edge", ua: "Mozilla/5.0 Edg/120.0.0.0", want: "Edge"},
|
{name: "edge", ua: "Mozilla/5.0 Edg/120.0.0.0", want: "Edge"},
|
||||||
{name: "wechat", ua: "MicroMessenger/8.0", want: "WeChat"},
|
{name: "wechat", ua: "MicroMessenger/8.0", want: "WeChat"},
|
||||||
{name: "postman", ua: "PostmanRuntime/7.36.0", want: "Postman"},
|
{name: "postman", ua: "PostmanRuntime/7.36.0", want: "Postman"},
|
||||||
{name: "other", ua: "curl/8.0", want: "Other"},
|
{name: "cli", ua: "curl/8.0", want: "CLI"},
|
||||||
|
{name: "other", ua: "CustomClient/1.0", want: "Other"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -5,26 +5,116 @@ package analytics
|
|||||||
|
|
||||||
import "strings"
|
import "strings"
|
||||||
|
|
||||||
|
const (
|
||||||
|
uaLabelUnknown = "Unknown"
|
||||||
|
uaLabelBot = "Bot"
|
||||||
|
uaLabelOther = "Other"
|
||||||
|
uaTokenBot = "bot"
|
||||||
|
uaTokenAndroid = "android"
|
||||||
|
uaTokenSpider = "spider"
|
||||||
|
uaTokenCrawler = "crawler"
|
||||||
|
)
|
||||||
|
|
||||||
|
type uaMatchRule struct {
|
||||||
|
label string
|
||||||
|
contains []string
|
||||||
|
allOf []string
|
||||||
|
noneOf []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchUARules(uaLower string, rules []uaMatchRule, fallback string) string {
|
||||||
|
if uaLower == "" {
|
||||||
|
return uaLabelUnknown
|
||||||
|
}
|
||||||
|
for _, rule := range rules {
|
||||||
|
matched := false
|
||||||
|
for _, token := range rule.contains {
|
||||||
|
if strings.Contains(uaLower, token) {
|
||||||
|
matched = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !matched && len(rule.allOf) > 0 {
|
||||||
|
matched = true
|
||||||
|
for _, token := range rule.allOf {
|
||||||
|
if !strings.Contains(uaLower, token) {
|
||||||
|
matched = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !matched {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
excluded := false
|
||||||
|
for _, token := range rule.noneOf {
|
||||||
|
if strings.Contains(uaLower, token) {
|
||||||
|
excluded = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if excluded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return rule.label
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
var browserRules = []uaMatchRule{
|
||||||
|
{label: "WeChat", contains: []string{"micromessenger"}},
|
||||||
|
{label: "Postman", contains: []string{"postman"}},
|
||||||
|
{label: "CLI", contains: []string{"curl/", "wget/"}},
|
||||||
|
{label: "Edge", contains: []string{"edg/", "edgios/", "edga/"}},
|
||||||
|
{label: "Opera", contains: []string{"opr/", "opera"}},
|
||||||
|
{label: "Firefox", contains: []string{"firefox", "fxios"}},
|
||||||
|
{label: "Chrome", contains: []string{"crios", "chrome"}, noneOf: []string{"chromium"}},
|
||||||
|
{label: "Chromium", contains: []string{"chromium"}},
|
||||||
|
{label: "Safari", contains: []string{"safari"}},
|
||||||
|
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
var osRules = []uaMatchRule{
|
||||||
|
{label: "Android", contains: []string{uaTokenAndroid}},
|
||||||
|
{label: "iOS", contains: []string{"iphone", "ipad", "ipod", "ios"}},
|
||||||
|
{label: "Windows", contains: []string{"windows"}},
|
||||||
|
{label: "macOS", contains: []string{"mac os x", "macintosh", "macos"}},
|
||||||
|
{label: "Chrome OS", contains: []string{"cros"}},
|
||||||
|
{label: "Linux", contains: []string{"linux"}},
|
||||||
|
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler}},
|
||||||
|
}
|
||||||
|
|
||||||
|
var deviceRules = []uaMatchRule{
|
||||||
|
{
|
||||||
|
label: uaLabelBot,
|
||||||
|
contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp", "curl/", "wget/", "python-requests", "go-http-client", "postman"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Tablet",
|
||||||
|
contains: []string{"ipad", "tablet"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Tablet",
|
||||||
|
allOf: []string{uaTokenAndroid},
|
||||||
|
noneOf: []string{"mobile"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Mobile",
|
||||||
|
contains: []string{"mobi", "iphone", "ipod", uaTokenAndroid},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// ParseBrowserName performs lightweight User-Agent browser identification.
|
// ParseBrowserName performs lightweight User-Agent browser identification.
|
||||||
func ParseBrowserName(ua string) string {
|
func ParseBrowserName(ua string) string {
|
||||||
uaLower := strings.ToLower(ua)
|
return matchUARules(strings.ToLower(ua), browserRules, uaLabelOther)
|
||||||
if strings.Contains(uaLower, "micromessenger") {
|
}
|
||||||
return "WeChat"
|
|
||||||
}
|
// ParseOSName performs lightweight User-Agent OS identification.
|
||||||
if strings.Contains(uaLower, "postman") {
|
func ParseOSName(ua string) string {
|
||||||
return "Postman"
|
return matchUARules(strings.ToLower(ua), osRules, uaLabelOther)
|
||||||
}
|
}
|
||||||
if strings.Contains(uaLower, "edg/") || strings.Contains(uaLower, "edge") {
|
|
||||||
return "Edge"
|
// ParseDeviceType performs lightweight User-Agent device type identification.
|
||||||
}
|
func ParseDeviceType(ua string) string {
|
||||||
if strings.Contains(uaLower, "firefox") {
|
return matchUARules(strings.ToLower(ua), deviceRules, "Desktop")
|
||||||
return "Firefox"
|
|
||||||
}
|
|
||||||
if strings.Contains(uaLower, "chrome") {
|
|
||||||
return "Chrome"
|
|
||||||
}
|
|
||||||
if strings.Contains(uaLower, "safari") {
|
|
||||||
return "Safari"
|
|
||||||
}
|
|
||||||
return "Other"
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]anal
|
|||||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||||
tableName := nodeAccessLogTableName()
|
tableName := nodeAccessLogTableName()
|
||||||
sql := fmt.Sprintf(`
|
sql := fmt.Sprintf(`
|
||||||
SELECT id, node_id, logged_at, remote_addr, region, host, path, status_code, bytes_sent, request_length, request_time_ms, created_at
|
SELECT id, node_id, logged_at, remote_addr, region, host, path, user_agent, status_code, bytes_sent, request_length, request_time_ms, created_at
|
||||||
FROM %s
|
FROM %s
|
||||||
WHERE %s
|
WHERE %s
|
||||||
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
|
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
|
||||||
@@ -55,6 +55,7 @@ ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.
|
|||||||
return scanNodeAccessLogRows(rows)
|
return scanNodeAccessLogRows(rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
|
||||||
func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, error) {
|
func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, error) {
|
||||||
var result []analyticsmodel.NodeAccessLog
|
var result []analyticsmodel.NodeAccessLog
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
@@ -67,6 +68,7 @@ func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, er
|
|||||||
&item.Region,
|
&item.Region,
|
||||||
&item.Host,
|
&item.Host,
|
||||||
&item.Path,
|
&item.Path,
|
||||||
|
&item.UserAgent,
|
||||||
&item.StatusCode,
|
&item.StatusCode,
|
||||||
&item.BytesSent,
|
&item.BytesSent,
|
||||||
&item.RequestLength,
|
&item.RequestLength,
|
||||||
@@ -206,7 +208,7 @@ WHERE %s`, tableName, clause)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
|
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
|
||||||
// Allowed columns: status_code, host, path, remote_addr.
|
// Allowed columns: status_code, host, path, remote_addr, user_agent.
|
||||||
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
|
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
|
||||||
conn, err := nodeAccessLogConn()
|
conn, err := nodeAccessLogConn()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -261,6 +263,8 @@ func nodeAccessLogValueCountExpr(column string) (string, bool) {
|
|||||||
return "trim(" + nodeAccessLogColumnPath + ")", true
|
return "trim(" + nodeAccessLogColumnPath + ")", true
|
||||||
case nodeAccessLogColumnRemoteAddr:
|
case nodeAccessLogColumnRemoteAddr:
|
||||||
return "trim(" + nodeAccessLogColumnRemoteAddr + ")", true
|
return "trim(" + nodeAccessLogColumnRemoteAddr + ")", true
|
||||||
|
case nodeAccessLogColumnUserAgent:
|
||||||
|
return "trim(" + nodeAccessLogColumnUserAgent + ")", true
|
||||||
default:
|
default:
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ const (
|
|||||||
nodeAccessLogColumnStatusCode = "status_code"
|
nodeAccessLogColumnStatusCode = "status_code"
|
||||||
nodeAccessLogColumnHost = "host"
|
nodeAccessLogColumnHost = "host"
|
||||||
nodeAccessLogColumnPath = "path"
|
nodeAccessLogColumnPath = "path"
|
||||||
|
nodeAccessLogColumnUserAgent = "user_agent"
|
||||||
nodeAccessLogColumnLoggedAt = "logged_at"
|
nodeAccessLogColumnLoggedAt = "logged_at"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -49,8 +49,9 @@ func TestBatchInsertNodeAccessLogs_UsesModelBatchSQL(t *testing.T) {
|
|||||||
assert.True(t, mockBatch.sendCalled)
|
assert.True(t, mockBatch.sendCalled)
|
||||||
require.Len(t, mockBatch.rows, 1)
|
require.Len(t, mockBatch.rows, 1)
|
||||||
assert.Equal(t, "node-a", mockBatch.rows[0][1])
|
assert.Equal(t, "node-a", mockBatch.rows[0][1])
|
||||||
require.Len(t, mockBatch.rows[0], 12)
|
require.Len(t, mockBatch.rows[0], 13)
|
||||||
assert.Equal(t, uint64(2048), mockBatch.rows[0][8]) // bytes_sent
|
assert.Equal(t, "", mockBatch.rows[0][7]) // user_agent
|
||||||
assert.Equal(t, uint64(0), mockBatch.rows[0][9]) // request_length
|
assert.Equal(t, uint64(2048), mockBatch.rows[0][9]) // bytes_sent
|
||||||
assert.Equal(t, uint32(0), mockBatch.rows[0][10]) // request_time_ms
|
assert.Equal(t, uint64(0), mockBatch.rows[0][10]) // request_length
|
||||||
|
assert.Equal(t, uint32(0), mockBatch.rows[0][11]) // request_time_ms
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAc
|
|||||||
logItem.Region,
|
logItem.Region,
|
||||||
logItem.Host,
|
logItem.Host,
|
||||||
logItem.Path,
|
logItem.Path,
|
||||||
|
strings.TrimSpace(logItem.UserAgent),
|
||||||
logItem.StatusCode,
|
logItem.StatusCode,
|
||||||
logItem.BytesSent,
|
logItem.BytesSent,
|
||||||
logItem.RequestLength,
|
logItem.RequestLength,
|
||||||
|
|||||||
@@ -159,6 +159,7 @@ ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
|||||||
return scanNodeObsFrpcRows(rows)
|
return scanNodeObsFrpcRows(rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
|
||||||
func scanNodeMetricSnapshotRows(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
func scanNodeMetricSnapshotRows(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||||
var result []analyticsmodel.NodeMetricSnapshot
|
var result []analyticsmodel.NodeMetricSnapshot
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
|
|||||||
@@ -138,6 +138,7 @@ type NodeAccessLog struct {
|
|||||||
RemoteAddr string `json:"remote_addr"`
|
RemoteAddr string `json:"remote_addr"`
|
||||||
Host string `json:"host"`
|
Host string `json:"host"`
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
|
UserAgent string `json:"user_agent,omitempty"`
|
||||||
StatusCode int `json:"status_code"`
|
StatusCode int `json:"status_code"`
|
||||||
BytesSent int64 `json:"bytes_sent"` // body bytes = 已提供数据
|
BytesSent int64 `json:"bytes_sent"` // body bytes = 已提供数据
|
||||||
RequestLength int64 `json:"request_length"` // 接收数据
|
RequestLength int64 `json:"request_length"` // 接收数据
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ http {
|
|||||||
fastcgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/fastcgi_temp;
|
fastcgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/fastcgi_temp;
|
||||||
uwsgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/uwsgi_temp;
|
uwsgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/uwsgi_temp;
|
||||||
scgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/scgi_temp;
|
scgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/scgi_temp;
|
||||||
{{OpenRestyConnectionUpgradeMap}}{{OpenRestyDefaultServerBlock}} log_format openflare_json escape=json '{"ts":"$time_iso8601","host":"$host","path":"$request_uri","remote_addr":"$remote_addr","status":$status,"request_time":$request_time,"bytes_sent":$body_bytes_sent,"request_length":$request_length}';
|
{{OpenRestyConnectionUpgradeMap}}{{OpenRestyDefaultServerBlock}} log_format openflare_json escape=json '{"ts":"$time_iso8601","host":"$host","path":"$request_uri","remote_addr":"$remote_addr","status":$status,"request_time":$request_time,"bytes_sent":$body_bytes_sent,"request_length":$request_length,"user_agent":"$http_user_agent"}';
|
||||||
access_log {{OpenRestyAccessLogPath}} openflare_json;
|
access_log {{OpenRestyAccessLogPath}} openflare_json;
|
||||||
sendfile on;
|
sendfile on;
|
||||||
tcp_nopush on;
|
tcp_nopush on;
|
||||||
|
|||||||
Reference in New Issue
Block a user