[优化] 添加访问日志地区解析功能,更新相关数据结构和前端展示

This commit is contained in:
ryan
2026-03-15 15:21:00 +08:00
parent 640dd6c82c
commit 7c89ad8c7b
14 changed files with 278 additions and 34 deletions
+24
View File
@@ -7,6 +7,7 @@ type NodeAccessLog struct {
NodeID string `json:"node_id" gorm:"index;size:64;not null"`
LoggedAt time.Time `json:"logged_at" gorm:"index"`
RemoteAddr string `json:"remote_addr" gorm:"size:128"`
Region string `json:"region" gorm:"size:128"`
Host string `json:"host" gorm:"size:255"`
Path string `json:"path" gorm:"size:2048"`
StatusCode int `json:"status_code"`
@@ -14,6 +15,11 @@ type NodeAccessLog struct {
CreatedAt time.Time `json:"created_at"`
}
type NodeAccessLogRegionCount struct {
Region string `json:"region"`
Count int64 `json:"count"`
}
func ListNodeAccessLogs(nodeID string, since time.Time, offset int, limit int) (logs []*NodeAccessLog, err error) {
query := DB.Order("logged_at desc, id desc")
if nodeID != "" {
@@ -51,3 +57,21 @@ func CountNodeAccessLogs(nodeID string, since time.Time) (totalRecords int64, to
}
return totalRecords, totalIPs, nil
}
func ListNodeAccessLogRegionCounts(nodeID string, since time.Time, limit int) (items []*NodeAccessLogRegionCount, err error) {
query := DB.Model(&NodeAccessLog{}).
Select("region as region, count(*) as count").
Where("region <> ''")
if nodeID != "" {
query = query.Where("node_id = ?", nodeID)
}
if !since.IsZero() {
query = query.Where("logged_at >= ?", since)
}
query = query.Group("region").Order("count desc, region asc")
if limit > 0 {
query = query.Limit(limit)
}
err = query.Scan(&items).Error
return items, err
}
+2
View File
@@ -17,6 +17,7 @@ type AccessLogView struct {
NodeName string `json:"node_name"`
LoggedAt time.Time `json:"logged_at"`
RemoteAddr string `json:"remote_addr"`
Region string `json:"region"`
Host string `json:"host"`
Path string `json:"path"`
StatusCode int `json:"status_code"`
@@ -76,6 +77,7 @@ func ListAccessLogs(nodeID string, page int, pageSize int) (*AccessLogList, erro
NodeName: nodeNames[item.NodeID],
LoggedAt: item.LoggedAt,
RemoteAddr: item.RemoteAddr,
Region: item.Region,
Host: item.Host,
Path: item.Path,
StatusCode: item.StatusCode,
+91
View File
@@ -0,0 +1,91 @@
package service
import (
"atsflare/utils/geoip"
"log/slog"
"net"
"strings"
)
var accessLogGeoProviderFactory = func() (geoip.GeoIPService, error) {
return geoip.NewMaxMindGeoIPService()
}
type accessLogRegionResolver struct {
provider geoip.GeoIPService
cache map[string]string
}
func newAccessLogRegionResolver() (*accessLogRegionResolver, error) {
provider, err := accessLogGeoProviderFactory()
if err != nil {
return nil, err
}
return &accessLogRegionResolver{
provider: provider,
cache: make(map[string]string),
}, nil
}
func (r *accessLogRegionResolver) Close() {
if r == nil || r.provider == nil {
return
}
if err := r.provider.Close(); err != nil {
slog.Warn("close access log geo provider failed", "error", err)
}
}
func (r *accessLogRegionResolver) Resolve(rawIP string) string {
if r == nil || r.provider == nil {
return ""
}
normalizedIP := normalizeAccessLogIP(rawIP)
if normalizedIP == "" {
return ""
}
if cached, ok := r.cache[normalizedIP]; ok {
return cached
}
info, err := r.provider.GetGeoInfo(net.ParseIP(normalizedIP))
if err != nil || info == nil {
r.cache[normalizedIP] = ""
return ""
}
region := strings.TrimSpace(info.Name)
if region == "" {
region = strings.TrimSpace(info.ISOCode)
}
r.cache[normalizedIP] = region
return region
}
func normalizeAccessLogIP(raw string) string {
trimmed := strings.TrimSpace(raw)
if trimmed == "" {
return ""
}
if ip := net.ParseIP(trimmed); ip != nil {
return ip.String()
}
trimmed = strings.TrimPrefix(trimmed, "[")
trimmed = strings.TrimSuffix(trimmed, "]")
if ip := net.ParseIP(trimmed); ip != nil {
return ip.String()
}
host, _, err := net.SplitHostPort(strings.TrimSpace(raw))
if err != nil {
return ""
}
host = strings.TrimPrefix(host, "[")
host = strings.TrimSuffix(host, "]")
if ip := net.ParseIP(host); ip != nil {
return ip.String()
}
return ""
}
+6
View File
@@ -28,6 +28,7 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) {
NodeID: "node-a",
LoggedAt: now.Add(-5 * time.Minute),
RemoteAddr: "1.1.1.1",
Region: "United States",
Host: "a.example.com",
Path: "/alpha",
StatusCode: 200,
@@ -36,6 +37,7 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) {
NodeID: "node-a",
LoggedAt: now.Add(-4 * time.Minute),
RemoteAddr: "2.2.2.2",
Region: "China",
Host: "a.example.com",
Path: "/beta",
StatusCode: 404,
@@ -44,6 +46,7 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) {
NodeID: "node-b",
LoggedAt: now.Add(-3 * time.Minute),
RemoteAddr: "1.1.1.1",
Region: "United States",
Host: "b.example.com",
Path: "/gamma",
StatusCode: 502,
@@ -74,6 +77,9 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) {
if len(result.Items) != 2 {
t.Fatalf("expected current page items=2, got %d", len(result.Items))
}
if result.Items[1].Region == "" {
t.Fatalf("expected region to be returned, got %+v", result.Items[1])
}
if !result.HasMore {
t.Fatal("expected has_more to be true")
}
+5 -1
View File
@@ -84,6 +84,10 @@ func GetDashboardOverview() (*DashboardOverviewView, error) {
if err != nil {
return nil, err
}
accessLogRegions, err := model.ListNodeAccessLogRegionCounts("", since, 8)
if err != nil {
return nil, err
}
activeEvents, err := model.ListActiveNodeHealthEvents()
if err != nil {
return nil, err
@@ -92,7 +96,7 @@ func GetDashboardOverview() (*DashboardOverviewView, error) {
view := &DashboardOverviewView{
GeneratedAt: now,
Nodes: make([]DashboardNodeHealth, 0, len(nodes)),
Distributions: buildTrafficDistributions(reports, 8),
Distributions: buildTrafficDistributions(reports, accessLogRegions, 8),
Trends: DashboardTrends{
Traffic24h: buildTrafficTrendPoints(now, reports),
Capacity24h: buildCapacityTrendPoints(now, snapshots),
+5 -1
View File
@@ -68,6 +68,10 @@ func GetNodeObservability(id uint, query NodeObservabilityQuery) (*NodeObservabi
if err != nil {
return nil, err
}
accessLogRegions, err := model.ListNodeAccessLogRegionCounts(node.NodeID, since, 8)
if err != nil {
return nil, err
}
trendSnapshots, err := model.ListNodeMetricSnapshots(node.NodeID, now.Add(-24*time.Hour), 0)
if err != nil {
return nil, err
@@ -89,7 +93,7 @@ func GetNodeObservability(id uint, query NodeObservabilityQuery) (*NodeObservabi
HealthEvents: events,
Analytics: NodeObservabilityAnalytics{
Traffic: buildTrafficWindowSummary(latestTrafficReport(reports)),
Distributions: buildTrafficDistributions(reports, 8),
Distributions: buildTrafficDistributions(reports, accessLogRegions, 8),
Health: buildObservabilityHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events),
},
Trends: NodeObservabilityTrends{
+99 -19
View File
@@ -19,6 +19,10 @@ type fakeGeoIPProvider struct {
info *geoip.GeoInfo
}
type fakeAccessLogGeoProvider struct {
info *geoip.GeoInfo
}
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
@@ -39,6 +43,22 @@ func (f *fakeGeoIPProvider) Close() error {
return nil
}
func (f *fakeAccessLogGeoProvider) Name() string {
return "fake-access-log-geoip"
}
func (f *fakeAccessLogGeoProvider) GetGeoInfo(ip net.IP) (*geoip.GeoInfo, error) {
return f.info, nil
}
func (f *fakeAccessLogGeoProvider) UpdateDatabase() error {
return nil
}
func (f *fakeAccessLogGeoProvider) Close() error {
return nil
}
func withFakeGeoIPProvider(t *testing.T, info *geoip.GeoInfo) {
t.Helper()
previous := geoip.CurrentProvider
@@ -48,6 +68,17 @@ func withFakeGeoIPProvider(t *testing.T, info *geoip.GeoInfo) {
})
}
func withFakeAccessLogGeoProvider(t *testing.T, info *geoip.GeoInfo) {
t.Helper()
previous := accessLogGeoProviderFactory
accessLogGeoProviderFactory = func() (geoip.GeoIPService, error) {
return &fakeAccessLogGeoProvider{info: info}, nil
}
t.Cleanup(func() {
accessLogGeoProviderFactory = previous
})
}
func geoipFloat(value float64) *float64 {
return &value
}
@@ -564,6 +595,10 @@ func TestListNodeViewsDoesNotPersistComputedStatus(t *testing.T) {
func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) {
setupServiceTestDB(t)
withFakeAccessLogGeoProvider(t, &geoip.GeoInfo{
ISOCode: "US",
Name: "United States",
})
node := &model.Node{
NodeID: "node-observe-1",
@@ -680,6 +715,9 @@ func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) {
if len(accessLogs) != 2 || accessLogs[0].Path == "" {
t.Fatalf("unexpected access logs: %+v", accessLogs)
}
if accessLogs[0].Region == "" || accessLogs[1].Region == "" {
t.Fatalf("expected access log region to persist: %+v", accessLogs)
}
events, err := model.ListNodeHealthEvents(node.NodeID, true, 10)
if err != nil {
@@ -692,6 +730,10 @@ func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) {
func TestHeartbeatNodePersistsBufferedObservabilityPayload(t *testing.T) {
setupServiceTestDB(t)
withFakeAccessLogGeoProvider(t, &geoip.GeoInfo{
ISOCode: "CN",
Name: "China",
})
node := &model.Node{
NodeID: "node-observe-buffered",
@@ -787,6 +829,9 @@ func TestHeartbeatNodePersistsBufferedObservabilityPayload(t *testing.T) {
if len(accessLogs) != 1 || accessLogs[0].Path != "/buffered" {
t.Fatalf("expected buffered access logs to persist, got %+v", accessLogs)
}
if accessLogs[0].Region != "China" {
t.Fatalf("expected buffered access log region to persist, got %+v", accessLogs[0])
}
_, err = HeartbeatNode(node, AgentNodePayload{
NodeID: node.NodeID,
@@ -858,6 +903,7 @@ func TestListAccessLogsUsesPagination(t *testing.T) {
NodeID: node.NodeID,
LoggedAt: now.Add(-10 * time.Second),
RemoteAddr: "203.0.113.1",
Region: "United States",
Host: "example.com",
Path: "/one",
StatusCode: 200,
@@ -866,6 +912,7 @@ func TestListAccessLogsUsesPagination(t *testing.T) {
NodeID: node.NodeID,
LoggedAt: now.Add(-9 * time.Second),
RemoteAddr: "203.0.113.2",
Region: "China",
Host: "example.com",
Path: "/two",
StatusCode: 200,
@@ -874,6 +921,7 @@ func TestListAccessLogsUsesPagination(t *testing.T) {
NodeID: node.NodeID,
LoggedAt: now.Add(-8 * time.Second),
RemoteAddr: "203.0.113.3",
Region: "Japan",
Host: "example.com",
Path: "/three",
StatusCode: 502,
@@ -892,6 +940,9 @@ func TestListAccessLogsUsesPagination(t *testing.T) {
if pageOne.Items[0].Path != "/three" || pageOne.Items[1].Path != "/two" {
t.Fatalf("unexpected first page ordering: %+v", pageOne.Items)
}
if pageOne.Items[0].Region != "Japan" {
t.Fatalf("expected paged access log region to be returned, got %+v", pageOne.Items[0])
}
pageTwo, err := ListAccessLogs(node.NodeID, 1, 2)
if err != nil {
@@ -1252,31 +1303,60 @@ func TestGetDashboardOverview(t *testing.T) {
}
if err := (&model.NodeRequestReport{
NodeID: "node-dashboard-a",
WindowStartedAt: now.Add(-time.Minute),
WindowEndedAt: now,
RequestCount: 600,
ErrorCount: 6,
UniqueVisitorCount: 120,
StatusCodesJSON: `{"200":570,"502":6,"304":24}`,
TopDomainsJSON: `{"app.example.com":420,"api.example.com":180}`,
SourceCountriesJSON: `{"CN":320,"SG":280}`,
NodeID: "node-dashboard-a",
WindowStartedAt: now.Add(-time.Minute),
WindowEndedAt: now,
RequestCount: 600,
ErrorCount: 6,
UniqueVisitorCount: 120,
StatusCodesJSON: `{"200":570,"502":6,"304":24}`,
TopDomainsJSON: `{"app.example.com":420,"api.example.com":180}`,
}).Insert(); err != nil {
t.Fatalf("failed to insert node a traffic report: %v", err)
}
if err := (&model.NodeRequestReport{
NodeID: "node-dashboard-b",
WindowStartedAt: now.Add(-time.Minute),
WindowEndedAt: now,
RequestCount: 300,
ErrorCount: 30,
UniqueVisitorCount: 80,
StatusCodesJSON: `{"200":240,"500":18,"502":12,"404":30}`,
TopDomainsJSON: `{"app.example.com":140,"edge.example.com":160}`,
SourceCountriesJSON: `{"US":180,"CN":120}`,
NodeID: "node-dashboard-b",
WindowStartedAt: now.Add(-time.Minute),
WindowEndedAt: now,
RequestCount: 300,
ErrorCount: 30,
UniqueVisitorCount: 80,
StatusCodesJSON: `{"200":240,"500":18,"502":12,"404":30}`,
TopDomainsJSON: `{"app.example.com":140,"edge.example.com":160}`,
}).Insert(); err != nil {
t.Fatalf("failed to insert node b traffic report: %v", err)
}
if err := model.DB.Create([]*model.NodeAccessLog{
{
NodeID: "node-dashboard-a",
LoggedAt: now.Add(-30 * time.Minute),
RemoteAddr: "203.0.113.11",
Region: "China",
Host: "app.example.com",
Path: "/",
StatusCode: 200,
},
{
NodeID: "node-dashboard-a",
LoggedAt: now.Add(-20 * time.Minute),
RemoteAddr: "203.0.113.12",
Region: "China",
Host: "app.example.com",
Path: "/login",
StatusCode: 200,
},
{
NodeID: "node-dashboard-b",
LoggedAt: now.Add(-10 * time.Minute),
RemoteAddr: "198.51.100.8",
Region: "United States",
Host: "edge.example.com",
Path: "/edge",
StatusCode: 502,
},
}).Error; err != nil {
t.Fatalf("failed to seed dashboard access logs: %v", err)
}
if err := model.DB.Create(&model.NodeHealthEvent{
NodeID: "node-dashboard-b",
@@ -1331,7 +1411,7 @@ func TestGetDashboardOverview(t *testing.T) {
if len(view.Distributions.StatusCodes) == 0 || view.Distributions.StatusCodes[0].Key != "200" {
t.Fatalf("unexpected dashboard status distributions: %+v", view.Distributions.StatusCodes)
}
if len(view.Distributions.SourceCountries) == 0 || view.Distributions.SourceCountries[0].Key != "CN" {
if len(view.Distributions.SourceCountries) == 0 || view.Distributions.SourceCountries[0].Key != "China" {
t.Fatalf("unexpected dashboard source distributions: %+v", view.Distributions.SourceCountries)
}
if len(view.Distributions.TopDomains) == 0 || view.Distributions.TopDomains[0].Key != "app.example.com" {
+11
View File
@@ -206,16 +206,27 @@ func persistNodeAccessLogs(tx *gorm.DB, nodeID string, logs []AgentNodeAccessLog
if len(logs) == 0 {
return nil
}
resolver, err := newAccessLogRegionResolver()
if err != nil {
slog.Warn("initialize access log geo resolver failed", "node_id", nodeID, "error", err)
}
if resolver != nil {
defer resolver.Close()
}
for _, item := range logs {
record := &model.NodeAccessLog{
NodeID: nodeID,
LoggedAt: timeFromUnix(item.LoggedAtUnix, reportedAt),
RemoteAddr: strings.TrimSpace(item.RemoteAddr),
Region: "",
Host: strings.TrimSpace(item.Host),
Path: strings.TrimSpace(item.Path),
StatusCode: item.StatusCode,
RawJSON: marshalJSON(item),
}
if resolver != nil {
record.Region = resolver.Resolve(record.RemoteAddr)
}
if err := tx.Where(
"node_id = ? AND logged_at = ? AND remote_addr = ? AND host = ? AND path = ? AND status_code = ?",
nodeID,
+17 -3
View File
@@ -62,14 +62,28 @@ func buildTrafficWindowSummary(report *model.NodeRequestReport) TrafficWindowSum
return summary
}
func buildTrafficDistributions(reports []*model.NodeRequestReport, limit int) TrafficDistributions {
func buildTrafficDistributions(
reports []*model.NodeRequestReport,
accessLogRegions []*model.NodeAccessLogRegionCount,
limit int,
) TrafficDistributions {
statusCodes := make(distributionAccumulator)
topDomains := make(distributionAccumulator)
sourceCountries := make(distributionAccumulator)
reportSourceCountries := make(distributionAccumulator)
for _, report := range reports {
mergeJSONCounts(statusCodes, report.StatusCodesJSON)
mergeJSONCounts(topDomains, report.TopDomainsJSON)
mergeJSONCounts(sourceCountries, report.SourceCountriesJSON)
mergeJSONCounts(reportSourceCountries, report.SourceCountriesJSON)
}
sourceCountries := reportSourceCountries
if len(accessLogRegions) > 0 {
sourceCountries = make(distributionAccumulator, len(accessLogRegions))
for _, item := range accessLogRegions {
if item == nil || strings.TrimSpace(item.Region) == "" || item.Count <= 0 {
continue
}
sourceCountries[item.Region] = item.Count
}
}
return TrafficDistributions{
StatusCodes: toDistributionItems(statusCodes, limit),
@@ -69,7 +69,7 @@ export function AccessLogsPage() {
<div className="space-y-6">
<PageHeader
title="日志"
description="查看最近时间窗口内的访问记录,包含来源 IP、访问域名、路径、命中节点与响应状态码。"
description="查看最近时间窗口内的访问记录,包含来源 IP、归属地、访问域名、路径、命中节点与响应状态码。"
action={
<Link
href="/node"
@@ -185,7 +185,14 @@ export function AccessLogsPage() {
</div>
</td>
<td className="px-3 py-4 font-medium text-[var(--foreground-primary)]">
{item.remote_addr || '—'}
<div>{item.remote_addr || '—'}</div>
{item.region ? (
<div className="mt-2">
<span className="inline-flex rounded-full border border-[var(--border-default)] bg-[var(--surface-elevated)] px-2.5 py-1 text-[11px] font-medium text-[var(--foreground-secondary)]">
{item.region}
</span>
</div>
) : null}
</td>
<td className="px-3 py-4 text-[var(--foreground-secondary)]">
{item.host || '—'}
@@ -4,6 +4,7 @@ export interface AccessLogItem {
node_name: string;
logged_at: string;
remote_addr: string;
region: string;
host: string;
path: string;
status_code: number;
@@ -1054,14 +1054,15 @@ export function SettingsPage() {
IP 归属方式
</p>
<p className="text-sm text-[var(--foreground-muted)]">
控制世界地图等场景使用的 IP 归属解析来源。选择 MaxMind
时会按需下载本地 mmdb 数据库。
控制节点地图等场景使用的 IP
归属解析来源。访客访问记录归属地入库固定使用 MaxMind
mmdb。
</p>
</div>
<div className="mt-4">
<ResourceField
label="归属方式"
hint="disabled 关闭解析;mmdb 使用本地数据库;其余选项调用外部 GeoIP 服务。"
hint="disabled 关闭节点归属解析;mmdb 使用本地数据库;其余选项调用外部 GeoIP 服务。"
>
<ResourceSelect
value={operationFields.GeoIPProvider}
@@ -1082,9 +1083,7 @@ export function SettingsPage() {
</div>
<div className="mt-5 rounded-2xl border border-[var(--border-default)] bg-[var(--surface-elevated)] p-5">
<div className="flex flex-col gap-3 lg:flex-row lg:items-end">
<ResourceField
label="测试 IP"
>
<ResourceField label="测试 IP">
<ResourceInput
value={geoIPTestIP}
onChange={(event) =>
+2 -1
View File
@@ -111,7 +111,7 @@ volumes:
| `AgentHeartbeatInterval` | Agent 心跳间隔(毫秒) | `10000` |
| `NodeOfflineThreshold` | 节点离线判定阈值(毫秒) | `120000` |
| `AgentUpdateRepo` | Agent 自更新仓库 | `Rain-kl/ATSFlare` |
| `GeoIPProvider` | IP 归属解析方式;支持 `disabled`、`mmdb`、`ip-api`、`geojs`、`ipinfo` | `ipinfo` |
| `GeoIPProvider` | 节点/IP 归属解析方式;支持 `disabled`、`mmdb`、`ip-api`、`geojs`、`ipinfo` | `ipinfo` |
| `GlobalApiRateLimitNum` / `GlobalApiRateLimitDuration` | 全局 API 限流次数 / 时间窗口(秒) | `300` / `180` |
| `GlobalWebRateLimitNum` / `GlobalWebRateLimitDuration` | 全局 Web 限流次数 / 时间窗口(秒) | `300` / `180` |
| `UploadRateLimitNum` / `UploadRateLimitDuration` | 上传接口限流次数 / 时间窗口(秒) | `50` / `60` |
@@ -123,6 +123,7 @@ volumes:
* 限流窗口上限不能超过 `RateLimitKeyExpirationDuration`,当前为 20 分钟
* 限流按来源 IP 统计,若前置了 Nginx/CDN/LB,应正确透传真实客户端 IP
* `GeoIPProvider=mmdb` 时,Server 会按需下载并使用本地 MaxMind Country 数据库;其余非 `disabled` 选项会直接请求对应外部 GeoIP 服务
* 管理端“日志 -> 访问记录”中的访客 IP 归属地入库固定使用 MaxMind mmdb,不跟随 `GeoIPProvider` 切换
### 1.2.2 第五版当前支持的 OpenResty 优化配置项
+1 -1
View File
@@ -204,7 +204,7 @@ ATSFlare Agent (register / heartbeat / sync / apply / update)
* 节点请求数据必须随 heartbeat 按批次上报,Server 不主动反向拉取节点日志文件
* 指标看板使用服务端聚合结果,不在前端重复做大规模统计计算
* 节点资源快照与请求明细必须能按时间排序并绑定节点,保证 24 小时趋势与节点详情可回放
* 管理端日志页只展示受控保留窗口内的必要访问字段,如原始来源 IP、访问域名、路径、命中的节点与响应状态码,不演变为通用日志检索平台
* 管理端日志页只展示受控保留窗口内的必要访问字段,如原始来源 IP、来源地区、访问域名、路径、命中的节点与响应状态码,不演变为通用日志检索平台
* 原始请求明细、聚合统计与节点基础状态必须在时间窗口上可对齐
* 首页总览的系统状态必须基于统一服务端口径生成,不能由多个历史列表接口在前端临时拼装推导
* 健康事件必须支持“触发中/已恢复”状态,避免首页异常永远累积