diff --git a/atsf_server/model/node_access_log.go b/atsf_server/model/node_access_log.go index 72ef51f0..e5355092 100644 --- a/atsf_server/model/node_access_log.go +++ b/atsf_server/model/node_access_log.go @@ -7,6 +7,7 @@ type NodeAccessLog struct { NodeID string `json:"node_id" gorm:"index;size:64;not null"` LoggedAt time.Time `json:"logged_at" gorm:"index"` RemoteAddr string `json:"remote_addr" gorm:"size:128"` + Region string `json:"region" gorm:"size:128"` Host string `json:"host" gorm:"size:255"` Path string `json:"path" gorm:"size:2048"` StatusCode int `json:"status_code"` @@ -14,6 +15,11 @@ type NodeAccessLog struct { CreatedAt time.Time `json:"created_at"` } +type NodeAccessLogRegionCount struct { + Region string `json:"region"` + Count int64 `json:"count"` +} + func ListNodeAccessLogs(nodeID string, since time.Time, offset int, limit int) (logs []*NodeAccessLog, err error) { query := DB.Order("logged_at desc, id desc") if nodeID != "" { @@ -51,3 +57,21 @@ func CountNodeAccessLogs(nodeID string, since time.Time) (totalRecords int64, to } return totalRecords, totalIPs, nil } + +func ListNodeAccessLogRegionCounts(nodeID string, since time.Time, limit int) (items []*NodeAccessLogRegionCount, err error) { + query := DB.Model(&NodeAccessLog{}). + Select("region as region, count(*) as count"). + Where("region <> ''") + if nodeID != "" { + query = query.Where("node_id = ?", nodeID) + } + if !since.IsZero() { + query = query.Where("logged_at >= ?", since) + } + query = query.Group("region").Order("count desc, region asc") + if limit > 0 { + query = query.Limit(limit) + } + err = query.Scan(&items).Error + return items, err +} diff --git a/atsf_server/service/access_log.go b/atsf_server/service/access_log.go index 498551b2..45afcdf1 100644 --- a/atsf_server/service/access_log.go +++ b/atsf_server/service/access_log.go @@ -17,6 +17,7 @@ type AccessLogView struct { NodeName string `json:"node_name"` LoggedAt time.Time `json:"logged_at"` RemoteAddr string `json:"remote_addr"` + Region string `json:"region"` Host string `json:"host"` Path string `json:"path"` StatusCode int `json:"status_code"` @@ -76,6 +77,7 @@ func ListAccessLogs(nodeID string, page int, pageSize int) (*AccessLogList, erro NodeName: nodeNames[item.NodeID], LoggedAt: item.LoggedAt, RemoteAddr: item.RemoteAddr, + Region: item.Region, Host: item.Host, Path: item.Path, StatusCode: item.StatusCode, diff --git a/atsf_server/service/access_log_region.go b/atsf_server/service/access_log_region.go new file mode 100644 index 00000000..3568689a --- /dev/null +++ b/atsf_server/service/access_log_region.go @@ -0,0 +1,91 @@ +package service + +import ( + "atsflare/utils/geoip" + "log/slog" + "net" + "strings" +) + +var accessLogGeoProviderFactory = func() (geoip.GeoIPService, error) { + return geoip.NewMaxMindGeoIPService() +} + +type accessLogRegionResolver struct { + provider geoip.GeoIPService + cache map[string]string +} + +func newAccessLogRegionResolver() (*accessLogRegionResolver, error) { + provider, err := accessLogGeoProviderFactory() + if err != nil { + return nil, err + } + return &accessLogRegionResolver{ + provider: provider, + cache: make(map[string]string), + }, nil +} + +func (r *accessLogRegionResolver) Close() { + if r == nil || r.provider == nil { + return + } + if err := r.provider.Close(); err != nil { + slog.Warn("close access log geo provider failed", "error", err) + } +} + +func (r *accessLogRegionResolver) Resolve(rawIP string) string { + if r == nil || r.provider == nil { + return "" + } + normalizedIP := normalizeAccessLogIP(rawIP) + if normalizedIP == "" { + return "" + } + if cached, ok := r.cache[normalizedIP]; ok { + return cached + } + + info, err := r.provider.GetGeoInfo(net.ParseIP(normalizedIP)) + if err != nil || info == nil { + r.cache[normalizedIP] = "" + return "" + } + + region := strings.TrimSpace(info.Name) + if region == "" { + region = strings.TrimSpace(info.ISOCode) + } + r.cache[normalizedIP] = region + return region +} + +func normalizeAccessLogIP(raw string) string { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "" + } + + if ip := net.ParseIP(trimmed); ip != nil { + return ip.String() + } + + trimmed = strings.TrimPrefix(trimmed, "[") + trimmed = strings.TrimSuffix(trimmed, "]") + if ip := net.ParseIP(trimmed); ip != nil { + return ip.String() + } + + host, _, err := net.SplitHostPort(strings.TrimSpace(raw)) + if err != nil { + return "" + } + host = strings.TrimPrefix(host, "[") + host = strings.TrimSuffix(host, "]") + if ip := net.ParseIP(host); ip != nil { + return ip.String() + } + return "" +} diff --git a/atsf_server/service/access_log_test.go b/atsf_server/service/access_log_test.go index 862b2848..deebc493 100644 --- a/atsf_server/service/access_log_test.go +++ b/atsf_server/service/access_log_test.go @@ -28,6 +28,7 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) { NodeID: "node-a", LoggedAt: now.Add(-5 * time.Minute), RemoteAddr: "1.1.1.1", + Region: "United States", Host: "a.example.com", Path: "/alpha", StatusCode: 200, @@ -36,6 +37,7 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) { NodeID: "node-a", LoggedAt: now.Add(-4 * time.Minute), RemoteAddr: "2.2.2.2", + Region: "China", Host: "a.example.com", Path: "/beta", StatusCode: 404, @@ -44,6 +46,7 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) { NodeID: "node-b", LoggedAt: now.Add(-3 * time.Minute), RemoteAddr: "1.1.1.1", + Region: "United States", Host: "b.example.com", Path: "/gamma", StatusCode: 502, @@ -74,6 +77,9 @@ func TestListAccessLogsIncludesSummaryTotals(t *testing.T) { if len(result.Items) != 2 { t.Fatalf("expected current page items=2, got %d", len(result.Items)) } + if result.Items[1].Region == "" { + t.Fatalf("expected region to be returned, got %+v", result.Items[1]) + } if !result.HasMore { t.Fatal("expected has_more to be true") } diff --git a/atsf_server/service/dashboard.go b/atsf_server/service/dashboard.go index 552370ea..4a4f05e1 100644 --- a/atsf_server/service/dashboard.go +++ b/atsf_server/service/dashboard.go @@ -84,6 +84,10 @@ func GetDashboardOverview() (*DashboardOverviewView, error) { if err != nil { return nil, err } + accessLogRegions, err := model.ListNodeAccessLogRegionCounts("", since, 8) + if err != nil { + return nil, err + } activeEvents, err := model.ListActiveNodeHealthEvents() if err != nil { return nil, err @@ -92,7 +96,7 @@ func GetDashboardOverview() (*DashboardOverviewView, error) { view := &DashboardOverviewView{ GeneratedAt: now, Nodes: make([]DashboardNodeHealth, 0, len(nodes)), - Distributions: buildTrafficDistributions(reports, 8), + Distributions: buildTrafficDistributions(reports, accessLogRegions, 8), Trends: DashboardTrends{ Traffic24h: buildTrafficTrendPoints(now, reports), Capacity24h: buildCapacityTrendPoints(now, snapshots), diff --git a/atsf_server/service/node_observability.go b/atsf_server/service/node_observability.go index 7eaf3101..f30b5637 100644 --- a/atsf_server/service/node_observability.go +++ b/atsf_server/service/node_observability.go @@ -68,6 +68,10 @@ func GetNodeObservability(id uint, query NodeObservabilityQuery) (*NodeObservabi if err != nil { return nil, err } + accessLogRegions, err := model.ListNodeAccessLogRegionCounts(node.NodeID, since, 8) + if err != nil { + return nil, err + } trendSnapshots, err := model.ListNodeMetricSnapshots(node.NodeID, now.Add(-24*time.Hour), 0) if err != nil { return nil, err @@ -89,7 +93,7 @@ func GetNodeObservability(id uint, query NodeObservabilityQuery) (*NodeObservabi HealthEvents: events, Analytics: NodeObservabilityAnalytics{ Traffic: buildTrafficWindowSummary(latestTrafficReport(reports)), - Distributions: buildTrafficDistributions(reports, 8), + Distributions: buildTrafficDistributions(reports, accessLogRegions, 8), Health: buildObservabilityHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events), }, Trends: NodeObservabilityTrends{ diff --git a/atsf_server/service/node_update_test.go b/atsf_server/service/node_update_test.go index c5e5cad3..7fabaee6 100644 --- a/atsf_server/service/node_update_test.go +++ b/atsf_server/service/node_update_test.go @@ -19,6 +19,10 @@ type fakeGeoIPProvider struct { info *geoip.GeoInfo } +type fakeAccessLogGeoProvider struct { + info *geoip.GeoInfo +} + func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } @@ -39,6 +43,22 @@ func (f *fakeGeoIPProvider) Close() error { return nil } +func (f *fakeAccessLogGeoProvider) Name() string { + return "fake-access-log-geoip" +} + +func (f *fakeAccessLogGeoProvider) GetGeoInfo(ip net.IP) (*geoip.GeoInfo, error) { + return f.info, nil +} + +func (f *fakeAccessLogGeoProvider) UpdateDatabase() error { + return nil +} + +func (f *fakeAccessLogGeoProvider) Close() error { + return nil +} + func withFakeGeoIPProvider(t *testing.T, info *geoip.GeoInfo) { t.Helper() previous := geoip.CurrentProvider @@ -48,6 +68,17 @@ func withFakeGeoIPProvider(t *testing.T, info *geoip.GeoInfo) { }) } +func withFakeAccessLogGeoProvider(t *testing.T, info *geoip.GeoInfo) { + t.Helper() + previous := accessLogGeoProviderFactory + accessLogGeoProviderFactory = func() (geoip.GeoIPService, error) { + return &fakeAccessLogGeoProvider{info: info}, nil + } + t.Cleanup(func() { + accessLogGeoProviderFactory = previous + }) +} + func geoipFloat(value float64) *float64 { return &value } @@ -564,6 +595,10 @@ func TestListNodeViewsDoesNotPersistComputedStatus(t *testing.T) { func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) { setupServiceTestDB(t) + withFakeAccessLogGeoProvider(t, &geoip.GeoInfo{ + ISOCode: "US", + Name: "United States", + }) node := &model.Node{ NodeID: "node-observe-1", @@ -680,6 +715,9 @@ func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) { if len(accessLogs) != 2 || accessLogs[0].Path == "" { t.Fatalf("unexpected access logs: %+v", accessLogs) } + if accessLogs[0].Region == "" || accessLogs[1].Region == "" { + t.Fatalf("expected access log region to persist: %+v", accessLogs) + } events, err := model.ListNodeHealthEvents(node.NodeID, true, 10) if err != nil { @@ -692,6 +730,10 @@ func TestHeartbeatNodePersistsObservabilityPayload(t *testing.T) { func TestHeartbeatNodePersistsBufferedObservabilityPayload(t *testing.T) { setupServiceTestDB(t) + withFakeAccessLogGeoProvider(t, &geoip.GeoInfo{ + ISOCode: "CN", + Name: "China", + }) node := &model.Node{ NodeID: "node-observe-buffered", @@ -787,6 +829,9 @@ func TestHeartbeatNodePersistsBufferedObservabilityPayload(t *testing.T) { if len(accessLogs) != 1 || accessLogs[0].Path != "/buffered" { t.Fatalf("expected buffered access logs to persist, got %+v", accessLogs) } + if accessLogs[0].Region != "China" { + t.Fatalf("expected buffered access log region to persist, got %+v", accessLogs[0]) + } _, err = HeartbeatNode(node, AgentNodePayload{ NodeID: node.NodeID, @@ -858,6 +903,7 @@ func TestListAccessLogsUsesPagination(t *testing.T) { NodeID: node.NodeID, LoggedAt: now.Add(-10 * time.Second), RemoteAddr: "203.0.113.1", + Region: "United States", Host: "example.com", Path: "/one", StatusCode: 200, @@ -866,6 +912,7 @@ func TestListAccessLogsUsesPagination(t *testing.T) { NodeID: node.NodeID, LoggedAt: now.Add(-9 * time.Second), RemoteAddr: "203.0.113.2", + Region: "China", Host: "example.com", Path: "/two", StatusCode: 200, @@ -874,6 +921,7 @@ func TestListAccessLogsUsesPagination(t *testing.T) { NodeID: node.NodeID, LoggedAt: now.Add(-8 * time.Second), RemoteAddr: "203.0.113.3", + Region: "Japan", Host: "example.com", Path: "/three", StatusCode: 502, @@ -892,6 +940,9 @@ func TestListAccessLogsUsesPagination(t *testing.T) { if pageOne.Items[0].Path != "/three" || pageOne.Items[1].Path != "/two" { t.Fatalf("unexpected first page ordering: %+v", pageOne.Items) } + if pageOne.Items[0].Region != "Japan" { + t.Fatalf("expected paged access log region to be returned, got %+v", pageOne.Items[0]) + } pageTwo, err := ListAccessLogs(node.NodeID, 1, 2) if err != nil { @@ -1252,31 +1303,60 @@ func TestGetDashboardOverview(t *testing.T) { } if err := (&model.NodeRequestReport{ - NodeID: "node-dashboard-a", - WindowStartedAt: now.Add(-time.Minute), - WindowEndedAt: now, - RequestCount: 600, - ErrorCount: 6, - UniqueVisitorCount: 120, - StatusCodesJSON: `{"200":570,"502":6,"304":24}`, - TopDomainsJSON: `{"app.example.com":420,"api.example.com":180}`, - SourceCountriesJSON: `{"CN":320,"SG":280}`, + NodeID: "node-dashboard-a", + WindowStartedAt: now.Add(-time.Minute), + WindowEndedAt: now, + RequestCount: 600, + ErrorCount: 6, + UniqueVisitorCount: 120, + StatusCodesJSON: `{"200":570,"502":6,"304":24}`, + TopDomainsJSON: `{"app.example.com":420,"api.example.com":180}`, }).Insert(); err != nil { t.Fatalf("failed to insert node a traffic report: %v", err) } if err := (&model.NodeRequestReport{ - NodeID: "node-dashboard-b", - WindowStartedAt: now.Add(-time.Minute), - WindowEndedAt: now, - RequestCount: 300, - ErrorCount: 30, - UniqueVisitorCount: 80, - StatusCodesJSON: `{"200":240,"500":18,"502":12,"404":30}`, - TopDomainsJSON: `{"app.example.com":140,"edge.example.com":160}`, - SourceCountriesJSON: `{"US":180,"CN":120}`, + NodeID: "node-dashboard-b", + WindowStartedAt: now.Add(-time.Minute), + WindowEndedAt: now, + RequestCount: 300, + ErrorCount: 30, + UniqueVisitorCount: 80, + StatusCodesJSON: `{"200":240,"500":18,"502":12,"404":30}`, + TopDomainsJSON: `{"app.example.com":140,"edge.example.com":160}`, }).Insert(); err != nil { t.Fatalf("failed to insert node b traffic report: %v", err) } + if err := model.DB.Create([]*model.NodeAccessLog{ + { + NodeID: "node-dashboard-a", + LoggedAt: now.Add(-30 * time.Minute), + RemoteAddr: "203.0.113.11", + Region: "China", + Host: "app.example.com", + Path: "/", + StatusCode: 200, + }, + { + NodeID: "node-dashboard-a", + LoggedAt: now.Add(-20 * time.Minute), + RemoteAddr: "203.0.113.12", + Region: "China", + Host: "app.example.com", + Path: "/login", + StatusCode: 200, + }, + { + NodeID: "node-dashboard-b", + LoggedAt: now.Add(-10 * time.Minute), + RemoteAddr: "198.51.100.8", + Region: "United States", + Host: "edge.example.com", + Path: "/edge", + StatusCode: 502, + }, + }).Error; err != nil { + t.Fatalf("failed to seed dashboard access logs: %v", err) + } if err := model.DB.Create(&model.NodeHealthEvent{ NodeID: "node-dashboard-b", @@ -1331,7 +1411,7 @@ func TestGetDashboardOverview(t *testing.T) { if len(view.Distributions.StatusCodes) == 0 || view.Distributions.StatusCodes[0].Key != "200" { t.Fatalf("unexpected dashboard status distributions: %+v", view.Distributions.StatusCodes) } - if len(view.Distributions.SourceCountries) == 0 || view.Distributions.SourceCountries[0].Key != "CN" { + if len(view.Distributions.SourceCountries) == 0 || view.Distributions.SourceCountries[0].Key != "China" { t.Fatalf("unexpected dashboard source distributions: %+v", view.Distributions.SourceCountries) } if len(view.Distributions.TopDomains) == 0 || view.Distributions.TopDomains[0].Key != "app.example.com" { diff --git a/atsf_server/service/observability.go b/atsf_server/service/observability.go index b7cba337..f389f83f 100644 --- a/atsf_server/service/observability.go +++ b/atsf_server/service/observability.go @@ -206,16 +206,27 @@ func persistNodeAccessLogs(tx *gorm.DB, nodeID string, logs []AgentNodeAccessLog if len(logs) == 0 { return nil } + resolver, err := newAccessLogRegionResolver() + if err != nil { + slog.Warn("initialize access log geo resolver failed", "node_id", nodeID, "error", err) + } + if resolver != nil { + defer resolver.Close() + } for _, item := range logs { record := &model.NodeAccessLog{ NodeID: nodeID, LoggedAt: timeFromUnix(item.LoggedAtUnix, reportedAt), RemoteAddr: strings.TrimSpace(item.RemoteAddr), + Region: "", Host: strings.TrimSpace(item.Host), Path: strings.TrimSpace(item.Path), StatusCode: item.StatusCode, RawJSON: marshalJSON(item), } + if resolver != nil { + record.Region = resolver.Resolve(record.RemoteAddr) + } if err := tx.Where( "node_id = ? AND logged_at = ? AND remote_addr = ? AND host = ? AND path = ? AND status_code = ?", nodeID, diff --git a/atsf_server/service/observability_analytics.go b/atsf_server/service/observability_analytics.go index b045631a..9ee096a0 100644 --- a/atsf_server/service/observability_analytics.go +++ b/atsf_server/service/observability_analytics.go @@ -62,14 +62,28 @@ func buildTrafficWindowSummary(report *model.NodeRequestReport) TrafficWindowSum return summary } -func buildTrafficDistributions(reports []*model.NodeRequestReport, limit int) TrafficDistributions { +func buildTrafficDistributions( + reports []*model.NodeRequestReport, + accessLogRegions []*model.NodeAccessLogRegionCount, + limit int, +) TrafficDistributions { statusCodes := make(distributionAccumulator) topDomains := make(distributionAccumulator) - sourceCountries := make(distributionAccumulator) + reportSourceCountries := make(distributionAccumulator) for _, report := range reports { mergeJSONCounts(statusCodes, report.StatusCodesJSON) mergeJSONCounts(topDomains, report.TopDomainsJSON) - mergeJSONCounts(sourceCountries, report.SourceCountriesJSON) + mergeJSONCounts(reportSourceCountries, report.SourceCountriesJSON) + } + sourceCountries := reportSourceCountries + if len(accessLogRegions) > 0 { + sourceCountries = make(distributionAccumulator, len(accessLogRegions)) + for _, item := range accessLogRegions { + if item == nil || strings.TrimSpace(item.Region) == "" || item.Count <= 0 { + continue + } + sourceCountries[item.Region] = item.Count + } } return TrafficDistributions{ StatusCodes: toDistributionItems(statusCodes, limit), diff --git a/atsf_server/web/features/access-logs/components/access-logs-page.tsx b/atsf_server/web/features/access-logs/components/access-logs-page.tsx index f1e6b7f9..71450b9b 100644 --- a/atsf_server/web/features/access-logs/components/access-logs-page.tsx +++ b/atsf_server/web/features/access-logs/components/access-logs-page.tsx @@ -69,7 +69,7 @@ export function AccessLogsPage() {
- 控制世界地图等场景使用的 IP 归属解析来源。选择 MaxMind - 时会按需下载本地 mmdb 数据库。 + 控制节点地图等场景使用的 IP + 归属解析来源。访客访问记录归属地入库固定使用 MaxMind + mmdb。