#60 GeoIP 共享单例化:消除访问日志 region 解析每批次的 mmdb 重建开销与无界缓存,ctx 贯穿下载路径

Result: {"status":"keep","total_issues":8,"eslint_errors":0,"eslint_problems":0,"eslint_warnings":0,"golint_canonicalheader":0,"golint_errname":0,"golint_errorlint":1,"golint_exhaustive":0,"golint_forcetypeassert":0,"golint_gosec":0,"golint_intrange":0,"golint_modernize":3,"golint_nilnil":3,"golint_perfsprint":0,"golint_prealloc":0,"golint_recvcheck":1,"golint_test_testifylint":0,"golint_test_thelper":0,"golint_test_total":0,"golint_test_usetesting":0,"golint_total":8,"golint_usestdlibvars":0,"golint_vetx_total":0,"golint_wastedassign":0,"measure_s":81,"tsc_errors":0,"vitest_failed":0,"vitest_total":126}
This commit is contained in:
ryan
2026-08-26 13:48:44 +08:00
parent b56f27632a
commit 2b69f4d8d7
6 changed files with 56 additions and 60 deletions
@@ -6,65 +6,58 @@
package agent
import (
"context"
"log/slog"
"net"
"strings"
"sync"
pkggeoip "github.com/Rain-kl/Wavelet/pkg/geoip"
)
var accessLogGeoProviderFactory = func() (pkggeoip.Service, error) {
return pkggeoip.NewMaxMindGeoIPService()
// 共享一个 GeoIP 服务实例:mmdb 打开(mmap + 解析元数据)成本不低,缺文件时还会
// 同步下载,绝不能每个上报批次重建。maxminddb.Reader 并发安全,无需额外加锁。
// 初始化失败不锁存:下一批上报会重试(与旧行为一致)。
var (
sharedAccessLogGeoMu sync.Mutex
sharedAccessLogGeoInstance pkggeoip.Service
)
func sharedAccessLogGeoService(ctx context.Context) pkggeoip.Service {
sharedAccessLogGeoMu.Lock()
defer sharedAccessLogGeoMu.Unlock()
if sharedAccessLogGeoInstance == nil {
service, err := pkggeoip.NewMaxMindGeoIPServiceWithContext(ctx, "", "")
if err != nil {
slog.WarnContext(ctx, "initialize access log geo service failed", "error", err)
return nil
}
sharedAccessLogGeoInstance = service
}
return sharedAccessLogGeoInstance
}
type accessLogRegionResolver struct {
provider pkggeoip.Service
cache map[string]string
}
func newAccessLogRegionResolver() (*accessLogRegionResolver, error) {
provider, err := accessLogGeoProviderFactory()
if err != nil {
return nil, err
}
return &accessLogRegionResolver{
provider: provider,
cache: make(map[string]string),
}, nil
}
func (r *accessLogRegionResolver) Close() {
if r == nil || r.provider == nil {
return
}
if err := r.provider.Close(); err != nil {
slog.Warn("close access log geo provider failed", "error", err)
}
}
func (r *accessLogRegionResolver) Resolve(rawIP string) string {
if r == nil || r.provider == nil {
return ""
}
// resolveAccessLogRegion resolves the region name for an access-log remote address.
// mmdb Lookup 本身是内存映射 trie 查找(微秒级),无需再建应用层 IP 缓存。
// resolveAccessLogRegion resolves the region name for an access-log remote address.
// mmdb Lookup 本身是内存映射 trie 查找(微秒级),无需再建应用层 IP 缓存。
func resolveAccessLogRegion(ctx context.Context, rawIP string) string {
normalizedIP := normalizeAccessLogIP(rawIP)
if normalizedIP == "" {
return ""
}
if cached, ok := r.cache[normalizedIP]; ok {
return cached
}
info, err := r.provider.GetGeoInfo(net.ParseIP(normalizedIP))
if err != nil || info == nil {
r.cache[normalizedIP] = ""
service := sharedAccessLogGeoService(ctx)
if service == nil {
return ""
}
info, err := service.GetGeoInfo(net.ParseIP(normalizedIP))
if err != nil || info == nil {
return ""
}
region := strings.TrimSpace(info.Name)
if region == "" {
region = strings.TrimSpace(info.ISOCode)
}
r.cache[normalizedIP] = region
return region
}
+3 -15
View File
@@ -6,7 +6,6 @@ package agent
import (
"context"
"encoding/json"
"log/slog"
"strings"
"time"
@@ -36,7 +35,7 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
return
}
accessLogRecords, err := buildNodeAccessLogRecords(nodeID, payload.AccessLogs, payload.Buffered, reportedAt)
accessLogRecords, err := buildNodeAccessLogRecords(ctx, nodeID, payload.AccessLogs, payload.Buffered, reportedAt)
if err != nil {
zap.L().Error("build heartbeat access logs failed", zap.String("node_id", nodeID), zap.Error(err))
return
@@ -159,7 +158,7 @@ func persistNodeMetricSnapshot(ctx context.Context, nodeID string, snapshot *Nod
return repository.InsertOpenFlareMetricSnapshot(ctx, record)
}
func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered []BufferedObservabilityRecord, reportedAt time.Time) ([]*model.OpenFlareAccessLog, error) {
func buildNodeAccessLogRecords(ctx context.Context, nodeID string, direct []NodeAccessLog, buffered []BufferedObservabilityRecord, reportedAt time.Time) ([]*model.OpenFlareAccessLog, error) {
total := len(direct)
for _, record := range buffered {
total += len(record.AccessLogs)
@@ -168,14 +167,6 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
return nil, nil
}
resolver, err := newAccessLogRegionResolver()
if err != nil {
slog.Warn("initialize access log geo resolver failed", "node_id", nodeID, "error", err)
}
if resolver != nil {
defer resolver.Close()
}
records := make([]*model.OpenFlareAccessLog, 0, total)
appendLogs := func(logs []NodeAccessLog) {
for _, item := range logs {
@@ -186,7 +177,7 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
NodeID: nodeID,
LoggedAt: timeFromUnix(item.LoggedAtUnix, reportedAt),
RemoteAddr: strings.TrimSpace(item.RemoteAddr),
Region: "",
Region: resolveAccessLogRegion(ctx, item.RemoteAddr),
Host: strings.TrimSpace(item.Host),
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
UserAgent: truncateForDatabase(strings.TrimSpace(item.UserAgent), accessLogUserAgentMaxLength),
@@ -196,9 +187,6 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
RequestLength: requestLength,
RequestTimeMs: requestTimeMs,
}
if resolver != nil {
record.Region = resolver.Resolve(record.RemoteAddr)
}
records = append(records, record)
}
}
@@ -4,6 +4,7 @@
package agent
import (
"context"
"testing"
"time"
)
@@ -11,7 +12,7 @@ import (
func TestBuildNodeAccessLogRecordsPreservesBytesSent(t *testing.T) {
reportedAt := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC)
records, err := buildNodeAccessLogRecords("node-a", []NodeAccessLog{
records, err := buildNodeAccessLogRecords(context.Background(), "node-a", []NodeAccessLog{
{
LoggedAtUnix: reportedAt.Unix(),
RemoteAddr: "203.0.113.10",