mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-02 23:06:36 +08:00
#60 GeoIP 共享单例化:消除访问日志 region 解析每批次的 mmdb 重建开销与无界缓存,ctx 贯穿下载路径
Result: {"status":"keep","total_issues":8,"eslint_errors":0,"eslint_problems":0,"eslint_warnings":0,"golint_canonicalheader":0,"golint_errname":0,"golint_errorlint":1,"golint_exhaustive":0,"golint_forcetypeassert":0,"golint_gosec":0,"golint_intrange":0,"golint_modernize":3,"golint_nilnil":3,"golint_perfsprint":0,"golint_prealloc":0,"golint_recvcheck":1,"golint_test_testifylint":0,"golint_test_thelper":0,"golint_test_total":0,"golint_test_usetesting":0,"golint_total":8,"golint_usestdlibvars":0,"golint_vetx_total":0,"golint_wastedassign":0,"measure_s":81,"tsc_errors":0,"vitest_failed":0,"vitest_total":126}
This commit is contained in:
@@ -6,65 +6,58 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
pkggeoip "github.com/Rain-kl/Wavelet/pkg/geoip"
|
||||
)
|
||||
|
||||
var accessLogGeoProviderFactory = func() (pkggeoip.Service, error) {
|
||||
return pkggeoip.NewMaxMindGeoIPService()
|
||||
// 共享一个 GeoIP 服务实例:mmdb 打开(mmap + 解析元数据)成本不低,缺文件时还会
|
||||
// 同步下载,绝不能每个上报批次重建。maxminddb.Reader 并发安全,无需额外加锁。
|
||||
// 初始化失败不锁存:下一批上报会重试(与旧行为一致)。
|
||||
var (
|
||||
sharedAccessLogGeoMu sync.Mutex
|
||||
sharedAccessLogGeoInstance pkggeoip.Service
|
||||
)
|
||||
|
||||
func sharedAccessLogGeoService(ctx context.Context) pkggeoip.Service {
|
||||
sharedAccessLogGeoMu.Lock()
|
||||
defer sharedAccessLogGeoMu.Unlock()
|
||||
if sharedAccessLogGeoInstance == nil {
|
||||
service, err := pkggeoip.NewMaxMindGeoIPServiceWithContext(ctx, "", "")
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "initialize access log geo service failed", "error", err)
|
||||
return nil
|
||||
}
|
||||
sharedAccessLogGeoInstance = service
|
||||
}
|
||||
return sharedAccessLogGeoInstance
|
||||
}
|
||||
|
||||
type accessLogRegionResolver struct {
|
||||
provider pkggeoip.Service
|
||||
cache map[string]string
|
||||
}
|
||||
|
||||
func newAccessLogRegionResolver() (*accessLogRegionResolver, error) {
|
||||
provider, err := accessLogGeoProviderFactory()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &accessLogRegionResolver{
|
||||
provider: provider,
|
||||
cache: make(map[string]string),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *accessLogRegionResolver) Close() {
|
||||
if r == nil || r.provider == nil {
|
||||
return
|
||||
}
|
||||
if err := r.provider.Close(); err != nil {
|
||||
slog.Warn("close access log geo provider failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *accessLogRegionResolver) Resolve(rawIP string) string {
|
||||
if r == nil || r.provider == nil {
|
||||
return ""
|
||||
}
|
||||
// resolveAccessLogRegion resolves the region name for an access-log remote address.
|
||||
// mmdb Lookup 本身是内存映射 trie 查找(微秒级),无需再建应用层 IP 缓存。
|
||||
// resolveAccessLogRegion resolves the region name for an access-log remote address.
|
||||
// mmdb Lookup 本身是内存映射 trie 查找(微秒级),无需再建应用层 IP 缓存。
|
||||
func resolveAccessLogRegion(ctx context.Context, rawIP string) string {
|
||||
normalizedIP := normalizeAccessLogIP(rawIP)
|
||||
if normalizedIP == "" {
|
||||
return ""
|
||||
}
|
||||
if cached, ok := r.cache[normalizedIP]; ok {
|
||||
return cached
|
||||
}
|
||||
|
||||
info, err := r.provider.GetGeoInfo(net.ParseIP(normalizedIP))
|
||||
if err != nil || info == nil {
|
||||
r.cache[normalizedIP] = ""
|
||||
service := sharedAccessLogGeoService(ctx)
|
||||
if service == nil {
|
||||
return ""
|
||||
}
|
||||
info, err := service.GetGeoInfo(net.ParseIP(normalizedIP))
|
||||
if err != nil || info == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
region := strings.TrimSpace(info.Name)
|
||||
if region == "" {
|
||||
region = strings.TrimSpace(info.ISOCode)
|
||||
}
|
||||
r.cache[normalizedIP] = region
|
||||
return region
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -36,7 +35,7 @@ func PersistHeartbeatObservability(ctx context.Context, nodeID string, payload N
|
||||
return
|
||||
}
|
||||
|
||||
accessLogRecords, err := buildNodeAccessLogRecords(nodeID, payload.AccessLogs, payload.Buffered, reportedAt)
|
||||
accessLogRecords, err := buildNodeAccessLogRecords(ctx, nodeID, payload.AccessLogs, payload.Buffered, reportedAt)
|
||||
if err != nil {
|
||||
zap.L().Error("build heartbeat access logs failed", zap.String("node_id", nodeID), zap.Error(err))
|
||||
return
|
||||
@@ -159,7 +158,7 @@ func persistNodeMetricSnapshot(ctx context.Context, nodeID string, snapshot *Nod
|
||||
return repository.InsertOpenFlareMetricSnapshot(ctx, record)
|
||||
}
|
||||
|
||||
func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered []BufferedObservabilityRecord, reportedAt time.Time) ([]*model.OpenFlareAccessLog, error) {
|
||||
func buildNodeAccessLogRecords(ctx context.Context, nodeID string, direct []NodeAccessLog, buffered []BufferedObservabilityRecord, reportedAt time.Time) ([]*model.OpenFlareAccessLog, error) {
|
||||
total := len(direct)
|
||||
for _, record := range buffered {
|
||||
total += len(record.AccessLogs)
|
||||
@@ -168,14 +167,6 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
resolver, err := newAccessLogRegionResolver()
|
||||
if err != nil {
|
||||
slog.Warn("initialize access log geo resolver failed", "node_id", nodeID, "error", err)
|
||||
}
|
||||
if resolver != nil {
|
||||
defer resolver.Close()
|
||||
}
|
||||
|
||||
records := make([]*model.OpenFlareAccessLog, 0, total)
|
||||
appendLogs := func(logs []NodeAccessLog) {
|
||||
for _, item := range logs {
|
||||
@@ -186,7 +177,7 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
|
||||
NodeID: nodeID,
|
||||
LoggedAt: timeFromUnix(item.LoggedAtUnix, reportedAt),
|
||||
RemoteAddr: strings.TrimSpace(item.RemoteAddr),
|
||||
Region: "",
|
||||
Region: resolveAccessLogRegion(ctx, item.RemoteAddr),
|
||||
Host: strings.TrimSpace(item.Host),
|
||||
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
|
||||
UserAgent: truncateForDatabase(strings.TrimSpace(item.UserAgent), accessLogUserAgentMaxLength),
|
||||
@@ -196,9 +187,6 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
|
||||
RequestLength: requestLength,
|
||||
RequestTimeMs: requestTimeMs,
|
||||
}
|
||||
if resolver != nil {
|
||||
record.Region = resolver.Resolve(record.RemoteAddr)
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
func TestBuildNodeAccessLogRecordsPreservesBytesSent(t *testing.T) {
|
||||
reportedAt := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
records, err := buildNodeAccessLogRecords("node-a", []NodeAccessLog{
|
||||
records, err := buildNodeAccessLogRecords(context.Background(), "node-a", []NodeAccessLog{
|
||||
{
|
||||
LoggedAtUnix: reportedAt.Unix(),
|
||||
RemoteAddr: "203.0.113.10",
|
||||
|
||||
Reference in New Issue
Block a user