From 6f221b042e7c18e9b74734e2ae79e1ea1df012c8 Mon Sep 17 00:00:00 2001 From: ryan Date: Sat, 18 Jul 2026 22:56:10 +0800 Subject: [PATCH] =?UTF-8?q?fix(agent):=20=E8=A7=82=E6=B5=8B=E7=BC=93?= =?UTF-8?q?=E5=86=B2=E5=8E=BB=E9=87=8D=E7=BA=B3=E5=85=A5=20cache=5Fstatus?= =?UTF-8?q?=20=E5=B9=B6=E4=BF=9D=E7=95=99=E5=8E=9F=E5=A7=8B=20-?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 避免同一请求不同缓存状态被合并丢弃；OpenResty 的 - 原样入库便于详情区分。 --- docs/changelog/index.md | 3 ++- docs/plan/20260718-access-log-cache-status.md | 3 ++- internal/apps/agent/observability/traffic.go | 7 ++----- internal/apps/agent/observability/traffic_test.go | 12 ++++++++++++ internal/apps/agent/state/observability_buffer.go | 2 +- .../apps/agent/state/observability_buffer_test.go | 10 ++++++---- 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/docs/changelog/index.md b/docs/changelog/index.md index f5d0fbf5..c48221d9 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -28,7 +28,8 @@ sidebar: false - 边缘访问日志现支持上报并存储 User-Agent；概览新增设备类型饼图，以及浏览器、操作系统、User-Agent 排行。 - 访问日志明细支持查看单条请求详情，可查看 User-Agent、IP 访问趋势与分析，并快捷将 IP 加入或移出 IP 组。 - 访问日志页签状态改为 URL 参数记忆，筛选后不会再跳回概览。 -- 访问日志支持边缘缓存状态（`cache_status`）：明细列表与详情可查看命中 / 回源 / 未缓存。 +- 访问日志支持边缘缓存状态（`cache_status`）：明细列表与详情可查看命中 / 回源 / 未缓存。启用需执行 ClickHouse 迁移并重新发布节点配置使新日志格式生效。 +- 修复 Agent 观测缓冲在合并访问日志时忽略 `cache_status` 导致 HIT/MISS 被去重丢弃的问题。 - 优化访问日志概览的设备类型与状态码饼图布局，使设备类型与状态码饼图在 1024px（lg）以上屏幕保持双列并排展示，提升页面紧凑度。 - 修复访问日志概览在 ClickHouse 查询发生错误时静默吞掉报错的缺陷，引入错误日志输出，方便定位问题。 diff --git a/docs/plan/20260718-access-log-cache-status.md b/docs/plan/20260718-access-log-cache-status.md index 0afd124a..e08e5ff4 100644 --- a/docs/plan/20260718-access-log-cache-status.md +++ b/docs/plan/20260718-access-log-cache-status.md @@ -59,4 +59,5 @@ * [x] log_format + protocol + agent parse * [x] CH migration + 写入/读取 * [x] API + 前端明细展示 -* [ ] 测试与提交 +* [x] 缓冲去重 key 含 cache_status；保留 `-` 原始值 +* [x] 测试与提交 diff --git a/internal/apps/agent/observability/traffic.go b/internal/apps/agent/observability/traffic.go index 1cec9f60..193d4f08 100644 --- a/internal/apps/agent/observability/traffic.go +++ b/internal/apps/agent/observability/traffic.go @@ -205,11 +205,8 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) { } func normalizeCacheStatus(value string) string { - trimmed := strings.TrimSpace(value) - if trimmed == "-" { - return "" - } - return trimmed + // Keep OpenResty "-" as-is so details can distinguish it from a missing field. + return strings.TrimSpace(value) } func parseCombinedAccessLogRecord(raw string) (parsedAccessLogRecord, bool) { diff --git a/internal/apps/agent/observability/traffic_test.go b/internal/apps/agent/observability/traffic_test.go index 3097addd..451b7bb3 100644 --- a/internal/apps/agent/observability/traffic_test.go +++ b/internal/apps/agent/observability/traffic_test.go @@ -56,6 +56,18 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) { } } +func TestNormalizeCacheStatusKeepsDash(t *testing.T) { + if got := normalizeCacheStatus(" - "); got != "-" { + t.Fatalf("normalizeCacheStatus dash = %q, want -", got) + } + if got := normalizeCacheStatus(" HIT "); got != "HIT" { + t.Fatalf("normalizeCacheStatus hit = %q, want HIT", got) + } + if got := normalizeCacheStatus(""); got != "" { + t.Fatalf("normalizeCacheStatus empty = %q, want empty", got) + } +} + func TestCollectAccessLogsResetsOffsetAfterTruncate(t *testing.T) { tempDir := t.TempDir() logPath := filepath.Join(tempDir, "openflare_access.log") diff --git a/internal/apps/agent/state/observability_buffer.go b/internal/apps/agent/state/observability_buffer.go index 4132018d..2509e183 100644 --- a/internal/apps/agent/state/observability_buffer.go +++ b/internal/apps/agent/state/observability_buffer.go @@ -117,7 +117,7 @@ func mergeAccessLogs(existing []protocol.NodeAccessLog, incoming []protocol.Node } func accessLogKey(item protocol.NodeAccessLog) string { - return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + item.UserAgent + "|" + strconv.Itoa(item.StatusCode) + return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + item.UserAgent + "|" + item.CacheStatus + "|" + strconv.Itoa(item.StatusCode) } // Replayable returns buffered records from windows before currentWindowStartedAtUnix. diff --git a/internal/apps/agent/state/observability_buffer_test.go b/internal/apps/agent/state/observability_buffer_test.go index 3b0aacf4..75e5b2ae 100644 --- a/internal/apps/agent/state/observability_buffer_test.go +++ b/internal/apps/agent/state/observability_buffer_test.go @@ -65,7 +65,7 @@ func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) { if err := store.Upsert(ObservabilityBufferRecord{ WindowStartedAtUnix: 1710403200, AccessLogs: []protocol.NodeAccessLog{ - {LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200}, + {LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200, CacheStatus: "HIT"}, }, }, 1710403000); err != nil { t.Fatalf("first upsert failed: %v", err) @@ -73,7 +73,9 @@ func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) { if err := store.Upsert(ObservabilityBufferRecord{ WindowStartedAtUnix: 1710403200, AccessLogs: []protocol.NodeAccessLog{ - {LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200}, + // Same identity except cache status — must not collapse HIT/MISS. + {LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200, CacheStatus: "HIT"}, + {LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200, CacheStatus: "MISS"}, {LoggedAtUnix: 1710403205, RemoteAddr: "10.0.0.2", Host: "app.example.com", Path: "/b", StatusCode: 502}, }, }, 1710403000); err != nil { @@ -84,8 +86,8 @@ func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) { if err != nil { t.Fatalf("Replayable failed: %v", err) } - if len(records) != 1 || len(records[0].AccessLogs) != 2 { - t.Fatalf("expected merged access logs, got %+v", records) + if len(records) != 1 || len(records[0].AccessLogs) != 3 { + t.Fatalf("expected merged access logs with distinct cache_status, got %+v", records) } }