fix(agent): 观测缓冲去重纳入 cache_status 并保留原始 -

避免同一请求不同缓存状态被合并丢弃;OpenResty 的 - 原样入库便于详情区分。
This commit is contained in:
ryan
2026-07-18 22:56:10 +08:00
parent fb5a4e5b59
commit 6f221b042e
6 changed files with 25 additions and 12 deletions
+2 -1
View File
@@ -28,7 +28,8 @@ sidebar: false
- 边缘访问日志现支持上报并存储 User-Agent;概览新增设备类型饼图,以及浏览器、操作系统、User-Agent 排行。
- 访问日志明细支持查看单条请求详情,可查看 User-Agent、IP 访问趋势与分析,并快捷将 IP 加入或移出 IP 组。
- 访问日志页签状态改为 URL 参数记忆,筛选后不会再跳回概览。
- 访问日志支持边缘缓存状态(`cache_status`):明细列表与详情可查看命中 / 回源 / 未缓存。
- 访问日志支持边缘缓存状态(`cache_status`):明细列表与详情可查看命中 / 回源 / 未缓存。启用需执行 ClickHouse 迁移并重新发布节点配置使新日志格式生效。
- 修复 Agent 观测缓冲在合并访问日志时忽略 `cache_status` 导致 HIT/MISS 被去重丢弃的问题。
- 优化访问日志概览的设备类型与状态码饼图布局,使设备类型与状态码饼图在 1024px(lg)以上屏幕保持双列并排展示,提升页面紧凑度。
- 修复访问日志概览在 ClickHouse 查询发生错误时静默吞掉报错的缺陷,引入错误日志输出,方便定位问题。
@@ -59,4 +59,5 @@
* [x] log_format + protocol + agent parse
* [x] CH migration + 写入/读取
* [x] API + 前端明细展示
* [ ] 测试与提交
* [x] 缓冲去重 key 含 cache_status;保留 `-` 原始值
* [x] 测试与提交
+2 -5
View File
@@ -205,11 +205,8 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
}
func normalizeCacheStatus(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "-" {
return ""
}
return trimmed
// Keep OpenResty "-" as-is so details can distinguish it from a missing field.
return strings.TrimSpace(value)
}
func parseCombinedAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
@@ -56,6 +56,18 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
}
}
func TestNormalizeCacheStatusKeepsDash(t *testing.T) {
if got := normalizeCacheStatus(" - "); got != "-" {
t.Fatalf("normalizeCacheStatus dash = %q, want -", got)
}
if got := normalizeCacheStatus(" HIT "); got != "HIT" {
t.Fatalf("normalizeCacheStatus hit = %q, want HIT", got)
}
if got := normalizeCacheStatus(""); got != "" {
t.Fatalf("normalizeCacheStatus empty = %q, want empty", got)
}
}
func TestCollectAccessLogsResetsOffsetAfterTruncate(t *testing.T) {
tempDir := t.TempDir()
logPath := filepath.Join(tempDir, "openflare_access.log")
@@ -117,7 +117,7 @@ func mergeAccessLogs(existing []protocol.NodeAccessLog, incoming []protocol.Node
}
func accessLogKey(item protocol.NodeAccessLog) string {
return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + item.UserAgent + "|" + strconv.Itoa(item.StatusCode)
return strconv.FormatInt(item.LoggedAtUnix, 10) + "|" + item.RemoteAddr + "|" + item.Host + "|" + item.Path + "|" + item.UserAgent + "|" + item.CacheStatus + "|" + strconv.Itoa(item.StatusCode)
}
// Replayable returns buffered records from windows before currentWindowStartedAtUnix.
@@ -65,7 +65,7 @@ func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) {
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
AccessLogs: []protocol.NodeAccessLog{
{LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200},
{LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200, CacheStatus: "HIT"},
},
}, 1710403000); err != nil {
t.Fatalf("first upsert failed: %v", err)
@@ -73,7 +73,9 @@ func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) {
if err := store.Upsert(ObservabilityBufferRecord{
WindowStartedAtUnix: 1710403200,
AccessLogs: []protocol.NodeAccessLog{
{LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200},
// Same identity except cache status — must not collapse HIT/MISS.
{LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200, CacheStatus: "HIT"},
{LoggedAtUnix: 1710403201, RemoteAddr: "10.0.0.1", Host: "app.example.com", Path: "/a", StatusCode: 200, CacheStatus: "MISS"},
{LoggedAtUnix: 1710403205, RemoteAddr: "10.0.0.2", Host: "app.example.com", Path: "/b", StatusCode: 502},
},
}, 1710403000); err != nil {
@@ -84,8 +86,8 @@ func TestObservabilityBufferStoreMergesAccessLogsWithinWindow(t *testing.T) {
if err != nil {
t.Fatalf("Replayable failed: %v", err)
}
if len(records) != 1 || len(records[0].AccessLogs) != 2 {
t.Fatalf("expected merged access logs, got %+v", records)
if len(records) != 1 || len(records[0].AccessLogs) != 3 {
t.Fatalf("expected merged access logs with distinct cache_status, got %+v", records)
}
}