tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-01 14:46:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(access-logs): 上报并展示边缘缓存状态 cache_status

Browse Source 
OpenResty 日志输出 $upstream_cache_status；Agent/协议/ClickHouse 贯通入库。
明细列表与详情按 HIT/MISS 等推导命中、回源、未缓存三态标签。
This commit is contained in:
ryan
2026-07-18 22:50:46 +08:00
parent ee9d651c8a
commit fb5a4e5b59
24 changed files with 271 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apps/agent/observability/traffic.go
+12
View File
@@ -23,6 +23,7 @@ type accessLogRecord struct {
RemoteAddr string `json:"remote_addr"`
Path string `json:"path"`
UserAgent string `json:"user_agent"`
CacheStatus string `json:"cache_status"`
Status int `json:"status"`
BytesSent int64 `json:"bytes_sent"`
RequestLength int64 `json:"request_length"`
@@ -147,6 +148,7 @@ func (aggregate *trafficAggregate) consume(line []byte) {
Host: strings.TrimSpace(record.Host),
Path: normalizeAccessLogPath(record.Path),
UserAgent: strings.TrimSpace(record.UserAgent),
CacheStatus: normalizeCacheStatus(record.CacheStatus),
StatusCode: record.Status,
BytesSent: record.BytesSent,
RequestLength: record.RequestLength,
@@ -160,6 +162,7 @@ type parsedAccessLogRecord struct {
RemoteAddr string
Path string
UserAgent string
CacheStatus string
Status int
BytesSent int64
RequestLength int64
@@ -193,6 +196,7 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
RemoteAddr: strings.TrimSpace(record.RemoteAddr),
Path: normalizeAccessLogPath(record.Path),
UserAgent: strings.TrimSpace(record.UserAgent),
CacheStatus: normalizeCacheStatus(record.CacheStatus),
Status: record.Status,
BytesSent: record.BytesSent,
RequestLength: record.RequestLength,
@@ -200,6 +204,14 @@ func parseJSONAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
}, true
}
func normalizeCacheStatus(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "-" {
return ""
}
return trimmed
}
func parseCombinedAccessLogRecord(raw string) (parsedAccessLogRecord, bool) {
matches := combinedAccessLogPattern.FindStringSubmatch(raw)
if len(matches) != combinedAccessLogMatchGroupCount {
internal/apps/agent/observability/traffic_test.go
+5 -2
View File
@@ -14,8 +14,8 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
tempDir := t.TempDir()
logPath := filepath.Join(tempDir, "openflare_access.log")
content := []byte(
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512,\"request_time\":0.015,\"user_agent\":\"Mozilla/5.0\"}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256,\"request_time\":0.008,\"user_agent\":\"curl/8.0\"}\n",
"{\"ts\":\"2026-03-14T08:00:00Z\",\"host\":\"app.example.com\",\"path\":\"/login\",\"remote_addr\":\"10.0.0.1\",\"status\":200,\"request_length\":128,\"bytes_sent\":512,\"request_time\":0.015,\"user_agent\":\"Mozilla/5.0\",\"cache_status\":\"HIT\"}\n" +
"{\"ts\":\"2026-03-14T08:00:05Z\",\"host\":\"api.example.com\",\"path\":\"/v1/ping\",\"remote_addr\":\"10.0.0.2\",\"status\":502,\"request_length\":64,\"bytes_sent\":256,\"request_time\":0.008,\"user_agent\":\"curl/8.0\",\"cache_status\":\"MISS\"}\n",
)
if err := os.WriteFile(logPath, content, 0o644); err != nil {
t.Fatalf("WriteFile failed: %v", err)
@@ -38,6 +38,9 @@ func TestCollectAccessLogsReturnsFactsOnly(t *testing.T) {
if accessLogs[0].UserAgent != "Mozilla/5.0" || accessLogs[1].UserAgent != "curl/8.0" {
t.Fatalf("unexpected user agents: %+v", accessLogs)
}
if accessLogs[0].CacheStatus != "HIT" || accessLogs[1].CacheStatus != "MISS" {
t.Fatalf("unexpected cache status: %+v", accessLogs)
}
snapshot, err := stateStore.Load()
if err != nil {
internal/apps/openflare/agent/observability.go
+3 -1
View File
@@ -24,7 +24,8 @@ const (
healthSeverityWarning = "warning"
healthSeverityCritical = "critical"
accessLogPathMaxLength = 100
accessLogUserAgentMaxLength = 512
accessLogUserAgentMaxLength = 512
accessLogCacheStatusMaxLength = 32
healthEventMessageMaxLength = 4096
)
@@ -210,6 +211,7 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
Host: strings.TrimSpace(item.Host),
Path: truncateForDatabase(strings.TrimSpace(item.Path), accessLogPathMaxLength),
UserAgent: truncateForDatabase(strings.TrimSpace(item.UserAgent), accessLogUserAgentMaxLength),
CacheStatus: truncateForDatabase(strings.TrimSpace(item.CacheStatus), accessLogCacheStatusMaxLength),
StatusCode: item.StatusCode,
BytesSent: bytesSent,
RequestLength: requestLength,
internal/apps/openflare/observability/access_log_logics.go
+22 -20
View File
@@ -46,16 +46,17 @@ type AccessLogQuery struct {
// AccessLogView is a single access log row.
type AccessLogView struct {
ID string `json:"id"`
NodeID string `json:"node_id"`
NodeName string `json:"node_name"`
LoggedAt time.Time `json:"logged_at"`
RemoteAddr string `json:"remote_addr"`
Region string `json:"region"`
Host string `json:"host"`
Path string `json:"path"`
UserAgent string `json:"user_agent"`
StatusCode int `json:"status_code"`
ID string `json:"id"`
NodeID string `json:"node_id"`
NodeName string `json:"node_name"`
LoggedAt time.Time `json:"logged_at"`
RemoteAddr string `json:"remote_addr"`
Region string `json:"region"`
Host string `json:"host"`
Path string `json:"path"`
UserAgent string `json:"user_agent"`
CacheStatus string `json:"cache_status"`
StatusCode int `json:"status_code"`
}
// AccessLogList is a paginated access log response.
@@ -476,16 +477,17 @@ func ListAccessLogs(ctx context.Context, input AccessLogQuery) (*AccessLogList,
continue
}
views = append(views, AccessLogView{
ID: formatAccessLogID(item.ID),
NodeID: item.NodeID,
NodeName: nodeNames[item.NodeID],
LoggedAt: item.LoggedAt,
RemoteAddr: item.RemoteAddr,
Region: item.Region,
Host: item.Host,
Path: item.Path,
UserAgent: item.UserAgent,
StatusCode: item.StatusCode,
ID: formatAccessLogID(item.ID),
NodeID: item.NodeID,
NodeName: nodeNames[item.NodeID],
LoggedAt: item.LoggedAt,
RemoteAddr: item.RemoteAddr,
Region: item.Region,
Host: item.Host,
Path: item.Path,
UserAgent: item.UserAgent,
CacheStatus: item.CacheStatus,
StatusCode: item.StatusCode,
})
}
return &AccessLogList{
internal/db/migrator/goose/clickhouse/202607180005_access_log_cache_status.sql
+6
View File
@@ -0,0 +1,6 @@
-- +goose Up
ALTER TABLE of_node_access_logs
ADD COLUMN IF NOT EXISTS cache_status String DEFAULT '';
-- +goose Down
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS cache_status;
internal/model/analytics/node_access_log.go
+2 -1
View File
@@ -10,7 +10,7 @@ import (
const (
nodeAccessLogTableName = "of_node_access_logs"
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, user_agent, status_code, bytes_sent, request_length, request_time_ms, created_at"
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, user_agent, cache_status, status_code, bytes_sent, request_length, request_time_ms, created_at"
)
// NodeAccessLog stores OpenFlare edge node access records in ClickHouse.
@@ -23,6 +23,7 @@ type NodeAccessLog struct {
Host string `gorm:"column:host"`
Path string `gorm:"column:path"`
UserAgent string `gorm:"column:user_agent"`
CacheStatus string `gorm:"column:cache_status"`
StatusCode int32 `gorm:"column:status_code"`
BytesSent uint64 `gorm:"column:bytes_sent"`
RequestLength uint64 `gorm:"column:request_length"`
internal/model/openflare_access_log_store.go
+2
View File
@@ -285,6 +285,7 @@ func toAnalyticsNodeAccessLog(record *OpenFlareAccessLog) analyticsmodel.NodeAcc
Host: record.Host,
Path: record.Path,
UserAgent: record.UserAgent,
CacheStatus: record.CacheStatus,
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
BytesSent: bytesSent,
RequestLength: requestLength,
@@ -317,6 +318,7 @@ func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*OpenFla
Host: row.Host,
Path: row.Path,
UserAgent: row.UserAgent,
CacheStatus: row.CacheStatus,
StatusCode: int(row.StatusCode),
BytesSent: bytesSent,
RequestLength: requestLength,
internal/model/openflare_observability.go
+1
View File
@@ -48,6 +48,7 @@ type OpenFlareAccessLog struct {
Host string `json:"host" gorm:"index;size:255"`
Path string `json:"path" gorm:"size:2048"`
UserAgent string `json:"user_agent" gorm:"column:user_agent;size:512"`
CacheStatus string `json:"cache_status" gorm:"column:cache_status;size:32"`
StatusCode int `json:"status_code" gorm:"index"`
BytesSent int64 `json:"bytes_sent" gorm:"column:bytes_sent;not null;default:0"`
RequestLength int64 `json:"request_length" gorm:"column:request_length;not null;default:0"`
internal/model/openflare_option.go
+1 -1
View File
@@ -38,7 +38,7 @@ http {
fastcgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/fastcgi_temp;
uwsgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/uwsgi_temp;
scgi_temp_path __OPENFLARE_NGINX_CACHE_DIR__/scgi_temp;
{{OpenRestyConnectionUpgradeMap}}{{OpenRestyDefaultServerBlock}} log_format openflare_json escape=json '{"ts":"$time_iso8601","host":"$host","path":"$request_uri","remote_addr":"$remote_addr","status":$status,"request_time":$request_time,"bytes_sent":$body_bytes_sent,"request_length":$request_length,"user_agent":"$http_user_agent"}';
{{OpenRestyConnectionUpgradeMap}}{{OpenRestyDefaultServerBlock}} log_format openflare_json escape=json '{"ts":"$time_iso8601","host":"$host","path":"$request_uri","remote_addr":"$remote_addr","status":$status,"request_time":$request_time,"bytes_sent":$body_bytes_sent,"request_length":$request_length,"user_agent":"$http_user_agent","cache_status":"$upstream_cache_status"}';
access_log {{OpenRestyAccessLogPath}} openflare_json;
sendfile on;
tcp_nopush on;
internal/repository/analytics/node_access_log.go
+2 -1
View File
@@ -36,7 +36,7 @@ func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]anal
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, logged_at, remote_addr, region, host, path, user_agent, status_code, bytes_sent, request_length, request_time_ms, created_at
SELECT id, node_id, logged_at, remote_addr, region, host, path, user_agent, cache_status, status_code, bytes_sent, request_length, request_time_ms, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
@@ -69,6 +69,7 @@ func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, er
&item.Host,
&item.Path,
&item.UserAgent,
&item.CacheStatus,
&item.StatusCode,
&item.BytesSent,
&item.RequestLength,
internal/repository/analytics/node_access_log_test.go
+6 -5
View File
@@ -49,9 +49,10 @@ func TestBatchInsertNodeAccessLogs_UsesModelBatchSQL(t *testing.T) {
assert.True(t, mockBatch.sendCalled)
require.Len(t, mockBatch.rows, 1)
assert.Equal(t, "node-a", mockBatch.rows[0][1])
require.Len(t, mockBatch.rows[0], 13)
assert.Equal(t, "", mockBatch.rows[0][7]) // user_agent
assert.Equal(t, uint64(2048), mockBatch.rows[0][9]) // bytes_sent
assert.Equal(t, uint64(0), mockBatch.rows[0][10]) // request_length
assert.Equal(t, uint32(0), mockBatch.rows[0][11]) // request_time_ms
require.Len(t, mockBatch.rows[0], 14)
assert.Equal(t, "", mockBatch.rows[0][7]) // user_agent
assert.Equal(t, "", mockBatch.rows[0][8]) // cache_status
assert.Equal(t, uint64(2048), mockBatch.rows[0][10]) // bytes_sent
assert.Equal(t, uint64(0), mockBatch.rows[0][11]) // request_length
assert.Equal(t, uint32(0), mockBatch.rows[0][12]) // request_time_ms
}
internal/repository/analytics/node_access_log_writer.go
+1
View File
@@ -47,6 +47,7 @@ func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAc
logItem.Host,
logItem.Path,
strings.TrimSpace(logItem.UserAgent),
strings.TrimSpace(logItem.CacheStatus),
logItem.StatusCode,
logItem.BytesSent,
logItem.RequestLength,