feat(obs): 访问日志 SSOT 与 edge_health,去掉协议兼容层

Agent 仅上报 host_metrics/edge_health/access_logs;业务流量与 UV 由
Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly,
删除 request_reports/openresty 吞吐路径;API 不再暴露 traffic_reports
与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
ryan
2026-07-18 11:53:11 +08:00
parent 9a0974cce8
commit f0e234df1f
60 changed files with 1799 additions and 2164 deletions
@@ -20,10 +20,8 @@ const (
DatabaseCleanupTargetAccessLogs = "node_access_logs"
// DatabaseCleanupTargetMetricSnapshots is the API cleanup target for metric snapshots.
DatabaseCleanupTargetMetricSnapshots = "node_metric_snapshots"
// DatabaseCleanupTargetRequestReports is the API cleanup target for request reports.
DatabaseCleanupTargetRequestReports = "node_request_reports"
// DatabaseCleanupTargetObsOpenresty is the API cleanup target for OpenResty observations.
DatabaseCleanupTargetObsOpenresty = "node_obs_openresty"
// DatabaseCleanupTargetEdgeHealth is the API cleanup target for OpenResty edge health (connections).
DatabaseCleanupTargetEdgeHealth = "node_edge_health"
// DatabaseCleanupTargetObsFrps is the API cleanup target for FRPS observations.
DatabaseCleanupTargetObsFrps = "node_obs_frps"
// DatabaseCleanupTargetObsFrpc is the API cleanup target for FRPC observations.
@@ -33,8 +31,7 @@ const (
var databaseCleanupTargets = map[string]string{
DatabaseCleanupTargetAccessLogs: "访问日志",
DatabaseCleanupTargetMetricSnapshots: "性能快照",
DatabaseCleanupTargetRequestReports: "请求聚合",
DatabaseCleanupTargetObsOpenresty: "OpenResty 观测",
DatabaseCleanupTargetEdgeHealth: "OpenResty 健康(连接)",
DatabaseCleanupTargetObsFrps: "FRPS 观测",
DatabaseCleanupTargetObsFrpc: "FRPC 观测",
}
@@ -43,8 +40,7 @@ var databaseCleanupTargets = map[string]string{
var databaseCleanupTableTTLDays = map[string]int{
DatabaseCleanupTargetAccessLogs: analyticsrepo.TableTTLDaysNodeAccessLogs,
DatabaseCleanupTargetMetricSnapshots: analyticsrepo.TableTTLDaysNodeMetricSnapshots,
DatabaseCleanupTargetRequestReports: analyticsrepo.TableTTLDaysNodeRequestReports,
DatabaseCleanupTargetObsOpenresty: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetEdgeHealth: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetObsFrps: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetObsFrpc: analyticsrepo.TableTTLDaysNodeObs,
}
@@ -165,8 +161,7 @@ func RunDatabaseAutoCleanupOnce(ctx context.Context, now time.Time) (*DatabaseAu
for _, target := range []string{
DatabaseCleanupTargetAccessLogs,
DatabaseCleanupTargetMetricSnapshots,
DatabaseCleanupTargetRequestReports,
DatabaseCleanupTargetObsOpenresty,
DatabaseCleanupTargetEdgeHealth,
DatabaseCleanupTargetObsFrps,
DatabaseCleanupTargetObsFrpc,
} {
@@ -201,10 +196,8 @@ func deleteAllObservabilityRows(ctx context.Context, target string) (int64, stri
deleted, err = model.DeleteAllOpenFlareAccessLogs(ctx)
case DatabaseCleanupTargetMetricSnapshots:
deleted, err = model.DeleteAllOpenFlareMetricSnapshots(ctx)
case DatabaseCleanupTargetRequestReports:
deleted, err = model.DeleteAllOpenFlareRequestReports(ctx)
case DatabaseCleanupTargetObsOpenresty:
deleted, err = model.DeleteAllOpenFlareNodeObservationOpenresty(ctx)
case DatabaseCleanupTargetEdgeHealth:
deleted, err = model.DeleteAllOpenFlareEdgeHealth(ctx)
case DatabaseCleanupTargetObsFrps:
deleted, err = model.DeleteAllOpenFlareNodeObservationFrps(ctx)
case DatabaseCleanupTargetObsFrpc:
@@ -236,10 +229,8 @@ func materializeObservabilityTableTTL(ctx context.Context, target string) (int64
eligible, err = model.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
case DatabaseCleanupTargetMetricSnapshots:
eligible, err = model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
case DatabaseCleanupTargetRequestReports:
eligible, err = model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
case DatabaseCleanupTargetObsOpenresty:
eligible, err = model.DeleteOpenFlareNodeObservationOpenrestyBefore(ctx, cutoff)
case DatabaseCleanupTargetEdgeHealth:
eligible, err = model.DeleteOpenFlareEdgeHealthBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrps:
eligible, err = model.DeleteOpenFlareNodeObservationFrpsBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrpc:
@@ -157,11 +157,11 @@ func TestRunDatabaseAutoCleanupOnceClampsRetentionToTableTTL(t *testing.T) {
CapturedAt: now.Add(-40 * 24 * time.Hour),
CPUUsagePercent: 10,
}))
require.NoError(t, model.InsertOpenFlareRequestReport(ctx, &model.OpenFlareRequestReport{
NodeID: "node-a",
WindowStartedAt: now.Add(-41 * 24 * time.Hour),
WindowEndedAt: now.Add(-40 * 24 * time.Hour),
RequestCount: 15,
require.NoError(t, model.InsertOpenFlareEdgeHealth(ctx, &model.OpenFlareEdgeHealth{
NodeID: "node-a",
CapturedAt: now.Add(-40 * 24 * time.Hour),
Status: "healthy",
Connections: 2,
}))
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled, "true"))
@@ -170,7 +170,7 @@ func TestRunDatabaseAutoCleanupOnceClampsRetentionToTableTTL(t *testing.T) {
summary, err := RunDatabaseAutoCleanupOnce(ctx, now)
require.NoError(t, err)
require.NotNil(t, summary)
require.Len(t, summary.Results, 6)
require.Len(t, summary.Results, 5)
assert.Equal(t, 1, summary.RetentionDays)
for _, result := range summary.Results {
@@ -189,9 +189,9 @@ func TestRunDatabaseAutoCleanupOnceClampsRetentionToTableTTL(t *testing.T) {
require.NoError(t, err)
assert.Empty(t, metricSnapshots)
requestReports, err := model.ListOpenFlareRequestReportsSince(ctx, "", time.Time{}, 0)
edgeHealth, err := model.ListOpenFlareEdgeHealth(ctx, "", time.Time{}, 0)
require.NoError(t, err)
assert.Empty(t, requestReports)
assert.Empty(t, edgeHealth)
}
func TestTableTTLDaysForCleanupTarget(t *testing.T) {