feat(obs): 访问日志 SSOT 与 edge_health,去掉协议兼容层

Agent 仅上报 host_metrics/edge_health/access_logs;业务流量与 UV 由
Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly,
删除 request_reports/openresty 吞吐路径;API 不再暴露 traffic_reports
与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
ryan
2026-07-18 11:53:11 +08:00
parent 9a0974cce8
commit f0e234df1f
60 changed files with 1799 additions and 2164 deletions
@@ -14,6 +14,8 @@ import (
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
const edgeHealthStatusUnknown = "unknown"
// InsertNodeMetricSnapshot writes a single metric snapshot via the batch API.
func InsertNodeMetricSnapshot(ctx context.Context, snapshot analyticsmodel.NodeMetricSnapshot) error {
if strings.TrimSpace(snapshot.NodeID) == "" {
@@ -78,105 +80,49 @@ func BatchInsertNodeMetricSnapshots(ctx context.Context, snapshots []analyticsmo
return nil
}
// InsertNodeRequestReport writes a single request report via the batch API.
func InsertNodeRequestReport(ctx context.Context, report analyticsmodel.NodeRequestReport) error {
if strings.TrimSpace(report.NodeID) == "" {
return nil
func normalizeEdgeHealthStatus(status string) string {
status = strings.TrimSpace(status)
if status == "" {
return edgeHealthStatusUnknown
}
return BatchInsertNodeRequestReports(ctx, []analyticsmodel.NodeRequestReport{report})
return status
}
// BatchInsertNodeRequestReports writes request reports to ClickHouse.
func BatchInsertNodeRequestReports(ctx context.Context, reports []analyticsmodel.NodeRequestReport) error {
if len(reports) == 0 {
// InsertNodeEdgeHealth writes a single edge health snapshot.
func InsertNodeEdgeHealth(ctx context.Context, row analyticsmodel.NodeEdgeHealth) error {
if strings.TrimSpace(row.NodeID) == "" {
return nil
}
return BatchInsertNodeEdgeHealth(ctx, []analyticsmodel.NodeEdgeHealth{row})
}
// BatchInsertNodeEdgeHealth writes L2 OpenResty health snapshots to ClickHouse.
func BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
if len(rows) == 0 {
return nil
}
if db.ChConn == nil {
return fmt.Errorf("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeRequestReport{}.BatchInsertSQL())
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
now := time.Now().UTC()
for _, report := range reports {
nodeID := strings.TrimSpace(report.NodeID)
for _, row := range rows {
nodeID := strings.TrimSpace(row.NodeID)
if nodeID == "" {
continue
}
id := report.ID
id := row.ID
if id == 0 {
id = idgen.NextUint64ID()
}
createdAt := report.CreatedAt
createdAt := row.CreatedAt
if createdAt.IsZero() {
createdAt = now
}
if err := batch.Append(
id,
nodeID,
report.WindowStartedAt.UTC(),
report.WindowEndedAt.UTC(),
report.RequestCount,
report.ErrorCount,
report.UniqueVisitorCount,
report.StatusCodesJSON,
report.TopDomainsJSON,
report.SourceCountriesJSON,
createdAt.UTC(),
); err != nil {
return fmt.Errorf("append node request report to batch: %w", err)
}
}
if batch.Rows() == 0 {
return nil
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send clickhouse batch: %w", err)
}
return nil
}
// InsertNodeObsOpenresty writes a single OpenResty observation via the batch API.
func InsertNodeObsOpenresty(ctx context.Context, obs analyticsmodel.NodeObsOpenresty) error {
if strings.TrimSpace(obs.NodeID) == "" {
return nil
}
return BatchInsertNodeObsOpenresty(ctx, []analyticsmodel.NodeObsOpenresty{obs})
}
// BatchInsertNodeObsOpenresty writes OpenResty observations to ClickHouse.
func BatchInsertNodeObsOpenresty(ctx context.Context, observations []analyticsmodel.NodeObsOpenresty) error {
if len(observations) == 0 {
return nil
}
if db.ChConn == nil {
return fmt.Errorf("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsOpenresty{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
now := time.Now().UTC()
for _, obs := range observations {
nodeID := strings.TrimSpace(obs.NodeID)
if nodeID == "" {
continue
}
id := obs.ID
if id == 0 {
id = idgen.NextUint64ID()
}
createdAt := obs.CreatedAt
if createdAt.IsZero() {
createdAt = now
}
capturedAt := obs.CapturedAt.UTC()
capturedAt := row.CapturedAt.UTC()
if capturedAt.IsZero() {
capturedAt = now
}
@@ -184,15 +130,13 @@ func BatchInsertNodeObsOpenresty(ctx context.Context, observations []analyticsmo
id,
nodeID,
capturedAt,
obs.OpenrestyRxBytes,
obs.OpenrestyTxBytes,
obs.OpenrestyConnections,
normalizeEdgeHealthStatus(row.Status),
row.Connections,
createdAt.UTC(),
); err != nil {
return fmt.Errorf("append node openresty observation to batch: %w", err)
return fmt.Errorf("append node edge health to batch: %w", err)
}
}
if batch.Rows() == 0 {
return nil
}