mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-02 06:56:36 +08:00
feat(obs): 访问日志 SSOT 与 edge_health，去掉协议兼容层
Agent 仅上报 host_metrics/edge_health/access_logs；业务流量与 UV 由 Server 侧访问日志聚合。新增 of_node_edge_health 与 of_access_log_hourly， 删除 request_reports/openresty 吞吐路径；API 不再暴露 traffic_reports 与 openresty_rx|tx。心跳/离线默认阈值与回填迁移一并入库。
This commit is contained in:
ryan
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
-- L1 access log: request body/header length (接收数据) and optional request duration.
|
||||
ALTER TABLE of_node_access_logs
|
||||
ADD COLUMN IF NOT EXISTS request_length UInt64 DEFAULT 0;
|
||||
|
||||
ALTER TABLE of_node_access_logs
|
||||
ADD COLUMN IF NOT EXISTS request_time_ms UInt32 DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS request_time_ms;
|
||||
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS request_length;
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
-- +goose Up
|
||||
-- M5: L2 edge health table, L1 access-log hourly rollup, drop deprecated pre-aggregation tables.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_edge_health
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
status LowCardinality(String),
|
||||
connections Int64,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
TTL toDateTime(captured_at) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- Hourly business traffic from access logs (Server-side only; Agent never writes this).
|
||||
-- SummingMergeTree merges request/error/bytes; UV is not stored (query raw for exact UV).
|
||||
CREATE TABLE IF NOT EXISTS of_access_log_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime('UTC'),
|
||||
host String,
|
||||
request_count UInt64,
|
||||
error_count UInt64,
|
||||
bytes_sent UInt64,
|
||||
request_length UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour, host)
|
||||
TTL hour + INTERVAL 90 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_access_log_hourly_mv
|
||||
TO of_access_log_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(logged_at) AS hour,
|
||||
host,
|
||||
toUInt64(count()) AS request_count,
|
||||
toUInt64(countIf(status_code >= 500)) AS error_count,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length
|
||||
FROM of_node_access_logs
|
||||
GROUP BY node_id, hour, host;
|
||||
|
||||
-- Deprecated pre-aggregation paths (business traffic is access logs; OR throughput is not authoritative).
|
||||
DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_traffic_hourly;
|
||||
DROP VIEW IF EXISTS of_node_openresty_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_openresty_hourly;
|
||||
DROP TABLE IF EXISTS of_node_request_reports;
|
||||
DROP TABLE IF EXISTS of_node_obs_openresty;
|
||||
|
||||
-- +goose Down
|
||||
CREATE TABLE IF NOT EXISTS of_node_obs_openresty
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
openresty_rx_bytes Int64,
|
||||
openresty_tx_bytes Int64,
|
||||
openresty_connections Int64,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
TTL toDateTime(captured_at) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_request_reports
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
window_started_at DateTime64(3, 'UTC'),
|
||||
window_ended_at DateTime64(3, 'UTC'),
|
||||
request_count Int64,
|
||||
error_count Int64,
|
||||
unique_visitor_count Int64,
|
||||
status_codes_json String,
|
||||
top_domains_json String,
|
||||
source_countries_json String,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(window_ended_at)
|
||||
ORDER BY (node_id, window_ended_at, window_started_at, id)
|
||||
TTL toDateTime(window_ended_at) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_traffic_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
request_count UInt64,
|
||||
error_count UInt64,
|
||||
unique_visitor_count UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_traffic_hourly_mv
|
||||
TO of_node_traffic_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(window_ended_at) AS hour,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
max(unique_visitor_count) AS unique_visitor_count
|
||||
FROM of_node_request_reports
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_openresty_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
openresty_rx_min SimpleAggregateFunction(min, Int64),
|
||||
openresty_rx_max SimpleAggregateFunction(max, Int64),
|
||||
openresty_tx_min SimpleAggregateFunction(min, Int64),
|
||||
openresty_tx_max SimpleAggregateFunction(max, Int64)
|
||||
)
|
||||
ENGINE = AggregatingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour)
|
||||
TTL hour + INTERVAL 30 DAY;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_openresty_hourly_mv
|
||||
TO of_node_openresty_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(captured_at) AS hour,
|
||||
min(openresty_rx_bytes) AS openresty_rx_min,
|
||||
max(openresty_rx_bytes) AS openresty_rx_max,
|
||||
min(openresty_tx_bytes) AS openresty_tx_min,
|
||||
max(openresty_tx_bytes) AS openresty_tx_max
|
||||
FROM of_node_obs_openresty
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
DROP VIEW IF EXISTS of_access_log_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_access_log_hourly;
|
||||
DROP TABLE IF EXISTS of_node_edge_health;
|
||||
@@ -0,0 +1,40 @@
|
||||
-- +goose Up
|
||||
-- One-time historical backfill for of_access_log_hourly.
|
||||
-- MV only ingests rows after creation; without this, 24h charts fall back to raw access logs.
|
||||
-- ANTI JOIN avoids double-counting (node_id, hour, host) already filled by the live MV.
|
||||
-- UV is intentionally NOT stored in of_access_log_hourly (SummingMergeTree counts only).
|
||||
|
||||
INSERT INTO of_access_log_hourly
|
||||
SELECT
|
||||
s.node_id,
|
||||
toStartOfHour(s.logged_at) AS hour,
|
||||
s.host,
|
||||
toUInt64(count()) AS request_count,
|
||||
toUInt64(countIf(s.status_code >= 500)) AS error_count,
|
||||
sum(s.bytes_sent) AS bytes_sent,
|
||||
sum(s.request_length) AS request_length
|
||||
FROM of_node_access_logs AS s
|
||||
ANTI JOIN
|
||||
(
|
||||
SELECT
|
||||
node_id,
|
||||
hour,
|
||||
host
|
||||
FROM of_access_log_hourly
|
||||
GROUP BY
|
||||
node_id,
|
||||
hour,
|
||||
host
|
||||
) AS existing
|
||||
ON s.node_id = existing.node_id
|
||||
AND toStartOfHour(s.logged_at) = existing.hour
|
||||
AND s.host = existing.host
|
||||
WHERE s.logged_at >= now() - INTERVAL 90 DAY
|
||||
GROUP BY
|
||||
s.node_id,
|
||||
hour,
|
||||
s.host;
|
||||
|
||||
-- +goose Down
|
||||
-- Backfill is additive; down does not remove historical rollup rows (TTL still applies).
|
||||
SELECT 1;
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
-- +goose Up
|
||||
-- 将仍为历史默认值的心跳/离线配置升级为新默认：心跳 3s、离线 60s。
|
||||
-- 已由管理员改成其他值的配置不受影响。
|
||||
UPDATE w_system_configs
|
||||
SET value = '3000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'agent_heartbeat_interval'
|
||||
AND value = '10000';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = '60000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'node_offline_threshold'
|
||||
AND value = '120000';
|
||||
|
||||
-- +goose Down
|
||||
UPDATE w_system_configs
|
||||
SET value = '10000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'agent_heartbeat_interval'
|
||||
AND value = '3000';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = '120000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'node_offline_threshold'
|
||||
AND value = '60000';
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
-- +goose Up
|
||||
-- 将仍为历史默认值的心跳/离线配置升级为新默认：心跳 3s、离线 60s。
|
||||
-- 已由管理员改成其他值的配置不受影响。
|
||||
UPDATE w_system_configs
|
||||
SET value = '3000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'agent_heartbeat_interval'
|
||||
AND value = '10000';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = '60000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'node_offline_threshold'
|
||||
AND value = '120000';
|
||||
|
||||
-- +goose Down
|
||||
UPDATE w_system_configs
|
||||
SET value = '10000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'agent_heartbeat_interval'
|
||||
AND value = '3000';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = '120000',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'node_offline_threshold'
|
||||
AND value = '60000';
|
||||
Reference in New Issue
Block a user