mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-09 09:06:36 +08:00
refactor(backend): rename OpenFlare directory to lowercase openflare
This commit is contained in:
ryan
+20
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS of_node_access_logs
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
logged_at DateTime64(3, 'UTC'),
|
||||
remote_addr String,
|
||||
region String,
|
||||
host String,
|
||||
path String,
|
||||
status_code Int32,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(logged_at)
|
||||
ORDER BY (node_id, logged_at, remote_addr, host, path, status_code)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS of_node_access_logs;
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS of_node_metric_snapshots
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
cpu_usage_percent Float64,
|
||||
memory_used_bytes Int64,
|
||||
memory_total_bytes Int64,
|
||||
storage_used_bytes Int64,
|
||||
storage_total_bytes Int64,
|
||||
disk_read_bytes Int64,
|
||||
disk_write_bytes Int64,
|
||||
network_rx_bytes Int64,
|
||||
network_tx_bytes Int64,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_request_reports
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
window_started_at DateTime64(3, 'UTC'),
|
||||
window_ended_at DateTime64(3, 'UTC'),
|
||||
request_count Int64,
|
||||
error_count Int64,
|
||||
unique_visitor_count Int64,
|
||||
status_codes_json String,
|
||||
top_domains_json String,
|
||||
source_countries_json String,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(window_ended_at)
|
||||
ORDER BY (node_id, window_ended_at, window_started_at, id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_obs_openresty
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
openresty_rx_bytes Int64,
|
||||
openresty_tx_bytes Int64,
|
||||
openresty_connections Int64,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_obs_frps
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
frps_connections Int32,
|
||||
frps_proxy_count Int32,
|
||||
frps_client_count Int32,
|
||||
frps_proxies String,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_obs_frpc
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
tunnel_status String,
|
||||
connected_relays_count Int32,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS of_node_obs_frpc;
|
||||
DROP TABLE IF EXISTS of_node_obs_frps;
|
||||
DROP TABLE IF EXISTS of_node_obs_openresty;
|
||||
DROP TABLE IF EXISTS of_node_request_reports;
|
||||
DROP TABLE IF EXISTS of_node_metric_snapshots;
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
-- +goose Up
|
||||
-- Add TTL policies to analytics tables so ClickHouse can expire rows automatically.
|
||||
-- DateTime64 columns must be cast for TTL (ClickHouse requires DateTime/Date in TTL expr).
|
||||
ALTER TABLE of_node_access_logs MODIFY TTL toDateTime(logged_at) + INTERVAL 90 DAY;
|
||||
|
||||
ALTER TABLE of_node_metric_snapshots MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_request_reports MODIFY TTL toDateTime(window_ended_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_obs_openresty MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_obs_frps MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_obs_frpc MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
-- +goose Down
|
||||
-- TTL changes cannot be safely reversed without recreating tables.
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS of_node_traffic_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
request_count UInt64,
|
||||
error_count UInt64,
|
||||
unique_visitor_count UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_traffic_hourly_mv
|
||||
TO of_node_traffic_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(window_ended_at) AS hour,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
sum(unique_visitor_count) AS unique_visitor_count
|
||||
FROM of_node_request_reports
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_traffic_hourly;
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
-- +goose Up
|
||||
-- Hourly capacity rollups (avg CPU/memory + counter min/max for in-hour delta approximation).
|
||||
-- Network/disk counters are cumulative; max-min within an hour approximates that hour's delta
|
||||
-- (cross-hour continuity is intentionally approximate for dashboard trends).
|
||||
CREATE TABLE IF NOT EXISTS of_node_metric_capacity_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
cpu_usage_sum SimpleAggregateFunction(sum, Float64),
|
||||
cpu_usage_count SimpleAggregateFunction(sum, UInt64),
|
||||
memory_usage_sum SimpleAggregateFunction(sum, Float64),
|
||||
memory_usage_count SimpleAggregateFunction(sum, UInt64),
|
||||
network_rx_min SimpleAggregateFunction(min, Int64),
|
||||
network_rx_max SimpleAggregateFunction(max, Int64),
|
||||
network_tx_min SimpleAggregateFunction(min, Int64),
|
||||
network_tx_max SimpleAggregateFunction(max, Int64),
|
||||
disk_read_min SimpleAggregateFunction(min, Int64),
|
||||
disk_read_max SimpleAggregateFunction(max, Int64),
|
||||
disk_write_min SimpleAggregateFunction(min, Int64),
|
||||
disk_write_max SimpleAggregateFunction(max, Int64)
|
||||
)
|
||||
ENGINE = AggregatingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour)
|
||||
TTL hour + INTERVAL 30 DAY;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_metric_capacity_hourly_mv
|
||||
TO of_node_metric_capacity_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(captured_at) AS hour,
|
||||
sum(cpu_usage_percent) AS cpu_usage_sum,
|
||||
toUInt64(count()) AS cpu_usage_count,
|
||||
sum(if(memory_total_bytes > 0, (memory_used_bytes * 100.0) / memory_total_bytes, 0)) AS memory_usage_sum,
|
||||
toUInt64(countIf(memory_total_bytes > 0)) AS memory_usage_count,
|
||||
min(network_rx_bytes) AS network_rx_min,
|
||||
max(network_rx_bytes) AS network_rx_max,
|
||||
min(network_tx_bytes) AS network_tx_min,
|
||||
max(network_tx_bytes) AS network_tx_max,
|
||||
min(disk_read_bytes) AS disk_read_min,
|
||||
max(disk_read_bytes) AS disk_read_max,
|
||||
min(disk_write_bytes) AS disk_write_min,
|
||||
max(disk_write_bytes) AS disk_write_max
|
||||
FROM of_node_metric_snapshots
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
-- Hourly OpenResty counter rollups (min/max per node-hour for delta approximation).
|
||||
CREATE TABLE IF NOT EXISTS of_node_openresty_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
openresty_rx_min SimpleAggregateFunction(min, Int64),
|
||||
openresty_rx_max SimpleAggregateFunction(max, Int64),
|
||||
openresty_tx_min SimpleAggregateFunction(min, Int64),
|
||||
openresty_tx_max SimpleAggregateFunction(max, Int64)
|
||||
)
|
||||
ENGINE = AggregatingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour)
|
||||
TTL hour + INTERVAL 30 DAY;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_openresty_hourly_mv
|
||||
TO of_node_openresty_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(captured_at) AS hour,
|
||||
min(openresty_rx_bytes) AS openresty_rx_min,
|
||||
max(openresty_rx_bytes) AS openresty_rx_max,
|
||||
min(openresty_tx_bytes) AS openresty_tx_min,
|
||||
max(openresty_tx_bytes) AS openresty_tx_max
|
||||
FROM of_node_obs_openresty
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW IF EXISTS of_node_openresty_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_openresty_hourly;
|
||||
DROP VIEW IF EXISTS of_node_metric_capacity_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_metric_capacity_hourly;
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
-- +goose Up
|
||||
-- Hourly traffic rollups: 30d TTL + UV aggregation semantics.
|
||||
--
|
||||
-- unique_visitor_count on of_node_request_reports is per short report window
|
||||
-- (agent local distinct count for that window only). Summing those values in the
|
||||
-- MV (and again via SummingMergeTree part merges) invents a "true UV" number that
|
||||
-- double-counts visitors across windows. Prefer max() as a peak-window estimate;
|
||||
-- still NOT distinct visitors across the hour — UI/API must not overclaim.
|
||||
|
||||
ALTER TABLE of_node_traffic_hourly
|
||||
MODIFY TTL toDateTime(hour) + INTERVAL 30 DAY;
|
||||
|
||||
DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
|
||||
|
||||
CREATE MATERIALIZED VIEW of_node_traffic_hourly_mv
|
||||
TO of_node_traffic_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(window_ended_at) AS hour,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
-- Peak per-window UV estimate for the hour; not true cross-window distinct UV.
|
||||
max(unique_visitor_count) AS unique_visitor_count
|
||||
FROM of_node_request_reports
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
-- +goose Down
|
||||
-- TTL reverse is not safe without table rewrite; restore prior MV definition only.
|
||||
DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
|
||||
|
||||
CREATE MATERIALIZED VIEW of_node_traffic_hourly_mv
|
||||
TO of_node_traffic_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(window_ended_at) AS hour,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
sum(unique_visitor_count) AS unique_visitor_count
|
||||
FROM of_node_request_reports
|
||||
GROUP BY node_id, hour;
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
-- +goose Up
|
||||
-- One-time historical backfill for hours not yet present in rollup tables.
|
||||
-- MV only ingests rows after creation; without this, 24h charts rely on raw merge forever.
|
||||
-- ANTI JOIN avoids double-counting hours already filled by the live MV.
|
||||
|
||||
INSERT INTO of_node_metric_capacity_hourly
|
||||
SELECT
|
||||
s.node_id,
|
||||
toStartOfHour(s.captured_at) AS hour,
|
||||
sum(s.cpu_usage_percent) AS cpu_usage_sum,
|
||||
toUInt64(count()) AS cpu_usage_count,
|
||||
sum(if(s.memory_total_bytes > 0, (s.memory_used_bytes * 100.0) / s.memory_total_bytes, 0)) AS memory_usage_sum,
|
||||
toUInt64(countIf(s.memory_total_bytes > 0)) AS memory_usage_count,
|
||||
min(s.network_rx_bytes) AS network_rx_min,
|
||||
max(s.network_rx_bytes) AS network_rx_max,
|
||||
min(s.network_tx_bytes) AS network_tx_min,
|
||||
max(s.network_tx_bytes) AS network_tx_max,
|
||||
min(s.disk_read_bytes) AS disk_read_min,
|
||||
max(s.disk_read_bytes) AS disk_read_max,
|
||||
min(s.disk_write_bytes) AS disk_write_min,
|
||||
max(s.disk_write_bytes) AS disk_write_max
|
||||
FROM of_node_metric_snapshots AS s
|
||||
ANTI JOIN
|
||||
(
|
||||
SELECT
|
||||
node_id,
|
||||
hour
|
||||
FROM of_node_metric_capacity_hourly
|
||||
GROUP BY
|
||||
node_id,
|
||||
hour
|
||||
) AS existing
|
||||
ON s.node_id = existing.node_id AND toStartOfHour(s.captured_at) = existing.hour
|
||||
WHERE s.captured_at >= now() - INTERVAL 30 DAY
|
||||
GROUP BY
|
||||
s.node_id,
|
||||
hour;
|
||||
|
||||
INSERT INTO of_node_openresty_hourly
|
||||
SELECT
|
||||
s.node_id,
|
||||
toStartOfHour(s.captured_at) AS hour,
|
||||
min(s.openresty_rx_bytes) AS openresty_rx_min,
|
||||
max(s.openresty_rx_bytes) AS openresty_rx_max,
|
||||
min(s.openresty_tx_bytes) AS openresty_tx_min,
|
||||
max(s.openresty_tx_bytes) AS openresty_tx_max
|
||||
FROM of_node_obs_openresty AS s
|
||||
ANTI JOIN
|
||||
(
|
||||
SELECT
|
||||
node_id,
|
||||
hour
|
||||
FROM of_node_openresty_hourly
|
||||
GROUP BY
|
||||
node_id,
|
||||
hour
|
||||
) AS existing
|
||||
ON s.node_id = existing.node_id AND toStartOfHour(s.captured_at) = existing.hour
|
||||
WHERE s.captured_at >= now() - INTERVAL 30 DAY
|
||||
GROUP BY
|
||||
s.node_id,
|
||||
hour;
|
||||
|
||||
-- +goose Down
|
||||
-- Backfill is additive; down does not remove historical rollup rows (TTL still applies).
|
||||
SELECT 1;
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE of_node_access_logs ADD COLUMN IF NOT EXISTS bytes_sent UInt64;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS bytes_sent;
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
-- +goose Up
|
||||
-- L1 access log: request body/header length (接收数据) and optional request duration.
|
||||
ALTER TABLE of_node_access_logs
|
||||
ADD COLUMN IF NOT EXISTS request_length UInt64 DEFAULT 0;
|
||||
|
||||
ALTER TABLE of_node_access_logs
|
||||
ADD COLUMN IF NOT EXISTS request_time_ms UInt32 DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS request_time_ms;
|
||||
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS request_length;
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
-- +goose Up
|
||||
-- M5: L2 edge health table, L1 access-log hourly rollup, drop deprecated pre-aggregation tables.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_edge_health
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
status LowCardinality(String),
|
||||
connections Int64,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
TTL toDateTime(captured_at) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- Hourly business traffic from access logs (Server-side only; Agent never writes this).
|
||||
-- SummingMergeTree merges request/error/bytes; UV is not stored (query raw for exact UV).
|
||||
CREATE TABLE IF NOT EXISTS of_access_log_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime('UTC'),
|
||||
host String,
|
||||
request_count UInt64,
|
||||
error_count UInt64,
|
||||
bytes_sent UInt64,
|
||||
request_length UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour, host)
|
||||
TTL hour + INTERVAL 90 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_access_log_hourly_mv
|
||||
TO of_access_log_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(logged_at) AS hour,
|
||||
host,
|
||||
toUInt64(count()) AS request_count,
|
||||
toUInt64(countIf(status_code >= 500)) AS error_count,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length
|
||||
FROM of_node_access_logs
|
||||
GROUP BY node_id, hour, host;
|
||||
|
||||
-- Deprecated pre-aggregation paths (business traffic is access logs; OR throughput is not authoritative).
|
||||
DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_traffic_hourly;
|
||||
DROP VIEW IF EXISTS of_node_openresty_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_openresty_hourly;
|
||||
DROP TABLE IF EXISTS of_node_request_reports;
|
||||
DROP TABLE IF EXISTS of_node_obs_openresty;
|
||||
|
||||
-- +goose Down
|
||||
CREATE TABLE IF NOT EXISTS of_node_obs_openresty
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
captured_at DateTime64(3, 'UTC'),
|
||||
openresty_rx_bytes Int64,
|
||||
openresty_tx_bytes Int64,
|
||||
openresty_connections Int64,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(captured_at)
|
||||
ORDER BY (node_id, captured_at, id)
|
||||
TTL toDateTime(captured_at) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_request_reports
|
||||
(
|
||||
id UInt64,
|
||||
node_id String,
|
||||
window_started_at DateTime64(3, 'UTC'),
|
||||
window_ended_at DateTime64(3, 'UTC'),
|
||||
request_count Int64,
|
||||
error_count Int64,
|
||||
unique_visitor_count Int64,
|
||||
status_codes_json String,
|
||||
top_domains_json String,
|
||||
source_countries_json String,
|
||||
created_at DateTime64(3, 'UTC')
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(window_ended_at)
|
||||
ORDER BY (node_id, window_ended_at, window_started_at, id)
|
||||
TTL toDateTime(window_ended_at) + INTERVAL 30 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_traffic_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
request_count UInt64,
|
||||
error_count UInt64,
|
||||
unique_visitor_count UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_traffic_hourly_mv
|
||||
TO of_node_traffic_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(window_ended_at) AS hour,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
max(unique_visitor_count) AS unique_visitor_count
|
||||
FROM of_node_request_reports
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS of_node_openresty_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
openresty_rx_min SimpleAggregateFunction(min, Int64),
|
||||
openresty_rx_max SimpleAggregateFunction(max, Int64),
|
||||
openresty_tx_min SimpleAggregateFunction(min, Int64),
|
||||
openresty_tx_max SimpleAggregateFunction(max, Int64)
|
||||
)
|
||||
ENGINE = AggregatingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour)
|
||||
TTL hour + INTERVAL 30 DAY;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_openresty_hourly_mv
|
||||
TO of_node_openresty_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(captured_at) AS hour,
|
||||
min(openresty_rx_bytes) AS openresty_rx_min,
|
||||
max(openresty_rx_bytes) AS openresty_rx_max,
|
||||
min(openresty_tx_bytes) AS openresty_tx_min,
|
||||
max(openresty_tx_bytes) AS openresty_tx_max
|
||||
FROM of_node_obs_openresty
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
DROP VIEW IF EXISTS of_access_log_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_access_log_hourly;
|
||||
DROP TABLE IF EXISTS of_node_edge_health;
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
-- +goose Up
|
||||
-- One-time historical backfill for of_access_log_hourly.
|
||||
-- MV only ingests rows after creation; without this, 24h charts fall back to raw access logs.
|
||||
-- ANTI JOIN avoids double-counting (node_id, hour, host) already filled by the live MV.
|
||||
-- UV is intentionally NOT stored in of_access_log_hourly (SummingMergeTree counts only).
|
||||
|
||||
INSERT INTO of_access_log_hourly
|
||||
SELECT
|
||||
s.node_id,
|
||||
toStartOfHour(s.logged_at) AS hour,
|
||||
s.host,
|
||||
toUInt64(count()) AS request_count,
|
||||
toUInt64(countIf(s.status_code >= 500)) AS error_count,
|
||||
sum(s.bytes_sent) AS bytes_sent,
|
||||
sum(s.request_length) AS request_length
|
||||
FROM of_node_access_logs AS s
|
||||
ANTI JOIN
|
||||
(
|
||||
SELECT
|
||||
node_id,
|
||||
hour,
|
||||
host
|
||||
FROM of_access_log_hourly
|
||||
GROUP BY
|
||||
node_id,
|
||||
hour,
|
||||
host
|
||||
) AS existing
|
||||
ON s.node_id = existing.node_id
|
||||
AND toStartOfHour(s.logged_at) = existing.hour
|
||||
AND s.host = existing.host
|
||||
WHERE s.logged_at >= now() - INTERVAL 90 DAY
|
||||
GROUP BY
|
||||
s.node_id,
|
||||
hour,
|
||||
s.host;
|
||||
|
||||
-- +goose Down
|
||||
-- Backfill is additive; down does not remove historical rollup rows (TTL still applies).
|
||||
SELECT 1;
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE of_node_access_logs
|
||||
ADD COLUMN IF NOT EXISTS user_agent String DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS user_agent;
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE of_node_access_logs
|
||||
ADD COLUMN IF NOT EXISTS cache_status String DEFAULT '';
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE of_node_access_logs DROP COLUMN IF EXISTS cache_status;
|
||||
Reference in New Issue
Block a user