mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-09-28 21:56:36 +08:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b312460ddf | |||
| 50f7257d93 | |||
| 336185f01c | |||
| 44bba0f19a | |||
| f0eca028f9 | |||
| 58624db397 | |||
| 38946d1af5 | |||
| caf2ffcff4 | |||
| 0e86fe3547 | |||
| 6525bef15d | |||
| 3e910f1961 | |||
| 28c14eb054 | |||
| ae618905a3 | |||
| 5ad151469c | |||
| 6467b32d8e | |||
| cf72420815 | |||
| 34225cb88a | |||
| 389f02b6b0 | |||
| 2fcbb945fb | |||
| 23501259b2 | |||
| c561e65cd3 |
+3
-3
@@ -29,8 +29,8 @@ DB_ENABLED=true
|
||||
# SQLITE_PATH=./data/openflare.db
|
||||
DB_HOST=postgres
|
||||
DB_PORT=5432
|
||||
DB_USERNAME=postgres
|
||||
DB_PASSWORD=postgres
|
||||
DB_USERNAME=openflare
|
||||
DB_PASSWORD=replace-with-strong-password
|
||||
DB_NAME=openflare
|
||||
DB_SSL_MODE=disable
|
||||
DB_TIMEZONE=Asia/Shanghai
|
||||
@@ -51,7 +51,7 @@ REDIS_KEY_PREFIX=openflare:
|
||||
# ─── ClickHouse(必需)────────────────────────────────────────────────────
|
||||
CLICKHOUSE_HOST=clickhouse:9000
|
||||
CLICKHOUSE_USERNAME=default
|
||||
CLICKHOUSE_PASSWORD=123456
|
||||
CLICKHOUSE_PASSWORD=replace-with-clickhouse-password
|
||||
CLICKHOUSE_NAME=openflare
|
||||
|
||||
# ─── 日志 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
+19
-13
@@ -56,7 +56,7 @@ Quick links:
|
||||
```yaml
|
||||
services:
|
||||
openflare:
|
||||
image: ghcr.io/rain-kl/openflare-server:latest
|
||||
image: ghcr.io/rain-kl/openflare:latest
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
environment:
|
||||
@@ -64,7 +64,7 @@ services:
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./uploads:/app/uploads
|
||||
- openflare_uploads:/app/uploads
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -77,13 +77,13 @@ services:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: openflare
|
||||
POSTGRES_USER: openflare
|
||||
POSTGRES_PASSWORD: replace-with-strong-password
|
||||
POSTGRES_DB: ${DB_NAME:-openflare}
|
||||
POSTGRES_USER: ${DB_USERNAME:-openflare}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-replace-with-strong-password}
|
||||
volumes:
|
||||
- ./data/postgres_data:/var/lib/postgresql/data
|
||||
- openflare_postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openflare -d openflare"]
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-openflare} -d ${DB_NAME:-openflare}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -93,7 +93,7 @@ services:
|
||||
restart: unless-stopped
|
||||
command: ["valkey-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- ./data/valkey:/data
|
||||
- openflare_redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
@@ -105,19 +105,25 @@ services:
|
||||
image: clickhouse/clickhouse-server:25.3-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CLICKHOUSE_DB: openflare
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: 123456
|
||||
CLICKHOUSE_DB: ${CLICKHOUSE_NAME:-openflare}
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USERNAME:-default}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
||||
TZ: ${TZ:-Asia/Shanghai}
|
||||
volumes:
|
||||
- ./data/clickhouse_data:/var/lib/clickhouse
|
||||
- openflare_clickhouse_data:/var/lib/clickhouse
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
||||
test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USERNAME:-default}", "--password", "${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}", "--query", "SELECT 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 15s
|
||||
|
||||
volumes:
|
||||
openflare_uploads:
|
||||
openflare_postgres_data:
|
||||
openflare_redis_data:
|
||||
openflare_clickhouse_data:
|
||||
```
|
||||
|
||||
```bash
|
||||
|
||||
@@ -48,16 +48,36 @@ OpenFlare 是开源 CDN 编排与边缘安全平台。它支持反向代理、
|
||||
* **SSO 单点登录**:支持 GitHub OAuth 与标准 OIDC 协议,无缝接入企业身份提供商实现统一登录。
|
||||
* **统一观测**:聚合节点请求指标、实时访问日志明细、宿主机与 Nginx 资源快照、健康事件以及网络波动补传缓冲。
|
||||
|
||||
## 界面预览
|
||||
|
||||
### 仪表盘总览
|
||||
|
||||

|
||||
|
||||
### 节点详情
|
||||
|
||||

|
||||
|
||||
### 配置新增
|
||||
|
||||

|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 启动 Server
|
||||
|
||||
使用 docker-compose
|
||||
|
||||
```bash
|
||||
# 下载环境变量模板并创建 .env 文件
|
||||
curl -o .env.example https://raw.githubusercontent.com/Rain-kl/OpenFlare/refs/heads/main/.env.example
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
```yaml
|
||||
services:
|
||||
openflare:
|
||||
image: ghcr.io/rain-kl/openflare-server:latest
|
||||
image: ghcr.io/rain-kl/openflare:latest
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
environment:
|
||||
@@ -65,7 +85,7 @@ services:
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./uploads:/app/uploads
|
||||
- openflare_uploads:/app/uploads
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -78,13 +98,13 @@ services:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: openflare
|
||||
POSTGRES_USER: openflare
|
||||
POSTGRES_PASSWORD: replace-with-strong-password
|
||||
POSTGRES_DB: ${DB_NAME:-openflare}
|
||||
POSTGRES_USER: ${DB_USERNAME:-openflare}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-replace-with-strong-password}
|
||||
volumes:
|
||||
- ./data/postgres_data:/var/lib/postgresql/data
|
||||
- openflare_postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openflare -d openflare"]
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-openflare} -d ${DB_NAME:-openflare}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -94,7 +114,7 @@ services:
|
||||
restart: unless-stopped
|
||||
command: ["valkey-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- ./data/valkey:/data
|
||||
- openflare_redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
@@ -106,19 +126,25 @@ services:
|
||||
image: clickhouse/clickhouse-server:25.3-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CLICKHOUSE_DB: openflare
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: 123456
|
||||
CLICKHOUSE_DB: ${CLICKHOUSE_NAME:-openflare}
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USERNAME:-default}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
||||
TZ: ${TZ:-Asia/Shanghai}
|
||||
volumes:
|
||||
- ./data/clickhouse_data:/var/lib/clickhouse
|
||||
- openflare_clickhouse_data:/var/lib/clickhouse
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
||||
test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USERNAME:-default}", "--password", "${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}", "--query", "SELECT 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 15s
|
||||
|
||||
volumes:
|
||||
openflare_uploads:
|
||||
openflare_postgres_data:
|
||||
openflare_redis_data:
|
||||
openflare_clickhouse_data:
|
||||
```
|
||||
|
||||
详细部署说明见 [部署文档](https://open-flare.pages.dev/deployment/deployment)。
|
||||
@@ -150,20 +176,6 @@ docker run -d --name openflare-agent --restart unless-stopped \
|
||||
ghcr.io/rain-kl/openflare-agent:latest
|
||||
```
|
||||
|
||||
## 界面预览
|
||||
|
||||
### 仪表盘总览
|
||||
|
||||

|
||||
|
||||
### 节点详情
|
||||
|
||||

|
||||
|
||||
### 配置新增
|
||||
|
||||

|
||||
|
||||
## 开源协议
|
||||
|
||||
本项目采用 [Apache License 2.0](./LICENSE) 开源。
|
||||
|
||||
+3
-3
@@ -106,8 +106,8 @@ clickhouse:
|
||||
username: "default"
|
||||
password: "123456"
|
||||
database: "openflare"
|
||||
max_idle_conn: 10
|
||||
max_open_conn: 100
|
||||
max_idle_conn: 20
|
||||
max_open_conn: 50
|
||||
conn_max_lifetime: 3600
|
||||
dial_timeout: 5
|
||||
block_buffer_size: 10
|
||||
block_buffer_size: 100
|
||||
|
||||
+14
-9
@@ -5,7 +5,7 @@ services:
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
VERSION: v0.9.9
|
||||
# image: ghcr.io/rain-kl/openflare-server:latest
|
||||
# image: ghcr.io/rain-kl/openflare:latest
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
environment:
|
||||
@@ -34,13 +34,13 @@ services:
|
||||
ports:
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_DB: openflare
|
||||
POSTGRES_USER: openflare
|
||||
POSTGRES_PASSWORD: replace-with-strong-password
|
||||
POSTGRES_DB: ${DB_NAME:-openflare}
|
||||
POSTGRES_USER: ${DB_USERNAME:-openflare}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-replace-with-strong-password}
|
||||
volumes:
|
||||
- ./data/postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openflare -d openflare"]
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-openflare} -d ${DB_NAME:-openflare}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -74,18 +74,23 @@ services:
|
||||
image: clickhouse/clickhouse-server:25.3-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CLICKHOUSE_DB: openflare
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: 123456
|
||||
CLICKHOUSE_DB: ${CLICKHOUSE_NAME:-openflare}
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USERNAME:-default}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
||||
TZ: ${TZ:-Asia/Shanghai}
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 262144
|
||||
hard: 262144
|
||||
ports:
|
||||
- "${CLICKHOUSE_HTTP_PORT:-8123}:8123"
|
||||
- "${CLICKHOUSE_NATIVE_PORT:-9000}:9000"
|
||||
volumes:
|
||||
- ./data/clickhouse_data:/var/lib/clickhouse
|
||||
- ./docker/clickhouse/config.d:/etc/clickhouse-server/config.d
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
||||
test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USERNAME:-default}", "--password", "${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}", "--query", "SELECT 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0"?>
|
||||
<clickhouse>
|
||||
<max_concurrent_queries>50</max_concurrent_queries>
|
||||
<background_pool_size>8</background_pool_size>
|
||||
<background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
|
||||
</clickhouse>
|
||||
@@ -18,6 +18,33 @@ sidebar: false
|
||||
|
||||
## [unreleased]
|
||||
|
||||
## [v3.1.1] - 2026-07-06
|
||||
|
||||
### 修改
|
||||
|
||||
- 将 `cap_login_enabled` 默认值由 `true` 变更为 `false`,默认关闭登录界面 PoW 人机验证。
|
||||
|
||||
## [v3.1.0] - 2026-07-04
|
||||
|
||||
### 修改
|
||||
|
||||
- 修复 ClickHouse TTL 迁移:`DateTime64` 时间列通过 `toDateTime()` 转换后再设置 TTL,避免 goose 启动报错;移除 `MODIFY ORDER BY`(ClickHouse 不允许将排序键缩短至短于隐式主键前缀)。
|
||||
- ClickHouse 遗留治理 Phase 2:保留期清理改为 TTL `MATERIALIZE TTL`(全量清理使用 `TRUNCATE`),消除定时 `ALTER DELETE` mutation;移除 GORM 双连接池并统一 `ChConn` 读路径;查询侧去除 `trim(remote_addr)`;`wait_for_async_insert` 调整为 1;新增 `/admin/status/clickhouse` 运维指标与 `of_node_traffic_hourly` 预聚合 MV。
|
||||
- ClickHouse 写入路径优化:移除 Agent 心跳路径中的同步 `ALTER DELETE` 保留清理;`batchwriter` 新增 `MinBatchSize` 抑制过小批次定时 flush;可观测 writer 批次提升至 500、flush 间隔 5s,并为 OpenResty/FRPS/FRPC 补全去重。
|
||||
- ClickHouse 客户端启用 `async_insert` 异步写入缓冲,并调高 `block_buffer_size` 与连接池默认值,降低小 part 与连接争用。
|
||||
- Dashboard 与节点可观测 API 消除无 `LIMIT` 全表扫描、增加短 TTL 内存缓存,前端轮询间隔分别调整为 60s/30s。
|
||||
- 访问日志与 WAF IP 组同步改为 ClickHouse 侧聚合与 SQL 分页,默认查询窗口限制为近 7 天,浏览器分布查询增加 Top 100 限制。
|
||||
- ClickHouse 分析表新增 TTL 自动过期策略:`w_user_access_logs` 180 天、`of_node_access_logs` 90 天,其余节点观测与聚合表 30 天。
|
||||
- 节点访问日志写入 ClickHouse 时对 `remote_addr` 执行 `TrimSpace` 规范化,避免首尾空白影响 IP 汇总统计。
|
||||
- 数据库自动清理任务新增 OpenResty、FRPS、FRPC 观测表清理目标。
|
||||
- Docker 部署为 ClickHouse 服务增加 `nofile` ulimits 与 `docker/clickhouse/config.d/performance.xml` 性能配置挂载,限制 `max_concurrent_queries`、`background_pool_size` 与 `background_merges_mutations_concurrency_ratio`,降低高负载下的合并与查询争用。
|
||||
- 审计访问日志写入 ClickHouse 时仅保留安全相关请求头(Authorization、Cookie、X-Forwarded-For、X-Real-IP、User-Agent、Content-Type),敏感头字段以 SHA-256 摘要脱敏,并将序列化后的 headers 载荷上限收紧至 2KB,减小 `w_user_access_logs` 行宽与 merge CPU 开销。
|
||||
- 隐藏侧边栏“文档库”分组中的“规范示例”与“接口文档”,并将“使用文档”及其他相关页面的文档链接统一跳转至外部文档 https://open-flare.pages.dev/
|
||||
- 修复全局搜索数据源覆盖不全的问题,补全了所有核心业务控制台页面(节点、规则、域名、证书、DNS、源站、WAF、IP组、Pages、版本发布、访问日志、应用记录和性能调优)及缺失的管理员专有页面(存储、数据、推送、日志)的搜索检索支持。
|
||||
- 修复系统自更新(Updater)检测上游 GitHub Action Release 时,因资产包名称前缀(`openflare-server`)与仓库名不完全一致导致匹配失败并报错“未找到兼容的 Release”的问题。
|
||||
- 修复系统设置页面(`/admin/settings`)基于 URL `tab` 参数的定位逻辑,补全缺失的 `openflare-ops` (OpenFlare) Tab,且在不带参数时默认选中 OpenFlare 选项卡。
|
||||
- 移除系统设置中 OpenFlare 标签页下的“版本信息”卡片及对应的升级管理弹窗逻辑。
|
||||
|
||||
## [v3.0.2] - 2026-06-30
|
||||
|
||||
### 修复
|
||||
|
||||
@@ -81,6 +81,7 @@ Agent:
|
||||
仓库根目录已提供完整 `docker-compose.yaml`(含 PostgreSQL、Redis、ClickHouse、Jaeger)。
|
||||
|
||||
```bash
|
||||
curl -o .env.example https://raw.githubusercontent.com/Rain-kl/OpenFlare/refs/heads/main/.env.example
|
||||
cp .env.example .env
|
||||
# 编辑 .env,至少修改 APP_SESSION_SECRET 与数据库密码
|
||||
docker compose up -d
|
||||
|
||||
+41
-27
@@ -27,7 +27,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
openflare:
|
||||
image: ghcr.io/rain-kl/openflare-server:latest
|
||||
image: ghcr.io/rain-kl/openflare:latest
|
||||
container_name: openflare-server
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
@@ -100,7 +100,7 @@ docker compose up -d
|
||||
```yaml
|
||||
services:
|
||||
openflare:
|
||||
image: ghcr.io/rain-kl/openflare-server:latest
|
||||
image: ghcr.io/rain-kl/openflare:latest
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
environment:
|
||||
@@ -108,7 +108,7 @@ services:
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./uploads:/app/uploads
|
||||
- openflare_uploads:/app/uploads
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -121,13 +121,13 @@ services:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: openflare
|
||||
POSTGRES_USER: openflare
|
||||
POSTGRES_PASSWORD: replace-with-strong-password
|
||||
POSTGRES_DB: ${DB_NAME:-openflare}
|
||||
POSTGRES_USER: ${DB_USERNAME:-openflare}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-replace-with-strong-password}
|
||||
volumes:
|
||||
- ./data/postgres_data:/var/lib/postgresql/data
|
||||
- openflare_postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openflare -d openflare"]
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-openflare} -d ${DB_NAME:-openflare}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -137,7 +137,7 @@ services:
|
||||
restart: unless-stopped
|
||||
command: ["valkey-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- ./data/valkey:/data
|
||||
- openflare_redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
@@ -149,24 +149,31 @@ services:
|
||||
image: clickhouse/clickhouse-server:25.3-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CLICKHOUSE_DB: openflare
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: replace-with-clickhouse-password
|
||||
CLICKHOUSE_DB: ${CLICKHOUSE_NAME:-openflare}
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USERNAME:-default}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
||||
TZ: ${TZ:-Asia/Shanghai}
|
||||
volumes:
|
||||
- ./data/clickhouse_data:/var/lib/clickhouse
|
||||
- openflare_clickhouse_data:/var/lib/clickhouse
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
||||
test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USERNAME:-default}", "--password", "${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}", "--query", "SELECT 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 15s
|
||||
|
||||
volumes:
|
||||
openflare_uploads:
|
||||
openflare_postgres_data:
|
||||
openflare_redis_data:
|
||||
openflare_clickhouse_data:
|
||||
```
|
||||
|
||||
创建对应的 `.env` 文件来配置系统环境变量(可复制并修改根目录下的 `.env.example`):
|
||||
|
||||
```bash
|
||||
curl -o .env.example https://raw.githubusercontent.com/Rain-kl/OpenFlare/refs/heads/main/.env.example
|
||||
cp .env.example .env
|
||||
# 编辑 .env 文件,填入对应的数据库、Redis、ClickHouse 连接地址、密码与 APP_SESSION_SECRET
|
||||
|
||||
@@ -188,7 +195,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
openflare:
|
||||
image: ghcr.io/rain-kl/openflare-server:latest
|
||||
image: ghcr.io/rain-kl/openflare:latest
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
environment:
|
||||
@@ -199,7 +206,7 @@ services:
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./uploads:/app/uploads
|
||||
- openflare_uploads:/app/uploads
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -214,13 +221,13 @@ services:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: openflare
|
||||
POSTGRES_USER: openflare
|
||||
POSTGRES_PASSWORD: replace-with-strong-password
|
||||
POSTGRES_DB: ${DB_NAME:-openflare}
|
||||
POSTGRES_USER: ${DB_USERNAME:-openflare}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-replace-with-strong-password}
|
||||
volumes:
|
||||
- ./data/postgres_data:/var/lib/postgresql/data
|
||||
- openflare_postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openflare -d openflare"]
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-openflare} -d ${DB_NAME:-openflare}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -230,7 +237,7 @@ services:
|
||||
restart: unless-stopped
|
||||
command: ["valkey-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- ./data/valkey:/data
|
||||
- openflare_redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
@@ -252,15 +259,21 @@ services:
|
||||
image: clickhouse/clickhouse-server:25.3-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CLICKHOUSE_DB: openflare
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: replace-with-clickhouse-password
|
||||
CLICKHOUSE_DB: ${CLICKHOUSE_NAME:-openflare}
|
||||
CLICKHOUSE_USER: ${CLICKHOUSE_USERNAME:-default}
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
||||
TZ: ${TZ:-Asia/Shanghai}
|
||||
volumes:
|
||||
- ./data/clickhouse_data:/var/lib/clickhouse
|
||||
- openflare_clickhouse_data:/var/lib/clickhouse
|
||||
|
||||
volumes:
|
||||
openflare_uploads:
|
||||
openflare_postgres_data:
|
||||
openflare_redis_data:
|
||||
openflare_clickhouse_data:
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
||||
test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USERNAME:-default}", "--password", "${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}", "--query", "SELECT 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -270,6 +283,7 @@ services:
|
||||
启动并验证:
|
||||
|
||||
```bash
|
||||
curl -o .env.example https://raw.githubusercontent.com/Rain-kl/OpenFlare/refs/heads/main/.env.example
|
||||
cp .env.example .env
|
||||
# 编辑 .env 文件并确保设置好 APP_SESSION_SECRET 密码
|
||||
|
||||
|
||||
+219
-5
@@ -309,6 +309,7 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "认证源 ID 或名称",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -386,6 +387,7 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "认证源 ID 或名称",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -453,6 +455,7 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "认证源 ID 或名称",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -1363,6 +1366,7 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "通道ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -1416,6 +1420,7 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "通道ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -1872,6 +1877,67 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/status/clickhouse": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"SessionCookie": []
|
||||
}
|
||||
],
|
||||
"description": "返回 ClickHouse parts、mutation、async_insert 队列等运维指标,需要管理员权限",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"admin"
|
||||
],
|
||||
"summary": "获取 ClickHouse 运行指标",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "获取成功",
|
||||
"schema": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/response.Any"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"$ref": "#/definitions/analytics.ClickHouseOperationalStats"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "ClickHouse 未启用",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "未登录",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "无管理员权限",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "内部错误",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/system-configs": {
|
||||
"get": {
|
||||
"security": [
|
||||
@@ -3368,6 +3434,7 @@ const docTemplate = `{
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "上传用户 ID",
|
||||
"name": "user_id",
|
||||
"in": "query"
|
||||
@@ -3691,6 +3758,11 @@ const docTemplate = `{
|
||||
],
|
||||
"summary": "获取用户列表",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"name": "email",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"minimum": 1,
|
||||
"type": "integer",
|
||||
@@ -3909,6 +3981,92 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"put": {
|
||||
"security": [
|
||||
{
|
||||
"SessionCookie": []
|
||||
}
|
||||
],
|
||||
"description": "更新指定用户的昵称、邮箱、管理员权限,并可选重置密码,需要管理员权限",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"admin"
|
||||
],
|
||||
"summary": "更新用户信息",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"description": "用户 ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"description": "更新参数",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/user.updateUserRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "更新成功",
|
||||
"schema": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/response.Any"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "参数错误",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "未登录",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "无管理员权限或尝试修改自身权限",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "用户不存在",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "内部错误",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"security": [
|
||||
{
|
||||
@@ -11144,6 +11302,7 @@ const docTemplate = `{
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "外部帐号绑定记录 ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -12948,6 +13107,29 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"analytics.ClickHouseOperationalStats": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"active_parts": {
|
||||
"type": "integer"
|
||||
},
|
||||
"async_insert_bytes": {
|
||||
"type": "integer"
|
||||
},
|
||||
"async_insert_queue": {
|
||||
"type": "integer"
|
||||
},
|
||||
"database": {
|
||||
"type": "string"
|
||||
},
|
||||
"pending_mutations": {
|
||||
"type": "integer"
|
||||
},
|
||||
"total_rows": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"apply_log.CleanupInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -13775,19 +13957,22 @@ const docTemplate = `{
|
||||
"source_countries": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
},
|
||||
"status_codes": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
},
|
||||
"top_domains": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
},
|
||||
"unique_visitor_count": {
|
||||
@@ -14217,7 +14402,7 @@ const docTemplate = `{
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer"
|
||||
"type": "string"
|
||||
},
|
||||
"is_active": {
|
||||
"type": "boolean"
|
||||
@@ -14252,7 +14437,7 @@ const docTemplate = `{
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer"
|
||||
"type": "string"
|
||||
},
|
||||
"is_active": {
|
||||
"type": "boolean"
|
||||
@@ -15092,6 +15277,11 @@ const docTemplate = `{
|
||||
"UploadStatusPending": "待使用",
|
||||
"UploadStatusUsed": "已使用"
|
||||
},
|
||||
"x-enum-descriptions": [
|
||||
"待使用",
|
||||
"已使用",
|
||||
"已删除"
|
||||
],
|
||||
"x-enum-varnames": [
|
||||
"UploadStatusPending",
|
||||
"UploadStatusUsed",
|
||||
@@ -18085,6 +18275,30 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"user.updateUserRequest": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"email"
|
||||
],
|
||||
"properties": {
|
||||
"email": {
|
||||
"type": "string",
|
||||
"maxLength": 255
|
||||
},
|
||||
"is_admin": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"nickname": {
|
||||
"type": "string",
|
||||
"maxLength": 64
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"minLength": 8
|
||||
}
|
||||
}
|
||||
},
|
||||
"user.updateUserStatusRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
+20
-14
@@ -37,22 +37,22 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
openflare:
|
||||
image: ghcr.io/rain-kl/openflare-server:latest
|
||||
image: ghcr.io/rain-kl/openflare:latest
|
||||
container_name: openflare-server
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./uploads:/app/uploads
|
||||
- openflare_uploads:/app/uploads
|
||||
environment:
|
||||
TZ: Asia/Shanghai
|
||||
APP_SESSION_SECRET: 'replace-with-a-long-random-string' # 生产环境请替换为长随机字符串
|
||||
DB_ENABLED: "true"
|
||||
DB_HOST: "postgres"
|
||||
DB_PORT: "5432"
|
||||
DB_USERNAME: "openflare"
|
||||
DB_PASSWORD: "replace-with-strong-password"
|
||||
DB_NAME: "openflare"
|
||||
DB_USERNAME: "${DB_USERNAME:-openflare}"
|
||||
DB_PASSWORD: "${DB_PASSWORD:-replace-with-strong-password}"
|
||||
DB_NAME: "${DB_NAME:-openflare}"
|
||||
REDIS_ENABLED: "true"
|
||||
REDIS_ADDRS: "redis:6379"
|
||||
CLICKHOUSE_ENABLED: "true"
|
||||
@@ -69,13 +69,13 @@ services:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: openflare
|
||||
POSTGRES_USER: openflare
|
||||
POSTGRES_PASSWORD: replace-with-strong-password
|
||||
POSTGRES_DB: ${DB_NAME:-openflare}
|
||||
POSTGRES_USER: ${DB_USERNAME:-openflare}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-replace-with-strong-password}
|
||||
volumes:
|
||||
- ./data/postgres_data:/var/lib/postgresql/data
|
||||
- openflare_postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U openflare -d openflare"]
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-openflare} -d ${DB_NAME:-openflare}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -85,7 +85,7 @@ services:
|
||||
restart: unless-stopped
|
||||
command: ["valkey-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- ./data/valkey:/data
|
||||
- openflare_redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
@@ -98,17 +98,23 @@ services:
|
||||
environment:
|
||||
CLICKHOUSE_DB: openflare
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: 123456
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
||||
TZ: Asia/Shanghai
|
||||
volumes:
|
||||
- ./data/clickhouse_data:/var/lib/clickhouse
|
||||
- openflare_clickhouse_data:/var/lib/clickhouse
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
||||
test: ["CMD", "clickhouse-client", "--user", "${CLICKHOUSE_USERNAME:-default}", "--password", "${CLICKHOUSE_PASSWORD:-replace-with-clickhouse-password}", "--query", "SELECT 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 15s
|
||||
|
||||
volumes:
|
||||
openflare_uploads:
|
||||
openflare_postgres_data:
|
||||
openflare_redis_data:
|
||||
openflare_clickhouse_data:
|
||||
```
|
||||
|
||||
启动服务:
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# ClickHouse CPU 性能优化计划
|
||||
|
||||
> PLAN_ID: `63ba981b`
|
||||
> 状态: 已完成(含 Phase 2 遗留治理)
|
||||
> 目标: 完成 P0–P2 优化,降低 ClickHouse CPU 占用
|
||||
|
||||
## 背景
|
||||
|
||||
ClickHouse CPU 偏高由写入侧(小 part 频繁 flush、心跳同步 DELETE mutation)与查询侧(无 LIMIT 全表扫、高频轮询、WAF 全量拉日志)叠加导致。
|
||||
|
||||
## PR Plan
|
||||
|
||||
### PR 1: 写入路径 P0 优化
|
||||
|
||||
- **Description:** 移除心跳路径同步 `ALTER DELETE`;为 `batchwriter` 增加 `MinBatchSize`;调大可观测 writer 批次与 flush 间隔;为 openresty/frps/frpc 补全去重。
|
||||
- **Files/components affected:** `internal/apps/openflare/agent/observability.go`, `internal/db/batchwriter/`, `internal/apps/openflare/chwriter/`, `internal/db/batchwriter/*_test.go`
|
||||
- **Dependencies:** None
|
||||
|
||||
### PR 2: ClickHouse 客户端与配置 P1
|
||||
|
||||
- **Description:** 启用 `async_insert` 等写入优化 settings;提高 `block_buffer_size` 默认值;更新 `config.example.yaml` 与配置模型注释。
|
||||
- **Files/components affected:** `internal/db/clickhouse.go`, `internal/config/model.go`, `internal/config/config.go`, `config.example.yaml`
|
||||
- **Dependencies:** None
|
||||
|
||||
### PR 3: Dashboard 与可观测查询 P0
|
||||
|
||||
- **Description:** 消除 `limit=0` 无界查询;复用已有限制数据构建趋势;增加服务端短 TTL 缓存;降低前端轮询频率。
|
||||
- **Files/components affected:** `internal/apps/openflare/dashboard/logics.go`, `internal/apps/openflare/observability/node_logics.go`, `frontend/app/(main)/page.tsx`, `frontend/app/(main)/nodes/components/node-observability.tsx`
|
||||
- **Dependencies:** None
|
||||
|
||||
### PR 4: 访问日志与 WAF 查询 P0/P1
|
||||
|
||||
- **Description:** WAF IP 同步改为 ClickHouse 侧聚合;IP 汇总与折叠日志 SQL 分页;消除 count 重复全量扫描;列表 API 强制默认时间窗口。
|
||||
- **Files/components affected:** `internal/apps/openflare/waf/ip_group_sync.go`, `internal/repository/analytics/node_access_log_stats.go`, `internal/model/openflare_access_log.go`, `internal/apps/openflare/observability/access_log_logics.go`, `internal/repository/analytics/access_log_stats.go`
|
||||
- **Dependencies:** None
|
||||
|
||||
### PR 5: ClickHouse DDL 与数据规范化 P1
|
||||
|
||||
- **Description:** 为 7 张分析表添加 TTL;收窄 `of_node_access_logs` ORDER BY;插入时规范化 `remote_addr`(去 trim 查询);将可观测 obs 三表纳入自动清理。
|
||||
- **Files/components affected:** `internal/db/migrator/goose/clickhouse/`, `internal/repository/analytics/node_access_log_writer.go`, `internal/apps/openflare/tasks/database_cleanup.go`, `internal/model/analytics/`
|
||||
- **Dependencies:** PR 1
|
||||
|
||||
### PR 6: 基础设施与审计减负 P2
|
||||
|
||||
- **Description:** Docker ClickHouse 服务端基础调优;审计日志 headers 截断/精简;更新 changelog。
|
||||
- **Files/components affected:** `docker-compose.yaml`, `docker/clickhouse/` (if needed), `internal/apps/risk_control/middleware.go`, `docs/changelog/index.md`
|
||||
- **Dependencies:** None
|
||||
@@ -161,7 +161,7 @@ Server 的所有核心基础配置定义在 `config.yaml` 中,且均支持环
|
||||
### 2. 人机安全校验 (PoW Captcha)
|
||||
| 配置键 (Key) | 数据类型 | 作用说明 | 默认值 |
|
||||
| --- | --- | --- | --- |
|
||||
| `cap_login_enabled` | `bool` | 是否在登录界面强制要求进行本地 PoW 算力防爆破人机验证 | `true` |
|
||||
| `cap_login_enabled` | `bool` | 是否在登录界面强制要求进行本地 PoW 算力防爆破人机验证 | `false` |
|
||||
| `cap_auto_solve` | `bool` | 打开页面后是否由浏览器自动开始后台背景计算算力(无需用户手动点击)| `true` |
|
||||
| `cap_challenge_count` | `int` | 人机验证所需的计算难题数。数量越大,计算要求时间越长(推荐 1~5) | `1` |
|
||||
| `cap_challenge_difficulty`| `int`| 每次计算所需的 PoW 哈希前缀匹配难度。推荐数值在 3-5 之间 | `4` |
|
||||
|
||||
+219
-5
@@ -302,6 +302,7 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "认证源 ID 或名称",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -379,6 +380,7 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "认证源 ID 或名称",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -446,6 +448,7 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "认证源 ID 或名称",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -1356,6 +1359,7 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "通道ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -1409,6 +1413,7 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "通道ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -1865,6 +1870,67 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/status/clickhouse": {
|
||||
"get": {
|
||||
"security": [
|
||||
{
|
||||
"SessionCookie": []
|
||||
}
|
||||
],
|
||||
"description": "返回 ClickHouse parts、mutation、async_insert 队列等运维指标,需要管理员权限",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"admin"
|
||||
],
|
||||
"summary": "获取 ClickHouse 运行指标",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "获取成功",
|
||||
"schema": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/response.Any"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"$ref": "#/definitions/analytics.ClickHouseOperationalStats"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "ClickHouse 未启用",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "未登录",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "无管理员权限",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "内部错误",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/admin/system-configs": {
|
||||
"get": {
|
||||
"security": [
|
||||
@@ -3361,6 +3427,7 @@
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "上传用户 ID",
|
||||
"name": "user_id",
|
||||
"in": "query"
|
||||
@@ -3684,6 +3751,11 @@
|
||||
],
|
||||
"summary": "获取用户列表",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"name": "email",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"minimum": 1,
|
||||
"type": "integer",
|
||||
@@ -3902,6 +3974,92 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"put": {
|
||||
"security": [
|
||||
{
|
||||
"SessionCookie": []
|
||||
}
|
||||
],
|
||||
"description": "更新指定用户的昵称、邮箱、管理员权限,并可选重置密码,需要管理员权限",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"admin"
|
||||
],
|
||||
"summary": "更新用户信息",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"description": "用户 ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"description": "更新参数",
|
||||
"name": "request",
|
||||
"in": "body",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"$ref": "#/definitions/user.updateUserRequest"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "更新成功",
|
||||
"schema": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/response.Any"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "参数错误",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "未登录",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "无管理员权限或尝试修改自身权限",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "用户不存在",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "内部错误",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/response.Any"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"security": [
|
||||
{
|
||||
@@ -11137,6 +11295,7 @@
|
||||
"parameters": [
|
||||
{
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "外部帐号绑定记录 ID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
@@ -12941,6 +13100,29 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"analytics.ClickHouseOperationalStats": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"active_parts": {
|
||||
"type": "integer"
|
||||
},
|
||||
"async_insert_bytes": {
|
||||
"type": "integer"
|
||||
},
|
||||
"async_insert_queue": {
|
||||
"type": "integer"
|
||||
},
|
||||
"database": {
|
||||
"type": "string"
|
||||
},
|
||||
"pending_mutations": {
|
||||
"type": "integer"
|
||||
},
|
||||
"total_rows": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"apply_log.CleanupInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -13768,19 +13950,22 @@
|
||||
"source_countries": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
},
|
||||
"status_codes": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
},
|
||||
"top_domains": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "integer"
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
},
|
||||
"unique_visitor_count": {
|
||||
@@ -14210,7 +14395,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer"
|
||||
"type": "string"
|
||||
},
|
||||
"is_active": {
|
||||
"type": "boolean"
|
||||
@@ -14245,7 +14430,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer"
|
||||
"type": "string"
|
||||
},
|
||||
"is_active": {
|
||||
"type": "boolean"
|
||||
@@ -15085,6 +15270,11 @@
|
||||
"UploadStatusPending": "待使用",
|
||||
"UploadStatusUsed": "已使用"
|
||||
},
|
||||
"x-enum-descriptions": [
|
||||
"待使用",
|
||||
"已使用",
|
||||
"已删除"
|
||||
],
|
||||
"x-enum-varnames": [
|
||||
"UploadStatusPending",
|
||||
"UploadStatusUsed",
|
||||
@@ -18078,6 +18268,30 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"user.updateUserRequest": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"email"
|
||||
],
|
||||
"properties": {
|
||||
"email": {
|
||||
"type": "string",
|
||||
"maxLength": 255
|
||||
},
|
||||
"is_admin": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"nickname": {
|
||||
"type": "string",
|
||||
"maxLength": 64
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"minLength": 8
|
||||
}
|
||||
}
|
||||
},
|
||||
"user.updateUserStatusRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
+140
-2
@@ -169,6 +169,21 @@ definitions:
|
||||
$ref: '#/definitions/github_com_Rain-kl_Wavelet_pkg_protocol.WAFIPGroup'
|
||||
type: array
|
||||
type: object
|
||||
analytics.ClickHouseOperationalStats:
|
||||
properties:
|
||||
active_parts:
|
||||
type: integer
|
||||
async_insert_bytes:
|
||||
type: integer
|
||||
async_insert_queue:
|
||||
type: integer
|
||||
database:
|
||||
type: string
|
||||
pending_mutations:
|
||||
type: integer
|
||||
total_rows:
|
||||
type: integer
|
||||
type: object
|
||||
apply_log.CleanupInput:
|
||||
properties:
|
||||
delete_all:
|
||||
@@ -713,14 +728,17 @@ definitions:
|
||||
type: integer
|
||||
source_countries:
|
||||
additionalProperties:
|
||||
format: int64
|
||||
type: integer
|
||||
type: object
|
||||
status_codes:
|
||||
additionalProperties:
|
||||
format: int64
|
||||
type: integer
|
||||
type: object
|
||||
top_domains:
|
||||
additionalProperties:
|
||||
format: int64
|
||||
type: integer
|
||||
type: object
|
||||
unique_visitor_count:
|
||||
@@ -1004,7 +1022,7 @@ definitions:
|
||||
created_by:
|
||||
type: string
|
||||
id:
|
||||
type: integer
|
||||
type: string
|
||||
is_active:
|
||||
type: boolean
|
||||
main_config:
|
||||
@@ -1027,7 +1045,7 @@ definitions:
|
||||
created_by:
|
||||
type: string
|
||||
id:
|
||||
type: integer
|
||||
type: string
|
||||
is_active:
|
||||
type: boolean
|
||||
version:
|
||||
@@ -1593,6 +1611,10 @@ definitions:
|
||||
UploadStatusDeleted: 已删除
|
||||
UploadStatusPending: 待使用
|
||||
UploadStatusUsed: 已使用
|
||||
x-enum-descriptions:
|
||||
- 待使用
|
||||
- 已使用
|
||||
- 已删除
|
||||
x-enum-varnames:
|
||||
- UploadStatusPending
|
||||
- UploadStatusUsed
|
||||
@@ -3577,6 +3599,23 @@ definitions:
|
||||
website:
|
||||
type: string
|
||||
type: object
|
||||
user.updateUserRequest:
|
||||
properties:
|
||||
email:
|
||||
maxLength: 255
|
||||
type: string
|
||||
is_admin:
|
||||
type: boolean
|
||||
nickname:
|
||||
maxLength: 64
|
||||
type: string
|
||||
password:
|
||||
maxLength: 64
|
||||
minLength: 8
|
||||
type: string
|
||||
required:
|
||||
- email
|
||||
type: object
|
||||
user.updateUserStatusRequest:
|
||||
properties:
|
||||
is_active:
|
||||
@@ -4085,6 +4124,7 @@ paths:
|
||||
description: 删除指定认证源及其关联的所有外部帐号绑定记录,警告:删除后相关用户将无法通过该源登录,需要管理员权限
|
||||
parameters:
|
||||
- description: 认证源 ID 或名称
|
||||
format: int64
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
@@ -4124,6 +4164,7 @@ paths:
|
||||
description: 更新指定 ID 的认证源配置。若 client_secret 字段为空,则保留原有密钥不变,需要管理员权限
|
||||
parameters:
|
||||
- description: 认证源 ID 或名称
|
||||
format: int64
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
@@ -4174,6 +4215,7 @@ paths:
|
||||
description: 启用或禁用指定认证源。尝试启用时将验证 Client ID 和 Client Secret 是否已配置,需要管理员权限
|
||||
parameters:
|
||||
- description: 认证源 ID 或名称
|
||||
format: int64
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
@@ -4670,6 +4712,7 @@ paths:
|
||||
description: 根据ID删除消息通道,需要管理员权限
|
||||
parameters:
|
||||
- description: 通道ID
|
||||
format: int64
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
@@ -4692,6 +4735,7 @@ paths:
|
||||
description: 修改消息通道配置,需要管理员权限
|
||||
parameters:
|
||||
- description: 通道ID
|
||||
format: int64
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
@@ -5016,6 +5060,42 @@ paths:
|
||||
summary: 获取系统状态信息
|
||||
tags:
|
||||
- admin
|
||||
/api/v1/admin/status/clickhouse:
|
||||
get:
|
||||
description: 返回 ClickHouse parts、mutation、async_insert 队列等运维指标,需要管理员权限
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: 获取成功
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: '#/definitions/response.Any'
|
||||
- properties:
|
||||
data:
|
||||
$ref: '#/definitions/analytics.ClickHouseOperationalStats'
|
||||
type: object
|
||||
"400":
|
||||
description: ClickHouse 未启用
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
"401":
|
||||
description: 未登录
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
"403":
|
||||
description: 无管理员权限
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
"500":
|
||||
description: 内部错误
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
security:
|
||||
- SessionCookie: []
|
||||
summary: 获取 ClickHouse 运行指标
|
||||
tags:
|
||||
- admin
|
||||
/api/v1/admin/system-configs:
|
||||
get:
|
||||
description: 返回所有系统配置列表,支持按配置类型(system/business)过滤,需要管理员权限
|
||||
@@ -5912,6 +5992,7 @@ paths:
|
||||
name: extension
|
||||
type: string
|
||||
- description: 上传用户 ID
|
||||
format: int64
|
||||
in: query
|
||||
name: user_id
|
||||
type: integer
|
||||
@@ -6108,6 +6189,9 @@ paths:
|
||||
get:
|
||||
description: 分页返回用户列表,支持按用户 ID 和用户名筛选,需要管理员权限
|
||||
parameters:
|
||||
- in: query
|
||||
name: email
|
||||
type: string
|
||||
- in: query
|
||||
minimum: 1
|
||||
name: page
|
||||
@@ -6291,6 +6375,59 @@ paths:
|
||||
summary: 获取用户详情
|
||||
tags:
|
||||
- admin
|
||||
put:
|
||||
consumes:
|
||||
- application/json
|
||||
description: 更新指定用户的昵称、邮箱、管理员权限,并可选重置密码,需要管理员权限
|
||||
parameters:
|
||||
- description: 用户 ID
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
type: integer
|
||||
- description: 更新参数
|
||||
in: body
|
||||
name: request
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/user.updateUserRequest'
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: 更新成功
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: '#/definitions/response.Any'
|
||||
- properties:
|
||||
data:
|
||||
type: string
|
||||
type: object
|
||||
"400":
|
||||
description: 参数错误
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
"401":
|
||||
description: 未登录
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
"403":
|
||||
description: 无管理员权限或尝试修改自身权限
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
"404":
|
||||
description: 用户不存在
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
"500":
|
||||
description: 内部错误
|
||||
schema:
|
||||
$ref: '#/definitions/response.Any'
|
||||
security:
|
||||
- SessionCookie: []
|
||||
summary: 更新用户信息
|
||||
tags:
|
||||
- admin
|
||||
/api/v1/admin/users/{id}/status:
|
||||
put:
|
||||
consumes:
|
||||
@@ -10642,6 +10779,7 @@ paths:
|
||||
description: 解除当前登录用户与指定外部帐号的绑定关系,需要登录
|
||||
parameters:
|
||||
- description: 外部帐号绑定记录 ID
|
||||
format: int64
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
} from "@/components/ui/alert-dialog"
|
||||
import {Badge} from "@/components/ui/badge"
|
||||
import {Button} from "@/components/ui/button"
|
||||
import {Card, CardContent, CardDescription, CardHeader, CardTitle} from "@/components/ui/card"
|
||||
import {Input} from "@/components/ui/input"
|
||||
@@ -28,9 +27,6 @@ import {ErrorInline} from "@/components/layout/error"
|
||||
import {LoadingStateWithBorder} from "@/components/layout/loading"
|
||||
import type {DatabaseCleanupTarget} from "@/lib/services/openflare"
|
||||
import {NodeService, OptionService, StatusService, UptimeKumaService,} from "@/lib/services/openflare"
|
||||
import {AdminStatusService} from "@/lib/services/admin"
|
||||
import {VersionUpgradeDialog} from "@/app/(main)/components/version-upgrade-dialog"
|
||||
import {adminUpdateStatusQueryKey, openflarePublicStatusQueryKey,} from "@/lib/hooks/use-openflare-server-upgrade"
|
||||
|
||||
import {
|
||||
agentOptionEntries,
|
||||
@@ -47,6 +43,7 @@ import {
|
||||
import {UptimeKumaSiteSelectModal} from "./uptimekuma-site-modal"
|
||||
|
||||
const optionsQueryKey = ["openflare", "options"] as const
|
||||
const openflarePublicStatusQueryKey = ["openflare", "public-status"] as const
|
||||
|
||||
const cleanupTargets: Array<{
|
||||
target: DatabaseCleanupTarget
|
||||
@@ -85,7 +82,6 @@ export function OpenFlareOpsSettings() {
|
||||
label: string
|
||||
} | null>(null)
|
||||
const [cleanupRetentionDays, setCleanupRetentionDays] = useState("")
|
||||
const [versionDialogOpen, setVersionDialogOpen] = useState(false)
|
||||
|
||||
const optionsQuery = useQuery({
|
||||
queryKey: optionsQueryKey,
|
||||
@@ -102,10 +98,6 @@ export function OpenFlareOpsSettings() {
|
||||
queryFn: () => NodeService.getBootstrapToken(),
|
||||
})
|
||||
|
||||
const releaseQuery = useQuery({
|
||||
queryKey: adminUpdateStatusQueryKey,
|
||||
queryFn: () => AdminStatusService.getUpdateStatus(),
|
||||
})
|
||||
|
||||
useEffect(() => {
|
||||
if (!optionsQuery.data) return
|
||||
@@ -605,50 +597,6 @@ export function OpenFlareOpsSettings() {
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<Card className="border-dashed shadow-none">
|
||||
<CardHeader className="flex flex-row items-center justify-between gap-4">
|
||||
<div>
|
||||
<CardTitle className="text-base">版本信息</CardTitle>
|
||||
<CardDescription>
|
||||
检查上游 GitHub Release 并升级当前服务。
|
||||
</CardDescription>
|
||||
</div>
|
||||
<Button type="button" size="sm" onClick={() => setVersionDialogOpen(true)}>
|
||||
管理升级
|
||||
</Button>
|
||||
</CardHeader>
|
||||
<CardContent className="grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<InfoCell label="当前版本" value={statusQuery.data?.version ?? releaseQuery.data?.current_version ?? "—"} />
|
||||
<InfoCell
|
||||
label="最新 Release"
|
||||
value={releaseQuery.data?.latest_version ?? "—"}
|
||||
/>
|
||||
<div className="rounded-lg border border-dashed px-3 py-2">
|
||||
<p className="text-[10px] uppercase tracking-wider text-muted-foreground">更新状态</p>
|
||||
<div className="mt-2">
|
||||
{releaseQuery.data?.update_available ? (
|
||||
<Badge variant="secondary">有新版本</Badge>
|
||||
) : (
|
||||
<Badge variant="outline">已是最新</Badge>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<InfoCell
|
||||
label="启动时间"
|
||||
value={
|
||||
statusQuery.data?.start_time
|
||||
? new Date(statusQuery.data.start_time * 1000).toLocaleString()
|
||||
: "—"
|
||||
}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<VersionUpgradeDialog
|
||||
open={versionDialogOpen}
|
||||
onOpenChange={setVersionDialogOpen}
|
||||
canUpgrade
|
||||
/>
|
||||
|
||||
<UptimeKumaSiteSelectModal
|
||||
open={uptimeKumaModalOpen}
|
||||
|
||||
@@ -7,7 +7,7 @@ import dynamic from "next/dynamic"
|
||||
import {useEffect, useMemo} from "react"
|
||||
import {useQuery} from "@tanstack/react-query"
|
||||
import {Loader2, Settings} from "lucide-react"
|
||||
import {useRouter} from "next/navigation"
|
||||
import {useRouter, useSearchParams} from "next/navigation"
|
||||
import {motion} from "motion/react"
|
||||
|
||||
import {Tabs, TabsContent, TabsList, TabsTrigger} from "@/components/ui/tabs"
|
||||
@@ -64,6 +64,17 @@ function systemConfigMap(configs: SystemConfig[]) {
|
||||
export function AdminSettingsPageClient() {
|
||||
const { user, loading } = useAuth()
|
||||
const router = useRouter()
|
||||
const searchParams = useSearchParams()
|
||||
|
||||
const activeTab = useMemo(() => {
|
||||
const rawTab = searchParams.get("tab")
|
||||
const validTabs = ["openflare-ops", "security", "operation", "system", "other", "status", "info"]
|
||||
return rawTab && validTabs.includes(rawTab) ? rawTab : "openflare-ops"
|
||||
}, [searchParams])
|
||||
|
||||
const handleTabChange = (value: string) => {
|
||||
router.push(`/admin/settings?tab=${value}`)
|
||||
}
|
||||
|
||||
const systemConfigsQuery = useQuery({
|
||||
queryKey: ["admin", "system-configs"],
|
||||
@@ -103,7 +114,7 @@ export function AdminSettingsPageClient() {
|
||||
<h1 className="text-2xl font-semibold tracking-tight">系统设置</h1>
|
||||
</div>
|
||||
</div>
|
||||
<Tabs defaultValue="security" className="w-full">
|
||||
<Tabs value={activeTab} onValueChange={handleTabChange} className="w-full">
|
||||
<TabsList variant="line" className="w-fit inline-flex gap-8 mb-6">
|
||||
<TabsTrigger value="openflare-ops" className="px-0 pb-2 text-xs font-semibold">
|
||||
OpenFlare
|
||||
|
||||
@@ -1,8 +1,20 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import {Suspense} from "react"
|
||||
import {Loader2} from "lucide-react"
|
||||
import {AdminSettingsPageClient} from "./page-client"
|
||||
|
||||
export default function AdminSettingsPage() {
|
||||
return <AdminSettingsPageClient />
|
||||
}
|
||||
return (
|
||||
<Suspense
|
||||
fallback={
|
||||
<div className="flex items-center justify-center min-h-[400px]">
|
||||
<Loader2 className="size-6 animate-spin text-primary" />
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<AdminSettingsPageClient />
|
||||
</Suspense>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
'use client';
|
||||
|
||||
import {useEffect} from 'react';
|
||||
import {ExternalLink, Loader2} from 'lucide-react';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import remarkGfm from 'remark-gfm';
|
||||
|
||||
import {Badge} from '@/components/ui/badge';
|
||||
import {Button} from '@/components/ui/button';
|
||||
import {Card, CardContent, CardDescription, CardHeader, CardTitle} from '@/components/ui/card';
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogAction,
|
||||
AlertDialogCancel,
|
||||
AlertDialogContent,
|
||||
AlertDialogDescription,
|
||||
AlertDialogFooter,
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
AlertDialogTrigger,
|
||||
} from '@/components/ui/alert-dialog';
|
||||
import {Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle} from '@/components/ui/dialog';
|
||||
import {useOpenFlareServerUpgrade} from '@/lib/hooks/use-openflare-server-upgrade';
|
||||
import type {AppUpdateStatus} from '@/lib/services/admin/types';
|
||||
import {formatDateTime} from '@/lib/utils';
|
||||
import {formatRelativeTime} from '@/app/(main)/nodes/components/node-utils';
|
||||
|
||||
function getUpgradeBadge(update: AppUpdateStatus | null | undefined) {
|
||||
if (!update) {
|
||||
return { label: '未检查', variant: 'outline' as const };
|
||||
}
|
||||
if (update.update_available) {
|
||||
return { label: '可升级', variant: 'secondary' as const };
|
||||
}
|
||||
return { label: '最新', variant: 'default' as const };
|
||||
}
|
||||
|
||||
export function VersionUpgradeDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
canUpgrade = true,
|
||||
}: {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
canUpgrade?: boolean;
|
||||
}) {
|
||||
const {
|
||||
currentVersion,
|
||||
update,
|
||||
releaseErrorMessage,
|
||||
isInitialLoading,
|
||||
isChecking,
|
||||
isUpgrading,
|
||||
handleOpen,
|
||||
handleCheckRelease,
|
||||
handleUpgrade,
|
||||
} = useOpenFlareServerUpgrade({ open, canUpgrade });
|
||||
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
handleOpen();
|
||||
}
|
||||
}, [open, handleOpen]);
|
||||
|
||||
const upgradeBadge = getUpgradeBadge(update);
|
||||
const isBusy = isChecking || isUpgrading;
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-3xl max-h-[90vh] overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>服务端版本</DialogTitle>
|
||||
<DialogDescription>
|
||||
检查上游 GitHub Release 并升级当前服务。升级开始后服务会短暂重启。
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div className="grid gap-4 md:grid-cols-2">
|
||||
<Card className="border-dashed shadow-none py-4 gap-3">
|
||||
<CardHeader className="px-4 pb-0">
|
||||
<CardTitle className="text-sm">当前版本</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="px-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<p className="text-sm font-medium">{currentVersion}</p>
|
||||
<Badge variant={upgradeBadge.variant}>{upgradeBadge.label}</Badge>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card className="border-dashed shadow-none py-4 gap-3">
|
||||
<CardHeader className="px-4 pb-0">
|
||||
<CardTitle className="text-sm">最新版本</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="px-4 space-y-3">
|
||||
<p className="text-sm font-medium">{update?.latest_version || '未检查'}</p>
|
||||
{canUpgrade ? (
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={isBusy}
|
||||
onClick={handleCheckRelease}
|
||||
>
|
||||
{isChecking ? '检查中...' : '检查更新'}
|
||||
</Button>
|
||||
) : null}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
{isInitialLoading ? (
|
||||
<div className="flex items-center justify-center py-8 text-sm text-muted-foreground">
|
||||
<Loader2 className="size-4 mr-2 animate-spin" />
|
||||
加载版本信息...
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{!isInitialLoading && releaseErrorMessage ? (
|
||||
<div className="rounded-lg border border-destructive/30 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
||||
{releaseErrorMessage}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{!isInitialLoading && !releaseErrorMessage && !update ? (
|
||||
<div className="rounded-lg border border-dashed px-4 py-8 text-center text-sm text-muted-foreground">
|
||||
尚未检查更新,点击「检查更新」后展示 GitHub Release 信息。
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{update ? (
|
||||
<Card className="border-dashed shadow-none py-4 gap-3">
|
||||
<CardHeader className="px-4 pb-0">
|
||||
<CardTitle className="text-sm">GitHub Release · {update.latest_version}</CardTitle>
|
||||
<CardDescription>
|
||||
{update.published_at
|
||||
? `发布时间:${formatRelativeTime(update.published_at)} · ${formatDateTime(update.published_at)}`
|
||||
: '未提供发布时间'}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="px-4 space-y-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge variant={update.update_available ? 'secondary' : 'default'}>
|
||||
{update.update_available ? '发现新版本' : '已经是最新版本'}
|
||||
</Badge>
|
||||
{update.prerelease ? (
|
||||
<Badge variant="secondary">Preview 发布</Badge>
|
||||
) : (
|
||||
<Badge variant="outline">正式发布</Badge>
|
||||
)}
|
||||
{!update.can_upgrade ? (
|
||||
<Badge variant="destructive">当前平台不支持自动升级</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="prose prose-sm dark:prose-invert max-w-none text-sm">
|
||||
<ReactMarkdown remarkPlugins={[remarkGfm]}>
|
||||
{update.release_notes || '暂无更新说明'}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
|
||||
{update.release_url ? (
|
||||
<a
|
||||
href={update.release_url}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="inline-flex items-center text-sm text-primary hover:underline"
|
||||
>
|
||||
查看发布详情
|
||||
<ExternalLink className="size-3 ml-1" />
|
||||
</a>
|
||||
) : null}
|
||||
|
||||
{canUpgrade ? (
|
||||
<div className="flex justify-end">
|
||||
<AlertDialog>
|
||||
<AlertDialogTrigger asChild>
|
||||
<Button
|
||||
type="button"
|
||||
disabled={
|
||||
!update.update_available ||
|
||||
isUpgrading ||
|
||||
!update.can_upgrade ||
|
||||
isBusy
|
||||
}
|
||||
>
|
||||
{isUpgrading ? '升级中...' : '立即升级'}
|
||||
</Button>
|
||||
</AlertDialogTrigger>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>升级到 {update.latest_version}?</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
服务将下载并校验 {update.asset_name},随后替换当前二进制并重启。请确保安装目录可写,且服务允许原地重启。
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>取消</AlertDialogCancel>
|
||||
<AlertDialogAction onClick={handleUpgrade}>确认升级</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{!update.can_upgrade ? (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{update.current_version === 'dev'
|
||||
? '开发构建没有可比较的 Release 版本,不能执行自动升级。'
|
||||
: update.update_available
|
||||
? '当前平台暂不支持自动替换二进制,请从 Release 页面手动升级。'
|
||||
: '当前版本无需升级。'}
|
||||
</p>
|
||||
) : null}
|
||||
</CardContent>
|
||||
</Card>
|
||||
) : null}
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -289,7 +289,7 @@ export function NodeObservability({
|
||||
const observabilityQuery = useQuery({
|
||||
queryKey: ['openflare', 'node-observability', nodeId],
|
||||
queryFn: () => NodeService.getObservability(nodeId, { hours: 24, limit: 48 }),
|
||||
refetchInterval: 10000,
|
||||
refetchInterval: 30000,
|
||||
});
|
||||
|
||||
const cleanupMutation = useMutation({
|
||||
|
||||
@@ -28,7 +28,7 @@ export default function OpenFlareDashboardPage() {
|
||||
const overviewQuery = useQuery({
|
||||
queryKey: dashboardQueryKey,
|
||||
queryFn: () => DashboardService.getOverview(),
|
||||
refetchInterval: 30_000,
|
||||
refetchInterval: 60_000,
|
||||
});
|
||||
|
||||
const overview = overviewQuery.data;
|
||||
|
||||
@@ -73,7 +73,7 @@ export function LoginForm({ onOTPStateChange }: { onOTPStateChange?: (show: bool
|
||||
queryFn: () => AuthService.getAuthSources(),
|
||||
})
|
||||
|
||||
const capEnabled = configBool(publicConfigQuery.data?.cap_login_enabled, true)
|
||||
const capEnabled = configBool(publicConfigQuery.data?.cap_login_enabled, false)
|
||||
const capAutoSolve = configBool(publicConfigQuery.data?.cap_auto_solve, true)
|
||||
|
||||
const loginMutation = useMutation({
|
||||
|
||||
@@ -68,7 +68,7 @@ export function RegisterForm() {
|
||||
|
||||
const emailRegisterEnabled = configBool(publicConfigQuery.data?.email_register_verification_enabled, false)
|
||||
|
||||
const capEnabled = configBool(publicConfigQuery.data?.cap_login_enabled, true)
|
||||
const capEnabled = configBool(publicConfigQuery.data?.cap_login_enabled, false)
|
||||
const capAutoSolve = configBool(publicConfigQuery.data?.cap_auto_solve, true)
|
||||
|
||||
const [capScope, setCapScope] = useState<'send_email_code' | 'register'>('send_email_code')
|
||||
|
||||
@@ -245,7 +245,7 @@ export const apiSections: PolicySection[] = [
|
||||
"registration_enabled": "false",
|
||||
"password_login_enabled": "true",
|
||||
"password_register_enabled": "false",
|
||||
"cap_login_enabled": "true",
|
||||
"cap_login_enabled": "false",
|
||||
"oidc_login_enabled": "true"
|
||||
}
|
||||
}`}
|
||||
|
||||
@@ -5,7 +5,7 @@ import {AnimatePresence, motion} from "motion/react"
|
||||
import {useUser} from "@/contexts/user-context"
|
||||
import {Card, CardHeader, CardTitle} from "@/components/ui/card"
|
||||
import {Button} from "@/components/ui/button"
|
||||
import {ArrowRight, ExternalLink, FileText, HelpCircle, Layers, Shield, ShieldCheck, Terminal, User} from "lucide-react"
|
||||
import {ArrowRight, ExternalLink, HelpCircle, Layers, Shield, ShieldCheck, Terminal, User} from "lucide-react"
|
||||
import Link from "next/link"
|
||||
|
||||
export function HomeMain() {
|
||||
@@ -22,21 +22,11 @@ export function HomeMain() {
|
||||
bgColor: "bg-blue-500/10",
|
||||
borderColor: "hover:border-blue-500/30",
|
||||
},
|
||||
{
|
||||
title: "开发接口文档",
|
||||
description: "查看开放平台的 RESTful 接口规格说明",
|
||||
icon: FileText,
|
||||
url: "/docs/api",
|
||||
color: "text-emerald-500",
|
||||
bgColor: "bg-emerald-500/10",
|
||||
borderColor: "hover:border-emerald-500/30",
|
||||
external: true,
|
||||
},
|
||||
{
|
||||
title: "使用文档",
|
||||
description: "学习如何集成 API 及日常操作帮助指南",
|
||||
icon: HelpCircle,
|
||||
url: "/docs/how-to-use",
|
||||
url: "https://open-flare.pages.dev/",
|
||||
color: "text-purple-500",
|
||||
bgColor: "bg-purple-500/10",
|
||||
borderColor: "hover:border-purple-500/30",
|
||||
|
||||
@@ -4,8 +4,6 @@ import {ComponentType, useMemo} from "react"
|
||||
import {useMutation, useQueryClient} from "@tanstack/react-query"
|
||||
import {
|
||||
Bell,
|
||||
Code,
|
||||
CreditCard,
|
||||
Database,
|
||||
FileText,
|
||||
FolderOpen,
|
||||
@@ -61,9 +59,7 @@ const MENU_GROUPS: MenuGroup[] = [
|
||||
{
|
||||
name: "文档菜单",
|
||||
items: [
|
||||
{ path: "/admin/demo", label: "规范示例", description: "内置 UI 组件与设计规范的展示、调试与参考", icon: Code },
|
||||
{ path: "/docs/api", label: "接口文档", description: "系统 Swagger 交互式 API 接口文档", icon: CreditCard },
|
||||
{ path: "/docs/how-to-use", label: "使用文档", description: "面向开发与运营的部署使用指南", icon: FileText },
|
||||
{ path: "https://open-flare.pages.dev/", label: "使用文档", description: "面向开发与运营的部署使用指南", icon: FileText },
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -126,10 +126,10 @@ curl -X POST https://api.example.com/api/v1/auth/register \\
|
||||
</ul>
|
||||
|
||||
<div className="flex flex-wrap gap-4">
|
||||
<Link href="/docs/api">
|
||||
<Link href="https://open-flare.pages.dev/" target="_blank" rel="noopener noreferrer">
|
||||
<Button variant="secondary" className="rounded-full text-xs hover:bg-muted-foreground/10">
|
||||
<Book className="w-3 h-3" />
|
||||
API 文档
|
||||
使用文档
|
||||
</Button>
|
||||
</Link>
|
||||
</div>
|
||||
|
||||
@@ -43,8 +43,7 @@ export const FooterSection = React.memo(function FooterSection({ className }: Fo
|
||||
<div className="lg:col-span-1">
|
||||
<h3 className="font-semibold text-foreground mb-6">开发</h3>
|
||||
<ul className="space-y-4 text-sm text-muted-foreground">
|
||||
<li><FooterLink href="/docs/how-to-use">快速开始</FooterLink></li>
|
||||
<li><FooterLink href="/docs/api">API 文档</FooterLink></li>
|
||||
<li><FooterLink href="https://open-flare.pages.dev/">使用文档</FooterLink></li>
|
||||
<li><FooterLink href="https://github.com/Rain-kl/OpenFlare">源代码</FooterLink></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
@@ -72,7 +72,7 @@ export const HeroSection = React.memo(function HeroSection({ className }: HeroSe
|
||||
</Button>
|
||||
</Link>
|
||||
|
||||
<Link href="/docs/how-to-use" className="w-full sm:w-auto">
|
||||
<Link href="https://open-flare.pages.dev/" target="_blank" rel="noopener noreferrer" className="w-full sm:w-auto">
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="lg"
|
||||
|
||||
@@ -47,8 +47,6 @@ import {
|
||||
ArrowUpRight,
|
||||
Bell,
|
||||
ChevronDown,
|
||||
Code,
|
||||
CreditCard,
|
||||
Database,
|
||||
FileQuestionMark,
|
||||
FileText,
|
||||
@@ -79,9 +77,7 @@ const data = {
|
||||
{ title: "系统设置", url: "/admin/settings", icon: Settings },
|
||||
],
|
||||
document: [
|
||||
{ title: "规范示例", url: "/admin/demo", icon: Code },
|
||||
{ title: "接口文档", url: "/docs/api", icon: CreditCard, external: true },
|
||||
{ title: "使用文档", url: "/docs/how-to-use", icon: FileText, external: true },
|
||||
{ title: "使用文档", url: "https://open-flare.pages.dev/", icon: FileText, external: true },
|
||||
],
|
||||
}
|
||||
|
||||
@@ -281,7 +277,7 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator className="my-2" />
|
||||
<DropdownMenuItem onClick={() => {
|
||||
router.push("/docs/how-to-use")
|
||||
window.open("https://open-flare.pages.dev/", "_blank", "noopener,noreferrer")
|
||||
handleCloseSidebar()
|
||||
}}>
|
||||
<FileQuestionMark className="mr-2 size-4" />
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
'use client';
|
||||
|
||||
import {useMutation, useQuery} from '@tanstack/react-query';
|
||||
import {useCallback, useEffect, useRef, useState} from 'react';
|
||||
|
||||
import {AdminStatusService} from '@/lib/services/admin';
|
||||
import type {AppUpdateStatus} from '@/lib/services/admin/types';
|
||||
import {StatusService} from '@/lib/services/openflare';
|
||||
|
||||
export const openflarePublicStatusQueryKey = ['openflare', 'public-status'] as const;
|
||||
|
||||
export const adminUpdateStatusQueryKey = ['admin', 'update'] as const;
|
||||
|
||||
export function useOpenFlareServerUpgrade({
|
||||
open,
|
||||
canUpgrade,
|
||||
}: {
|
||||
open: boolean;
|
||||
canUpgrade: boolean;
|
||||
}) {
|
||||
const [feedback, setFeedback] = useState<string | null>(null);
|
||||
|
||||
const upgradeReloadStartedRef = useRef(false);
|
||||
const upgradeReloadTimerRef = useRef<number | null>(null);
|
||||
|
||||
const statusQuery = useQuery({
|
||||
queryKey: openflarePublicStatusQueryKey,
|
||||
queryFn: () => StatusService.getPublicStatus(),
|
||||
enabled: open,
|
||||
});
|
||||
|
||||
const updateQuery = useQuery({
|
||||
queryKey: adminUpdateStatusQueryKey,
|
||||
queryFn: () => AdminStatusService.getUpdateStatus(),
|
||||
enabled: open && canUpgrade,
|
||||
staleTime: 5 * 60 * 1000,
|
||||
});
|
||||
|
||||
const scheduleUpgradePageReload = useCallback(() => {
|
||||
if (upgradeReloadStartedRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
upgradeReloadStartedRef.current = true;
|
||||
setFeedback('服务升级已进入重启阶段,页面将在服务恢复后自动刷新。');
|
||||
|
||||
const reloadWhenServerReady = async () => {
|
||||
try {
|
||||
await StatusService.getPublicStatus();
|
||||
window.location.reload();
|
||||
} catch {
|
||||
upgradeReloadTimerRef.current = window.setTimeout(reloadWhenServerReady, 1500);
|
||||
}
|
||||
};
|
||||
|
||||
upgradeReloadTimerRef.current = window.setTimeout(reloadWhenServerReady, 1200);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (upgradeReloadTimerRef.current !== null) {
|
||||
window.clearTimeout(upgradeReloadTimerRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const upgradeMutation = useMutation({
|
||||
mutationFn: () => AdminStatusService.applyUpdate(),
|
||||
onSuccess: () => {
|
||||
scheduleUpgradePageReload();
|
||||
setFeedback('升级包已校验完成,服务正在重启。');
|
||||
},
|
||||
onError: (error) => {
|
||||
setFeedback(error instanceof Error ? error.message : '升级失败,请稍后重试。');
|
||||
},
|
||||
});
|
||||
|
||||
const resetTransientState = useCallback(() => {
|
||||
setFeedback(null);
|
||||
upgradeReloadStartedRef.current = false;
|
||||
}, []);
|
||||
|
||||
const handleOpen = useCallback(() => {
|
||||
resetTransientState();
|
||||
if (canUpgrade) {
|
||||
void updateQuery.refetch();
|
||||
}
|
||||
}, [canUpgrade, resetTransientState, updateQuery]);
|
||||
|
||||
const handleCheckRelease = useCallback(() => {
|
||||
setFeedback(null);
|
||||
if (!canUpgrade) {
|
||||
return;
|
||||
}
|
||||
void updateQuery.refetch();
|
||||
}, [canUpgrade, updateQuery]);
|
||||
|
||||
const handleUpgrade = useCallback(() => {
|
||||
setFeedback(null);
|
||||
upgradeMutation.mutate();
|
||||
}, [upgradeMutation]);
|
||||
|
||||
const update = updateQuery.data;
|
||||
const releaseErrorMessage =
|
||||
feedback ||
|
||||
(updateQuery.isError
|
||||
? updateQuery.error instanceof Error
|
||||
? updateQuery.error.message
|
||||
: '版本检查失败,请稍后重试。'
|
||||
: undefined);
|
||||
|
||||
const currentVersion = statusQuery.data?.version || update?.current_version || 'unknown';
|
||||
|
||||
return {
|
||||
currentVersion,
|
||||
update,
|
||||
releaseErrorMessage,
|
||||
isInitialLoading: updateQuery.isLoading && !updateQuery.data && canUpgrade,
|
||||
isChecking: updateQuery.isFetching,
|
||||
isUpgrading: upgradeMutation.isPending,
|
||||
handleOpen,
|
||||
handleCheckRelease,
|
||||
handleUpgrade,
|
||||
};
|
||||
}
|
||||
|
||||
export type {AppUpdateStatus};
|
||||
@@ -56,23 +56,121 @@ export const searchData: SearchItem[] = [
|
||||
},
|
||||
|
||||
// ==================== 文档库 ====================
|
||||
{
|
||||
id: 'docs-api',
|
||||
title: '开发接口文档',
|
||||
description: '查看 RESTful API 接口规格定义',
|
||||
url: '/docs/api',
|
||||
category: 'page',
|
||||
keywords: ['api', 'docs', '文档', '接口', 'specification'],
|
||||
},
|
||||
{
|
||||
id: 'docs-how-to-use',
|
||||
title: '使用帮助文档',
|
||||
description: '查看新手教程和集成示例',
|
||||
url: '/docs/how-to-use',
|
||||
url: 'https://open-flare.pages.dev/',
|
||||
category: 'page',
|
||||
keywords: ['docs', '文档', '使用', 'how to', 'tutorial', '教程', 'help'],
|
||||
},
|
||||
|
||||
// ==================== 业务控制台 ====================
|
||||
{
|
||||
id: 'console-nodes',
|
||||
title: '节点管理',
|
||||
description: '管理边缘节点、中继节点与内网穿透通道',
|
||||
url: '/nodes',
|
||||
category: 'page',
|
||||
keywords: ['node', '节点', '边缘节点', '中继', '内网穿透', 'tunnel', '服务器'],
|
||||
},
|
||||
{
|
||||
id: 'console-proxy-routes',
|
||||
title: '规则管理',
|
||||
description: '配置反向代理、路由匹配规则、WAF 策略与缓存设置',
|
||||
url: '/proxy-routes',
|
||||
category: 'page',
|
||||
keywords: ['route', '规则', '路由', '代理', '反向代理', 'proxy'],
|
||||
},
|
||||
{
|
||||
id: 'console-websites',
|
||||
title: '域名列表',
|
||||
description: '管理托管域名及证书绑定与监听配置',
|
||||
url: '/websites',
|
||||
category: 'page',
|
||||
keywords: ['website', 'domain', '网站', '域名', '站点'],
|
||||
},
|
||||
{
|
||||
id: 'console-certificates',
|
||||
title: 'TLS 证书',
|
||||
description: '申请与管理 SSL/TLS 证书,支持自动续期',
|
||||
url: '/certificates',
|
||||
category: 'page',
|
||||
keywords: ['certificate', 'ssl', 'tls', '证书', 'https', '加密'],
|
||||
},
|
||||
{
|
||||
id: 'console-dns-accounts',
|
||||
title: 'DNS 账号',
|
||||
description: '配置 DNS 服务商 API 凭证以自动申请证书及管理解析',
|
||||
url: '/dns-accounts',
|
||||
category: 'page',
|
||||
keywords: ['dns', 'dns account', '账号', '域名解析', 'cloudflare', 'aliyun', 'tencent'],
|
||||
},
|
||||
{
|
||||
id: 'console-origins',
|
||||
title: '源站地址',
|
||||
description: '管理反向代理的目标后端服务器与负载均衡组',
|
||||
url: '/origins',
|
||||
category: 'page',
|
||||
keywords: ['origin', '源站', '后端', 'backend', '服务器', '负载均衡'],
|
||||
},
|
||||
{
|
||||
id: 'console-waf',
|
||||
title: 'WAF 防火墙',
|
||||
description: '配置 Web 应用防火墙规则,阻断恶意请求',
|
||||
url: '/waf',
|
||||
category: 'page',
|
||||
keywords: ['waf', '防火墙', '安全', 'security', '拦截', '规则'],
|
||||
},
|
||||
{
|
||||
id: 'console-ip-groups',
|
||||
title: 'IP 组',
|
||||
description: '定义 IP 地址列表以在 WAF 或路由中实现黑白名单控制',
|
||||
url: '/ip-groups',
|
||||
category: 'page',
|
||||
keywords: ['ip', 'ip group', 'ip组', '黑名单', '白名单', '访问控制'],
|
||||
},
|
||||
{
|
||||
id: 'console-pages',
|
||||
title: 'Pages 静态托管',
|
||||
description: '上传或部署静态网页,提供全球 CDN 加速托管',
|
||||
url: '/pages',
|
||||
category: 'page',
|
||||
keywords: ['pages', '静态托管', 'cdn', '网站', '部署', 'static'],
|
||||
},
|
||||
{
|
||||
id: 'console-config-versions',
|
||||
title: '版本发布',
|
||||
description: '查看、对比、发布与回滚系统配置版本',
|
||||
url: '/config-versions',
|
||||
category: 'page',
|
||||
keywords: ['version', 'config', '版本', '发布', '回滚', '对比', '部署'],
|
||||
},
|
||||
{
|
||||
id: 'console-access-logs',
|
||||
title: '访问日志',
|
||||
description: '查看并检索全量网站访问请求日志与网络分析数据',
|
||||
url: '/access-logs',
|
||||
category: 'page',
|
||||
keywords: ['log', 'logs', '访问日志', '分析', '流量', '请求'],
|
||||
},
|
||||
{
|
||||
id: 'console-apply-logs',
|
||||
title: '应用记录',
|
||||
description: '查看节点配置下发、同步与生效的历史记录',
|
||||
url: '/apply-logs',
|
||||
category: 'page',
|
||||
keywords: ['apply', 'log', 'logs', '应用记录', '配置下发', '同步', '部署历史'],
|
||||
},
|
||||
{
|
||||
id: 'console-performance',
|
||||
title: '性能调优',
|
||||
description: '调优网络连接、代理超时与核心系统性能参数',
|
||||
url: '/performance',
|
||||
category: 'page',
|
||||
keywords: ['performance', '性能', '调优', '优化', '参数', '连接', '超时'],
|
||||
},
|
||||
|
||||
// ==================== 个人设置 ====================
|
||||
{
|
||||
id: 'settings',
|
||||
@@ -98,7 +196,6 @@ export const searchData: SearchItem[] = [
|
||||
category: 'setting',
|
||||
keywords: ['appearance', '外观', '主题', 'theme', 'dark', 'light'],
|
||||
},
|
||||
// ==================== 管理员 ====================
|
||||
{
|
||||
id: 'admin-settings',
|
||||
title: '系统设置',
|
||||
@@ -131,6 +228,38 @@ export const searchData: SearchItem[] = [
|
||||
category: 'admin',
|
||||
keywords: ['admin', '管理员', '任务', '异步', 'tasks', 'scheduler', 'worker'],
|
||||
},
|
||||
{
|
||||
id: 'admin-files',
|
||||
title: '存储管理',
|
||||
description: '查看、检索与清理上传到对象存储中的文件 (管理员专属)',
|
||||
url: '/admin/files',
|
||||
category: 'admin',
|
||||
keywords: ['admin', '管理员', '存储', '文件', 'files', 'upload', 's3'],
|
||||
},
|
||||
{
|
||||
id: 'admin-database',
|
||||
title: '数据管理',
|
||||
description: '监控数据库表大小、分页浏览物理表内容并支持交互式 SQL (管理员专属)',
|
||||
url: '/admin/database',
|
||||
category: 'admin',
|
||||
keywords: ['admin', '管理员', '数据库', 'database', 'sql', 'query', 'gorm'],
|
||||
},
|
||||
{
|
||||
id: 'admin-push',
|
||||
title: '通知推送',
|
||||
description: '配置与下发邮件、Lark 和 Telegram 渠道通知推送 (管理员专属)',
|
||||
url: '/admin/push',
|
||||
category: 'admin',
|
||||
keywords: ['admin', '管理员', '推送', '通知', 'push', 'mail', 'telegram', 'lark'],
|
||||
},
|
||||
{
|
||||
id: 'admin-logs',
|
||||
title: '系统日志',
|
||||
description: '查看系统日志与后台异步任务执行日志 (管理员专属)',
|
||||
url: '/admin/logs',
|
||||
category: 'admin',
|
||||
keywords: ['admin', '管理员', '日志', 'logs', 'system log', 'terminal'],
|
||||
},
|
||||
]
|
||||
|
||||
/**
|
||||
|
||||
@@ -248,7 +248,7 @@ func enrichAccessLogsWithUsers(ctx context.Context, list []accessLogItem) {
|
||||
// @Router /api/v1/admin/logs/access [get]
|
||||
func GetAccessLogs(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
if !config.Config.ClickHouse.Enabled || db.ChDB(ctx) == nil {
|
||||
if !config.Config.ClickHouse.Enabled || !db.ChConnReady() {
|
||||
response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用,无法检索访问日志")
|
||||
return
|
||||
}
|
||||
@@ -348,7 +348,7 @@ type logsAnalyticsResponse struct {
|
||||
// @Router /api/v1/admin/logs/analytics [get]
|
||||
func GetLogsAnalytics(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
if !config.Config.ClickHouse.Enabled || db.ChDB(ctx) == nil {
|
||||
if !config.Config.ClickHouse.Enabled || !db.ChConnReady() {
|
||||
response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用,无法获取分析数据")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package status
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/common/response"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// GetClickHouseStatus returns ClickHouse operational metrics for administrators.
|
||||
// @Summary 获取 ClickHouse 运行指标
|
||||
// @Description 返回 ClickHouse parts、mutation、async_insert 队列等运维指标,需要管理员权限
|
||||
// @Tags admin
|
||||
// @Produce json
|
||||
// @Security SessionCookie
|
||||
// @Success 200 {object} response.Any{data=analyticsrepo.ClickHouseOperationalStats} "获取成功"
|
||||
// @Failure 400 {object} response.Any "ClickHouse 未启用"
|
||||
// @Failure 401 {object} response.Any "未登录"
|
||||
// @Failure 403 {object} response.Any "无管理员权限"
|
||||
// @Failure 500 {object} response.Any "内部错误"
|
||||
// @Router /api/v1/admin/status/clickhouse [get]
|
||||
func GetClickHouseStatus(c *gin.Context) {
|
||||
if !config.Config.ClickHouse.Enabled || !db.ChConnReady() {
|
||||
response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用")
|
||||
return
|
||||
}
|
||||
|
||||
stats, err := analyticsrepo.GetClickHouseOperationalStats(c.Request.Context())
|
||||
if err != nil {
|
||||
response.AbortInternal(c, "获取 ClickHouse 运行指标失败")
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, response.OK(stats))
|
||||
}
|
||||
@@ -144,6 +144,8 @@ func expectedAssetNames(repository, tag string) []string {
|
||||
extension = "zip"
|
||||
}
|
||||
names = append(names, fmt.Sprintf("%s_%s_%s_%s.%s", repoName, tag, runtime.GOOS, runtime.GOARCH, extension))
|
||||
names = append(names, fmt.Sprintf("%s_%s_%s_%s.%s", strings.ToLower(repoName), tag, runtime.GOOS, runtime.GOARCH, extension))
|
||||
names = append(names, fmt.Sprintf("%s-server_%s_%s_%s.%s", strings.ToLower(repoName), tag, runtime.GOOS, runtime.GOARCH, extension))
|
||||
}
|
||||
}
|
||||
return names
|
||||
|
||||
@@ -56,13 +56,13 @@ func TestProtectionEnabledReflectsLoginSwitch(t *testing.T) {
|
||||
|
||||
ResetRuntimeSettingsForTest()
|
||||
|
||||
if !ProtectionEnabled(ctx) {
|
||||
t.Fatal("ProtectionEnabled() = false, want true from seed defaults")
|
||||
if ProtectionEnabled(ctx) {
|
||||
t.Fatal("ProtectionEnabled() = true, want false from seed defaults")
|
||||
}
|
||||
|
||||
if err := db.DB(ctx).Model(&model.SystemConfig{}).
|
||||
Where("key = ?", model.ConfigKeyCapLoginEnabled).
|
||||
Update("value", "false").Error; err != nil {
|
||||
Update("value", "true").Error; err != nil {
|
||||
t.Fatalf("Update(cap_login_enabled) error = %v", err)
|
||||
}
|
||||
if err := repository.InvalidateSystemConfigCache(ctx, model.ConfigKeyCapLoginEnabled); err != nil {
|
||||
@@ -70,8 +70,8 @@ func TestProtectionEnabledReflectsLoginSwitch(t *testing.T) {
|
||||
}
|
||||
InvalidateRuntimeSettings()
|
||||
|
||||
if ProtectionEnabled(ctx) {
|
||||
t.Fatal("ProtectionEnabled() = true, want false after config update")
|
||||
if !ProtectionEnabled(ctx) {
|
||||
t.Fatal("ProtectionEnabled() = false, want true after config update")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,8 +24,6 @@ const (
|
||||
healthSeverityInfo = "info"
|
||||
healthSeverityWarning = "warning"
|
||||
healthSeverityCritical = "critical"
|
||||
nodeAccessLogRetentionDays = 90
|
||||
nodeAccessLogRetentionWindow = nodeAccessLogRetentionDays * 24 * time.Hour
|
||||
accessLogPathMaxLength = 100
|
||||
healthEventMessageMaxLength = 4096
|
||||
)
|
||||
@@ -237,15 +235,11 @@ func buildNodeAccessLogRecords(nodeID string, direct []NodeAccessLog, buffered [
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func persistNodeAccessLogs(ctx context.Context, nodeID string, records []*model.OpenFlareAccessLog, reportedAt time.Time) error {
|
||||
func persistNodeAccessLogs(ctx context.Context, _ string, records []*model.OpenFlareAccessLog, _ time.Time) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := model.InsertOpenFlareAccessLogsBatch(ctx, records); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := model.DeleteOpenFlareAccessLogsByNodeBefore(ctx, nodeID, reportedAt.Add(-nodeAccessLogRetentionWindow))
|
||||
return err
|
||||
return model.InsertOpenFlareAccessLogsBatch(ctx, records)
|
||||
}
|
||||
|
||||
func reconcileNodeHealthEvents(tx *gorm.DB, nodeID string, events []NodeHealthEvent, reportedAt time.Time) error {
|
||||
|
||||
@@ -21,8 +21,8 @@ import (
|
||||
|
||||
const (
|
||||
observabilityQueueSize = 5_000
|
||||
observabilityMaxBatchSize = 200
|
||||
observabilityFlushEvery = 2 * time.Second
|
||||
observabilityMaxBatchSize = 500
|
||||
observabilityFlushEvery = 5 * time.Second
|
||||
|
||||
nodeAccessLogQueueSize = 10_000
|
||||
nodeAccessLogMaxBatchSize = 1_000
|
||||
@@ -41,6 +41,9 @@ var (
|
||||
|
||||
metricSnapshotDedup *dedupSet
|
||||
requestReportDedup *dedupSet
|
||||
openrestyDedup *dedupSet
|
||||
frpsDedup *dedupSet
|
||||
frpcDedup *dedupSet
|
||||
)
|
||||
|
||||
// Init starts OpenFlare ClickHouse batch writers. Safe to call multiple times.
|
||||
@@ -52,6 +55,9 @@ func Init(ctx context.Context) {
|
||||
initOnce.Do(func() {
|
||||
metricSnapshotDedup = newDedupSet()
|
||||
requestReportDedup = newDedupSet()
|
||||
openrestyDedup = newDedupSet()
|
||||
frpsDedup = newDedupSet()
|
||||
frpcDedup = newDedupSet()
|
||||
|
||||
metricSnapshotWriter = mustNewObservabilityWriter("metric_snapshots", analyticsrepo.BatchInsertNodeMetricSnapshots)
|
||||
requestReportWriter = mustNewObservabilityWriter("request_reports", analyticsrepo.BatchInsertNodeRequestReports)
|
||||
@@ -130,6 +136,10 @@ func QueueOpenrestyObservation(observation analyticsmodel.NodeObsOpenresty) {
|
||||
if openrestyWriter == nil {
|
||||
return
|
||||
}
|
||||
key := fmt.Sprintf("%s|%d", observation.NodeID, observation.CapturedAt.UTC().UnixNano())
|
||||
if !openrestyDedup.markIfNew(key) {
|
||||
return
|
||||
}
|
||||
openrestyWriter.TryEnqueue(observation)
|
||||
}
|
||||
|
||||
@@ -138,6 +148,10 @@ func QueueFrpsObservation(observation analyticsmodel.NodeObsFrps) {
|
||||
if frpsWriter == nil {
|
||||
return
|
||||
}
|
||||
key := fmt.Sprintf("%s|%d", observation.NodeID, observation.CapturedAt.UTC().UnixNano())
|
||||
if !frpsDedup.markIfNew(key) {
|
||||
return
|
||||
}
|
||||
frpsWriter.TryEnqueue(observation)
|
||||
}
|
||||
|
||||
@@ -146,6 +160,10 @@ func QueueFrpcObservation(observation analyticsmodel.NodeObsFrpc) {
|
||||
if frpcWriter == nil {
|
||||
return
|
||||
}
|
||||
key := fmt.Sprintf("%s|%d", observation.NodeID, observation.CapturedAt.UTC().UnixNano())
|
||||
if !frpcDedup.markIfNew(key) {
|
||||
return
|
||||
}
|
||||
frpcWriter.TryEnqueue(observation)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const overviewCacheTTL = 30 * time.Second
|
||||
|
||||
var overviewCache struct {
|
||||
mu sync.Mutex
|
||||
payload *OverviewPayload
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
func getCachedOverview() (*OverviewPayload, bool) {
|
||||
overviewCache.mu.Lock()
|
||||
defer overviewCache.mu.Unlock()
|
||||
if overviewCache.payload == nil || time.Now().After(overviewCache.expiresAt) {
|
||||
return nil, false
|
||||
}
|
||||
return overviewCache.payload, true
|
||||
}
|
||||
|
||||
func setCachedOverview(payload *OverviewPayload) {
|
||||
overviewCache.mu.Lock()
|
||||
defer overviewCache.mu.Unlock()
|
||||
overviewCache.payload = payload
|
||||
overviewCache.expiresAt = time.Now().Add(overviewCacheTTL)
|
||||
}
|
||||
@@ -18,6 +18,7 @@ const (
|
||||
nodeStatusPending = "pending"
|
||||
|
||||
dashboardDistributionLimit = 8
|
||||
dashboardOverviewSnapshotLimit = 500
|
||||
highCPUUsagePercentThreshold = 80
|
||||
highMemoryUsagePercentThreshold = 85
|
||||
highStorageUsagePercentThreshold = 85
|
||||
|
||||
@@ -97,11 +97,16 @@ type trendsPayload struct {
|
||||
|
||||
// GetOverview aggregates dashboard overview data from nodes and observability tables.
|
||||
func GetOverview(ctx context.Context) (*OverviewPayload, error) {
|
||||
if payload, ok := getCachedOverview(); ok {
|
||||
return payload, nil
|
||||
}
|
||||
view, err := buildOverviewView(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return compressOverview(view), nil
|
||||
payload := compressOverview(view)
|
||||
setCachedOverview(payload)
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func buildOverviewView(ctx context.Context) (*OverviewView, error) {
|
||||
@@ -112,11 +117,11 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
snapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", since, 0)
|
||||
snapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, "", since, dashboardOverviewSnapshotLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := model.ListOpenFlareRequestReportsSince(ctx, "", since, 0)
|
||||
reports, err := model.ListOpenFlareRequestReportsSince(ctx, "", since, dashboardOverviewSnapshotLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -128,17 +133,21 @@ func buildOverviewView(ctx context.Context) (*OverviewView, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
openrestySnapshots, err := model.ListOpenFlareNodeObservationOpenresty(ctx, "", since, 0)
|
||||
openrestySnapshots, err := model.ListOpenFlareNodeObservationOpenresty(ctx, "", since, dashboardOverviewSnapshotLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
trafficTrend := observability.BuildTrafficTrendPoints(now, reports)
|
||||
if trafficHourly, hourlyErr := model.ListOpenFlareTrafficHourlySince(ctx, "", since); hourlyErr == nil && len(trafficHourly) > 0 {
|
||||
trafficTrend = observability.BuildTrafficTrendPointsFromHourly(now, trafficHourly)
|
||||
}
|
||||
|
||||
view := &OverviewView{
|
||||
GeneratedAt: now,
|
||||
Nodes: make([]NodeHealth, 0, len(nodes)),
|
||||
Distributions: observability.BuildTrafficDistributions(reports, accessLogRegions, dashboardDistributionLimit),
|
||||
Trends: observability.NodeTrends{
|
||||
Traffic24h: observability.BuildTrafficTrendPoints(now, reports),
|
||||
Traffic24h: trafficTrend,
|
||||
Capacity24h: observability.BuildCapacityTrendPoints(now, snapshots),
|
||||
Network24h: observability.BuildNetworkTrendPoints(now, snapshots, openrestySnapshots),
|
||||
DiskIO24h: observability.BuildDiskIOTrendPoints(now, snapshots),
|
||||
|
||||
@@ -21,11 +21,12 @@ const (
|
||||
defaultIPTrendBucketMinute = 30
|
||||
maxIPTrendHours = 168
|
||||
nodeAccessLogRetentionDays = 90
|
||||
defaultAccessLogQueryDays = 7
|
||||
accessLogFieldRemoteAddr = "remote_addr"
|
||||
accessLogFieldRequestCount = "request_count"
|
||||
)
|
||||
|
||||
var nodeAccessLogRetentionWindow = nodeAccessLogRetentionDays * 24 * time.Hour
|
||||
var defaultAccessLogQueryWindow = defaultAccessLogQueryDays * 24 * time.Hour
|
||||
|
||||
// AccessLogQuery filters access log list queries.
|
||||
type AccessLogQuery struct {
|
||||
@@ -346,7 +347,7 @@ func ListFoldedAccessLogIPs(ctx context.Context, input FoldedAccessLogIPQuery) (
|
||||
// ListAccessLogIPSummaries returns paginated IP summaries.
|
||||
func ListAccessLogIPSummaries(ctx context.Context, input AccessLogIPSummaryQuery) (*AccessLogIPSummaryList, error) {
|
||||
normalized := normalizeAccessLogIPSummaryQuery(input)
|
||||
since := time.Now().UTC().Add(-nodeAccessLogRetentionWindow)
|
||||
since := defaultAccessLogSince()
|
||||
recentSince := time.Now().UTC().Add(-3 * time.Hour)
|
||||
query := model.OpenFlareAccessLogIPSummaryQuery{
|
||||
NodeID: strings.TrimSpace(normalized.NodeID),
|
||||
@@ -453,7 +454,7 @@ func buildModelAccessLogQuery(input AccessLogQuery) model.OpenFlareAccessLogQuer
|
||||
RemoteAddr: strings.TrimSpace(input.RemoteAddr),
|
||||
Host: strings.TrimSpace(input.Host),
|
||||
Path: strings.TrimSpace(input.Path),
|
||||
Since: time.Now().UTC().Add(-nodeAccessLogRetentionWindow),
|
||||
Since: defaultAccessLogSince(),
|
||||
Page: input.Page,
|
||||
PageSize: input.PageSize,
|
||||
SortBy: input.SortBy,
|
||||
@@ -461,6 +462,10 @@ func buildModelAccessLogQuery(input AccessLogQuery) model.OpenFlareAccessLogQuer
|
||||
}
|
||||
}
|
||||
|
||||
func defaultAccessLogSince() time.Time {
|
||||
return time.Now().UTC().Add(-defaultAccessLogQueryWindow)
|
||||
}
|
||||
|
||||
func listNodeNameMap(ctx context.Context, logs []*model.OpenFlareAccessLog) (map[string]string, error) {
|
||||
nodeIDs := make([]string, 0, len(logs))
|
||||
seen := make(map[string]struct{}, len(logs))
|
||||
|
||||
@@ -257,6 +257,28 @@ func buildHealthSummary(
|
||||
return summary
|
||||
}
|
||||
|
||||
// BuildTrafficTrendPointsFromHourly builds 24h traffic trend buckets from hourly rollups.
|
||||
func BuildTrafficTrendPointsFromHourly(now time.Time, hourly []*model.OpenFlareTrafficHourly) []TrafficTrendPoint {
|
||||
start := trendWindowStart(now)
|
||||
points := make([]TrafficTrendPoint, observabilityTrendBuckets)
|
||||
for index := range points {
|
||||
points[index].BucketStartedAt = start.Add(time.Duration(index) * time.Hour)
|
||||
}
|
||||
for _, row := range hourly {
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
index, ok := trendBucketIndex(row.Hour, start)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
points[index].RequestCount += row.RequestCount
|
||||
points[index].ErrorCount += row.ErrorCount
|
||||
points[index].UniqueVisitorCount += row.UniqueVisitorCount
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
// BuildTrafficTrendPoints builds 24h traffic trend buckets.
|
||||
func BuildTrafficTrendPoints(now time.Time, reports []*model.OpenFlareRequestReport) []TrafficTrendPoint {
|
||||
start := trendWindowStart(now)
|
||||
|
||||
@@ -10,6 +10,23 @@ import (
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
)
|
||||
|
||||
func TestBuildTrafficTrendPointsFromHourlyBucketsByHour(t *testing.T) {
|
||||
now := time.Date(2026, 7, 2, 15, 30, 0, 0, time.UTC)
|
||||
hourly := []*model.OpenFlareTrafficHourly{
|
||||
{
|
||||
NodeID: "node-a",
|
||||
Hour: now.Add(-2 * time.Hour).Truncate(time.Hour),
|
||||
RequestCount: 12,
|
||||
ErrorCount: 1,
|
||||
UniqueVisitorCount: 4,
|
||||
},
|
||||
}
|
||||
points := BuildTrafficTrendPointsFromHourly(now, hourly)
|
||||
if len(points) != observabilityTrendBuckets {
|
||||
t.Fatalf("BuildTrafficTrendPointsFromHourly() len = %d, want %d", len(points), observabilityTrendBuckets)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTrafficTrendPointsBucketsByHour(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
@@ -18,8 +19,19 @@ const (
|
||||
defaultObservabilityLimit = 120
|
||||
maxObservabilityLimit = 500
|
||||
defaultTrafficDistributionLimit = 8
|
||||
nodeObservabilityCacheTTL = 15 * time.Second
|
||||
)
|
||||
|
||||
var nodeObservabilityCache struct {
|
||||
mu sync.Mutex
|
||||
views map[string]cachedNodeObservability
|
||||
}
|
||||
|
||||
type cachedNodeObservability struct {
|
||||
view *NodeView
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
// NodeQuery filters node observability data.
|
||||
type NodeQuery struct {
|
||||
Hours int `json:"hours"`
|
||||
@@ -87,6 +99,9 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if view, ok := getCachedNodeObservability(node.NodeID); ok {
|
||||
return view, nil
|
||||
}
|
||||
|
||||
limit := normalizeObservabilityLimit(query.Limit)
|
||||
since := now.Add(-normalizeObservabilityWindow(query.Hours))
|
||||
@@ -115,22 +130,14 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
trendSnapshots, err := model.ListOpenFlareMetricSnapshotsSince(ctx, node.NodeID, now.Add(-24*time.Hour), 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
trendOpenresty, err := model.ListOpenFlareNodeObservationOpenresty(ctx, node.NodeID, now.Add(-24*time.Hour), 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
trendReports, err := model.ListOpenFlareRequestReportsSince(ctx, node.NodeID, now.Add(-24*time.Hour), 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events, err := model.ListOpenFlareHealthEvents(ctx, node.NodeID, false, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
trafficTrend := BuildTrafficTrendPoints(now, reports)
|
||||
if trafficHourly, hourlyErr := model.ListOpenFlareTrafficHourlySince(ctx, node.NodeID, now.Add(-24*time.Hour)); hourlyErr == nil && len(trafficHourly) > 0 {
|
||||
trafficTrend = BuildTrafficTrendPointsFromHourly(now, trafficHourly)
|
||||
}
|
||||
|
||||
view := &NodeView{
|
||||
NodeID: node.NodeID,
|
||||
@@ -144,10 +151,10 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
|
||||
Health: buildHealthSummary(latestMetricSnapshot(snapshots), latestTrafficReport(reports), events),
|
||||
},
|
||||
Trends: NodeTrends{
|
||||
Traffic24h: BuildTrafficTrendPoints(now, trendReports),
|
||||
Capacity24h: BuildCapacityTrendPoints(now, trendSnapshots),
|
||||
Network24h: BuildNetworkTrendPoints(now, trendSnapshots, trendOpenresty),
|
||||
DiskIO24h: BuildDiskIOTrendPoints(now, trendSnapshots),
|
||||
Traffic24h: trafficTrend,
|
||||
Capacity24h: BuildCapacityTrendPoints(now, snapshots),
|
||||
Network24h: BuildNetworkTrendPoints(now, snapshots, openrestyObs),
|
||||
DiskIO24h: BuildDiskIOTrendPoints(now, snapshots),
|
||||
},
|
||||
}
|
||||
if node.NodeType == "tunnel_relay" {
|
||||
@@ -161,9 +168,35 @@ func GetNodeObservability(ctx context.Context, id uint, query NodeQuery) (*NodeV
|
||||
}
|
||||
view.RelayDashboard = buildRelayDashboardSnapshot(node, latestFrps)
|
||||
}
|
||||
setCachedNodeObservability(node.NodeID, view)
|
||||
return view, nil
|
||||
}
|
||||
|
||||
func getCachedNodeObservability(nodeID string) (*NodeView, bool) {
|
||||
nodeObservabilityCache.mu.Lock()
|
||||
defer nodeObservabilityCache.mu.Unlock()
|
||||
if nodeObservabilityCache.views == nil {
|
||||
return nil, false
|
||||
}
|
||||
entry, ok := nodeObservabilityCache.views[nodeID]
|
||||
if !ok || time.Now().After(entry.expiresAt) {
|
||||
return nil, false
|
||||
}
|
||||
return entry.view, true
|
||||
}
|
||||
|
||||
func setCachedNodeObservability(nodeID string, view *NodeView) {
|
||||
nodeObservabilityCache.mu.Lock()
|
||||
defer nodeObservabilityCache.mu.Unlock()
|
||||
if nodeObservabilityCache.views == nil {
|
||||
nodeObservabilityCache.views = make(map[string]cachedNodeObservability)
|
||||
}
|
||||
nodeObservabilityCache.views[nodeID] = cachedNodeObservability{
|
||||
view: view,
|
||||
expiresAt: time.Now().Add(nodeObservabilityCacheTTL),
|
||||
}
|
||||
}
|
||||
|
||||
// CleanupHealthEvents removes all health events for a node.
|
||||
func CleanupHealthEvents(ctx context.Context, id uint) (*HealthEventCleanupResult, error) {
|
||||
node, err := model.GetOpenFlareNodeByID(ctx, id)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -21,12 +22,21 @@ const (
|
||||
DatabaseCleanupTargetMetricSnapshots = "node_metric_snapshots"
|
||||
// DatabaseCleanupTargetRequestReports is the API cleanup target for request reports.
|
||||
DatabaseCleanupTargetRequestReports = "node_request_reports"
|
||||
// DatabaseCleanupTargetObsOpenresty is the API cleanup target for OpenResty observations.
|
||||
DatabaseCleanupTargetObsOpenresty = "node_obs_openresty"
|
||||
// DatabaseCleanupTargetObsFrps is the API cleanup target for FRPS observations.
|
||||
DatabaseCleanupTargetObsFrps = "node_obs_frps"
|
||||
// DatabaseCleanupTargetObsFrpc is the API cleanup target for FRPC observations.
|
||||
DatabaseCleanupTargetObsFrpc = "node_obs_frpc"
|
||||
)
|
||||
|
||||
var databaseCleanupTargets = map[string]string{
|
||||
DatabaseCleanupTargetAccessLogs: "访问日志",
|
||||
DatabaseCleanupTargetMetricSnapshots: "性能快照",
|
||||
DatabaseCleanupTargetRequestReports: "请求聚合",
|
||||
DatabaseCleanupTargetObsOpenresty: "OpenResty 观测",
|
||||
DatabaseCleanupTargetObsFrps: "FRPS 观测",
|
||||
DatabaseCleanupTargetObsFrpc: "FRPC 观测",
|
||||
}
|
||||
|
||||
// DatabaseCleanupInput describes a manual observability cleanup request.
|
||||
@@ -40,6 +50,7 @@ type DatabaseCleanupResult struct {
|
||||
Target string `json:"target"`
|
||||
TargetLabel string `json:"target_label"`
|
||||
DeletedCount int64 `json:"deleted_count"`
|
||||
CleanupMode string `json:"cleanup_mode,omitempty"`
|
||||
DeleteAll bool `json:"delete_all"`
|
||||
RetentionDays *int `json:"retention_days,omitempty"`
|
||||
Cutoff *time.Time `json:"cutoff,omitempty"`
|
||||
@@ -70,21 +81,23 @@ func CleanupDatabaseObservability(ctx context.Context, input DatabaseCleanupInpu
|
||||
}
|
||||
|
||||
if input.RetentionDays == nil {
|
||||
deleted, err := deleteAllObservabilityRows(ctx, target)
|
||||
deleted, mode, err := deleteAllObservabilityRows(ctx, target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.DeletedCount = deleted
|
||||
result.CleanupMode = mode
|
||||
return result, nil
|
||||
}
|
||||
|
||||
retentionDays := *input.RetentionDays
|
||||
cutoff := time.Now().UTC().Add(-time.Duration(retentionDays) * 24 * time.Hour)
|
||||
deleted, err := deleteObservabilityRowsBefore(ctx, target, cutoff)
|
||||
deleted, mode, err := deleteObservabilityRowsBefore(ctx, target, cutoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.DeletedCount = deleted
|
||||
result.CleanupMode = mode
|
||||
result.RetentionDays = &retentionDays
|
||||
result.Cutoff = &cutoff
|
||||
return result, nil
|
||||
@@ -111,6 +124,9 @@ func RunDatabaseAutoCleanupOnce(ctx context.Context, now time.Time) (*DatabaseAu
|
||||
DatabaseCleanupTargetAccessLogs,
|
||||
DatabaseCleanupTargetMetricSnapshots,
|
||||
DatabaseCleanupTargetRequestReports,
|
||||
DatabaseCleanupTargetObsOpenresty,
|
||||
DatabaseCleanupTargetObsFrps,
|
||||
DatabaseCleanupTargetObsFrpc,
|
||||
} {
|
||||
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
|
||||
Target: target,
|
||||
@@ -129,28 +145,56 @@ func RunDatabaseAutoCleanupOnce(ctx context.Context, now time.Time) (*DatabaseAu
|
||||
}, nil
|
||||
}
|
||||
|
||||
func deleteAllObservabilityRows(ctx context.Context, target string) (int64, error) {
|
||||
func deleteAllObservabilityRows(ctx context.Context, target string) (int64, string, error) {
|
||||
var (
|
||||
deleted int64
|
||||
err error
|
||||
)
|
||||
switch target {
|
||||
case DatabaseCleanupTargetAccessLogs:
|
||||
return model.DeleteAllOpenFlareAccessLogs(ctx)
|
||||
deleted, err = model.DeleteAllOpenFlareAccessLogs(ctx)
|
||||
case DatabaseCleanupTargetMetricSnapshots:
|
||||
return model.DeleteAllOpenFlareMetricSnapshots(ctx)
|
||||
deleted, err = model.DeleteAllOpenFlareMetricSnapshots(ctx)
|
||||
case DatabaseCleanupTargetRequestReports:
|
||||
return model.DeleteAllOpenFlareRequestReports(ctx)
|
||||
deleted, err = model.DeleteAllOpenFlareRequestReports(ctx)
|
||||
case DatabaseCleanupTargetObsOpenresty:
|
||||
deleted, err = model.DeleteAllOpenFlareNodeObservationOpenresty(ctx)
|
||||
case DatabaseCleanupTargetObsFrps:
|
||||
deleted, err = model.DeleteAllOpenFlareNodeObservationFrps(ctx)
|
||||
case DatabaseCleanupTargetObsFrpc:
|
||||
deleted, err = model.DeleteAllOpenFlareNodeObservationFrpc(ctx)
|
||||
default:
|
||||
return 0, errors.New("unsupported cleanup target")
|
||||
return 0, "", errors.New("unsupported cleanup target")
|
||||
}
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return deleted, analyticsrepo.CleanupModeTruncate, nil
|
||||
}
|
||||
|
||||
func deleteObservabilityRowsBefore(ctx context.Context, target string, cutoff time.Time) (int64, error) {
|
||||
func deleteObservabilityRowsBefore(ctx context.Context, target string, cutoff time.Time) (int64, string, error) {
|
||||
var (
|
||||
deleted int64
|
||||
err error
|
||||
)
|
||||
switch target {
|
||||
case DatabaseCleanupTargetAccessLogs:
|
||||
return model.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
|
||||
deleted, err = model.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
|
||||
case DatabaseCleanupTargetMetricSnapshots:
|
||||
return model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
|
||||
deleted, err = model.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
|
||||
case DatabaseCleanupTargetRequestReports:
|
||||
return model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
|
||||
deleted, err = model.DeleteOpenFlareRequestReportsBefore(ctx, cutoff)
|
||||
case DatabaseCleanupTargetObsOpenresty:
|
||||
deleted, err = model.DeleteOpenFlareNodeObservationOpenrestyBefore(ctx, cutoff)
|
||||
case DatabaseCleanupTargetObsFrps:
|
||||
deleted, err = model.DeleteOpenFlareNodeObservationFrpsBefore(ctx, cutoff)
|
||||
case DatabaseCleanupTargetObsFrpc:
|
||||
deleted, err = model.DeleteOpenFlareNodeObservationFrpcBefore(ctx, cutoff)
|
||||
default:
|
||||
return 0, errors.New("unsupported cleanup target")
|
||||
return 0, "", errors.New("unsupported cleanup target")
|
||||
}
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return deleted, analyticsrepo.CleanupModeTTLMaterialize, nil
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ func TestRunDatabaseAutoCleanupOnceDeletesAllObservabilityTargets(t *testing.T)
|
||||
summary, err := RunDatabaseAutoCleanupOnce(ctx, now)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, summary)
|
||||
require.Len(t, summary.Results, 3)
|
||||
require.Len(t, summary.Results, 6)
|
||||
|
||||
accessLogs, err := model.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -234,15 +234,15 @@ func evaluateParsedIPGroupAutoConfig(ctx context.Context, config ipGroupAutoConf
|
||||
}
|
||||
programs = append(programs, program)
|
||||
}
|
||||
logs, err := model.ListOpenFlareAccessLogsForWAFIPGroup(ctx, model.OpenFlareAccessLogQuery{
|
||||
aggregates, err := model.ListOpenFlareAccessLogWAFIPAggregates(ctx, model.OpenFlareAccessLogQuery{
|
||||
Since: now.Add(-time.Duration(config.LookbackMinutes) * time.Minute),
|
||||
Until: now,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accumulators := make(map[string]*ipGroupAutoAccumulator)
|
||||
for _, item := range logs {
|
||||
accumulators := make(map[string]*ipGroupAutoAccumulator, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
@@ -250,30 +250,23 @@ func evaluateParsedIPGroupAutoConfig(ctx context.Context, config ipGroupAutoConf
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
acc := accumulators[ip]
|
||||
if acc == nil {
|
||||
acc = &ipGroupAutoAccumulator{
|
||||
ip: ip,
|
||||
statusCounts: make(map[int]int),
|
||||
}
|
||||
accumulators[ip] = acc
|
||||
lastSeen := time.Time{}
|
||||
if item.LastSeenEpoch > 0 {
|
||||
lastSeen = time.Unix(item.LastSeenEpoch, 0).UTC()
|
||||
}
|
||||
acc.requestCount++
|
||||
acc.statusCounts[item.StatusCode]++
|
||||
if item.StatusCode == http.StatusNotFound {
|
||||
acc.status404Count++
|
||||
statusCounts := make(map[int]int, len(item.StatusCounts))
|
||||
for code, count := range item.StatusCounts {
|
||||
statusCounts[code] = count
|
||||
}
|
||||
if item.StatusCode >= 400 && item.StatusCode < 500 {
|
||||
acc.clientErrorCount++
|
||||
}
|
||||
if item.StatusCode >= http.StatusInternalServerError {
|
||||
acc.serverErrorCount++
|
||||
}
|
||||
if hostIsIPLiteral(item.Host) {
|
||||
acc.ipHostCount++
|
||||
}
|
||||
if item.LoggedAt.After(acc.lastSeen) {
|
||||
acc.lastSeen = item.LoggedAt
|
||||
accumulators[ip] = &ipGroupAutoAccumulator{
|
||||
ip: ip,
|
||||
requestCount: item.RequestCount,
|
||||
status404Count: item.Status404Count,
|
||||
ipHostCount: item.IPHostCount,
|
||||
clientErrorCount: item.ClientErrorCount,
|
||||
serverErrorCount: item.ServerErrorCount,
|
||||
lastSeen: lastSeen,
|
||||
statusCounts: statusCounts,
|
||||
}
|
||||
}
|
||||
matched := make([]string, 0)
|
||||
@@ -336,11 +329,6 @@ func normalizeIPLiteral(value string) (string, bool) {
|
||||
return addr.String(), true
|
||||
}
|
||||
|
||||
func hostIsIPLiteral(value string) bool {
|
||||
_, ok := normalizeIPLiteral(value)
|
||||
return ok
|
||||
}
|
||||
|
||||
func downloadIPGroupSubscription(ctx context.Context, rawURL string) ([]byte, error) {
|
||||
if err := validateSubscriptionURL(rawURL); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -5,8 +5,11 @@
|
||||
package risk_control
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
|
||||
@@ -18,6 +21,61 @@ import (
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const maxAuditLogHeadersBytes = 2 * 1024
|
||||
|
||||
var auditLogHeaderAllowlist = map[string]struct{}{
|
||||
"Authorization": {},
|
||||
"Cookie": {},
|
||||
"X-Forwarded-For": {},
|
||||
"X-Real-Ip": {},
|
||||
"User-Agent": {},
|
||||
"Content-Type": {},
|
||||
}
|
||||
|
||||
func marshalAuditLogHeaders(headers http.Header) string {
|
||||
if headers == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
filtered := make(http.Header)
|
||||
for key, values := range headers {
|
||||
if _, ok := auditLogHeaderAllowlist[key]; !ok {
|
||||
continue
|
||||
}
|
||||
filtered[key] = redactAuditLogHeaderValues(key, values)
|
||||
}
|
||||
|
||||
headersBytes, err := json.Marshal(filtered)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if len(headersBytes) <= maxAuditLogHeadersBytes {
|
||||
return string(headersBytes)
|
||||
}
|
||||
return string(headersBytes[:maxAuditLogHeadersBytes])
|
||||
}
|
||||
|
||||
func redactAuditLogHeaderValues(key string, values []string) []string {
|
||||
switch key {
|
||||
case "Authorization", "Cookie":
|
||||
redacted := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
redacted[i] = hashAuditLogSensitiveValue(value)
|
||||
}
|
||||
return redacted
|
||||
default:
|
||||
return values
|
||||
}
|
||||
}
|
||||
|
||||
func hashAuditLogSensitiveValue(value string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return "sha256:" + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// RiskControlMiddleware 全局日志采集中间件
|
||||
func RiskControlMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
@@ -47,19 +105,7 @@ func RiskControlMiddleware() gin.HandlerFunc {
|
||||
// 4. 计算耗时并异步推送到缓冲队列
|
||||
latency := time.Since(start).Milliseconds()
|
||||
|
||||
var headersStr string
|
||||
if c.Request.Header != nil {
|
||||
// 克隆 Header,避免污染原 HTTP 请求的 Header 对象
|
||||
clonedHeaders := make(http.Header)
|
||||
for k, v := range c.Request.Header {
|
||||
clonedHeaders[k] = v
|
||||
}
|
||||
clonedHeaders.Del("Cookie")
|
||||
|
||||
if headersBytes, err := json.Marshal(clonedHeaders); err == nil {
|
||||
headersStr = string(headersBytes)
|
||||
}
|
||||
}
|
||||
headersStr := marshalAuditLogHeaders(c.Request.Header)
|
||||
|
||||
const maxHTTPStatus = 999
|
||||
status := c.Writer.Status()
|
||||
|
||||
@@ -94,6 +94,8 @@ func TestRiskControlMiddleware(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "/test", nil)
|
||||
req.Header.Set("X-Test-Header", "hello")
|
||||
req.Header.Set("Cookie", "session_id=abcdef123456")
|
||||
req.Header.Set("Authorization", "Bearer secret-token")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
@@ -106,8 +108,11 @@ func TestRiskControlMiddleware(t *testing.T) {
|
||||
assert.Equal(t, http.MethodGet, logItem.Method)
|
||||
assert.Equal(t, int32(http.StatusOK), logItem.Status)
|
||||
assert.NotEmpty(t, logItem.Headers)
|
||||
assert.Contains(t, logItem.Headers, "X-Test-Header")
|
||||
assert.NotContains(t, logItem.Headers, "Cookie")
|
||||
assert.NotContains(t, logItem.Headers, "X-Test-Header")
|
||||
assert.Contains(t, logItem.Headers, "Content-Type")
|
||||
assert.Contains(t, logItem.Headers, "sha256:")
|
||||
assert.NotContains(t, logItem.Headers, "secret-token")
|
||||
assert.NotContains(t, logItem.Headers, "session_id=abcdef123456")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
t.Fatal("expected flushed log item, but got none")
|
||||
}
|
||||
|
||||
@@ -135,10 +135,10 @@ func applyClickHouseDefaults(c *configModel) {
|
||||
c.ClickHouse.Username = "default"
|
||||
}
|
||||
if c.ClickHouse.MaxIdleConn <= 0 {
|
||||
c.ClickHouse.MaxIdleConn = 10
|
||||
c.ClickHouse.MaxIdleConn = 20
|
||||
}
|
||||
if c.ClickHouse.MaxOpenConn <= 0 {
|
||||
c.ClickHouse.MaxOpenConn = 100
|
||||
c.ClickHouse.MaxOpenConn = 50
|
||||
}
|
||||
if c.ClickHouse.ConnMaxLifetime <= 0 {
|
||||
c.ClickHouse.ConnMaxLifetime = 3600
|
||||
@@ -147,7 +147,7 @@ func applyClickHouseDefaults(c *configModel) {
|
||||
c.ClickHouse.DialTimeout = 5
|
||||
}
|
||||
if c.ClickHouse.BlockBufferSize == 0 {
|
||||
c.ClickHouse.BlockBufferSize = 10
|
||||
c.ClickHouse.BlockBufferSize = 100
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,9 +9,10 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultQueueSize = 10_000
|
||||
defaultMaxBatchSize = 1_000
|
||||
defaultFlushEvery = time.Second
|
||||
defaultQueueSize = 10_000
|
||||
defaultMaxBatchSize = 1_000
|
||||
defaultMinBatchSize = 50
|
||||
defaultFlushEvery = time.Second
|
||||
)
|
||||
|
||||
// Config controls queue capacity and flush thresholds for a Writer instance.
|
||||
@@ -25,6 +26,10 @@ type Config struct {
|
||||
// MaxBatchSize triggers a flush when the in-memory batch reaches this count.
|
||||
MaxBatchSize int
|
||||
|
||||
// MinBatchSize is the minimum in-memory batch size for time-based flushes.
|
||||
// Zero disables the threshold and preserves legacy interval flush behavior.
|
||||
MinBatchSize int
|
||||
|
||||
// FlushInterval triggers a time-based flush even when the batch is smaller.
|
||||
FlushInterval time.Duration
|
||||
}
|
||||
@@ -34,6 +39,7 @@ func DefaultConfig() Config {
|
||||
return Config{
|
||||
QueueSize: defaultQueueSize,
|
||||
MaxBatchSize: defaultMaxBatchSize,
|
||||
MinBatchSize: defaultMinBatchSize,
|
||||
FlushInterval: defaultFlushEvery,
|
||||
}
|
||||
}
|
||||
@@ -45,6 +51,9 @@ func (c Config) validate() error {
|
||||
if c.MaxBatchSize <= 0 {
|
||||
return fmt.Errorf("batchwriter: max batch size must be positive")
|
||||
}
|
||||
if c.MinBatchSize < 0 {
|
||||
return fmt.Errorf("batchwriter: min batch size must be non-negative")
|
||||
}
|
||||
if c.FlushInterval <= 0 {
|
||||
return fmt.Errorf("batchwriter: flush interval must be positive")
|
||||
}
|
||||
|
||||
@@ -202,7 +202,9 @@ func (w *Writer[T]) run() {
|
||||
flush()
|
||||
}
|
||||
case <-ticker.C:
|
||||
flush()
|
||||
if len(batch) > 0 && (w.cfg.MinBatchSize == 0 || len(batch) >= w.cfg.MinBatchSize) {
|
||||
flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +98,7 @@ func TestWriterFlushesOnInterval(t *testing.T) {
|
||||
)
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxBatchSize = 100
|
||||
cfg.MinBatchSize = 0
|
||||
cfg.FlushInterval = 20 * time.Millisecond
|
||||
|
||||
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
|
||||
@@ -219,6 +220,112 @@ func TestWriterStopDrainsQueuedItems(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterSkipsIntervalFlushBelowMinBatchSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
batch []int
|
||||
)
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxBatchSize = 100
|
||||
cfg.MinBatchSize = 5
|
||||
cfg.FlushInterval = 20 * time.Millisecond
|
||||
|
||||
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
batch = append([]int(nil), items...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
writer.Start(context.Background())
|
||||
t.Cleanup(func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := writer.Stop(stopCtx); err != nil {
|
||||
t.Fatalf("Stop() error = %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for i := range 3 {
|
||||
if !writer.TryEnqueue(i + 1) {
|
||||
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
got := batch
|
||||
mu.Unlock()
|
||||
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("interval flush with below-min batch = %v, want no flush", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterFlushesOnIntervalWhenMinBatchSizeReached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
batch []int
|
||||
)
|
||||
cfg := DefaultConfig()
|
||||
cfg.MaxBatchSize = 100
|
||||
cfg.MinBatchSize = 3
|
||||
cfg.FlushInterval = 20 * time.Millisecond
|
||||
|
||||
writer, err := New[int](cfg, func(_ context.Context, items []int) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
batch = append([]int(nil), items...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
writer.Start(context.Background())
|
||||
t.Cleanup(func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := writer.Stop(stopCtx); err != nil {
|
||||
t.Fatalf("Stop() error = %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
for i := range 3 {
|
||||
if !writer.TryEnqueue(i + 1) {
|
||||
t.Fatalf("TryEnqueue(%d) = false, want true", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
ready := len(batch) == 3
|
||||
mu.Unlock()
|
||||
if ready || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
got := batch
|
||||
mu.Unlock()
|
||||
|
||||
want := []int{1, 2, 3}
|
||||
if diff := cmp.Diff(want, got); diff != "" {
|
||||
t.Fatalf("interval flush at min batch size mismatch (-want +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterInvokesFlushErrorHandler(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+13
-76
@@ -7,32 +7,24 @@ package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
clickhouseDriver "gorm.io/driver/clickhouse"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/plugin/opentelemetry/tracing"
|
||||
)
|
||||
|
||||
const (
|
||||
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
|
||||
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
|
||||
clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒)
|
||||
clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数
|
||||
clickhouseAsyncInsertMaxDataSize = 10_000_000
|
||||
clickhouseAsyncInsertBusyTimeoutMs = 1000
|
||||
)
|
||||
|
||||
var (
|
||||
// ChConn ClickHouse 原生连接实例,用于批量写入
|
||||
// ChConn ClickHouse 原生连接实例,用于批量写入与查询
|
||||
ChConn driver.Conn
|
||||
|
||||
chDB *gorm.DB
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -57,36 +49,6 @@ func init() {
|
||||
log.Fatalf("[ClickHouse] ping failed: %v\n", err)
|
||||
}
|
||||
|
||||
chDB, err = gorm.Open(clickhouseDriver.New(clickhouseDriver.Config{
|
||||
DSN: buildClickHouseDSN(),
|
||||
}), &gorm.Config{
|
||||
SkipDefaultTransaction: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] init gorm connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = chDB.Use(
|
||||
tracing.NewPlugin(
|
||||
tracing.WithoutMetrics(),
|
||||
tracing.WithAttributes(
|
||||
attribute.String("db.instance", cfg.Database),
|
||||
attribute.String("db.system", "ClickHouse"),
|
||||
),
|
||||
),
|
||||
); err != nil {
|
||||
log.Fatalf("[ClickHouse] init trace failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB, err := chDB.DB()
|
||||
if err != nil {
|
||||
log.Fatalf("[ClickHouse] load sql db failed: %v\n", err)
|
||||
}
|
||||
|
||||
sqlDB.SetMaxIdleConns(cfg.MaxIdleConn)
|
||||
sqlDB.SetMaxOpenConns(cfg.MaxOpenConn)
|
||||
sqlDB.SetConnMaxLifetime(time.Duration(cfg.ConnMaxLifetime) * time.Second)
|
||||
|
||||
log.Println("[ClickHouse] connection established successfully")
|
||||
}
|
||||
|
||||
@@ -101,7 +63,11 @@ func buildClickHouseOptions() *clickhouse.Options {
|
||||
Password: cfg.Password,
|
||||
},
|
||||
Settings: clickhouse.Settings{
|
||||
"max_execution_time": clickhouseMaxExecTime,
|
||||
"max_execution_time": clickhouseMaxExecTime,
|
||||
"async_insert": 1,
|
||||
"wait_for_async_insert": 1,
|
||||
"async_insert_max_data_size": clickhouseAsyncInsertMaxDataSize,
|
||||
"async_insert_busy_timeout_ms": clickhouseAsyncInsertBusyTimeoutMs,
|
||||
},
|
||||
Compression: &clickhouse.Compression{
|
||||
Method: clickhouse.CompressionLZ4,
|
||||
@@ -115,38 +81,9 @@ func buildClickHouseOptions() *clickhouse.Options {
|
||||
}
|
||||
}
|
||||
|
||||
func buildClickHouseDSN() string {
|
||||
cfg := config.Config.ClickHouse
|
||||
|
||||
chURL := &url.URL{
|
||||
Scheme: "clickhouse",
|
||||
Host: strings.Join(cfg.Hosts, ","),
|
||||
Path: "/" + cfg.Database,
|
||||
}
|
||||
if cfg.Username != "" || cfg.Password != "" {
|
||||
chURL.User = url.UserPassword(cfg.Username, cfg.Password)
|
||||
}
|
||||
|
||||
query := chURL.Query()
|
||||
query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout))
|
||||
query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor))
|
||||
query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime))
|
||||
chURL.RawQuery = query.Encode()
|
||||
|
||||
return chURL.String()
|
||||
}
|
||||
|
||||
// ChDB returns a context-aware GORM ClickHouse instance.
|
||||
func ChDB(ctx context.Context) *gorm.DB {
|
||||
if chDB == nil {
|
||||
return nil
|
||||
}
|
||||
return chDB.WithContext(ctx)
|
||||
}
|
||||
|
||||
// SetChDBForTest sets the package-level ClickHouse GORM instance for testing.
|
||||
func SetChDBForTest(d *gorm.DB) {
|
||||
chDB = d
|
||||
// ChConnReady reports whether the native ClickHouse connection is initialized.
|
||||
func ChConnReady() bool {
|
||||
return ChConn != nil
|
||||
}
|
||||
|
||||
// SetChConnForTest sets the package-level native ClickHouse connection for testing.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
-- +goose Up
|
||||
-- Add TTL policies to analytics tables so ClickHouse can expire rows automatically.
|
||||
ALTER TABLE w_user_access_logs MODIFY TTL created_at + INTERVAL 180 DAY;
|
||||
|
||||
-- DateTime64 columns must be cast for TTL (ClickHouse requires DateTime/Date in TTL expr).
|
||||
ALTER TABLE of_node_access_logs MODIFY TTL toDateTime(logged_at) + INTERVAL 90 DAY;
|
||||
|
||||
ALTER TABLE of_node_metric_snapshots MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_request_reports MODIFY TTL toDateTime(window_ended_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_obs_openresty MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_obs_frps MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
ALTER TABLE of_node_obs_frpc MODIFY TTL toDateTime(captured_at) + INTERVAL 30 DAY;
|
||||
|
||||
-- +goose Down
|
||||
-- TTL changes cannot be safely reversed without recreating tables.
|
||||
@@ -0,0 +1,28 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE IF NOT EXISTS of_node_traffic_hourly
|
||||
(
|
||||
node_id String,
|
||||
hour DateTime,
|
||||
request_count UInt64,
|
||||
error_count UInt64,
|
||||
unique_visitor_count UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY toYYYYMM(hour)
|
||||
ORDER BY (node_id, hour);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS of_node_traffic_hourly_mv
|
||||
TO of_node_traffic_hourly
|
||||
AS
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(window_ended_at) AS hour,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
sum(unique_visitor_count) AS unique_visitor_count
|
||||
FROM of_node_request_reports
|
||||
GROUP BY node_id, hour;
|
||||
|
||||
-- +goose Down
|
||||
DROP VIEW IF EXISTS of_node_traffic_hourly_mv;
|
||||
DROP TABLE IF EXISTS of_node_traffic_hourly;
|
||||
@@ -137,7 +137,7 @@ CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
|
||||
|
||||
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
|
||||
('cap_login_enabled', 'true', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
|
||||
@@ -7,10 +7,6 @@ UPDATE w_system_configs
|
||||
SET value = 'false', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'password_register_enabled' AND value = 'true';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = 'true', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'cap_login_enabled' AND value = 'false';
|
||||
|
||||
-- +goose Down
|
||||
UPDATE w_system_configs
|
||||
SET value = 'true', updated_at = CURRENT_TIMESTAMP
|
||||
@@ -18,8 +14,4 @@ WHERE key = 'registration_enabled' AND value = 'false';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = 'true', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'password_register_enabled' AND value = 'false';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = 'false', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'cap_login_enabled' AND value = 'true';
|
||||
WHERE key = 'password_register_enabled' AND value = 'false';
|
||||
@@ -137,7 +137,7 @@ CREATE INDEX IF NOT EXISTS idx_templates_created_at ON templates (created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_templates_updated_at ON templates (updated_at);
|
||||
|
||||
INSERT INTO system_configs (key, value, type, visibility, description, created_at, updated_at) VALUES
|
||||
('cap_login_enabled', 'true', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_login_enabled', 'false', 'system', 1, '是否启用登录人机验证(true/false)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_auto_solve', 'true', 'system', 1, '打开页面后是否自动开始计算,关闭则需用户手动点击触发', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_count', '1', 'system', 0, '客户端需求解的 PoW 难题总数,默认 1,推荐 1~5', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
('cap_challenge_size', '32', 'system', 0, '人机验证盐值长度', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
|
||||
|
||||
@@ -7,10 +7,6 @@ UPDATE w_system_configs
|
||||
SET value = 'false', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'password_register_enabled' AND value = 'true';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = 'true', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'cap_login_enabled' AND value = 'false';
|
||||
|
||||
-- +goose Down
|
||||
UPDATE w_system_configs
|
||||
SET value = 'true', updated_at = CURRENT_TIMESTAMP
|
||||
@@ -18,8 +14,4 @@ WHERE key = 'registration_enabled' AND value = 'false';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = 'true', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'password_register_enabled' AND value = 'false';
|
||||
|
||||
UPDATE w_system_configs
|
||||
SET value = 'false', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE key = 'cap_login_enabled' AND value = 'true';
|
||||
WHERE key = 'password_register_enabled' AND value = 'false';
|
||||
@@ -24,6 +24,8 @@ type openFlareAccessLogBucketAggregateRow struct {
|
||||
SuccessCount int64 `gorm:"column:success_count"`
|
||||
ClientErrorCount int64 `gorm:"column:client_error_count"`
|
||||
ServerErrorCount int64 `gorm:"column:server_error_count"`
|
||||
UniqueIPCount int64 `gorm:"column:unique_ip_count"`
|
||||
UniqueHostCount int64 `gorm:"column:unique_host_count"`
|
||||
}
|
||||
|
||||
type openFlareAccessLogBucketDimensionRow struct {
|
||||
@@ -52,9 +54,45 @@ type openFlareAccessLogIPTrendRow struct {
|
||||
RequestCount int64 `gorm:"column:request_count"`
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogsForWAFIPGroup lists access logs in a time window for automatic IP group rules.
|
||||
func ListOpenFlareAccessLogsForWAFIPGroup(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLog, error) {
|
||||
return ListOpenFlareAccessLogs(ctx, query)
|
||||
type openFlareAccessLogWAFIPAggregateRow struct {
|
||||
RemoteAddr string
|
||||
RequestCount int64
|
||||
Status404Count int64
|
||||
ClientErrorCount int64
|
||||
ServerErrorCount int64
|
||||
IPHostCount int64
|
||||
LastSeenEpoch int64
|
||||
StatusCounts map[int]int64
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogWAFIPAggregates returns per-IP aggregates for WAF automatic rules.
|
||||
func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLogWAFIPAggregate, error) {
|
||||
rows, err := currentAccessLogStore().WAFIPAggregates(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*OpenFlareAccessLogWAFIPAggregate, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
statusCounts := make(map[int]int, len(row.StatusCounts))
|
||||
for code, count := range row.StatusCounts {
|
||||
statusCounts[code] = int(count)
|
||||
}
|
||||
result = append(result, &OpenFlareAccessLogWAFIPAggregate{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: int(row.RequestCount),
|
||||
Status404Count: int(row.Status404Count),
|
||||
ClientErrorCount: int(row.ClientErrorCount),
|
||||
ServerErrorCount: int(row.ServerErrorCount),
|
||||
IPHostCount: int(row.IPHostCount),
|
||||
LastSeenEpoch: row.LastSeenEpoch,
|
||||
StatusCounts: statusCounts,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// InsertOpenFlareAccessLogsBatch inserts access log rows into ClickHouse.
|
||||
@@ -79,24 +117,17 @@ func ListOpenFlareAccessLogRegionCounts(ctx context.Context, nodeID string, sinc
|
||||
|
||||
// ListOpenFlareAccessLogBuckets lists folded access log buckets.
|
||||
func ListOpenFlareAccessLogBuckets(ctx context.Context, query OpenFlareAccessLogBucketQuery) ([]*OpenFlareAccessLogBucketRow, error) {
|
||||
rows, err := buildOpenFlareAccessLogBucketRows(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
|
||||
if start >= len(rows) {
|
||||
return []*OpenFlareAccessLogBucketRow{}, nil
|
||||
}
|
||||
return rows[start:end], nil
|
||||
return buildOpenFlareAccessLogBucketRows(ctx, query)
|
||||
}
|
||||
|
||||
// CountOpenFlareAccessLogBuckets counts folded access log buckets.
|
||||
func CountOpenFlareAccessLogBuckets(ctx context.Context, query OpenFlareAccessLogBucketQuery) (int64, error) {
|
||||
rows, err := buildOpenFlareAccessLogBucketRows(ctx, query)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
filter := openFlareAccessLogQueryFromBucket(query)
|
||||
bucketSeconds := int64(query.FoldMinutes * secondsPerMinute)
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 180
|
||||
}
|
||||
return int64(len(rows)), nil
|
||||
return currentAccessLogStore().CountBuckets(ctx, filter, bucketSeconds)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogBucketIPs lists folded IP rows for a bucket window.
|
||||
@@ -123,24 +154,13 @@ func CountOpenFlareAccessLogBucketIPs(ctx context.Context, query OpenFlareAccess
|
||||
|
||||
// ListOpenFlareAccessLogIPSummaries lists IP summaries.
|
||||
func ListOpenFlareAccessLogIPSummaries(ctx context.Context, query OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*OpenFlareAccessLogIPSummaryRow, error) {
|
||||
rows, err := buildOpenFlareAccessLogIPSummaryRows(ctx, query, recentSince)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
|
||||
if start >= len(rows) {
|
||||
return []*OpenFlareAccessLogIPSummaryRow{}, nil
|
||||
}
|
||||
return rows[start:end], nil
|
||||
return buildOpenFlareAccessLogIPSummaryRows(ctx, query, recentSince)
|
||||
}
|
||||
|
||||
// CountOpenFlareAccessLogIPSummaries counts IP summaries.
|
||||
func CountOpenFlareAccessLogIPSummaries(ctx context.Context, query OpenFlareAccessLogIPSummaryQuery) (int64, error) {
|
||||
rows, err := buildOpenFlareAccessLogIPSummaryRows(ctx, query, time.Time{})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(len(rows)), nil
|
||||
filter := openFlareAccessLogQueryFromIPSummary(query)
|
||||
return currentAccessLogStore().CountIPSummaries(ctx, filter)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogIPTrend lists IP trend points.
|
||||
@@ -195,75 +215,22 @@ func buildOpenFlareAccessLogBucketRows(ctx context.Context, query OpenFlareAcces
|
||||
bucketSeconds = 180
|
||||
}
|
||||
|
||||
type bucketAccumulator struct {
|
||||
requestCount int64
|
||||
uniqueIPs map[string]struct{}
|
||||
uniqueHosts map[string]struct{}
|
||||
successCount int64
|
||||
clientErrorCount int64
|
||||
serverErrorCount int64
|
||||
}
|
||||
accumulators := make(map[int64]*bucketAccumulator)
|
||||
|
||||
partials, err := currentAccessLogStore().BucketAggregates(ctx, filter, bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([]*OpenFlareAccessLogBucketRow, 0, len(partials))
|
||||
for _, partial := range partials {
|
||||
accumulator := accumulators[partial.BucketEpoch]
|
||||
if accumulator == nil {
|
||||
accumulator = &bucketAccumulator{
|
||||
uniqueIPs: make(map[string]struct{}),
|
||||
uniqueHosts: make(map[string]struct{}),
|
||||
}
|
||||
accumulators[partial.BucketEpoch] = accumulator
|
||||
}
|
||||
accumulator.requestCount += partial.RequestCount
|
||||
accumulator.successCount += partial.SuccessCount
|
||||
accumulator.clientErrorCount += partial.ClientErrorCount
|
||||
accumulator.serverErrorCount += partial.ServerErrorCount
|
||||
}
|
||||
|
||||
for _, column := range []string{columnRemoteAddr, columnHost} {
|
||||
dimensions, err := currentAccessLogStore().BucketDimensions(ctx, filter, column, bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, item := range dimensions {
|
||||
accumulator := accumulators[item.BucketEpoch]
|
||||
if accumulator == nil {
|
||||
accumulator = &bucketAccumulator{
|
||||
uniqueIPs: make(map[string]struct{}),
|
||||
uniqueHosts: make(map[string]struct{}),
|
||||
}
|
||||
accumulators[item.BucketEpoch] = accumulator
|
||||
}
|
||||
trimmed := strings.TrimSpace(item.Value)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
switch column {
|
||||
case columnRemoteAddr:
|
||||
accumulator.uniqueIPs[trimmed] = struct{}{}
|
||||
case columnHost:
|
||||
accumulator.uniqueHosts[trimmed] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rows := make([]*OpenFlareAccessLogBucketRow, 0, len(accumulators))
|
||||
for bucketEpoch, accumulator := range accumulators {
|
||||
rows = append(rows, &OpenFlareAccessLogBucketRow{
|
||||
BucketEpoch: bucketEpoch,
|
||||
RequestCount: accumulator.requestCount,
|
||||
UniqueIPCount: int64(len(accumulator.uniqueIPs)),
|
||||
UniqueHostCount: int64(len(accumulator.uniqueHosts)),
|
||||
SuccessCount: accumulator.successCount,
|
||||
ClientErrorCount: accumulator.clientErrorCount,
|
||||
ServerErrorCount: accumulator.serverErrorCount,
|
||||
BucketEpoch: partial.BucketEpoch,
|
||||
RequestCount: partial.RequestCount,
|
||||
UniqueIPCount: partial.UniqueIPCount,
|
||||
UniqueHostCount: partial.UniqueHostCount,
|
||||
SuccessCount: partial.SuccessCount,
|
||||
ClientErrorCount: partial.ClientErrorCount,
|
||||
ServerErrorCount: partial.ServerErrorCount,
|
||||
})
|
||||
}
|
||||
sortOpenFlareAccessLogBucketRows(rows, query.SortBy, query.SortOrder)
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
@@ -293,12 +260,7 @@ func buildOpenFlareAccessLogBucketIPRows(ctx context.Context, query OpenFlareAcc
|
||||
}
|
||||
|
||||
func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*OpenFlareAccessLogIPSummaryRow, error) {
|
||||
filter := OpenFlareAccessLogQuery{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: query.RemoteAddr,
|
||||
Host: query.Host,
|
||||
Since: query.Since,
|
||||
}
|
||||
filter := openFlareAccessLogQueryFromIPSummary(query)
|
||||
partials, err := currentAccessLogStore().IPSummaries(ctx, filter, recentSince)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -316,7 +278,6 @@ func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query OpenFlareAc
|
||||
LastSeenEpoch: partial.LastSeenEpoch,
|
||||
})
|
||||
}
|
||||
sortOpenFlareAccessLogIPSummaryRows(rows, query.SortBy, query.SortOrder)
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
@@ -350,6 +311,23 @@ func openFlareAccessLogQueryFromBucket(query OpenFlareAccessLogBucketQuery) Open
|
||||
Host: query.Host,
|
||||
Path: query.Path,
|
||||
Since: query.Since,
|
||||
Page: query.Page,
|
||||
PageSize: query.PageSize,
|
||||
SortBy: query.SortBy,
|
||||
SortOrder: query.SortOrder,
|
||||
}
|
||||
}
|
||||
|
||||
func openFlareAccessLogQueryFromIPSummary(query OpenFlareAccessLogIPSummaryQuery) OpenFlareAccessLogQuery {
|
||||
return OpenFlareAccessLogQuery{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: query.RemoteAddr,
|
||||
Host: query.Host,
|
||||
Since: query.Since,
|
||||
Page: query.Page,
|
||||
PageSize: query.PageSize,
|
||||
SortBy: query.SortBy,
|
||||
SortOrder: query.SortOrder,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,9 +19,12 @@ type accessLogStore interface {
|
||||
Count(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, error)
|
||||
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareAccessLogRegionCount, error)
|
||||
BucketAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error)
|
||||
CountBuckets(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
|
||||
BucketDimensions(ctx context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error)
|
||||
IPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error)
|
||||
WAFIPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error)
|
||||
IPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error)
|
||||
CountIPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery) (int64, error)
|
||||
IPTrend(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error)
|
||||
DeleteAll(ctx context.Context) (int64, error)
|
||||
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
@@ -116,11 +119,17 @@ func (clickhouseAccessLogStore) BucketAggregates(ctx context.Context, filter Ope
|
||||
SuccessCount: row.SuccessCount,
|
||||
ClientErrorCount: row.ClientErrorCount,
|
||||
ServerErrorCount: row.ServerErrorCount,
|
||||
UniqueIPCount: row.UniqueIPCount,
|
||||
UniqueHostCount: row.UniqueHostCount,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) CountBuckets(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
|
||||
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) BucketDimensions(ctx context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
|
||||
rows, err := analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, bucketSeconds)
|
||||
if err != nil {
|
||||
@@ -172,6 +181,31 @@ func (clickhouseAccessLogStore) IPSummaries(ctx context.Context, filter OpenFlar
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) CountIPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery) (int64, error) {
|
||||
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) WAFIPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
|
||||
rows, err := analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]openFlareAccessLogWAFIPAggregateRow, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = openFlareAccessLogWAFIPAggregateRow{
|
||||
RemoteAddr: row.RemoteAddr,
|
||||
RequestCount: row.RequestCount,
|
||||
Status404Count: row.Status404Count,
|
||||
ClientErrorCount: row.ClientErrorCount,
|
||||
ServerErrorCount: row.ServerErrorCount,
|
||||
IPHostCount: row.IPHostCount,
|
||||
LastSeenEpoch: row.LastSeenEpoch,
|
||||
StatusCounts: row.StatusCounts,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) IPTrend(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
|
||||
rows, err := analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
|
||||
if err != nil {
|
||||
|
||||
@@ -5,6 +5,9 @@ package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -99,12 +102,21 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
aggregates := make(map[int64]*openFlareAccessLogBucketAggregateRow)
|
||||
type bucketAccumulator struct {
|
||||
openFlareAccessLogBucketAggregateRow
|
||||
uniqueIPs map[string]struct{}
|
||||
uniqueHosts map[string]struct{}
|
||||
}
|
||||
aggregates := make(map[int64]*bucketAccumulator)
|
||||
for _, row := range rows {
|
||||
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
|
||||
item := aggregates[bucketEpoch]
|
||||
if item == nil {
|
||||
item = &openFlareAccessLogBucketAggregateRow{BucketEpoch: bucketEpoch}
|
||||
item = &bucketAccumulator{
|
||||
openFlareAccessLogBucketAggregateRow: openFlareAccessLogBucketAggregateRow{BucketEpoch: bucketEpoch},
|
||||
uniqueIPs: make(map[string]struct{}),
|
||||
uniqueHosts: make(map[string]struct{}),
|
||||
}
|
||||
aggregates[bucketEpoch] = item
|
||||
}
|
||||
item.RequestCount++
|
||||
@@ -116,14 +128,61 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
|
||||
default:
|
||||
item.ServerErrorCount++
|
||||
}
|
||||
if remoteAddr := strings.TrimSpace(row.RemoteAddr); remoteAddr != "" {
|
||||
item.uniqueIPs[remoteAddr] = struct{}{}
|
||||
}
|
||||
if host := strings.TrimSpace(row.Host); host != "" {
|
||||
item.uniqueHosts[host] = struct{}{}
|
||||
}
|
||||
}
|
||||
result := make([]openFlareAccessLogBucketAggregateRow, 0, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
result = append(result, *item)
|
||||
item.UniqueIPCount = int64(len(item.uniqueIPs))
|
||||
item.UniqueHostCount = int64(len(item.uniqueHosts))
|
||||
result = append(result, item.openFlareAccessLogBucketAggregateRow)
|
||||
}
|
||||
bucketRows := make([]*OpenFlareAccessLogBucketRow, len(result))
|
||||
for index := range result {
|
||||
bucketRows[index] = &OpenFlareAccessLogBucketRow{
|
||||
BucketEpoch: result[index].BucketEpoch,
|
||||
RequestCount: result[index].RequestCount,
|
||||
UniqueIPCount: result[index].UniqueIPCount,
|
||||
UniqueHostCount: result[index].UniqueHostCount,
|
||||
SuccessCount: result[index].SuccessCount,
|
||||
ClientErrorCount: result[index].ClientErrorCount,
|
||||
ServerErrorCount: result[index].ServerErrorCount,
|
||||
}
|
||||
}
|
||||
sortOpenFlareAccessLogBucketRows(bucketRows, filter.SortBy, filter.SortOrder)
|
||||
for index := range result {
|
||||
result[index] = openFlareAccessLogBucketAggregateRow{
|
||||
BucketEpoch: bucketRows[index].BucketEpoch,
|
||||
RequestCount: bucketRows[index].RequestCount,
|
||||
UniqueIPCount: bucketRows[index].UniqueIPCount,
|
||||
UniqueHostCount: bucketRows[index].UniqueHostCount,
|
||||
SuccessCount: bucketRows[index].SuccessCount,
|
||||
ClientErrorCount: bucketRows[index].ClientErrorCount,
|
||||
ServerErrorCount: bucketRows[index].ServerErrorCount,
|
||||
}
|
||||
}
|
||||
if filter.PageSize > 0 {
|
||||
start, end := openFlareAccessLogPaginateBounds(len(result), filter.Page, filter.PageSize)
|
||||
return result[start:end], nil
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) CountBuckets(_ context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
seen := make(map[int64]struct{})
|
||||
for _, row := range rows {
|
||||
seen[memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)] = struct{}{}
|
||||
}
|
||||
return int64(len(seen)), nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) BucketDimensions(_ context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
@@ -222,9 +281,91 @@ func (s *memoryAccessLogStore) IPSummaries(_ context.Context, filter OpenFlareAc
|
||||
item.LastSeenEpoch = epoch
|
||||
}
|
||||
}
|
||||
result := make([]openFlareAccessLogIPSummaryRow, 0, len(aggregates))
|
||||
summaryRows := make([]*OpenFlareAccessLogIPSummaryRow, 0, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
result = append(result, *item)
|
||||
summaryRows = append(summaryRows, &OpenFlareAccessLogIPSummaryRow{
|
||||
RemoteAddr: item.RemoteAddr,
|
||||
TotalRequests: item.TotalRequests,
|
||||
RecentRequests: item.RecentRequests,
|
||||
LastSeenEpoch: item.LastSeenEpoch,
|
||||
})
|
||||
}
|
||||
sortOpenFlareAccessLogIPSummaryRows(summaryRows, filter.SortBy, filter.SortOrder)
|
||||
if filter.PageSize > 0 {
|
||||
start, end := openFlareAccessLogPaginateBounds(len(summaryRows), filter.Page, filter.PageSize)
|
||||
summaryRows = summaryRows[start:end]
|
||||
}
|
||||
result := make([]openFlareAccessLogIPSummaryRow, len(summaryRows))
|
||||
for index, item := range summaryRows {
|
||||
result[index] = openFlareAccessLogIPSummaryRow{
|
||||
RemoteAddr: item.RemoteAddr,
|
||||
TotalRequests: item.TotalRequests,
|
||||
RecentRequests: item.RecentRequests,
|
||||
LastSeenEpoch: item.LastSeenEpoch,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) CountIPSummaries(_ context.Context, filter OpenFlareAccessLogQuery) (int64, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
seen := make(map[string]struct{})
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
seen[remoteAddr] = struct{}{}
|
||||
}
|
||||
return int64(len(seen)), nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) WAFIPAggregates(_ context.Context, filter OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
aggregates := make(map[string]*openFlareAccessLogWAFIPAggregateRow)
|
||||
order := make([]string, 0)
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
item := aggregates[remoteAddr]
|
||||
if item == nil {
|
||||
item = &openFlareAccessLogWAFIPAggregateRow{
|
||||
RemoteAddr: remoteAddr,
|
||||
StatusCounts: make(map[int]int64),
|
||||
}
|
||||
aggregates[remoteAddr] = item
|
||||
order = append(order, remoteAddr)
|
||||
}
|
||||
item.RequestCount++
|
||||
item.StatusCounts[row.StatusCode]++
|
||||
if row.StatusCode == http.StatusNotFound {
|
||||
item.Status404Count++
|
||||
}
|
||||
if row.StatusCode >= 400 && row.StatusCode < 500 {
|
||||
item.ClientErrorCount++
|
||||
}
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
item.ServerErrorCount++
|
||||
}
|
||||
if memoryAccessLogHostIsIPLiteral(row.Host) {
|
||||
item.IPHostCount++
|
||||
}
|
||||
epoch := row.LoggedAt.UTC().Unix()
|
||||
if epoch > item.LastSeenEpoch {
|
||||
item.LastSeenEpoch = epoch
|
||||
}
|
||||
}
|
||||
result := make([]openFlareAccessLogWAFIPAggregateRow, 0, len(order))
|
||||
for _, remoteAddr := range order {
|
||||
if item := aggregates[remoteAddr]; item != nil {
|
||||
result = append(result, *item)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -324,6 +465,19 @@ func memoryAccessLogMatches(row *OpenFlareAccessLog, query OpenFlareAccessLogQue
|
||||
return true
|
||||
}
|
||||
|
||||
func memoryAccessLogHostIsIPLiteral(value string) bool {
|
||||
host := strings.TrimSpace(value)
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
if parsedHost, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = parsedHost
|
||||
}
|
||||
host = strings.Trim(host, "[]")
|
||||
_, err := netip.ParseAddr(host)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func memoryAccessLogBucketEpoch(loggedAt time.Time, bucketSeconds int64) int64 {
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 180
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -277,6 +278,18 @@ type OpenFlareAccessLogIPTrendRow struct {
|
||||
RequestCount int64 `json:"request_count"`
|
||||
}
|
||||
|
||||
// OpenFlareAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
|
||||
type OpenFlareAccessLogWAFIPAggregate struct {
|
||||
RemoteAddr string
|
||||
RequestCount int
|
||||
Status404Count int
|
||||
ClientErrorCount int
|
||||
ServerErrorCount int
|
||||
IPHostCount int
|
||||
LastSeenEpoch int64
|
||||
StatusCounts map[int]int
|
||||
}
|
||||
|
||||
func isMissingTableError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
@@ -325,6 +338,37 @@ func ListOpenFlareRequestReportsSince(ctx context.Context, nodeID string, since
|
||||
return currentObservabilityStore().ListRequestReports(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// OpenFlareTrafficHourly is an hourly traffic rollup row.
|
||||
type OpenFlareTrafficHourly struct {
|
||||
NodeID string `json:"node_id"`
|
||||
Hour time.Time `json:"hour"`
|
||||
RequestCount int64 `json:"request_count"`
|
||||
ErrorCount int64 `json:"error_count"`
|
||||
UniqueVisitorCount int64 `json:"unique_visitor_count"`
|
||||
}
|
||||
|
||||
// ListOpenFlareTrafficHourlySince returns hourly traffic rollup rows since the given time.
|
||||
func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*OpenFlareTrafficHourly, error) {
|
||||
rows, err := analyticsrepo.ListNodeTrafficHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*OpenFlareTrafficHourly, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &OpenFlareTrafficHourly{
|
||||
NodeID: row.NodeID,
|
||||
Hour: row.Hour,
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueVisitorCount: row.UniqueVisitorCount,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareActiveHealthEvents returns active health events across all nodes.
|
||||
func ListOpenFlareActiveHealthEvents(ctx context.Context) ([]*OpenFlareHealthEvent, error) {
|
||||
conn := db.DB(ctx)
|
||||
@@ -384,6 +428,36 @@ func DeleteAllOpenFlareRequestReports(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllRequestReports(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationOpenrestyBefore deletes OpenResty observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteNodeObservationOpenrestyBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationOpenresty deletes all OpenResty observations.
|
||||
func DeleteAllOpenFlareNodeObservationOpenresty(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllNodeObservationOpenresty(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteNodeObservationFrpsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationFrps deletes all FRPS observations.
|
||||
func DeleteAllOpenFlareNodeObservationFrps(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllNodeObservationFrps(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationFrpcBefore deletes FRPC observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteNodeObservationFrpcBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationFrpc deletes all FRPC observations.
|
||||
func DeleteAllOpenFlareNodeObservationFrpc(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllNodeObservationFrpc(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareHealthEventsByNodeID deletes all health events for a node.
|
||||
func DeleteOpenFlareHealthEventsByNodeID(ctx context.Context, nodeID string) (int64, error) {
|
||||
conn := db.DB(ctx)
|
||||
|
||||
@@ -7,41 +7,51 @@ package analytics
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func userAccessLogConn() error {
|
||||
if db.ChConn == nil {
|
||||
return fmt.Errorf("clickhouse native connection is not initialized")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CountAccessLogs returns the number of access logs matching filter.
|
||||
func CountAccessLogs(ctx context.Context, filter AccessLogFilter) (uint64, error) {
|
||||
ch := db.ChDB(ctx)
|
||||
if ch == nil {
|
||||
return 0, fmt.Errorf("clickhouse gorm connection is not initialized")
|
||||
clause, args, ok := buildUserAccessLogFilterClause(filter)
|
||||
if !ok {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var count int64
|
||||
query := applyFilter(ch.Model(&analyticsmodel.UserAccessLog{}), filter)
|
||||
if err := query.Count(&count).Error; err != nil {
|
||||
if err := userAccessLogConn(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tableName := analyticsmodel.UserAccessLog{}.TableName()
|
||||
sql := fmt.Sprintf("SELECT count() FROM %s WHERE %s", tableName, clause)
|
||||
var count uint64
|
||||
if err := db.ChConn.QueryRow(ctx, sql, args...).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count access logs: %w", err)
|
||||
}
|
||||
return safeUint64Count(count), nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// ListAccessLogs returns paginated access logs and the total match count.
|
||||
func ListAccessLogs(ctx context.Context, filter AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
||||
ch := db.ChDB(ctx)
|
||||
if ch == nil {
|
||||
return nil, 0, fmt.Errorf("clickhouse gorm connection is not initialized")
|
||||
}
|
||||
|
||||
if filter.UserIDs != nil && len(filter.UserIDs) == 0 {
|
||||
clause, args, ok := buildUserAccessLogFilterClause(filter)
|
||||
if !ok {
|
||||
return []analyticsmodel.UserAccessLog{}, 0, nil
|
||||
}
|
||||
if err := userAccessLogConn(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var total int64
|
||||
baseQuery := applyFilter(ch.Model(&analyticsmodel.UserAccessLog{}), filter)
|
||||
if err := baseQuery.Count(&total).Error; err != nil {
|
||||
tableName := analyticsmodel.UserAccessLog{}.TableName()
|
||||
countSQL := fmt.Sprintf("SELECT count() FROM %s WHERE %s", tableName, clause)
|
||||
var total uint64
|
||||
if err := db.ChConn.QueryRow(ctx, countSQL, args...).Scan(&total); err != nil {
|
||||
return nil, 0, fmt.Errorf("count access logs: %w", err)
|
||||
}
|
||||
if total == 0 {
|
||||
@@ -56,34 +66,41 @@ func ListAccessLogs(ctx context.Context, filter AccessLogFilter, page, pageSize
|
||||
}
|
||||
offset := (page - 1) * pageSize
|
||||
|
||||
var logs []analyticsmodel.UserAccessLog
|
||||
err := applyFilter(ch.Model(&analyticsmodel.UserAccessLog{}), filter).
|
||||
Order("created_at DESC, id DESC").
|
||||
Limit(pageSize).
|
||||
Offset(offset).
|
||||
Find(&logs).Error
|
||||
listSQL := fmt.Sprintf(`
|
||||
SELECT id, user_id, path, method, ip, user_agent, headers, status, latency, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT ? OFFSET ?`, tableName, clause)
|
||||
listArgs := append(append([]any{}, args...), pageSize, offset)
|
||||
rows, err := db.ChConn.Query(ctx, listSQL, listArgs...)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("list access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
return logs, safeUint64Count(total), nil
|
||||
}
|
||||
|
||||
func applyFilter(query *gorm.DB, filter AccessLogFilter) *gorm.DB {
|
||||
if filter.UserIDs != nil {
|
||||
if len(filter.UserIDs) == 0 {
|
||||
return query.Where("1 = 0")
|
||||
logs := make([]analyticsmodel.UserAccessLog, 0, pageSize)
|
||||
for rows.Next() {
|
||||
var (
|
||||
item analyticsmodel.UserAccessLog
|
||||
createdAt time.Time
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.UserID,
|
||||
&item.Path,
|
||||
&item.Method,
|
||||
&item.IP,
|
||||
&item.UserAgent,
|
||||
&item.Headers,
|
||||
&item.Status,
|
||||
&item.Latency,
|
||||
&createdAt,
|
||||
); err != nil {
|
||||
return nil, 0, fmt.Errorf("scan access log row: %w", err)
|
||||
}
|
||||
query = query.Where("user_id IN ?", filter.UserIDs)
|
||||
item.CreatedAt = createdAt
|
||||
logs = append(logs, item)
|
||||
}
|
||||
if filter.Path != "" {
|
||||
query = query.Where("path LIKE ?", "%"+filter.Path+"%")
|
||||
}
|
||||
if filter.StartTime != nil {
|
||||
query = query.Where("created_at >= ?", *filter.StartTime)
|
||||
}
|
||||
if filter.EndTime != nil {
|
||||
query = query.Where("created_at <= ?", *filter.EndTime)
|
||||
}
|
||||
return query
|
||||
return logs, total, nil
|
||||
}
|
||||
@@ -3,7 +3,13 @@
|
||||
|
||||
package analytics
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const userAccessLogFilterClauseCapacity = 4
|
||||
|
||||
// AccessLogFilter scopes ClickHouse user access log queries.
|
||||
type AccessLogFilter struct {
|
||||
@@ -14,4 +20,37 @@ type AccessLogFilter struct {
|
||||
StartTime *time.Time
|
||||
// EndTime filters created_at <= EndTime when non-nil.
|
||||
EndTime *time.Time
|
||||
}
|
||||
|
||||
func buildUserAccessLogFilterClause(filter AccessLogFilter) (string, []any, bool) {
|
||||
if filter.UserIDs != nil && len(filter.UserIDs) == 0 {
|
||||
return "", nil, false
|
||||
}
|
||||
|
||||
parts := make([]string, 0, userAccessLogFilterClauseCapacity)
|
||||
args := make([]any, 0, userAccessLogFilterClauseCapacity)
|
||||
if filter.UserIDs != nil {
|
||||
placeholders := make([]string, len(filter.UserIDs))
|
||||
for index, userID := range filter.UserIDs {
|
||||
placeholders[index] = "?"
|
||||
args = append(args, userID)
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("user_id IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
if trimmed := strings.TrimSpace(filter.Path); trimmed != "" {
|
||||
parts = append(parts, "path LIKE ?")
|
||||
args = append(args, "%"+trimmed+"%")
|
||||
}
|
||||
if filter.StartTime != nil {
|
||||
parts = append(parts, "created_at >= ?")
|
||||
args = append(args, *filter.StartTime)
|
||||
}
|
||||
if filter.EndTime != nil {
|
||||
parts = append(parts, "created_at <= ?")
|
||||
args = append(args, *filter.EndTime)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "1", args, true
|
||||
}
|
||||
return strings.Join(parts, " AND "), args, true
|
||||
}
|
||||
@@ -38,15 +38,12 @@ func GetDailyTrend(ctx context.Context, days int) ([]DailyTrend, error) {
|
||||
if days < 1 {
|
||||
days = 7
|
||||
}
|
||||
|
||||
ch := db.ChDB(ctx)
|
||||
if ch == nil {
|
||||
return nil, fmt.Errorf("clickhouse gorm connection is not initialized")
|
||||
if err := userAccessLogConn(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
startTime := time.Now().AddDate(0, 0, -(days - 1)).Truncate(hoursInDay * time.Hour)
|
||||
tableName := analyticsmodel.UserAccessLog{}.TableName()
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
SELECT toDate(created_at) AS date, count() AS count
|
||||
FROM %s
|
||||
@@ -55,24 +52,26 @@ func GetDailyTrend(ctx context.Context, days int) ([]DailyTrend, error) {
|
||||
ORDER BY date ASC
|
||||
`, tableName)
|
||||
|
||||
type trendRow struct {
|
||||
Date time.Time
|
||||
Count uint64
|
||||
}
|
||||
|
||||
var rows []trendRow
|
||||
if err := ch.Raw(query, startTime).Scan(&rows).Error; err != nil {
|
||||
rows, err := db.ChConn.Query(ctx, query, startTime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get daily trend: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
trendMap := make(map[string]uint64, days)
|
||||
for i := 0; i < days; i++ {
|
||||
dateStr := time.Now().AddDate(0, 0, -i).Format("2006-01-02")
|
||||
trendMap[dateStr] = 0
|
||||
}
|
||||
for _, row := range rows {
|
||||
dateStr := row.Date.Format("2006-01-02")
|
||||
trendMap[dateStr] = row.Count
|
||||
for rows.Next() {
|
||||
var (
|
||||
date time.Time
|
||||
count uint64
|
||||
)
|
||||
if err := rows.Scan(&date, &count); err != nil {
|
||||
return nil, fmt.Errorf("scan daily trend row: %w", err)
|
||||
}
|
||||
trendMap[date.Format("2006-01-02")] = count
|
||||
}
|
||||
|
||||
result := make([]DailyTrend, 0, days)
|
||||
@@ -88,9 +87,8 @@ func GetDailyTrend(ctx context.Context, days int) ([]DailyTrend, error) {
|
||||
|
||||
// GetBrowserDistribution returns browser-grouped access counts since startTime.
|
||||
func GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]BrowserShare, error) {
|
||||
ch := db.ChDB(ctx)
|
||||
if ch == nil {
|
||||
return nil, fmt.Errorf("clickhouse gorm connection is not initialized")
|
||||
if err := userAccessLogConn(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tableName := analyticsmodel.UserAccessLog{}.TableName()
|
||||
@@ -99,22 +97,27 @@ func GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]Browser
|
||||
FROM %s
|
||||
WHERE created_at >= ?
|
||||
GROUP BY user_agent
|
||||
ORDER BY count DESC
|
||||
LIMIT 100
|
||||
`, tableName)
|
||||
|
||||
type uaRow struct {
|
||||
UserAgent string
|
||||
Count uint64
|
||||
}
|
||||
|
||||
var rows []uaRow
|
||||
if err := ch.Raw(query, startTime).Scan(&rows).Error; err != nil {
|
||||
rows, err := db.ChConn.Query(ctx, query, startTime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get browser distribution: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
browserCounts := make(map[string]uint64)
|
||||
for _, row := range rows {
|
||||
browser := ParseBrowserName(row.UserAgent)
|
||||
browserCounts[browser] += row.Count
|
||||
for rows.Next() {
|
||||
var (
|
||||
userAgent string
|
||||
count uint64
|
||||
)
|
||||
if err := rows.Scan(&userAgent, &count); err != nil {
|
||||
return nil, fmt.Errorf("scan browser distribution row: %w", err)
|
||||
}
|
||||
browser := ParseBrowserName(userAgent)
|
||||
browserCounts[browser] += count
|
||||
}
|
||||
|
||||
result := make([]BrowserShare, 0, len(browserCounts))
|
||||
@@ -135,10 +138,8 @@ func GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]T
|
||||
if limit < 1 {
|
||||
limit = 10
|
||||
}
|
||||
|
||||
ch := db.ChDB(ctx)
|
||||
if ch == nil {
|
||||
return nil, fmt.Errorf("clickhouse gorm connection is not initialized")
|
||||
if err := userAccessLogConn(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tableName := analyticsmodel.UserAccessLog{}.TableName()
|
||||
@@ -151,9 +152,19 @@ func GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]T
|
||||
LIMIT ?
|
||||
`, tableName)
|
||||
|
||||
var users []TopUser
|
||||
if err := ch.Raw(query, startTime, limit).Scan(&users).Error; err != nil {
|
||||
rows, err := db.ChConn.Query(ctx, query, startTime, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get top active users: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var users []TopUser
|
||||
for rows.Next() {
|
||||
var item TopUser
|
||||
if err := rows.Scan(&item.UserID, &item.Count); err != nil {
|
||||
return nil, fmt.Errorf("scan top active user row: %w", err)
|
||||
}
|
||||
users = append(users, item)
|
||||
}
|
||||
return users, nil
|
||||
}
|
||||
@@ -12,24 +12,10 @@ import (
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func setupChGormDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
|
||||
gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gormDB.AutoMigrate(&analyticsmodel.UserAccessLog{}))
|
||||
db.SetChDBForTest(gormDB)
|
||||
return gormDB
|
||||
}
|
||||
|
||||
func TestParseBrowserName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -52,56 +38,24 @@ func TestParseBrowserName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountAccessLogs_EmptyUserIDs(t *testing.T) {
|
||||
setupChGormDB(t)
|
||||
t.Cleanup(func() { db.SetChDBForTest(nil) })
|
||||
func TestBuildUserAccessLogFilterClause_EmptyUserIDs(t *testing.T) {
|
||||
_, _, ok := buildUserAccessLogFilterClause(AccessLogFilter{UserIDs: []uint64{}})
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestCountAccessLogs_EmptyUserIDs(t *testing.T) {
|
||||
count, err := CountAccessLogs(context.Background(), AccessLogFilter{UserIDs: []uint64{}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(0), count)
|
||||
}
|
||||
|
||||
func TestListAccessLogs_EmptyUserIDs(t *testing.T) {
|
||||
setupChGormDB(t)
|
||||
t.Cleanup(func() { db.SetChDBForTest(nil) })
|
||||
|
||||
logs, total, err := ListAccessLogs(context.Background(), AccessLogFilter{UserIDs: []uint64{}}, 1, 20)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(0), total)
|
||||
assert.Empty(t, logs)
|
||||
}
|
||||
|
||||
func TestListAccessLogs_WithFilters(t *testing.T) {
|
||||
gormDB := setupChGormDB(t)
|
||||
t.Cleanup(func() { db.SetChDBForTest(nil) })
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
logs := []analyticsmodel.UserAccessLog{
|
||||
{ID: 1, UserID: 10, Path: "/api/v1/users", Method: "GET", Status: 200, CreatedAt: now},
|
||||
{ID: 2, UserID: 20, Path: "/api/v1/admin/logs", Method: "GET", Status: 200, CreatedAt: now},
|
||||
{ID: 3, UserID: 10, Path: "/api/v1/other", Method: "POST", Status: 201, CreatedAt: now},
|
||||
}
|
||||
require.NoError(t, gormDB.Create(&logs).Error)
|
||||
|
||||
start := now.Add(-time.Hour)
|
||||
filter := AccessLogFilter{
|
||||
UserIDs: []uint64{10},
|
||||
Path: "users",
|
||||
StartTime: &start,
|
||||
}
|
||||
|
||||
count, err := CountAccessLogs(context.Background(), filter)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(1), count)
|
||||
|
||||
result, total, err := ListAccessLogs(context.Background(), filter, 1, 10)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(1), total)
|
||||
require.Len(t, result, 1)
|
||||
assert.Equal(t, uint64(1), result[0].ID)
|
||||
assert.Equal(t, "/api/v1/users", result[0].Path)
|
||||
}
|
||||
|
||||
func TestBatchInsert_Empty(t *testing.T) {
|
||||
err := BatchInsert(context.Background(), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -5,13 +5,6 @@ package analytics
|
||||
|
||||
import "math"
|
||||
|
||||
func safeUint64Count(count int64) uint64 {
|
||||
if count < 0 {
|
||||
return 0
|
||||
}
|
||||
return uint64(count)
|
||||
}
|
||||
|
||||
func safeInt64Count(count uint64) int64 {
|
||||
if count > math.MaxInt64 {
|
||||
return math.MaxInt64
|
||||
|
||||
@@ -31,24 +31,3 @@ func TestSafeInt64Count(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeUint64Count(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
count int64
|
||||
want uint64
|
||||
}{
|
||||
{name: "zero", count: 0, want: 0},
|
||||
{name: "positive", count: 42, want: 42},
|
||||
{name: "negative clamps", count: -1, want: 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := safeUint64Count(tt.count); got != tt.want {
|
||||
t.Fatalf("safeUint64Count(%d) = %d, want %d", tt.count, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
const (
|
||||
// CleanupModeTTLMaterialize expires rows via table TTL instead of ALTER DELETE mutations.
|
||||
CleanupModeTTLMaterialize = "ttl_materialize"
|
||||
// CleanupModeTruncate removes all rows via TRUNCATE TABLE.
|
||||
CleanupModeTruncate = "truncate"
|
||||
)
|
||||
|
||||
// CleanupOutcome describes a non-mutation ClickHouse cleanup operation.
|
||||
type CleanupOutcome struct {
|
||||
EligibleCount int64
|
||||
Mode string
|
||||
}
|
||||
|
||||
func countClickHouseRows(ctx context.Context, conn driver.Conn, countSQL string, countArgs []any) (int64, error) {
|
||||
var count uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count clickhouse rows: %w", err)
|
||||
}
|
||||
return safeInt64Count(count), nil
|
||||
}
|
||||
|
||||
func materializeTableTTL(ctx context.Context, conn driver.Conn, tableName string) error {
|
||||
sql := fmt.Sprintf("ALTER TABLE %s MATERIALIZE TTL", tableName)
|
||||
if err := conn.Exec(ctx, sql); err != nil {
|
||||
return fmt.Errorf("materialize ttl on %s: %w", tableName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func expireRowsViaTTL(ctx context.Context, conn driver.Conn, tableName string, countSQL string, countArgs []any) (CleanupOutcome, error) {
|
||||
count, err := countClickHouseRows(ctx, conn, countSQL, countArgs)
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
if count == 0 {
|
||||
return CleanupOutcome{Mode: CleanupModeTTLMaterialize}, nil
|
||||
}
|
||||
if err := materializeTableTTL(ctx, conn, tableName); err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
return CleanupOutcome{
|
||||
EligibleCount: count,
|
||||
Mode: CleanupModeTTLMaterialize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func truncateClickHouseTable(ctx context.Context, conn driver.Conn, tableName string) (CleanupOutcome, error) {
|
||||
count, err := countClickHouseRows(ctx, conn, "SELECT count() FROM "+tableName, nil)
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
if count == 0 {
|
||||
return CleanupOutcome{Mode: CleanupModeTruncate}, nil
|
||||
}
|
||||
if err := conn.Exec(ctx, "TRUNCATE TABLE "+tableName); err != nil {
|
||||
return CleanupOutcome{}, fmt.Errorf("truncate %s: %w", tableName, err)
|
||||
}
|
||||
return CleanupOutcome{
|
||||
EligibleCount: count,
|
||||
Mode: CleanupModeTruncate,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/config"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
)
|
||||
|
||||
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure.
|
||||
type ClickHouseOperationalStats struct {
|
||||
Database string `json:"database"`
|
||||
ActiveParts int64 `json:"active_parts"`
|
||||
TotalRows int64 `json:"total_rows"`
|
||||
PendingMutations int64 `json:"pending_mutations"`
|
||||
AsyncInsertQueue int64 `json:"async_insert_queue"`
|
||||
AsyncInsertBytes int64 `json:"async_insert_bytes"`
|
||||
}
|
||||
|
||||
// GetClickHouseOperationalStats returns operational metrics for the configured database.
|
||||
func GetClickHouseOperationalStats(ctx context.Context) (*ClickHouseOperationalStats, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, fmt.Errorf("clickhouse native connection is not initialized")
|
||||
}
|
||||
database := config.Config.ClickHouse.Database
|
||||
stats := &ClickHouseOperationalStats{Database: database}
|
||||
|
||||
partsSQL := `
|
||||
SELECT
|
||||
count() AS active_parts,
|
||||
ifNull(sum(rows), 0) AS total_rows
|
||||
FROM system.parts
|
||||
WHERE active AND database = ?`
|
||||
var activeParts, totalRows uint64
|
||||
if err := db.ChConn.QueryRow(ctx, partsSQL, database).Scan(&activeParts, &totalRows); err != nil {
|
||||
return nil, fmt.Errorf("query system.parts: %w", err)
|
||||
}
|
||||
stats.ActiveParts = safeInt64Count(activeParts)
|
||||
stats.TotalRows = safeInt64Count(totalRows)
|
||||
|
||||
mutationsSQL := `
|
||||
SELECT count()
|
||||
FROM system.mutations
|
||||
WHERE is_done = 0 AND database = ?`
|
||||
if err := db.ChConn.QueryRow(ctx, mutationsSQL, database).Scan(&stats.PendingMutations); err != nil {
|
||||
return nil, fmt.Errorf("query system.mutations: %w", err)
|
||||
}
|
||||
|
||||
asyncSQL := `
|
||||
SELECT
|
||||
count() AS queue_entries,
|
||||
ifNull(sum(bytes), 0) AS queue_bytes
|
||||
FROM system.asynchronous_inserts
|
||||
WHERE database = ?`
|
||||
var queueEntries, queueBytes uint64
|
||||
if err := db.ChConn.QueryRow(ctx, asyncSQL, database).Scan(&queueEntries, &queueBytes); err != nil {
|
||||
// Older ClickHouse versions may not expose asynchronous_inserts; treat as optional.
|
||||
stats.AsyncInsertQueue = 0
|
||||
stats.AsyncInsertBytes = 0
|
||||
} else {
|
||||
stats.AsyncInsertQueue = safeInt64Count(queueEntries)
|
||||
stats.AsyncInsertBytes = safeInt64Count(queueBytes)
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
@@ -87,23 +87,16 @@ func CountNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
|
||||
var totalRecords uint64
|
||||
countSQL := fmt.Sprintf("SELECT count() FROM %s WHERE %s", tableName, clause)
|
||||
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords); err != nil {
|
||||
countSQL := fmt.Sprintf(`
|
||||
SELECT
|
||||
count() AS total_records,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS total_ips
|
||||
FROM %s
|
||||
WHERE %s`, tableName, clause)
|
||||
var totalRecords, totalIPs uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords, &totalIPs); err != nil {
|
||||
return 0, 0, fmt.Errorf("count node access logs: %w", err)
|
||||
}
|
||||
|
||||
ipSQL := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT trim(remote_addr) AS trimmed_remote_addr
|
||||
FROM %s
|
||||
WHERE %s AND trim(remote_addr) != ''
|
||||
GROUP BY trimmed_remote_addr
|
||||
)`, tableName, clause)
|
||||
var totalIPs uint64
|
||||
if err := conn.QueryRow(ctx, ipSQL, args...).Scan(&totalIPs); err != nil {
|
||||
return 0, 0, fmt.Errorf("count node access log ips: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalRecords), safeInt64Count(totalIPs), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -11,50 +11,55 @@ import (
|
||||
|
||||
// DeleteAllNodeAccessLogs deletes all node access logs.
|
||||
func DeleteAllNodeAccessLogs(ctx context.Context) (int64, error) {
|
||||
tableName := nodeAccessLogTableName()
|
||||
return deleteNodeAccessLogsWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsBefore deletes logs older than cutoff.
|
||||
func DeleteNodeAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeAccessLogTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeAccessLogsWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE logged_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsByNodeBefore deletes logs for a node older than cutoff.
|
||||
func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
tableName := nodeAccessLogTableName()
|
||||
before = before.UTC()
|
||||
return deleteNodeAccessLogsWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
|
||||
[]any{nodeID, before},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE node_id = ? AND logged_at < ?", tableName),
|
||||
nodeID, before,
|
||||
)
|
||||
}
|
||||
|
||||
func deleteNodeAccessLogsWithCount(ctx context.Context, countSQL string, countArgs []any, deleteSQL string, deleteArgs ...any) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var count uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count node access logs for delete: %w", err)
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeAccessLogTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if count == 0 {
|
||||
return 0, nil
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsBefore expires logs older than cutoff via table TTL.
|
||||
func DeleteNodeAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := conn.Exec(ctx, deleteSQL, deleteArgs...); err != nil {
|
||||
return 0, fmt.Errorf("delete node access logs: %w", err)
|
||||
tableName := nodeAccessLogTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
outcome, err := expireRowsViaTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return safeInt64Count(count), nil
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsByNodeBefore expires logs for a node older than cutoff via table TTL.
|
||||
func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tableName := nodeAccessLogTableName()
|
||||
before = before.UTC()
|
||||
outcome, err := expireRowsViaTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
|
||||
[]any{nodeID, before},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
@@ -9,7 +9,15 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const nodeAccessLogFilterClauseCapacity = 6
|
||||
const (
|
||||
nodeAccessLogFilterClauseCapacity = 6
|
||||
|
||||
nodeAccessLogSortDesc = "DESC"
|
||||
nodeAccessLogSortAsc = "ASC"
|
||||
nodeAccessLogSortAscInput = "asc"
|
||||
|
||||
nodeAccessLogColumnRemoteAddr = "remote_addr"
|
||||
)
|
||||
|
||||
// NodeAccessLogFilter scopes ClickHouse node access log queries.
|
||||
type NodeAccessLogFilter struct {
|
||||
@@ -32,7 +40,7 @@ func buildNodeAccessLogFilterClause(filter NodeAccessLogFilter) (string, []any)
|
||||
parts = append(parts, "node_id = ?")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if trimmed := strings.TrimSpace(filter.RemoteAddr); trimmed != "" {
|
||||
if trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr); trimmed != "" {
|
||||
parts = append(parts, "remote_addr LIKE ?")
|
||||
args = append(args, trimmed+"%")
|
||||
}
|
||||
@@ -66,16 +74,16 @@ func combineNodeAccessLogSQLClauses(left string, right string) string {
|
||||
}
|
||||
|
||||
func nodeAccessLogOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := "DESC"
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == "asc" {
|
||||
direction = "ASC"
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
column := "logged_at"
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "status_code":
|
||||
column = "status_code"
|
||||
case "remote_addr":
|
||||
column = "remote_addr"
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
column = nodeAccessLogColumnRemoteAddr
|
||||
case "host":
|
||||
column = "host"
|
||||
case "path":
|
||||
@@ -87,6 +95,10 @@ func nodeAccessLogOrderClause(sortBy string, sortOrder string) string {
|
||||
return column + " " + direction + ", logged_at " + direction + ", id " + direction
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogRemoteAddr(value string) string {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogSortOrder(sortOrder string) string {
|
||||
if strings.EqualFold(strings.TrimSpace(sortOrder), "asc") {
|
||||
return "asc"
|
||||
@@ -102,6 +114,43 @@ func nodeAccessLogEpochExpr() string {
|
||||
return "toInt64(toUnixTimestamp(logged_at))"
|
||||
}
|
||||
|
||||
func nodeAccessLogHostIsIPLiteralExpr() string {
|
||||
return `(
|
||||
toIPv4OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
|
||||
OR toIPv6OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
|
||||
)`
|
||||
}
|
||||
|
||||
func nodeAccessLogBucketOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "request_count":
|
||||
return "request_count " + direction + ", bucket_epoch DESC"
|
||||
default:
|
||||
return "bucket_epoch " + direction
|
||||
}
|
||||
}
|
||||
|
||||
func nodeAccessLogIPSummaryOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
column := "total_requests"
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "recent_requests":
|
||||
column = "recent_requests"
|
||||
case "last_seen_at":
|
||||
column = "last_seen_epoch"
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
column = nodeAccessLogColumnRemoteAddr
|
||||
}
|
||||
return column + " " + direction + ", last_seen_epoch DESC, remote_addr ASC"
|
||||
}
|
||||
|
||||
func nodeAccessLogTableName() string {
|
||||
return "of_node_access_logs"
|
||||
}
|
||||
|
||||
@@ -17,6 +17,20 @@ type NodeAccessLogBucketAggregate struct {
|
||||
SuccessCount int64
|
||||
ClientErrorCount int64
|
||||
ServerErrorCount int64
|
||||
UniqueIPCount int64
|
||||
UniqueHostCount int64
|
||||
}
|
||||
|
||||
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
|
||||
type NodeAccessLogWAFIPAggregate struct {
|
||||
RemoteAddr string
|
||||
RequestCount int64
|
||||
Status404Count int64
|
||||
ClientErrorCount int64
|
||||
ServerErrorCount int64
|
||||
IPHostCount int64
|
||||
LastSeenEpoch int64
|
||||
StatusCounts map[int]int64
|
||||
}
|
||||
|
||||
// NodeAccessLogBucketDimension is a bucket dimension value.
|
||||
@@ -49,7 +63,7 @@ type NodeAccessLogIPTrend struct {
|
||||
RequestCount int64
|
||||
}
|
||||
|
||||
// BucketAggregatesNodeAccessLogs returns folded bucket aggregates.
|
||||
// BucketAggregatesNodeAccessLogs returns folded bucket aggregates with unique IP/host counts.
|
||||
func BucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogBucketAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
@@ -64,10 +78,20 @@ SELECT
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ip_count,
|
||||
uniqExactIf(host, host != '') AS unique_host_count
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY bucket_epoch`, bucketExpr, tableName, clause)
|
||||
GROUP BY bucket_epoch
|
||||
ORDER BY %s`, bucketExpr, tableName, clause, nodeAccessLogBucketOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bucket aggregates node access logs: %w", err)
|
||||
@@ -77,10 +101,10 @@ GROUP BY bucket_epoch`, bucketExpr, tableName, clause)
|
||||
var result []NodeAccessLogBucketAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
bucketEpoch int64
|
||||
requestCount, successCount, clientErrorCount, serverErrorCount uint64
|
||||
bucketEpoch int64
|
||||
requestCount, successCount, clientErrorCount, serverErrorCount, uniqueIPCount, uniqueHostCount uint64
|
||||
)
|
||||
if err := rows.Scan(&bucketEpoch, &requestCount, &successCount, &clientErrorCount, &serverErrorCount); err != nil {
|
||||
if err := rows.Scan(&bucketEpoch, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &uniqueIPCount, &uniqueHostCount); err != nil {
|
||||
return nil, fmt.Errorf("scan bucket aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogBucketAggregate{
|
||||
@@ -89,11 +113,36 @@ GROUP BY bucket_epoch`, bucketExpr, tableName, clause)
|
||||
SuccessCount: safeInt64Count(successCount),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPCount),
|
||||
UniqueHostCount: safeInt64Count(uniqueHostCount),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountBucketAggregatesNodeAccessLogs returns the number of folded buckets matching filter.
|
||||
func CountBucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT 1
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY %s
|
||||
)`, tableName, clause, bucketExpr)
|
||||
var totalBuckets uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalBuckets); err != nil {
|
||||
return 0, fmt.Errorf("count bucket aggregates node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalBuckets), nil
|
||||
}
|
||||
|
||||
// BucketDimensionsNodeAccessLogs returns bucket dimension values.
|
||||
func BucketDimensionsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, bucketSeconds int64) ([]NodeAccessLogBucketDimension, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
@@ -137,26 +186,26 @@ func IPAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter,
|
||||
queryClause := clause
|
||||
queryArgs := append([]any{}, args...)
|
||||
if exactRemoteAddr {
|
||||
trimmed := strings.TrimSpace(filter.RemoteAddr)
|
||||
trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr)
|
||||
if trimmed == "" {
|
||||
return []NodeAccessLogIPAggregate{}, nil
|
||||
}
|
||||
queryClause = combineNodeAccessLogSQLClauses(queryClause, "trim(remote_addr) = ?")
|
||||
queryClause = combineNodeAccessLogSQLClauses(queryClause, "remote_addr = ?")
|
||||
queryArgs = append(queryArgs, trimmed)
|
||||
}
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
trim(remote_addr) AS trimmed_remote_addr,
|
||||
remote_addr,
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND trim(remote_addr) != ''
|
||||
GROUP BY trimmed_remote_addr`, lastSeenExpr, tableName, queryClause)
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, lastSeenExpr, tableName, queryClause)
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip aggregates node access logs: %w", err)
|
||||
@@ -185,7 +234,7 @@ GROUP BY trimmed_remote_addr`, lastSeenExpr, tableName, queryClause)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// IPSummariesNodeAccessLogs returns IP summary rows.
|
||||
// IPSummariesNodeAccessLogs returns paginated IP summary rows.
|
||||
func IPSummariesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, recentSince time.Time) ([]NodeAccessLogIPSummary, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
@@ -203,13 +252,21 @@ func IPSummariesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter,
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
trim(remote_addr) AS trimmed_remote_addr,
|
||||
remote_addr,
|
||||
count() AS total_requests,
|
||||
sum(%s) AS recent_requests,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND trim(remote_addr) != ''
|
||||
GROUP BY trimmed_remote_addr`, recentClause, lastSeenExpr, tableName, clause)
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr
|
||||
ORDER BY %s`, recentClause, lastSeenExpr, tableName, clause, nodeAccessLogIPSummaryOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
queryArgs = append(queryArgs, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip summaries node access logs: %w", err)
|
||||
@@ -236,6 +293,140 @@ GROUP BY trimmed_remote_addr`, recentClause, lastSeenExpr, tableName, clause)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountIPSummaryNodeAccessLogs returns the number of distinct IPs matching filter.
|
||||
func CountIPSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT 1
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr
|
||||
)`, tableName, clause)
|
||||
var totalIPs uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalIPs); err != nil {
|
||||
return 0, fmt.Errorf("count ip summary node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalIPs), nil
|
||||
}
|
||||
|
||||
// IPAggregatesForWAFNodeAccessLogs returns per-IP aggregates for WAF automatic rules.
|
||||
func IPAggregatesForWAFNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogWAFIPAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
hostIsIPExpr := nodeAccessLogHostIsIPLiteralExpr()
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
count() AS request_count,
|
||||
countIf(status_code = 404) AS status_404_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
countIf(%s) AS ip_host_count,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, hostIsIPExpr, lastSeenExpr, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip aggregates for waf node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
aggregates := make(map[string]*NodeAccessLogWAFIPAggregate)
|
||||
order := make([]string, 0)
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
lastSeenEpoch int64
|
||||
requestCount, status404Count, clientErrorCount, serverErrorCount, ipHostCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &requestCount, &status404Count, &clientErrorCount, &serverErrorCount, &ipHostCount, &lastSeenEpoch); err != nil {
|
||||
return nil, fmt.Errorf("scan waf ip aggregate row: %w", err)
|
||||
}
|
||||
remoteAddr = strings.TrimSpace(remoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
aggregates[remoteAddr] = &NodeAccessLogWAFIPAggregate{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
Status404Count: safeInt64Count(status404Count),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
IPHostCount: safeInt64Count(ipHostCount),
|
||||
LastSeenEpoch: lastSeenEpoch,
|
||||
StatusCounts: make(map[int]int64),
|
||||
}
|
||||
order = append(order, remoteAddr)
|
||||
}
|
||||
if err := mergeWAFIPStatusCodeCounts(ctx, filter, aggregates); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]NodeAccessLogWAFIPAggregate, 0, len(order))
|
||||
for _, remoteAddr := range order {
|
||||
if aggregate := aggregates[remoteAddr]; aggregate != nil {
|
||||
result = append(result, *aggregate)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func mergeWAFIPStatusCodeCounts(ctx context.Context, filter NodeAccessLogFilter, aggregates map[string]*NodeAccessLogWAFIPAggregate) error {
|
||||
if len(aggregates) == 0 {
|
||||
return nil
|
||||
}
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
status_code,
|
||||
count() AS status_count
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr, status_code`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("waf ip status code counts: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
statusCode int32
|
||||
statusCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &statusCode, &statusCount); err != nil {
|
||||
return fmt.Errorf("scan waf ip status code row: %w", err)
|
||||
}
|
||||
remoteAddr = strings.TrimSpace(remoteAddr)
|
||||
aggregate := aggregates[remoteAddr]
|
||||
if aggregate == nil {
|
||||
continue
|
||||
}
|
||||
if aggregate.StatusCounts == nil {
|
||||
aggregate.StatusCounts = make(map[int]int64)
|
||||
}
|
||||
aggregate.StatusCounts[int(statusCode)] = safeInt64Count(statusCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IPTrendNodeAccessLogs returns IP trend bucket rows.
|
||||
func IPTrendNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogIPTrend, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
|
||||
@@ -6,6 +6,7 @@ package analytics
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
@@ -41,7 +42,7 @@ func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAc
|
||||
id,
|
||||
logItem.NodeID,
|
||||
logItem.LoggedAt.UTC(),
|
||||
logItem.RemoteAddr,
|
||||
strings.TrimSpace(logItem.RemoteAddr),
|
||||
logItem.Region,
|
||||
logItem.Host,
|
||||
logItem.Path,
|
||||
|
||||
@@ -6,6 +6,7 @@ package analytics
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
@@ -244,6 +245,59 @@ func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
const nodeTrafficHourlyTableName = "of_node_traffic_hourly"
|
||||
|
||||
// NodeTrafficHourly is an hourly traffic rollup row.
|
||||
type NodeTrafficHourly struct {
|
||||
NodeID string
|
||||
Hour time.Time
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueVisitorCount int64
|
||||
}
|
||||
|
||||
// ListNodeTrafficHourly returns hourly traffic rollup rows matching filter.
|
||||
func ListNodeTrafficHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeTrafficHourly, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "hour")
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
node_id,
|
||||
hour,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
sum(unique_visitor_count) AS unique_visitor_count
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY node_id, hour
|
||||
ORDER BY hour ASC`, nodeTrafficHourlyTableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node traffic hourly: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
result := make([]NodeTrafficHourly, 0)
|
||||
for rows.Next() {
|
||||
var (
|
||||
item NodeTrafficHourly
|
||||
requestCount, errorCount, uniqueVisitorCount uint64
|
||||
)
|
||||
if err := rows.Scan(&item.NodeID, &item.Hour, &requestCount, &errorCount, &uniqueVisitorCount); err != nil {
|
||||
return nil, fmt.Errorf("scan node traffic hourly row: %w", err)
|
||||
}
|
||||
item.Hour = item.Hour.UTC()
|
||||
item.RequestCount = safeInt64Count(requestCount)
|
||||
item.ErrorCount = safeInt64Count(errorCount)
|
||||
item.UniqueVisitorCount = safeInt64Count(uniqueVisitorCount)
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsFrpcRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
var result []analyticsmodel.NodeObsFrpc
|
||||
for rows.Next() {
|
||||
|
||||
@@ -11,113 +11,170 @@ import (
|
||||
|
||||
// DeleteAllNodeMetricSnapshots deletes all node metric snapshots.
|
||||
func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeMetricSnapshotsBefore deletes metric snapshots captured before cutoff.
|
||||
func DeleteNodeMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeRequestReports deletes all node request reports.
|
||||
func DeleteAllNodeRequestReports(ctx context.Context) (int64, error) {
|
||||
tableName := nodeRequestReportTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeRequestReportsBefore deletes request reports ending before cutoff.
|
||||
func DeleteNodeRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeRequestReportTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE window_ended_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE window_ended_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsOpenresty deletes all OpenResty observations.
|
||||
func DeleteAllNodeObsOpenresty(ctx context.Context) (int64, error) {
|
||||
tableName := nodeObsOpenrestyTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeObsOpenrestyBefore deletes OpenResty observations captured before cutoff.
|
||||
func DeleteNodeObsOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeObsOpenrestyTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrps deletes all FRPS observations.
|
||||
func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
|
||||
tableName := nodeObsFrpsTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpsBefore deletes FRPS observations captured before cutoff.
|
||||
func DeleteNodeObsFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeObsFrpsTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrpc deletes all FRPC observations.
|
||||
func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
|
||||
tableName := nodeObsFrpcTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpcBefore deletes FRPC observations captured before cutoff.
|
||||
func DeleteNodeObsFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeObsFrpcTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
func deleteNodeObservabilityWithCount(ctx context.Context, countSQL string, countArgs []any, deleteSQL string, deleteArgs ...any) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var count uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count node observability rows for delete: %w", err)
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeMetricSnapshotTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if count == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := conn.Exec(ctx, deleteSQL, deleteArgs...); err != nil {
|
||||
return 0, fmt.Errorf("delete node observability rows: %w", err)
|
||||
}
|
||||
return safeInt64Count(count), nil
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeMetricSnapshotsBefore expires metric snapshots captured before cutoff via table TTL.
|
||||
func DeleteNodeMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
outcome, err := expireRowsViaTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteAllNodeRequestReports deletes all node request reports.
|
||||
func DeleteAllNodeRequestReports(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeRequestReportTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeRequestReportsBefore expires request reports ending before cutoff via table TTL.
|
||||
func DeleteNodeRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tableName := nodeRequestReportTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
outcome, err := expireRowsViaTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE window_ended_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsOpenresty deletes all OpenResty observations.
|
||||
func DeleteAllNodeObsOpenresty(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsOpenrestyTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeObsOpenrestyBefore expires OpenResty observations captured before cutoff via table TTL.
|
||||
func DeleteNodeObsOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tableName := nodeObsOpenrestyTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
outcome, err := expireRowsViaTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrps deletes all FRPS observations.
|
||||
func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpsTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpsBefore expires FRPS observations captured before cutoff via table TTL.
|
||||
func DeleteNodeObsFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tableName := nodeObsFrpsTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
outcome, err := expireRowsViaTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrpc deletes all FRPC observations.
|
||||
func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpcTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpcBefore expires FRPC observations captured before cutoff via table TTL.
|
||||
func DeleteNodeObsFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tableName := nodeObsFrpcTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
outcome, err := expireRowsViaTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
@@ -51,6 +51,7 @@ func RegisterAdminRoutes(apiV1Router *gin.RouterGroup) {
|
||||
func registerAdminDiagnosticRoutes(adminRouter *gin.RouterGroup) {
|
||||
// System status
|
||||
adminRouter.GET("/status", admin_status.GetSystemStatus)
|
||||
adminRouter.GET("/status/clickhouse", admin_status.GetClickHouseStatus)
|
||||
|
||||
// Database basic info & backup export
|
||||
adminRouter.GET("/db-info", admin_status.GetDatabaseInfo)
|
||||
|
||||
@@ -142,7 +142,7 @@ func getSeedConfigsPart1() []model.SystemConfig {
|
||||
},
|
||||
{
|
||||
Key: model.ConfigKeyCapLoginEnabled,
|
||||
Value: configValueTrue,
|
||||
Value: configValueFalse,
|
||||
Type: configTypeSystem,
|
||||
Description: "是否启用登录人机验证(true/false)",
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user