Merge branch 'codex/log-database-decoupling'

# Conflicts:
#	docs/changelog/index.md
#	frontend/app/(main)/error-pages/page.tsx
#	internal/infra/persistence/migrator/migrator_test.go
This commit is contained in:
ryan
2026-08-09 08:37:37 +08:00
95 changed files with 10697 additions and 3744 deletions
+4 -10
View File
@@ -22,21 +22,15 @@ sidebar: false
## [Unreleased]
## [v3.5.0] - 2026-08-08
### 新增
- 支持 Service Worker 离线兜底:为启用 HTTPS 的网站下发 Service Worker 并缓存离线页,域名无法访问时浏览器展示离线兜底页面,减少用户流失。可指定生效域名范围(仅对选中的 HTTPS 域名生效),配置位于「响应页面」-「离线页」,可在版本发布中批量生效。
- 日志存储解耦:新增日志存储抽象(`internal/repository/logstore`),ClickHouse 变为可选项,不启用时由 PostgreSQL/SQLite 承担全部日志功能;新增「切换日志数据库」任务支持 PostgreSQL/SQLite 与 ClickHouse 间数据迁移(迁移期间冻结日志写入,成功后自动切换主库并保留源数据);日志保留时间改为按存储库在业务配置中设置(`log_retention_days_*`),过期清理并入系统垃圾清理每日任务。
### 修复
- 修复 PoW 挑战页潜在 XSS:错误提示与状态文案改用纯文本渲染,挑战通过后的 `redir` 跳转参数仅允许 http/https 协议,防止异常文本被当作 HTML 执行或跳转到危险协议。
- 修复邮件发送的邮件头注入风险:标题、发件人、收件人在写入邮件头前清除 CR/LF 换行符,防止注入额外邮件头(CWE-93)。
- 修复 UptimeKuma 同步调试日志泄露凭据:输出日志前对 password/token/secret 等敏感字段打码,避免凭据进入日志。
### 改进
- 离线页预制模板支持:新增离线页内置预制模板套件(「极简白底」、「线框拓扑」、「包豪斯」),与源站错误页模板风格保持一致,可在编辑界面一键加载与预览。
- 「切换日志数据库」迁移任务补齐第 6 张表(用户访问日志)的复制与清空;目标为 PostgreSQL 时按源库时间范围预建分区,历史日志可正常迁移;冻结前先排空批写入队列,避免在途日志丢失。
- 节点监控最新指标读取仅在 ClickHouse 为当前日志库时走 ClickHouse 快速路径,日志库切换后仪表盘不再读到旧库数据。
- `log_database` / `log_db_migration` 为受保护配置,管理端创建/修改接口均拒绝手动写入。
## [v3.4.5] - 2026-08-08
+89 -167
View File
@@ -1024,7 +1024,7 @@ const docTemplate = `{
"SessionCookie": []
}
],
"description": "分页并按照用户、接口路径、时间范围等维度检索 ClickHouse 用户访问日志列表(需要管理员权限,ClickHouse 未启用时报错)",
"description": "分页并按照用户、接口路径、时间范围等维度检索用户访问日志列表(需要管理员权限,日志存储未启用时报错)",
"produces": [
"application/json"
],
@@ -1092,7 +1092,7 @@ const docTemplate = `{
}
},
"400": {
"description": "ClickHouse 未启用或参数错误",
"description": "日志存储未启用或参数错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
@@ -1119,7 +1119,7 @@ const docTemplate = `{
"SessionCookie": []
}
],
"description": "聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,ClickHouse 未启用时报错)",
"description": "聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,日志存储未启用时报错)",
"produces": [
"application/json"
],
@@ -1147,7 +1147,7 @@ const docTemplate = `{
}
},
"400": {
"description": "ClickHouse 未启用",
"description": "日志存储未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
@@ -1877,21 +1877,21 @@ const docTemplate = `{
}
}
},
"/api/v1/admin/status/clickhouse": {
"/api/v1/admin/status/log-database": {
"get": {
"security": [
{
"SessionCookie": []
}
],
"description": "返回 ClickHouse parts、mutation、async_insert 队列及进程内 batch writer 指标,需要管理员权限",
"description": "返回当前日志主库、迁移状态、各库保留天数与合法迁移目标,需要管理员权限",
"produces": [
"application/json"
],
"tags": [
"admin"
],
"summary": "获取 ClickHouse 运行指标",
"summary": "获取日志数据库状态",
"responses": {
"200": {
"description": "获取成功",
@@ -1904,19 +1904,13 @@ const docTemplate = `{
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/analytics.ClickHouseOperationalStats"
"$ref": "#/definitions/status.LogDatabaseStatus"
}
}
}
]
}
},
"400": {
"description": "ClickHouse 未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
@@ -2393,6 +2387,18 @@ const docTemplate = `{
"name": "task_type",
"in": "query"
},
{
"type": "string",
"description": "任务类型前缀筛选(与 task_type / task_types 互斥,精确类型优先)",
"name": "task_type_prefix",
"in": "query"
},
{
"type": "string",
"description": "逗号分隔的精确任务类型列表(IN 筛选,优先于前缀)",
"name": "task_types",
"in": "query"
},
{
"type": "integer",
"default": 1,
@@ -8305,80 +8311,6 @@ const docTemplate = `{
}
}
},
"/api/v1/d/option/database/cleanup": {
"post": {
"security": [
{
"SessionCookie": []
}
],
"description": "按目标与保留天数清理可观测性相关数据表,需要管理员权限",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"openflare-option"
],
"summary": "清理可观测性数据库",
"parameters": [
{
"description": "清理参数",
"name": "request",
"in": "body",
"schema": {
"$ref": "#/definitions/option.databaseCleanupInput"
}
}
],
"responses": {
"200": {
"description": "清理结果",
"schema": {
"allOf": [
{
"$ref": "#/definitions/response.Any"
},
{
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/option.databaseCleanupResult"
}
}
}
]
}
},
"400": {
"description": "参数错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"404": {
"description": "无权限或不存在",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"500": {
"description": "内部错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
}
}
}
},
"/api/v1/d/option/geoip/lookup": {
"post": {
"security": [
@@ -14906,33 +14838,26 @@ const docTemplate = `{
}
}
},
"analytics.ClickHouseOperationalStats": {
"analytics.BatchWriterStats": {
"type": "object",
"properties": {
"active_parts": {
"cap": {
"type": "integer"
},
"async_insert_bytes": {
"depth": {
"type": "integer"
},
"async_insert_queue": {
"drops": {
"type": "integer"
},
"batch_writers": {
"description": "BatchWriters reports in-process queue depth/drops/flush errors for CH writers.",
"type": "array",
"items": {
"$ref": "#/definitions/batchwriter.Stats"
}
"flush_errors": {
"type": "integer"
},
"database": {
"name": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
"running": {
"type": "boolean"
}
}
},
@@ -15024,29 +14949,6 @@ const docTemplate = `{
}
}
},
"batchwriter.Stats": {
"type": "object",
"properties": {
"cap": {
"type": "integer"
},
"depth": {
"type": "integer"
},
"drops": {
"type": "integer"
},
"flush_errors": {
"type": "integer"
},
"name": {
"type": "string"
},
"running": {
"type": "boolean"
}
}
},
"cache.updateCacheConfigRequest": {
"type": "object",
"required": [
@@ -15105,6 +15007,9 @@ const docTemplate = `{
"id": {
"type": "integer"
},
"zone_domain": {
"type": "string"
},
"zone_id": {
"type": "integer"
}
@@ -15826,6 +15731,36 @@ const docTemplate = `{
}
}
},
"github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats": {
"type": "object",
"properties": {
"active_parts": {
"type": "integer"
},
"async_insert_bytes": {
"type": "integer"
},
"async_insert_queue": {
"type": "integer"
},
"batch_writers": {
"description": "BatchWriters reports in-process queue depth/drops/flush errors for CH writers.",
"type": "array",
"items": {
"$ref": "#/definitions/analytics.BatchWriterStats"
}
},
"database": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
}
}
},
"github_com_Rain-kl_Wavelet_pkg_protocol.ActiveConfigMeta": {
"type": "object",
"properties": {
@@ -18481,46 +18416,6 @@ const docTemplate = `{
}
}
},
"option.databaseCleanupInput": {
"type": "object",
"properties": {
"retention_days": {
"type": "integer"
},
"target": {
"type": "string"
}
}
},
"option.databaseCleanupResult": {
"type": "object",
"properties": {
"cleanup_mode": {
"type": "string"
},
"delete_all": {
"type": "boolean"
},
"deleted_count": {
"type": "integer"
},
"eligible_count": {
"type": "integer"
},
"retention_days": {
"type": "integer"
},
"table_ttl_days": {
"type": "integer"
},
"target": {
"type": "string"
},
"target_label": {
"type": "string"
}
}
},
"option.geoIPLookupRequest": {
"type": "object",
"properties": {
@@ -19859,6 +19754,33 @@ const docTemplate = `{
}
}
},
"status.LogDatabaseStatus": {
"type": "object",
"properties": {
"active_database": {
"type": "string"
},
"available_targets": {
"type": "array",
"items": {
"type": "string"
}
},
"clickhouse": {
"$ref": "#/definitions/github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats"
},
"migration": {
"description": "idle | migrating",
"type": "string"
},
"retention_days": {
"type": "object",
"additionalProperties": {
"type": "integer"
}
}
}
},
"status.SystemStatusResponse": {
"type": "object",
"properties": {
+14 -2
View File
@@ -197,8 +197,6 @@ Server 的所有核心基础配置定义在 `config.yaml` 中,且均支持环
| `node_offline_threshold` | `int` | 在管理后台中判定节点失去心跳并标注为离线状态的无响应阈值(毫秒) | `60000` (60s) |
| `agent_update_repo` | `string` | Agent 节点更新下载自身二进制的 Release 仓库源 | `Rain-kl/OpenFlare` |
| `geoip_provider` | `string` | GeoIP 提供商,支持 `maxmind` 等,用于 WAF 防护时地域分析 | `ipinfo` |
| `database_auto_cleanup_enabled` | `bool` | 是否在每天凌晨 3:00 自动清理过期观测历史日志(降低数据库空间) | `true` |
| `database_auto_cleanup_retention_days` | `int` | 自动清理观测数据(访问日志、度量曲线、审计等)的默认保留天数 | `30` |
### 5. Uptime Kuma 监控联动同步
| 配置键 (Key) | 数据类型 | 作用说明 | 默认值 |
@@ -272,6 +270,20 @@ Server 的所有核心基础配置定义在 `config.yaml` 中,且均支持环
---
### 8. 日志存储(Log Database)
日志存储解耦后的运行时配置:日志主库由「切换日志数据库」任务管理(内部/受保护 key,禁止管理员手动修改),保留天数按存储库分别在业务配置中设置。
| 配置键 (Key) | 数据类型 | 作用说明 | 默认值 |
| --- | --- | --- | --- |
| `log_database` | `string` | 当前日志主库(`postgres` / `sqlite` / `clickhouse`)。**内部受保护 key**:仅「切换日志数据库」迁移任务写入,管理员不可手动创建/修改 | 随主库(PostgreSQL 启用时为 `postgres`,否则 `sqlite`;ClickHouse 启用时优先 `clickhouse`) |
| `log_db_migration` | `string` | 日志迁移冻结标记(`migrating` 或空)。**内部受保护 key**:仅迁移任务写入,置位期间日志写入返回 503「日志数据库迁移中,暂不可写」 | 空 |
| `log_retention_days_postgres` | `int` | PostgreSQL 日志库的过期清理保留天数(过期日志由系统垃圾清理每日任务删除) | `90` |
| `log_retention_days_sqlite` | `int` | SQLite 日志库的过期清理保留天数 | `90` |
| `log_retention_days_clickhouse` | `int` | ClickHouse 日志库的过期清理保留天数 | `90` |
---
## 前端构建环境变量
| 环境变量 | 作用 | 默认值 |
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,177 @@
# 日志数据库解耦设计(ClickHouse 可选化)
> 状态:已与用户逐段确认,待用户复核。
> 日期:2026-08-08
## 1. 背景与目标
当前系统日志/分析(访问日志、可观测时序)完全绑定 ClickHouse:`internal/repository/analytics` 直接操作 `db.ChConn`/`db.ChDB`,apps 层(`chwriter`、`risk_control`、`admin/logs`、`admin/status`)依赖 `config.ClickHouse.Enabled` 判断可用性。业务流量小、主机性能低时 ClickHouse 负担大。
目标:
1. **解耦**:ClickHouse 变为可选项;不启用时,主库(PostgreSQL;禁用时 SQLite)完整承接全部日志功能(写入、查询、聚合、清理)。
2. **代码级约束**:上层应用写日志不能直接调用底层库(`analyticsrepo` / `db.ChConn`),用接口 + import-lint 测试保证,而非 AGENTS.md 口头约束。
3. **可迁移**:提供用户触发的「切换日志数据库」任务,支持 PostgreSQL/SQLite ↔ ClickHouse 数据迁移。
4. **表结构**:CH 日志表迁入 PG/SQLite;CH 保持只有日志表的 SQL 脚本;PG/SQLite 包含全部表。
## 2. 现状要点
- 连接:`internal/infra/persistence/clickhouse.go`(`ChConn` 原生批量写 + `ChDB` GORM 查询),`init()` 依据 `clickhouse.enabled`。
- 分析域:`internal/repository/analytics/` 直接读写 CH;apps 通过 `batchwriter` 异步 flush(`chwriter`、`risk_control`)。
- 已有抽象雏形:`internal/repository/openflare_access_log_store.go` / `openflare_observability_store.go` 中的未导出 `accessLogStore` / `observabilityStore` 接口,默认 `clickhouseAccessLogStore{}`,测试可换 memory 实现——默认写死 CH、不可配置切换、接口未导出。
- 迁移:主库 goose(`goose/postgres` + `goose/sqlite` 双方言)与 CH 单方言(`goose/clickhouse`)分离。
- 历史:PG/SQLite 曾有过 `of_node_metric_snapshots`、`of_node_access_logs` 等观测表(`202606190010_create_of_observability_tables.sql`),后由 `202606200005_drop_of_node_observability_timeseries.sql` 删除(迁去 CH)。**旧 DDL 可复活改造**。
- 任务:Asynq + `task.RegisterHandler`/`RegisterTaskMeta`;`system_cleanup`(系统垃圾清理)每日任务已存在;`of_database_auto_cleanup`(可观测清理,schedule id=102)存在。
- 系统配置:`system_configs` 表(key/type/visibility),现有 `database_auto_cleanup_enabled` / `database_auto_cleanup_retention_days`(business)。
## 3. 已确认的核心决策
| # | 决策 |
|---|---|
| 1 | 范围:CH 不启用时,PG(或 SQLite)承担**全部**日志功能;聚合在 PG/SQLite 查询时实时计算,不物理建 MV 同构表。 |
| 2 | 实现:接口定义在 repository 层;PG 用 GORM 全新实现;CH 保留现有原生批量优化(`PrepareBatch`)包进同一接口。 |
| 3 | SQLite 是一等公民:`log_database` ∈ {`postgres`, `sqlite`, `clickhouse`};迁移方向 PG→CH、SQLite→CH、CH→PG、CH→SQLite。 |
| 4 | 日志库只有两种合法状态:**随主库**(`database.enabled` → postgres,否则 sqlite)或 **clickhouse**;不存在主库 PG + 日志 SQLite 的组合。 |
| 5 | 迁移任务「切换日志数据库」:纯复制、**源数据不删除**、可重试;迁移期间**冻结日志写入**(拒绝,不排队积压);全部成功才翻转主库标记。 |
| 6 | 清理统一到 `system_cleanup`(每日一次,日志过期无需实时);保留时间按**存储库**配置(`type=business`)。 |
## 4. 包结构与接口(方案一)
新增 `internal/repository/logstore/`,职责唯一:日志存储抽象。
```
internal/repository/logstore/
├── logstore.go # 导出接口:AccessLogStore / ObservabilityStore / UserAccessLogStore / CleanupStore / StatusStore
├── provider.go # Open(ctx) 按当前日志主库返回实现;ActiveDatabase() 供状态/UI;测试可注入
├── postgres_store.go # GORM 实现(PG 与 SQLite 共用一套,方言差异只在 goose DDL + dialect_* 小文件)
├── dialect_postgres.go # PG 方言 SQL 片段(date_trunc / FILTER / 分区清理)
├── dialect_sqlite.go # SQLite 方言 SQL 片段(strftime / unixepoch)
└── clickhouse_store.go # 把现有 analyticsrepo 原生批量 + GORM 查询包进接口(零性能损耗)
```
- **接口划分**(避免 40+ 方法巨型接口,合成 `logstore.Store` 结构体持有):
- `AccessLogStore`:节点访问日志的 InsertBatch / List / Count / RegionCounts / BucketAggregates / CountBuckets / BucketDimensions / IPAggregates / IPSummaries / CountIPSummaries / WAFIPAggregates / IPTrend / TrafficSummary / ValueCounts / NodeAggregates / DeleteAll / DeleteBefore / DeleteByNodeBefore。
- `ObservabilityStore`:4 表(metric snapshots / edge health / frps / frpc)的 Insert / List / Delete。
- `UserAccessLogStore`:`w_user_access_logs` 的 BatchInsert / Count / List / 统计(DailyTrend / BrowserDistribution / TopActiveUsers 等)。
- `CleanupStore`:按保留天数清理过期数据(PG=分区 DROP + 分批 DELETE;SQLite=分批 DELETE;CH=MODIFY TTL + materialize)。
- `StatusStore`:当前库状态、CH 运行指标(激活时)、GORM 写入器状态。
- **消费面**:`internal/repository` 现有公开函数(`ListOpenFlareAccessLogs`、`InsertOpenFlareAccessLogsBatch`、`InsertOpenFlareMetricSnapshot` 等)**保留签名、改为一行委托 `logstore`**,apps 调用面几乎不动;apps 里现有 `analyticsrepo` 直连(`risk_control`、`chwriter`、`tasks/database_cleanup.go`、`observability/access_log_logics.go`、`admin/logs`、`admin/status`)全部改走 repository/logstore。
- **import-lint 测试**:新增 `go test`,扫描 `internal/apps/**` 的 import,发现 `internal/repository/analytics` 或 `internal/infra/persistence`(`batchwriter` 白名单除外)即失败。这是「代码层面规避」的验收。
- `analyticsrepo` 保留,仅被 `logstore/clickhouse_store.go` 引用(CH 实现细节)。
### 主库标记与启动校验
- `system_configs` 新增内部 key:
- `log_database`(`postgres`/`sqlite`/`clickhouse`):当前日志主库,仅迁移任务写入。
- `log_db_migration`(`"migrating"`/空):迁移冻结标记,仅迁移任务写入。
- **首次 seed**(bootstrap Go 侧,因依赖运行时主库选择):key 缺失时,`clickhouse.enabled` → `clickhouse`(保持现状、不丢现有 CH 数据);否则 → 当前主库(`database.enabled` → `postgres`,否则 `sqlite`)。
- **启动校验**(bootstrap):
- `log_database=clickhouse` 但 `clickhouse.enabled=false` → 启动报错:「当前日志主库为 ClickHouse 但 ClickHouse 未启用。请先重新启用 ClickHouse 配置并启动,在任务管理运行『切换日志数据库』迁移到 PostgreSQL/SQLite 后再禁用 ClickHouse」。
- `log_database=postgres` 但 `database.enabled=false`,或 `log_database=sqlite` 但 `database.enabled=true` → 启动报错(违反「随主库或随 CH」规则)。
- **key 保护**:`log_database`、`log_db_migration` 在配置更新接口(admin system-configs / option 校验)拒绝修改;仅迁移任务可写;启动校验兜底被篡改组合。
- **热切换**:`logstore` 通过系统配置缓存(Redis,更新即失效)读取 `log_database`;翻转后 API 进程自动切到新实现,无需自定义跨进程协议。
## 5. PG/SQLite 表结构与优化
**新建原始日志表(PG + SQLite 双方言 goose,同版本号)**——只建原始表,**不建** CH 物化视图/聚合表(`of_access_log_hourly`、`of_node_metric_capacity_hourly` 等),PG/SQLite 查询时实时聚合:
| 表 | 说明 |
|---|---|
| `w_user_access_logs` | 用户访问日志 |
| `of_node_access_logs` | 节点访问日志(含 user_agent/cache_status/bytes_sent/request_length/request_time_ms 现行列) |
| `of_node_metric_snapshots` | 资源指标 |
| `of_node_edge_health` | 边缘健康 |
| `of_node_obs_frps` | FRPS 观测 |
| `of_node_obs_frpc` | FRPC 观测 |
- **ID**:沿用 snowflake uint64(DDL 用 BIGINT,与 CH UInt64 对齐);不换自增,保证迁移 ID 原样保留、无冲突。
- **时间**:PG `TIMESTAMPTZ`;SQLite `DATETIME`。
- **复合主键**:分区表主键 `(id, 时间列)`(满足 PG 分区键进唯一索引要求)。
### PG 优化
1. **分区**:仅 `of_node_access_logs`、`w_user_access_logs` 两个高频表用 PG 原生 `PARTITION BY RANGE` **按月分区**;可观测 4 表数据量小,普通表 + 索引。SQLite 无原生分区 → 普通表 + 组合索引(方言差异只留在 goose DDL,运行时 GORM 代码共用)。
2. **批量写入**:PG/SQLite 统一 GORM `CreateInBatches`(批次 500–1000);CH 维持原生 `PrepareBatch`。
3. **索引**:
- `of_node_access_logs`:`(logged_at DESC)`、`(node_id, logged_at DESC)`、`(host, logged_at DESC)`;
- `w_user_access_logs`:`(created_at DESC)`、`(user_id, created_at DESC)`;
- 可观测表:`(node_id, captured_at DESC)`。
4. **聚合查询重写**:PG 用 `date_trunc` / `count(DISTINCT)` / `FILTER (WHERE ...)` 等价替换 CH 的 `toStartOfHour` / `uniqExact` / `countIf`;SQLite 用 `strftime` / `unixepoch`。时间分桶等少量方言 SQL 拆到 `dialect_postgres.go` / `dialect_sqlite.go`,store 主体方言中立。
### goose 迁移
- PG/SQLite 各新增一组建表迁移(复活并改造 `202606190010` 旧 DDL,按 database-migration 技能双方言、同版本号规则)。
- CH 目录不动(本来就只有日志表脚本,满足「CH 保持只有日志表 SQL」)。
## 6. 清理(并入 system_cleanup)
- 日志过期清理并入 `system_cleanup`(系统垃圾清理)每日任务;`of_database_auto_cleanup` 专用 schedule(id=102)与任务下线。
- 新增 `type=business` 配置(替换旧 `database_auto_cleanup_enabled` / `database_auto_cleanup_retention_days`):
- `log_retention_days_postgres`(默认 90)
- `log_retention_days_sqlite`(默认 90)
- `log_retention_days_clickhouse`(默认 90)
- `CleanupStore` 按当前生效库读取对应值执行:
- PG:分区 DROP(整月)+ 分批 DELETE(不满月);
- SQLite:分批 DELETE;
- CH:`ALTER TABLE ... MODIFY TTL toDateTime(...) + INTERVAL N DAY` + materialize(保留期由配置驱动,不再依赖 DDL 写死)。
- 旧 key `database_auto_cleanup_*` 由 goose 迁移删除,前端同步清理。
## 7. 迁移任务「切换日志数据库」
**元数据**:Asynq `openflare:log_db_switch`,管理类型 `of_log_db_switch`,名称「切换日志数据库」,参数 `target`(`postgres`/`sqlite`/`clickhouse`),`Retryable: true`。UI 按当前日志主库只展示合法目标(当前=CH → 「主库」;当前=主库 → 「ClickHouse」)。
**执行流程(worker 进程)**:
1. **校验**:`target == 当前主库` → 拒绝;`target=clickhouse` 但 CH 未启用 / `target=postgres` 但 `database.enabled=false` / `target=sqlite` 但 `database.enabled=true` → 拒绝。
2. **写冻结**:写 `log_db_migration = "migrating"`;先让 batchwriter 把在途批次 flush 完;此后 API 进程所有日志写入路径(`risk_control`、`chwriter` 队列、agent 上报落库)检查该 key → 返回明确错误(HTTP 503「日志数据库迁移中,暂不可写」),不排队积压。
3. **复制**:6 张原始日志表逐表、按 id 分批(每批 ~1000)读源 → 写目标(CH→主库用 GORM `CreateInBatches`;主库→CH 用原生 `PrepareBatch`);ID 原样保留;每表/每批 `task.AppendLog` 进度。
- **幂等前提**:开始复制前**清空目标库日志表**(任务参数「覆盖目标库已有日志」默认开启;目标库通常为空,仅「切回去」场景有旧数据)——保证失败重试可重跑不重复。
4. **翻转**:全部成功 → 更新 `log_database = target`、清除迁移标记 → `logstore` 缓存失效自动切到新实现 → 写入恢复(走新库)。
5. **失败**:返回错误触发 Asynq 重试;**失败时清除迁移标记**,写入继续走源库(不丢功能);重试时重新清空目标 + 复制。
**双进程一致性**:迁移标记与主库标记落在 `system_configs`(Redis 缓存,worker 更新后 API 进程自动失效重读)。
## 8. API 与前端
**后端**:
- `GET /api/v1/admin/status/log-database`(改造现有 `/clickhouse` 状态端点):返回当前日志主库、迁移状态(`idle`/`migrating`)、各库保留天数、当前合法迁移目标;CH 为主时附带现有 CH 运行指标,主库为主时附带 GORM 写入器状态。
- 任务「切换日志数据库」走现有任务管理通用派发 API(`RegisterTaskMeta` + Params),无需新派发接口;执行记录/进度复用任务框架。
- 系统配置:新增 3 个 `log_retention_days_*`(business)图形化 + 参数表可见;新增内部 `log_database`、`log_db_migration`(system、visibility=0、受保护);下线 `database_auto_cleanup_*`。
**前端**:
- 任务管理页:出现「切换日志数据库」,参数下拉只显示合法目标;页面展示当前日志主库与迁移状态。
- `/admin/settings` 业务配置:新增「日志保留时间」分组(PG/SQLite/CH 三个数字输入)。
- 状态/仪表盘:日志库状态卡片(当前库 + 迁移中提示)。
## 9. 测试与验证
- **import-lint 测试**:`internal/apps/**` 不得 import `internal/repository/analytics`、`internal/infra/persistence`(`batchwriter` 白名单除外),违规即失败。
- **logstore 单测**:GORM 实现用 SQLite 全量跑;PG 专属(分区 DROP 等)走既有集成测试路径;CH 实现复用现有 analyticsrepo 测试。
- **迁移任务测试**:目标/组合校验、批处理与 ID 保留、清空目标、翻转标记、失败清标记回退、冻结期写入拒绝——用 memory/sqlite 双端模拟,不依赖真实 CH。
- **清理测试**:`system_cleanup` 日志清理步骤(PG 分区 DROP / SQLite 分批 DELETE / CH TTL 修改)与保留配置读取。
- **迁移验证**:goose 空库 Up 全量(PG/SQLite/CH 三套)、`go test ./...`、`make swagger`(API 变更)、`make code-check`、`make format`。
## 10. 非目标(YAGNI)
- 不在 PG/SQLite 物理建聚合/物化视图表(查询实时聚合)。
- 不做 PG ↔ SQLite 日志互迁(非法组合,启动校验拒绝)。
- 迁移成功不自动删除源库数据(保留,后续提供手动清理入口)。
- 不引入 PG COPY 协议(GORM `CreateInBatches` 对低流量足够)。
- 不引入自定义跨进程迁移协议(`system_configs` + Redis 缓存即可)。
## 11. 里程碑建议(供实现计划分解)
1. **M1 抽象与改造**:`logstore` 接口 + PG/SQLite 实现 + `clickhouse_store` 包装 + import-lint 测试 + repository 委托改造 + apps 直连改造 + `log_database`/`log_db_migration` key 与启动校验。
2. **M2 表与清理**:goose 双方言建表迁移 + 保留配置 key + `system_cleanup` 日志清理步骤 + 下线 `of_database_auto_cleanup` 与旧配置。
3. **M3 迁移任务与展示**:迁移任务 Handler + 状态端点 + 任务管理页/业务配置前端 + 日志库状态卡片。
4. **M4 收尾**:全量验证(goose 三套、单测、`make code-check`/`swagger`/`format`)、文档同步(中文)、changelog `[Unreleased]`。
## 12. 实现归档说明(Task 18,2026-08-08)
- 设计稿第 4 节 provider 入口写作 `Open(ctx)`,实现命名为 `Active(ctx)`(按 `log_database` 解析并缓存,配置翻转后重建),另导出 `Build(ctx, database)` / `BuildForMigration(ctx, database)` 供迁移任务构造目标库 store;`ActiveDatabase(ctx)` 供状态端点。
- 设计稿第 4 节列出的 `CleanupStore` 接口未单独落地:清理实现为包级 `CleanupExpired(ctx)`(按当前激活库保留天数删除过期日志并预建 PG 分区),由 `system_cleanup` 每日任务调用。
- 设计稿第 4 节列举的 `tasks/database_cleanup.go` 已随 M2 下线(`of_database_auto_cleanup` 配置与前端 UI 一并移除),日志清理职责并入 `system_cleanup`。
- 迁移复制按 id 升序分页,`copyObservability` 以每批最后一条 id 作为下一批游标(修正计划中 `lastID += n` 的近似写法);失败回退由 `defer setMigrationFlag("")` 保证源库恢复可写,重试前先清空目标库保证幂等。
- 其余实现决策(`SetConfigReader` 注入、`ensureWritable` 统一冻结、解析 helper 迁至 `model/analytics` 等)见计划「自检记录」,与本文档一致。
+89 -167
View File
@@ -1017,7 +1017,7 @@
"SessionCookie": []
}
],
"description": "分页并按照用户、接口路径、时间范围等维度检索 ClickHouse 用户访问日志列表(需要管理员权限,ClickHouse 未启用时报错)",
"description": "分页并按照用户、接口路径、时间范围等维度检索用户访问日志列表(需要管理员权限,日志存储未启用时报错)",
"produces": [
"application/json"
],
@@ -1085,7 +1085,7 @@
}
},
"400": {
"description": "ClickHouse 未启用或参数错误",
"description": "日志存储未启用或参数错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
@@ -1112,7 +1112,7 @@
"SessionCookie": []
}
],
"description": "聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,ClickHouse 未启用时报错)",
"description": "聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,日志存储未启用时报错)",
"produces": [
"application/json"
],
@@ -1140,7 +1140,7 @@
}
},
"400": {
"description": "ClickHouse 未启用",
"description": "日志存储未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
@@ -1870,21 +1870,21 @@
}
}
},
"/api/v1/admin/status/clickhouse": {
"/api/v1/admin/status/log-database": {
"get": {
"security": [
{
"SessionCookie": []
}
],
"description": "返回 ClickHouse parts、mutation、async_insert 队列及进程内 batch writer 指标,需要管理员权限",
"description": "返回当前日志主库、迁移状态、各库保留天数与合法迁移目标,需要管理员权限",
"produces": [
"application/json"
],
"tags": [
"admin"
],
"summary": "获取 ClickHouse 运行指标",
"summary": "获取日志数据库状态",
"responses": {
"200": {
"description": "获取成功",
@@ -1897,19 +1897,13 @@
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/analytics.ClickHouseOperationalStats"
"$ref": "#/definitions/status.LogDatabaseStatus"
}
}
}
]
}
},
"400": {
"description": "ClickHouse 未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
@@ -2386,6 +2380,18 @@
"name": "task_type",
"in": "query"
},
{
"type": "string",
"description": "任务类型前缀筛选(与 task_type / task_types 互斥,精确类型优先)",
"name": "task_type_prefix",
"in": "query"
},
{
"type": "string",
"description": "逗号分隔的精确任务类型列表(IN 筛选,优先于前缀)",
"name": "task_types",
"in": "query"
},
{
"type": "integer",
"default": 1,
@@ -8298,80 +8304,6 @@
}
}
},
"/api/v1/d/option/database/cleanup": {
"post": {
"security": [
{
"SessionCookie": []
}
],
"description": "按目标与保留天数清理可观测性相关数据表,需要管理员权限",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"openflare-option"
],
"summary": "清理可观测性数据库",
"parameters": [
{
"description": "清理参数",
"name": "request",
"in": "body",
"schema": {
"$ref": "#/definitions/option.databaseCleanupInput"
}
}
],
"responses": {
"200": {
"description": "清理结果",
"schema": {
"allOf": [
{
"$ref": "#/definitions/response.Any"
},
{
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/option.databaseCleanupResult"
}
}
}
]
}
},
"400": {
"description": "参数错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"404": {
"description": "无权限或不存在",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"500": {
"description": "内部错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
}
}
}
},
"/api/v1/d/option/geoip/lookup": {
"post": {
"security": [
@@ -14899,33 +14831,26 @@
}
}
},
"analytics.ClickHouseOperationalStats": {
"analytics.BatchWriterStats": {
"type": "object",
"properties": {
"active_parts": {
"cap": {
"type": "integer"
},
"async_insert_bytes": {
"depth": {
"type": "integer"
},
"async_insert_queue": {
"drops": {
"type": "integer"
},
"batch_writers": {
"description": "BatchWriters reports in-process queue depth/drops/flush errors for CH writers.",
"type": "array",
"items": {
"$ref": "#/definitions/batchwriter.Stats"
}
"flush_errors": {
"type": "integer"
},
"database": {
"name": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
"running": {
"type": "boolean"
}
}
},
@@ -15017,29 +14942,6 @@
}
}
},
"batchwriter.Stats": {
"type": "object",
"properties": {
"cap": {
"type": "integer"
},
"depth": {
"type": "integer"
},
"drops": {
"type": "integer"
},
"flush_errors": {
"type": "integer"
},
"name": {
"type": "string"
},
"running": {
"type": "boolean"
}
}
},
"cache.updateCacheConfigRequest": {
"type": "object",
"required": [
@@ -15098,6 +15000,9 @@
"id": {
"type": "integer"
},
"zone_domain": {
"type": "string"
},
"zone_id": {
"type": "integer"
}
@@ -15819,6 +15724,36 @@
}
}
},
"github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats": {
"type": "object",
"properties": {
"active_parts": {
"type": "integer"
},
"async_insert_bytes": {
"type": "integer"
},
"async_insert_queue": {
"type": "integer"
},
"batch_writers": {
"description": "BatchWriters reports in-process queue depth/drops/flush errors for CH writers.",
"type": "array",
"items": {
"$ref": "#/definitions/analytics.BatchWriterStats"
}
},
"database": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
}
}
},
"github_com_Rain-kl_Wavelet_pkg_protocol.ActiveConfigMeta": {
"type": "object",
"properties": {
@@ -18474,46 +18409,6 @@
}
}
},
"option.databaseCleanupInput": {
"type": "object",
"properties": {
"retention_days": {
"type": "integer"
},
"target": {
"type": "string"
}
}
},
"option.databaseCleanupResult": {
"type": "object",
"properties": {
"cleanup_mode": {
"type": "string"
},
"delete_all": {
"type": "boolean"
},
"deleted_count": {
"type": "integer"
},
"eligible_count": {
"type": "integer"
},
"retention_days": {
"type": "integer"
},
"table_ttl_days": {
"type": "integer"
},
"target": {
"type": "string"
},
"target_label": {
"type": "string"
}
}
},
"option.geoIPLookupRequest": {
"type": "object",
"properties": {
@@ -19852,6 +19747,33 @@
}
}
},
"status.LogDatabaseStatus": {
"type": "object",
"properties": {
"active_database": {
"type": "string"
},
"available_targets": {
"type": "array",
"items": {
"type": "string"
}
},
"clickhouse": {
"$ref": "#/definitions/github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats"
},
"migration": {
"description": "idle | migrating",
"type": "string"
},
"retention_days": {
"type": "object",
"additionalProperties": {
"type": "integer"
}
}
}
},
"status.SystemStatusResponse": {
"type": "object",
"properties": {
+66 -112
View File
@@ -169,26 +169,20 @@ definitions:
$ref: '#/definitions/github_com_Rain-kl_Wavelet_pkg_protocol.WAFIPGroup'
type: array
type: object
analytics.ClickHouseOperationalStats:
analytics.BatchWriterStats:
properties:
active_parts:
cap:
type: integer
async_insert_bytes:
depth:
type: integer
async_insert_queue:
drops:
type: integer
batch_writers:
description: BatchWriters reports in-process queue depth/drops/flush errors
for CH writers.
items:
$ref: '#/definitions/batchwriter.Stats'
type: array
database:
flush_errors:
type: integer
name:
type: string
pending_mutations:
type: integer
total_rows:
type: integer
running:
type: boolean
type: object
apply_log.CleanupInput:
properties:
@@ -247,21 +241,6 @@ definitions:
is_active:
type: boolean
type: object
batchwriter.Stats:
properties:
cap:
type: integer
depth:
type: integer
drops:
type: integer
flush_errors:
type: integer
name:
type: string
running:
type: boolean
type: object
cache.updateCacheConfigRequest:
properties:
lru_enabled:
@@ -301,6 +280,8 @@ definitions:
type: string
id:
type: integer
zone_domain:
type: string
zone_id:
type: integer
type: object
@@ -774,6 +755,27 @@ definitions:
token:
type: string
type: object
github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats:
properties:
active_parts:
type: integer
async_insert_bytes:
type: integer
async_insert_queue:
type: integer
batch_writers:
description: BatchWriters reports in-process queue depth/drops/flush errors
for CH writers.
items:
$ref: '#/definitions/analytics.BatchWriterStats'
type: array
database:
type: string
pending_mutations:
type: integer
total_rows:
type: integer
type: object
github_com_Rain-kl_Wavelet_pkg_protocol.ActiveConfigMeta:
properties:
checksum:
@@ -2533,32 +2535,6 @@ definitions:
window_started_at:
type: string
type: object
option.databaseCleanupInput:
properties:
retention_days:
type: integer
target:
type: string
type: object
option.databaseCleanupResult:
properties:
cleanup_mode:
type: string
delete_all:
type: boolean
deleted_count:
type: integer
eligible_count:
type: integer
retention_days:
type: integer
table_ttl_days:
type: integer
target:
type: string
target_label:
type: string
type: object
option.geoIPLookupRequest:
properties:
ip:
@@ -3442,6 +3418,24 @@ definitions:
version:
type: string
type: object
status.LogDatabaseStatus:
properties:
active_database:
type: string
available_targets:
items:
type: string
type: array
clickhouse:
$ref: '#/definitions/github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats'
migration:
description: idle | migrating
type: string
retention_days:
additionalProperties:
type: integer
type: object
type: object
status.SystemStatusResponse:
properties:
alloc:
@@ -4935,7 +4929,7 @@ paths:
- admin
/api/v1/admin/logs/access:
get:
description: 分页并按照用户、接口路径、时间范围等维度检索 ClickHouse 用户访问日志列表(需要管理员权限,ClickHouse 未启用时报错)
description: 分页并按照用户、接口路径、时间范围等维度检索用户访问日志列表(需要管理员权限,日志存储未启用时报错)
parameters:
- default: 1
description: 页码
@@ -4976,7 +4970,7 @@ paths:
$ref: '#/definitions/logs.accessLogsResponse'
type: object
"400":
description: ClickHouse 未启用或参数错误
description: 日志存储未启用或参数错误
schema:
$ref: '#/definitions/response.Any'
"401":
@@ -4994,7 +4988,7 @@ paths:
- admin
/api/v1/admin/logs/analytics:
get:
description: 聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,ClickHouse 未启用时报错)
description: 聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,日志存储未启用时报错)
produces:
- application/json
responses:
@@ -5008,7 +5002,7 @@ paths:
$ref: '#/definitions/logs.logsAnalyticsResponse'
type: object
"400":
description: ClickHouse 未启用
description: 日志存储未启用
schema:
$ref: '#/definitions/response.Any'
"401":
@@ -5434,9 +5428,9 @@ paths:
summary: 获取系统状态信息
tags:
- admin
/api/v1/admin/status/clickhouse:
/api/v1/admin/status/log-database:
get:
description: 返回 ClickHouse parts、mutation、async_insert 队列及进程内 batch writer 指标,需要管理员权限
description: 返回当前日志主库、迁移状态、各库保留天数与合法迁移目标,需要管理员权限
produces:
- application/json
responses:
@@ -5447,12 +5441,8 @@ paths:
- $ref: '#/definitions/response.Any'
- properties:
data:
$ref: '#/definitions/analytics.ClickHouseOperationalStats'
$ref: '#/definitions/status.LogDatabaseStatus'
type: object
"400":
description: ClickHouse 未启用
schema:
$ref: '#/definitions/response.Any'
"401":
description: 未登录
schema:
@@ -5467,7 +5457,7 @@ paths:
$ref: '#/definitions/response.Any'
security:
- SessionCookie: []
summary: 获取 ClickHouse 运行指标
summary: 获取日志数据库状态
tags:
- admin
/api/v1/admin/system-configs:
@@ -5738,6 +5728,14 @@ paths:
in: query
name: task_type
type: string
- description: 任务类型前缀筛选(与 task_type / task_types 互斥,精确类型优先)
in: query
name: task_type_prefix
type: string
- description: 逗号分隔的精确任务类型列表(IN 筛选,优先于前缀)
in: query
name: task_types
type: string
- default: 1
description: 页码
in: query
@@ -9285,50 +9283,6 @@ paths:
summary: 列出 OpenFlare 配置项
tags:
- openflare-option
/api/v1/d/option/database/cleanup:
post:
consumes:
- application/json
description: 按目标与保留天数清理可观测性相关数据表,需要管理员权限
parameters:
- description: 清理参数
in: body
name: request
schema:
$ref: '#/definitions/option.databaseCleanupInput'
produces:
- application/json
responses:
"200":
description: 清理结果
schema:
allOf:
- $ref: '#/definitions/response.Any'
- properties:
data:
$ref: '#/definitions/option.databaseCleanupResult'
type: object
"400":
description: 参数错误
schema:
$ref: '#/definitions/response.Any'
"401":
description: 未登录
schema:
$ref: '#/definitions/response.Any'
"404":
description: 无权限或不存在
schema:
$ref: '#/definitions/response.Any'
"500":
description: 内部错误
schema:
$ref: '#/definitions/response.Any'
security:
- SessionCookie: []
summary: 清理可观测性数据库
tags:
- openflare-option
/api/v1/d/option/geoip/lookup:
post:
consumes:
@@ -18,8 +18,6 @@ export type OpenFlareOpsFields = {
uptime_kuma_retry: string;
uptime_kuma_retry_interval: string;
uptime_kuma_timeout: string;
database_auto_cleanup_enabled: boolean;
database_auto_cleanup_retention_days: string;
pages_max_package_size_mb: string;
pages_max_history_count: string;
};
@@ -42,8 +40,6 @@ export const defaultOpenFlareOpsFields: OpenFlareOpsFields = {
uptime_kuma_retry: '0',
uptime_kuma_retry_interval: '60',
uptime_kuma_timeout: '48',
database_auto_cleanup_enabled: false,
database_auto_cleanup_retention_days: '30',
pages_max_package_size_mb: '100',
pages_max_history_count: '20',
};
@@ -88,12 +84,6 @@ export function mapOptionsToOpsFields(
uptime_kuma_retry: optionMap.uptime_kuma_retry ?? '0',
uptime_kuma_retry_interval: optionMap.uptime_kuma_retry_interval ?? '60',
uptime_kuma_timeout: optionMap.uptime_kuma_timeout ?? '48',
database_auto_cleanup_enabled: toBoolean(
optionMap.database_auto_cleanup_enabled,
false,
),
database_auto_cleanup_retention_days:
optionMap.database_auto_cleanup_retention_days ?? '30',
pages_max_package_size_mb: optionMap.pages_max_package_size_mb ?? '100',
pages_max_history_count: optionMap.pages_max_history_count ?? '20',
};
@@ -165,16 +155,6 @@ export function validateUptimeKumaFields(fields: OpenFlareOpsFields) {
throw new Error('请求超时必须为正整数。');
}
export function validateDatabaseAutoCleanup(fields: OpenFlareOpsFields) {
const retentionDays = Number.parseInt(
fields.database_auto_cleanup_retention_days,
10,
);
if (Number.isNaN(retentionDays) || retentionDays < 1) {
throw new Error('自动清理保留天数至少为 1 天。');
}
}
export function agentOptionEntries(fields: OpenFlareOpsFields): OptionItem[] {
validateAgentFields(fields);
return [
@@ -220,22 +200,6 @@ export function uptimeKumaOptionEntries(
];
}
export function databaseAutoCleanupEntries(
fields: OpenFlareOpsFields,
): OptionItem[] {
validateDatabaseAutoCleanup(fields);
return [
{
key: 'database_auto_cleanup_enabled',
value: String(fields.database_auto_cleanup_enabled),
},
{
key: 'database_auto_cleanup_retention_days',
value: fields.database_auto_cleanup_retention_days,
},
];
}
export function validatePagesFields(fields: OpenFlareOpsFields) {
const packageSize = Number.parseInt(fields.pages_max_package_size_mb, 10);
const historyCount = Number.parseInt(fields.pages_max_history_count, 10);
@@ -11,20 +11,8 @@ import {
RotateCw,
Save,
Server,
Trash2,
} from 'lucide-react';
import { toast } from 'sonner';
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from '@/components/ui/alert-dialog';
import { Button } from '@/components/ui/button';
import {
Card,
@@ -46,7 +34,6 @@ import { Switch } from '@/components/ui/switch';
import { Textarea } from '@/components/ui/textarea';
import { ErrorInline } from '@/components/layout/error';
import { LoadingStateWithBorder } from '@/components/layout/loading';
import type { DatabaseCleanupTarget } from '@/lib/services/openflare';
import {
NodeService,
OptionService,
@@ -57,7 +44,6 @@ import {
import {
agentOptionEntries,
buildDiscoveryCommand,
databaseAutoCleanupEntries,
defaultOpenFlareOpsFields,
formatDurationLabel,
getBrowserOrigin,
@@ -72,31 +58,6 @@ import { UptimeKumaSiteSelectModal } from './uptimekuma-site-modal';
const optionsQueryKey = ['openflare', 'options'] as const;
const openflarePublicStatusQueryKey = ['openflare', 'public-status'] as const;
const cleanupTargets: Array<{
target: DatabaseCleanupTarget;
label: string;
description: string;
}> = [
{
target: 'node_access_logs',
label: '访问日志',
description:
'清理 node_access_logs,影响访问明细与 IP 汇总;表 TTL 为 90 天。',
},
{
target: 'node_metric_snapshots',
label: '性能快照',
description:
'清理 node_metric_snapshots,影响节点资源趋势;表 TTL 为 30 天。',
},
{
target: 'node_edge_health',
label: 'OpenResty 健康',
description:
'清理 node_edge_health(OpenResty 连接/健康快照);表 TTL 为 30 天。业务流量请清理访问日志。',
},
];
async function copyText(value: string) {
await navigator.clipboard.writeText(value);
}
@@ -109,11 +70,6 @@ export function OpenFlareOpsSettings() {
const [savingSection, setSavingSection] = useState<string | null>(null);
const [geoIPTestIP, setGeoIPTestIP] = useState('8.8.8.8');
const [uptimeKumaModalOpen, setUptimeKumaModalOpen] = useState(false);
const [cleanupTarget, setCleanupTarget] = useState<{
target: DatabaseCleanupTarget;
label: string;
} | null>(null);
const [cleanupRetentionDays, setCleanupRetentionDays] = useState('');
const optionsQuery = useQuery({
queryKey: optionsQueryKey,
@@ -194,25 +150,6 @@ export function OpenFlareOpsSettings() {
toast.error(error instanceof Error ? error.message : '同步失败'),
});
const cleanupMutation = useMutation({
mutationFn: (payload: {
target: DatabaseCleanupTarget;
retention_days?: number;
}) => OptionService.cleanupDatabase(payload),
onSuccess: (result) => {
setCleanupTarget(null);
setCleanupRetentionDays('');
toast.success(
result.delete_all
? `已清空${result.target_label},共删除 ${result.deleted_count} 条。`
: `已清理${result.target_label},共删除 ${result.deleted_count} 条。`,
);
},
onError: (error) => {
toast.error(error instanceof Error ? error.message : '清理失败');
},
});
const discoveryToken = bootstrapQuery.data?.discovery_token ?? '';
const discoveryCommand = useMemo(() => {
if (!fields.server_address || !discoveryToken) return '';
@@ -248,17 +185,6 @@ export function OpenFlareOpsSettings() {
}
};
const saveDatabaseAutoCleanup = () => {
try {
saveMutation.mutate({
section: 'database-auto',
entries: databaseAutoCleanupEntries(fields),
});
} catch (error) {
toast.error(error instanceof Error ? error.message : '参数校验失败');
}
};
const savePagesSettings = () => {
try {
saveMutation.mutate({
@@ -690,86 +616,6 @@ export function OpenFlareOpsSettings() {
</CardContent>
</Card>
<div className='grid gap-6 xl:grid-cols-2'>
<Card className='border-dashed shadow-none'>
<CardHeader className='flex flex-row items-center justify-between gap-4'>
<div>
<CardTitle className='text-base'>数据库自动清理</CardTitle>
<CardDescription>
每天凌晨 3 点物化 ClickHouse 表 TTL;访问日志至少保留 90
天,其它观测数据至少保留 30 天。
</CardDescription>
</div>
<Button
size='sm'
disabled={savingSection === 'database-auto'}
onClick={saveDatabaseAutoCleanup}
>
保存
</Button>
</CardHeader>
<CardContent className='space-y-4'>
<ToggleRow
label='启用每日自动清理'
checked={fields.database_auto_cleanup_enabled}
onChange={(value) =>
updateField('database_auto_cleanup_enabled', value)
}
/>
<div className='space-y-1.5'>
<FieldInput
label='期望保留天数'
value={fields.database_auto_cleanup_retention_days}
type='number'
onChange={(value) =>
updateField('database_auto_cleanup_retention_days', value)
}
/>
<p className='text-xs text-muted-foreground'>
小于表 TTL 时自动按下限执行:访问日志 90 天,其它观测数据 30
天。
</p>
</div>
</CardContent>
</Card>
<Card className='border-dashed shadow-none'>
<CardHeader>
<CardTitle className='text-base'>手动数据清理</CardTitle>
<CardDescription>
按表 TTL 清理;输入天数不能小于对应表
TTL,留空时将删除该类数据的全部历史记录。
</CardDescription>
</CardHeader>
<CardContent className='space-y-3'>
{cleanupTargets.map((item) => (
<div
key={item.target}
className='flex items-start justify-between gap-3 rounded-lg border border-dashed p-3'
>
<div>
<p className='text-sm font-medium'>{item.label}</p>
<p className='mt-1 text-xs text-muted-foreground'>
{item.description}
</p>
</div>
<Button
type='button'
variant='destructive'
size='sm'
onClick={() =>
setCleanupTarget({ target: item.target, label: item.label })
}
>
<Trash2 className='size-3.5 mr-1' />
清理
</Button>
</div>
))}
</CardContent>
</Card>
</div>
<UptimeKumaSiteSelectModal
open={uptimeKumaModalOpen}
selectedSites={
@@ -782,56 +628,6 @@ export function OpenFlareOpsSettings() {
updateField('uptime_kuma_selected_sites', sites.join(','))
}
/>
<AlertDialog
open={cleanupTarget !== null}
onOpenChange={(open) => !open && setCleanupTarget(null)}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>清理{cleanupTarget?.label}</AlertDialogTitle>
<AlertDialogDescription>
输入保留天数后会按该表 TTL 物化过期数据;小于表 TTL
的天数会被拒绝。留空则删除全部历史记录,操作不可恢复。
</AlertDialogDescription>
</AlertDialogHeader>
<FieldInput
label='保留天数'
value={cleanupRetentionDays}
type='number'
onChange={setCleanupRetentionDays}
placeholder='留空则全部删除'
/>
<AlertDialogFooter>
<AlertDialogCancel disabled={cleanupMutation.isPending}>
取消
</AlertDialogCancel>
<AlertDialogAction
disabled={cleanupMutation.isPending}
onClick={(event) => {
event.preventDefault();
if (!cleanupTarget) return;
const trimmed = cleanupRetentionDays.trim();
if (trimmed !== '') {
const retentionDays = Number.parseInt(trimmed, 10);
if (Number.isNaN(retentionDays) || retentionDays < 1) {
toast.error('保留天数至少为 1 天');
return;
}
cleanupMutation.mutate({
target: cleanupTarget.target,
retention_days: retentionDays,
});
return;
}
cleanupMutation.mutate({ target: cleanupTarget.target });
}}
>
{cleanupMutation.isPending ? '清理中...' : '确认清理'}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
);
}
@@ -1,6 +1,6 @@
'use client';
import { useCallback, useEffect, useState } from 'react';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { toast } from 'sonner';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
@@ -8,6 +8,13 @@ import { Label } from '@/components/ui/label';
import { Textarea } from '@/components/ui/textarea';
import { Switch } from '@/components/ui/switch';
import { Spinner } from '@/components/ui/spinner';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import {
Dialog,
DialogContent,
@@ -19,12 +26,17 @@ import {
import {
Calendar as CalendarIcon,
Clock,
Database,
Info,
Layers,
Play,
} from 'lucide-react';
import type { DispatchTaskRequest, TaskMeta } from '@/lib/services/admin';
import type {
DispatchTaskRequest,
LogDatabaseStatus,
TaskMeta,
} from '@/lib/services/admin';
import services from '@/lib/services';
import { buildTaskPayload } from '@/lib/task-param-utils';
import { ErrorInline } from '@/components/layout/error';
@@ -68,6 +80,18 @@ const TASK_CONFIGS: Record<
gradient:
'from-rose-500/10 via-rose-500/5 to-transparent border-rose-200/50 dark:border-rose-800/50 hover:border-rose-400 dark:hover:border-rose-500',
},
of_log_db_switch: {
icon: Database,
color: 'text-teal-600 dark:text-teal-400',
gradient:
'from-teal-500/10 via-teal-500/5 to-transparent border-teal-200/50 dark:border-teal-800/50 hover:border-teal-400 dark:hover:border-teal-500',
},
};
const LOG_DATABASE_LABELS: Record<string, string> = {
postgres: 'PostgreSQL(主库)',
sqlite: 'SQLite(主库)',
clickhouse: 'ClickHouse',
};
const DEFAULT_TASK_CONFIG = {
@@ -192,9 +216,38 @@ export function TaskManager() {
}
}, []);
const [logDbStatus, setLogDbStatus] = useState<LogDatabaseStatus | null>(
null,
);
// 日志库状态用于「切换日志数据库」卡片与 target 下拉;获取失败不阻塞任务列表。
const fetchLogDbStatus = useCallback(async () => {
try {
const data = await services.adminStatus.getLogDatabaseStatus();
setLogDbStatus(data);
} catch {
setLogDbStatus(null);
}
}, []);
useEffect(() => {
fetchTaskTypes();
}, [fetchTaskTypes]);
fetchLogDbStatus();
}, [fetchTaskTypes, fetchLogDbStatus]);
const availableLogDbTargets = useMemo(
() => logDbStatus?.available_targets ?? [],
[logDbStatus],
);
const retentionSummary = useMemo(() => {
const days = logDbStatus?.retention_days ?? {};
const parts: string[] = [];
if (days.postgres != null) parts.push(`PG ${days.postgres}`);
if (days.sqlite != null) parts.push(`SQLite ${days.sqlite}`);
if (days.clickhouse != null) parts.push(`CH ${days.clickhouse}`);
return parts.join(' / ');
}, [logDbStatus]);
useEffect(() => {
if (selectedTaskType) {
@@ -344,6 +397,45 @@ export function TaskManager() {
</div>
</div>
{task.type === 'of_log_db_switch' && logDbStatus && (
<div className='pt-3 mt-3 border-t border-border/50 space-y-1.5'>
<div className='flex items-center justify-between gap-2'>
<span className='text-[10px] text-muted-foreground shrink-0'>
日志主库
</span>
<span className='text-[10px] font-mono text-foreground truncate'>
{LOG_DATABASE_LABELS[logDbStatus.active_database] ||
logDbStatus.active_database}
</span>
</div>
<div className='flex items-center justify-between gap-2'>
<span className='text-[10px] text-muted-foreground shrink-0'>
保留天数
</span>
<span className='text-[10px] font-mono text-muted-foreground truncate'>
{retentionSummary || '-'}
</span>
</div>
<div className='flex items-center justify-between gap-2'>
<span className='text-[10px] text-muted-foreground shrink-0'>
迁移状态
</span>
<Badge
variant={
logDbStatus.migration === 'migrating'
? 'default'
: 'outline'
}
className='text-[10px] h-5 px-1.5'
>
{logDbStatus.migration === 'migrating'
? '迁移中'
: '空闲'}
</Badge>
</div>
</div>
)}
<div className='pt-4 mt-1'>
<Button
className='w-full h-7 text-xs'
@@ -436,70 +528,104 @@ export function TaskManager() {
}
return (
<div className='space-y-4'>
{targetTask.params.map((param) => (
<div key={param.name} className='grid gap-2'>
<Label
htmlFor={`param-${param.name}`}
className='flex items-center gap-1'
>
{param.label}
{param.required && (
<span className='text-destructive font-bold'>*</span>
)}
</Label>
{param.type === 'text' ? (
<Textarea
id={`param-${param.name}`}
placeholder={param.placeholder}
className='text-xs min-h-[80px]'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: e.target.value,
}))
}
/>
) : param.type === 'boolean' ? (
<div className='flex items-center gap-2 pt-1 h-9'>
<Switch
{targetTask.params.map((param) => {
const isSwitchTarget =
param.name === 'target' &&
getSelectedTaskMeta()?.type === 'of_log_db_switch';
return (
<div key={param.name} className='grid gap-2'>
<Label
htmlFor={`param-${param.name}`}
className='flex items-center gap-1'
>
{param.label}
{param.required && (
<span className='text-destructive font-bold'>
*
</span>
)}
</Label>
{param.type === 'text' ? (
<Textarea
id={`param-${param.name}`}
checked={paramValues[param.name] === 'true'}
onCheckedChange={(checked) =>
placeholder={param.placeholder}
className='text-xs min-h-[80px]'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: checked ? 'true' : 'false',
[param.name]: e.target.value,
}))
}
/>
<span className='text-xs text-muted-foreground'>
{paramValues[param.name] === 'true'
? '开启'
: '关闭'}
</span>
</div>
) : (
<Input
id={`param-${param.name}`}
type={param.type === 'number' ? 'number' : 'text'}
placeholder={param.placeholder}
className='text-xs'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: e.target.value,
}))
}
/>
)}
{param.description && (
<p className='text-[10px] text-muted-foreground'>
{param.description}
</p>
)}
</div>
))}
) : param.type === 'boolean' ? (
<div className='flex items-center gap-2 pt-1 h-9'>
<Switch
id={`param-${param.name}`}
checked={paramValues[param.name] === 'true'}
onCheckedChange={(checked) =>
setParamValues((prev) => ({
...prev,
[param.name]: checked ? 'true' : 'false',
}))
}
/>
<span className='text-xs text-muted-foreground'>
{paramValues[param.name] === 'true'
? '开启'
: '关闭'}
</span>
</div>
) : isSwitchTarget &&
availableLogDbTargets.length > 0 ? (
<Select
value={paramValues[param.name] || ''}
onValueChange={(value) =>
setParamValues((prev) => ({
...prev,
[param.name]: value,
}))
}
disabled={dispatching}
>
<SelectTrigger
id={`param-${param.name}`}
className='w-full text-xs'
size='sm'
>
<SelectValue placeholder='选择目标日志库...' />
</SelectTrigger>
<SelectContent>
{availableLogDbTargets.map((target) => (
<SelectItem key={target} value={target}>
{LOG_DATABASE_LABELS[target] || target}
</SelectItem>
))}
</SelectContent>
</Select>
) : (
<Input
id={`param-${param.name}`}
type={param.type === 'number' ? 'number' : 'text'}
placeholder={param.placeholder}
className='text-xs'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: e.target.value,
}))
}
/>
)}
{param.description && (
<p className='text-[10px] text-muted-foreground'>
{param.description}
</p>
)}
</div>
);
})}
</div>
);
})()}
@@ -1,13 +1,13 @@
'use client';
import { useMemo } from 'react';
import { useEffect, useMemo, useState } from 'react';
import {
useMutation,
useQuery,
useQueryClient,
type UseQueryResult,
} from '@tanstack/react-query';
import { KeyRound, ShieldAlert, X } from 'lucide-react';
import { Database, KeyRound, Save, ShieldAlert, X } from 'lucide-react';
import {
Card,
CardContent,
@@ -16,6 +16,10 @@ import {
CardTitle,
} from '@/components/ui/card';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Spinner } from '@/components/ui/spinner';
import { Label } from '@/components/ui/label';
import {
Select,
SelectContent,
@@ -28,6 +32,24 @@ import type { SystemConfig } from '@/lib/services/admin';
import { TemplatesManager } from './templates';
import { toast } from 'sonner';
const LOG_RETENTION_FIELDS = [
{
key: 'log_retention_days_postgres',
label: 'PostgreSQL',
description: '访问日志与可观测指标统一保留天数',
},
{
key: 'log_retention_days_sqlite',
label: 'SQLite',
description: 'SQLite 日志保留天数',
},
{
key: 'log_retention_days_clickhouse',
label: 'ClickHouse',
description: 'ClickHouse 日志保留天数',
},
] as const;
interface OperationTabProps {
configs: Record<string, SystemConfig>;
systemConfigsQuery: UseQueryResult<SystemConfig[], Error>;
@@ -44,6 +66,65 @@ export function OperationTab({
queryFn: () => services.adminSystemConfig.listUploadTypes(),
});
const businessConfigsQuery = useQuery({
queryKey: ['admin', 'system-configs', 'business'],
queryFn: () => services.adminSystemConfig.listSystemConfigs('business'),
});
const businessConfigs = useMemo(() => {
return (businessConfigsQuery.data ?? []).reduce<
Record<string, SystemConfig>
>((acc, config) => {
acc[config.key] = config;
return acc;
}, {});
}, [businessConfigsQuery.data]);
const [retentionValues, setRetentionValues] = useState<
Record<string, string>
>({});
useEffect(() => {
if (!businessConfigsQuery.data) return;
setRetentionValues((prev) => {
const next: Record<string, string> = {};
LOG_RETENTION_FIELDS.forEach((field) => {
const config = businessConfigs[field.key];
next[field.key] = config?.value || prev[field.key] || '90';
});
return next;
});
}, [businessConfigsQuery.data, businessConfigs]);
const updateRetentionMutation = useMutation({
mutationFn: async (values: Record<string, string>) => {
for (const field of LOG_RETENTION_FIELDS) {
const raw = (values[field.key] ?? '').trim();
const num = Number(raw);
if (!raw || !Number.isInteger(num) || num < 1) {
throw new Error(`${field.label}必须为大于等于 1 的整数`);
}
const config = businessConfigs[field.key];
if (!config) {
throw new Error(`缺少配置项: ${field.key}`);
}
await services.adminSystemConfig.updateSystemConfig(field.key, {
value: String(num),
description: config.description,
});
}
},
onSuccess: async () => {
await queryClient.invalidateQueries({
queryKey: ['admin', 'system-configs'],
});
toast.success('日志保留时间已更新');
},
onError: (error: Error) => {
toast.error(error.message || '更新日志保留时间失败');
},
});
const updateWhitelistMutation = useMutation({
mutationFn: async (newValue: string) => {
const config = configs['file_access_whitelist'];
@@ -208,6 +289,77 @@ export function OperationTab({
</CardContent>
</Card>
{/* 日志保留时间设置 */}
<Card className='border border-dashed shadow-sm'>
<CardHeader className='border-b border-dashed pb-4'>
<div className='flex items-center gap-2'>
<div className='p-1.5 rounded-lg bg-primary/10 text-primary'>
<Database className='size-4' />
</div>
<div>
<CardTitle className='text-base font-semibold'>
日志保留时间
</CardTitle>
<CardDescription className='text-xs'>
配置各日志数据库的日志保留天数,切换日志数据库后自动按对应配置清理过期日志。
</CardDescription>
</div>
</div>
</CardHeader>
<CardContent className='pt-6'>
<div className='grid gap-4 sm:grid-cols-3'>
{LOG_RETENTION_FIELDS.map((field) => (
<div key={field.key} className='grid gap-2'>
<Label htmlFor={field.key}>{field.label}</Label>
<div className='flex items-center gap-2'>
<Input
id={field.key}
type='number'
min={1}
className='text-xs'
value={retentionValues[field.key] ?? ''}
disabled={
updateRetentionMutation.isPending ||
businessConfigsQuery.isPending
}
onChange={(e) =>
setRetentionValues((prev) => ({
...prev,
[field.key]: e.target.value,
}))
}
/>
<span className='text-xs text-muted-foreground whitespace-nowrap'>
天
</span>
</div>
<p className='text-[10px] text-muted-foreground'>
{field.description}
</p>
</div>
))}
</div>
<div className='mt-4 flex justify-end'>
<Button
type='button'
size='sm'
onClick={() => updateRetentionMutation.mutate(retentionValues)}
disabled={
updateRetentionMutation.isPending ||
businessConfigsQuery.isPending
}
>
{updateRetentionMutation.isPending ? (
<Spinner className='size-3' />
) : (
<Save className='size-3' />
)}
保存
</Button>
</div>
</CardContent>
</Card>
{/* 通知模板管理 */}
<TemplatesManager />
</div>
+1
View File
@@ -29,6 +29,7 @@ export type {
UpdateUserStatusRequest,
UpdateUserRequest,
SystemStatus,
LogDatabaseStatus,
AppUpdateStatus,
Schedule,
CreateScheduleRequest,
@@ -1,5 +1,5 @@
import { BaseService } from '@/lib/services/core';
import type { AppUpdateStatus, SystemStatus } from './types';
import type { AppUpdateStatus, LogDatabaseStatus, SystemStatus } from './types';
export class AdminStatusService extends BaseService {
protected static readonly basePath = '/api/v1/admin';
@@ -8,6 +8,10 @@ export class AdminStatusService extends BaseService {
return this.get<SystemStatus>('/status');
}
static async getLogDatabaseStatus(): Promise<LogDatabaseStatus> {
return this.get<LogDatabaseStatus>('/status/log-database');
}
static async getUpdateStatus(): Promise<AppUpdateStatus> {
return this.get<AppUpdateStatus>('/update');
}
+16
View File
@@ -405,6 +405,22 @@ export interface ToggleAuthSourceRequest {
is_active: boolean;
}
/**
* 日志数据库状态
*/
export interface LogDatabaseStatus {
/** 当前日志主库:postgres | sqlite | clickhouse */
active_database: string;
/** 迁移状态:idle | migrating */
migration: string;
/** 各日志库保留天数 */
retention_days: Record<string, number>;
/** 当前主库的合法迁移目标 */
available_targets: string[];
/** ClickHouse 运行指标(仅主库为 ClickHouse 时) */
clickhouse?: unknown;
}
/**
* 系统状态信息
*/
+1 -1
View File
@@ -158,6 +158,7 @@ export type {
CreateUserRequest,
UpdateUserRequest,
SystemStatus,
LogDatabaseStatus,
AppUpdateStatus,
Schedule,
CreateScheduleRequest,
@@ -260,6 +261,5 @@ export type {
AccessLogOverview,
OptionItem,
GeoIPLookupResult,
DatabaseCleanupResult,
OpenFlarePublicStatus,
} from './openflare';
-3
View File
@@ -104,9 +104,6 @@ export type {
FoldedAccessLogList,
OptionItem,
GeoIPLookupResult,
DatabaseCleanupPayload,
DatabaseCleanupResult,
DatabaseCleanupTarget,
OpenFlarePublicStatus,
OriginDetail,
OriginItem,
@@ -1,7 +1,5 @@
import { OpenFlareBaseService } from './base.service';
import type {
DatabaseCleanupPayload,
DatabaseCleanupResult,
GeoIPLookupResult,
OptionBatchPayload,
OptionItem,
@@ -26,10 +24,4 @@ export class OptionService extends OpenFlareBaseService {
static lookupGeoIP(provider: string, ip: string): Promise<GeoIPLookupResult> {
return this.post<GeoIPLookupResult>('/geoip/lookup', { provider, ip });
}
static cleanupDatabase(
payload: DatabaseCleanupPayload,
): Promise<DatabaseCleanupResult> {
return this.post<DatabaseCleanupResult>('/database/cleanup', payload);
}
}
-21
View File
@@ -866,27 +866,6 @@ export interface GeoIPLookupResult {
longitude?: number | null;
}
export type DatabaseCleanupTarget =
| 'node_access_logs'
| 'node_metric_snapshots'
| 'node_edge_health'
| 'node_obs_frps'
| 'node_obs_frpc';
export interface DatabaseCleanupPayload {
target: DatabaseCleanupTarget;
retention_days?: number;
}
export interface DatabaseCleanupResult {
target: DatabaseCleanupTarget;
target_label: string;
deleted_count: number;
delete_all: boolean;
retention_days?: number;
cutoff?: string;
}
export interface OpenFlarePublicStatus {
version: string;
start_time: number;
+20 -17
View File
@@ -14,10 +14,9 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/apps/admin"
"github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/repository"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
@@ -158,8 +157,8 @@ type accessLogsResponse struct {
List []accessLogItem `json:"list"`
}
func buildAccessLogFilter(ctx context.Context, c *gin.Context) (analyticsrepo.AccessLogFilter, error) {
filter := analyticsrepo.AccessLogFilter{}
func buildAccessLogFilter(ctx context.Context, c *gin.Context) (analyticsmodel.AccessLogFilter, error) {
filter := analyticsmodel.AccessLogFilter{}
username := c.Query("username")
if username != "" {
@@ -227,7 +226,7 @@ func enrichAccessLogsWithUsers(ctx context.Context, list []accessLogItem) {
// GetAccessLogs 获取 ClickHouse 异步采集的访问日志
// @Summary 获取用户访问日志
// @Description 分页并按照用户、接口路径、时间范围等维度检索 ClickHouse 用户访问日志列表(需要管理员权限,ClickHouse 未启用时报错)
// @Description 分页并按照用户、接口路径、时间范围等维度检索用户访问日志列表(需要管理员权限,日志存储未启用时报错)
// @Tags admin
// @Produce json
// @Security SessionCookie
@@ -238,14 +237,16 @@ func enrichAccessLogsWithUsers(ctx context.Context, list []accessLogItem) {
// @Param start_time query string false "起始时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
// @Param end_time query string false "结束时间(RFC3339 或 YYYY-MM-DD HH:MM:SS)"
// @Success 200 {object} response.Any{data=logs.accessLogsResponse} "访问日志列表"
// @Failure 400 {object} response.Any "ClickHouse 未启用或参数错误"
// @Failure 400 {object} response.Any "日志存储未启用或参数错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 403 {object} response.Any "无管理员权限"
// @Router /api/v1/admin/logs/access [get]
func GetAccessLogs(c *gin.Context) {
ctx := c.Request.Context()
if !config.Config.ClickHouse.Enabled || !db.ChConnReady() {
response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用,无法检索访问日志")
store, err := logstore.Active(ctx)
if err != nil {
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法检索访问日志")
return
}
@@ -271,7 +272,7 @@ func GetAccessLogs(c *gin.Context) {
return
}
logs, total, err := analyticsrepo.ListAccessLogs(ctx, filter, page, pageSize)
logs, total, err := store.UserAccessLogs.List(ctx, filter, page, pageSize)
if err != nil {
response.AbortWithError(c, http.StatusInternalServerError, err.Error())
return
@@ -333,25 +334,27 @@ type logsAnalyticsResponse struct {
// GetLogsAnalytics 获取 ClickHouse 访问日志图表聚合指标
// @Summary 获取访问日志分析数据
// @Description 聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,ClickHouse 未启用时报错)
// @Description 聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行(需要管理员权限,日志存储未启用时报错)
// @Tags admin
// @Produce json
// @Security SessionCookie
// @Success 200 {object} response.Any{data=logs.logsAnalyticsResponse} "分析统计数据"
// @Failure 400 {object} response.Any "ClickHouse 未启用"
// @Failure 400 {object} response.Any "日志存储未启用"
// @Failure 401 {object} response.Any "未登录"
// @Failure 403 {object} response.Any "无管理员权限"
// @Router /api/v1/admin/logs/analytics [get]
func GetLogsAnalytics(c *gin.Context) {
ctx := c.Request.Context()
if !config.Config.ClickHouse.Enabled || !db.ChConnReady() {
response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用,无法获取分析数据")
store, err := logstore.Active(ctx)
if err != nil {
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
response.AbortWithError(c, http.StatusBadRequest, "日志存储未启用,无法获取分析数据")
return
}
startTime := time.Now().AddDate(0, 0, -(analyticsDays - 1)).Truncate(hoursInDay * time.Hour)
trendPoints, err := analyticsrepo.GetDailyTrend(ctx, analyticsDays)
trendPoints, err := store.UserAccessLogs.GetDailyTrend(ctx, analyticsDays)
if err != nil {
response.AbortWithError(c, http.StatusInternalServerError, "查询访问趋势失败: "+err.Error())
return
@@ -364,7 +367,7 @@ func GetLogsAnalytics(c *gin.Context) {
}
}
browserPoints, err := analyticsrepo.GetBrowserDistribution(ctx, startTime)
browserPoints, err := store.UserAccessLogs.GetBrowserDistribution(ctx, startTime)
if err != nil {
response.AbortWithError(c, http.StatusInternalServerError, "查询浏览器分布失败: "+err.Error())
return
@@ -377,7 +380,7 @@ func GetLogsAnalytics(c *gin.Context) {
}
}
topUserPoints, err := analyticsrepo.GetTopActiveUsers(ctx, startTime, topActiveLimit)
topUserPoints, err := store.UserAccessLogs.GetTopActiveUsers(ctx, startTime, topActiveLimit)
if err != nil {
response.AbortWithError(c, http.StatusInternalServerError, "查询活跃用户失败: "+err.Error())
return
+99 -15
View File
@@ -4,43 +4,127 @@
package status
import (
"context"
"errors"
"net/http"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
"github.com/Rain-kl/Wavelet/internal/apps/risk_control"
"github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
// GetClickHouseStatus returns ClickHouse operational metrics for administrators.
// @Summary 获取 ClickHouse 运行指标
// @Description 返回 ClickHouse parts、mutation、async_insert 队列及进程内 batch writer 指标,需要管理员权限
// 日志库名取值(与 model 配置值、logstore provider 分支保持一致)。
const (
logDBNamePostgres = "postgres"
logDBNameSQLite = "sqlite"
logDBNameClickHouse = "clickhouse"
)
// defaultLogRetentionDays 日志保留天数配置缺失时的兜底值(与 seed 默认一致)。
const defaultLogRetentionDays = 90
// LogDatabaseStatus 日志库状态。
type LogDatabaseStatus struct {
ActiveDatabase string `json:"active_database"`
Migration string `json:"migration"` // idle | migrating
RetentionDays map[string]int `json:"retention_days"`
AvailableTargets []string `json:"available_targets"`
ClickHouse *analyticsmodel.ClickHouseOperationalStats `json:"clickhouse,omitempty"`
}
// GetLogDatabaseStatus 返回当前日志库状态。
// @Summary 获取日志数据库状态
// @Description 返回当前日志主库、迁移状态、各库保留天数与合法迁移目标,需要管理员权限
// @Tags admin
// @Produce json
// @Security SessionCookie
// @Success 200 {object} response.Any{data=analyticsrepo.ClickHouseOperationalStats} "获取成功"
// @Failure 400 {object} response.Any "ClickHouse 未启用"
// @Success 200 {object} response.Any{data=status.LogDatabaseStatus} "获取成功"
// @Failure 401 {object} response.Any "未登录"
// @Failure 403 {object} response.Any "无管理员权限"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/admin/status/clickhouse [get]
func GetClickHouseStatus(c *gin.Context) {
if !config.Config.ClickHouse.Enabled || !db.ChConnReady() {
response.AbortWithError(c, http.StatusBadRequest, "ClickHouse 存储服务未启用")
// @Router /api/v1/admin/status/log-database [get]
func GetLogDatabaseStatus(c *gin.Context) {
ctx := c.Request.Context()
store, err := logstore.Active(ctx)
if err != nil {
logger.ErrorF(ctx, "获取日志存储实例失败: %v", err)
response.AbortInternal(c, "日志存储初始化失败")
return
}
stats, err := analyticsrepo.GetClickHouseOperationalStats(c.Request.Context())
// 分支判定复用同一 store 实例的 ActiveDatabase,避免与 Active 解析之间出现 TOCTOU。
activeDB, err := store.Status.ActiveDatabase(ctx)
if err != nil {
response.AbortInternal(c, "获取 ClickHouse 运行指标失败")
logger.ErrorF(ctx, "获取日志库状态失败: %v", err)
response.AbortInternal(c, "获取日志库状态失败")
return
}
stats.BatchWriters = collectBatchWriterStats()
c.JSON(http.StatusOK, response.OK(stats))
migration := "idle"
if logstore.Migrating(ctx) {
migration = "migrating"
}
out := LogDatabaseStatus{
ActiveDatabase: activeDB,
Migration: migration,
RetentionDays: map[string]int{
logDBNamePostgres: retentionOr(ctx, model.ConfigKeyLogRetentionDaysPostgres),
logDBNameSQLite: retentionOr(ctx, model.ConfigKeyLogRetentionDaysSQLite),
logDBNameClickHouse: retentionOr(ctx, model.ConfigKeyLogRetentionDaysClickHouse),
},
AvailableTargets: availableTargets(activeDB),
}
if activeDB == logDBNameClickHouse {
stats, err := store.Status.ClickHouseOperationalStats(ctx)
if err != nil {
logger.ErrorF(ctx, "获取 ClickHouse 运行指标失败: %v", err)
} else {
stats.BatchWriters = collectBatchWriterStats()
out.ClickHouse = stats
}
}
c.JSON(http.StatusOK, response.OK(out))
}
// retentionOr 读取保留天数配置,缺失或非法时返回默认值。
func retentionOr(ctx context.Context, key string) int {
v, err := repository.GetIntByKey(ctx, key)
if err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
logger.ErrorF(ctx, "读取日志保留天数配置失败 key=%s: %v", key, err)
}
return defaultLogRetentionDays
}
if v < 1 {
return defaultLogRetentionDays
}
return v
}
// availableTargets 返回当前日志主库的合法迁移目标(复用调用方已解析的 active):
// 当前为 clickhouse 时目标为主库(postgres/sqlite 按启动配置);当前为主库时目标为 clickhouse(仅 CH 启用时)。
func availableTargets(active string) []string {
if active == logDBNameClickHouse {
if config.Config.Database.Enabled {
return []string{logDBNamePostgres}
}
return []string{logDBNameSQLite}
}
if config.Config.ClickHouse.Enabled {
return []string{logDBNameClickHouse}
}
return []string{}
}
func collectBatchWriterStats() []batchwriter.Stats {
@@ -0,0 +1,117 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package status
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/gin-gonic/gin"
)
// restoreConfig 恢复测试中临时修改的全局配置。
func restoreConfig(t *testing.T) {
t.Helper()
dbEnabled := config.Config.Database.Enabled
chEnabled := config.Config.ClickHouse.Enabled
t.Cleanup(func() {
config.Config.Database.Enabled = dbEnabled
config.Config.ClickHouse.Enabled = chEnabled
})
}
func TestAvailableTargets(t *testing.T) {
restoreConfig(t)
logstore.ResetForTest()
t.Cleanup(logstore.ResetForTest)
// 当前 clickhouse → 主库(postgres/sqlite 按启动配置)。
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == "log_database" {
return logDBNameClickHouse, nil
}
return "", nil
})
config.Config.Database.Enabled = true
config.Config.ClickHouse.Enabled = true
if got := availableTargets(logDBNameClickHouse); !reflect.DeepEqual(got, []string{logDBNamePostgres}) {
t.Fatalf("clickhouse active + postgres main: got %v, want [postgres]", got)
}
config.Config.Database.Enabled = false
if got := availableTargets(logDBNameClickHouse); !reflect.DeepEqual(got, []string{logDBNameSQLite}) {
t.Fatalf("clickhouse active + sqlite main: got %v, want [sqlite]", got)
}
// 当前主库 → clickhouse(CH 启用时)。
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == "log_database" {
return logDBNamePostgres, nil
}
return "", nil
})
config.Config.ClickHouse.Enabled = true
if got := availableTargets(logDBNamePostgres); !reflect.DeepEqual(got, []string{logDBNameClickHouse}) {
t.Fatalf("postgres active: got %v, want [clickhouse]", got)
}
// CH 禁用时排除 clickhouse。
config.Config.ClickHouse.Enabled = false
if got := availableTargets(logDBNamePostgres); len(got) != 0 {
t.Fatalf("CH disabled: got %v, want empty", got)
}
}
// TestGetLogDatabaseStatusSmoke 覆盖 handler 的 CH 激活分支(无需 DB/CH 连接)。
func TestGetLogDatabaseStatusSmoke(t *testing.T) {
restoreConfig(t)
config.Config.Database.Enabled = false
config.Config.ClickHouse.Enabled = true
logstore.ResetForTest()
t.Cleanup(logstore.ResetForTest)
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == "log_database" {
return logDBNameClickHouse, nil
}
return "", nil
})
gin.SetMode(gin.TestMode)
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/admin/status/log-database", nil)
GetLogDatabaseStatus(c)
if w.Code != http.StatusOK {
t.Fatalf("status code = %d, want 200; body=%s", w.Code, w.Body.String())
}
var resp struct {
Data LogDatabaseStatus `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("unmarshal body: %v", err)
}
if resp.Data.ActiveDatabase != logDBNameClickHouse {
t.Fatalf("active_database = %q, want clickhouse", resp.Data.ActiveDatabase)
}
if resp.Data.Migration != "idle" {
t.Fatalf("migration = %q, want idle", resp.Data.Migration)
}
for _, key := range []string{logDBNamePostgres, logDBNameSQLite, logDBNameClickHouse} {
if got := resp.Data.RetentionDays[key]; got != defaultLogRetentionDays {
t.Fatalf("retention_days[%s] = %d, want default %d", key, got, defaultLogRetentionDays)
}
}
if got := resp.Data.AvailableTargets; !reflect.DeepEqual(got, []string{logDBNameSQLite}) {
t.Fatalf("available_targets = %v, want [sqlite] (test main DB disabled)", got)
}
}
@@ -17,6 +17,11 @@ import (
)
func createSystemConfig(ctx context.Context, req CreateSystemConfigRequest) error {
// 防御:受保护 key(log_database / log_db_migration)仅允许内部写入,Handler 已拦截。
if isProtectedConfigKey(req.Key) {
return errors.New(protectedConfigKeyMessage)
}
exists, err := repository.SystemConfigExists(ctx, req.Key)
if err != nil {
return err
@@ -27,6 +27,17 @@ import (
const maskedConfigValue = "******"
// protectedConfigKeyMessage 命中受保护 key 时返回给管理员的业务错误文案。
const protectedConfigKeyMessage = "该配置项由系统任务管理,禁止手动修改"
// protectedConfigKeys 仅允许内部(迁移任务/bootstrap)写入的 key。
var protectedConfigKeys = map[string]bool{
model.ConfigKeyLogDatabase: true,
model.ConfigKeyLogDBMigration: true,
}
func isProtectedConfigKey(key string) bool { return protectedConfigKeys[key] }
// CreateSystemConfigRequest 创建系统配置请求
type CreateSystemConfigRequest struct {
Key string `json:"key" binding:"required,max=64"`
@@ -64,11 +75,21 @@ func CreateSystemConfig(c *gin.Context) {
return
}
// 与 PUT 路径一致:log_database / log_db_migration 仅允许内部(迁移任务/bootstrap)写入。
if isProtectedConfigKey(req.Key) {
response.AbortBadRequest(c, protectedConfigKeyMessage)
return
}
if err := createSystemConfig(c.Request.Context(), req); err != nil {
if err.Error() == ConfigKeyExists {
response.AbortBadRequest(c, ConfigKeyExists)
return
}
if err.Error() == protectedConfigKeyMessage {
response.AbortBadRequest(c, protectedConfigKeyMessage)
return
}
response.AbortInternal(c, err.Error())
return
}
@@ -155,6 +176,10 @@ func UpdateSystemConfig(c *gin.Context) {
}
key := c.Param("key")
if isProtectedConfigKey(key) {
response.AbortBadRequest(c, protectedConfigKeyMessage)
return
}
if err := updateSystemConfig(c.Request.Context(), key, req); err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
response.AbortNotFound(c, SystemConfigNotFound)
@@ -27,7 +27,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/shared/response"
)
const expectedDefaultConfigsCount = 34
const expectedDefaultConfigsCount = 37
func setupTestRouter(authUser *model.User) *gin.Engine {
r := testhelper.NewTestGinEngine()
@@ -168,8 +168,8 @@ func TestListSystemConfigs(t *testing.T) {
var configs []model.SystemConfig
_ = json.Unmarshal(dataBytes, &configs)
if len(configs) != 5 {
t.Errorf("expected 5 business configs, got %d: %v", len(configs), configs)
if len(configs) != 8 {
t.Errorf("expected 8 business configs, got %d: %v", len(configs), configs)
}
})
}
+21 -62
View File
@@ -21,11 +21,6 @@ const (
// TaskTypeSSLRenew is the admin task type for SSL renewal.
TaskTypeSSLRenew = "of_ssl_renew"
// DatabaseAutoCleanupTask prunes observability tables by retention policy.
DatabaseAutoCleanupTask = "openflare:database_auto_cleanup"
// TaskTypeDatabaseAutoCleanup is the admin task type for observability cleanup.
TaskTypeDatabaseAutoCleanup = "of_database_auto_cleanup"
// WAFIPGroupSyncTask syncs due automatic/subscription WAF IP groups.
WAFIPGroupSyncTask = "openflare:waf_ip_group_sync"
// TaskTypeWAFIPGroupSync is the admin task type for WAF IP group sync.
@@ -35,6 +30,11 @@ const (
UptimeKumaSyncTask = "openflare:uptime_kuma_sync"
// TaskTypeUptimeKumaSync is the admin task type for Uptime Kuma sync.
TaskTypeUptimeKumaSync = "of_uptime_kuma_sync"
// LogDBSwitchTask 切换日志数据库任务标识。
LogDBSwitchTask = "openflare:log_db_switch"
// TaskTypeLogDBSwitch is the admin task type for log database switch.
TaskTypeLogDBSwitch = "of_log_db_switch"
)
var (
@@ -54,18 +54,6 @@ var SSLRenewMeta = task.TaskMeta{
Retryable: true,
}
// DatabaseAutoCleanupMeta describes the observability auto-cleanup task.
var DatabaseAutoCleanupMeta = task.TaskMeta{
Type: TaskTypeDatabaseAutoCleanup,
AsynqTask: DatabaseAutoCleanupTask,
Name: "OpenFlare 可观测数据自动清理",
Description: "按保留天数清理访问日志、性能快照与请求聚合数据",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
}
// WAFIPGroupSyncMeta describes the WAF IP group sync task.
var WAFIPGroupSyncMeta = task.TaskMeta{
Type: TaskTypeWAFIPGroupSync,
@@ -90,6 +78,22 @@ var UptimeKumaSyncMeta = task.TaskMeta{
Retryable: true,
}
// LogDBSwitchMeta 描述切换日志数据库任务。
var LogDBSwitchMeta = task.TaskMeta{
Type: TaskTypeLogDBSwitch,
AsynqTask: LogDBSwitchTask,
Name: "切换日志数据库",
Description: "复制迁移日志数据并在成功后切换日志主库(期间禁止日志写入)",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
Params: []task.TaskParam{
{Name: "target", Label: "目标日志库", Type: "string", Required: true,
Placeholder: "postgres|sqlite|clickhouse", Description: "迁移目标:postgres(主库为 PG 时)、sqlite(主库为 SQLite 时)或 clickhouse"},
},
}
// SSLRenewHandler renews due TLS certificates.
type SSLRenewHandler struct{}
@@ -105,51 +109,6 @@ func (h *SSLRenewHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResu
return &task.TaskResult{Message: msg}, nil
}
// DatabaseAutoCleanupHandler prunes observability data when auto-cleanup is enabled.
type DatabaseAutoCleanupHandler struct{}
// Execute runs retention-based cleanup for all observability targets.
func (h *DatabaseAutoCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
// 从 SystemConfig 读取自动清理配置
enabled, _ := repository.GetBoolByKey(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled)
if !enabled {
msg := "自动清理未启用,跳过执行"
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
retentionDays, _ := repository.GetIntByKey(ctx, model.ConfigKeyDatabaseAutoCleanupRetentionDays)
if retentionDays <= 0 {
retentionDays = 30 // 默认保留 30 天
}
task.AppendLog(ctx, "开始执行可观测数据自动清理,保留天数=%d", retentionDays)
summary, err := tasks.RunDatabaseAutoCleanupOnce(ctx, time.Now())
if err != nil {
task.AppendLog(ctx, "可观测数据自动清理失败: %v", err)
return nil, err
}
if summary == nil {
msg := "自动清理未启用,跳过执行"
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
var totalDeleted int64
for _, item := range summary.Results {
totalDeleted += item.DeletedCount
task.AppendLog(ctx, "清理 %s:删除 %d 条", item.TargetLabel, item.DeletedCount)
}
msg := fmt.Sprintf(
"可观测数据自动清理完成,保留 %d 天,共删除 %d 条",
summary.RetentionDays,
totalDeleted,
)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
// WAFIPGroupSyncHandler syncs due WAF IP groups to agents.
type WAFIPGroupSyncHandler struct{}
@@ -6,7 +6,6 @@ package openflare
import (
"context"
"testing"
"time"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -17,63 +16,6 @@ import (
"gorm.io/gorm"
)
func TestDatabaseAutoCleanupHandlerSkipsWhenDisabled(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, sqliteDB.AutoMigrate(&model.SystemConfig{}))
db.SetDB(sqliteDB)
t.Cleanup(func() { db.SetDB(nil) })
ctx := context.Background()
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled, "false"))
result, err := (&DatabaseAutoCleanupHandler{}).Execute(ctx, nil)
require.NoError(t, err)
require.NotNil(t, result)
assert.Contains(t, result.Message, "未启用")
}
func TestDatabaseAutoCleanupHandlerDeletesRowsWhenEnabled(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, sqliteDB.AutoMigrate(&model.SystemConfig{}))
db.SetDB(sqliteDB)
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
resetObservabilityStore := repository.SetObservabilityStoreForTest(repository.NewMemoryObservabilityStore())
t.Cleanup(func() {
resetObservabilityStore()
resetAccessLogStore()
db.SetDB(nil)
})
ctx := context.Background()
now := time.Now().UTC()
require.NoError(t, repository.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{{
NodeID: "node-a",
LoggedAt: now.Add(-95 * 24 * time.Hour),
RemoteAddr: "203.0.113.10",
Host: "example.com",
Path: "/access",
StatusCode: 200,
}}))
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled, "true"))
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyDatabaseAutoCleanupRetentionDays, "1"))
result, err := (&DatabaseAutoCleanupHandler{}).Execute(ctx, nil)
require.NoError(t, err)
require.NotNil(t, result)
assert.Contains(t, result.Message, "共删除")
rows, err := repository.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
require.NoError(t, err)
assert.Empty(t, rows)
}
func TestUptimeKumaSyncHandlerSkipsWhenDisabled(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
+102 -20
View File
@@ -11,13 +11,10 @@ import (
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/platform/lifecycle"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
@@ -56,12 +53,10 @@ var (
frpcDedup *dedupSet
)
// Init starts OpenFlare ClickHouse batch writers. Safe to call multiple times.
// Init starts OpenFlare log batch writers. Safe to call multiple times.
// Writers always initialize regardless of ClickHouse.enabled; the active log
// store is resolved via logstore at flush time (PG/SQLite when CH is not active).
func Init(ctx context.Context) {
if !config.Config.ClickHouse.Enabled {
return
}
initOnce.Do(func() {
metricSnapshotDedup = newDedupSet()
edgeHealthDedup = newDedupSet()
@@ -70,25 +65,25 @@ func Init(ctx context.Context) {
metricSnapshotWriter = mustNewObservabilityWriter(
"metric_snapshots",
withFlushRetries(analyticsrepo.BatchInsertNodeMetricSnapshots),
withFlushRetries(flushNodeMetricSnapshots),
metricSnapshotDedup,
metricSnapshotKey,
)
edgeHealthWriter = mustNewObservabilityWriter(
"edge_health",
withFlushRetries(analyticsrepo.BatchInsertNodeEdgeHealth),
withFlushRetries(flushNodeEdgeHealth),
edgeHealthDedup,
edgeHealthKey,
)
frpsWriter = mustNewObservabilityWriter(
"frps_obs",
withFlushRetries(analyticsrepo.BatchInsertNodeObsFrps),
withFlushRetries(flushNodeObsFrps),
frpsDedup,
frpsKey,
)
frpcWriter = mustNewObservabilityWriter(
"frpc_obs",
withFlushRetries(analyticsrepo.BatchInsertNodeObsFrpc),
withFlushRetries(flushNodeObsFrpc),
frpcDedup,
frpcKey,
)
@@ -129,6 +124,51 @@ func Stop(ctx context.Context) error {
return firstErr
}
// Drain 等待所有 OpenFlare 日志 writer 的在途批次落库:轮询队列 Depth 归零后
// 再保持一个最大 flush 周期(observabilityFlushEvery)持续为空才返回;
// 不停止 writer(迁移冻结后由 ensureWritable 拒绝新写入)。未初始化时直接返回 nil。
func Drain(ctx context.Context) error {
return drainWriters(ctx, WriterStats, observabilityFlushEvery)
}
// drainWriters 轮询 stats 直至所有队列 Depth=0 并持续 quietPeriod 无新积压。
func drainWriters(ctx context.Context, stats func() []batchwriter.Stats, quietPeriod time.Duration) error {
if !running() {
return nil
}
ticker := time.NewTicker(drainPollInterval)
defer ticker.Stop()
var quietSince time.Time
for {
if allDepthZero(stats()) {
if quietSince.IsZero() {
quietSince = time.Now()
} else if time.Since(quietSince) >= quietPeriod {
return nil
}
} else {
quietSince = time.Time{}
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
}
// drainPollInterval 队列轮询间隔。
const drainPollInterval = 50 * time.Millisecond
func allDepthZero(stats []batchwriter.Stats) bool {
for _, s := range stats {
if s.Depth > 0 {
return false
}
}
return true
}
// WriterStats returns queue depth and failure counters for all OpenFlare writers.
func WriterStats() []batchwriter.Stats {
writers := []statsProvider{
@@ -237,7 +277,7 @@ func mustNewNodeAccessLogWriter() *batchwriter.Writer[analyticsmodel.NodeAccessL
}
writer, err := batchwriter.New[analyticsmodel.NodeAccessLog](
cfg,
withFlushRetries(analyticsrepo.BatchInsertNodeAccessLogs),
withFlushRetries(flushNodeAccessLogs),
batchwriter.WithDropHandler[analyticsmodel.NodeAccessLog](func(item analyticsmodel.NodeAccessLog) {
logger.WarnF(context.Background(), "[OpenFlare] node access log queue full, dropping log for node %s path %s", item.NodeID, item.Path)
}),
@@ -280,17 +320,59 @@ func withFlushRetries[T any](flush batchwriter.FlushFunc[T]) batchwriter.FlushFu
}
func wireModelInsertHooks() {
repository.SetObservabilityInsertHooks(repository.ObservabilityInsertHooks{
QueueMetricSnapshot: QueueMetricSnapshot,
QueueEdgeHealth: QueueEdgeHealth,
QueueFrpsObservation: QueueFrpsObservation,
QueueFrpcObservation: QueueFrpcObservation,
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{
QueueMetricSnapshot: QueueMetricSnapshot,
QueueEdgeHealth: QueueEdgeHealth,
QueueNodeObsFrps: QueueFrpsObservation,
QueueNodeObsFrpc: QueueFrpcObservation,
})
repository.SetAccessLogInsertHooks(repository.AccessLogInsertHooks{
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
QueueNodeAccessLogs: QueueNodeAccessLogs,
})
}
// 以下 flush 函数作为 batchwriter 的落库目标:激活库由 logstore 在 flush 时决定。
func flushNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.BatchInsertNodeMetricSnapshots(ctx, rows)
}
func flushNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.BatchInsertNodeEdgeHealth(ctx, rows)
}
func flushNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.BatchInsertNodeObsFrps(ctx, rows)
}
func flushNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.BatchInsertNodeObsFrpc(ctx, rows)
}
func flushNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.AccessLogs.BatchInsertNodeAccessLogs(ctx, rows)
}
func metricSnapshotKey(snapshot analyticsmodel.NodeMetricSnapshot) string {
return fmt.Sprintf("%s|%d", snapshot.NodeID, snapshot.CapturedAt.UTC().UnixNano())
}
@@ -9,6 +9,7 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -26,11 +27,8 @@ func setupDashboardTestDB(t *testing.T) func() {
require.NoError(t, sqliteDB.AutoMigrate(&model.OpenFlareNode{}))
db.SetDB(sqliteDB)
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
resetObservabilityStore := repository.SetObservabilityStoreForTest(repository.NewMemoryObservabilityStore())
testhelper.SetupLogStoresForTest(t)
return func() {
resetObservabilityStore()
resetAccessLogStore()
db.SetDB(nil)
}
}
@@ -40,15 +40,12 @@ func setupProtocolTestEnv(t *testing.T) (*gin.Engine, func()) {
db.SetDB(sqliteDB)
agent.ResetAuthCacheForTest()
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
resetObservabilityStore := repository.SetObservabilityStoreForTest(repository.NewMemoryObservabilityStore())
testhelper.SetupLogStoresForTest(t)
engine := testhelper.NewTestGinEngine()
mountOpenFlareTestRoutes(engine)
cleanup := func() {
resetObservabilityStore()
resetAccessLogStore()
db.SetDB(nil)
agent.ResetAuthCacheForTest()
}
+2 -4
View File
@@ -15,6 +15,7 @@ import (
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -41,12 +42,9 @@ func setupNodeTestDB(t *testing.T) func() {
))
db.SetDB(sqliteDB)
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
resetObservabilityStore := repository.SetObservabilityStoreForTest(repository.NewMemoryObservabilityStore())
testhelper.SetupLogStoresForTest(t)
return func() {
resetObservabilityStore()
resetAccessLogStore()
db.SetDB(nil)
}
}
@@ -11,7 +11,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
@@ -448,9 +448,9 @@ func buildAccessLogUADistributions(
for _, row := range uaRows {
ua := row.Key
count := row.Value
deviceAcc[analyticsrepo.ParseDeviceType(ua)] += count
browserAcc[analyticsrepo.ParseBrowserName(ua)] += count
osAcc[analyticsrepo.ParseOSName(ua)] += count
deviceAcc[analyticsmodel.ParseDeviceType(ua)] += count
browserAcc[analyticsmodel.ParseBrowserName(ua)] += count
osAcc[analyticsmodel.ParseOSName(ua)] += count
}
topUserAgents = make([]DistributionItem, 0, accessLogOverviewTopLimit)
-43
View File
@@ -10,7 +10,6 @@ import (
"strings"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/geoip"
oftasks "github.com/Rain-kl/Wavelet/internal/apps/openflare/tasks"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/uptimekuma"
"github.com/Rain-kl/Wavelet/internal/buildinfo"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -50,22 +49,6 @@ type geoIPLookupView struct {
Longitude *float64 `json:"longitude,omitempty"`
}
type databaseCleanupInput struct {
Target string `json:"target"`
RetentionDays *int `json:"retention_days"`
}
type databaseCleanupResult struct {
Target string `json:"target"`
TargetLabel string `json:"target_label"`
DeletedCount int64 `json:"deleted_count"`
EligibleCount int64 `json:"eligible_count,omitempty"`
CleanupMode string `json:"cleanup_mode,omitempty"`
TableTTLDays int `json:"table_ttl_days,omitempty"`
DeleteAll bool `json:"delete_all"`
RetentionDays *int `json:"retention_days,omitempty"`
}
type optionBatchPayload struct {
Options []model.OpenFlareOption `json:"options"`
}
@@ -183,32 +166,6 @@ func lookupGeoIP(_ context.Context, provider, rawIP string) (*geoIPLookupView, e
}, nil
}
func cleanupDatabaseObservability(ctx context.Context, input databaseCleanupInput) (*databaseCleanupResult, error) {
target := strings.TrimSpace(input.Target)
if target == "" {
return nil, errors.New(errInvalidParams)
}
result, err := oftasks.CleanupDatabaseObservability(ctx, oftasks.DatabaseCleanupInput{
Target: target,
RetentionDays: input.RetentionDays,
})
if err != nil {
return nil, err
}
return &databaseCleanupResult{
Target: result.Target,
TargetLabel: result.TargetLabel,
DeletedCount: result.DeletedCount,
EligibleCount: result.EligibleCount,
CleanupMode: result.CleanupMode,
TableTTLDays: result.TableTTLDays,
DeleteAll: result.DeleteAll,
RetentionDays: result.RetentionDays,
}, nil
}
func syncUptimeKuma(ctx context.Context) error {
return uptimekuma.SyncToUptimeKuma(ctx)
}
@@ -6,7 +6,6 @@ package option
import (
"context"
"testing"
"time"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -117,55 +116,3 @@ func TestLookupGeoIPDisabledProvider(t *testing.T) {
assert.Equal(t, "disabled", view.Provider)
assert.Equal(t, "8.8.8.8", view.IP)
}
func TestCleanupDatabaseObservabilityDeletesRows(t *testing.T) {
cleanup := setupOptionTestDB(t)
defer cleanup()
ctx := context.Background()
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
defer resetAccessLogStore()
now := time.Now().UTC()
require.NoError(t, repository.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
{
NodeID: "node-a",
LoggedAt: now.Add(-10 * 24 * time.Hour),
RemoteAddr: "203.0.113.1",
Host: "example.com",
Path: "/old",
StatusCode: 200,
},
{
NodeID: "node-a",
LoggedAt: now.Add(-2 * time.Hour),
RemoteAddr: "203.0.113.2",
Host: "example.com",
Path: "/recent",
StatusCode: 200,
},
}))
// Retention shorter than table TTL (90d for access logs) must be rejected.
shortRetention := 7
_, err := cleanupDatabaseObservability(ctx, databaseCleanupInput{
Target: "node_access_logs",
RetentionDays: &shortRetention,
})
require.Error(t, err)
// Full truncate still hard-deletes all rows.
result, err := cleanupDatabaseObservability(ctx, databaseCleanupInput{
Target: "node_access_logs",
})
require.NoError(t, err)
assert.Equal(t, "node_access_logs", result.Target)
assert.Equal(t, "访问日志", result.TargetLabel)
assert.Equal(t, int64(2), result.DeletedCount)
assert.True(t, result.DeleteAll)
assert.Equal(t, "truncate", result.CleanupMode)
rows, err := repository.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
require.NoError(t, err)
assert.Empty(t, rows)
}
-37
View File
@@ -4,9 +4,6 @@
package option
import (
"encoding/json"
"errors"
"io"
"net/http"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/apiutil"
@@ -128,33 +125,6 @@ func LookupGeoIPHandler(c *gin.Context) {
c.JSON(http.StatusOK, response.OK(view))
}
// CleanupDatabaseHandler 清理可观测性数据库数据。
// @Summary 清理可观测性数据库
// @Description 按目标与保留天数清理可观测性相关数据表,需要管理员权限
// @Tags openflare-option
// @Accept json
// @Produce json
// @Security SessionCookie
// @Param request body option.databaseCleanupInput false "清理参数"
// @Success 200 {object} response.Any{data=option.databaseCleanupResult} "清理结果"
// @Failure 400 {object} response.Any "参数错误"
// @Failure 401 {object} response.Any "未登录"
// @Failure 404 {object} response.Any "无权限或不存在"
// @Failure 500 {object} response.Any "内部错误"
// @Router /api/v1/d/option/database/cleanup [post]
func CleanupDatabaseHandler(c *gin.Context) {
var input databaseCleanupInput
if err := bindOptionalJSON(c.Request.Body, &input); err != nil {
response.AbortBadRequest(c, errInvalidParams)
return
}
result, err := cleanupDatabaseObservability(c.Request.Context(), input)
if apiutil.AbortBadRequestOnError(c, err) {
return
}
c.JSON(http.StatusOK, response.OK(result))
}
// SyncUptimeKumaHandler 同步 Uptime Kuma 监控。
// @Summary 同步 Uptime Kuma
// @Description 将 OpenFlare 节点同步到 Uptime Kuma,需要管理员权限
@@ -174,10 +144,3 @@ func SyncUptimeKumaHandler(c *gin.Context) {
}
c.JSON(http.StatusOK, response.OK("同步成功"))
}
func bindOptionalJSON(body io.Reader, target any) error {
if err := json.NewDecoder(body).Decode(target); err != nil && !errors.Is(err, io.EOF) {
return err
}
return nil
}
+17 -5
View File
@@ -27,6 +27,17 @@ var (
const optionValueTrue = "true"
// protectedConfigKeyMessage 命中受保护 key 时返回给管理员的业务错误文案。
const protectedConfigKeyMessage = "该配置项由系统任务管理,禁止手动修改"
// protectedConfigKeys 仅允许内部(迁移任务/bootstrap)写入的 key。
var protectedConfigKeys = map[string]bool{
model.ConfigKeyLogDatabase: true,
model.ConfigKeyLogDBMigration: true,
}
func isProtectedConfigKey(key string) bool { return protectedConfigKeys[key] }
func buildOptionValidationState(ctx context.Context, options []model.OpenFlareOption) map[string]string {
// 从 SystemConfig 读取所有业务配置构建状态
configs, err := repository.ListAdminSystemConfigs(ctx, "business")
@@ -53,7 +64,7 @@ func validateOptionWithState(ctx context.Context, option model.OpenFlareOption,
if err := validateGeoIPOption(option.Key, option.Value); err != nil {
return err
}
if err := validateDatabaseCleanupOption(option.Key, option.Value); err != nil {
if err := validateLogRetentionOption(option.Key, option.Value); err != nil {
return err
}
if err := validateAgentOption(option.Key, option.Value); err != nil {
@@ -100,11 +111,9 @@ func validateGeoIPOption(key, value string) error {
return fmt.Errorf("%s 仅支持 disabled、mmdb、ip-api、geojs、ipinfo", key)
}
func validateDatabaseCleanupOption(key, value string) error {
func validateLogRetentionOption(key, value string) error {
switch key {
case model.ConfigKeyDatabaseAutoCleanupEnabled:
return validateBooleanOption(key, value)
case model.ConfigKeyDatabaseAutoCleanupRetentionDays:
case model.ConfigKeyLogRetentionDaysPostgres, model.ConfigKeyLogRetentionDaysSQLite, model.ConfigKeyLogRetentionDaysClickHouse:
intValue, err := strconv.Atoi(value)
if err != nil || intValue < 1 {
return fmt.Errorf("%s 必须为大于等于 1 的整数天", key)
@@ -221,6 +230,9 @@ func validateOptions(ctx context.Context, options []model.OpenFlareOption) error
if strings.TrimSpace(option.Key) == "" {
return errors.New(errInvalidParams)
}
if isProtectedConfigKey(option.Key) {
return errors.New(protectedConfigKeyMessage)
}
if err := validateOptionWithState(ctx, option, state); err != nil {
return err
}
+2 -2
View File
@@ -10,6 +10,7 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/agent"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
@@ -38,10 +39,9 @@ func setupRelayTestDB(t *testing.T) func() {
db.SetDB(sqliteDB)
agent.ResetAuthCacheForTest()
resetObservabilityStore := repository.SetObservabilityStoreForTest(repository.NewMemoryObservabilityStore())
testhelper.SetupLogStoresForTest(t)
return func() {
resetObservabilityStore()
db.SetDB(nil)
agent.ResetAuthCacheForTest()
}
@@ -1,245 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package tasks
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
const (
// DatabaseCleanupTargetAccessLogs is the API cleanup target for access logs.
DatabaseCleanupTargetAccessLogs = "node_access_logs"
// DatabaseCleanupTargetMetricSnapshots is the API cleanup target for metric snapshots.
DatabaseCleanupTargetMetricSnapshots = "node_metric_snapshots"
// DatabaseCleanupTargetEdgeHealth is the API cleanup target for OpenResty edge health (connections).
DatabaseCleanupTargetEdgeHealth = "node_edge_health"
// DatabaseCleanupTargetObsFrps is the API cleanup target for FRPS observations.
DatabaseCleanupTargetObsFrps = "node_obs_frps"
// DatabaseCleanupTargetObsFrpc is the API cleanup target for FRPC observations.
DatabaseCleanupTargetObsFrpc = "node_obs_frpc"
)
var databaseCleanupTargets = map[string]string{
DatabaseCleanupTargetAccessLogs: "访问日志",
DatabaseCleanupTargetMetricSnapshots: "性能快照",
DatabaseCleanupTargetEdgeHealth: "OpenResty 健康(连接)",
DatabaseCleanupTargetObsFrps: "FRPS 观测",
DatabaseCleanupTargetObsFrpc: "FRPC 观测",
}
// databaseCleanupTableTTLDays maps API targets to ClickHouse DDL TTL days.
var databaseCleanupTableTTLDays = map[string]int{
DatabaseCleanupTargetAccessLogs: analyticsrepo.TableTTLDaysNodeAccessLogs,
DatabaseCleanupTargetMetricSnapshots: analyticsrepo.TableTTLDaysNodeMetricSnapshots,
DatabaseCleanupTargetEdgeHealth: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetObsFrps: analyticsrepo.TableTTLDaysNodeObs,
DatabaseCleanupTargetObsFrpc: analyticsrepo.TableTTLDaysNodeObs,
}
// DatabaseCleanupInput describes a manual observability cleanup request.
type DatabaseCleanupInput struct {
Target string `json:"target"`
RetentionDays *int `json:"retention_days"`
}
// DatabaseCleanupResult summarizes a manual observability cleanup run.
//
// Semantics:
// - delete_all / cleanup_mode=truncate: DeletedCount is hard-deleted rows (TRUNCATE).
// - retention path / cleanup_mode=ttl_materialize: DeletedCount is always 0;
// EligibleCount estimates rows past the table DDL TTL (not an arbitrary younger cutoff).
type DatabaseCleanupResult struct {
Target string `json:"target"`
TargetLabel string `json:"target_label"`
DeletedCount int64 `json:"deleted_count"`
EligibleCount int64 `json:"eligible_count,omitempty"`
CleanupMode string `json:"cleanup_mode,omitempty"`
TableTTLDays int `json:"table_ttl_days,omitempty"`
DeleteAll bool `json:"delete_all"`
RetentionDays *int `json:"retention_days,omitempty"`
Cutoff *time.Time `json:"cutoff,omitempty"`
}
// DatabaseAutoCleanupSummary summarizes a scheduled auto-cleanup run.
type DatabaseAutoCleanupSummary struct {
RetentionDays int `json:"retention_days"`
ExecutedAt time.Time `json:"executed_at"`
Results []DatabaseCleanupResult `json:"results"`
}
// TableTTLDaysForCleanupTarget returns the DDL TTL days for a cleanup target.
func TableTTLDaysForCleanupTarget(target string) (int, bool) {
days, ok := databaseCleanupTableTTLDays[strings.TrimSpace(target)]
return days, ok
}
// CleanupDatabaseObservability deletes observability rows for the given target.
//
// When RetentionDays is nil, rows are hard-deleted via TRUNCATE.
// When RetentionDays is set, ClickHouse only force-materializes the table TTL policy:
// retention_days shorter than the table TTL is rejected (do not fake success).
func CleanupDatabaseObservability(ctx context.Context, input DatabaseCleanupInput) (*DatabaseCleanupResult, error) {
target := strings.TrimSpace(input.Target)
targetLabel, ok := databaseCleanupTargets[target]
if !ok {
return nil, errors.New("unsupported cleanup target")
}
if input.RetentionDays != nil && *input.RetentionDays <= 0 {
return nil, errors.New("retention_days 必须为大于 0 的整数")
}
tableTTLDays := databaseCleanupTableTTLDays[target]
result := &DatabaseCleanupResult{
Target: target,
TargetLabel: targetLabel,
DeleteAll: input.RetentionDays == nil,
TableTTLDays: tableTTLDays,
}
if input.RetentionDays == nil {
deleted, mode, err := deleteAllObservabilityRows(ctx, target)
if err != nil {
return nil, err
}
result.DeletedCount = deleted
result.EligibleCount = deleted
result.CleanupMode = mode
return result, nil
}
retentionDays := *input.RetentionDays
if retentionDays < tableTTLDays {
return nil, fmt.Errorf(
"retention_days 不能小于表 TTL(%d 天);ClickHouse 仅支持按表 TTL 物化过期,更短保留请使用清空全部或调整 DDL",
tableTTLDays,
)
}
// MATERIALIZE TTL only enforces DDL policy; cutoff reported is the table TTL boundary.
tableCutoff := time.Now().UTC().Add(-time.Duration(tableTTLDays) * 24 * time.Hour)
eligible, mode, err := materializeObservabilityTableTTL(ctx, target)
if err != nil {
return nil, err
}
result.DeletedCount = 0
result.EligibleCount = eligible
result.CleanupMode = mode
result.RetentionDays = &retentionDays
result.Cutoff = &tableCutoff
return result, nil
}
// RunDatabaseAutoCleanupOnce runs retention-based cleanup for all observability targets.
//
// Configured retention shorter than a target's table TTL is clamped up to the table TTL
// so the scheduled job can force-materialize each table policy without failing.
func RunDatabaseAutoCleanupOnce(ctx context.Context, now time.Time) (*DatabaseAutoCleanupSummary, error) {
enabled, err := repository.GetBoolByKey(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled)
if err != nil {
return nil, fmt.Errorf("failed to read database_auto_cleanup_enabled: %w", err)
}
if !enabled {
return nil, nil
}
retentionDays, err := repository.GetIntByKey(ctx, model.ConfigKeyDatabaseAutoCleanupRetentionDays)
if err != nil || retentionDays <= 0 {
// Use default value 30 if config read fails or value is invalid
retentionDays = 30
}
results := make([]DatabaseCleanupResult, 0, len(databaseCleanupTargets))
for _, target := range []string{
DatabaseCleanupTargetAccessLogs,
DatabaseCleanupTargetMetricSnapshots,
DatabaseCleanupTargetEdgeHealth,
DatabaseCleanupTargetObsFrps,
DatabaseCleanupTargetObsFrpc,
} {
effectiveDays := retentionDays
if ttl, ok := databaseCleanupTableTTLDays[target]; ok && effectiveDays < ttl {
effectiveDays = ttl
}
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
Target: target,
RetentionDays: &effectiveDays,
})
if err != nil {
return nil, err
}
results = append(results, *result)
}
return &DatabaseAutoCleanupSummary{
RetentionDays: retentionDays,
ExecutedAt: now.UTC(),
Results: results,
}, nil
}
func deleteAllObservabilityRows(ctx context.Context, target string) (int64, string, error) {
var (
deleted int64
err error
)
switch target {
case DatabaseCleanupTargetAccessLogs:
deleted, err = repository.DeleteAllOpenFlareAccessLogs(ctx)
case DatabaseCleanupTargetMetricSnapshots:
deleted, err = repository.DeleteAllOpenFlareMetricSnapshots(ctx)
case DatabaseCleanupTargetEdgeHealth:
deleted, err = repository.DeleteAllOpenFlareEdgeHealth(ctx)
case DatabaseCleanupTargetObsFrps:
deleted, err = repository.DeleteAllOpenFlareNodeObservationFrps(ctx)
case DatabaseCleanupTargetObsFrpc:
deleted, err = repository.DeleteAllOpenFlareNodeObservationFrpc(ctx)
default:
return 0, "", errors.New("unsupported cleanup target")
}
if err != nil {
return 0, "", err
}
return deleted, analyticsrepo.CleanupModeTruncate, nil
}
// materializeObservabilityTableTTL triggers table-TTL materialize (or memory-store delete-before
// with the table TTL cutoff for tests) and returns the eligible/estimate row count.
func materializeObservabilityTableTTL(ctx context.Context, target string) (int64, string, error) {
ttlDays, ok := databaseCleanupTableTTLDays[target]
if !ok {
return 0, "", errors.New("unsupported cleanup target")
}
cutoff := time.Now().UTC().Add(-time.Duration(ttlDays) * 24 * time.Hour)
var (
eligible int64
err error
)
switch target {
case DatabaseCleanupTargetAccessLogs:
eligible, err = repository.DeleteOpenFlareAccessLogsBefore(ctx, cutoff)
case DatabaseCleanupTargetMetricSnapshots:
eligible, err = repository.DeleteOpenFlareMetricSnapshotsBefore(ctx, cutoff)
case DatabaseCleanupTargetEdgeHealth:
eligible, err = repository.DeleteOpenFlareEdgeHealthBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrps:
eligible, err = repository.DeleteOpenFlareNodeObservationFrpsBefore(ctx, cutoff)
case DatabaseCleanupTargetObsFrpc:
eligible, err = repository.DeleteOpenFlareNodeObservationFrpcBefore(ctx, cutoff)
default:
return 0, "", errors.New("unsupported cleanup target")
}
if err != nil {
return 0, "", err
}
return eligible, analyticsrepo.CleanupModeTTLMaterialize, nil
}
@@ -1,208 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package tasks
import (
"context"
"testing"
"time"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func setupDatabaseCleanupTestDB(t *testing.T) context.Context {
t.Helper()
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, sqliteDB.AutoMigrate(&model.SystemConfig{}))
db.SetDB(sqliteDB)
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
resetObservabilityStore := repository.SetObservabilityStoreForTest(repository.NewMemoryObservabilityStore())
t.Cleanup(func() {
resetObservabilityStore()
resetAccessLogStore()
db.SetDB(nil)
})
return context.Background()
}
func TestCleanupDatabaseObservabilityRejectsRetentionShorterThanTableTTL(t *testing.T) {
ctx := setupDatabaseCleanupTestDB(t)
retentionDays := 7 // metric snapshots DDL TTL is 30 days
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
Target: DatabaseCleanupTargetMetricSnapshots,
RetentionDays: &retentionDays,
})
require.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "不能小于表 TTL")
assert.Contains(t, err.Error(), "30")
}
func TestCleanupDatabaseObservabilityRejectsAccessLogRetentionShorterThanTableTTL(t *testing.T) {
ctx := setupDatabaseCleanupTestDB(t)
retentionDays := 30 // access logs DDL TTL is 90 days
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
Target: DatabaseCleanupTargetAccessLogs,
RetentionDays: &retentionDays,
})
require.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "90")
}
func TestCleanupDatabaseObservabilityMaterializeDoesNotClaimHardDelete(t *testing.T) {
ctx := setupDatabaseCleanupTestDB(t)
now := time.Now().UTC()
// One row past metric table TTL (30d), one still inside the window.
require.NoError(t, repository.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
NodeID: "node-a",
CapturedAt: now.Add(-40 * 24 * time.Hour),
CPUUsagePercent: 10,
}))
require.NoError(t, repository.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
NodeID: "node-a",
CapturedAt: now.Add(-12 * time.Hour),
CPUUsagePercent: 20,
}))
retentionDays := analyticsrepo.TableTTLDaysNodeMetricSnapshots
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
Target: DatabaseCleanupTargetMetricSnapshots,
RetentionDays: &retentionDays,
})
require.NoError(t, err)
assert.False(t, result.DeleteAll)
assert.Equal(t, analyticsrepo.CleanupModeTTLMaterialize, result.CleanupMode)
assert.Equal(t, analyticsrepo.TableTTLDaysNodeMetricSnapshots, result.TableTTLDays)
// MATERIALIZE is not a counted hard delete.
assert.Equal(t, int64(0), result.DeletedCount)
assert.Equal(t, int64(1), result.EligibleCount)
require.NotNil(t, result.Cutoff)
assert.True(t, result.Cutoff.Before(now.Add(-29*24*time.Hour)))
// Memory store applies the table-TTL cutoff for tests; only the recent row remains.
rows, err := repository.ListOpenFlareMetricSnapshotsSince(ctx, "", time.Time{}, 0)
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, float64(20), rows[0].CPUUsagePercent)
}
func TestCleanupDatabaseObservabilityDeletesAllRowsWhenRetentionMissing(t *testing.T) {
ctx := setupDatabaseCleanupTestDB(t)
now := time.Now().UTC()
require.NoError(t, repository.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{
{
NodeID: "node-a",
LoggedAt: now.Add(-3 * time.Hour),
RemoteAddr: "203.0.113.1",
Host: "example.com",
Path: "/one",
StatusCode: 200,
},
{
NodeID: "node-a",
LoggedAt: now.Add(-2 * time.Hour),
RemoteAddr: "203.0.113.2",
Host: "example.com",
Path: "/two",
StatusCode: 502,
},
}))
result, err := CleanupDatabaseObservability(ctx, DatabaseCleanupInput{
Target: DatabaseCleanupTargetAccessLogs,
})
require.NoError(t, err)
assert.True(t, result.DeleteAll)
assert.Equal(t, analyticsrepo.CleanupModeTruncate, result.CleanupMode)
assert.Equal(t, int64(2), result.DeletedCount)
assert.Equal(t, int64(2), result.EligibleCount)
rows, err := repository.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
require.NoError(t, err)
assert.Empty(t, rows)
}
func TestRunDatabaseAutoCleanupOnceClampsRetentionToTableTTL(t *testing.T) {
ctx := setupDatabaseCleanupTestDB(t)
now := time.Now().UTC()
// Access logs TTL=90d, metrics TTL=30d. Config retention=1 must clamp, not reject.
require.NoError(t, repository.InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{{
NodeID: "node-a",
LoggedAt: now.Add(-100 * 24 * time.Hour),
RemoteAddr: "203.0.113.10",
Host: "example.com",
Path: "/access",
StatusCode: 200,
}}))
require.NoError(t, repository.InsertOpenFlareMetricSnapshot(ctx, &model.OpenFlareMetricSnapshot{
NodeID: "node-a",
CapturedAt: now.Add(-40 * 24 * time.Hour),
CPUUsagePercent: 10,
}))
require.NoError(t, repository.InsertOpenFlareEdgeHealth(ctx, &model.OpenFlareEdgeHealth{
NodeID: "node-a",
CapturedAt: now.Add(-40 * 24 * time.Hour),
Status: "healthy",
Connections: 2,
}))
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled, "true"))
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyDatabaseAutoCleanupRetentionDays, "1"))
summary, err := RunDatabaseAutoCleanupOnce(ctx, now)
require.NoError(t, err)
require.NotNil(t, summary)
require.Len(t, summary.Results, 5)
assert.Equal(t, 1, summary.RetentionDays)
for _, result := range summary.Results {
assert.Equal(t, analyticsrepo.CleanupModeTTLMaterialize, result.CleanupMode)
assert.Equal(t, int64(0), result.DeletedCount, "target %s must not claim hard delete", result.Target)
assert.GreaterOrEqual(t, result.TableTTLDays, 30)
require.NotNil(t, result.RetentionDays)
assert.GreaterOrEqual(t, *result.RetentionDays, result.TableTTLDays)
}
accessLogs, err := repository.ListOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Page: 0, PageSize: 10})
require.NoError(t, err)
assert.Empty(t, accessLogs)
metricSnapshots, err := repository.ListOpenFlareMetricSnapshotsSince(ctx, "", time.Time{}, 0)
require.NoError(t, err)
assert.Empty(t, metricSnapshots)
edgeHealth, err := repository.ListOpenFlareEdgeHealth(ctx, "", time.Time{}, 0)
require.NoError(t, err)
assert.Empty(t, edgeHealth)
}
func TestTableTTLDaysForCleanupTarget(t *testing.T) {
days, ok := TableTTLDaysForCleanupTarget(DatabaseCleanupTargetAccessLogs)
require.True(t, ok)
assert.Equal(t, 90, days)
days, ok = TableTTLDaysForCleanupTarget(DatabaseCleanupTargetMetricSnapshots)
require.True(t, ok)
assert.Equal(t, 30, days)
_, ok = TableTTLDaysForCleanupTarget("unknown")
assert.False(t, ok)
}
@@ -0,0 +1,402 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package tasks
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
"github.com/Rain-kl/Wavelet/internal/apps/risk_control"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
const copyBatchSize = 1000
// 迁移目标库名常量(normalizeTarget 归一化后的取值)。
const (
targetPostgres = "postgres"
targetSQLite = "sqlite"
targetClickHouse = "clickhouse"
)
type logDBSwitchPayload struct {
Target string `json:"target"`
}
// LogDBSwitchHandler 切换日志数据库任务处理器。
type LogDBSwitchHandler struct{}
// ValidatePayload 校验并规范化参数。
func (h *LogDBSwitchHandler) ValidatePayload(payload []byte) ([]byte, error) {
var p logDBSwitchPayload
if err := json.Unmarshal(payload, &p); err != nil {
return nil, fmt.Errorf("参数解析失败: %w", err)
}
p.Target = normalizeTarget(p.Target)
if !validTarget(p.Target) {
return nil, fmt.Errorf("目标日志库不合法: %s", p.Target)
}
out, err := json.Marshal(p)
if err != nil {
return nil, err
}
return out, nil
}
func normalizeTarget(v string) string {
switch v {
case targetPostgres, "postgresql":
return targetPostgres
case targetSQLite, "sqlite3":
return targetSQLite
case targetClickHouse, "ch":
return targetClickHouse
}
return v
}
func validTarget(v string) bool {
return v == targetPostgres || v == targetSQLite || v == targetClickHouse
}
// Execute 执行迁移。
func (h *LogDBSwitchHandler) Execute(ctx context.Context, payload []byte) (*task.TaskResult, error) {
var p logDBSwitchPayload
if err := json.Unmarshal(payload, &p); err != nil {
return nil, fmt.Errorf("参数解析失败: %w", err)
}
p.Target = normalizeTarget(p.Target)
if err := validateSwitch(ctx, p.Target); err != nil {
return nil, err
}
source, err := currentLogDatabase(ctx)
if err != nil {
task.AppendLog(ctx, "读取日志主库失败: %v", err)
return nil, err
}
task.AppendLog(ctx, "开始切换日志数据库:%s -> %s", source, p.Target)
// 设置迁移冻结标记(置位后由 ensureWritable 拒绝新写入)。
if err := setMigrationFlag(ctx, "migrating"); err != nil {
return nil, err
}
// 失败也清除(SaveOrUpdateSystemConfig 会失效 RAM 缓存并广播),保持源库可写。
defer func() {
if err := setMigrationFlag(ctx, ""); err != nil {
logger.ErrorF(ctx, "清除日志迁移冻结标记失败: %v", err)
}
}()
// 冻结标记置位后再排空在途批次(chwriter + 用户访问日志 writer),
// 保证排空完成后不再有新批次进入源库。
if err := drainLogWriters(ctx); err != nil {
return nil, fmt.Errorf("排空日志写入队列失败: %w", err)
}
src, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
dst, err := buildTargetStore(ctx, p.Target)
if err != nil {
return nil, err
}
// 清空目标库日志表(幂等重试前提)。
if err := clearTargetLogTables(ctx, dst); err != nil {
return nil, err
}
// PG 目标:按源库时间范围预建分区,避免历史数据复制报 "no partition of relation found"。
if err := ensureTargetPartitions(ctx, src, dst, p.Target); err != nil {
return nil, err
}
// 逐表复制(6 张日志表)。
if err := copyAccessLogs(ctx, src, dst); err != nil {
return nil, err
}
if err := copyUserAccessLogs(ctx, src, dst); err != nil {
return nil, err
}
if err := copyObservability(ctx, src, dst); err != nil {
return nil, err
}
// 翻转主库标记。
if err := flipLogDatabase(ctx, p.Target); err != nil {
return nil, err
}
task.AppendLog(ctx, "日志数据库已切换为 %s,写入恢复", p.Target)
return &task.TaskResult{Message: fmt.Sprintf("日志数据库已从 %s 切换为 %s", source, p.Target)}, nil
}
func validateSwitch(ctx context.Context, target string) error {
source, err := currentLogDatabase(ctx)
if err != nil {
return err
}
if source == target {
return errors.New("目标日志库与当前日志库相同,无需迁移")
}
switch target {
case "clickhouse":
if !config.Config.ClickHouse.Enabled {
return errors.New("ClickHouse 未启用,无法迁移到 ClickHouse")
}
case "postgres":
if !config.Config.Database.Enabled {
return errors.New("PostgreSQL 未启用(当前主库为 SQLite),无法迁移到 PostgreSQL")
}
case "sqlite":
if config.Config.Database.Enabled {
return errors.New("当前主库为 PostgreSQL,日志库不能设置为 SQLite")
}
}
return nil
}
func currentLogDatabase(ctx context.Context) (string, error) {
cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
if err != nil {
return "", fmt.Errorf("读取日志主库失败: %w", err)
}
if cfg.Value == "" {
return "", errors.New("日志主库配置为空")
}
return cfg.Value, nil
}
// drainLogWriters 等待 chwriter(节点访问日志 + 可观测 4 表)与 risk_control
// (用户访问日志)的在途批次全部落库。见设计 §7.2:先排空再冻结。
func drainLogWriters(ctx context.Context) error {
if err := chwriter.Drain(ctx); err != nil {
return err
}
return risk_control.DrainLogWriter(ctx)
}
// setMigrationFlag 写入迁移冻结标记。用 SaveOrUpdateSystemConfig:行缺失时 upsert,
// 并失效 RAM 缓存 + 广播其他节点,保证 logstore.Migrating/resolveDatabase 立即生效。
func setMigrationFlag(ctx context.Context, v string) error {
return repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDBMigration, v)
}
// flipLogDatabase 翻转日志主库。同上用 SaveOrUpdateSystemConfig,确保各进程缓存失效后指向新库。
func flipLogDatabase(ctx context.Context, target string) error {
return repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDatabase, target)
}
// buildTargetStore 构造目标库 Store(不经过 Active 缓存,直接 Build)。
// 迁移期间冻结标记已置位,目标库的清空/复制写入必须放行,故使用 BuildForMigration。
func buildTargetStore(ctx context.Context, database string) (*logstore.Store, error) {
return logstore.BuildForMigration(ctx, database)
}
func clearTargetLogTables(ctx context.Context, dst *logstore.Store) error {
// 依次清空 6 张表:AccessLogs.DeleteAll、UserAccessLogs.DeleteAll、Observability.DeleteAll*
// (SQLite/PG 用 DeleteAll;CH 用 TRUNCATE 语义)。
if _, err := dst.AccessLogs.DeleteAll(ctx); err != nil {
return fmt.Errorf("清空目标访问日志失败: %w", err)
}
if _, err := dst.UserAccessLogs.DeleteAll(ctx); err != nil {
return fmt.Errorf("清空目标用户访问日志失败: %w", err)
}
for _, fn := range []func(context.Context) (int64, error){
dst.Observability.DeleteAllMetricSnapshots,
dst.Observability.DeleteAllEdgeHealth,
dst.Observability.DeleteAllNodeObservationFrps,
dst.Observability.DeleteAllNodeObservationFrpc,
} {
if _, err := fn(ctx); err != nil {
return err
}
}
return nil
}
// ensureTargetPartitions 目标为 PG 时,按源库时间范围(两表合并)预建分区,
// 否则复制历史数据会报 "no partition of relation found";目标非 PG 为 no-op。
func ensureTargetPartitions(ctx context.Context, src, dst *logstore.Store, target string) error {
if target != targetPostgres {
return nil
}
from, to, err := migrationRange(ctx, src)
if err != nil {
return err
}
if from.IsZero() || to.IsZero() {
task.AppendLog(ctx, "源库无日志数据,跳过分区预建")
return nil
}
if err := dst.AccessLogs.EnsurePartitions(ctx, from, to.AddDate(0, 1, 0)); err != nil {
return fmt.Errorf("预建目标 PG 分区失败: %w", err)
}
task.AppendLog(ctx, "已为目标 PG 预建分区 %s ~ %s", from.Format("2006-01"), to.Format("2006-01"))
return nil
}
// migrationRange 合并源库节点访问日志(logged_at)与用户访问日志(created_at)
// 的最小/最大时间;任一表为空时忽略该表。
func migrationRange(ctx context.Context, src *logstore.Store) (time.Time, time.Time, error) {
fromAccess, toAccess, err := src.AccessLogs.MigrationRange(ctx)
if err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("读取源访问日志时间范围失败: %w", err)
}
fromUser, toUser, err := src.UserAccessLogs.MigrationRange(ctx)
if err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("读取源用户访问日志时间范围失败: %w", err)
}
return minTime(fromAccess, fromUser), maxTime(toAccess, toUser), nil
}
func minTime(a, b time.Time) time.Time {
switch {
case a.IsZero():
return b
case b.IsZero():
return a
case a.Before(b):
return a
default:
return b
}
}
func maxTime(a, b time.Time) time.Time {
switch {
case a.IsZero():
return b
case b.IsZero():
return a
case a.After(b):
return a
default:
return b
}
}
// copyAccessLogs 从 src 复制节点访问日志到 dst。
func copyAccessLogs(ctx context.Context, src, dst *logstore.Store) error {
// 注意:迁移期间 src 已冻结,但复制读取不受冻结影响;每批按 id 升序扫描。
var lastID uint64
for {
rows, err := listNodeAccessLogsByID(ctx, src, lastID, copyBatchSize)
if err != nil {
return err
}
if len(rows) == 0 {
break
}
if err := dst.AccessLogs.BatchInsertNodeAccessLogs(ctx, rows); err != nil {
return fmt.Errorf("写入目标访问日志失败(批 %d): %w", lastID, err)
}
task.AppendLog(ctx, "已复制访问日志 %d 条(截至 id=%d)", len(rows), rows[len(rows)-1].ID)
lastID = rows[len(rows)-1].ID
if len(rows) < copyBatchSize {
break
}
}
return nil
}
func listNodeAccessLogsByID(ctx context.Context, src *logstore.Store, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error) {
return src.AccessLogs.ListForMigration(ctx, afterID, limit)
}
// copyUserAccessLogs 从 src 复制用户访问日志到 dst(按 id 升序分批)。
func copyUserAccessLogs(ctx context.Context, src, dst *logstore.Store) error {
var lastID uint64
for {
rows, err := src.UserAccessLogs.ListForMigration(ctx, lastID, copyBatchSize)
if err != nil {
return err
}
if len(rows) == 0 {
return nil
}
if err := dst.UserAccessLogs.BatchInsert(ctx, rows); err != nil {
return fmt.Errorf("写入目标用户访问日志失败(批 %d): %w", lastID, err)
}
lastID = rows[len(rows)-1].ID
task.AppendLog(ctx, "已复制用户访问日志 %d 条(截至 id=%d)", len(rows), lastID)
if len(rows) < copyBatchSize {
return nil
}
}
}
// copyObservability 复制 4 张可观测表,每张表按 id 升序分批复制,
// 以每批最后一条 id 作为下一批游标(不使用 len 近似)。
func copyObservability(ctx context.Context, src, dst *logstore.Store) error {
if err := copyObsTable(ctx, "metric_snapshots",
src.Observability.ListMetricSnapshotsForMigration,
dst.Observability.BatchInsertNodeMetricSnapshots,
lastMetricSnapshotID); err != nil {
return err
}
if err := copyObsTable(ctx, "edge_health",
src.Observability.ListEdgeHealthForMigration,
dst.Observability.BatchInsertNodeEdgeHealth,
lastEdgeHealthID); err != nil {
return err
}
if err := copyObsTable(ctx, "obs_frps",
src.Observability.ListNodeObsFrpsForMigration,
dst.Observability.BatchInsertNodeObsFrps,
lastObsFrpsID); err != nil {
return err
}
if err := copyObsTable(ctx, "obs_frpc",
src.Observability.ListNodeObsFrpcForMigration,
dst.Observability.BatchInsertNodeObsFrpc,
lastObsFrpcID); err != nil {
return err
}
return nil
}
// copyObsTable 按 id 升序分批复制单张可观测表;idOf 返回批内最后一条 id。
func copyObsTable[T any](ctx context.Context, name string,
list func(context.Context, uint64, int) ([]T, error),
insert func(context.Context, []T) error,
idOf func([]T) uint64,
) error {
var lastID uint64
for {
rows, err := list(ctx, lastID, copyBatchSize)
if err != nil {
return fmt.Errorf("复制 %s 失败: %w", name, err)
}
if len(rows) == 0 {
return nil
}
if err := insert(ctx, rows); err != nil {
return fmt.Errorf("复制 %s 失败: %w", name, err)
}
lastID = idOf(rows)
task.AppendLog(ctx, "已复制 %s %d 条(截至 id=%d)", name, len(rows), lastID)
if len(rows) < copyBatchSize {
return nil
}
}
}
func lastMetricSnapshotID(rows []analyticsmodel.NodeMetricSnapshot) uint64 {
return rows[len(rows)-1].ID
}
func lastEdgeHealthID(rows []analyticsmodel.NodeEdgeHealth) uint64 { return rows[len(rows)-1].ID }
func lastObsFrpsID(rows []analyticsmodel.NodeObsFrps) uint64 { return rows[len(rows)-1].ID }
func lastObsFrpcID(rows []analyticsmodel.NodeObsFrpc) uint64 { return rows[len(rows)-1].ID }
@@ -0,0 +1,358 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package tasks
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync/atomic"
"testing"
"time"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
)
var logDBSwitchDBSeq int64
// newLogDBSwitchDB 构造内存 sqlite 库(含日志 5 表 + 系统配置表)。
func newLogDBSwitchDB(t *testing.T) *gorm.DB {
t.Helper()
dsn := fmt.Sprintf("file:log-db-switch-%d?mode=memory&cache=shared", atomic.AddInt64(&logDBSwitchDBSeq, 1))
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
require.NoError(t, err)
require.NoError(t, gdb.AutoMigrate(
&model.SystemConfig{},
&analyticsmodel.NodeAccessLog{},
&analyticsmodel.NodeMetricSnapshot{},
&analyticsmodel.NodeEdgeHealth{},
&analyticsmodel.NodeObsFrps{},
&analyticsmodel.NodeObsFrpc{},
&analyticsmodel.UserAccessLog{},
))
return gdb
}
// TestCopyAccessLogsPreservesIDs sqlite→sqlite 模拟:源 store 3 条,目标空库,
// copyAccessLogs 后 ID 保留、数量一致。
func TestCopyAccessLogsPreservesIDs(t *testing.T) {
oldDB, oldCH := config.Config.Database.Enabled, config.Config.ClickHouse.Enabled
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = false, false
t.Cleanup(func() {
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = oldDB, oldCH
})
logstore.ResetForTest()
defer logstore.ResetForTest()
ctx := context.Background()
srcDB := newLogDBSwitchDB(t)
dstDB := newLogDBSwitchDB(t)
db.SetDB(srcDB)
src, err := logstore.Active(ctx) // 无 reader 时按 seed 规则解析为 sqlite
require.NoError(t, err)
db.SetDB(dstDB)
dst, err := logstore.BuildForMigration(ctx, "sqlite")
require.NoError(t, err)
t.Cleanup(func() { db.SetDB(nil) })
now := time.Now().UTC()
rows := []analyticsmodel.NodeAccessLog{
{ID: 101, NodeID: "n1", LoggedAt: now, RemoteAddr: "1.1.1.1", Host: "a.example.com", Path: "/"},
{ID: 202, NodeID: "n2", LoggedAt: now, RemoteAddr: "2.2.2.2", Host: "b.example.com", Path: "/x"},
{ID: 303, NodeID: "n1", LoggedAt: now, RemoteAddr: "3.3.3.3", Host: "c.example.com", Path: "/y"},
}
require.NoError(t, src.AccessLogs.BatchInsertNodeAccessLogs(ctx, rows))
require.NoError(t, copyAccessLogs(ctx, src, dst))
var got []analyticsmodel.NodeAccessLog
require.NoError(t, dstDB.Order("id ASC").Find(&got).Error)
require.Len(t, got, 3)
for i, wantID := range []uint64{101, 202, 303} {
assert.Equal(t, wantID, got[i].ID, "row %d id preserved", i)
}
assert.Equal(t, "n1", got[0].NodeID)
assert.Equal(t, "n2", got[1].NodeID)
assert.Equal(t, "n1", got[2].NodeID)
assert.Equal(t, "1.1.1.1", got[0].RemoteAddr)
// 源库保持不变。
var srcCount int64
require.NoError(t, srcDB.Model(&analyticsmodel.NodeAccessLog{}).Count(&srcCount).Error)
assert.Equal(t, int64(3), srcCount)
}
// TestCopyUserAccessLogsPreservesIDs sqlite→sqlite 模拟:源库用户访问日志按 id 升序
// 复制到目标库,ID 保留、数量一致,且源库保持不变。
func TestCopyUserAccessLogsPreservesIDs(t *testing.T) {
oldDB, oldCH := config.Config.Database.Enabled, config.Config.ClickHouse.Enabled
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = false, false
t.Cleanup(func() {
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = oldDB, oldCH
})
logstore.ResetForTest()
defer logstore.ResetForTest()
ctx := context.Background()
srcDB := newLogDBSwitchDB(t)
dstDB := newLogDBSwitchDB(t)
db.SetDB(srcDB)
src, err := logstore.Active(ctx)
require.NoError(t, err)
db.SetDB(dstDB)
dst, err := logstore.BuildForMigration(ctx, "sqlite")
require.NoError(t, err)
t.Cleanup(func() { db.SetDB(nil) })
now := time.Now().UTC()
rows := []analyticsmodel.UserAccessLog{
{ID: 11, UserID: 1, Path: "/a", CreatedAt: now},
{ID: 22, UserID: 2, Path: "/b", CreatedAt: now.Add(time.Second)},
{ID: 33, UserID: 1, Path: "/c", CreatedAt: now.Add(2 * time.Second)},
}
require.NoError(t, src.UserAccessLogs.BatchInsert(ctx, rows))
require.NoError(t, copyUserAccessLogs(ctx, src, dst))
var got []analyticsmodel.UserAccessLog
require.NoError(t, dstDB.Order("id ASC").Find(&got).Error)
require.Len(t, got, 3)
for i, wantID := range []uint64{11, 22, 33} {
assert.Equal(t, wantID, got[i].ID, "row %d id preserved", i)
}
var srcCount int64
require.NoError(t, srcDB.Model(&analyticsmodel.UserAccessLog{}).Count(&srcCount).Error)
assert.Equal(t, int64(3), srcCount)
}
// TestClearTargetLogTablesClearsUserAccessLogs 验证清空目标包含用户访问日志表
// (6 张日志表之一),迁移「覆盖目标库已有日志」幂等前提成立。
func TestClearTargetLogTablesClearsUserAccessLogs(t *testing.T) {
oldDB, oldCH := config.Config.Database.Enabled, config.Config.ClickHouse.Enabled
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = false, false
t.Cleanup(func() {
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = oldDB, oldCH
})
logstore.ResetForTest()
defer logstore.ResetForTest()
ctx := context.Background()
dstDB := newLogDBSwitchDB(t)
db.SetDB(dstDB)
t.Cleanup(func() { db.SetDB(nil) })
dst, err := logstore.BuildForMigration(ctx, "sqlite")
require.NoError(t, err)
now := time.Now().UTC()
require.NoError(t, dst.UserAccessLogs.BatchInsert(ctx, []analyticsmodel.UserAccessLog{
{ID: 1, UserID: 1, Path: "/a", CreatedAt: now},
{ID: 2, UserID: 2, Path: "/b", CreatedAt: now},
}))
require.NoError(t, clearTargetLogTables(ctx, dst))
var count int64
require.NoError(t, dstDB.Model(&analyticsmodel.UserAccessLog{}).Count(&count).Error)
assert.Zero(t, count, "用户访问日志应被清空")
}
// TestValidateSwitch 各非法组合报错。
func TestValidateSwitch(t *testing.T) {
oldDB, oldCH := config.Config.Database.Enabled, config.Config.ClickHouse.Enabled
t.Cleanup(func() {
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = oldDB, oldCH
})
gdb := newLogDBSwitchDB(t)
db.SetDB(gdb)
t.Cleanup(func() { db.SetDB(nil) })
ctx := context.Background()
setLogDB := func(v string) {
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDatabase, v))
}
t.Run("same target rejected", func(t *testing.T) {
setLogDB("sqlite")
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = false, false
err := validateSwitch(ctx, "sqlite")
require.Error(t, err)
assert.Contains(t, err.Error(), "相同")
})
t.Run("clickhouse disabled rejected", func(t *testing.T) {
setLogDB("sqlite")
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = false, false
err := validateSwitch(ctx, "clickhouse")
require.Error(t, err)
assert.Contains(t, err.Error(), "ClickHouse 未启用")
})
t.Run("postgres requires main db enabled", func(t *testing.T) {
setLogDB("sqlite")
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = false, false
err := validateSwitch(ctx, "postgres")
require.Error(t, err)
assert.Contains(t, err.Error(), "PostgreSQL 未启用")
})
t.Run("sqlite rejected when main db is postgres", func(t *testing.T) {
setLogDB("postgres")
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = true, false
err := validateSwitch(ctx, "sqlite")
require.Error(t, err)
assert.Contains(t, err.Error(), "SQLite")
})
t.Run("valid postgres migration", func(t *testing.T) {
setLogDB("sqlite")
config.Config.Database.Enabled, config.Config.ClickHouse.Enabled = true, false
require.NoError(t, validateSwitch(ctx, "postgres"))
})
}
// TestLogDBSwitchValidatePayload 参数归一化与非法值拒绝。
func TestLogDBSwitchValidatePayload(t *testing.T) {
h := &LogDBSwitchHandler{}
cases := []struct {
name string
in string
want string
ok bool
}{
{name: "postgresql normalized", in: `{"target":"postgresql"}`, want: "postgres", ok: true},
{name: "sqlite3 normalized", in: `{"target":"sqlite3"}`, want: "sqlite", ok: true},
{name: "ch normalized", in: `{"target":"ch"}`, want: "clickhouse", ok: true},
{name: "postgres passthrough", in: `{"target":"postgres"}`, want: "postgres", ok: true},
{name: "invalid target", in: `{"target":"mysql"}`, ok: false},
{name: "malformed json", in: `not-json`, ok: false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
out, err := h.ValidatePayload([]byte(c.in))
if !c.ok {
require.Error(t, err)
return
}
require.NoError(t, err)
var p logDBSwitchPayload
require.NoError(t, json.Unmarshal(out, &p))
assert.Equal(t, c.want, p.Target)
})
}
}
// TestExecuteFailureClearsMigrationFlag 迁移失败后 log_db_migration 冻结标记被清除。
// 在 FRESH DB(不预置 log_db_migration 行)上验证:setMigrationFlag 必须 upsert 建行,
// 且失败后经缓存路径(GetSystemConfigByKey)可观察为空。
func TestExecuteFailureClearsMigrationFlag(t *testing.T) {
oldDB := config.Config.Database.Enabled
config.Config.Database.Enabled = true
t.Cleanup(func() { config.Config.Database.Enabled = oldDB })
logstore.ResetForTest()
defer logstore.ResetForTest()
gdb := newLogDBSwitchDB(t)
db.SetDB(gdb)
t.Cleanup(func() { db.SetDB(nil) })
ctx := context.Background()
// FRESH DB:log_db_migration 行不存在(不预置),log_database 预置为 sqlite。
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDatabase, "sqlite"))
_, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDBMigration)
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
// configReader 对 log_database 报错,使 logstore.Active 在冻结标记置位后失败。
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == model.ConfigKeyLogDatabase {
return "", errors.New("reader error")
}
return "", nil
})
_, err = (&LogDBSwitchHandler{}).Execute(ctx, []byte(`{"target":"postgres"}`))
require.Error(t, err)
assert.Contains(t, err.Error(), "reader error")
// 冻结标记必须被 upsert 持久化(行存在)并经缓存路径可观察为空,源库恢复可写。
cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDBMigration)
require.NoError(t, err, "setMigrationFlag 应 upsert 创建 log_db_migration 行")
assert.Empty(t, cfg.Value, "失败后冻结标记必须清除,源库保持可写")
assert.False(t, logstore.Migrating(ctx))
}
// TestSetMigrationFlagObservableThroughCache 在 FRESH DB 上验证 setMigrationFlag 写入
// 经缓存路径(logstore.Migrating → repository 读取)实时反映:置位 true、清除 false。
func TestSetMigrationFlagObservableThroughCache(t *testing.T) {
logstore.ResetForTest()
defer logstore.ResetForTest()
gdb := newLogDBSwitchDB(t)
db.SetDB(gdb)
t.Cleanup(func() { db.SetDB(nil) })
ctx := context.Background()
// 按 bootstrap 同款注入 repository 读取,走 RAM 缓存路径。
logstore.SetConfigReader(func(ctx context.Context, key string) (string, error) {
cfg, err := repository.GetSystemConfigByKey(ctx, key)
if err != nil {
return "", err
}
return cfg.Value, nil
})
// FRESH DB:行缺失 → fail-open false。
assert.False(t, logstore.Migrating(ctx))
require.NoError(t, setMigrationFlag(ctx, "migrating"))
assert.True(t, logstore.Migrating(ctx), "置位后缓存路径必须立即观察到 migrating")
require.NoError(t, setMigrationFlag(ctx, ""))
assert.False(t, logstore.Migrating(ctx), "清除后缓存路径必须立即观察到非 migrating")
}
// TestFlipLogDatabaseRefreshesCachedConfig 验证翻转日志主库后缓存路径立即反映新库
// (logstore.ActiveDatabase / GetSystemConfigByKey),防止各进程继续写旧库(split-brain)。
func TestFlipLogDatabaseRefreshesCachedConfig(t *testing.T) {
logstore.ResetForTest()
defer logstore.ResetForTest()
gdb := newLogDBSwitchDB(t)
db.SetDB(gdb)
t.Cleanup(func() { db.SetDB(nil) })
ctx := context.Background()
logstore.SetConfigReader(func(ctx context.Context, key string) (string, error) {
cfg, err := repository.GetSystemConfigByKey(ctx, key)
if err != nil {
return "", err
}
return cfg.Value, nil
})
require.NoError(t, repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDatabase, "sqlite"))
active, err := logstore.ActiveDatabase(ctx)
require.NoError(t, err)
assert.Equal(t, "sqlite", active) // 预热缓存
require.NoError(t, flipLogDatabase(ctx, "postgres"))
active, err = logstore.ActiveDatabase(ctx)
require.NoError(t, err)
assert.Equal(t, "postgres", active, "翻转后缓存路径必须立即反映新库")
cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
require.NoError(t, err)
assert.Equal(t, "postgres", cfg.Value)
}
@@ -12,6 +12,7 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -34,9 +35,8 @@ func setupIPGroupSyncTestDB(t *testing.T) func() {
))
db.SetDB(sqliteDB)
resetAccessLogStore := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
testhelper.SetupLogStoresForTest(t)
return func() {
resetAccessLogStore()
db.SetDB(nil)
}
}
+2 -2
View File
@@ -9,6 +9,7 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/testhelper"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
@@ -61,8 +62,7 @@ func TestLegacyImportUsesEffectiveTLDPlusOne(t *testing.T) {
func TestGetStatsAggregatesZoneHosts(t *testing.T) {
ctx := setupZoneDB(t)
reset := repository.SetAccessLogStoreForTest(repository.NewMemoryAccessLogStore())
t.Cleanup(reset)
testhelper.SetupLogStoresForTest(t)
zone, err := Create(ctx, Input{Domain: "example.com"})
require.NoError(t, err)
+43 -10
View File
@@ -8,11 +8,10 @@ import (
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/platform/lifecycle"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
@@ -26,12 +25,10 @@ var (
logWriter *batchwriter.Writer[*analytics.UserAccessLog]
)
// InitLogWriter initializes the ClickHouse access-log batch writer.
// InitLogWriter initializes the user access-log batch writer.
// The active log store is resolved via logstore at flush time, so the writer
// runs for PG/SQLite as well as ClickHouse.
func InitLogWriter(ctx context.Context) {
if !config.Config.ClickHouse.Enabled {
return
}
logWriterMu.Lock()
defer logWriterMu.Unlock()
if logWriter != nil {
@@ -49,7 +46,11 @@ func InitLogWriter(ctx context.Context) {
}
rows = append(rows, *item)
}
return analyticsrepo.BatchInsert(ctx, rows)
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.UserAccessLogs.BatchInsert(ctx, rows)
},
batchwriter.WithDropHandler[*analytics.UserAccessLog](func(item *analytics.UserAccessLog) {
path := ""
@@ -59,7 +60,7 @@ func InitLogWriter(ctx context.Context) {
logger.WarnF(context.Background(), "[RiskControl] Log queue full, dropping log item for path: %s", path)
}),
batchwriter.WithFlushErrorHandler[*analytics.UserAccessLog](func(ctx context.Context, items []*analytics.UserAccessLog, err error) {
logger.ErrorF(ctx, "[RiskControl] Send ClickHouse batch failed (batch=%d): %v", len(items), err)
logger.ErrorF(ctx, "[RiskControl] Send log batch failed (batch=%d): %v", len(items), err)
}),
)
if err != nil {
@@ -72,7 +73,7 @@ func InitLogWriter(ctx context.Context) {
lifecycle.OnShutdown("risk_control_log_writer", StopLogWriter)
}
// StopLogWriter stops the ClickHouse access-log batch writer and drains pending logs.
// StopLogWriter stops the user access-log batch writer and drains pending logs.
func StopLogWriter(ctx context.Context) error {
writer := currentLogWriter()
if writer == nil {
@@ -81,6 +82,38 @@ func StopLogWriter(ctx context.Context) error {
return writer.Stop(ctx)
}
// DrainLogWriter 等待用户访问日志 writer 的在途批次落库:队列 Depth 归零后
// 再保持一个 flush 周期(1s)持续为空才返回;不停止 writer(迁移冻结后由
// ensureWritable 拒绝新写入)。writer 未初始化时直接返回 nil。
func DrainLogWriter(ctx context.Context) error {
writer := currentLogWriter()
if writer == nil {
return nil
}
ticker := time.NewTicker(drainPollInterval)
defer ticker.Stop()
var quietSince time.Time
for {
if writer.Stats().Depth == 0 {
if quietSince.IsZero() {
quietSince = time.Now()
} else if time.Since(quietSince) >= batchwriter.DefaultConfig().FlushInterval {
return nil
}
} else {
quietSince = time.Time{}
}
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
}
// drainPollInterval 用户访问日志队列轮询间隔。
const drainPollInterval = 50 * time.Millisecond
// IsBufferFull reports whether the access-log queue has no remaining capacity.
func IsBufferFull() bool {
writer := currentLogWriter()
+5 -3
View File
@@ -13,11 +13,12 @@ import (
"time"
"github.com/Rain-kl/Wavelet/internal/apps/oauth"
"github.com/Rain-kl/Wavelet/internal/infra/config"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/internal/shared/response"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
)
@@ -79,8 +80,9 @@ func hashAuditLogSensitiveValue(value string) string {
// RiskControlMiddleware 全局日志采集中间件
func RiskControlMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
// 如果未启用 ClickHouse,直接放行
if !config.Config.ClickHouse.Enabled {
// 日志库迁移冻结期跳过采集,不阻断业务请求。
if logstore.Migrating(c.Request.Context()) {
logger.WarnF(c.Request.Context(), "[RiskControl] log DB migrating, skip audit log")
c.Next()
return
}
+13
View File
@@ -18,6 +18,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
@@ -129,6 +130,18 @@ func (h *SystemCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.Tas
)
}
task.AppendLog(ctx, "开始清理过期日志(按当前日志库保留天数)...")
summary, err := logstore.CleanupExpired(ctx)
switch {
case err != nil:
logger.ErrorF(ctx, "清理过期日志失败: %v", err)
task.AppendLog(ctx, "清理过期日志失败: %v", err)
case summary.Deleted == 0:
task.AppendLog(ctx, "没有需要清理的过期日志 (保留 %d 天)", summary.RetentionDays)
default:
task.AppendLog(ctx, "日志清理完成:保留 %d 天,删除 %d 条", summary.RetentionDays, summary.Deleted)
}
msg := fmt.Sprintf("系统清理完成。成功清理未使用的上传文件 %d/%d 个;清理历史推送审计日志 %d 条;清理任务执行日志 %d 条。",
totalDeleted,
totalProcessed,
@@ -11,6 +11,8 @@ import (
"sync"
"sync/atomic"
"time"
"github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// FlushFunc persists a batch of queued items. It is invoked from the worker goroutine.
@@ -22,14 +24,8 @@ type FlushFunc[T any] func(ctx context.Context, items []T) error
type FlushErrorHandler[T any] func(ctx context.Context, items []T, err error)
// Stats is a point-in-time snapshot of Writer queue and failure counters.
type Stats struct {
Name string `json:"name"`
Depth int `json:"depth"`
Cap int `json:"cap"`
Drops int64 `json:"drops"`
FlushErrors int64 `json:"flush_errors"`
Running bool `json:"running"`
}
// It is an alias of analyticsmodel.BatchWriterStats (moved to keep model pure data).
type Stats = analytics.BatchWriterStats
// Writer buffers items and flushes them by size or interval.
type Writer[T any] struct {
@@ -0,0 +1,115 @@
-- +goose Up
-- 节点访问日志:按月 RANGE 分区,复合主键 (id, logged_at) 满足分区键进唯一索引要求。
CREATE TABLE IF NOT EXISTS of_node_access_logs (
id BIGINT NOT NULL,
node_id VARCHAR(64) NOT NULL DEFAULT '',
logged_at TIMESTAMPTZ NOT NULL,
remote_addr VARCHAR(128) NOT NULL DEFAULT '',
region VARCHAR(128) NOT NULL DEFAULT '',
host VARCHAR(255) NOT NULL DEFAULT '',
path VARCHAR(2048) NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
cache_status VARCHAR(64) NOT NULL DEFAULT '',
status_code INTEGER NOT NULL DEFAULT 0,
bytes_sent BIGINT NOT NULL DEFAULT 0,
request_length BIGINT NOT NULL DEFAULT 0,
request_time_ms INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, logged_at)
) PARTITION BY RANGE (logged_at);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_node_id ON of_node_access_logs (node_id, logged_at DESC);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_host ON of_node_access_logs (host, logged_at DESC);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_remote_addr ON of_node_access_logs (remote_addr, logged_at DESC);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_status_code ON of_node_access_logs (status_code, logged_at DESC);
-- 用户访问日志:按月分区。
CREATE TABLE IF NOT EXISTS w_user_access_logs (
id BIGINT NOT NULL,
user_id BIGINT NOT NULL DEFAULT 0,
path VARCHAR(2048) NOT NULL DEFAULT '',
method VARCHAR(16) NOT NULL DEFAULT '',
ip VARCHAR(128) NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
headers TEXT NOT NULL DEFAULT '',
status INTEGER NOT NULL DEFAULT 0,
latency BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, created_at)
) PARTITION BY RANGE (created_at);
CREATE INDEX IF NOT EXISTS idx_w_user_access_logs_user_id ON w_user_access_logs (user_id, created_at DESC);
-- 可观测 4 表:普通表 + (node_id, captured_at DESC) 索引。
CREATE TABLE IF NOT EXISTS of_node_metric_snapshots (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
cpu_usage_percent DOUBLE PRECISION NOT NULL DEFAULT 0,
memory_used_bytes BIGINT NOT NULL DEFAULT 0,
memory_total_bytes BIGINT NOT NULL DEFAULT 0,
storage_used_bytes BIGINT NOT NULL DEFAULT 0,
storage_total_bytes BIGINT NOT NULL DEFAULT 0,
disk_read_bytes BIGINT NOT NULL DEFAULT 0,
disk_write_bytes BIGINT NOT NULL DEFAULT 0,
network_rx_bytes BIGINT NOT NULL DEFAULT 0,
network_tx_bytes BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_metric_snapshots_node ON of_node_metric_snapshots (node_id, captured_at DESC);
CREATE TABLE IF NOT EXISTS of_node_edge_health (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
status VARCHAR(64) NOT NULL DEFAULT '',
connections BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_edge_health_node ON of_node_edge_health (node_id, captured_at DESC);
CREATE TABLE IF NOT EXISTS of_node_obs_frps (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
frps_connections INTEGER NOT NULL DEFAULT 0,
frps_proxy_count INTEGER NOT NULL DEFAULT 0,
frps_client_count INTEGER NOT NULL DEFAULT 0,
frps_proxies TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_obs_frps_node ON of_node_obs_frps (node_id, captured_at DESC);
CREATE TABLE IF NOT EXISTS of_node_obs_frpc (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
tunnel_status VARCHAR(16) NOT NULL DEFAULT '',
connected_relays_count INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_obs_frpc_node ON of_node_obs_frpc (node_id, captured_at DESC);
-- 分区预建:创建当月及未来 2 个月分区(共 3 个月)。
-- +goose StatementBegin
DO $$
DECLARE
d date;
BEGIN
FOR d IN SELECT generate_series(date_trunc('month', now())::date, (date_trunc('month', now()) + interval '2 months')::date, interval '1 month')::date
LOOP
EXECUTE format('CREATE TABLE IF NOT EXISTS of_node_access_logs_%s PARTITION OF of_node_access_logs FOR VALUES FROM (%L) TO (%L)',
to_char(d, 'YYYYMM'), d, d + interval '1 month');
EXECUTE format('CREATE TABLE IF NOT EXISTS w_user_access_logs_%s PARTITION OF w_user_access_logs FOR VALUES FROM (%L) TO (%L)',
to_char(d, 'YYYYMM'), d, d + interval '1 month');
END LOOP;
END $$;
-- +goose StatementEnd
-- +goose Down
DROP TABLE IF EXISTS w_user_access_logs;
DROP TABLE IF EXISTS of_node_access_logs;
DROP TABLE IF EXISTS of_node_metric_snapshots;
DROP TABLE IF EXISTS of_node_edge_health;
DROP TABLE IF EXISTS of_node_obs_frps;
DROP TABLE IF EXISTS of_node_obs_frpc;
@@ -0,0 +1,22 @@
-- +goose Up
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
SELECT k, COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, descr, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
FROM (VALUES
('log_retention_days_postgres', 'PostgreSQL 日志保留天数(访问日志与可观测统一)'),
('log_retention_days_sqlite', 'SQLite 日志保留天数'),
('log_retention_days_clickhouse', 'ClickHouse 日志保留天数')
) AS v(k, descr)
ON CONFLICT (key) DO NOTHING;
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
-- +goose Down
INSERT INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
VALUES
('database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP),
('database_auto_cleanup_retention_days', COALESCE((SELECT value FROM w_system_configs WHERE key = 'log_retention_days_postgres'), '30'), 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (key) DO NOTHING;
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
@@ -0,0 +1,19 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_schedules_backup_of_database_auto_cleanup AS
SELECT * FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
-- +goose Down
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at
FROM w_schedules_backup_of_database_auto_cleanup
ON CONFLICT (id) DO NOTHING;
INSERT INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT 102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
WHERE NOT EXISTS (SELECT 1 FROM w_schedules WHERE task_type = 'of_database_auto_cleanup')
ON CONFLICT (id) DO NOTHING;
DROP TABLE IF EXISTS w_schedules_backup_of_database_auto_cleanup;
@@ -0,0 +1,93 @@
-- +goose Up
-- 节点访问日志:普通表(同 PG 语义,索引名保持一致)。
CREATE TABLE IF NOT EXISTS of_node_access_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id TEXT NOT NULL DEFAULT '',
logged_at DATETIME NOT NULL,
remote_addr TEXT NOT NULL DEFAULT '',
region TEXT NOT NULL DEFAULT '',
host TEXT NOT NULL DEFAULT '',
path TEXT NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
cache_status TEXT NOT NULL DEFAULT '',
status_code INTEGER NOT NULL DEFAULT 0,
bytes_sent INTEGER NOT NULL DEFAULT 0,
request_length INTEGER NOT NULL DEFAULT 0,
request_time_ms INTEGER NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_node_id ON of_node_access_logs (node_id, logged_at DESC);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_host ON of_node_access_logs (host, logged_at DESC);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_remote_addr ON of_node_access_logs (remote_addr, logged_at DESC);
CREATE INDEX IF NOT EXISTS idx_of_node_access_logs_status_code ON of_node_access_logs (status_code, logged_at DESC);
CREATE TABLE IF NOT EXISTS w_user_access_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL DEFAULT 0,
path TEXT NOT NULL DEFAULT '',
method TEXT NOT NULL DEFAULT '',
ip TEXT NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
headers TEXT NOT NULL DEFAULT '',
status INTEGER NOT NULL DEFAULT 0,
latency INTEGER NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_w_user_access_logs_user_id ON w_user_access_logs (user_id, created_at DESC);
CREATE TABLE IF NOT EXISTS of_node_metric_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id TEXT NOT NULL DEFAULT '',
captured_at DATETIME NOT NULL,
cpu_usage_percent REAL NOT NULL DEFAULT 0,
memory_used_bytes INTEGER NOT NULL DEFAULT 0,
memory_total_bytes INTEGER NOT NULL DEFAULT 0,
storage_used_bytes INTEGER NOT NULL DEFAULT 0,
storage_total_bytes INTEGER NOT NULL DEFAULT 0,
disk_read_bytes INTEGER NOT NULL DEFAULT 0,
disk_write_bytes INTEGER NOT NULL DEFAULT 0,
network_rx_bytes INTEGER NOT NULL DEFAULT 0,
network_tx_bytes INTEGER NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_metric_snapshots_node ON of_node_metric_snapshots (node_id, captured_at DESC);
CREATE TABLE IF NOT EXISTS of_node_edge_health (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id TEXT NOT NULL DEFAULT '',
captured_at DATETIME NOT NULL,
status TEXT NOT NULL DEFAULT '',
connections INTEGER NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_edge_health_node ON of_node_edge_health (node_id, captured_at DESC);
CREATE TABLE IF NOT EXISTS of_node_obs_frps (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id TEXT NOT NULL DEFAULT '',
captured_at DATETIME NOT NULL,
frps_connections INTEGER NOT NULL DEFAULT 0,
frps_proxy_count INTEGER NOT NULL DEFAULT 0,
frps_client_count INTEGER NOT NULL DEFAULT 0,
frps_proxies TEXT NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_obs_frps_node ON of_node_obs_frps (node_id, captured_at DESC);
CREATE TABLE IF NOT EXISTS of_node_obs_frpc (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id TEXT NOT NULL DEFAULT '',
captured_at DATETIME NOT NULL,
tunnel_status TEXT NOT NULL DEFAULT '',
connected_relays_count INTEGER NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_of_node_obs_frpc_node ON of_node_obs_frpc (node_id, captured_at DESC);
-- +goose Down
DROP TABLE IF EXISTS of_node_obs_frpc;
DROP TABLE IF EXISTS of_node_obs_frps;
DROP TABLE IF EXISTS of_node_edge_health;
DROP TABLE IF EXISTS of_node_metric_snapshots;
DROP TABLE IF EXISTS w_user_access_logs;
DROP TABLE IF EXISTS of_node_access_logs;
@@ -0,0 +1,19 @@
-- +goose Up
-- 日志保留天数配置(business),替换旧的 database_auto_cleanup_* 键。
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
SELECT 'log_retention_days_postgres', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'PostgreSQL 日志保留天数(访问日志与可观测统一)', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
UNION ALL
SELECT 'log_retention_days_sqlite', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'SQLite 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
UNION ALL
SELECT 'log_retention_days_clickhouse', COALESCE((SELECT value FROM w_system_configs WHERE key = 'database_auto_cleanup_retention_days'), '90'), 'business', 0, 'ClickHouse 日志保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP;
DELETE FROM w_system_configs WHERE key IN ('database_auto_cleanup_enabled', 'database_auto_cleanup_retention_days');
-- +goose Down
INSERT OR IGNORE INTO w_system_configs (key, value, type, visibility, description, created_at, updated_at)
SELECT 'database_auto_cleanup_enabled', 'true', 'business', 0, '数据库自动清理开关', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
UNION ALL
SELECT 'database_auto_cleanup_retention_days', COALESCE((SELECT value FROM w_system_configs WHERE key = 'log_retention_days_sqlite'), '30'), 'business', 0, '数据库保留天数', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP;
DELETE FROM w_system_configs WHERE key IN ('log_retention_days_postgres', 'log_retention_days_sqlite', 'log_retention_days_clickhouse');
@@ -0,0 +1,17 @@
-- +goose Up
CREATE TABLE IF NOT EXISTS w_schedules_backup_of_database_auto_cleanup AS
SELECT * FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
DELETE FROM w_schedules WHERE task_type = 'of_database_auto_cleanup';
-- +goose Down
INSERT OR IGNORE INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT id, name, task_type, cron, payload, is_active, created_at, updated_at
FROM w_schedules_backup_of_database_auto_cleanup;
INSERT OR IGNORE INTO w_schedules (id, name, task_type, cron, payload, is_active, created_at, updated_at)
SELECT 102, 'OpenFlare 可观测数据自动清理', 'of_database_auto_cleanup', '0 3 * * *', '{}', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
WHERE NOT EXISTS (SELECT 1 FROM w_schedules WHERE task_type = 'of_database_auto_cleanup');
DROP TABLE IF EXISTS w_schedules_backup_of_database_auto_cleanup;
@@ -21,10 +21,8 @@ import (
// expectedMigratedSystemConfigCount 包含初始 32 项系统配置、202606220004
// 从 of_options 迁移过来的 48 项业务配置、Pages 的 2 项业务配置、
// OpenResty 默认限流的 3 项业务配置、单 IP 请求频率限制 1 项业务配置、
// 源站错误页的 4 项业务配置,以及 Service Worker 离线兜底的 2 项业务配置、
// SW 离线兜底生效域名的 1 项业务配置。
const expectedMigratedSystemConfigCount = 93
// OpenResty 默认限流的 3 项业务配置,以及单 IP 请求频率限制 1 项业务配置。
const expectedMigratedSystemConfigCount = 86
func TestMigrateInitializesSQLiteDatabase(t *testing.T) {
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
+4 -3
View File
@@ -10,6 +10,7 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/openflare"
cf "github.com/Rain-kl/Wavelet/internal/apps/openflare/cloudflare"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/pages"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/tasks"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/tls"
"github.com/Rain-kl/Wavelet/internal/apps/upload"
"github.com/Rain-kl/Wavelet/internal/apps/user"
@@ -44,15 +45,15 @@ func Register() {
task.RegisterHandler(openflare.SSLRenewTask, &openflare.SSLRenewHandler{})
task.RegisterTaskMeta(openflare.SSLRenewMeta)
task.RegisterHandler(openflare.DatabaseAutoCleanupTask, &openflare.DatabaseAutoCleanupHandler{})
task.RegisterTaskMeta(openflare.DatabaseAutoCleanupMeta)
task.RegisterHandler(openflare.WAFIPGroupSyncTask, &openflare.WAFIPGroupSyncHandler{})
task.RegisterTaskMeta(openflare.WAFIPGroupSyncMeta)
task.RegisterHandler(openflare.UptimeKumaSyncTask, &openflare.UptimeKumaSyncHandler{})
task.RegisterTaskMeta(openflare.UptimeKumaSyncMeta)
task.RegisterHandler(openflare.LogDBSwitchTask, &tasks.LogDBSwitchHandler{})
task.RegisterTaskMeta(openflare.LogDBSwitchMeta)
task.RegisterHandler(cf.SyncMemberTask, &cf.SyncMemberTaskHandler{})
task.RegisterTaskMeta(cf.SyncMemberMeta)
task.RegisterHandler(cf.SyncGroupTask, &cf.SyncGroupTaskHandler{})
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics defines ClickHouse analytics domain models and query DTOs
// (pure data, no IO).
package analytics
import "time"
// AccessLogFilter scopes user access log queries.
// 单一权威字段集(CH 原字段,Task 1 迁入):禁止追加仅某实现使用的字段(避免双字段集分叉)。
type AccessLogFilter struct {
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
UserIDs []uint64
Path string
// StartTime filters created_at >= StartTime when non-nil.
StartTime *time.Time
// EndTime filters created_at <= EndTime when non-nil(闭区间,与 CH/GORM 实现一致)。
EndTime *time.Time
}
// NodeAccessLogFilter scopes ClickHouse node access log queries.
type NodeAccessLogFilter struct {
NodeID string
RemoteAddr string
Host string
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
Hosts []string
Path string
Since time.Time
Until time.Time
Page int
PageSize int
SortBy string
SortOrder string
}
// NodeObservabilityFilter scopes ClickHouse node observability queries.
type NodeObservabilityFilter struct {
NodeID string
Since time.Time
Limit int
}
// DailyTrend is a single day's access count.
type DailyTrend struct {
Date string
Count uint64
}
// BrowserShare is a browser group's share of access logs.
type BrowserShare struct {
Browser string
Count uint64
}
// TopUser is an active user ranked by access count.
type TopUser struct {
UserID uint64
Count uint64
}
// NodeAccessLogRegionCount aggregates access log regions.
type NodeAccessLogRegionCount struct {
Region string
Count int64
}
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
type NodeAccessLogTrafficSummary struct {
RequestCount int64
ErrorCount int64
UniqueIPCount int64
BytesSent int64
RequestLength int64
NodeCount int64
}
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
type NodeAccessLogValueCount struct {
Value string
Count int64
}
// NodeAccessLogNodeAggregate is per-node traffic over a window.
type NodeAccessLogNodeAggregate struct {
NodeID string
RequestCount int64
ErrorCount int64
UniqueIPCount int64
}
// BatchWriterStats is a point-in-time snapshot of a batch writer queue and failure counters.
type BatchWriterStats struct {
Name string `json:"name"`
Depth int `json:"depth"`
Cap int `json:"cap"`
Drops int64 `json:"drops"`
FlushErrors int64 `json:"flush_errors"`
Running bool `json:"running"`
}
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
// and in-process batch writer queue health.
type ClickHouseOperationalStats struct {
Database string `json:"database"`
ActiveParts int64 `json:"active_parts"`
TotalRows int64 `json:"total_rows"`
PendingMutations int64 `json:"pending_mutations"`
AsyncInsertQueue int64 `json:"async_insert_queue"`
AsyncInsertBytes int64 `json:"async_insert_bytes"`
// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
BatchWriters []BatchWriterStats `json:"batch_writers,omitempty"`
}
@@ -90,6 +90,34 @@ type AccessLogHourly struct {
RequestLength int64 `gorm:"column:request_length"`
}
// NodeTrafficHourly is an hourly traffic rollup row.
//
// UniqueVisitorCount is always 0 when sourced from of_access_log_hourly
// (true UV requires raw uniqExact on access logs).
type NodeTrafficHourly struct {
NodeID string
Hour time.Time
RequestCount int64
ErrorCount int64
UniqueVisitorCount int64
}
// NodeMetricHourly is an hourly metric snapshot aggregation row.
//
// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
// lagInFrame samples per node (negative deltas after counter reset are dropped).
type NodeMetricHourly struct {
Hour time.Time
AverageCPUUsagePercent float64
AverageMemoryUsagePercent float64
NetworkRxBytes int64
NetworkTxBytes int64
DiskReadBytes int64
DiskWriteBytes int64
ReportedNodes int
}
// NodeObsFrps stores FRPS observability snapshots in ClickHouse.
type NodeObsFrps struct {
ID uint64 `gorm:"column:id"`
@@ -1,7 +1,6 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics defines ClickHouse analytics domain models.
package analytics
import (
+124
View File
@@ -0,0 +1,124 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import "strings"
// User-Agent 浏览器/OS/设备分类(纯函数,无 IO)。
// 与 internal/repository/analytics/browser.go 的判定逻辑保持一致(Task 4 复制,
// 因为 model 不得 import analyticsrepo);后续若移除旧 CH 实现,可让 analyticsrepo 改以别名复用本包。
const (
uaLabelUnknown = "Unknown"
uaLabelBot = "Bot"
uaLabelOther = "Other"
uaTokenBot = "bot"
uaTokenAndroid = "android"
uaTokenSpider = "spider"
uaTokenCrawler = "crawler"
)
type uaMatchRule struct {
label string
contains []string
allOf []string
noneOf []string
}
func matchUARules(uaLower string, rules []uaMatchRule, fallback string) string {
if uaLower == "" {
return uaLabelUnknown
}
for _, rule := range rules {
matched := false
for _, token := range rule.contains {
if strings.Contains(uaLower, token) {
matched = true
break
}
}
if !matched && len(rule.allOf) > 0 {
matched = true
for _, token := range rule.allOf {
if !strings.Contains(uaLower, token) {
matched = false
break
}
}
}
if !matched {
continue
}
excluded := false
for _, token := range rule.noneOf {
if strings.Contains(uaLower, token) {
excluded = true
break
}
}
if excluded {
continue
}
return rule.label
}
return fallback
}
var browserRules = []uaMatchRule{
{label: "WeChat", contains: []string{"micromessenger"}},
{label: "Postman", contains: []string{"postman"}},
{label: "CLI", contains: []string{"curl/", "wget/"}},
{label: "Edge", contains: []string{"edg/", "edgios/", "edga/"}},
{label: "Opera", contains: []string{"opr/", "opera"}},
{label: "Firefox", contains: []string{"firefox", "fxios"}},
{label: "Chrome", contains: []string{"crios", "chrome"}, noneOf: []string{"chromium"}},
{label: "Chromium", contains: []string{"chromium"}},
{label: "Safari", contains: []string{"safari"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp"}},
}
var osRules = []uaMatchRule{
{label: "Android", contains: []string{uaTokenAndroid}},
{label: "iOS", contains: []string{"iphone", "ipad", "ipod", "ios"}},
{label: "Windows", contains: []string{"windows"}},
{label: "macOS", contains: []string{"mac os x", "macintosh", "macos"}},
{label: "Chrome OS", contains: []string{"cros"}},
{label: "Linux", contains: []string{"linux"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler}},
}
var deviceRules = []uaMatchRule{
{
label: uaLabelBot,
contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp", "curl/", "wget/", "python-requests", "go-http-client", "postman"},
},
{
label: "Tablet",
contains: []string{"ipad", "tablet"},
},
{
label: "Tablet",
allOf: []string{uaTokenAndroid},
noneOf: []string{"mobile"},
},
{
label: "Mobile",
contains: []string{"mobi", "iphone", "ipod", uaTokenAndroid},
},
}
// ParseBrowserName performs lightweight User-Agent browser identification.
func ParseBrowserName(ua string) string {
return matchUARules(strings.ToLower(ua), browserRules, uaLabelOther)
}
// ParseOSName performs lightweight User-Agent OS identification.
func ParseOSName(ua string) string {
return matchUARules(strings.ToLower(ua), osRules, uaLabelOther)
}
// ParseDeviceType performs lightweight User-Agent device type identification.
func ParseDeviceType(ua string) string {
return matchUARules(strings.ToLower(ua), deviceRules, "Desktop")
}
+15 -8
View File
@@ -41,14 +41,12 @@ const (
ConfigKeyRelayFRPSWebUIPort = "relay_frps_web_ui_port" // FRPS 内置 Web 界面端口
// OpenFlare 业务配置(从 of_options 迁移)
ConfigKeyAgentDiscoveryToken = "agent_discovery_token" //nolint:gosec // false positive: config key name. Agent 发现令牌
ConfigKeyAgentHeartbeatInterval = "agent_heartbeat_interval" // Agent 心跳间隔(毫秒)
ConfigKeyAgentWebsocketUpgradeEnabled = "agent_websocket_upgrade_enabled" // Agent WebSocket 升级开关
ConfigKeyNodeOfflineThreshold = "node_offline_threshold" // 节点离线阈值(毫秒)
ConfigKeyAgentUpdateRepo = "agent_update_repo" // Agent 更新仓库
ConfigKeyGeoIPProvider = "geoip_provider" // GeoIP 服务商
ConfigKeyDatabaseAutoCleanupEnabled = "database_auto_cleanup_enabled" // 数据库自动清理开关
ConfigKeyDatabaseAutoCleanupRetentionDays = "database_auto_cleanup_retention_days" // 数据库保留天数
ConfigKeyAgentDiscoveryToken = "agent_discovery_token" //nolint:gosec // false positive: config key name. Agent 发现令牌
ConfigKeyAgentHeartbeatInterval = "agent_heartbeat_interval" // Agent 心跳间隔(毫秒)
ConfigKeyAgentWebsocketUpgradeEnabled = "agent_websocket_upgrade_enabled" // Agent WebSocket 升级开关
ConfigKeyNodeOfflineThreshold = "node_offline_threshold" // 节点离线阈值(毫秒)
ConfigKeyAgentUpdateRepo = "agent_update_repo" // Agent 更新仓库
ConfigKeyGeoIPProvider = "geoip_provider" // GeoIP 服务商
// Pages 静态托管配置
ConfigKeyPagesMaxPackageSizeMB = "pages_max_package_size_mb" // Pages 部署包上传大小上限(MiB)
@@ -122,6 +120,15 @@ const (
ConfigKeySWOfflineDomains = "sw_offline_domains" // 离线兜底生效域名列表(JSON 数组,空则仅总开关无效)
)
// 日志数据库解耦
const (
ConfigKeyLogDatabase = "log_database" // 当前日志主库:postgres|sqlite|clickhouse(仅迁移任务写入)
ConfigKeyLogDBMigration = "log_db_migration" // 迁移冻结标记:"migrating" 或空
ConfigKeyLogRetentionDaysPostgres = "log_retention_days_postgres" // PostgreSQL 日志保留天数
ConfigKeyLogRetentionDaysSQLite = "log_retention_days_sqlite" // SQLite 日志保留天数
ConfigKeyLogRetentionDaysClickHouse = "log_retention_days_clickhouse" // ClickHouse 日志保留天数
)
const (
// ConfigVisibilityHidden 表示配置不通过公共配置接口暴露
ConfigVisibilityHidden = 0
+63
View File
@@ -7,6 +7,9 @@ package bootstrap
import (
"context"
"errors"
"fmt"
"log"
"sync"
admin_push "github.com/Rain-kl/Wavelet/internal/apps/admin/push"
@@ -14,11 +17,15 @@ import (
"github.com/Rain-kl/Wavelet/internal/apps/openflare/chwriter"
ofgeoip "github.com/Rain-kl/Wavelet/internal/apps/openflare/geoip"
"github.com/Rain-kl/Wavelet/internal/apps/risk_control"
"github.com/Rain-kl/Wavelet/internal/infra/config"
taskhandlers "github.com/Rain-kl/Wavelet/internal/infra/task/handlers"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/platform/lifecycle"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/cache/ram"
"github.com/Rain-kl/Wavelet/pkg/logger"
"gorm.io/gorm"
)
// Options selects role-specific runtime bootstrap steps for the current process.
@@ -129,6 +136,21 @@ func RegisterAll() {
// Call from cmd entry points after wiring registration and database migration, not from router.
func Init(ctx context.Context, opts Options) {
initRuntimeOnce.Do(func() {
if err := validateAndSeedLogDatabase(ctx); err != nil {
logger.ErrorF(ctx, "[Bootstrap] 日志主库配置校验失败: %v", err)
log.Fatalf("[Bootstrap] 日志主库配置校验失败: %v", err)
}
// 注入 logstore 配置读取(避免 logstore ↔ repository 循环依赖),并预热激活 store。
logstore.SetConfigReader(func(ctx context.Context, key string) (string, error) {
cfg, err := repository.GetSystemConfigByKey(ctx, key)
if err != nil {
return "", err
}
return cfg.Value, nil
})
logstore.Init(ctx)
// Register config cache loader
RegisterCache(repository.ConfigCacheType, CacheRegistry{
Loader: repository.ConfigLoader{},
@@ -152,6 +174,47 @@ func Init(ctx context.Context, opts Options) {
})
}
// validateAndSeedLogDatabase 校验日志主库标记与运行配置的一致性,首次启动 seed。
func validateAndSeedLogDatabase(ctx context.Context) error {
cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("读取日志主库配置失败: %w", err)
}
current := cfg.Value
if current == "" {
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
current = "sqlite"
if config.Config.Database.Enabled {
current = "postgres"
}
if config.Config.ClickHouse.Enabled {
current = "clickhouse"
}
// 行缺失时 UpdateSystemConfigFields 仅为 UPDATE 无法插入,改用可创建可更新的 SaveOrUpdateSystemConfig。
if err := repository.SaveOrUpdateSystemConfig(ctx, model.ConfigKeyLogDatabase, current); err != nil {
return fmt.Errorf("初始化日志主库配置失败: %w", err)
}
return nil
}
switch current {
case "clickhouse":
if !config.Config.ClickHouse.Enabled {
return errors.New("当前日志主库为 ClickHouse 但 ClickHouse 未启用。请先重新启用 ClickHouse 配置并启动,在任务管理运行『切换日志数据库』迁移到 PostgreSQL/SQLite 后再禁用 ClickHouse")
}
case "postgres":
if !config.Config.Database.Enabled {
return errors.New("当前日志主库为 PostgreSQL 但 PostgreSQL 未启用(当前为 SQLite 主库)。请运行『切换日志数据库』迁回 SQLite 或启用 PostgreSQL")
}
case "sqlite":
if config.Config.Database.Enabled {
return errors.New("当前日志主库为 SQLite 但当前主库为 PostgreSQL。请运行『切换日志数据库』迁移到 PostgreSQL")
}
default:
return fmt.Errorf("未知的日志主库配置: %s", current)
}
return nil
}
// Stop stops all batch writers and background resources.
func Stop(ctx context.Context) {
lifecycle.Stop(ctx)
@@ -5,10 +5,15 @@ package bootstrap
import (
"context"
"strings"
"testing"
admin_push "github.com/Rain-kl/Wavelet/internal/apps/admin/push"
"github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/internal/testhelper"
)
@@ -50,3 +55,170 @@ func TestInitSyncsPushEventsOnce(t *testing.T) {
t.Fatalf("admin_login name = %q, want %q", adminLogin.Name, "管理员登录")
}
}
func TestValidateAndSeedLogDatabaseSeedsDefault(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
prevDB := config.Config.Database.Enabled
prevCH := config.Config.ClickHouse.Enabled
t.Cleanup(func() {
config.Config.Database.Enabled = prevDB
config.Config.ClickHouse.Enabled = prevCH
})
tests := []struct {
name string
dbEnabled bool
chEnabled bool
want string
}{
{name: "sqlite default", dbEnabled: false, chEnabled: false, want: "sqlite"},
{name: "postgres default", dbEnabled: true, chEnabled: false, want: "postgres"},
{name: "clickhouse default", dbEnabled: true, chEnabled: true, want: "clickhouse"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config.Config.Database.Enabled = tt.dbEnabled
config.Config.ClickHouse.Enabled = tt.chEnabled
ctx := context.Background()
// 清掉标记行,模拟首次启动。
if err := db.DB(ctx).Where("key = ?", model.ConfigKeyLogDatabase).Delete(&model.SystemConfig{}).Error; err != nil {
t.Fatalf("delete log_database marker failed: %v", err)
}
repository.ResetSystemConfigRAMCacheForTest()
if err := validateAndSeedLogDatabase(ctx); err != nil {
t.Fatalf("validateAndSeedLogDatabase() error = %v", err)
}
repository.ResetSystemConfigRAMCacheForTest()
cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
if err != nil {
t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err)
}
if cfg.Value != tt.want {
t.Fatalf("seeded log_database = %q, want %q", cfg.Value, tt.want)
}
})
}
}
func TestValidateAndSeedLogDatabaseUpdatesEmptyMarker(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
dbPrev := config.Config.Database.Enabled
chPrev := config.Config.ClickHouse.Enabled
config.Config.Database.Enabled = true
config.Config.ClickHouse.Enabled = false
t.Cleanup(func() {
config.Config.Database.Enabled = dbPrev
config.Config.ClickHouse.Enabled = chPrev
})
ctx := context.Background()
// 标记行已存在但值为空,等同首次启动,应写入默认值(走更新路径)。
if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: "", Type: "system"}); err != nil {
t.Fatalf("create empty marker failed: %v", err)
}
repository.ResetSystemConfigRAMCacheForTest()
if err := validateAndSeedLogDatabase(ctx); err != nil {
t.Fatalf("validateAndSeedLogDatabase() error = %v", err)
}
repository.ResetSystemConfigRAMCacheForTest()
cfg, err := repository.GetSystemConfigByKey(ctx, model.ConfigKeyLogDatabase)
if err != nil {
t.Fatalf("GetSystemConfigByKey(%s) error = %v", model.ConfigKeyLogDatabase, err)
}
want := "postgres"
if cfg.Value != want {
t.Fatalf("log_database = %q, want %q", cfg.Value, want)
}
}
func TestValidateAndSeedLogDatabaseRejectsInconsistentConfig(t *testing.T) {
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
prevDB := config.Config.Database.Enabled
prevCH := config.Config.ClickHouse.Enabled
t.Cleanup(func() {
config.Config.Database.Enabled = prevDB
config.Config.ClickHouse.Enabled = prevCH
})
seedMarker := func(t *testing.T, value string) {
t.Helper()
ctx := context.Background()
if err := db.DB(ctx).Where("key = ?", model.ConfigKeyLogDatabase).Delete(&model.SystemConfig{}).Error; err != nil {
t.Fatalf("delete log_database marker failed: %v", err)
}
if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDatabase, Value: value, Type: "system"}); err != nil {
t.Fatalf("create log_database marker failed: %v", err)
}
repository.ResetSystemConfigRAMCacheForTest()
}
tests := []struct {
name string
marker string
dbEnabled bool
chEnabled bool
wantErr string
}{
{name: "clickhouse marker but disabled", marker: "clickhouse", dbEnabled: true, chEnabled: false, wantErr: "ClickHouse 未启用"},
{name: "postgres marker but disabled", marker: "postgres", dbEnabled: false, chEnabled: false, wantErr: "PostgreSQL 未启用"},
{name: "sqlite marker but postgres primary", marker: "sqlite", dbEnabled: true, chEnabled: false, wantErr: "SQLite"},
{name: "unknown marker", marker: "mysql", dbEnabled: false, chEnabled: false, wantErr: "未知的日志主库配置"},
{name: "consistent sqlite", marker: "sqlite", dbEnabled: false, chEnabled: false, wantErr: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config.Config.Database.Enabled = tt.dbEnabled
config.Config.ClickHouse.Enabled = tt.chEnabled
seedMarker(t, tt.marker)
err := validateAndSeedLogDatabase(context.Background())
if tt.wantErr == "" {
if err != nil {
t.Fatalf("validateAndSeedLogDatabase() error = %v, want nil", err)
}
return
}
if err == nil {
t.Fatalf("validateAndSeedLogDatabase() = nil, want error containing %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("validateAndSeedLogDatabase() error = %q, want contains %q", err.Error(), tt.wantErr)
}
})
}
}
func TestInitWiresLogstoreConfigReader(t *testing.T) {
ResetInitRuntimeOnceForTest()
t.Cleanup(ResetInitRuntimeOnceForTest)
logstore.ResetForTest()
t.Cleanup(logstore.ResetForTest)
_, _, cleanup := testhelper.SetupTestEnvironment(t)
defer cleanup()
ctx := context.Background()
// 插入迁移标记:bootstrap 注入的 reader 应能经 repository 读到该值(区分未装配时的兜底行为)。
if err := repository.CreateSystemConfig(ctx, &model.SystemConfig{Key: model.ConfigKeyLogDBMigration, Value: "migrating", Type: "system"}); err != nil {
t.Fatalf("create log_db_migration marker failed: %v", err)
}
repository.ResetSystemConfigRAMCacheForTest()
Init(ctx, Options{})
repository.ResetSystemConfigRAMCacheForTest()
if !logstore.Migrating(ctx) {
t.Fatal("logstore config reader not wired after bootstrap.Init: Migrating() = false, want true")
}
}
@@ -38,6 +38,18 @@ func CountAccessLogs(ctx context.Context, filter AccessLogFilter) (uint64, error
return count, nil
}
// DeleteAllUserAccessLogs hard-deletes all user access logs via TRUNCATE.
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
if err := userAccessLogConn(); err != nil {
return 0, err
}
outcome, err := truncateClickHouseTable(ctx, db.ChConn, analyticsmodel.UserAccessLog{}.TableName())
if err != nil {
return 0, err
}
return outcome.DeletedCount, nil
}
// ListAccessLogs returns paginated access logs and the total match count.
func ListAccessLogs(ctx context.Context, filter AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
clause, args, ok := buildUserAccessLogFilterClause(filter)
@@ -6,21 +6,14 @@ package analytics
import (
"fmt"
"strings"
"time"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
const userAccessLogFilterClauseCapacity = 4
// AccessLogFilter scopes ClickHouse user access log queries.
type AccessLogFilter struct {
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
UserIDs []uint64
Path string
// StartTime filters created_at >= StartTime when non-nil.
StartTime *time.Time
// EndTime filters created_at <= EndTime when non-nil.
EndTime *time.Time
}
type AccessLogFilter = analyticsmodel.AccessLogFilter
func buildUserAccessLogFilterClause(filter AccessLogFilter) (string, []any, bool) {
if filter.UserIDs != nil && len(filter.UserIDs) == 0 {
@@ -16,22 +16,13 @@ import (
const hoursInDay = 24
// DailyTrend is a single day's access count.
type DailyTrend struct {
Date string
Count uint64
}
type DailyTrend = analyticsmodel.DailyTrend
// BrowserShare is a browser group's share of access logs.
type BrowserShare struct {
Browser string
Count uint64
}
type BrowserShare = analyticsmodel.BrowserShare
// TopUser is an active user ranked by access count.
type TopUser struct {
UserID uint64
Count uint64
}
type TopUser = analyticsmodel.TopUser
// GetDailyTrend returns per-day access counts for the last days days (inclusive of today).
func GetDailyTrend(ctx context.Context, days int) ([]DailyTrend, error) {
@@ -9,21 +9,12 @@ import (
"github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
// and in-process batch writer queue health.
type ClickHouseOperationalStats struct {
Database string `json:"database"`
ActiveParts int64 `json:"active_parts"`
TotalRows int64 `json:"total_rows"`
PendingMutations int64 `json:"pending_mutations"`
AsyncInsertQueue int64 `json:"async_insert_queue"`
AsyncInsertBytes int64 `json:"async_insert_bytes"`
// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
BatchWriters []batchwriter.Stats `json:"batch_writers,omitempty"`
}
type ClickHouseOperationalStats = analyticsmodel.ClickHouseOperationalStats
// GetClickHouseOperationalStats returns operational metrics for the configured database.
func GetClickHouseOperationalStats(ctx context.Context) (*ClickHouseOperationalStats, error) {
@@ -15,10 +15,7 @@ import (
)
// NodeAccessLogRegionCount aggregates access log regions.
type NodeAccessLogRegionCount struct {
Region string
Count int64
}
type NodeAccessLogRegionCount = analyticsmodel.NodeAccessLogRegionCount
func nodeAccessLogConn() (driver.Conn, error) {
if db.ChConn == nil {
@@ -151,28 +148,13 @@ ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
}
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
type NodeAccessLogTrafficSummary struct {
RequestCount int64
ErrorCount int64
UniqueIPCount int64
BytesSent int64
RequestLength int64
NodeCount int64
}
type NodeAccessLogTrafficSummary = analyticsmodel.NodeAccessLogTrafficSummary
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
type NodeAccessLogValueCount struct {
Value string
Count int64
}
type NodeAccessLogValueCount = analyticsmodel.NodeAccessLogValueCount
// NodeAccessLogNodeAggregate is per-node traffic over a window.
type NodeAccessLogNodeAggregate struct {
NodeID string
RequestCount int64
ErrorCount int64
UniqueIPCount int64
}
type NodeAccessLogNodeAggregate = analyticsmodel.NodeAccessLogNodeAggregate
// TrafficSummaryNodeAccessLogs returns request/error/UV/bytes/node counts for the filter.
func TrafficSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (NodeAccessLogTrafficSummary, error) {
@@ -6,7 +6,8 @@ package analytics
import (
"fmt"
"strings"
"time"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
const (
@@ -25,20 +26,7 @@ const (
)
// NodeAccessLogFilter scopes ClickHouse node access log queries.
type NodeAccessLogFilter struct {
NodeID string
RemoteAddr string
Host string
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
Hosts []string
Path string
Since time.Time
Until time.Time
Page int
PageSize int
SortBy string
SortOrder string
}
type NodeAccessLogFilter = analyticsmodel.NodeAccessLogFilter
func buildNodeAccessLogFilterClause(filter NodeAccessLogFilter) (string, []any) {
parts := make([]string, 0, nodeAccessLogFilterClauseCapacity)
@@ -211,33 +211,11 @@ func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error)
return result, nil
}
// NodeTrafficHourly is an hourly traffic rollup row.
//
// UniqueVisitorCount is always 0 when sourced from of_access_log_hourly
// (true UV requires raw uniqExact on access logs).
type NodeTrafficHourly struct {
NodeID string
Hour time.Time
RequestCount int64
ErrorCount int64
UniqueVisitorCount int64
}
// NodeTrafficHourly 为小时级流量汇总行(别名,定义见 model/analytics)。
type NodeTrafficHourly = analyticsmodel.NodeTrafficHourly
// NodeMetricHourly is an hourly metric snapshot aggregation row.
//
// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
// lagInFrame samples per node (negative deltas after counter reset are dropped).
type NodeMetricHourly struct {
Hour time.Time
AverageCPUUsagePercent float64
AverageMemoryUsagePercent float64
NetworkRxBytes int64
NetworkTxBytes int64
DiskReadBytes int64
DiskWriteBytes int64
ReportedNodes int
}
// NodeMetricHourly 为小时级指标聚合行(别名,定义见 model/analytics)。
type NodeMetricHourly = analyticsmodel.NodeMetricHourly
// ListNodeTrafficHourly returns hourly traffic from of_access_log_hourly (M5).
// UniqueVisitorCount is always 0 here (UV requires raw uniqExact on access logs).
@@ -5,17 +5,14 @@ package analytics
import (
"strings"
"time"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
const nodeObservabilityFilterClauseCapacity = 3
// NodeObservabilityFilter scopes ClickHouse node observability queries.
type NodeObservabilityFilter struct {
NodeID string
Since time.Time
Limit int
}
type NodeObservabilityFilter = analyticsmodel.NodeObservabilityFilter
func buildNodeObservabilityFilterClause(filter NodeObservabilityFilter, sinceColumn string) (string, []any) {
parts := make([]string, 0, nodeObservabilityFilterClauseCapacity)
+135
View File
@@ -0,0 +1,135 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"strconv"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
// CleanupSummary 汇总本次清理结果。
type CleanupSummary struct {
ActiveDatabase string `json:"active_database"`
RetentionDays int `json:"retention_days"`
Deleted int64 `json:"deleted"`
// Tables 记录本次清理的物理表简写名(去掉 of_ 前缀,如 node_access_logs 对应
// of_node_access_logs;CH 侧物理表名相同,简写仅便于状态展示)。
Tables []string `json:"tables"`
}
// defaultLogRetentionDays 默认日志保留天数(配置缺失/非法时回退)。
const defaultLogRetentionDays = 90
// partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。
const partitionLeadMonths = 2
// retentionDaysForDatabase 按给定日志库读取保留天数(默认 90)。
func retentionDaysForDatabase(ctx context.Context, dbName string) int {
key := model.ConfigKeyLogRetentionDaysPostgres
switch dbName {
case dbNameSQLite:
key = model.ConfigKeyLogRetentionDaysSQLite
case dbNameClickHouse:
key = model.ConfigKeyLogRetentionDaysClickHouse
}
v, err := getConfig(ctx, key)
if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "读取日志保留天数配置失败(key=%s),回退默认 %d 天: %v", key, defaultLogRetentionDays, err)
}
return defaultLogRetentionDays
}
days, perr := strconv.Atoi(v)
if perr != nil || days <= 0 {
logger.ErrorF(ctx, "日志保留天数配置非法(key=%s, value=%q),回退默认 %d 天", key, v, defaultLogRetentionDays)
return defaultLogRetentionDays
}
return days
}
// CleanupExpired 按当前激活库保留天数清理过期日志(每日由 system_cleanup 调用)。
func CleanupExpired(ctx context.Context) (*CleanupSummary, error) {
dbName, err := resolveDatabase(ctx)
if err != nil {
return nil, fmt.Errorf("resolve active database: %w", err)
}
s, err := Active(ctx)
if err != nil {
return nil, err
}
days := retentionDaysForDatabase(ctx, dbName)
cutoff := time.Now().AddDate(0, 0, -days)
summary := &CleanupSummary{ActiveDatabase: dbName, RetentionDays: days, Tables: []string{}}
// PG 分区表仅在迁移时预建「当前+2 月」分区,此处确保分区持续存在,
// 否则跨月后新写入会报 "no partition of relation found"(SQLite/CH 为 no-op)。
now := time.Now().UTC()
if err := s.AccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
return nil, fmt.Errorf("ensure partitions: %w", err)
}
if err := cleanupTable("node_access_logs", func() (int64, error) {
return s.AccessLogs.DeleteBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
if err := cleanupTable("metric_snapshots", func() (int64, error) {
return s.Observability.DeleteMetricSnapshotsBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
if err := cleanupTable("edge_health", func() (int64, error) {
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
if err := cleanupTable("obs_frps", func() (int64, error) {
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
if err := cleanupTable("obs_frpc", func() (int64, error) {
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
return summary, nil
}
func cleanupTable(name string, fn func() (int64, error), summary *CleanupSummary) error {
n, err := fn()
if err != nil {
return fmt.Errorf("cleanup %s: %w", name, err)
}
summary.Deleted += n
summary.Tables = append(summary.Tables, name)
return nil
}
// partitionStatementsRange 生成覆盖 [from, to] 全部月份的两表分区 DDL,
// 幂等 CREATE TABLE IF NOT EXISTS ... PARTITION OF ... FOR VALUES FROM ... TO ...。
// 入参为任意时间点:按各自所在月份生成,含 from 月与 to 月(to 常用 max+1 月兜底)。
func partitionStatementsRange(from, to time.Time) []string {
var out []string
start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC)
end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 1, 0)
for ; start.Before(end); start = start.AddDate(0, 1, 0) {
monthEnd := start.AddDate(0, 1, 0)
suffix := start.Format("200601")
fromDay := start.Format("2006-01-02")
toDay := monthEnd.Format("2006-01-02")
for _, table := range []string{"of_node_access_logs", "w_user_access_logs"} {
out = append(out, fmt.Sprintf(
"CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')",
table, suffix, table, fromDay, toDay))
}
}
return out
}
@@ -0,0 +1,223 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// cleanupTestModels 清理涉及的 5 张日志/可观测表。
func cleanupTestModels() []any {
return []any{
&analyticsmodel.NodeAccessLog{},
&analyticsmodel.NodeMetricSnapshot{},
&analyticsmodel.NodeEdgeHealth{},
&analyticsmodel.NodeObsFrps{},
&analyticsmodel.NodeObsFrpc{},
}
}
// newCleanupTestDB 构造内存 sqlite 库并注入 db.DB(CleanupExpired 经 Active → buildStore 使用)。
func newCleanupTestDB(t *testing.T) *gorm.DB {
t.Helper()
dsn := fmt.Sprintf("file:logstore-cleanup-%d?mode=memory&cache=shared", atomic.AddInt64(&testGormStoreSeq, 1))
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := gdb.AutoMigrate(cleanupTestModels()...); err != nil {
t.Fatalf("automigrate: %v", err)
}
db.SetDB(gdb)
t.Cleanup(func() { db.SetDB(nil) })
return gdb
}
// TestCleanupExpiredSQLite 验证 sqlite 激活库的过期日志清理:
// 注入 log_retention_days_sqlite=30,40 天前的 5 表记录被删、昨天的保留。
func TestCleanupExpiredSQLite(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case logDatabaseKey:
return "sqlite", nil
case model.ConfigKeyLogRetentionDaysSQLite:
return "30", nil
}
return "", nil
})
defer ResetForTest()
gdb := newCleanupTestDB(t)
ctx := context.Background()
old := time.Now().AddDate(0, 0, -40).UTC()
recent := time.Now().AddDate(0, 0, -1).UTC()
if err := gdb.Create([]analyticsmodel.NodeAccessLog{
{ID: 1, NodeID: "n1", LoggedAt: old, RemoteAddr: "1.1.1.1"},
{ID: 2, NodeID: "n1", LoggedAt: recent, RemoteAddr: "2.2.2.2"},
}).Error; err != nil {
t.Fatalf("seed node access logs: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeMetricSnapshot{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed metric snapshots: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeEdgeHealth{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed edge health: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeObsFrps{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed obs frps: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeObsFrpc{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed obs frpc: %v", err)
}
summary, err := CleanupExpired(ctx)
if err != nil {
t.Fatalf("CleanupExpired: %v", err)
}
if summary.ActiveDatabase != "sqlite" {
t.Fatalf("ActiveDatabase = %q, want sqlite", summary.ActiveDatabase)
}
if summary.RetentionDays != 30 {
t.Fatalf("RetentionDays = %d, want 30", summary.RetentionDays)
}
if summary.Deleted != 5 {
t.Fatalf("Deleted = %d, want 5", summary.Deleted)
}
if len(summary.Tables) != 5 {
t.Fatalf("Tables = %v, want 5 tables", summary.Tables)
}
assertCount := func(m any, want int64, label string) {
t.Helper()
var n int64
if err := gdb.Model(m).Count(&n).Error; err != nil {
t.Fatalf("count %s: %v", label, err)
}
if n != want {
t.Fatalf("%s count = %d, want %d", label, n, want)
}
}
assertCount(&analyticsmodel.NodeAccessLog{}, 1, "node_access_logs")
assertCount(&analyticsmodel.NodeMetricSnapshot{}, 1, "metric_snapshots")
assertCount(&analyticsmodel.NodeEdgeHealth{}, 1, "edge_health")
assertCount(&analyticsmodel.NodeObsFrps{}, 1, "obs_frps")
assertCount(&analyticsmodel.NodeObsFrpc{}, 1, "obs_frpc")
var kept analyticsmodel.NodeAccessLog
if err := gdb.First(&kept).Error; err != nil {
t.Fatalf("recent node access log missing: %v", err)
}
if kept.ID != 2 {
t.Fatalf("kept log ID = %d, want 2 (recent)", kept.ID)
}
}
// TestRetentionDaysForDatabase 覆盖保留天数读取:按激活库选 key、非法值回退默认 90。
func TestRetentionDaysForDatabase(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case logDatabaseKey:
return "sqlite", nil
case model.ConfigKeyLogRetentionDaysSQLite:
return "30", nil
}
return "", nil
})
if got := retentionDaysForDatabase(context.Background(), "sqlite"); got != 30 {
t.Fatalf("retentionDaysForDatabase = %d, want 30", got)
}
// 非法值(非数字/<=0)回退默认 90。
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case logDatabaseKey:
return "postgres", nil
case model.ConfigKeyLogRetentionDaysPostgres:
return "abc", nil
}
return "", nil
})
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
t.Fatalf("retentionDaysForDatabase invalid value = %d, want 90", got)
}
// reader 报错回退默认 90。
SetConfigReader(func(_ context.Context, _ string) (string, error) {
return "", fmt.Errorf("boom")
})
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
t.Fatalf("retentionDaysForDatabase reader error = %d, want 90", got)
}
}
// TestPartitionStatements 验证 PG 分区 DDL 生成:当前月 + 未来 2 个月 × 2 表,
// 幂等 PARTITION OF 语句与迁移 SQL 命名一致(含跨年)。
func TestPartitionStatements(t *testing.T) {
now := time.Date(2026, 8, 15, 10, 0, 0, 0, time.UTC)
stmts := partitionStatementsRange(now, now.AddDate(0, 2, 0))
if len(stmts) != 6 {
t.Fatalf("partitionStatements len = %d, want 6", len(stmts))
}
want := []string{
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202608 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202608 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202609 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202609 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202610 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202610 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
}
for i, w := range want {
if stmts[i] != w {
t.Fatalf("stmt[%d] = %q, want %q", i, stmts[i], w)
}
}
// 跨年:2026-11 → 202611, 202612, 202701。
nov := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
suffixes := []string{"202611", "202612", "202701"}
for _, stmt := range partitionStatementsRange(nov, nov.AddDate(0, 2, 0)) {
if !hasAnySuffix(stmt, suffixes) {
t.Fatalf("statement lacks expected month suffix: %s", stmt)
}
}
}
func hasAnySuffix(stmt string, suffixes []string) bool {
for _, table := range []string{"of_node_access_logs", "w_user_access_logs"} {
for _, suf := range suffixes {
if strings.Contains(stmt, table+"_"+suf) {
return true
}
}
}
return false
}
@@ -0,0 +1,731 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"math"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
// clickhouseLogStore 实现 AccessLogStore / ObservabilityStore / StatusStore,
// 逐方法委托 analyticsrepo(CH 原生 batch 写入,零性能损耗)。
// UserAccessLogStore 由 clickhouseUserAccessLogStore 实现(List/Count 方法名已被
// AccessLogStore 占用,Go 不允许同名不同签名方法)。
type clickhouseLogStore struct {
// skipFreeze 为 true 时跳过迁移冻结检查(仅迁移目标 store 使用)。
skipFreeze bool
}
func newClickHouseStore() *clickhouseLogStore { return &clickhouseLogStore{} }
// 编译期断言。
var (
_ AccessLogStore = (*clickhouseLogStore)(nil)
_ ObservabilityStore = (*clickhouseLogStore)(nil)
_ StatusStore = (*clickhouseLogStore)(nil)
_ UserAccessLogStore = (*clickhouseUserAccessLogStore)(nil)
)
func chConnErr() error {
if !db.ChConnReady() {
return errors.New("clickhouse connection is not initialized")
}
return nil
}
// ensureWritable 迁移冻结期拒绝写入。
func (s *clickhouseLogStore) ensureWritable(ctx context.Context) error {
if !s.skipFreeze && Migrating(ctx) {
return ErrMigrating
}
return nil
}
// ---- AccessLogStore ----
// InsertBatch 节点访问日志写入入口:冻结检查后经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
rows := make([]analyticsmodel.NodeAccessLog, 0, len(records))
for _, r := range records {
if r == nil {
continue
}
rows = append(rows, toAnalyticsNodeAccessLog(r))
}
if h := currentAccessLogHooks().QueueNodeAccessLogs; h != nil {
h(rows)
}
return nil
}
// BatchInsertNodeAccessLogs 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeAccessLogs(ctx, rows)
}
func (s *clickhouseLogStore) List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
if err != nil {
return nil, err
}
return fromAnalyticsNodeAccessLogs(rows), nil
}
func (s *clickhouseLogStore) Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
func (s *clickhouseLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
if err != nil {
return nil, err
}
out := make([]*model.OpenFlareAccessLogRegionCount, len(rows))
for i, r := range rows {
out[i] = &model.OpenFlareAccessLogRegionCount{Region: r.Region, Count: r.Count}
}
return out, nil
}
func (s *clickhouseLogStore) BucketAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error) {
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
}
func (s *clickhouseLogStore) CountBuckets(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
}
func (s *clickhouseLogStore) BucketDimensions(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error) {
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, bucketSeconds)
}
func (s *clickhouseLogStore) IPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error) {
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), exactRemoteAddr)
}
func (s *clickhouseLogStore) IPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error) {
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), recentSince)
}
func (s *clickhouseLogStore) CountIPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, error) {
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
func (s *clickhouseLogStore) WAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error) {
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
func (s *clickhouseLogStore) IPTrend(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error) {
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
}
func (s *clickhouseLogStore) TrafficSummary(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
if err != nil {
return model.OpenFlareAccessLogTrafficSummary{}, err
}
return model.OpenFlareAccessLogTrafficSummary{
RequestCount: row.RequestCount,
ErrorCount: row.ErrorCount,
UniqueIPCount: row.UniqueIPCount,
BytesSent: row.BytesSent,
RequestLength: row.RequestLength,
NodeCount: row.NodeCount,
}, nil
}
func (s *clickhouseLogStore) ValueCounts(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, limit)
if err != nil {
return nil, err
}
out := make([]model.OpenFlareAccessLogValueCount, len(rows))
for i, r := range rows {
out[i] = model.OpenFlareAccessLogValueCount{Value: r.Value, Count: r.Count}
}
return out, nil
}
func (s *clickhouseLogStore) NodeAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
if err != nil {
return nil, err
}
out := make([]model.OpenFlareAccessLogNodeAggregate, len(rows))
for i, r := range rows {
out[i] = model.OpenFlareAccessLogNodeAggregate{NodeID: r.NodeID, RequestCount: r.RequestCount, ErrorCount: r.ErrorCount, UniqueIPCount: r.UniqueIPCount}
}
return out, nil
}
func (s *clickhouseLogStore) DeleteAll(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
}
func (s *clickhouseLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
}
func (s *clickhouseLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
}
// ListForMigration 按 id 升序分页读取(迁移复制用):直接查询 CH 原生表。
func (s *clickhouseLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error) {
if err := chConnErr(); err != nil {
return nil, err
}
rows, err := db.ChConn.Query(ctx, `
SELECT `+analyticsmodel.NodeAccessLog{}.InsertColumns()+`
FROM `+analyticsmodel.NodeAccessLog{}.TableName()+`
WHERE id > ?
ORDER BY id ASC
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
if err != nil {
return nil, fmt.Errorf("list node access logs for migration: %w", err)
}
defer func() { _ = rows.Close() }()
var result []analyticsmodel.NodeAccessLog
for rows.Next() {
var item analyticsmodel.NodeAccessLog
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.LoggedAt,
&item.RemoteAddr,
&item.Region,
&item.Host,
&item.Path,
&item.UserAgent,
&item.CacheStatus,
&item.StatusCode,
&item.BytesSent,
&item.RequestLength,
&item.RequestTimeMs,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node access log row: %w", err)
}
item.LoggedAt = item.LoggedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
// ---- ObservabilityStore ----
// InsertMetricSnapshot 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueMetricSnapshot; h != nil {
h(toAnalyticsNodeMetricSnapshot(record))
}
return nil
}
func (s *clickhouseLogStore) ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
rows, err := analyticsrepo.ListNodeMetricSnapshots(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeMetricSnapshots(rows), nil
}
func (s *clickhouseLogStore) DeleteAllMetricSnapshots(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeMetricSnapshots(ctx)
}
// ListTrafficHourly 委托 analyticsrepo 读 of_access_log_hourly rollup(M5 口径,UV 恒 0)。
func (s *clickhouseLogStore) ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error) {
return analyticsrepo.ListNodeTrafficHourly(ctx, toNodeObservabilitySince(nodeID, since))
}
// ListAccessLogHourly 委托 analyticsrepo 读 of_access_log_hourly rollup。
func (s *clickhouseLogStore) ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error) {
return analyticsrepo.ListAccessLogHourly(ctx, toNodeObservabilitySince(nodeID, since))
}
// ListMetricHourly 委托 analyticsrepo ListNodeMetricHourly:rollup 覆盖窗口时读
// of_node_metric_capacity_hourly,否则按 mergeNodeMetricHourlyPreferRollup 合并 raw 兜底。
func (s *clickhouseLogStore) ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error) {
return analyticsrepo.ListNodeMetricHourly(ctx, toNodeObservabilitySince(nodeID, since))
}
func (s *clickhouseLogStore) DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
}
// BatchInsertNodeMetricSnapshots 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeMetricSnapshots(ctx, rows)
}
// InsertEdgeHealth 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueEdgeHealth; h != nil {
h(toAnalyticsNodeEdgeHealth(record))
}
return nil
}
func (s *clickhouseLogStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeEdgeHealths(rows), nil
}
func (s *clickhouseLogStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
}
func (s *clickhouseLogStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
}
// BatchInsertNodeEdgeHealth 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeEdgeHealth(ctx, rows)
}
// InsertNodeObservationFrps 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueNodeObsFrps; h != nil {
h(toAnalyticsNodeObsFrps(record))
}
return nil
}
func (s *clickhouseLogStore) ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
rows, err := analyticsrepo.ListNodeObsFrps(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeObsFrps(rows), nil
}
func (s *clickhouseLogStore) DeleteAllNodeObservationFrps(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeObsFrps(ctx)
}
func (s *clickhouseLogStore) DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeObsFrpsBefore(ctx, cutoff)
}
// BatchInsertNodeObsFrps 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeObsFrps(ctx, rows)
}
// InsertNodeObservationFrpc 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueNodeObsFrpc; h != nil {
h(toAnalyticsNodeObsFrpc(record))
}
return nil
}
func (s *clickhouseLogStore) ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
rows, err := analyticsrepo.ListNodeObsFrpc(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeObsFrpc(rows), nil
}
func (s *clickhouseLogStore) DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeObsFrpc(ctx)
}
func (s *clickhouseLogStore) DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeObsFrpcBefore(ctx, cutoff)
}
// MigrationRange 返回 of_node_access_logs.logged_at 的最小/最大值(空表返回零值)。
func (s *clickhouseLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
return chMigrationRange(ctx, analyticsmodel.NodeAccessLog{}.TableName(), "logged_at")
}
// EnsurePartitions 是 CH 分支 no-op(CH 无 PG 式分区)。
func (s *clickhouseLogStore) EnsurePartitions(_ context.Context, _, _ time.Time) error {
return nil
}
// chMigrationRange 查询 CH 表时间列 MIN/MAX;空表(NULL)返回零值。
func chMigrationRange(ctx context.Context, table, column string) (time.Time, time.Time, error) {
if err := chConnErr(); err != nil {
return time.Time{}, time.Time{}, err
}
var minTime, maxTime *time.Time
if err := db.ChConn.QueryRow(ctx,
"SELECT min("+column+"), max("+column+") FROM "+table,
).Scan(&minTime, &maxTime); err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("query migration range %s: %w", table, err)
}
if minTime == nil || maxTime == nil {
return time.Time{}, time.Time{}, nil
}
return minTime.UTC(), maxTime.UTC(), nil
}
// BatchInsertNodeObsFrpc 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeObsFrpc(ctx, rows)
}
// ListMetricSnapshotsForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error) {
return chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeMetricSnapshot{}.TableName(),
analyticsmodel.NodeMetricSnapshot{}.InsertColumns(),
func(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
var result []analyticsmodel.NodeMetricSnapshot
for rows.Next() {
var item analyticsmodel.NodeMetricSnapshot
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.CPUUsagePercent,
&item.MemoryUsedBytes,
&item.MemoryTotalBytes,
&item.StorageUsedBytes,
&item.StorageTotalBytes,
&item.DiskReadBytes,
&item.DiskWriteBytes,
&item.NetworkRxBytes,
&item.NetworkTxBytes,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
})
}
// chObsRow 迁移读取共用的双字段观测行(字符串状态 + 数值计数):
// edge_health(status/connections)与 obs_frpc(tunnel_status/connected_relays_count)同形状。
type chObsRow struct {
ID uint64
NodeID string
CapturedAt time.Time
Status string
Count int64
CreatedAt time.Time
}
// countToInt32 将观测计数转为 int32(防御溢出;观测计数远小于 int32 上限)。
func countToInt32(v int64) int32 {
if v > math.MaxInt32 {
return math.MaxInt32
}
if v < math.MinInt32 {
return math.MinInt32
}
return int32(v)
}
// scanChObsRow 扫描 chObsRow(含 UTC 归一化)。
func scanChObsRow(rows driver.Rows) ([]chObsRow, error) {
var result []chObsRow
for rows.Next() {
var item chObsRow
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.Status,
&item.Count,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan observation row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
// ListEdgeHealthForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error) {
rows, err := chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeEdgeHealth{}.TableName(),
analyticsmodel.NodeEdgeHealth{}.InsertColumns(),
scanChObsRow)
if err != nil {
return nil, err
}
out := make([]analyticsmodel.NodeEdgeHealth, len(rows))
for i, r := range rows {
out[i] = analyticsmodel.NodeEdgeHealth{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, Status: r.Status, Connections: r.Count, CreatedAt: r.CreatedAt}
}
return out, nil
}
// ListNodeObsFrpsForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error) {
return chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeObsFrps{}.TableName(),
analyticsmodel.NodeObsFrps{}.InsertColumns(),
func(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
var result []analyticsmodel.NodeObsFrps
for rows.Next() {
var item analyticsmodel.NodeObsFrps
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.FrpsConnections,
&item.FrpsProxyCount,
&item.FrpsClientCount,
&item.FrpsProxies,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node frps observation row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
})
}
// ListNodeObsFrpcForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error) {
rows, err := chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeObsFrpc{}.TableName(),
analyticsmodel.NodeObsFrpc{}.InsertColumns(),
scanChObsRow)
if err != nil {
return nil, err
}
out := make([]analyticsmodel.NodeObsFrpc, len(rows))
for i, r := range rows {
out[i] = analyticsmodel.NodeObsFrpc{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, TunnelStatus: r.Status, ConnectedRelaysCount: countToInt32(r.Count), CreatedAt: r.CreatedAt}
}
return out, nil
}
// chListForMigration 执行按 id 升序分页的 CH 原生表查询,并交给 scanner 扫描。
func chListForMigration[T any](ctx context.Context, afterID uint64, limit int, table, columns string, scanner func(driver.Rows) ([]T, error)) ([]T, error) {
if err := chConnErr(); err != nil {
return nil, err
}
rows, err := db.ChConn.Query(ctx, `
SELECT `+columns+`
FROM `+table+`
WHERE id > ?
ORDER BY id ASC
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
if err != nil {
return nil, fmt.Errorf("list %s for migration: %w", table, err)
}
defer func() { _ = rows.Close() }()
return scanner(rows)
}
// ---- StatusStore ----
// ActiveDatabase 返回当前日志主库名(CH 分支固定 clickhouse)。
func (s *clickhouseLogStore) ActiveDatabase(_ context.Context) (string, error) {
return dbNameClickHouse, nil
}
// ClickHouseOperationalStats 委托 analyticsrepo 汇总 CH 运行状态。
func (s *clickhouseLogStore) ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) {
return analyticsrepo.GetClickHouseOperationalStats(ctx)
}
// ---- UserAccessLogStore ----
// clickhouseUserAccessLogStore 实现 UserAccessLogStore。clickhouseLogStore 已占用
// List/Count 方法名(AccessLogStore 接口),Go 不允许同名不同签名方法,故用户访问日志
// 用独立类型嵌入同一 clickhouseLogStore(与 userAccessLogGormStore 同构),复用 ensureWritable。
type clickhouseUserAccessLogStore struct {
*clickhouseLogStore
}
func newClickHouseUserAccessLogStore() *clickhouseUserAccessLogStore {
return &clickhouseUserAccessLogStore{clickhouseLogStore: newClickHouseStore()}
}
// BatchInsert 是 batchwriter flush 目标:CH 原生批量写入;冻结期拒绝写入,空批次直接返回。
func (s *clickhouseUserAccessLogStore) BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
if len(logs) == 0 {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsert(ctx, logs)
}
// DeleteAll 清空全部用户访问日志(TRUNCATE 语义,迁移「覆盖目标库已有日志」幂等前提用)。
func (s *clickhouseUserAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllUserAccessLogs(ctx)
}
// ListForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseUserAccessLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error) {
return chListForMigration(ctx, afterID, limit,
analyticsmodel.UserAccessLog{}.TableName(),
analyticsmodel.UserAccessLog{}.InsertColumns(),
func(rows driver.Rows) ([]analyticsmodel.UserAccessLog, error) {
var result []analyticsmodel.UserAccessLog
for rows.Next() {
var item analyticsmodel.UserAccessLog
if err := rows.Scan(
&item.ID,
&item.UserID,
&item.Path,
&item.Method,
&item.IP,
&item.UserAgent,
&item.Headers,
&item.Status,
&item.Latency,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan user access log row: %w", err)
}
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
})
}
// MigrationRange 返回 w_user_access_logs.created_at 的最小/最大值(空表返回零值)。
func (s *clickhouseUserAccessLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
return chMigrationRange(ctx, analyticsmodel.UserAccessLog{}.TableName(), "created_at")
}
func (s *clickhouseUserAccessLogStore) Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
return analyticsrepo.CountAccessLogs(ctx, filter)
}
func (s *clickhouseUserAccessLogStore) List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
return analyticsrepo.ListAccessLogs(ctx, filter, page, pageSize)
}
func (s *clickhouseUserAccessLogStore) GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
return analyticsrepo.GetDailyTrend(ctx, days)
}
func (s *clickhouseUserAccessLogStore) GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
return analyticsrepo.GetBrowserDistribution(ctx, startTime)
}
func (s *clickhouseUserAccessLogStore) GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
return analyticsrepo.GetTopActiveUsers(ctx, startTime, limit)
}
// toNodeObservabilityFilter 构造 CH 可观测查询过滤器(limit<=0 表示不限制)。
func toNodeObservabilityFilter(nodeID string, since time.Time, limit int) analyticsmodel.NodeObservabilityFilter {
return analyticsmodel.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
Limit: limit,
}
}
// toNodeObservabilitySince 构造不带 limit 的可观测查询过滤器
// (小时级聚合读无需分页,避免传无意义的 0)。
func toNodeObservabilitySince(nodeID string, since time.Time) analyticsmodel.NodeObservabilityFilter {
return analyticsmodel.NodeObservabilityFilter{NodeID: nodeID, Since: since}
}
@@ -0,0 +1,40 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"strings"
"testing"
"time"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
)
// TestClickHouseHourlyDelegationRegression 验证 CH 后端小时级聚合读委托 analyticsrepo:
// 未初始化 CH 连接时返回 analyticsrepo 的 "clickhouse connection is not initialized" 错误
// (而非未实现/panic),证明 3 个方法都路由到 CH 原生查询。
func TestClickHouseHourlyDelegationRegression(t *testing.T) {
if db.ChConnReady() {
t.Skip("clickhouse connection initialized; skipping delegation regression")
}
s := newClickHouseStore()
ctx := context.Background()
now := time.Now()
check := func(name string, err error) {
t.Helper()
if err == nil {
t.Fatalf("%s: want clickhouse-not-initialized error, got nil", name)
}
if !strings.Contains(err.Error(), "clickhouse connection is not initialized") {
t.Fatalf("%s: unexpected error %v", name, err)
}
}
_, err := s.ListTrafficHourly(ctx, "n1", now)
check("ListTrafficHourly", err)
_, err = s.ListAccessLogHourly(ctx, "n1", now)
check("ListAccessLogHourly", err)
_, err = s.ListMetricHourly(ctx, "n1", now)
check("ListMetricHourly", err)
}
@@ -0,0 +1,33 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"strconv"
)
// timeBucketSQLPostgres 返回 PG 时间分桶表达式(epoch 秒 -> 分桶起点,int64)。
func timeBucketSQLPostgres(column string, bucketSeconds int64) string {
return "(floor(extract(epoch from " + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")::bigint"
}
// dailyTrendDateSQLPostgres 返回 PG 按日聚合的日期表达式。
func dailyTrendDateSQLPostgres() string {
return "to_char(created_at, 'YYYY-MM-DD')"
}
// epochSQLPostgres 返回 PG epoch 秒表达式(int64)。
func epochSQLPostgres(column string) string {
return "extract(epoch from " + column + ")::bigint"
}
// textCastSQLPostgres 返回 PG 数值列转文本表达式。
func textCastSQLPostgres(column string) string {
return column + "::text"
}
// distinctNonEmptyCountSQLPostgres 返回 PG 排除空串的 distinct 计数表达式。
func distinctNonEmptyCountSQLPostgres(column string) string {
return "COUNT(DISTINCT " + column + ") FILTER (WHERE " + column + " <> '')"
}
@@ -0,0 +1,85 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"strconv"
"gorm.io/gorm"
)
// isPostgresDialect 判断 gorm 句柄是否为 PostgreSQL 方言(否则按 SQLite 处理)。
// Dialector 经 gorm.Config 内嵌提升,Name() 可直接在 DB 上调用。
func isPostgresDialect(db *gorm.DB) bool {
return db != nil && db.Dialector != nil && db.Name() == "postgres"
}
// timeBucketSQLSQLite 返回 SQLite 时间分桶表达式(epoch 秒 -> 分桶起点)。
func timeBucketSQLSQLite(column string, bucketSeconds int64) string {
return "(floor(unixepoch(" + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")"
}
// dailyTrendDateSQLSQLite 返回 SQLite 按日聚合的日期表达式。
func dailyTrendDateSQLSQLite() string {
return "strftime('%Y-%m-%d', created_at)"
}
// epochSQLSQLite 返回 SQLite epoch 秒表达式(unixepoch 整数秒)。
func epochSQLSQLite(column string) string {
return "unixepoch(" + column + ")"
}
// textCastSQLSQLite 返回 SQLite 数值列转文本表达式。
func textCastSQLSQLite(column string) string {
return "CAST(" + column + " AS TEXT)"
}
// distinctNonEmptyCountSQLSQLite 返回 SQLite 排除空串的 distinct 计数表达式
// (SQLite 无 FILTER 语法,用 CASE 等价实现)。
func distinctNonEmptyCountSQLSQLite(column string) string {
return "COUNT(DISTINCT CASE WHEN " + column + " <> '' THEN " + column + " END)"
}
// distinctNonEmptyCountSQL 按当前方言返回排除空串的 distinct 计数表达式
// (运行时按 Dialector 分发,默认 SQLite)。
func distinctNonEmptyCountSQL(db *gorm.DB, column string) string {
if isPostgresDialect(db) {
return distinctNonEmptyCountSQLPostgres(column)
}
return distinctNonEmptyCountSQLSQLite(column)
}
// dailyTrendDateSQL 按当前方言返回按日聚合的日期表达式(运行时按 Dialector 分发,默认 SQLite)。
func dailyTrendDateSQL(db *gorm.DB) string {
if isPostgresDialect(db) {
return dailyTrendDateSQLPostgres()
}
return dailyTrendDateSQLSQLite()
}
// epochSQL 按当前方言返回 epoch 秒表达式(运行时按 Dialector 分发,默认 SQLite)。
func epochSQL(db *gorm.DB, column string) string {
if isPostgresDialect(db) {
return epochSQLPostgres(column)
}
return epochSQLSQLite(column)
}
// textCastSQL 按当前方言返回数值列转文本表达式(运行时按 Dialector 分发,默认 SQLite)。
func textCastSQL(db *gorm.DB, column string) string {
if isPostgresDialect(db) {
return textCastSQLPostgres(column)
}
return textCastSQLSQLite(column)
}
// timeBucketSQL 按当前方言返回时间分桶表达式。
// brief 将 PG/SQLite 两版写为同名函数,同包无法共存;log_database 为运行时配置,
// 不能使用编译期 build tag,故按 db.Dialector.Name() 运行时分发(默认 SQLite)。
func timeBucketSQL(db *gorm.DB, column string, bucketSeconds int64) string {
if isPostgresDialect(db) {
return timeBucketSQLPostgres(column, bucketSeconds)
}
return timeBucketSQLSQLite(column, bucketSeconds)
}
+57
View File
@@ -0,0 +1,57 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"sync"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// AccessLogHooks 节点访问日志异步入队回调(由 chwriter 装配)。
type AccessLogHooks struct {
QueueNodeAccessLogs func(logs []analyticsmodel.NodeAccessLog)
}
// ObservabilityHooks 可观测异步入队回调(由 chwriter 装配)。
type ObservabilityHooks struct {
QueueMetricSnapshot func(record analyticsmodel.NodeMetricSnapshot)
QueueEdgeHealth func(record analyticsmodel.NodeEdgeHealth)
QueueNodeObsFrps func(record analyticsmodel.NodeObsFrps)
QueueNodeObsFrpc func(record analyticsmodel.NodeObsFrpc)
}
var (
hooksMu sync.RWMutex
accessLogHooks AccessLogHooks
observabilityHooks ObservabilityHooks
)
// SetAccessLogHooks 注册节点访问日志异步入队回调。
func SetAccessLogHooks(h AccessLogHooks) {
hooksMu.Lock()
accessLogHooks = h
hooksMu.Unlock()
}
// SetObservabilityHooks 注册可观测异步入队回调。
func SetObservabilityHooks(h ObservabilityHooks) {
hooksMu.Lock()
observabilityHooks = h
hooksMu.Unlock()
}
// currentAccessLogHooks 返回当前 hooks 快照(未注册时为 zero value,调用方判空跳过)。
func currentAccessLogHooks() AccessLogHooks {
hooksMu.RLock()
defer hooksMu.RUnlock()
return accessLogHooks
}
// currentObservabilityHooks 返回当前 hooks 快照(未注册时为 zero value,调用方判空跳过)。
func currentObservabilityHooks() ObservabilityHooks {
hooksMu.RLock()
defer hooksMu.RUnlock()
return observabilityHooks
}
@@ -0,0 +1,69 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"os/exec"
"strings"
"testing"
)
// forbiddenImports 上层应用禁止直接触碰的底层日志实现。
var forbiddenImports = []string{
"github.com/Rain-kl/Wavelet/internal/repository/analytics",
}
// allowedAnalyticsDelegation 允许直接依赖 analyticsrepo 的委托层:
// - internal/repository:持久化门面,ListOpenFlareLatestMetricSnapshotsSince 的
// CH 快速路径仍直连 analyticsrepo(LIMIT 1 BY node_id);小时级聚合读已改走 logstore;
// - internal/repository/logstore:CH 后端实现按设计委托 analyticsrepo。
//
// 除此之外,依赖闭包内任何包都禁止引入 analyticsrepo。
var allowedAnalyticsDelegation = map[string]bool{
"github.com/Rain-kl/Wavelet/internal/repository": true,
"github.com/Rain-kl/Wavelet/internal/repository/logstore": true,
}
// allowedInfraPersistence 允许 apps 引入的 infra/persistence 子包。
var allowedInfraPersistence = []string{
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter", // batchwriter 统计类型
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen", // 雪花 ID 生成(无日志依赖)
}
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
t.Chdir("../../..") // module root,保证 ./internal/apps/... 可解析
out, err := exec.Command("go", "list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
if err != nil {
t.Fatalf("go list: %v", err)
}
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
pkg := fields[0]
if !strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/apps") {
continue
}
for _, imp := range fields[1:] {
for _, forbidden := range forbiddenImports {
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
}
}
if strings.HasPrefix(imp, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
allowed := false
for _, a := range allowedInfraPersistence {
if imp == a || strings.HasPrefix(imp, a+"/") {
allowed = true
break
}
}
if !allowed {
t.Errorf("%s must not import infra/persistence subpackage directly: %s", pkg, imp)
}
}
}
}
}
@@ -0,0 +1,65 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// TestGormAccessLogInsertBatchHooks 覆盖访问日志写入入口:
// 冻结检查、hook 入队、不直接落库、flush 后可见(行为与旧 repository clickhouse 包装一致)。
func TestGormAccessLogInsertBatchHooks(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", nil
})
defer ResetForTest()
s := newTestGormStore(t)
ctx := context.Background()
now := time.Now().UTC()
var hooked []analyticsmodel.NodeAccessLog
SetAccessLogHooks(AccessLogHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
hooked = append(hooked, logs...)
},
})
defer SetAccessLogHooks(AccessLogHooks{})
records := []*model.OpenFlareAccessLog{
{NodeID: "n1", LoggedAt: now, RemoteAddr: "1.1.1.1", StatusCode: 200, BytesSent: 100},
{NodeID: "n1", LoggedAt: now, RemoteAddr: "2.2.2.2", StatusCode: 404},
}
if err := s.InsertBatch(ctx, records); err != nil {
t.Fatalf("insert batch: %v", err)
}
if len(hooked) != 2 || hooked[0].RemoteAddr != "1.1.1.1" || hooked[0].BytesSent != 100 || hooked[1].StatusCode != 404 {
t.Fatalf("hook rows mismatch: %+v", hooked)
}
// 写入入口只入队、不直接落库。
rows, err := s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
if err != nil {
t.Fatalf("list: %v", err)
}
if len(rows) != 0 {
t.Fatalf("entry insert must not write rows, got %d", len(rows))
}
// flush 后可见。
if err := s.BatchInsertNodeAccessLogs(ctx, hooked); err != nil {
t.Fatalf("flush: %v", err)
}
rows, err = s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
if err != nil {
t.Fatalf("list after flush: %v", err)
}
if len(rows) != 2 {
t.Fatalf("list after flush want 2, got %d", len(rows))
}
}
+122
View File
@@ -0,0 +1,122 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package logstore 提供日志/分析存储抽象:上层只面向本包接口,
// 禁止直接 import internal/repository/analytics 或触碰 db.ChConn/db.ChDB。
package logstore
import (
"context"
"errors"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// ErrMigrating 表示日志数据库正在迁移,当前禁止写入。
var ErrMigrating = errors.New("log database is migrating, writes are disabled")
// AccessLogStore 节点访问日志(of_node_access_logs)。
type AccessLogStore interface {
// InsertBatch 为写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error
// BatchInsertNodeAccessLogs 为 batchwriter flush 目标:直接批量写入当前存储。
BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error
List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error)
Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error)
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error)
BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error)
CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error)
IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error)
IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error)
CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error)
WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error)
IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error)
TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error)
ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error)
NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error)
DeleteAll(ctx context.Context) (int64, error)
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error)
// ListForMigration 按 id 升序分页读取(迁移复制用)。
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error)
// MigrationRange 返回源表 logged_at 的最小/最大值(空表返回零值),迁移预建分区用。
MigrationRange(ctx context.Context) (from, to time.Time, err error)
// EnsurePartitions 幂等预建 PG 分区(按月),覆盖 [from, to] 月份;CH/SQLite 为 no-op。
// 目标为 PG 的迁移在复制前调用,避免历史数据写入报 "no partition of relation found"。
EnsurePartitions(ctx context.Context, from, to time.Time) error
}
// ObservabilityStore 可观测 4 表(metric snapshots / edge health / frps / frpc)。
type ObservabilityStore interface {
InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error
ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error)
DeleteAllMetricSnapshots(ctx context.Context) (int64, error)
DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error
// ListTrafficHourly 返回小时级流量汇总(按 node/hour 聚合,unique_visitor_count 恒 0)。
// CH 后端读 of_access_log_hourly rollup;PG/SQLite 从 of_node_access_logs 实时聚合。
ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error)
// ListAccessLogHourly 返回按 node/hour/host 的小时级访问日志汇总。
ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error)
// ListMetricHourly 返回小时级指标聚合(avg cpu/memory + 计数器增量,reported_nodes 去重节点数)。
ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error)
InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error
ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error)
DeleteAllEdgeHealth(ctx context.Context) (int64, error)
DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error
InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error
ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error)
DeleteAllNodeObservationFrps(ctx context.Context) (int64, error)
DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error
InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error
ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error)
DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error)
DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error
// 迁移复制用:按 id 升序分页读取。
ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error)
ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error)
ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error)
ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error)
}
// UserAccessLogStore 用户访问日志(w_user_access_logs)。
type UserAccessLogStore interface {
BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error
// DeleteAll 清空全部用户访问日志(迁移「覆盖目标库已有日志」幂等前提用)。
DeleteAll(ctx context.Context) (int64, error)
Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error)
List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error)
GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error)
GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error)
GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error)
// ListForMigration 按 id 升序分页读取(迁移复制用)。
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error)
// MigrationRange 返回源表 created_at 的最小/最大值(空表返回零值),迁移预建分区用。
MigrationRange(ctx context.Context) (from, to time.Time, err error)
}
// StatusStore 日志库状态(供管理端状态端点)。
type StatusStore interface {
ActiveDatabase(ctx context.Context) (string, error)
ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) // 仅 CH 激活时非 nil
}
// Store 聚合当前生效日志库的全部域存储。
type Store struct {
AccessLogs AccessLogStore
Observability ObservabilityStore
UserAccessLogs UserAccessLogStore
Status StatusStore
}
@@ -0,0 +1,183 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"fmt"
"os"
"regexp"
"strings"
"testing"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/logger"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// TestEnsurePartitionsPostgresInsertAcrossMonths 需要 TEST_POSTGRES_DSN(未设置时跳过):
// 验证 EnsurePartitions 预建任意月份范围分区后,跨月历史数据可写入 PG 分区表
// (对应迁移任务从 CH/SQLite 复制历史日志到 PG 时先预建分区的场景)。
func TestEnsurePartitionsPostgresInsertAcrossMonths(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("TEST_POSTGRES_DSN is not set")
}
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open postgres: %v", err)
}
sqlDB, err := gdb.DB()
if err != nil {
t.Fatalf("sql db: %v", err)
}
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("logstore_partition_%d", time.Now().UnixNano())
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
t.Fatalf("invalid schema: %s", schema)
}
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
t.Fatalf("create schema: %v", err)
}
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
t.Fatalf("set search_path: %v", err)
}
t.Cleanup(func() {
_ = gdb.Exec("SET search_path TO public").Error
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
_ = sqlDB.Close()
})
// 与 goose/postgres/202608080001_create_log_tables.sql 保持一致的分区父表 DDL。
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
if err := gdb.Exec(ddl).Error; err != nil {
t.Fatalf("create partitioned table: %v", err)
}
}
ResetForTest()
SetConfigReader(func(_ context.Context, _ string) (string, error) { return "", nil })
defer ResetForTest()
ctx := context.Background()
store := newGormStore(gdb)
ua := newUserAccessLogGormStore(gdb)
// 源范围跨 3 个月:2026-01-10 ~ 2026-03-20;to+1 月兜底生成 202601..202604 分区。
from := time.Date(2026, 1, 10, 8, 0, 0, 0, time.UTC)
max := time.Date(2026, 3, 20, 9, 30, 0, 0, time.UTC)
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
t.Fatalf("EnsurePartitions: %v", err)
}
// 幂等:重复调用不报错(CREATE TABLE IF NOT EXISTS ... PARTITION OF)。
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
t.Fatalf("EnsurePartitions idempotent: %v", err)
}
var partitionCount int64
if err := gdb.Raw(
"SELECT count(*) FROM pg_inherits WHERE inhrelid = to_regclass('of_node_access_logs')",
).Scan(&partitionCount).Error; err != nil {
t.Fatalf("count partitions: %v", err)
}
if partitionCount != 4 {
t.Fatalf("of_node_access_logs partitions = %d, want 4", partitionCount)
}
// 跨月插入:1/2/3 月各 2 条节点访问日志 + 2 条用户访问日志,均应命中已有分区。
nodeRows := []analyticsmodel.NodeAccessLog{
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.2"},
{ID: 3, NodeID: "n2", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.2"},
{ID: 4, NodeID: "n2", LoggedAt: time.Date(2026, 2, 12, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.3"},
{ID: 5, NodeID: "n1", LoggedAt: time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
{ID: 6, NodeID: "n1", LoggedAt: time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.4"},
}
if err := store.BatchInsertNodeAccessLogs(ctx, nodeRows); err != nil {
t.Fatalf("insert node access logs across months: %v", err)
}
userRows := []analyticsmodel.UserAccessLog{
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)},
{ID: 2, UserID: 102, Path: "/b", CreatedAt: time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)},
}
if err := ua.BatchInsert(ctx, userRows); err != nil {
t.Fatalf("insert user access logs across months: %v", err)
}
var nodeCount, userCount int64
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
t.Fatalf("count node access logs: %v", err)
}
if err := gdb.Model(&analyticsmodel.UserAccessLog{}).Count(&userCount).Error; err != nil {
t.Fatalf("count user access logs: %v", err)
}
if nodeCount != 6 {
t.Fatalf("node access log count = %d, want 6", nodeCount)
}
if userCount != 2 {
t.Fatalf("user access log count = %d, want 2", userCount)
}
// MigrationRange 返回跨月范围(覆盖两表)。
gotFrom, gotTo, err := store.MigrationRange(ctx)
if err != nil {
t.Fatalf("node MigrationRange: %v", err)
}
if !gotFrom.Equal(time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)) || !gotTo.Equal(time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC)) {
t.Fatalf("node MigrationRange = %s ~ %s, want 2026-01-15 ~ 2026-03-18", gotFrom, gotTo)
}
uaFrom, uaTo, err := ua.MigrationRange(ctx)
if err != nil {
t.Fatalf("user MigrationRange: %v", err)
}
if !uaFrom.Equal(time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)) || !uaTo.Equal(time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)) {
t.Fatalf("user MigrationRange = %s ~ %s", uaFrom, uaTo)
}
}
// postgresNodeAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
const postgresNodeAccessLogsDDL = `
CREATE TABLE IF NOT EXISTS of_node_access_logs (
id BIGINT NOT NULL,
node_id VARCHAR(64) NOT NULL DEFAULT '',
logged_at TIMESTAMPTZ NOT NULL,
remote_addr VARCHAR(128) NOT NULL DEFAULT '',
region VARCHAR(128) NOT NULL DEFAULT '',
host VARCHAR(255) NOT NULL DEFAULT '',
path VARCHAR(2048) NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
cache_status VARCHAR(64) NOT NULL DEFAULT '',
status_code INTEGER NOT NULL DEFAULT 0,
bytes_sent BIGINT NOT NULL DEFAULT 0,
request_length BIGINT NOT NULL DEFAULT 0,
request_time_ms INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, logged_at)
) PARTITION BY RANGE (logged_at)`
// postgresUserAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
const postgresUserAccessLogsDDL = `
CREATE TABLE IF NOT EXISTS w_user_access_logs (
id BIGINT NOT NULL,
user_id BIGINT NOT NULL DEFAULT 0,
path VARCHAR(2048) NOT NULL DEFAULT '',
method VARCHAR(16) NOT NULL DEFAULT '',
ip VARCHAR(128) NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
headers TEXT NOT NULL DEFAULT '',
status INTEGER NOT NULL DEFAULT 0,
latency BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, created_at)
) PARTITION BY RANGE (created_at)`
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+204
View File
@@ -0,0 +1,204 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
// logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。
const (
logDatabaseKey = model.ConfigKeyLogDatabase
logMigrationKey = model.ConfigKeyLogDBMigration
)
// 日志库名常量(与 model 配置值一致,集中避免散落字符串字面量)。
const (
dbNamePostgres = "postgres"
dbNameSQLite = "sqlite"
dbNameClickHouse = "clickhouse"
)
// errConfigReaderNotWired 表示 config reader 尚未注入(首启/测试场景按 seed 规则兜底)。
var errConfigReaderNotWired = errors.New("logstore: config reader not wired")
// ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。
type ConfigReader func(ctx context.Context, key string) (string, error)
const resolveCacheTTL = 1 * time.Second
var (
configReader ConfigReader
storeMu sync.RWMutex
active *Store
activeDB string
lastResolveDB string
lastResolveTime time.Time
)
// SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。
func SetConfigReader(fn ConfigReader) { configReader = fn }
func getConfig(ctx context.Context, key string) (string, error) {
if configReader == nil {
return "", errConfigReaderNotWired
}
return configReader(ctx, key)
}
// Active 返回当前生效的日志库 Store。按 log_database 系统配置惰性解析并缓存,
// 配置更新(含迁移任务翻转)后自动重建。
func Active(ctx context.Context) (*Store, error) {
current, err := resolveDatabase(ctx)
if err != nil {
return nil, err
}
storeMu.RLock()
if active != nil && activeDB == current {
s := active
storeMu.RUnlock()
return s, nil
}
storeMu.RUnlock()
storeMu.Lock()
defer storeMu.Unlock()
if active != nil && activeDB == current {
return active, nil
}
s, err := buildStore(ctx, current, false)
if err != nil {
return nil, err
}
active = s
activeDB = current
return s, nil
}
// Build 直接按目标构造 store(不经 Active 缓存)。
func Build(ctx context.Context, database string) (*Store, error) {
return buildStore(ctx, database, false)
}
// BuildForMigration 构造迁移目标 store:与 Build 相同但不做冻结检查
// (迁移期间 log_db_migration=migrating 已冻结源库写入,目标库的清空/复制写入必须放行)。
func BuildForMigration(ctx context.Context, database string) (*Store, error) {
return buildStore(ctx, database, true)
}
// buildStore 按目标构造实现。skipFreeze 为 true 时该 store 跳过冻结检查
// (仅迁移任务的目标 store 使用)。gorm 分支 UserAccessLogs 用独立包装类型
// (gormLogStore 已占用 List/Count 方法名,无法再实现 UserAccessLogStore)。
func buildStore(ctx context.Context, database string, skipFreeze bool) (*Store, error) {
switch database {
case dbNameClickHouse:
ch := newClickHouseStore()
ch.skipFreeze = skipFreeze
return &Store{
AccessLogs: ch,
Observability: ch,
UserAccessLogs: newClickHouseUserAccessLogStore(),
Status: ch,
}, nil
case dbNamePostgres, dbNameSQLite:
gdb := db.DB(ctx)
g := newGormStore(gdb)
g.skipFreeze = skipFreeze
return &Store{
AccessLogs: g,
Observability: g,
UserAccessLogs: newUserAccessLogGormStore(gdb),
Status: g,
}, nil
default:
return nil, fmt.Errorf("unsupported log database: %s", database)
}
}
// Migrating 返回日志库是否处于迁移冻结状态。
func Migrating(ctx context.Context) bool {
v, err := getConfig(ctx, logMigrationKey)
if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "read log migration config failed: %v", err)
}
return false
}
return v == "migrating"
}
// Init 在 bootstrap 阶段预热一次激活 store(幂等,失败不致命——首次使用时再解析)。
func Init(ctx context.Context) {
_, _ = Active(ctx)
}
// InvalidateCache 清空日志库解析缓存(在修改 log_database 配置后显式调用)。
func InvalidateCache() {
storeMu.Lock()
defer storeMu.Unlock()
lastResolveTime = time.Time{}
lastResolveDB = ""
}
// ResetForTest 清空缓存的激活 store 与 config reader,便于测试注入。
func ResetForTest() {
storeMu.Lock()
active = nil
activeDB = ""
lastResolveDB = ""
lastResolveTime = time.Time{}
storeMu.Unlock()
configReader = nil
}
// ActiveDatabase 返回当前日志主库名(postgres|sqlite|clickhouse)。
func ActiveDatabase(ctx context.Context) (string, error) {
return resolveDatabase(ctx)
}
// resolveDatabase 读取 log_database:值缺失或 reader 未装配(首启)时按启动规则 seed;
// 已装配 reader 的真实读取错误直接透出,避免把读失败当首次启动。
func resolveDatabase(ctx context.Context) (string, error) {
storeMu.RLock()
if active != nil && time.Since(lastResolveTime) < resolveCacheTTL {
db := lastResolveDB
storeMu.RUnlock()
return db, nil
}
storeMu.RUnlock()
v, err := getConfig(ctx, logDatabaseKey)
if err != nil && !errors.Is(err, errConfigReaderNotWired) {
return "", err
}
resolved := v
if resolved == "" {
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
resolved = dbNameSQLite
if config.Config.Database.Enabled {
resolved = dbNamePostgres
}
if config.Config.ClickHouse.Enabled {
resolved = dbNameClickHouse
}
}
storeMu.Lock()
lastResolveDB = resolved
lastResolveTime = time.Now()
storeMu.Unlock()
return resolved, nil
}
@@ -0,0 +1,109 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"testing"
"time"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
func TestMigratingReadsConfig(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logMigrationKey {
return "migrating", nil
}
return "", nil
})
if !Migrating(context.Background()) {
t.Fatal("Migrating() = false, want true when key=migrating")
}
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", nil
})
if Migrating(context.Background()) {
t.Fatal("Migrating() = true, want false when key empty")
}
}
func TestResolveDatabaseDefaults(t *testing.T) {
ResetForTest()
// 配置缺失(reader 返回空值)时按主库规则 seed(config.Config 默认值由既有测试基建决定)。
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", nil
})
got, err := resolveDatabase(context.Background())
if err != nil {
t.Fatalf("resolveDatabase: %v", err)
}
if got != "postgres" && got != "sqlite" && got != "clickhouse" {
t.Fatalf("unexpected default log database: %s", got)
}
}
func TestResolveDatabaseSurfacesReadError(t *testing.T) {
ResetForTest()
wantErr := errors.New("boom")
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", wantErr
})
if _, err := resolveDatabase(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("resolveDatabase error = %v, want %v", err, wantErr)
}
}
func TestActiveBuildsStore(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logDatabaseKey {
return "sqlite", nil
}
return "", nil
})
store, err := Active(context.Background())
if err != nil {
t.Fatalf("Active: %v", err)
}
if store == nil {
t.Fatal("Active() returned nil store")
}
if store.AccessLogs == nil || store.Observability == nil || store.UserAccessLogs == nil || store.Status == nil {
t.Fatalf("Active() store fields not fully wired: %+v", store)
}
// 再次调用应命中缓存。
again, err := Active(context.Background())
if err != nil {
t.Fatalf("Active (cached): %v", err)
}
if again != store {
t.Fatal("Active() did not return cached store")
}
}
// TestClickHouseUserAccessLogBatchInsertFreeze 覆盖 CH 用户访问日志 flush 的冻结检查:
// 冻结期非空批次返回 ErrMigrating(在触碰 CH 连接之前),空批次直接成功。
func TestClickHouseUserAccessLogBatchInsertFreeze(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logMigrationKey {
return "migrating", nil
}
return "", nil
})
defer ResetForTest()
s := newClickHouseUserAccessLogStore()
ctx := context.Background()
now := time.Now()
if err := s.BatchInsert(ctx, []analyticsmodel.UserAccessLog{{UserID: 1, CreatedAt: now}}); !errors.Is(err, ErrMigrating) {
t.Fatalf("BatchInsert during migration: want ErrMigrating, got %v", err)
}
if err := s.BatchInsert(ctx, nil); err != nil {
t.Fatalf("BatchInsert empty batch: %v", err)
}
}
+87 -118
View File
@@ -5,7 +5,6 @@ package repository
import (
"context"
"math"
"sort"
"strings"
"time"
@@ -13,25 +12,21 @@ import (
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
)
type openFlareAccessLogBucketAggregateRow = analyticsmodel.NodeAccessLogBucketAggregate
type openFlareAccessLogBucketDimensionRow = analyticsmodel.NodeAccessLogBucketDimension
type openFlareAccessLogIPAggregateRow = analyticsmodel.NodeAccessLogIPAggregate
type openFlareAccessLogIPSummaryRow = analyticsmodel.NodeAccessLogIPSummary
type openFlareAccessLogIPTrendRow = analyticsmodel.NodeAccessLogIPTrend
type openFlareAccessLogWAFIPAggregateRow = analyticsmodel.NodeAccessLogWAFIPAggregate
const (
sortOrderAsc = "asc"
columnRemoteAddr = "remote_addr"
columnHost = "host"
secondsPerMinute = 60
)
// ListOpenFlareAccessLogWAFIPAggregates returns per-IP aggregates for WAF automatic rules.
func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLogWAFIPAggregate, error) {
rows, err := currentAccessLogStore().WAFIPAggregates(ctx, query)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.AccessLogs.WAFIPAggregates(ctx, query)
if err != nil {
return nil, err
}
@@ -59,39 +54,67 @@ func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query model.Open
return result, nil
}
// InsertOpenFlareAccessLogsBatch inserts access log rows into ClickHouse.
// InsertOpenFlareAccessLogsBatch inserts access log rows into the active log store.
func InsertOpenFlareAccessLogsBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
return currentAccessLogStore().InsertBatch(ctx, records)
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.AccessLogs.InsertBatch(ctx, records)
}
// ListOpenFlareAccessLogs lists access logs matching the query.
func ListOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
return currentAccessLogStore().List(ctx, query)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.List(ctx, query)
}
// CountOpenFlareAccessLogs counts access logs, distinct IPs, and total bytes sent matching the query.
func CountOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
return currentAccessLogStore().Count(ctx, query)
s, err := logstore.Active(ctx)
if err != nil {
return 0, 0, 0, err
}
return s.AccessLogs.Count(ctx, query)
}
// TrafficSummaryOpenFlareAccessLogs returns window-level request/error/UV/bytes summary.
func TrafficSummaryOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
return currentAccessLogStore().TrafficSummary(ctx, query)
s, err := logstore.Active(ctx)
if err != nil {
return model.OpenFlareAccessLogTrafficSummary{}, err
}
return s.AccessLogs.TrafficSummary(ctx, query)
}
// ValueCountsOpenFlareAccessLogs groups logs by status_code, host, path, remote_addr, or user_agent.
func ValueCountsOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
return currentAccessLogStore().ValueCounts(ctx, query, column, limit)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.ValueCounts(ctx, query, column, limit)
}
// NodeAggregatesOpenFlareAccessLogs returns per-node request/error/UV for the window.
func NodeAggregatesOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
return currentAccessLogStore().NodeAggregates(ctx, query)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.NodeAggregates(ctx, query)
}
// ListOpenFlareAccessLogRegionCounts returns region counts for access logs.
func ListOpenFlareAccessLogRegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
return currentAccessLogStore().RegionCounts(ctx, nodeID, since, limit)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.RegionCounts(ctx, nodeID, since, limit)
}
// ListOpenFlareAccessLogBuckets lists folded access log buckets.
@@ -106,7 +129,11 @@ func CountOpenFlareAccessLogBuckets(ctx context.Context, query model.OpenFlareAc
if bucketSeconds <= 0 {
bucketSeconds = 180
}
return currentAccessLogStore().CountBuckets(ctx, filter, bucketSeconds)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.CountBuckets(ctx, filter, bucketSeconds)
}
// ListOpenFlareAccessLogBucketIPs lists folded IP rows for a bucket window.
@@ -139,7 +166,11 @@ func ListOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlar
// CountOpenFlareAccessLogIPSummaries counts IP summaries.
func CountOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery) (int64, error) {
filter := openFlareAccessLogQueryFromIPSummary(query)
return currentAccessLogStore().CountIPSummaries(ctx, filter)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.CountIPSummaries(ctx, filter)
}
// ListOpenFlareAccessLogIPTrend lists IP trend points.
@@ -158,7 +189,11 @@ func ListOpenFlareAccessLogIPTrend(ctx context.Context, query model.OpenFlareAcc
if bucketSeconds <= 0 {
bucketSeconds = 1800
}
rows, err := currentAccessLogStore().IPTrend(ctx, filter, bucketSeconds)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.AccessLogs.IPTrend(ctx, filter, bucketSeconds)
if err != nil {
return nil, err
}
@@ -174,17 +209,29 @@ func ListOpenFlareAccessLogIPTrend(ctx context.Context, query model.OpenFlareAcc
// DeleteAllOpenFlareAccessLogs deletes all access logs.
func DeleteAllOpenFlareAccessLogs(ctx context.Context) (int64, error) {
return currentAccessLogStore().DeleteAll(ctx)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.DeleteAll(ctx)
}
// DeleteOpenFlareAccessLogsBefore deletes access logs older than cutoff.
func DeleteOpenFlareAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return currentAccessLogStore().DeleteBefore(ctx, cutoff)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.DeleteBefore(ctx, cutoff)
}
// DeleteOpenFlareAccessLogsByNodeBefore deletes access logs for a node older than cutoff.
func DeleteOpenFlareAccessLogsByNodeBefore(ctx context.Context, nodeID string, cutoff time.Time) (int64, error) {
return currentAccessLogStore().DeleteByNodeBefore(ctx, nodeID, cutoff)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.DeleteByNodeBefore(ctx, nodeID, cutoff)
}
func buildOpenFlareAccessLogBucketRows(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) ([]*model.OpenFlareAccessLogBucketRow, error) {
@@ -193,8 +240,11 @@ func buildOpenFlareAccessLogBucketRows(ctx context.Context, query model.OpenFlar
if bucketSeconds <= 0 {
bucketSeconds = 180
}
partials, err := currentAccessLogStore().BucketAggregates(ctx, filter, bucketSeconds)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
partials, err := s.AccessLogs.BucketAggregates(ctx, filter, bucketSeconds)
if err != nil {
return nil, err
}
@@ -242,7 +292,11 @@ func buildOpenFlareAccessLogBucketIPRows(ctx context.Context, query model.OpenFl
func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*analyticsmodel.NodeAccessLogIPSummary, error) {
filter := openFlareAccessLogQueryFromIPSummary(query)
partials, err := currentAccessLogStore().IPSummaries(ctx, filter, recentSince)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
partials, err := s.AccessLogs.IPSummaries(ctx, filter, recentSince)
if err != nil {
return nil, err
}
@@ -268,7 +322,11 @@ func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query model.OpenF
}
func queryOpenFlareAccessLogIPAggregateRows(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
partials, err := currentAccessLogStore().IPAggregates(ctx, filter, exactRemoteAddr)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
partials, err := s.AccessLogs.IPAggregates(ctx, filter, exactRemoteAddr)
if err != nil {
return nil, err
}
@@ -385,92 +443,3 @@ func openFlareAccessLogCompareInt64(left int64, right int64) int {
return 0
}
}
func openFlareAccessLogStatusCodeToInt32(code int) int32 {
switch {
case code > math.MaxInt32:
return math.MaxInt32
case code < math.MinInt32:
return math.MinInt32
default:
return int32(code)
}
}
func sortOpenFlareAccessLogBucketRows(items []*model.OpenFlareAccessLogBucketRow, sortBy string, sortOrder string) {
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
var compare int
switch strings.TrimSpace(sortBy) {
case "request_count":
compare = openFlareAccessLogCompareInt64(left.RequestCount, right.RequestCount)
default:
compare = openFlareAccessLogCompareInt64(left.BucketEpoch, right.BucketEpoch)
}
if compare == 0 {
compare = openFlareAccessLogCompareInt64(left.BucketEpoch, right.BucketEpoch)
}
if desc {
return compare > 0
}
return compare < 0
})
}
func sortOpenFlareAccessLogIPSummaryRows(items []*model.OpenFlareAccessLogIPSummaryRow, sortBy string, sortOrder string) {
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
var compare int
switch strings.TrimSpace(sortBy) {
case "request_length", "bytes_received":
compare = openFlareAccessLogCompareInt64(left.BytesReceived, right.BytesReceived)
case "bytes_sent":
compare = openFlareAccessLogCompareInt64(left.BytesSent, right.BytesSent)
case "success_ratio":
compare = openFlareAccessLogCompareFloat64(left.SuccessRatio, right.SuccessRatio)
case "last_seen_at":
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
case "remote_addr":
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
default:
compare = openFlareAccessLogCompareInt64(left.TotalRequests, right.TotalRequests)
}
if compare == 0 {
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
}
if compare == 0 {
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
}
if desc {
return compare > 0
}
return compare < 0
})
}
func openFlareAccessLogCompareFloat64(left, right float64) int {
if left < right {
return -1
}
if left > right {
return 1
}
return 0
}
func openFlareAccessLogUintToInt64(value uint64) int64 {
if value > math.MaxInt64 {
return math.MaxInt64
}
return int64(value)
}
@@ -1,308 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"math"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
// AccessLogInsertHooks queues node access logs for async ClickHouse write.
// Wired from openflare/chwriter.Init so model never imports the apps layer.
type AccessLogInsertHooks struct {
QueueNodeAccessLogs func(logs []analyticsmodel.NodeAccessLog)
}
var (
accessLogInsertHooksMu sync.RWMutex
accessLogInsertHooks AccessLogInsertHooks
)
// SetAccessLogInsertHooks registers async queue callbacks for access log inserts.
func SetAccessLogInsertHooks(hooks AccessLogInsertHooks) {
accessLogInsertHooksMu.Lock()
accessLogInsertHooks = hooks
accessLogInsertHooksMu.Unlock()
}
func currentAccessLogInsertHooks() AccessLogInsertHooks {
accessLogInsertHooksMu.RLock()
defer accessLogInsertHooksMu.RUnlock()
return accessLogInsertHooks
}
type accessLogStore interface {
InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error
List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error)
Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error)
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error)
BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error)
CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error)
IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error)
WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error)
IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error)
CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error)
IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error)
TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error)
ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error)
NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error)
DeleteAll(ctx context.Context) (int64, error)
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error)
}
var (
accessLogStoreMu sync.RWMutex
accessLogStoreHolder accessLogStore
)
func currentAccessLogStore() accessLogStore {
accessLogStoreMu.RLock()
defer accessLogStoreMu.RUnlock()
if accessLogStoreHolder != nil {
return accessLogStoreHolder
}
return clickhouseAccessLogStore{}
}
// SetAccessLogStoreForTest swaps the access log store implementation for unit tests.
func SetAccessLogStoreForTest(store accessLogStore) func() {
accessLogStoreMu.Lock()
previous := accessLogStoreHolder
accessLogStoreHolder = store
accessLogStoreMu.Unlock()
return func() {
accessLogStoreMu.Lock()
accessLogStoreHolder = previous
accessLogStoreMu.Unlock()
}
}
// NewMemoryAccessLogStore returns an in-memory access log store for unit tests.
func NewMemoryAccessLogStore() accessLogStore {
return &memoryAccessLogStore{
records: make([]*model.OpenFlareAccessLog, 0),
}
}
type clickhouseAccessLogStore struct{}
func (clickhouseAccessLogStore) InsertBatch(_ context.Context, records []*model.OpenFlareAccessLog) error {
logs := make([]analyticsmodel.NodeAccessLog, 0, len(records))
for _, record := range records {
if record == nil {
continue
}
logs = append(logs, toAnalyticsNodeAccessLog(record))
}
if hook := currentAccessLogInsertHooks().QueueNodeAccessLogs; hook != nil {
hook(logs)
}
return nil
}
func (clickhouseAccessLogStore) List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
if err != nil {
return nil, err
}
return fromAnalyticsNodeAccessLogs(rows), nil
}
func (clickhouseAccessLogStore) Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
func (clickhouseAccessLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
if err != nil {
return nil, err
}
result := make([]*model.OpenFlareAccessLogRegionCount, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareAccessLogRegionCount{
Region: row.Region,
Count: row.Count,
}
}
return result, nil
}
func (clickhouseAccessLogStore) BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error) {
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
}
func (clickhouseAccessLogStore) CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
}
func (clickhouseAccessLogStore) BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, bucketSeconds)
}
func (clickhouseAccessLogStore) IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error) {
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), exactRemoteAddr)
}
func (clickhouseAccessLogStore) IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error) {
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), recentSince)
}
func (clickhouseAccessLogStore) CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error) {
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
}
func (clickhouseAccessLogStore) WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
}
func (clickhouseAccessLogStore) IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
}
func (clickhouseAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
}
func (clickhouseAccessLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
}
func (clickhouseAccessLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
}
func (clickhouseAccessLogStore) TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
if err != nil {
return model.OpenFlareAccessLogTrafficSummary{}, err
}
return model.OpenFlareAccessLogTrafficSummary{
RequestCount: row.RequestCount,
ErrorCount: row.ErrorCount,
UniqueIPCount: row.UniqueIPCount,
BytesSent: row.BytesSent,
RequestLength: row.RequestLength,
NodeCount: row.NodeCount,
}, nil
}
func (clickhouseAccessLogStore) ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, limit)
if err != nil {
return nil, err
}
result := make([]model.OpenFlareAccessLogValueCount, len(rows))
for i, row := range rows {
result[i] = model.OpenFlareAccessLogValueCount{Value: row.Value, Count: row.Count}
}
return result, nil
}
func (clickhouseAccessLogStore) NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
if err != nil {
return nil, err
}
result := make([]model.OpenFlareAccessLogNodeAggregate, len(rows))
for i, row := range rows {
result[i] = model.OpenFlareAccessLogNodeAggregate{
NodeID: row.NodeID,
RequestCount: row.RequestCount,
ErrorCount: row.ErrorCount,
UniqueIPCount: row.UniqueIPCount,
}
}
return result, nil
}
func toNodeAccessLogFilter(query model.OpenFlareAccessLogQuery) analyticsrepo.NodeAccessLogFilter {
return analyticsrepo.NodeAccessLogFilter{
NodeID: query.NodeID,
RemoteAddr: query.RemoteAddr,
Host: query.Host,
Hosts: query.Hosts,
Path: query.Path,
Since: query.Since,
Until: query.Until,
Page: query.Page,
PageSize: query.PageSize,
SortBy: query.SortBy,
SortOrder: query.SortOrder,
}
}
func toAnalyticsNodeAccessLog(record *model.OpenFlareAccessLog) analyticsmodel.NodeAccessLog {
var bytesSent uint64
if record.BytesSent > 0 {
bytesSent = uint64(record.BytesSent)
}
var requestLength uint64
if record.RequestLength > 0 {
requestLength = uint64(record.RequestLength)
}
var requestTimeMs uint32
if record.RequestTimeMs > 0 && record.RequestTimeMs <= int64(math.MaxUint32) {
requestTimeMs = uint32(record.RequestTimeMs)
}
return analyticsmodel.NodeAccessLog{
ID: record.ID,
NodeID: record.NodeID,
LoggedAt: record.LoggedAt,
RemoteAddr: record.RemoteAddr,
Region: record.Region,
Host: record.Host,
Path: record.Path,
UserAgent: record.UserAgent,
CacheStatus: record.CacheStatus,
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
BytesSent: bytesSent,
RequestLength: requestLength,
RequestTimeMs: requestTimeMs,
CreatedAt: record.CreatedAt,
}
}
func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*model.OpenFlareAccessLog {
result := make([]*model.OpenFlareAccessLog, len(rows))
for index, row := range rows {
var bytesSent int64
if row.BytesSent <= math.MaxInt64 {
bytesSent = int64(row.BytesSent)
} else {
bytesSent = math.MaxInt64
}
var requestLength int64
if row.RequestLength <= math.MaxInt64 {
requestLength = int64(row.RequestLength)
} else {
requestLength = math.MaxInt64
}
result[index] = &model.OpenFlareAccessLog{
ID: row.ID,
NodeID: row.NodeID,
LoggedAt: row.LoggedAt,
RemoteAddr: row.RemoteAddr,
Region: row.Region,
Host: row.Host,
Path: row.Path,
UserAgent: row.UserAgent,
CacheStatus: row.CacheStatus,
StatusCode: int(row.StatusCode),
BytesSent: bytesSent,
RequestLength: requestLength,
RequestTimeMs: int64(row.RequestTimeMs),
CreatedAt: row.CreatedAt,
}
}
return result
}
@@ -1,710 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"net"
"net/http"
"net/netip"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
)
const (
accessLogColumnStatusCode = "status_code"
accessLogColumnHost = "host"
accessLogColumnPath = "path"
accessLogColumnRemoteAddr = "remote_addr"
accessLogColumnUserAgent = "user_agent"
)
type memoryAccessLogStore struct {
mu sync.RWMutex
records []*model.OpenFlareAccessLog
}
func (s *memoryAccessLogStore) InsertBatch(_ context.Context, records []*model.OpenFlareAccessLog) error {
s.mu.Lock()
defer s.mu.Unlock()
now := time.Now().UTC()
for _, record := range records {
if record == nil {
continue
}
copyRecord := *record
if copyRecord.ID == 0 {
copyRecord.ID = idgen.NextUint64ID()
}
if copyRecord.CreatedAt.IsZero() {
copyRecord.CreatedAt = now
}
copyRecord.LoggedAt = copyRecord.LoggedAt.UTC()
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
s.records = append(s.records, &copyRecord)
}
return nil
}
func (s *memoryAccessLogStore) List(_ context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(query)
sortOpenFlareAccessLogRows(rows, query.SortBy, query.SortOrder)
if query.PageSize > 0 {
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
return cloneAccessLogSlice(rows[start:end]), nil
}
return cloneAccessLogSlice(rows), nil
}
func (s *memoryAccessLogStore) Count(_ context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(query)
ips := make(map[string]struct{})
var totalBytes int64
for _, row := range rows {
totalBytes += row.BytesSent
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
ips[remoteAddr] = struct{}{}
}
return int64(len(rows)), int64(len(ips)), totalBytes, nil
}
func (s *memoryAccessLogStore) RegionCounts(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(model.OpenFlareAccessLogQuery{NodeID: nodeID, Since: since})
counts := make(map[string]int64)
for _, row := range rows {
region := strings.TrimSpace(row.Region)
if region == "" {
continue
}
counts[region]++
}
result := make([]*model.OpenFlareAccessLogRegionCount, 0, len(counts))
for region, count := range counts {
result = append(result, &model.OpenFlareAccessLogRegionCount{Region: region, Count: count})
}
sort.Slice(result, func(i, j int) bool {
if result[i].Count == result[j].Count {
return result[i].Region < result[j].Region
}
return result[i].Count > result[j].Count
})
if limit > 0 && len(result) > limit {
result = result[:limit]
}
return result, nil
}
func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
type bucketAccumulator struct {
openFlareAccessLogBucketAggregateRow
uniqueIPs map[string]struct{}
uniqueHosts map[string]struct{}
}
aggregates := make(map[int64]*bucketAccumulator)
for _, row := range rows {
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
item := aggregates[bucketEpoch]
if item == nil {
item = &bucketAccumulator{
openFlareAccessLogBucketAggregateRow: openFlareAccessLogBucketAggregateRow{BucketEpoch: bucketEpoch},
uniqueIPs: make(map[string]struct{}),
uniqueHosts: make(map[string]struct{}),
}
aggregates[bucketEpoch] = item
}
item.RequestCount++
item.BytesSent += row.BytesSent
item.RequestLength += row.RequestLength
switch {
case row.StatusCode < 400:
item.SuccessCount++
case row.StatusCode < 500:
item.ClientErrorCount++
default:
item.ServerErrorCount++
}
if remoteAddr := strings.TrimSpace(row.RemoteAddr); remoteAddr != "" {
item.uniqueIPs[remoteAddr] = struct{}{}
}
if host := strings.TrimSpace(row.Host); host != "" {
item.uniqueHosts[host] = struct{}{}
}
}
result := make([]openFlareAccessLogBucketAggregateRow, 0, len(aggregates))
for _, item := range aggregates {
item.UniqueIPCount = int64(len(item.uniqueIPs))
item.UniqueHostCount = int64(len(item.uniqueHosts))
result = append(result, item.openFlareAccessLogBucketAggregateRow)
}
bucketRows := make([]*model.OpenFlareAccessLogBucketRow, len(result))
for index := range result {
bucketRows[index] = &model.OpenFlareAccessLogBucketRow{
BucketEpoch: result[index].BucketEpoch,
RequestCount: result[index].RequestCount,
UniqueIPCount: result[index].UniqueIPCount,
UniqueHostCount: result[index].UniqueHostCount,
SuccessCount: result[index].SuccessCount,
ClientErrorCount: result[index].ClientErrorCount,
ServerErrorCount: result[index].ServerErrorCount,
BytesSent: result[index].BytesSent,
RequestLength: result[index].RequestLength,
}
}
sortOpenFlareAccessLogBucketRows(bucketRows, filter.SortBy, filter.SortOrder)
for index := range result {
result[index] = openFlareAccessLogBucketAggregateRow{
BucketEpoch: bucketRows[index].BucketEpoch,
RequestCount: bucketRows[index].RequestCount,
UniqueIPCount: bucketRows[index].UniqueIPCount,
UniqueHostCount: bucketRows[index].UniqueHostCount,
SuccessCount: bucketRows[index].SuccessCount,
ClientErrorCount: bucketRows[index].ClientErrorCount,
ServerErrorCount: bucketRows[index].ServerErrorCount,
BytesSent: bucketRows[index].BytesSent,
RequestLength: bucketRows[index].RequestLength,
}
}
if filter.PageSize > 0 {
start, end := openFlareAccessLogPaginateBounds(len(result), filter.Page, filter.PageSize)
return result[start:end], nil
}
return result, nil
}
func (s *memoryAccessLogStore) CountBuckets(_ context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
seen := make(map[int64]struct{})
for _, row := range rows {
seen[memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)] = struct{}{}
}
return int64(len(seen)), nil
}
func (s *memoryAccessLogStore) BucketDimensions(_ context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
seen := make(map[int64]map[string]struct{})
var result []openFlareAccessLogBucketDimensionRow
for _, row := range rows {
var value string
switch column {
case columnRemoteAddr:
value = strings.TrimSpace(row.RemoteAddr)
case columnHost:
value = strings.TrimSpace(row.Host)
default:
continue
}
if value == "" {
continue
}
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
if seen[bucketEpoch] == nil {
seen[bucketEpoch] = make(map[string]struct{})
}
if _, ok := seen[bucketEpoch][value]; ok {
continue
}
seen[bucketEpoch][value] = struct{}{}
result = append(result, openFlareAccessLogBucketDimensionRow{BucketEpoch: bucketEpoch, Value: value})
}
return result, nil
}
func (s *memoryAccessLogStore) IPAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
if exactRemoteAddr && strings.TrimSpace(filter.RemoteAddr) == "" {
return []openFlareAccessLogIPAggregateRow{}, nil
}
rows := s.filterRecords(filter)
aggregates := make(map[string]*openFlareAccessLogIPAggregateRow)
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
if exactRemoteAddr && remoteAddr != strings.TrimSpace(filter.RemoteAddr) {
continue
}
item := aggregates[remoteAddr]
if item == nil {
item = &openFlareAccessLogIPAggregateRow{RemoteAddr: remoteAddr}
aggregates[remoteAddr] = item
}
item.RequestCount++
epoch := row.LoggedAt.UTC().Unix()
if epoch > item.LastSeenEpoch {
item.LastSeenEpoch = epoch
}
switch {
case row.StatusCode < 400:
item.SuccessCount++
case row.StatusCode < 500:
item.ClientErrorCount++
default:
item.ServerErrorCount++
}
}
result := make([]openFlareAccessLogIPAggregateRow, 0, len(aggregates))
for _, item := range aggregates {
result = append(result, *item)
}
return result, nil
}
func (s *memoryAccessLogStore) IPSummaries(_ context.Context, filter model.OpenFlareAccessLogQuery, _ time.Time) ([]openFlareAccessLogIPSummaryRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
type aggregate struct {
RemoteAddr string
Region string
RegionEpoch int64
TotalRequests int64
Success2xxCount int64
BytesReceived int64
BytesSent int64
LastSeenEpoch int64
}
aggregates := make(map[string]*aggregate)
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
item := aggregates[remoteAddr]
if item == nil {
item = &aggregate{RemoteAddr: remoteAddr}
aggregates[remoteAddr] = item
}
item.TotalRequests++
if row.StatusCode >= 200 && row.StatusCode < 300 {
item.Success2xxCount++
}
item.BytesReceived += row.RequestLength
item.BytesSent += row.BytesSent
epoch := row.LoggedAt.UTC().Unix()
if epoch > item.LastSeenEpoch {
item.LastSeenEpoch = epoch
}
if epoch >= item.RegionEpoch {
item.RegionEpoch = epoch
item.Region = strings.TrimSpace(row.Region)
}
}
summaryRows := make([]*model.OpenFlareAccessLogIPSummaryRow, 0, len(aggregates))
for _, item := range aggregates {
ratio := 0.0
if item.TotalRequests > 0 {
ratio = float64(item.Success2xxCount) / float64(item.TotalRequests)
}
summaryRows = append(summaryRows, &model.OpenFlareAccessLogIPSummaryRow{
RemoteAddr: item.RemoteAddr,
Region: item.Region,
TotalRequests: item.TotalRequests,
Success2xxCount: item.Success2xxCount,
SuccessRatio: ratio,
BytesReceived: item.BytesReceived,
BytesSent: item.BytesSent,
RecentRequests: 0,
LastSeenEpoch: item.LastSeenEpoch,
})
}
sortOpenFlareAccessLogIPSummaryRows(summaryRows, filter.SortBy, filter.SortOrder)
if filter.PageSize > 0 {
start, end := openFlareAccessLogPaginateBounds(len(summaryRows), filter.Page, filter.PageSize)
summaryRows = summaryRows[start:end]
}
result := make([]openFlareAccessLogIPSummaryRow, len(summaryRows))
for index, item := range summaryRows {
result[index] = openFlareAccessLogIPSummaryRow{
RemoteAddr: item.RemoteAddr,
Region: item.Region,
TotalRequests: item.TotalRequests,
Success2xxCount: item.Success2xxCount,
SuccessRatio: item.SuccessRatio,
BytesReceived: item.BytesReceived,
BytesSent: item.BytesSent,
RecentRequests: 0,
LastSeenEpoch: item.LastSeenEpoch,
}
}
return result, nil
}
func (s *memoryAccessLogStore) CountIPSummaries(_ context.Context, filter model.OpenFlareAccessLogQuery) (int64, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
seen := make(map[string]struct{})
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
seen[remoteAddr] = struct{}{}
}
return int64(len(seen)), nil
}
func (s *memoryAccessLogStore) WAFIPAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
aggregates := make(map[string]*openFlareAccessLogWAFIPAggregateRow)
order := make([]string, 0)
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
item := aggregates[remoteAddr]
if item == nil {
item = &openFlareAccessLogWAFIPAggregateRow{
RemoteAddr: remoteAddr,
StatusCounts: make(map[int]int64),
}
aggregates[remoteAddr] = item
order = append(order, remoteAddr)
}
item.RequestCount++
item.StatusCounts[row.StatusCode]++
if row.StatusCode == http.StatusNotFound {
item.Status404Count++
}
if row.StatusCode >= 400 && row.StatusCode < 500 {
item.ClientErrorCount++
}
if row.StatusCode >= http.StatusInternalServerError {
item.ServerErrorCount++
}
if memoryAccessLogHostIsIPLiteral(row.Host) {
item.IPHostCount++
}
epoch := row.LoggedAt.UTC().Unix()
if epoch > item.LastSeenEpoch {
item.LastSeenEpoch = epoch
}
}
result := make([]openFlareAccessLogWAFIPAggregateRow, 0, len(order))
for _, remoteAddr := range order {
if item := aggregates[remoteAddr]; item != nil {
result = append(result, *item)
}
}
return result, nil
}
func (s *memoryAccessLogStore) IPTrend(_ context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
aggregates := make(map[int64]int64)
for _, row := range rows {
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
aggregates[bucketEpoch]++
}
result := make([]openFlareAccessLogIPTrendRow, 0, len(aggregates))
for bucketEpoch, count := range aggregates {
result = append(result, openFlareAccessLogIPTrendRow{BucketEpoch: bucketEpoch, RequestCount: count})
}
sort.Slice(result, func(i, j int) bool { return result[i].BucketEpoch < result[j].BucketEpoch })
return result, nil
}
func (s *memoryAccessLogStore) DeleteAll(_ context.Context) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
count := int64(len(s.records))
s.records = nil
return count, nil
}
func (s *memoryAccessLogStore) DeleteBefore(_ context.Context, cutoff time.Time) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
cutoff = cutoff.UTC()
remaining := make([]*model.OpenFlareAccessLog, 0, len(s.records))
var deleted int64
for _, row := range s.records {
if row.LoggedAt.Before(cutoff) {
deleted++
continue
}
remaining = append(remaining, row)
}
s.records = remaining
return deleted, nil
}
func (s *memoryAccessLogStore) DeleteByNodeBefore(_ context.Context, nodeID string, before time.Time) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
before = before.UTC()
remaining := make([]*model.OpenFlareAccessLog, 0, len(s.records))
var deleted int64
for _, row := range s.records {
if row.NodeID == nodeID && row.LoggedAt.Before(before) {
deleted++
continue
}
remaining = append(remaining, row)
}
s.records = remaining
return deleted, nil
}
func (s *memoryAccessLogStore) TrafficSummary(_ context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
ips := make(map[string]struct{})
nodes := make(map[string]struct{})
var summary model.OpenFlareAccessLogTrafficSummary
for _, row := range rows {
summary.RequestCount++
summary.BytesSent += row.BytesSent
summary.RequestLength += row.RequestLength
if row.StatusCode >= http.StatusInternalServerError {
summary.ErrorCount++
}
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
ips[ip] = struct{}{}
}
if id := strings.TrimSpace(row.NodeID); id != "" {
nodes[id] = struct{}{}
}
}
summary.UniqueIPCount = int64(len(ips))
summary.NodeCount = int64(len(nodes))
return summary, nil
}
func (s *memoryAccessLogStore) ValueCounts(_ context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
s.mu.RLock()
defer s.mu.RUnlock()
col := strings.TrimSpace(strings.ToLower(column))
switch col {
case accessLogColumnStatusCode, accessLogColumnHost, accessLogColumnPath, accessLogColumnRemoteAddr, accessLogColumnUserAgent:
default:
return nil, nil
}
rows := s.filterRecords(filter)
counts := make(map[string]int64)
for _, row := range rows {
var value string
switch col {
case accessLogColumnStatusCode:
value = strconv.Itoa(row.StatusCode)
case accessLogColumnHost:
value = strings.TrimSpace(row.Host)
case accessLogColumnPath:
value = strings.TrimSpace(row.Path)
case accessLogColumnRemoteAddr:
value = strings.TrimSpace(row.RemoteAddr)
case accessLogColumnUserAgent:
value = strings.TrimSpace(row.UserAgent)
}
if value == "" {
continue
}
counts[value]++
}
result := make([]model.OpenFlareAccessLogValueCount, 0, len(counts))
for value, count := range counts {
result = append(result, model.OpenFlareAccessLogValueCount{Value: value, Count: count})
}
sort.Slice(result, func(i, j int) bool {
if result[i].Count == result[j].Count {
return result[i].Value < result[j].Value
}
return result[i].Count > result[j].Count
})
if limit > 0 && len(result) > limit {
result = result[:limit]
}
return result, nil
}
func (s *memoryAccessLogStore) NodeAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
type acc struct {
model.OpenFlareAccessLogNodeAggregate
ips map[string]struct{}
}
byNode := make(map[string]*acc)
for _, row := range rows {
id := strings.TrimSpace(row.NodeID)
if id == "" {
continue
}
item := byNode[id]
if item == nil {
item = &acc{
OpenFlareAccessLogNodeAggregate: model.OpenFlareAccessLogNodeAggregate{NodeID: id},
ips: make(map[string]struct{}),
}
byNode[id] = item
}
item.RequestCount++
if row.StatusCode >= http.StatusInternalServerError {
item.ErrorCount++
}
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
item.ips[ip] = struct{}{}
}
}
result := make([]model.OpenFlareAccessLogNodeAggregate, 0, len(byNode))
for _, item := range byNode {
item.UniqueIPCount = int64(len(item.ips))
result = append(result, item.OpenFlareAccessLogNodeAggregate)
}
sort.Slice(result, func(i, j int) bool {
if result[i].RequestCount == result[j].RequestCount {
return result[i].NodeID < result[j].NodeID
}
return result[i].RequestCount > result[j].RequestCount
})
return result, nil
}
func (s *memoryAccessLogStore) filterRecords(query model.OpenFlareAccessLogQuery) []*model.OpenFlareAccessLog {
result := make([]*model.OpenFlareAccessLog, 0, len(s.records))
for _, row := range s.records {
if !memoryAccessLogMatches(row, query) {
continue
}
result = append(result, row)
}
return result
}
func memoryAccessLogMatches(row *model.OpenFlareAccessLog, query model.OpenFlareAccessLogQuery) bool {
if row == nil {
return false
}
if trimmed := strings.TrimSpace(query.NodeID); trimmed != "" && row.NodeID != trimmed {
return false
}
if trimmed := strings.TrimSpace(query.RemoteAddr); trimmed != "" && !strings.HasPrefix(strings.TrimSpace(row.RemoteAddr), trimmed) {
return false
}
if len(query.Hosts) > 0 {
rowHost := strings.ToLower(strings.TrimSpace(row.Host))
matched := false
for _, host := range query.Hosts {
if strings.ToLower(strings.TrimSpace(host)) == rowHost {
matched = true
break
}
}
if !matched {
return false
}
} else if trimmed := strings.TrimSpace(query.Host); trimmed != "" && !strings.HasPrefix(strings.TrimSpace(row.Host), trimmed) {
return false
}
if trimmed := strings.TrimSpace(query.Path); trimmed != "" && !strings.HasPrefix(strings.TrimSpace(row.Path), trimmed) {
return false
}
if !query.Since.IsZero() && row.LoggedAt.Before(query.Since) {
return false
}
if !query.Until.IsZero() && !row.LoggedAt.Before(query.Until) {
return false
}
return true
}
func memoryAccessLogHostIsIPLiteral(value string) bool {
host := strings.TrimSpace(value)
if host == "" {
return false
}
if parsedHost, _, err := net.SplitHostPort(host); err == nil {
host = parsedHost
}
host = strings.Trim(host, "[]")
_, err := netip.ParseAddr(host)
return err == nil
}
func memoryAccessLogBucketEpoch(loggedAt time.Time, bucketSeconds int64) int64 {
if bucketSeconds <= 0 {
bucketSeconds = 180
}
epoch := loggedAt.UTC().Unix()
return (epoch / bucketSeconds) * bucketSeconds
}
func cloneAccessLogSlice(rows []*model.OpenFlareAccessLog) []*model.OpenFlareAccessLog {
result := make([]*model.OpenFlareAccessLog, len(rows))
for index, row := range rows {
if row == nil {
continue
}
copyRecord := *row
result[index] = &copyRecord
}
return result
}
func sortOpenFlareAccessLogRows(items []*model.OpenFlareAccessLog, sortBy string, sortOrder string) {
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
var compare int
switch strings.TrimSpace(sortBy) {
case "status_code":
compare = left.StatusCode - right.StatusCode
case columnRemoteAddr:
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
case columnHost:
compare = strings.Compare(left.Host, right.Host)
case "path":
compare = strings.Compare(left.Path, right.Path)
default:
compare = openFlareAccessLogCompareInt64(left.LoggedAt.Unix(), right.LoggedAt.Unix())
}
if compare == 0 {
compare = openFlareAccessLogCompareInt64(left.LoggedAt.Unix(), right.LoggedAt.Unix())
}
if compare == 0 {
compare = openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(left.ID), openFlareAccessLogUintToInt64(right.ID))
}
if desc {
return compare > 0
}
return compare < 0
})
}
@@ -6,21 +6,60 @@ package repository
import (
"context"
"fmt"
"sync/atomic"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// accessLogTestDBSeq 保证每个测试获得独立的 sqlite 内存库(cache=shared 下同名 DSN 复用同一库)。
var accessLogTestDBSeq int64
func setupOpenFlareAccessLogTestEnvironment(t *testing.T) (context.Context, func()) {
t.Helper()
store := NewMemoryAccessLogStore()
reset := SetAccessLogStoreForTest(store)
return context.Background(), func() {
reset()
dsn := fmt.Sprintf("file:repo-access-log-test-%d?mode=memory&cache=shared", atomic.AddInt64(&accessLogTestDBSeq, 1))
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
require.NoError(t, err)
require.NoError(t, gdb.AutoMigrate(&analyticsmodel.NodeAccessLog{}))
db.SetDB(gdb)
logstore.ResetForTest()
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == model.ConfigKeyLogDatabase {
return "sqlite", nil
}
return "", nil
})
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{})
ctx := context.Background()
store, err := logstore.Active(ctx)
require.NoError(t, err)
// 写入入口只入队;测试环境立即 flush,保证后续查询可见。
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
require.NoError(t, store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs))
},
})
return ctx, func() {
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
logstore.ResetForTest()
db.SetDB(nil)
}
}
@@ -53,6 +92,7 @@ func TestListOpenFlareAccessLogsPaginated(t *testing.T) {
require.NoError(t, InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{record}))
}
// 0-based 分页与 CH ListNodeAccessLogs 一致:page=1 size=5 → OFFSET 5 → /path-05..09。
query := model.OpenFlareAccessLogQuery{
NodeID: "node-page",
Since: now.Add(-24 * time.Hour),
@@ -81,6 +121,7 @@ func TestCountOpenFlareAccessLogs(t *testing.T) {
totalRecords, totalIPs, _, err := CountOpenFlareAccessLogs(ctx, query)
require.NoError(t, err)
assert.Equal(t, int64(5), totalRecords)
// GORM 与 CH 一致:distinct IP 排除空 remote_addr(CH uniqExactIf(remote_addr, remote_addr != ''))。
assert.Equal(t, int64(3), totalIPs)
}
@@ -1,77 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
)
// Hook setters are process-global; keep these tests serial.
func TestObservabilityInsertHooksAreInvoked(t *testing.T) {
var gotSnapshot analyticsmodel.NodeMetricSnapshot
SetObservabilityInsertHooks(ObservabilityInsertHooks{
QueueMetricSnapshot: func(s analyticsmodel.NodeMetricSnapshot) {
gotSnapshot = s
},
})
t.Cleanup(func() {
SetObservabilityInsertHooks(ObservabilityInsertHooks{})
})
record := &model.OpenFlareMetricSnapshot{
NodeID: "node-1",
CapturedAt: time.Unix(100, 0).UTC(),
}
if err := (clickhouseObservabilityStore{}).InsertMetricSnapshot(context.Background(), record); err != nil {
t.Fatalf("InsertMetricSnapshot error = %v", err)
}
if gotSnapshot.NodeID != "node-1" {
t.Fatalf("hook node id = %q, want node-1", gotSnapshot.NodeID)
}
}
func TestAccessLogInsertHooksAreInvoked(t *testing.T) {
var got []analyticsmodel.NodeAccessLog
SetAccessLogInsertHooks(AccessLogInsertHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
got = append([]analyticsmodel.NodeAccessLog(nil), logs...)
},
})
t.Cleanup(func() {
SetAccessLogInsertHooks(AccessLogInsertHooks{})
})
records := []*model.OpenFlareAccessLog{
{NodeID: "n1", Path: "/a"},
{NodeID: "n1", Path: "/b"},
}
if err := (clickhouseAccessLogStore{}).InsertBatch(context.Background(), records); err != nil {
t.Fatalf("InsertBatch error = %v", err)
}
if len(got) != 2 {
t.Fatalf("hook logs = %d, want 2", len(got))
}
if got[0].Path != "/a" || got[1].Path != "/b" {
t.Fatalf("hook paths = %q/%q, want /a /b", got[0].Path, got[1].Path)
}
}
func TestInsertHooksNoopWhenUnset(t *testing.T) {
SetObservabilityInsertHooks(ObservabilityInsertHooks{})
SetAccessLogInsertHooks(AccessLogInsertHooks{})
if err := (clickhouseObservabilityStore{}).InsertMetricSnapshot(context.Background(), &model.OpenFlareMetricSnapshot{NodeID: "x"}); err != nil {
t.Fatalf("InsertMetricSnapshot with nil hook error = %v", err)
}
if err := (clickhouseAccessLogStore{}).InsertBatch(context.Background(), []*model.OpenFlareAccessLog{{NodeID: "x"}}); err != nil {
t.Fatalf("InsertBatch with nil hook error = %v", err)
}
}
+146 -39
View File
@@ -15,7 +15,10 @@ import (
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/Rain-kl/Wavelet/pkg/logger"
)
const (
@@ -25,6 +28,9 @@ const (
openFlareHealthSeverityWarning = "warning"
openFlareHealthSeverityCritical = "critical"
openFlareHealthEventMessageMaxLen = 4096
// logStoreNameClickHouse 与 logstore 内部 dbNameClickHouse 取值一致。
logStoreNameClickHouse = "clickhouse"
)
// OpenFlareHealthEventInput describes a desired active health event for reconciliation.
@@ -51,43 +57,72 @@ func isMissingTableError(err error) bool {
// InsertOpenFlareMetricSnapshot inserts a metric snapshot into ClickHouse.
func InsertOpenFlareMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
return currentObservabilityStore().InsertMetricSnapshot(ctx, record)
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertMetricSnapshot(ctx, record)
}
// InsertOpenFlareEdgeHealth inserts an L2 edge health snapshot into ClickHouse.
func InsertOpenFlareEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
return currentObservabilityStore().InsertEdgeHealth(ctx, record)
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertEdgeHealth(ctx, record)
}
// InsertOpenFlareNodeObservationFrps inserts an FRPS observation into ClickHouse.
func InsertOpenFlareNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
return currentObservabilityStore().InsertNodeObservationFrps(ctx, record)
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertNodeObservationFrps(ctx, record)
}
// InsertOpenFlareNodeObservationFrpc inserts an FRPC observation into ClickHouse.
func InsertOpenFlareNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
return currentObservabilityStore().InsertNodeObservationFrpc(ctx, record)
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertNodeObservationFrpc(ctx, record)
}
// ListOpenFlareMetricSnapshotsSince returns metric snapshots since the given time.
func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
return currentObservabilityStore().ListMetricSnapshots(ctx, nodeID, since, limit)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListMetricSnapshots(ctx, nodeID, since, limit)
}
// ListOpenFlareLatestMetricSnapshotsSince returns the latest metric snapshot per node.
// Prefer ClickHouse LIMIT 1 BY; on CH unavailability fall back to store list + reduce.
// The ClickHouse LIMIT 1 BY fast path is used only when ClickHouse is the ACTIVE log
// database; otherwise the request goes straight to the active log store (PG/SQLite),
// avoiding stale reads of the previous CH store after a migration.
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
rows, err := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
if err == nil {
return fromAnalyticsNodeMetricSnapshots(rows), nil
active, err := logstore.ActiveDatabase(ctx)
if err != nil {
logger.ErrorF(ctx, "failed to resolve active log database for latest metric snapshots: %v", err)
} else if active == logStoreNameClickHouse {
rows, chErr := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
if chErr == nil {
return fromAnalyticsNodeMetricSnapshots(rows), nil
}
logger.ErrorF(ctx, "clickhouse fast-path ListLatestNodeMetricSnapshots failed: %v", chErr)
return nil, chErr
}
// Fallback for unit tests (memory store) and environments without ClickHouse.
// Routes through the active log store (PG/SQLite active).
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
if listErr != nil {
return nil, err
return nil, listErr
}
return openFlareLatestMetricSnapshots(all), nil
}
@@ -110,13 +145,37 @@ func openFlareLatestMetricSnapshots(snapshots []*model.OpenFlareMetricSnapshot)
return result
}
// fromAnalyticsNodeMetricSnapshots converts analytics rows back to the business model.
func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot) []*model.OpenFlareMetricSnapshot {
result := make([]*model.OpenFlareMetricSnapshot, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareMetricSnapshot{
ID: uint(row.ID),
NodeID: row.NodeID,
CapturedAt: row.CapturedAt,
CPUUsagePercent: row.CPUUsagePercent,
MemoryUsedBytes: row.MemoryUsedBytes,
MemoryTotalBytes: row.MemoryTotalBytes,
StorageUsedBytes: row.StorageUsedBytes,
StorageTotalBytes: row.StorageTotalBytes,
DiskReadBytes: row.DiskReadBytes,
DiskWriteBytes: row.DiskWriteBytes,
NetworkRxBytes: row.NetworkRxBytes,
NetworkTxBytes: row.NetworkTxBytes,
CreatedAt: row.CreatedAt,
}
}
return result
}
// ListOpenFlareTrafficHourlySince returns hourly traffic rollup rows since the given time.
// Source: of_access_log_hourly (M5).
// CH 读 of_access_log_hourly rollup;PG/SQLite 经 logstore 从 of_node_access_logs 实时聚合。
func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareTrafficHourly, error) {
rows, err := analyticsrepo.ListNodeTrafficHourly(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.Observability.ListTrafficHourly(ctx, nodeID, since)
if err != nil {
return nil, err
}
@@ -133,12 +192,15 @@ func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since t
return result, nil
}
// ListOpenFlareAccessLogHourlySince returns of_access_log_hourly rows since the given time.
// ListOpenFlareAccessLogHourlySince returns hourly access-log rollups since the given time,
// read through logstore's active backend (ClickHouse of_access_log_hourly rollup;
// PostgreSQL/SQLite real-time aggregation from of_node_access_logs).
func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareAccessLogHourly, error) {
rows, err := analyticsrepo.ListAccessLogHourly(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.Observability.ListAccessLogHourly(ctx, nodeID, since)
if err != nil {
return nil, err
}
@@ -159,10 +221,11 @@ func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since
// ListOpenFlareMetricHourlySince returns hourly metric aggregates since the given time.
func ListOpenFlareMetricHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricHourly, error) {
rows, err := analyticsrepo.ListNodeMetricHourly(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.Observability.ListMetricHourly(ctx, nodeID, since)
if err != nil {
return nil, err
}
@@ -223,42 +286,74 @@ func ListOpenFlareHealthEvents(ctx context.Context, nodeID string, activeOnly bo
// DeleteOpenFlareMetricSnapshotsBefore deletes metric snapshots captured before cutoff.
func DeleteOpenFlareMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return currentObservabilityStore().DeleteMetricSnapshotsBefore(ctx, cutoff)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteMetricSnapshotsBefore(ctx, cutoff)
}
// DeleteAllOpenFlareMetricSnapshots deletes all metric snapshots.
func DeleteAllOpenFlareMetricSnapshots(ctx context.Context) (int64, error) {
return currentObservabilityStore().DeleteAllMetricSnapshots(ctx)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllMetricSnapshots(ctx)
}
// DeleteOpenFlareEdgeHealthBefore deletes edge health rows captured before cutoff.
func DeleteOpenFlareEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return currentObservabilityStore().DeleteEdgeHealthBefore(ctx, cutoff)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
}
// DeleteAllOpenFlareEdgeHealth deletes all edge health snapshots.
func DeleteAllOpenFlareEdgeHealth(ctx context.Context) (int64, error) {
return currentObservabilityStore().DeleteAllEdgeHealth(ctx)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllEdgeHealth(ctx)
}
// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
func DeleteOpenFlareNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return currentObservabilityStore().DeleteNodeObservationFrpsBefore(ctx, cutoff)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
}
// DeleteAllOpenFlareNodeObservationFrps deletes all FRPS observations.
func DeleteAllOpenFlareNodeObservationFrps(ctx context.Context) (int64, error) {
return currentObservabilityStore().DeleteAllNodeObservationFrps(ctx)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllNodeObservationFrps(ctx)
}
// DeleteOpenFlareNodeObservationFrpcBefore deletes FRPC observations captured before cutoff.
func DeleteOpenFlareNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return currentObservabilityStore().DeleteNodeObservationFrpcBefore(ctx, cutoff)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
}
// DeleteAllOpenFlareNodeObservationFrpc deletes all FRPC observations.
func DeleteAllOpenFlareNodeObservationFrpc(ctx context.Context) (int64, error) {
return currentObservabilityStore().DeleteAllNodeObservationFrpc(ctx)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllNodeObservationFrpc(ctx)
}
// DeleteOpenFlareHealthEventsByNodeID deletes all health events for a node.
@@ -523,15 +618,27 @@ func marshalOpenFlareHealthMetadata(value map[string]string) string {
// ListOpenFlareEdgeHealth returns L2 edge health snapshots.
func ListOpenFlareEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
return currentObservabilityStore().ListEdgeHealth(ctx, nodeID, since, limit)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListEdgeHealth(ctx, nodeID, since, limit)
}
// ListOpenFlareNodeObservationFrpc returns frpc observations.
func ListOpenFlareNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
return currentObservabilityStore().ListNodeObservationFrpc(ctx, nodeID, since, limit)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListNodeObservationFrpc(ctx, nodeID, since, limit)
}
// ListOpenFlareNodeObservationFrps returns frps observations.
func ListOpenFlareNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
return currentObservabilityStore().ListNodeObservationFrps(ctx, nodeID, since, limit)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListNodeObservationFrps(ctx, nodeID, since, limit)
}
@@ -1,355 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"math"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
)
// ObservabilityInsertHooks queues observability rows for async ClickHouse write.
// Wired from openflare/chwriter.Init so model never imports the apps layer.
type ObservabilityInsertHooks struct {
QueueMetricSnapshot func(analyticsmodel.NodeMetricSnapshot)
QueueEdgeHealth func(analyticsmodel.NodeEdgeHealth)
QueueFrpsObservation func(analyticsmodel.NodeObsFrps)
QueueFrpcObservation func(analyticsmodel.NodeObsFrpc)
}
var (
observabilityInsertHooksMu sync.RWMutex
observabilityInsertHooks ObservabilityInsertHooks
)
// SetObservabilityInsertHooks registers async queue callbacks for observability inserts.
func SetObservabilityInsertHooks(hooks ObservabilityInsertHooks) {
observabilityInsertHooksMu.Lock()
observabilityInsertHooks = hooks
observabilityInsertHooksMu.Unlock()
}
func currentObservabilityInsertHooks() ObservabilityInsertHooks {
observabilityInsertHooksMu.RLock()
defer observabilityInsertHooksMu.RUnlock()
return observabilityInsertHooks
}
type observabilityStore interface {
InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error
ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error)
DeleteAllMetricSnapshots(ctx context.Context) (int64, error)
DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error)
InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error
ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error)
DeleteAllEdgeHealth(ctx context.Context) (int64, error)
DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error)
InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error
ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error)
DeleteAllNodeObservationFrps(ctx context.Context) (int64, error)
DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error)
InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error
ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error)
DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error)
DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error)
}
var (
observabilityStoreMu sync.RWMutex
observabilityStoreHolder observabilityStore
)
func currentObservabilityStore() observabilityStore {
observabilityStoreMu.RLock()
defer observabilityStoreMu.RUnlock()
if observabilityStoreHolder != nil {
return observabilityStoreHolder
}
return clickhouseObservabilityStore{}
}
// SetObservabilityStoreForTest swaps the observability store implementation for unit tests.
func SetObservabilityStoreForTest(store observabilityStore) func() {
observabilityStoreMu.Lock()
previous := observabilityStoreHolder
observabilityStoreHolder = store
observabilityStoreMu.Unlock()
return func() {
observabilityStoreMu.Lock()
observabilityStoreHolder = previous
observabilityStoreMu.Unlock()
}
}
// NewMemoryObservabilityStore returns an in-memory observability store for unit tests.
func NewMemoryObservabilityStore() observabilityStore {
return &memoryObservabilityStore{}
}
type clickhouseObservabilityStore struct{}
func (clickhouseObservabilityStore) InsertMetricSnapshot(_ context.Context, record *model.OpenFlareMetricSnapshot) error {
if record == nil {
return nil
}
if hook := currentObservabilityInsertHooks().QueueMetricSnapshot; hook != nil {
hook(toAnalyticsNodeMetricSnapshot(record))
}
return nil
}
func (clickhouseObservabilityStore) ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
rows, err := analyticsrepo.ListNodeMetricSnapshots(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeMetricSnapshots(rows), nil
}
func (clickhouseObservabilityStore) DeleteAllMetricSnapshots(ctx context.Context) (int64, error) {
return analyticsrepo.DeleteAllNodeMetricSnapshots(ctx)
}
func (clickhouseObservabilityStore) DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
}
const edgeHealthStatusUnknown = "unknown"
func normalizeEdgeHealthStatus(status string) string {
status = strings.TrimSpace(status)
if status == "" {
return edgeHealthStatusUnknown
}
return status
}
func (clickhouseObservabilityStore) InsertEdgeHealth(_ context.Context, record *model.OpenFlareEdgeHealth) error {
if record == nil {
return nil
}
if hook := currentObservabilityInsertHooks().QueueEdgeHealth; hook != nil {
hook(toAnalyticsNodeEdgeHealth(record))
}
return nil
}
func (clickhouseObservabilityStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeEdgeHealth(rows), nil
}
func (clickhouseObservabilityStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
}
func (clickhouseObservabilityStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
}
func (clickhouseObservabilityStore) InsertNodeObservationFrps(_ context.Context, record *model.OpenFlareNodeObservationFrps) error {
if record == nil {
return nil
}
if hook := currentObservabilityInsertHooks().QueueFrpsObservation; hook != nil {
hook(toAnalyticsNodeObsFrps(record))
}
return nil
}
func (clickhouseObservabilityStore) ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
rows, err := analyticsrepo.ListNodeObsFrps(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeObsFrps(rows), nil
}
func (clickhouseObservabilityStore) DeleteAllNodeObservationFrps(ctx context.Context) (int64, error) {
return analyticsrepo.DeleteAllNodeObsFrps(ctx)
}
func (clickhouseObservabilityStore) DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return analyticsrepo.DeleteNodeObsFrpsBefore(ctx, cutoff)
}
func (clickhouseObservabilityStore) InsertNodeObservationFrpc(_ context.Context, record *model.OpenFlareNodeObservationFrpc) error {
if record == nil {
return nil
}
if hook := currentObservabilityInsertHooks().QueueFrpcObservation; hook != nil {
hook(toAnalyticsNodeObsFrpc(record))
}
return nil
}
func (clickhouseObservabilityStore) ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
rows, err := analyticsrepo.ListNodeObsFrpc(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeObsFrpc(rows), nil
}
func (clickhouseObservabilityStore) DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error) {
return analyticsrepo.DeleteAllNodeObsFrpc(ctx)
}
func (clickhouseObservabilityStore) DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
return analyticsrepo.DeleteNodeObsFrpcBefore(ctx, cutoff)
}
func toNodeObservabilityFilter(nodeID string, since time.Time, limit int) analyticsrepo.NodeObservabilityFilter {
return analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
Limit: limit,
}
}
func toAnalyticsNodeMetricSnapshot(record *model.OpenFlareMetricSnapshot) analyticsmodel.NodeMetricSnapshot {
return analyticsmodel.NodeMetricSnapshot{
ID: uint64(record.ID),
NodeID: record.NodeID,
CapturedAt: record.CapturedAt,
CPUUsagePercent: record.CPUUsagePercent,
MemoryUsedBytes: record.MemoryUsedBytes,
MemoryTotalBytes: record.MemoryTotalBytes,
StorageUsedBytes: record.StorageUsedBytes,
StorageTotalBytes: record.StorageTotalBytes,
DiskReadBytes: record.DiskReadBytes,
DiskWriteBytes: record.DiskWriteBytes,
NetworkRxBytes: record.NetworkRxBytes,
NetworkTxBytes: record.NetworkTxBytes,
CreatedAt: record.CreatedAt,
}
}
func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot) []*model.OpenFlareMetricSnapshot {
result := make([]*model.OpenFlareMetricSnapshot, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareMetricSnapshot{
ID: uint(row.ID),
NodeID: row.NodeID,
CapturedAt: row.CapturedAt,
CPUUsagePercent: row.CPUUsagePercent,
MemoryUsedBytes: row.MemoryUsedBytes,
MemoryTotalBytes: row.MemoryTotalBytes,
StorageUsedBytes: row.StorageUsedBytes,
StorageTotalBytes: row.StorageTotalBytes,
DiskReadBytes: row.DiskReadBytes,
DiskWriteBytes: row.DiskWriteBytes,
NetworkRxBytes: row.NetworkRxBytes,
NetworkTxBytes: row.NetworkTxBytes,
CreatedAt: row.CreatedAt,
}
}
return result
}
func toAnalyticsNodeEdgeHealth(record *model.OpenFlareEdgeHealth) analyticsmodel.NodeEdgeHealth {
return analyticsmodel.NodeEdgeHealth{
ID: uint64(record.ID),
NodeID: record.NodeID,
CapturedAt: record.CapturedAt,
Status: normalizeEdgeHealthStatus(record.Status),
Connections: record.Connections,
CreatedAt: record.CreatedAt,
}
}
func fromAnalyticsNodeEdgeHealth(rows []analyticsmodel.NodeEdgeHealth) []*model.OpenFlareEdgeHealth {
result := make([]*model.OpenFlareEdgeHealth, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareEdgeHealth{
ID: uint(row.ID),
NodeID: row.NodeID,
CapturedAt: row.CapturedAt,
Status: normalizeEdgeHealthStatus(row.Status),
Connections: row.Connections,
CreatedAt: row.CreatedAt,
}
}
return result
}
func toAnalyticsNodeObsFrps(record *model.OpenFlareNodeObservationFrps) analyticsmodel.NodeObsFrps {
return analyticsmodel.NodeObsFrps{
ID: uint64(record.ID),
NodeID: record.NodeID,
CapturedAt: record.CapturedAt,
FrpsConnections: openFlareObservabilityIntToInt32(record.FrpsConnections),
FrpsProxyCount: openFlareObservabilityIntToInt32(record.FrpsProxyCount),
FrpsClientCount: openFlareObservabilityIntToInt32(record.FrpsClientCount),
FrpsProxies: record.FrpsProxies,
CreatedAt: record.CreatedAt,
}
}
func fromAnalyticsNodeObsFrps(rows []analyticsmodel.NodeObsFrps) []*model.OpenFlareNodeObservationFrps {
result := make([]*model.OpenFlareNodeObservationFrps, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareNodeObservationFrps{
ID: uint(row.ID),
NodeID: row.NodeID,
CapturedAt: row.CapturedAt,
FrpsConnections: int(row.FrpsConnections),
FrpsProxyCount: int(row.FrpsProxyCount),
FrpsClientCount: int(row.FrpsClientCount),
FrpsProxies: row.FrpsProxies,
CreatedAt: row.CreatedAt,
}
}
return result
}
func toAnalyticsNodeObsFrpc(record *model.OpenFlareNodeObservationFrpc) analyticsmodel.NodeObsFrpc {
return analyticsmodel.NodeObsFrpc{
ID: uint64(record.ID),
NodeID: record.NodeID,
CapturedAt: record.CapturedAt,
TunnelStatus: record.TunnelStatus,
ConnectedRelaysCount: openFlareObservabilityIntToInt32(record.ConnectedRelaysCount),
CreatedAt: record.CreatedAt,
}
}
func openFlareObservabilityIntToInt32(value int) int32 {
switch {
case value > math.MaxInt32:
return math.MaxInt32
case value < math.MinInt32:
return math.MinInt32
default:
return int32(value)
}
}
func fromAnalyticsNodeObsFrpc(rows []analyticsmodel.NodeObsFrpc) []*model.OpenFlareNodeObservationFrpc {
result := make([]*model.OpenFlareNodeObservationFrpc, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareNodeObservationFrpc{
ID: uint(row.ID),
NodeID: row.NodeID,
CapturedAt: row.CapturedAt,
TunnelStatus: row.TunnelStatus,
ConnectedRelaysCount: int(row.ConnectedRelaysCount),
CreatedAt: row.CreatedAt,
}
}
return result
}
@@ -1,409 +0,0 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"sort"
"strings"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
)
type memoryObservabilityStore struct {
mu sync.RWMutex
metricSnapshots []*model.OpenFlareMetricSnapshot
edgeHealth []*model.OpenFlareEdgeHealth
frpsObs []*model.OpenFlareNodeObservationFrps
frpcObs []*model.OpenFlareNodeObservationFrpc
}
func (s *memoryObservabilityStore) InsertMetricSnapshot(_ context.Context, record *model.OpenFlareMetricSnapshot) error {
if record == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
copyRecord := cloneOpenFlareMetricSnapshot(record)
if memoryMetricSnapshotExists(s.metricSnapshots, copyRecord.NodeID, copyRecord.CapturedAt) {
return nil
}
s.metricSnapshots = append(s.metricSnapshots, copyRecord)
return nil
}
func (s *memoryObservabilityStore) ListMetricSnapshots(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := memoryFilterMetricSnapshots(s.metricSnapshots, nodeID, since)
sortOpenFlareMetricSnapshots(rows)
return memoryLimitObservabilityRows(rows, limit), nil
}
func (s *memoryObservabilityStore) DeleteAllMetricSnapshots(_ context.Context) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
count := int64(len(s.metricSnapshots))
s.metricSnapshots = nil
return count, nil
}
func (s *memoryObservabilityStore) DeleteMetricSnapshotsBefore(_ context.Context, cutoff time.Time) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
cutoff = cutoff.UTC()
remaining := make([]*model.OpenFlareMetricSnapshot, 0, len(s.metricSnapshots))
var deleted int64
for _, row := range s.metricSnapshots {
if row.CapturedAt.Before(cutoff) {
deleted++
continue
}
remaining = append(remaining, row)
}
s.metricSnapshots = remaining
return deleted, nil
}
func (s *memoryObservabilityStore) InsertEdgeHealth(_ context.Context, record *model.OpenFlareEdgeHealth) error {
if record == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.edgeHealth = append(s.edgeHealth, cloneOpenFlareEdgeHealth(record))
return nil
}
func (s *memoryObservabilityStore) ListEdgeHealth(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := memoryFilterEdgeHealth(s.edgeHealth, nodeID, since)
sortOpenFlareEdgeHealth(rows)
return memoryLimitObservabilityRows(rows, limit), nil
}
func (s *memoryObservabilityStore) DeleteAllEdgeHealth(_ context.Context) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
count := int64(len(s.edgeHealth))
s.edgeHealth = nil
return count, nil
}
func (s *memoryObservabilityStore) DeleteEdgeHealthBefore(_ context.Context, cutoff time.Time) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
cutoff = cutoff.UTC()
remaining := make([]*model.OpenFlareEdgeHealth, 0, len(s.edgeHealth))
var deleted int64
for _, row := range s.edgeHealth {
if row.CapturedAt.Before(cutoff) {
deleted++
continue
}
remaining = append(remaining, row)
}
s.edgeHealth = remaining
return deleted, nil
}
func (s *memoryObservabilityStore) InsertNodeObservationFrps(_ context.Context, record *model.OpenFlareNodeObservationFrps) error {
if record == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.frpsObs = append(s.frpsObs, cloneOpenFlareNodeObservationFrps(record))
return nil
}
func (s *memoryObservabilityStore) ListNodeObservationFrps(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := memoryFilterFrpsObservations(s.frpsObs, nodeID, since)
sortOpenFlareNodeObservationFrps(rows)
return memoryLimitObservabilityRows(rows, limit), nil
}
func (s *memoryObservabilityStore) DeleteAllNodeObservationFrps(_ context.Context) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
count := int64(len(s.frpsObs))
s.frpsObs = nil
return count, nil
}
func (s *memoryObservabilityStore) DeleteNodeObservationFrpsBefore(_ context.Context, cutoff time.Time) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
cutoff = cutoff.UTC()
remaining := make([]*model.OpenFlareNodeObservationFrps, 0, len(s.frpsObs))
var deleted int64
for _, row := range s.frpsObs {
if row.CapturedAt.Before(cutoff) {
deleted++
continue
}
remaining = append(remaining, row)
}
s.frpsObs = remaining
return deleted, nil
}
func (s *memoryObservabilityStore) InsertNodeObservationFrpc(_ context.Context, record *model.OpenFlareNodeObservationFrpc) error {
if record == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
s.frpcObs = append(s.frpcObs, cloneOpenFlareNodeObservationFrpc(record))
return nil
}
func (s *memoryObservabilityStore) ListNodeObservationFrpc(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := memoryFilterFrpcObservations(s.frpcObs, nodeID, since)
sortOpenFlareNodeObservationFrpc(rows)
return memoryLimitObservabilityRows(rows, limit), nil
}
func (s *memoryObservabilityStore) DeleteAllNodeObservationFrpc(_ context.Context) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
count := int64(len(s.frpcObs))
s.frpcObs = nil
return count, nil
}
func (s *memoryObservabilityStore) DeleteNodeObservationFrpcBefore(_ context.Context, cutoff time.Time) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
cutoff = cutoff.UTC()
remaining := make([]*model.OpenFlareNodeObservationFrpc, 0, len(s.frpcObs))
var deleted int64
for _, row := range s.frpcObs {
if row.CapturedAt.Before(cutoff) {
deleted++
continue
}
remaining = append(remaining, row)
}
s.frpcObs = remaining
return deleted, nil
}
func memoryFilterMetricSnapshots(rows []*model.OpenFlareMetricSnapshot, nodeID string, since time.Time) []*model.OpenFlareMetricSnapshot {
result := make([]*model.OpenFlareMetricSnapshot, 0, len(rows))
for _, row := range rows {
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
continue
}
if !since.IsZero() && row.CapturedAt.Before(since) {
continue
}
result = append(result, row)
}
return result
}
func memoryFilterEdgeHealth(rows []*model.OpenFlareEdgeHealth, nodeID string, since time.Time) []*model.OpenFlareEdgeHealth {
result := make([]*model.OpenFlareEdgeHealth, 0, len(rows))
for _, row := range rows {
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
continue
}
if !since.IsZero() && row.CapturedAt.Before(since) {
continue
}
result = append(result, row)
}
return result
}
func memoryFilterFrpsObservations(rows []*model.OpenFlareNodeObservationFrps, nodeID string, since time.Time) []*model.OpenFlareNodeObservationFrps {
result := make([]*model.OpenFlareNodeObservationFrps, 0, len(rows))
for _, row := range rows {
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
continue
}
if !since.IsZero() && row.CapturedAt.Before(since) {
continue
}
result = append(result, row)
}
return result
}
func memoryFilterFrpcObservations(rows []*model.OpenFlareNodeObservationFrpc, nodeID string, since time.Time) []*model.OpenFlareNodeObservationFrpc {
result := make([]*model.OpenFlareNodeObservationFrpc, 0, len(rows))
for _, row := range rows {
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
continue
}
if !since.IsZero() && row.CapturedAt.Before(since) {
continue
}
result = append(result, row)
}
return result
}
func memoryObservabilityMatchesNodeID(rowNodeID string, nodeID string) bool {
trimmed := strings.TrimSpace(nodeID)
if trimmed == "" {
return true
}
return rowNodeID == trimmed
}
func memoryMetricSnapshotExists(rows []*model.OpenFlareMetricSnapshot, nodeID string, capturedAt time.Time) bool {
capturedAt = capturedAt.UTC()
for _, row := range rows {
if row.NodeID == nodeID && row.CapturedAt.UTC().Equal(capturedAt) {
return true
}
}
return false
}
func sortOpenFlareMetricSnapshots(items []*model.OpenFlareMetricSnapshot) {
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
return compare > 0
}
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
})
}
func sortOpenFlareEdgeHealth(items []*model.OpenFlareEdgeHealth) {
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
return compare > 0
}
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
})
}
func sortOpenFlareNodeObservationFrps(items []*model.OpenFlareNodeObservationFrps) {
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
return compare > 0
}
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
})
}
func sortOpenFlareNodeObservationFrpc(items []*model.OpenFlareNodeObservationFrpc) {
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
return compare > 0
}
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
})
}
func memoryLimitObservabilityRows[T any](rows []T, limit int) []T {
if limit <= 0 || len(rows) <= limit {
result := make([]T, len(rows))
copy(result, rows)
return result
}
result := make([]T, limit)
copy(result, rows[:limit])
return result
}
func cloneOpenFlareMetricSnapshot(record *model.OpenFlareMetricSnapshot) *model.OpenFlareMetricSnapshot {
copyRecord := *record
if copyRecord.ID == 0 {
copyRecord.ID = uint(idgen.NextUint64ID())
}
now := time.Now().UTC()
if copyRecord.CreatedAt.IsZero() {
copyRecord.CreatedAt = now
}
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
return &copyRecord
}
func cloneOpenFlareEdgeHealth(record *model.OpenFlareEdgeHealth) *model.OpenFlareEdgeHealth {
copyRecord := *record
if copyRecord.ID == 0 {
copyRecord.ID = uint(idgen.NextUint64ID())
}
now := time.Now().UTC()
if copyRecord.CreatedAt.IsZero() {
copyRecord.CreatedAt = now
}
if copyRecord.CapturedAt.IsZero() {
copyRecord.CapturedAt = now
}
if strings.TrimSpace(copyRecord.Status) == "" {
copyRecord.Status = edgeHealthStatusUnknown
}
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
return &copyRecord
}
func cloneOpenFlareNodeObservationFrps(record *model.OpenFlareNodeObservationFrps) *model.OpenFlareNodeObservationFrps {
copyRecord := *record
if copyRecord.ID == 0 {
copyRecord.ID = uint(idgen.NextUint64ID())
}
now := time.Now().UTC()
if copyRecord.CreatedAt.IsZero() {
copyRecord.CreatedAt = now
}
if copyRecord.CapturedAt.IsZero() {
copyRecord.CapturedAt = now
}
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
return &copyRecord
}
func cloneOpenFlareNodeObservationFrpc(record *model.OpenFlareNodeObservationFrpc) *model.OpenFlareNodeObservationFrpc {
copyRecord := *record
if copyRecord.ID == 0 {
copyRecord.ID = uint(idgen.NextUint64ID())
}
now := time.Now().UTC()
if copyRecord.CreatedAt.IsZero() {
copyRecord.CreatedAt = now
}
if copyRecord.CapturedAt.IsZero() {
copyRecord.CapturedAt = now
}
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
return &copyRecord
}
+1 -1
View File
@@ -51,7 +51,7 @@ func RegisterAdminRoutes(apiV1Router *gin.RouterGroup) {
func registerAdminDiagnosticRoutes(adminRouter *gin.RouterGroup) {
// System status
adminRouter.GET("/status", admin_status.GetSystemStatus)
adminRouter.GET("/status/clickhouse", admin_status.GetClickHouseStatus)
adminRouter.GET("/status/log-database", admin_status.GetLogDatabaseStatus)
// Database basic info & backup export
adminRouter.GET("/db-info", admin_status.GetDatabaseInfo)
@@ -19,7 +19,6 @@ func registerOptionRoutes(apiGroup *gin.RouterGroup) {
optionRoute.POST("/update", option.UpdateOptionHandler)
optionRoute.POST("/update-batch", option.UpdateOptionsBatchHandler)
optionRoute.POST("/geoip/lookup", option.LookupGeoIPHandler)
optionRoute.POST("/database/cleanup", option.CleanupDatabaseHandler)
}
uptimeKumaRoute := apiGroup.Group("/uptimekuma")
+84
View File
@@ -11,11 +11,14 @@ import (
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
"github.com/alicebob/miniredis/v2"
"github.com/glebarez/sqlite"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
@@ -307,6 +310,24 @@ func getSeedConfigsPart2() []model.SystemConfig {
Type: configTypeBusiness,
Description: "Pages 每个项目最大历史部署保留数(0 表示不限制)",
},
{
Key: model.ConfigKeyLogRetentionDaysPostgres,
Value: "90",
Type: configTypeBusiness,
Description: "PostgreSQL 日志保留天数(访问日志与可观测统一)",
},
{
Key: model.ConfigKeyLogRetentionDaysSQLite,
Value: "90",
Type: configTypeBusiness,
Description: "SQLite 日志保留天数",
},
{
Key: model.ConfigKeyLogRetentionDaysClickHouse,
Value: "90",
Type: configTypeBusiness,
Description: "ClickHouse 日志保留天数",
},
}
}
@@ -352,3 +373,66 @@ func seedDefaultConfigs(t *testing.T, tx *gorm.DB) {
_ = db.HSetJSON(context.Background(), repository.SystemConfigRedisHashKey, config.Key, &config)
}
}
// SetupLogStoresForTest 将 logstore 指向测试已通过 db.SetDB 注入的 sqlite 库,
// 并注册立即 flush 的 hooks,使 repository 层日志写入对后续读取立即可见。
// 调用方必须先 db.SetDB(sqliteDB)(并迁移业务表),本函数负责迁移日志分析表。
func SetupLogStoresForTest(t *testing.T) {
t.Helper()
gdb := db.DB(context.Background())
require.NoError(t, gdb.AutoMigrate(
&analyticsmodel.NodeAccessLog{},
&analyticsmodel.UserAccessLog{},
&analyticsmodel.NodeMetricSnapshot{},
&analyticsmodel.NodeEdgeHealth{},
&analyticsmodel.NodeObsFrps{},
&analyticsmodel.NodeObsFrpc{},
))
logstore.ResetForTest()
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == model.ConfigKeyLogDatabase {
return "sqlite", nil
}
return "", nil
})
store, err := logstore.Active(context.Background())
require.NoError(t, err)
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
if err := store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs); err != nil {
t.Errorf("batch insert node access logs failed in test hook: %v", err)
}
},
})
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{
QueueMetricSnapshot: func(record analyticsmodel.NodeMetricSnapshot) {
if err := store.Observability.BatchInsertNodeMetricSnapshots(context.Background(), []analyticsmodel.NodeMetricSnapshot{record}); err != nil {
t.Errorf("batch insert node metric snapshots failed in test hook: %v", err)
}
},
QueueEdgeHealth: func(record analyticsmodel.NodeEdgeHealth) {
if err := store.Observability.BatchInsertNodeEdgeHealth(context.Background(), []analyticsmodel.NodeEdgeHealth{record}); err != nil {
t.Errorf("batch insert node edge health failed in test hook: %v", err)
}
},
QueueNodeObsFrps: func(record analyticsmodel.NodeObsFrps) {
if err := store.Observability.BatchInsertNodeObsFrps(context.Background(), []analyticsmodel.NodeObsFrps{record}); err != nil {
t.Errorf("batch insert node obs frps failed in test hook: %v", err)
}
},
QueueNodeObsFrpc: func(record analyticsmodel.NodeObsFrpc) {
if err := store.Observability.BatchInsertNodeObsFrpc(context.Background(), []analyticsmodel.NodeObsFrpc{record}); err != nil {
t.Errorf("batch insert node obs frpc failed in test hook: %v", err)
}
},
})
t.Cleanup(func() {
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{})
logstore.ResetForTest()
})
}