tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-03 23:06:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(log): decouple log storage from ClickHouse with switchable logstore

Browse Source 
- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/swagger.yaml
+65 -111
View File
@@ -169,26 +169,20 @@ definitions:
$ref: '#/definitions/github_com_Rain-kl_Wavelet_pkg_protocol.WAFIPGroup'
type: array
type: object
analytics.ClickHouseOperationalStats:
analytics.BatchWriterStats:
properties:
active_parts:
cap:
type: integer
async_insert_bytes:
depth:
type: integer
async_insert_queue:
drops:
type: integer
batch_writers:
description: BatchWriters reports in-process queue depth/drops/flush errors
for CH writers.
items:
$ref: '#/definitions/batchwriter.Stats'
type: array
database:
flush_errors:
type: integer
name:
type: string
pending_mutations:
type: integer
total_rows:
type: integer
running:
type: boolean
type: object
apply_log.CleanupInput:
properties:
@@ -247,21 +241,6 @@ definitions:
is_active:
type: boolean
type: object
batchwriter.Stats:
properties:
cap:
type: integer
depth:
type: integer
drops:
type: integer
flush_errors:
type: integer
name:
type: string
running:
type: boolean
type: object
cache.updateCacheConfigRequest:
properties:
lru_enabled:
@@ -301,6 +280,8 @@ definitions:
type: string
id:
type: integer
zone_domain:
type: string
zone_id:
type: integer
type: object
@@ -774,6 +755,27 @@ definitions:
token:
type: string
type: object
github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats:
properties:
active_parts:
type: integer
async_insert_bytes:
type: integer
async_insert_queue:
type: integer
batch_writers:
description: BatchWriters reports in-process queue depth/drops/flush errors
for CH writers.
items:
$ref: '#/definitions/analytics.BatchWriterStats'
type: array
database:
type: string
pending_mutations:
type: integer
total_rows:
type: integer
type: object
github_com_Rain-kl_Wavelet_pkg_protocol.ActiveConfigMeta:
properties:
checksum:
@@ -2533,32 +2535,6 @@ definitions:
window_started_at:
type: string
type: object
option.databaseCleanupInput:
properties:
retention_days:
type: integer
target:
type: string
type: object
option.databaseCleanupResult:
properties:
cleanup_mode:
type: string
delete_all:
type: boolean
deleted_count:
type: integer
eligible_count:
type: integer
retention_days:
type: integer
table_ttl_days:
type: integer
target:
type: string
target_label:
type: string
type: object
option.geoIPLookupRequest:
properties:
ip:
@@ -3442,6 +3418,24 @@ definitions:
version:
type: string
type: object
status.LogDatabaseStatus:
properties:
active_database:
type: string
available_targets:
items:
type: string
type: array
clickhouse:
$ref: '#/definitions/github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats'
migration:
description: idle | migrating
type: string
retention_days:
additionalProperties:
type: integer
type: object
type: object
status.SystemStatusResponse:
properties:
alloc:
@@ -4935,7 +4929,7 @@ paths:
- admin
/api/v1/admin/logs/access:
get:
description: 分页并按照用户、接口路径、时间范围等维度检索 ClickHouse 用户访问日志列表（需要管理员权限，ClickHouse 未启用时报错）
description: 分页并按照用户、接口路径、时间范围等维度检索用户访问日志列表（需要管理员权限，日志存储未启用时报错）
parameters:
- default: 1
description: 页码
@@ -4994,7 +4988,7 @@ paths:
- admin
/api/v1/admin/logs/analytics:
get:
description: 聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行（需要管理员权限，ClickHouse 未启用时报错）
description: 聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行（需要管理员权限，日志存储未启用时报错）
produces:
- application/json
responses:
@@ -5008,7 +5002,7 @@ paths:
$ref: '#/definitions/logs.logsAnalyticsResponse'
type: object
"400":
description: ClickHouse 未启用
description: 日志存储未启用
schema:
$ref: '#/definitions/response.Any'
"401":
@@ -5434,9 +5428,9 @@ paths:
summary: 获取系统状态信息
tags:
- admin
/api/v1/admin/status/clickhouse:
/api/v1/admin/status/log-database:
get:
description: 返回 ClickHouse parts、mutation、async_insert 队列及进程内 batch writer 指标，需要管理员权限
description: 返回当前日志主库、迁移状态、各库保留天数与合法迁移目标，需要管理员权限
produces:
- application/json
responses:
@@ -5447,12 +5441,8 @@ paths:
- $ref: '#/definitions/response.Any'
- properties:
data:
$ref: '#/definitions/analytics.ClickHouseOperationalStats'
$ref: '#/definitions/status.LogDatabaseStatus'
type: object
"400":
description: ClickHouse 未启用
schema:
$ref: '#/definitions/response.Any'
"401":
description: 未登录
schema:
@@ -5467,7 +5457,7 @@ paths:
$ref: '#/definitions/response.Any'
security:
- SessionCookie: []
summary: 获取 ClickHouse 运行指标
summary: 获取日志数据库状态
tags:
- admin
/api/v1/admin/system-configs:
@@ -5738,6 +5728,14 @@ paths:
in: query
name: task_type
type: string
- description: 任务类型前缀筛选（与 task_type / task_types 互斥，精确类型优先）
in: query
name: task_type_prefix
type: string
- description: 逗号分隔的精确任务类型列表（IN 筛选，优先于前缀）
in: query
name: task_types
type: string
- default: 1
description: 页码
in: query
@@ -9285,50 +9283,6 @@ paths:
summary: 列出 OpenFlare 配置项
tags:
- openflare-option
/api/v1/d/option/database/cleanup:
post:
consumes:
- application/json
description: 按目标与保留天数清理可观测性相关数据表，需要管理员权限
parameters:
- description: 清理参数
in: body
name: request
schema:
$ref: '#/definitions/option.databaseCleanupInput'
produces:
- application/json
responses:
"200":
description: 清理结果
schema:
allOf:
- $ref: '#/definitions/response.Any'
- properties:
data:
$ref: '#/definitions/option.databaseCleanupResult'
type: object
"400":
description: 参数错误
schema:
$ref: '#/definitions/response.Any'
"401":
description: 未登录
schema:
$ref: '#/definitions/response.Any'
"404":
description: 无权限或不存在
schema:
$ref: '#/definitions/response.Any'
"500":
description: 内部错误
schema:
$ref: '#/definitions/response.Any'
security:
- SessionCookie: []
summary: 清理可观测性数据库
tags:
- openflare-option
/api/v1/d/option/geoip/lookup:
post:
consumes: