tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-09-28 05:46:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(log): decouple log storage from ClickHouse with switchable logstore

Browse Source 
- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/docs.go
+88 -166
View File
@@ -1024,7 +1024,7 @@ const docTemplate = `{
"SessionCookie": []
}
],
"description": "分页并按照用户、接口路径、时间范围等维度检索 ClickHouse 用户访问日志列表（需要管理员权限，ClickHouse 未启用时报错）",
"description": "分页并按照用户、接口路径、时间范围等维度检索用户访问日志列表（需要管理员权限，日志存储未启用时报错）",
"produces": [
"application/json"
],
@@ -1119,7 +1119,7 @@ const docTemplate = `{
"SessionCookie": []
}
],
"description": "聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行（需要管理员权限，ClickHouse 未启用时报错）",
"description": "聚合统计最近 7 天的每日访问趋势、浏览器分布以及前 10 名最活跃用户排行（需要管理员权限，日志存储未启用时报错）",
"produces": [
"application/json"
],
@@ -1147,7 +1147,7 @@ const docTemplate = `{
}
},
"400": {
"description": "ClickHouse 未启用",
"description": "日志存储未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
@@ -1877,21 +1877,21 @@ const docTemplate = `{
}
}
},
"/api/v1/admin/status/clickhouse": {
"/api/v1/admin/status/log-database": {
"get": {
"security": [
{
"SessionCookie": []
}
],
"description": "返回 ClickHouse parts、mutation、async_insert 队列及进程内 batch writer 指标，需要管理员权限",
"description": "返回当前日志主库、迁移状态、各库保留天数与合法迁移目标，需要管理员权限",
"produces": [
"application/json"
],
"tags": [
"admin"
],
"summary": "获取 ClickHouse 运行指标",
"summary": "获取日志数据库状态",
"responses": {
"200": {
"description": "获取成功",
@@ -1904,19 +1904,13 @@ const docTemplate = `{
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/analytics.ClickHouseOperationalStats"
"$ref": "#/definitions/status.LogDatabaseStatus"
}
}
}
]
}
},
"400": {
"description": "ClickHouse 未启用",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
@@ -2393,6 +2387,18 @@ const docTemplate = `{
"name": "task_type",
"in": "query"
},
{
"type": "string",
"description": "任务类型前缀筛选（与 task_type / task_types 互斥，精确类型优先）",
"name": "task_type_prefix",
"in": "query"
},
{
"type": "string",
"description": "逗号分隔的精确任务类型列表（IN 筛选，优先于前缀）",
"name": "task_types",
"in": "query"
},
{
"type": "integer",
"default": 1,
@@ -8305,80 +8311,6 @@ const docTemplate = `{
}
}
},
"/api/v1/d/option/database/cleanup": {
"post": {
"security": [
{
"SessionCookie": []
}
],
"description": "按目标与保留天数清理可观测性相关数据表，需要管理员权限",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"openflare-option"
],
"summary": "清理可观测性数据库",
"parameters": [
{
"description": "清理参数",
"name": "request",
"in": "body",
"schema": {
"$ref": "#/definitions/option.databaseCleanupInput"
}
}
],
"responses": {
"200": {
"description": "清理结果",
"schema": {
"allOf": [
{
"$ref": "#/definitions/response.Any"
},
{
"type": "object",
"properties": {
"data": {
"$ref": "#/definitions/option.databaseCleanupResult"
}
}
}
]
}
},
"400": {
"description": "参数错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"401": {
"description": "未登录",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"404": {
"description": "无权限或不存在",
"schema": {
"$ref": "#/definitions/response.Any"
}
},
"500": {
"description": "内部错误",
"schema": {
"$ref": "#/definitions/response.Any"
}
}
}
}
},
"/api/v1/d/option/geoip/lookup": {
"post": {
"security": [
@@ -14906,33 +14838,26 @@ const docTemplate = `{
}
}
},
"analytics.ClickHouseOperationalStats": {
"analytics.BatchWriterStats": {
"type": "object",
"properties": {
"active_parts": {
"cap": {
"type": "integer"
},
"async_insert_bytes": {
"depth": {
"type": "integer"
},
"async_insert_queue": {
"drops": {
"type": "integer"
},
"batch_writers": {
"description": "BatchWriters reports in-process queue depth/drops/flush errors for CH writers.",
"type": "array",
"items": {
"$ref": "#/definitions/batchwriter.Stats"
}
"flush_errors": {
"type": "integer"
},
"database": {
"name": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
"running": {
"type": "boolean"
}
}
},
@@ -15024,29 +14949,6 @@ const docTemplate = `{
}
}
},
"batchwriter.Stats": {
"type": "object",
"properties": {
"cap": {
"type": "integer"
},
"depth": {
"type": "integer"
},
"drops": {
"type": "integer"
},
"flush_errors": {
"type": "integer"
},
"name": {
"type": "string"
},
"running": {
"type": "boolean"
}
}
},
"cache.updateCacheConfigRequest": {
"type": "object",
"required": [
@@ -15105,6 +15007,9 @@ const docTemplate = `{
"id": {
"type": "integer"
},
"zone_domain": {
"type": "string"
},
"zone_id": {
"type": "integer"
}
@@ -15826,6 +15731,36 @@ const docTemplate = `{
}
}
},
"github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats": {
"type": "object",
"properties": {
"active_parts": {
"type": "integer"
},
"async_insert_bytes": {
"type": "integer"
},
"async_insert_queue": {
"type": "integer"
},
"batch_writers": {
"description": "BatchWriters reports in-process queue depth/drops/flush errors for CH writers.",
"type": "array",
"items": {
"$ref": "#/definitions/analytics.BatchWriterStats"
}
},
"database": {
"type": "string"
},
"pending_mutations": {
"type": "integer"
},
"total_rows": {
"type": "integer"
}
}
},
"github_com_Rain-kl_Wavelet_pkg_protocol.ActiveConfigMeta": {
"type": "object",
"properties": {
@@ -18481,46 +18416,6 @@ const docTemplate = `{
}
}
},
"option.databaseCleanupInput": {
"type": "object",
"properties": {
"retention_days": {
"type": "integer"
},
"target": {
"type": "string"
}
}
},
"option.databaseCleanupResult": {
"type": "object",
"properties": {
"cleanup_mode": {
"type": "string"
},
"delete_all": {
"type": "boolean"
},
"deleted_count": {
"type": "integer"
},
"eligible_count": {
"type": "integer"
},
"retention_days": {
"type": "integer"
},
"table_ttl_days": {
"type": "integer"
},
"target": {
"type": "string"
},
"target_label": {
"type": "string"
}
}
},
"option.geoIPLookupRequest": {
"type": "object",
"properties": {
@@ -19859,6 +19754,33 @@ const docTemplate = `{
}
}
},
"status.LogDatabaseStatus": {
"type": "object",
"properties": {
"active_database": {
"type": "string"
},
"available_targets": {
"type": "array",
"items": {
"type": "string"
}
},
"clickhouse": {
"$ref": "#/definitions/github_com_Rain-kl_Wavelet_internal_model_analytics.ClickHouseOperationalStats"
},
"migration": {
"description": "idle | migrating",
"type": "string"
},
"retention_days": {
"type": "object",
"additionalProperties": {
"type": "integer"
}
}
}
},
"status.SystemStatusResponse": {
"type": "object",
"properties": {