mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-06 23:56:37 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -41,14 +41,12 @@ const (
|
||||
ConfigKeyRelayFRPSWebUIPort = "relay_frps_web_ui_port" // FRPS 内置 Web 界面端口
|
||||
|
||||
// OpenFlare 业务配置(从 of_options 迁移)
|
||||
ConfigKeyAgentDiscoveryToken = "agent_discovery_token" //nolint:gosec // false positive: config key name. Agent 发现令牌
|
||||
ConfigKeyAgentHeartbeatInterval = "agent_heartbeat_interval" // Agent 心跳间隔(毫秒)
|
||||
ConfigKeyAgentWebsocketUpgradeEnabled = "agent_websocket_upgrade_enabled" // Agent WebSocket 升级开关
|
||||
ConfigKeyNodeOfflineThreshold = "node_offline_threshold" // 节点离线阈值(毫秒)
|
||||
ConfigKeyAgentUpdateRepo = "agent_update_repo" // Agent 更新仓库
|
||||
ConfigKeyGeoIPProvider = "geoip_provider" // GeoIP 服务商
|
||||
ConfigKeyDatabaseAutoCleanupEnabled = "database_auto_cleanup_enabled" // 数据库自动清理开关
|
||||
ConfigKeyDatabaseAutoCleanupRetentionDays = "database_auto_cleanup_retention_days" // 数据库保留天数
|
||||
ConfigKeyAgentDiscoveryToken = "agent_discovery_token" //nolint:gosec // false positive: config key name. Agent 发现令牌
|
||||
ConfigKeyAgentHeartbeatInterval = "agent_heartbeat_interval" // Agent 心跳间隔(毫秒)
|
||||
ConfigKeyAgentWebsocketUpgradeEnabled = "agent_websocket_upgrade_enabled" // Agent WebSocket 升级开关
|
||||
ConfigKeyNodeOfflineThreshold = "node_offline_threshold" // 节点离线阈值(毫秒)
|
||||
ConfigKeyAgentUpdateRepo = "agent_update_repo" // Agent 更新仓库
|
||||
ConfigKeyGeoIPProvider = "geoip_provider" // GeoIP 服务商
|
||||
|
||||
// Pages 静态托管配置
|
||||
ConfigKeyPagesMaxPackageSizeMB = "pages_max_package_size_mb" // Pages 部署包上传大小上限(MiB)
|
||||
@@ -117,6 +115,15 @@ const (
|
||||
ConfigKeyOriginErrorPageGetOnly = "origin_error_page_get_only" // 是否仅对 GET 请求返回自定义错误页
|
||||
)
|
||||
|
||||
// 日志数据库解耦
|
||||
const (
|
||||
ConfigKeyLogDatabase = "log_database" // 当前日志主库:postgres|sqlite|clickhouse(仅迁移任务写入)
|
||||
ConfigKeyLogDBMigration = "log_db_migration" // 迁移冻结标记:"migrating" 或空
|
||||
ConfigKeyLogRetentionDaysPostgres = "log_retention_days_postgres" // PostgreSQL 日志保留天数
|
||||
ConfigKeyLogRetentionDaysSQLite = "log_retention_days_sqlite" // SQLite 日志保留天数
|
||||
ConfigKeyLogRetentionDaysClickHouse = "log_retention_days_clickhouse" // ClickHouse 日志保留天数
|
||||
)
|
||||
|
||||
const (
|
||||
// ConfigVisibilityHidden 表示配置不通过公共配置接口暴露
|
||||
ConfigVisibilityHidden = 0
|
||||
|
||||
Reference in New Issue
Block a user