feat(log): decouple log storage from ClickHouse with switchable logstore

- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
+1
View File
@@ -29,6 +29,7 @@ export type {
UpdateUserStatusRequest,
UpdateUserRequest,
SystemStatus,
LogDatabaseStatus,
AppUpdateStatus,
Schedule,
CreateScheduleRequest,
@@ -1,5 +1,5 @@
import { BaseService } from '@/lib/services/core';
import type { AppUpdateStatus, SystemStatus } from './types';
import type { AppUpdateStatus, LogDatabaseStatus, SystemStatus } from './types';
export class AdminStatusService extends BaseService {
protected static readonly basePath = '/api/v1/admin';
@@ -8,6 +8,10 @@ export class AdminStatusService extends BaseService {
return this.get<SystemStatus>('/status');
}
static async getLogDatabaseStatus(): Promise<LogDatabaseStatus> {
return this.get<LogDatabaseStatus>('/status/log-database');
}
static async getUpdateStatus(): Promise<AppUpdateStatus> {
return this.get<AppUpdateStatus>('/update');
}
+16
View File
@@ -405,6 +405,22 @@ export interface ToggleAuthSourceRequest {
is_active: boolean;
}
/**
* 日志数据库状态
*/
export interface LogDatabaseStatus {
/** 当前日志主库:postgres | sqlite | clickhouse */
active_database: string;
/** 迁移状态:idle | migrating */
migration: string;
/** 各日志库保留天数 */
retention_days: Record<string, number>;
/** 当前主库的合法迁移目标 */
available_targets: string[];
/** ClickHouse 运行指标(仅主库为 ClickHouse 时) */
clickhouse?: unknown;
}
/**
* 系统状态信息
*/
+1 -1
View File
@@ -158,6 +158,7 @@ export type {
CreateUserRequest,
UpdateUserRequest,
SystemStatus,
LogDatabaseStatus,
AppUpdateStatus,
Schedule,
CreateScheduleRequest,
@@ -260,6 +261,5 @@ export type {
AccessLogOverview,
OptionItem,
GeoIPLookupResult,
DatabaseCleanupResult,
OpenFlarePublicStatus,
} from './openflare';
-3
View File
@@ -104,9 +104,6 @@ export type {
FoldedAccessLogList,
OptionItem,
GeoIPLookupResult,
DatabaseCleanupPayload,
DatabaseCleanupResult,
DatabaseCleanupTarget,
OpenFlarePublicStatus,
OriginDetail,
OriginItem,
@@ -1,7 +1,5 @@
import { OpenFlareBaseService } from './base.service';
import type {
DatabaseCleanupPayload,
DatabaseCleanupResult,
GeoIPLookupResult,
OptionBatchPayload,
OptionItem,
@@ -26,10 +24,4 @@ export class OptionService extends OpenFlareBaseService {
static lookupGeoIP(provider: string, ip: string): Promise<GeoIPLookupResult> {
return this.post<GeoIPLookupResult>('/geoip/lookup', { provider, ip });
}
static cleanupDatabase(
payload: DatabaseCleanupPayload,
): Promise<DatabaseCleanupResult> {
return this.post<DatabaseCleanupResult>('/database/cleanup', payload);
}
}
-21
View File
@@ -866,27 +866,6 @@ export interface GeoIPLookupResult {
longitude?: number | null;
}
export type DatabaseCleanupTarget =
| 'node_access_logs'
| 'node_metric_snapshots'
| 'node_edge_health'
| 'node_obs_frps'
| 'node_obs_frpc';
export interface DatabaseCleanupPayload {
target: DatabaseCleanupTarget;
retention_days?: number;
}
export interface DatabaseCleanupResult {
target: DatabaseCleanupTarget;
target_label: string;
deleted_count: number;
delete_all: boolean;
retention_days?: number;
cutoff?: string;
}
export interface OpenFlarePublicStatus {
version: string;
start_time: number;