mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 23:26:38 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -29,6 +29,7 @@ export type {
|
||||
UpdateUserStatusRequest,
|
||||
UpdateUserRequest,
|
||||
SystemStatus,
|
||||
LogDatabaseStatus,
|
||||
AppUpdateStatus,
|
||||
Schedule,
|
||||
CreateScheduleRequest,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BaseService } from '@/lib/services/core';
|
||||
import type { AppUpdateStatus, SystemStatus } from './types';
|
||||
import type { AppUpdateStatus, LogDatabaseStatus, SystemStatus } from './types';
|
||||
|
||||
export class AdminStatusService extends BaseService {
|
||||
protected static readonly basePath = '/api/v1/admin';
|
||||
@@ -8,6 +8,10 @@ export class AdminStatusService extends BaseService {
|
||||
return this.get<SystemStatus>('/status');
|
||||
}
|
||||
|
||||
static async getLogDatabaseStatus(): Promise<LogDatabaseStatus> {
|
||||
return this.get<LogDatabaseStatus>('/status/log-database');
|
||||
}
|
||||
|
||||
static async getUpdateStatus(): Promise<AppUpdateStatus> {
|
||||
return this.get<AppUpdateStatus>('/update');
|
||||
}
|
||||
|
||||
@@ -405,6 +405,22 @@ export interface ToggleAuthSourceRequest {
|
||||
is_active: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* 日志数据库状态
|
||||
*/
|
||||
export interface LogDatabaseStatus {
|
||||
/** 当前日志主库:postgres | sqlite | clickhouse */
|
||||
active_database: string;
|
||||
/** 迁移状态:idle | migrating */
|
||||
migration: string;
|
||||
/** 各日志库保留天数 */
|
||||
retention_days: Record<string, number>;
|
||||
/** 当前主库的合法迁移目标 */
|
||||
available_targets: string[];
|
||||
/** ClickHouse 运行指标(仅主库为 ClickHouse 时) */
|
||||
clickhouse?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* 系统状态信息
|
||||
*/
|
||||
|
||||
@@ -158,6 +158,7 @@ export type {
|
||||
CreateUserRequest,
|
||||
UpdateUserRequest,
|
||||
SystemStatus,
|
||||
LogDatabaseStatus,
|
||||
AppUpdateStatus,
|
||||
Schedule,
|
||||
CreateScheduleRequest,
|
||||
@@ -260,6 +261,5 @@ export type {
|
||||
AccessLogOverview,
|
||||
OptionItem,
|
||||
GeoIPLookupResult,
|
||||
DatabaseCleanupResult,
|
||||
OpenFlarePublicStatus,
|
||||
} from './openflare';
|
||||
|
||||
@@ -104,9 +104,6 @@ export type {
|
||||
FoldedAccessLogList,
|
||||
OptionItem,
|
||||
GeoIPLookupResult,
|
||||
DatabaseCleanupPayload,
|
||||
DatabaseCleanupResult,
|
||||
DatabaseCleanupTarget,
|
||||
OpenFlarePublicStatus,
|
||||
OriginDetail,
|
||||
OriginItem,
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import { OpenFlareBaseService } from './base.service';
|
||||
import type {
|
||||
DatabaseCleanupPayload,
|
||||
DatabaseCleanupResult,
|
||||
GeoIPLookupResult,
|
||||
OptionBatchPayload,
|
||||
OptionItem,
|
||||
@@ -26,10 +24,4 @@ export class OptionService extends OpenFlareBaseService {
|
||||
static lookupGeoIP(provider: string, ip: string): Promise<GeoIPLookupResult> {
|
||||
return this.post<GeoIPLookupResult>('/geoip/lookup', { provider, ip });
|
||||
}
|
||||
|
||||
static cleanupDatabase(
|
||||
payload: DatabaseCleanupPayload,
|
||||
): Promise<DatabaseCleanupResult> {
|
||||
return this.post<DatabaseCleanupResult>('/database/cleanup', payload);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -866,27 +866,6 @@ export interface GeoIPLookupResult {
|
||||
longitude?: number | null;
|
||||
}
|
||||
|
||||
export type DatabaseCleanupTarget =
|
||||
| 'node_access_logs'
|
||||
| 'node_metric_snapshots'
|
||||
| 'node_edge_health'
|
||||
| 'node_obs_frps'
|
||||
| 'node_obs_frpc';
|
||||
|
||||
export interface DatabaseCleanupPayload {
|
||||
target: DatabaseCleanupTarget;
|
||||
retention_days?: number;
|
||||
}
|
||||
|
||||
export interface DatabaseCleanupResult {
|
||||
target: DatabaseCleanupTarget;
|
||||
target_label: string;
|
||||
deleted_count: number;
|
||||
delete_all: boolean;
|
||||
retention_days?: number;
|
||||
cutoff?: string;
|
||||
}
|
||||
|
||||
export interface OpenFlarePublicStatus {
|
||||
version: string;
|
||||
start_time: number;
|
||||
|
||||
Reference in New Issue
Block a user