feat(log): decouple log storage from ClickHouse with switchable logstore

- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
@@ -18,8 +18,6 @@ export type OpenFlareOpsFields = {
uptime_kuma_retry: string;
uptime_kuma_retry_interval: string;
uptime_kuma_timeout: string;
database_auto_cleanup_enabled: boolean;
database_auto_cleanup_retention_days: string;
pages_max_package_size_mb: string;
pages_max_history_count: string;
};
@@ -42,8 +40,6 @@ export const defaultOpenFlareOpsFields: OpenFlareOpsFields = {
uptime_kuma_retry: '0',
uptime_kuma_retry_interval: '60',
uptime_kuma_timeout: '48',
database_auto_cleanup_enabled: false,
database_auto_cleanup_retention_days: '30',
pages_max_package_size_mb: '100',
pages_max_history_count: '20',
};
@@ -88,12 +84,6 @@ export function mapOptionsToOpsFields(
uptime_kuma_retry: optionMap.uptime_kuma_retry ?? '0',
uptime_kuma_retry_interval: optionMap.uptime_kuma_retry_interval ?? '60',
uptime_kuma_timeout: optionMap.uptime_kuma_timeout ?? '48',
database_auto_cleanup_enabled: toBoolean(
optionMap.database_auto_cleanup_enabled,
false,
),
database_auto_cleanup_retention_days:
optionMap.database_auto_cleanup_retention_days ?? '30',
pages_max_package_size_mb: optionMap.pages_max_package_size_mb ?? '100',
pages_max_history_count: optionMap.pages_max_history_count ?? '20',
};
@@ -165,16 +155,6 @@ export function validateUptimeKumaFields(fields: OpenFlareOpsFields) {
throw new Error('请求超时必须为正整数。');
}
export function validateDatabaseAutoCleanup(fields: OpenFlareOpsFields) {
const retentionDays = Number.parseInt(
fields.database_auto_cleanup_retention_days,
10,
);
if (Number.isNaN(retentionDays) || retentionDays < 1) {
throw new Error('自动清理保留天数至少为 1 天。');
}
}
export function agentOptionEntries(fields: OpenFlareOpsFields): OptionItem[] {
validateAgentFields(fields);
return [
@@ -220,22 +200,6 @@ export function uptimeKumaOptionEntries(
];
}
export function databaseAutoCleanupEntries(
fields: OpenFlareOpsFields,
): OptionItem[] {
validateDatabaseAutoCleanup(fields);
return [
{
key: 'database_auto_cleanup_enabled',
value: String(fields.database_auto_cleanup_enabled),
},
{
key: 'database_auto_cleanup_retention_days',
value: fields.database_auto_cleanup_retention_days,
},
];
}
export function validatePagesFields(fields: OpenFlareOpsFields) {
const packageSize = Number.parseInt(fields.pages_max_package_size_mb, 10);
const historyCount = Number.parseInt(fields.pages_max_history_count, 10);