feat(log): decouple log storage from ClickHouse with switchable logstore

- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
@@ -1,6 +1,6 @@
'use client';
import { useCallback, useEffect, useState } from 'react';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { toast } from 'sonner';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
@@ -8,6 +8,13 @@ import { Label } from '@/components/ui/label';
import { Textarea } from '@/components/ui/textarea';
import { Switch } from '@/components/ui/switch';
import { Spinner } from '@/components/ui/spinner';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import {
Dialog,
DialogContent,
@@ -19,12 +26,17 @@ import {
import {
Calendar as CalendarIcon,
Clock,
Database,
Info,
Layers,
Play,
} from 'lucide-react';
import type { DispatchTaskRequest, TaskMeta } from '@/lib/services/admin';
import type {
DispatchTaskRequest,
LogDatabaseStatus,
TaskMeta,
} from '@/lib/services/admin';
import services from '@/lib/services';
import { buildTaskPayload } from '@/lib/task-param-utils';
import { ErrorInline } from '@/components/layout/error';
@@ -68,6 +80,18 @@ const TASK_CONFIGS: Record<
gradient:
'from-rose-500/10 via-rose-500/5 to-transparent border-rose-200/50 dark:border-rose-800/50 hover:border-rose-400 dark:hover:border-rose-500',
},
of_log_db_switch: {
icon: Database,
color: 'text-teal-600 dark:text-teal-400',
gradient:
'from-teal-500/10 via-teal-500/5 to-transparent border-teal-200/50 dark:border-teal-800/50 hover:border-teal-400 dark:hover:border-teal-500',
},
};
const LOG_DATABASE_LABELS: Record<string, string> = {
postgres: 'PostgreSQL(主库)',
sqlite: 'SQLite(主库)',
clickhouse: 'ClickHouse',
};
const DEFAULT_TASK_CONFIG = {
@@ -192,9 +216,38 @@ export function TaskManager() {
}
}, []);
const [logDbStatus, setLogDbStatus] = useState<LogDatabaseStatus | null>(
null,
);
// 日志库状态用于「切换日志数据库」卡片与 target 下拉;获取失败不阻塞任务列表。
const fetchLogDbStatus = useCallback(async () => {
try {
const data = await services.adminStatus.getLogDatabaseStatus();
setLogDbStatus(data);
} catch {
setLogDbStatus(null);
}
}, []);
useEffect(() => {
fetchTaskTypes();
}, [fetchTaskTypes]);
fetchLogDbStatus();
}, [fetchTaskTypes, fetchLogDbStatus]);
const availableLogDbTargets = useMemo(
() => logDbStatus?.available_targets ?? [],
[logDbStatus],
);
const retentionSummary = useMemo(() => {
const days = logDbStatus?.retention_days ?? {};
const parts: string[] = [];
if (days.postgres != null) parts.push(`PG ${days.postgres}`);
if (days.sqlite != null) parts.push(`SQLite ${days.sqlite}`);
if (days.clickhouse != null) parts.push(`CH ${days.clickhouse}`);
return parts.join(' / ');
}, [logDbStatus]);
useEffect(() => {
if (selectedTaskType) {
@@ -344,6 +397,45 @@ export function TaskManager() {
</div>
</div>
{task.type === 'of_log_db_switch' && logDbStatus && (
<div className='pt-3 mt-3 border-t border-border/50 space-y-1.5'>
<div className='flex items-center justify-between gap-2'>
<span className='text-[10px] text-muted-foreground shrink-0'>
日志主库
</span>
<span className='text-[10px] font-mono text-foreground truncate'>
{LOG_DATABASE_LABELS[logDbStatus.active_database] ||
logDbStatus.active_database}
</span>
</div>
<div className='flex items-center justify-between gap-2'>
<span className='text-[10px] text-muted-foreground shrink-0'>
保留天数
</span>
<span className='text-[10px] font-mono text-muted-foreground truncate'>
{retentionSummary || '-'}
</span>
</div>
<div className='flex items-center justify-between gap-2'>
<span className='text-[10px] text-muted-foreground shrink-0'>
迁移状态
</span>
<Badge
variant={
logDbStatus.migration === 'migrating'
? 'default'
: 'outline'
}
className='text-[10px] h-5 px-1.5'
>
{logDbStatus.migration === 'migrating'
? '迁移中'
: '空闲'}
</Badge>
</div>
</div>
)}
<div className='pt-4 mt-1'>
<Button
className='w-full h-7 text-xs'
@@ -436,70 +528,100 @@ export function TaskManager() {
}
return (
<div className='space-y-4'>
{targetTask.params.map((param) => (
<div key={param.name} className='grid gap-2'>
<Label
htmlFor={`param-${param.name}`}
className='flex items-center gap-1'
>
{param.label}
{param.required && (
<span className='text-destructive font-bold'>*</span>
)}
</Label>
{param.type === 'text' ? (
<Textarea
id={`param-${param.name}`}
placeholder={param.placeholder}
className='text-xs min-h-[80px]'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: e.target.value,
}))
}
/>
) : param.type === 'boolean' ? (
<div className='flex items-center gap-2 pt-1 h-9'>
<Switch
{targetTask.params.map((param) => {
const isSwitchTarget =
param.name === 'target' &&
getSelectedTaskMeta()?.type === 'of_log_db_switch';
return (
<div key={param.name} className='grid gap-2'>
<Label
htmlFor={`param-${param.name}`}
className='flex items-center gap-1'
>
{param.label}
{param.required && (
<span className='text-destructive font-bold'>
*
</span>
)}
</Label>
{param.type === 'text' ? (
<Textarea
id={`param-${param.name}`}
checked={paramValues[param.name] === 'true'}
onCheckedChange={(checked) =>
placeholder={param.placeholder}
className='text-xs min-h-[80px]'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: checked ? 'true' : 'false',
[param.name]: e.target.value,
}))
}
/>
<span className='text-xs text-muted-foreground'>
{paramValues[param.name] === 'true'
? '开启'
: '关闭'}
</span>
</div>
) : (
<Input
id={`param-${param.name}`}
type={param.type === 'number' ? 'number' : 'text'}
placeholder={param.placeholder}
className='text-xs'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: e.target.value,
}))
}
/>
)}
{param.description && (
<p className='text-[10px] text-muted-foreground'>
{param.description}
</p>
)}
</div>
))}
) : param.type === 'boolean' ? (
<div className='flex items-center gap-2 pt-1 h-9'>
<Switch
id={`param-${param.name}`}
checked={paramValues[param.name] === 'true'}
onCheckedChange={(checked) =>
setParamValues((prev) => ({
...prev,
[param.name]: checked ? 'true' : 'false',
}))
}
/>
<span className='text-xs text-muted-foreground'>
{paramValues[param.name] === 'true'
? '开启'
: '关闭'}
</span>
</div>
) : isSwitchTarget &&
availableLogDbTargets.length > 0 ? (
<Select
value={paramValues[param.name] || ''}
onValueChange={(value) =>
setParamValues((prev) => ({
...prev,
[param.name]: value,
}))
}
disabled={dispatching}
>
<SelectTrigger className='w-full text-xs' size='sm'>
<SelectValue placeholder='选择目标日志库...' />
</SelectTrigger>
<SelectContent>
{availableLogDbTargets.map((target) => (
<SelectItem key={target} value={target}>
{LOG_DATABASE_LABELS[target] || target}
</SelectItem>
))}
</SelectContent>
</Select>
) : (
<Input
id={`param-${param.name}`}
type={param.type === 'number' ? 'number' : 'text'}
placeholder={param.placeholder}
className='text-xs'
value={paramValues[param.name] || ''}
onChange={(e) =>
setParamValues((prev) => ({
...prev,
[param.name]: e.target.value,
}))
}
/>
)}
{param.description && (
<p className='text-[10px] text-muted-foreground'>
{param.description}
</p>
)}
</div>
);
})}
</div>
);
})()}