mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-07 16:16:37 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
'use client';
|
||||
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
@@ -8,6 +8,13 @@ import { Label } from '@/components/ui/label';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { Switch } from '@/components/ui/switch';
|
||||
import { Spinner } from '@/components/ui/spinner';
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
@@ -19,12 +26,17 @@ import {
|
||||
import {
|
||||
Calendar as CalendarIcon,
|
||||
Clock,
|
||||
Database,
|
||||
Info,
|
||||
Layers,
|
||||
Play,
|
||||
} from 'lucide-react';
|
||||
|
||||
import type { DispatchTaskRequest, TaskMeta } from '@/lib/services/admin';
|
||||
import type {
|
||||
DispatchTaskRequest,
|
||||
LogDatabaseStatus,
|
||||
TaskMeta,
|
||||
} from '@/lib/services/admin';
|
||||
import services from '@/lib/services';
|
||||
import { buildTaskPayload } from '@/lib/task-param-utils';
|
||||
import { ErrorInline } from '@/components/layout/error';
|
||||
@@ -68,6 +80,18 @@ const TASK_CONFIGS: Record<
|
||||
gradient:
|
||||
'from-rose-500/10 via-rose-500/5 to-transparent border-rose-200/50 dark:border-rose-800/50 hover:border-rose-400 dark:hover:border-rose-500',
|
||||
},
|
||||
of_log_db_switch: {
|
||||
icon: Database,
|
||||
color: 'text-teal-600 dark:text-teal-400',
|
||||
gradient:
|
||||
'from-teal-500/10 via-teal-500/5 to-transparent border-teal-200/50 dark:border-teal-800/50 hover:border-teal-400 dark:hover:border-teal-500',
|
||||
},
|
||||
};
|
||||
|
||||
const LOG_DATABASE_LABELS: Record<string, string> = {
|
||||
postgres: 'PostgreSQL(主库)',
|
||||
sqlite: 'SQLite(主库)',
|
||||
clickhouse: 'ClickHouse',
|
||||
};
|
||||
|
||||
const DEFAULT_TASK_CONFIG = {
|
||||
@@ -192,9 +216,38 @@ export function TaskManager() {
|
||||
}
|
||||
}, []);
|
||||
|
||||
const [logDbStatus, setLogDbStatus] = useState<LogDatabaseStatus | null>(
|
||||
null,
|
||||
);
|
||||
|
||||
// 日志库状态用于「切换日志数据库」卡片与 target 下拉;获取失败不阻塞任务列表。
|
||||
const fetchLogDbStatus = useCallback(async () => {
|
||||
try {
|
||||
const data = await services.adminStatus.getLogDatabaseStatus();
|
||||
setLogDbStatus(data);
|
||||
} catch {
|
||||
setLogDbStatus(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchTaskTypes();
|
||||
}, [fetchTaskTypes]);
|
||||
fetchLogDbStatus();
|
||||
}, [fetchTaskTypes, fetchLogDbStatus]);
|
||||
|
||||
const availableLogDbTargets = useMemo(
|
||||
() => logDbStatus?.available_targets ?? [],
|
||||
[logDbStatus],
|
||||
);
|
||||
|
||||
const retentionSummary = useMemo(() => {
|
||||
const days = logDbStatus?.retention_days ?? {};
|
||||
const parts: string[] = [];
|
||||
if (days.postgres != null) parts.push(`PG ${days.postgres}`);
|
||||
if (days.sqlite != null) parts.push(`SQLite ${days.sqlite}`);
|
||||
if (days.clickhouse != null) parts.push(`CH ${days.clickhouse}`);
|
||||
return parts.join(' / ');
|
||||
}, [logDbStatus]);
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedTaskType) {
|
||||
@@ -344,6 +397,45 @@ export function TaskManager() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{task.type === 'of_log_db_switch' && logDbStatus && (
|
||||
<div className='pt-3 mt-3 border-t border-border/50 space-y-1.5'>
|
||||
<div className='flex items-center justify-between gap-2'>
|
||||
<span className='text-[10px] text-muted-foreground shrink-0'>
|
||||
日志主库
|
||||
</span>
|
||||
<span className='text-[10px] font-mono text-foreground truncate'>
|
||||
{LOG_DATABASE_LABELS[logDbStatus.active_database] ||
|
||||
logDbStatus.active_database}
|
||||
</span>
|
||||
</div>
|
||||
<div className='flex items-center justify-between gap-2'>
|
||||
<span className='text-[10px] text-muted-foreground shrink-0'>
|
||||
保留天数
|
||||
</span>
|
||||
<span className='text-[10px] font-mono text-muted-foreground truncate'>
|
||||
{retentionSummary || '-'}
|
||||
</span>
|
||||
</div>
|
||||
<div className='flex items-center justify-between gap-2'>
|
||||
<span className='text-[10px] text-muted-foreground shrink-0'>
|
||||
迁移状态
|
||||
</span>
|
||||
<Badge
|
||||
variant={
|
||||
logDbStatus.migration === 'migrating'
|
||||
? 'default'
|
||||
: 'outline'
|
||||
}
|
||||
className='text-[10px] h-5 px-1.5'
|
||||
>
|
||||
{logDbStatus.migration === 'migrating'
|
||||
? '迁移中'
|
||||
: '空闲'}
|
||||
</Badge>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className='pt-4 mt-1'>
|
||||
<Button
|
||||
className='w-full h-7 text-xs'
|
||||
@@ -436,70 +528,100 @@ export function TaskManager() {
|
||||
}
|
||||
return (
|
||||
<div className='space-y-4'>
|
||||
{targetTask.params.map((param) => (
|
||||
<div key={param.name} className='grid gap-2'>
|
||||
<Label
|
||||
htmlFor={`param-${param.name}`}
|
||||
className='flex items-center gap-1'
|
||||
>
|
||||
{param.label}
|
||||
{param.required && (
|
||||
<span className='text-destructive font-bold'>*</span>
|
||||
)}
|
||||
</Label>
|
||||
{param.type === 'text' ? (
|
||||
<Textarea
|
||||
id={`param-${param.name}`}
|
||||
placeholder={param.placeholder}
|
||||
className='text-xs min-h-[80px]'
|
||||
value={paramValues[param.name] || ''}
|
||||
onChange={(e) =>
|
||||
setParamValues((prev) => ({
|
||||
...prev,
|
||||
[param.name]: e.target.value,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
) : param.type === 'boolean' ? (
|
||||
<div className='flex items-center gap-2 pt-1 h-9'>
|
||||
<Switch
|
||||
{targetTask.params.map((param) => {
|
||||
const isSwitchTarget =
|
||||
param.name === 'target' &&
|
||||
getSelectedTaskMeta()?.type === 'of_log_db_switch';
|
||||
return (
|
||||
<div key={param.name} className='grid gap-2'>
|
||||
<Label
|
||||
htmlFor={`param-${param.name}`}
|
||||
className='flex items-center gap-1'
|
||||
>
|
||||
{param.label}
|
||||
{param.required && (
|
||||
<span className='text-destructive font-bold'>
|
||||
*
|
||||
</span>
|
||||
)}
|
||||
</Label>
|
||||
{param.type === 'text' ? (
|
||||
<Textarea
|
||||
id={`param-${param.name}`}
|
||||
checked={paramValues[param.name] === 'true'}
|
||||
onCheckedChange={(checked) =>
|
||||
placeholder={param.placeholder}
|
||||
className='text-xs min-h-[80px]'
|
||||
value={paramValues[param.name] || ''}
|
||||
onChange={(e) =>
|
||||
setParamValues((prev) => ({
|
||||
...prev,
|
||||
[param.name]: checked ? 'true' : 'false',
|
||||
[param.name]: e.target.value,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
<span className='text-xs text-muted-foreground'>
|
||||
{paramValues[param.name] === 'true'
|
||||
? '开启'
|
||||
: '关闭'}
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
<Input
|
||||
id={`param-${param.name}`}
|
||||
type={param.type === 'number' ? 'number' : 'text'}
|
||||
placeholder={param.placeholder}
|
||||
className='text-xs'
|
||||
value={paramValues[param.name] || ''}
|
||||
onChange={(e) =>
|
||||
setParamValues((prev) => ({
|
||||
...prev,
|
||||
[param.name]: e.target.value,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{param.description && (
|
||||
<p className='text-[10px] text-muted-foreground'>
|
||||
{param.description}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
) : param.type === 'boolean' ? (
|
||||
<div className='flex items-center gap-2 pt-1 h-9'>
|
||||
<Switch
|
||||
id={`param-${param.name}`}
|
||||
checked={paramValues[param.name] === 'true'}
|
||||
onCheckedChange={(checked) =>
|
||||
setParamValues((prev) => ({
|
||||
...prev,
|
||||
[param.name]: checked ? 'true' : 'false',
|
||||
}))
|
||||
}
|
||||
/>
|
||||
<span className='text-xs text-muted-foreground'>
|
||||
{paramValues[param.name] === 'true'
|
||||
? '开启'
|
||||
: '关闭'}
|
||||
</span>
|
||||
</div>
|
||||
) : isSwitchTarget &&
|
||||
availableLogDbTargets.length > 0 ? (
|
||||
<Select
|
||||
value={paramValues[param.name] || ''}
|
||||
onValueChange={(value) =>
|
||||
setParamValues((prev) => ({
|
||||
...prev,
|
||||
[param.name]: value,
|
||||
}))
|
||||
}
|
||||
disabled={dispatching}
|
||||
>
|
||||
<SelectTrigger className='w-full text-xs' size='sm'>
|
||||
<SelectValue placeholder='选择目标日志库...' />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{availableLogDbTargets.map((target) => (
|
||||
<SelectItem key={target} value={target}>
|
||||
{LOG_DATABASE_LABELS[target] || target}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
) : (
|
||||
<Input
|
||||
id={`param-${param.name}`}
|
||||
type={param.type === 'number' ? 'number' : 'text'}
|
||||
placeholder={param.placeholder}
|
||||
className='text-xs'
|
||||
value={paramValues[param.name] || ''}
|
||||
onChange={(e) =>
|
||||
setParamValues((prev) => ({
|
||||
...prev,
|
||||
[param.name]: e.target.value,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{param.description && (
|
||||
<p className='text-[10px] text-muted-foreground'>
|
||||
{param.description}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
Reference in New Issue
Block a user