mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-06 23:56:37 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -21,11 +21,6 @@ const (
|
||||
// TaskTypeSSLRenew is the admin task type for SSL renewal.
|
||||
TaskTypeSSLRenew = "of_ssl_renew"
|
||||
|
||||
// DatabaseAutoCleanupTask prunes observability tables by retention policy.
|
||||
DatabaseAutoCleanupTask = "openflare:database_auto_cleanup"
|
||||
// TaskTypeDatabaseAutoCleanup is the admin task type for observability cleanup.
|
||||
TaskTypeDatabaseAutoCleanup = "of_database_auto_cleanup"
|
||||
|
||||
// WAFIPGroupSyncTask syncs due automatic/subscription WAF IP groups.
|
||||
WAFIPGroupSyncTask = "openflare:waf_ip_group_sync"
|
||||
// TaskTypeWAFIPGroupSync is the admin task type for WAF IP group sync.
|
||||
@@ -35,6 +30,11 @@ const (
|
||||
UptimeKumaSyncTask = "openflare:uptime_kuma_sync"
|
||||
// TaskTypeUptimeKumaSync is the admin task type for Uptime Kuma sync.
|
||||
TaskTypeUptimeKumaSync = "of_uptime_kuma_sync"
|
||||
|
||||
// LogDBSwitchTask 切换日志数据库任务标识。
|
||||
LogDBSwitchTask = "openflare:log_db_switch"
|
||||
// TaskTypeLogDBSwitch is the admin task type for log database switch.
|
||||
TaskTypeLogDBSwitch = "of_log_db_switch"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -54,18 +54,6 @@ var SSLRenewMeta = task.TaskMeta{
|
||||
Retryable: true,
|
||||
}
|
||||
|
||||
// DatabaseAutoCleanupMeta describes the observability auto-cleanup task.
|
||||
var DatabaseAutoCleanupMeta = task.TaskMeta{
|
||||
Type: TaskTypeDatabaseAutoCleanup,
|
||||
AsynqTask: DatabaseAutoCleanupTask,
|
||||
Name: "OpenFlare 可观测数据自动清理",
|
||||
Description: "按保留天数清理访问日志、性能快照与请求聚合数据",
|
||||
SupportsTime: false,
|
||||
MaxRetry: task.DefaultMaxRetry,
|
||||
Queue: task.QueueDefault,
|
||||
Retryable: true,
|
||||
}
|
||||
|
||||
// WAFIPGroupSyncMeta describes the WAF IP group sync task.
|
||||
var WAFIPGroupSyncMeta = task.TaskMeta{
|
||||
Type: TaskTypeWAFIPGroupSync,
|
||||
@@ -90,6 +78,22 @@ var UptimeKumaSyncMeta = task.TaskMeta{
|
||||
Retryable: true,
|
||||
}
|
||||
|
||||
// LogDBSwitchMeta 描述切换日志数据库任务。
|
||||
var LogDBSwitchMeta = task.TaskMeta{
|
||||
Type: TaskTypeLogDBSwitch,
|
||||
AsynqTask: LogDBSwitchTask,
|
||||
Name: "切换日志数据库",
|
||||
Description: "复制迁移日志数据并在成功后切换日志主库(期间禁止日志写入)",
|
||||
SupportsTime: false,
|
||||
MaxRetry: task.DefaultMaxRetry,
|
||||
Queue: task.QueueDefault,
|
||||
Retryable: true,
|
||||
Params: []task.TaskParam{
|
||||
{Name: "target", Label: "目标日志库", Type: "string", Required: true,
|
||||
Placeholder: "postgres|sqlite|clickhouse", Description: "迁移目标:postgres(主库为 PG 时)、sqlite(主库为 SQLite 时)或 clickhouse"},
|
||||
},
|
||||
}
|
||||
|
||||
// SSLRenewHandler renews due TLS certificates.
|
||||
type SSLRenewHandler struct{}
|
||||
|
||||
@@ -105,51 +109,6 @@ func (h *SSLRenewHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResu
|
||||
return &task.TaskResult{Message: msg}, nil
|
||||
}
|
||||
|
||||
// DatabaseAutoCleanupHandler prunes observability data when auto-cleanup is enabled.
|
||||
type DatabaseAutoCleanupHandler struct{}
|
||||
|
||||
// Execute runs retention-based cleanup for all observability targets.
|
||||
func (h *DatabaseAutoCleanupHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
|
||||
// 从 SystemConfig 读取自动清理配置
|
||||
enabled, _ := repository.GetBoolByKey(ctx, model.ConfigKeyDatabaseAutoCleanupEnabled)
|
||||
if !enabled {
|
||||
msg := "自动清理未启用,跳过执行"
|
||||
task.AppendLog(ctx, "%s", msg)
|
||||
return &task.TaskResult{Message: msg}, nil
|
||||
}
|
||||
|
||||
retentionDays, _ := repository.GetIntByKey(ctx, model.ConfigKeyDatabaseAutoCleanupRetentionDays)
|
||||
if retentionDays <= 0 {
|
||||
retentionDays = 30 // 默认保留 30 天
|
||||
}
|
||||
|
||||
task.AppendLog(ctx, "开始执行可观测数据自动清理,保留天数=%d", retentionDays)
|
||||
summary, err := tasks.RunDatabaseAutoCleanupOnce(ctx, time.Now())
|
||||
if err != nil {
|
||||
task.AppendLog(ctx, "可观测数据自动清理失败: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
if summary == nil {
|
||||
msg := "自动清理未启用,跳过执行"
|
||||
task.AppendLog(ctx, "%s", msg)
|
||||
return &task.TaskResult{Message: msg}, nil
|
||||
}
|
||||
|
||||
var totalDeleted int64
|
||||
for _, item := range summary.Results {
|
||||
totalDeleted += item.DeletedCount
|
||||
task.AppendLog(ctx, "清理 %s:删除 %d 条", item.TargetLabel, item.DeletedCount)
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf(
|
||||
"可观测数据自动清理完成,保留 %d 天,共删除 %d 条",
|
||||
summary.RetentionDays,
|
||||
totalDeleted,
|
||||
)
|
||||
task.AppendLog(ctx, "%s", msg)
|
||||
return &task.TaskResult{Message: msg}, nil
|
||||
}
|
||||
|
||||
// WAFIPGroupSyncHandler syncs due WAF IP groups to agents.
|
||||
type WAFIPGroupSyncHandler struct{}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user