Files
OpenFlare/internal/apps/openflare/async_tasks.go
T
ryan 7d71f1e4e1 feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
2026-08-08 19:43:01 +08:00

168 lines
5.8 KiB
Go

// Package openflare implements openflare configuration, service orchestration, and background tasks.
package openflare
import (
"context"
"fmt"
"sync"
"time"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/tasks"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/uptimekuma"
"github.com/Rain-kl/Wavelet/internal/apps/openflare/waf"
"github.com/Rain-kl/Wavelet/internal/infra/task"
"github.com/Rain-kl/Wavelet/internal/model"
"github.com/Rain-kl/Wavelet/internal/repository"
)
const (
// SSLRenewTask renews due ACME TLS certificates.
SSLRenewTask = "openflare:ssl_renew"
// TaskTypeSSLRenew is the admin task type for SSL renewal.
TaskTypeSSLRenew = "of_ssl_renew"
// WAFIPGroupSyncTask syncs due automatic/subscription WAF IP groups.
WAFIPGroupSyncTask = "openflare:waf_ip_group_sync"
// TaskTypeWAFIPGroupSync is the admin task type for WAF IP group sync.
TaskTypeWAFIPGroupSync = "of_waf_ip_group_sync"
// UptimeKumaSyncTask synchronizes proxy routes to Uptime Kuma monitors.
UptimeKumaSyncTask = "openflare:uptime_kuma_sync"
// TaskTypeUptimeKumaSync is the admin task type for Uptime Kuma sync.
TaskTypeUptimeKumaSync = "of_uptime_kuma_sync"
// LogDBSwitchTask 切换日志数据库任务标识。
LogDBSwitchTask = "openflare:log_db_switch"
// TaskTypeLogDBSwitch is the admin task type for log database switch.
TaskTypeLogDBSwitch = "of_log_db_switch"
)
var (
lastUptimeKumaSyncTime time.Time
uptimeKumaSyncMutex sync.Mutex
)
// SSLRenewMeta describes the SSL renewal task.
var SSLRenewMeta = task.TaskMeta{
Type: TaskTypeSSLRenew,
AsynqTask: SSLRenewTask,
Name: "OpenFlare SSL 自动续期",
Description: "扫描即将到期的 ACME 证书并触发自动续期",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
}
// WAFIPGroupSyncMeta describes the WAF IP group sync task.
var WAFIPGroupSyncMeta = task.TaskMeta{
Type: TaskTypeWAFIPGroupSync,
AsynqTask: WAFIPGroupSyncTask,
Name: "OpenFlare WAF IP 组同步",
Description: "同步到期的自动规则与订阅类型 WAF IP 组",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
}
// UptimeKumaSyncMeta describes the Uptime Kuma sync task.
var UptimeKumaSyncMeta = task.TaskMeta{
Type: TaskTypeUptimeKumaSync,
AsynqTask: UptimeKumaSyncTask,
Name: "OpenFlare Uptime Kuma 同步",
Description: "将启用的代理规则同步到 Uptime Kuma 监控",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
}
// LogDBSwitchMeta 描述切换日志数据库任务。
var LogDBSwitchMeta = task.TaskMeta{
Type: TaskTypeLogDBSwitch,
AsynqTask: LogDBSwitchTask,
Name: "切换日志数据库",
Description: "复制迁移日志数据并在成功后切换日志主库(期间禁止日志写入)",
SupportsTime: false,
MaxRetry: task.DefaultMaxRetry,
Queue: task.QueueDefault,
Retryable: true,
Params: []task.TaskParam{
{Name: "target", Label: "目标日志库", Type: "string", Required: true,
Placeholder: "postgres|sqlite|clickhouse", Description: "迁移目标:postgres(主库为 PG 时)、sqlite(主库为 SQLite 时)或 clickhouse"},
},
}
// SSLRenewHandler renews due TLS certificates.
type SSLRenewHandler struct{}
// Execute runs SSL certificate renewal for all due certificates.
func (h *SSLRenewHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
task.AppendLog(ctx, "开始扫描待续期证书")
if err := tasks.RunSSLRenewJob(ctx); err != nil {
task.AppendLog(ctx, "SSL 自动续期失败: %v", err)
return nil, err
}
msg := "SSL 自动续期任务完成"
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
// WAFIPGroupSyncHandler syncs due WAF IP groups to agents.
type WAFIPGroupSyncHandler struct{}
// Execute syncs all due automatic/subscription WAF IP groups.
func (h *WAFIPGroupSyncHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
task.AppendLog(ctx, "开始同步到期的 WAF IP 组")
if err := waf.SyncDueWAFIPGroups(ctx); err != nil {
task.AppendLog(ctx, "WAF IP 组同步失败: %v", err)
return nil, err
}
msg := "WAF IP 组同步完成"
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
// UptimeKumaSyncHandler synchronizes proxy routes to Uptime Kuma.
type UptimeKumaSyncHandler struct{}
// Execute runs Uptime Kuma sync when integration is enabled and the interval has elapsed.
func (h *UptimeKumaSyncHandler) Execute(ctx context.Context, _ []byte) (*task.TaskResult, error) {
// 从 SystemConfig 读取 UptimeKuma 配置
enabled, _ := repository.GetBoolByKey(ctx, model.ConfigKeyUptimeKumaEnabled)
if !enabled {
msg := "Uptime Kuma 集成未启用,跳过执行"
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
interval, _ := repository.GetIntByKey(ctx, model.ConfigKeyUptimeKumaSyncInterval)
if interval <= 0 {
interval = 5
}
if time.Since(lastUptimeKumaSyncTime) < time.Duration(interval)*time.Minute {
msg := fmt.Sprintf("距上次同步不足 %d 分钟,跳过执行", interval)
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
if !uptimeKumaSyncMutex.TryLock() {
msg := "Uptime Kuma 同步任务正在执行,跳过本次调度"
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}
defer uptimeKumaSyncMutex.Unlock()
task.AppendLog(ctx, "开始同步代理规则到 Uptime Kuma")
if err := uptimekuma.SyncToUptimeKuma(ctx); err != nil {
task.AppendLog(ctx, "Uptime Kuma 同步失败: %v", err)
return nil, err
}
lastUptimeKumaSyncTime = time.Now()
msg := "Uptime Kuma 同步完成"
task.AppendLog(ctx, "%s", msg)
return &task.TaskResult{Message: msg}, nil
}