mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-08 00:26:37 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
@@ -27,6 +27,17 @@ var (
|
||||
|
||||
const optionValueTrue = "true"
|
||||
|
||||
// protectedConfigKeyMessage 命中受保护 key 时返回给管理员的业务错误文案。
|
||||
const protectedConfigKeyMessage = "该配置项由系统任务管理,禁止手动修改"
|
||||
|
||||
// protectedConfigKeys 仅允许内部(迁移任务/bootstrap)写入的 key。
|
||||
var protectedConfigKeys = map[string]bool{
|
||||
model.ConfigKeyLogDatabase: true,
|
||||
model.ConfigKeyLogDBMigration: true,
|
||||
}
|
||||
|
||||
func isProtectedConfigKey(key string) bool { return protectedConfigKeys[key] }
|
||||
|
||||
func buildOptionValidationState(ctx context.Context, options []model.OpenFlareOption) map[string]string {
|
||||
// 从 SystemConfig 读取所有业务配置构建状态
|
||||
configs, err := repository.ListAdminSystemConfigs(ctx, "business")
|
||||
@@ -53,7 +64,7 @@ func validateOptionWithState(ctx context.Context, option model.OpenFlareOption,
|
||||
if err := validateGeoIPOption(option.Key, option.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDatabaseCleanupOption(option.Key, option.Value); err != nil {
|
||||
if err := validateLogRetentionOption(option.Key, option.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAgentOption(option.Key, option.Value); err != nil {
|
||||
@@ -100,11 +111,9 @@ func validateGeoIPOption(key, value string) error {
|
||||
return fmt.Errorf("%s 仅支持 disabled、mmdb、ip-api、geojs、ipinfo", key)
|
||||
}
|
||||
|
||||
func validateDatabaseCleanupOption(key, value string) error {
|
||||
func validateLogRetentionOption(key, value string) error {
|
||||
switch key {
|
||||
case model.ConfigKeyDatabaseAutoCleanupEnabled:
|
||||
return validateBooleanOption(key, value)
|
||||
case model.ConfigKeyDatabaseAutoCleanupRetentionDays:
|
||||
case model.ConfigKeyLogRetentionDaysPostgres, model.ConfigKeyLogRetentionDaysSQLite, model.ConfigKeyLogRetentionDaysClickHouse:
|
||||
intValue, err := strconv.Atoi(value)
|
||||
if err != nil || intValue < 1 {
|
||||
return fmt.Errorf("%s 必须为大于等于 1 的整数天", key)
|
||||
@@ -221,6 +230,9 @@ func validateOptions(ctx context.Context, options []model.OpenFlareOption) error
|
||||
if strings.TrimSpace(option.Key) == "" {
|
||||
return errors.New(errInvalidParams)
|
||||
}
|
||||
if isProtectedConfigKey(option.Key) {
|
||||
return errors.New(protectedConfigKeyMessage)
|
||||
}
|
||||
if err := validateOptionWithState(ctx, option, state); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user