mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-09-28 05:46:36 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
@@ -0,0 +1,114 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package analytics defines ClickHouse analytics domain models and query DTOs
|
||||
// (pure data, no IO).
|
||||
package analytics
|
||||
|
||||
import "time"
|
||||
|
||||
// AccessLogFilter scopes user access log queries.
|
||||
// 单一权威字段集（CH 原字段，Task 1 迁入）：禁止追加仅某实现使用的字段（避免双字段集分叉）。
|
||||
type AccessLogFilter struct {
|
||||
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
|
||||
UserIDs []uint64
|
||||
Path string
|
||||
// StartTime filters created_at >= StartTime when non-nil.
|
||||
StartTime *time.Time
|
||||
// EndTime filters created_at <= EndTime when non-nil（闭区间，与 CH/GORM 实现一致）。
|
||||
EndTime *time.Time
|
||||
}
|
||||
|
||||
// NodeAccessLogFilter scopes ClickHouse node access log queries.
|
||||
type NodeAccessLogFilter struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
|
||||
Hosts []string
|
||||
Path string
|
||||
Since time.Time
|
||||
Until time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
SortBy string
|
||||
SortOrder string
|
||||
}
|
||||
|
||||
// NodeObservabilityFilter scopes ClickHouse node observability queries.
|
||||
type NodeObservabilityFilter struct {
|
||||
NodeID string
|
||||
Since time.Time
|
||||
Limit int
|
||||
}
|
||||
|
||||
// DailyTrend is a single day's access count.
|
||||
type DailyTrend struct {
|
||||
Date string
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// BrowserShare is a browser group's share of access logs.
|
||||
type BrowserShare struct {
|
||||
Browser string
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// TopUser is an active user ranked by access count.
|
||||
type TopUser struct {
|
||||
UserID uint64
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// NodeAccessLogRegionCount aggregates access log regions.
|
||||
type NodeAccessLogRegionCount struct {
|
||||
Region string
|
||||
Count int64
|
||||
}
|
||||
|
||||
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
|
||||
type NodeAccessLogTrafficSummary struct {
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
BytesSent int64
|
||||
RequestLength int64
|
||||
NodeCount int64
|
||||
}
|
||||
|
||||
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
|
||||
type NodeAccessLogValueCount struct {
|
||||
Value string
|
||||
Count int64
|
||||
}
|
||||
|
||||
// NodeAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type NodeAccessLogNodeAggregate struct {
|
||||
NodeID string
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
}
|
||||
|
||||
// BatchWriterStats is a point-in-time snapshot of a batch writer queue and failure counters.
|
||||
type BatchWriterStats struct {
|
||||
Name string `json:"name"`
|
||||
Depth int `json:"depth"`
|
||||
Cap int `json:"cap"`
|
||||
Drops int64 `json:"drops"`
|
||||
FlushErrors int64 `json:"flush_errors"`
|
||||
Running bool `json:"running"`
|
||||
}
|
||||
|
||||
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
|
||||
// and in-process batch writer queue health.
|
||||
type ClickHouseOperationalStats struct {
|
||||
Database string `json:"database"`
|
||||
ActiveParts int64 `json:"active_parts"`
|
||||
TotalRows int64 `json:"total_rows"`
|
||||
PendingMutations int64 `json:"pending_mutations"`
|
||||
AsyncInsertQueue int64 `json:"async_insert_queue"`
|
||||
AsyncInsertBytes int64 `json:"async_insert_bytes"`
|
||||
// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
|
||||
BatchWriters []BatchWriterStats `json:"batch_writers,omitempty"`
|
||||
}
|
||||
@@ -90,6 +90,34 @@ type AccessLogHourly struct {
|
||||
RequestLength int64 `gorm:"column:request_length"`
|
||||
}
|
||||
|
||||
// NodeTrafficHourly is an hourly traffic rollup row.
|
||||
//
|
||||
// UniqueVisitorCount is always 0 when sourced from of_access_log_hourly
|
||||
// (true UV requires raw uniqExact on access logs).
|
||||
type NodeTrafficHourly struct {
|
||||
NodeID string
|
||||
Hour time.Time
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueVisitorCount int64
|
||||
}
|
||||
|
||||
// NodeMetricHourly is an hourly metric snapshot aggregation row.
|
||||
//
|
||||
// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
|
||||
// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
|
||||
// lagInFrame samples per node (negative deltas after counter reset are dropped).
|
||||
type NodeMetricHourly struct {
|
||||
Hour time.Time
|
||||
AverageCPUUsagePercent float64
|
||||
AverageMemoryUsagePercent float64
|
||||
NetworkRxBytes int64
|
||||
NetworkTxBytes int64
|
||||
DiskReadBytes int64
|
||||
DiskWriteBytes int64
|
||||
ReportedNodes int
|
||||
}
|
||||
|
||||
// NodeObsFrps stores FRPS observability snapshots in ClickHouse.
|
||||
type NodeObsFrps struct {
|
||||
ID uint64 `gorm:"column:id"`
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package analytics defines ClickHouse analytics domain models.
|
||||
package analytics
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import "strings"
|
||||
|
||||
// User-Agent 浏览器/OS/设备分类（纯函数，无 IO）。
|
||||
// 与 internal/repository/analytics/browser.go 的判定逻辑保持一致（Task 4 复制，
|
||||
// 因为 model 不得 import analyticsrepo）；后续若移除旧 CH 实现，可让 analyticsrepo 改以别名复用本包。
|
||||
|
||||
const (
|
||||
uaLabelUnknown = "Unknown"
|
||||
uaLabelBot = "Bot"
|
||||
uaLabelOther = "Other"
|
||||
uaTokenBot = "bot"
|
||||
uaTokenAndroid = "android"
|
||||
uaTokenSpider = "spider"
|
||||
uaTokenCrawler = "crawler"
|
||||
)
|
||||
|
||||
type uaMatchRule struct {
|
||||
label string
|
||||
contains []string
|
||||
allOf []string
|
||||
noneOf []string
|
||||
}
|
||||
|
||||
func matchUARules(uaLower string, rules []uaMatchRule, fallback string) string {
|
||||
if uaLower == "" {
|
||||
return uaLabelUnknown
|
||||
}
|
||||
for _, rule := range rules {
|
||||
matched := false
|
||||
for _, token := range rule.contains {
|
||||
if strings.Contains(uaLower, token) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched && len(rule.allOf) > 0 {
|
||||
matched = true
|
||||
for _, token := range rule.allOf {
|
||||
if !strings.Contains(uaLower, token) {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
excluded := false
|
||||
for _, token := range rule.noneOf {
|
||||
if strings.Contains(uaLower, token) {
|
||||
excluded = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if excluded {
|
||||
continue
|
||||
}
|
||||
return rule.label
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
var browserRules = []uaMatchRule{
|
||||
{label: "WeChat", contains: []string{"micromessenger"}},
|
||||
{label: "Postman", contains: []string{"postman"}},
|
||||
{label: "CLI", contains: []string{"curl/", "wget/"}},
|
||||
{label: "Edge", contains: []string{"edg/", "edgios/", "edga/"}},
|
||||
{label: "Opera", contains: []string{"opr/", "opera"}},
|
||||
{label: "Firefox", contains: []string{"firefox", "fxios"}},
|
||||
{label: "Chrome", contains: []string{"crios", "chrome"}, noneOf: []string{"chromium"}},
|
||||
{label: "Chromium", contains: []string{"chromium"}},
|
||||
{label: "Safari", contains: []string{"safari"}},
|
||||
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp"}},
|
||||
}
|
||||
|
||||
var osRules = []uaMatchRule{
|
||||
{label: "Android", contains: []string{uaTokenAndroid}},
|
||||
{label: "iOS", contains: []string{"iphone", "ipad", "ipod", "ios"}},
|
||||
{label: "Windows", contains: []string{"windows"}},
|
||||
{label: "macOS", contains: []string{"mac os x", "macintosh", "macos"}},
|
||||
{label: "Chrome OS", contains: []string{"cros"}},
|
||||
{label: "Linux", contains: []string{"linux"}},
|
||||
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler}},
|
||||
}
|
||||
|
||||
var deviceRules = []uaMatchRule{
|
||||
{
|
||||
label: uaLabelBot,
|
||||
contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp", "curl/", "wget/", "python-requests", "go-http-client", "postman"},
|
||||
},
|
||||
{
|
||||
label: "Tablet",
|
||||
contains: []string{"ipad", "tablet"},
|
||||
},
|
||||
{
|
||||
label: "Tablet",
|
||||
allOf: []string{uaTokenAndroid},
|
||||
noneOf: []string{"mobile"},
|
||||
},
|
||||
{
|
||||
label: "Mobile",
|
||||
contains: []string{"mobi", "iphone", "ipod", uaTokenAndroid},
|
||||
},
|
||||
}
|
||||
|
||||
// ParseBrowserName performs lightweight User-Agent browser identification.
|
||||
func ParseBrowserName(ua string) string {
|
||||
return matchUARules(strings.ToLower(ua), browserRules, uaLabelOther)
|
||||
}
|
||||
|
||||
// ParseOSName performs lightweight User-Agent OS identification.
|
||||
func ParseOSName(ua string) string {
|
||||
return matchUARules(strings.ToLower(ua), osRules, uaLabelOther)
|
||||
}
|
||||
|
||||
// ParseDeviceType performs lightweight User-Agent device type identification.
|
||||
func ParseDeviceType(ua string) string {
|
||||
return matchUARules(strings.ToLower(ua), deviceRules, "Desktop")
|
||||
}
|
||||
@@ -41,14 +41,12 @@ const (
|
||||
ConfigKeyRelayFRPSWebUIPort = "relay_frps_web_ui_port" // FRPS 内置 Web 界面端口
|
||||
|
||||
// OpenFlare 业务配置（从 of_options 迁移）
|
||||
ConfigKeyAgentDiscoveryToken = "agent_discovery_token" //nolint:gosec // false positive: config key name. Agent 发现令牌
|
||||
ConfigKeyAgentHeartbeatInterval = "agent_heartbeat_interval" // Agent 心跳间隔（毫秒）
|
||||
ConfigKeyAgentWebsocketUpgradeEnabled = "agent_websocket_upgrade_enabled" // Agent WebSocket 升级开关
|
||||
ConfigKeyNodeOfflineThreshold = "node_offline_threshold" // 节点离线阈值（毫秒）
|
||||
ConfigKeyAgentUpdateRepo = "agent_update_repo" // Agent 更新仓库
|
||||
ConfigKeyGeoIPProvider = "geoip_provider" // GeoIP 服务商
|
||||
ConfigKeyDatabaseAutoCleanupEnabled = "database_auto_cleanup_enabled" // 数据库自动清理开关
|
||||
ConfigKeyDatabaseAutoCleanupRetentionDays = "database_auto_cleanup_retention_days" // 数据库保留天数
|
||||
ConfigKeyAgentDiscoveryToken = "agent_discovery_token" //nolint:gosec // false positive: config key name. Agent 发现令牌
|
||||
ConfigKeyAgentHeartbeatInterval = "agent_heartbeat_interval" // Agent 心跳间隔（毫秒）
|
||||
ConfigKeyAgentWebsocketUpgradeEnabled = "agent_websocket_upgrade_enabled" // Agent WebSocket 升级开关
|
||||
ConfigKeyNodeOfflineThreshold = "node_offline_threshold" // 节点离线阈值（毫秒）
|
||||
ConfigKeyAgentUpdateRepo = "agent_update_repo" // Agent 更新仓库
|
||||
ConfigKeyGeoIPProvider = "geoip_provider" // GeoIP 服务商
|
||||
|
||||
// Pages 静态托管配置
|
||||
ConfigKeyPagesMaxPackageSizeMB = "pages_max_package_size_mb" // Pages 部署包上传大小上限（MiB）
|
||||
@@ -117,6 +115,15 @@ const (
|
||||
ConfigKeyOriginErrorPageGetOnly = "origin_error_page_get_only" // 是否仅对 GET 请求返回自定义错误页
|
||||
)
|
||||
|
||||
// 日志数据库解耦
|
||||
const (
|
||||
ConfigKeyLogDatabase = "log_database" // 当前日志主库：postgres|sqlite|clickhouse（仅迁移任务写入）
|
||||
ConfigKeyLogDBMigration = "log_db_migration" // 迁移冻结标记："migrating" 或空
|
||||
ConfigKeyLogRetentionDaysPostgres = "log_retention_days_postgres" // PostgreSQL 日志保留天数
|
||||
ConfigKeyLogRetentionDaysSQLite = "log_retention_days_sqlite" // SQLite 日志保留天数
|
||||
ConfigKeyLogRetentionDaysClickHouse = "log_retention_days_clickhouse" // ClickHouse 日志保留天数
|
||||
)
|
||||
|
||||
const (
|
||||
// ConfigVisibilityHidden 表示配置不通过公共配置接口暴露
|
||||
ConfigVisibilityHidden = 0
|
||||
|
||||
Reference in New Issue
Block a user