feat(log): decouple log storage from ClickHouse with switchable logstore

- New internal/repository/logstore abstraction: exported domain interfaces
  (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore),
  config-driven provider (Active/Build/Migrating/SetConfigReader), GORM
  implementation for PostgreSQL/SQLite (incl. hourly rollups computed in
  real time, migration listers, PG partition maintenance), and a ClickHouse
  wrapper preserving the native batch path; repository facade delegates to
  logstore; import-lint test enforces apps never import analyticsrepo.
- ClickHouse is now optional: the log DB is either the main DB (postgres
  when database.enabled, else sqlite) or clickhouse; boot validation +
  first-run seed; log_database / log_db_migration are protected keys.
- New user task 切换日志数据库 (of_log_db_switch): freeze log writes,
  drain batch writers, copy all 6 raw log tables by id (preserving IDs)
  with target-partition pre-creation for PG, flip log_database on success,
  clear the freeze flag on failure.
- Per-store retention (log_retention_days_*) with expiry cleanup folded
  into the daily system_cleanup task; legacy database_auto_cleanup_* and
  of_database_auto_cleanup decommissioned.
- goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite,
  retention config seeds, schedule cleanup; GET
  /api/v1/admin/status/log-database endpoint; frontend retention settings,
  switch-task UI and status badge; changelog and docs updated.

docs(plan): log database decoupling implementation plan

docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
2026-08-08 11:36:56 +08:00
parent 734fe45baa
commit 7d71f1e4e1
95 changed files with 10569 additions and 3728 deletions
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics defines ClickHouse analytics domain models and query DTOs
// (pure data, no IO).
package analytics
import "time"
// AccessLogFilter scopes user access log queries.
// 单一权威字段集(CH 原字段,Task 1 迁入):禁止追加仅某实现使用的字段(避免双字段集分叉)。
type AccessLogFilter struct {
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
UserIDs []uint64
Path string
// StartTime filters created_at >= StartTime when non-nil.
StartTime *time.Time
// EndTime filters created_at <= EndTime when non-nil(闭区间,与 CH/GORM 实现一致)。
EndTime *time.Time
}
// NodeAccessLogFilter scopes ClickHouse node access log queries.
type NodeAccessLogFilter struct {
NodeID string
RemoteAddr string
Host string
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
Hosts []string
Path string
Since time.Time
Until time.Time
Page int
PageSize int
SortBy string
SortOrder string
}
// NodeObservabilityFilter scopes ClickHouse node observability queries.
type NodeObservabilityFilter struct {
NodeID string
Since time.Time
Limit int
}
// DailyTrend is a single day's access count.
type DailyTrend struct {
Date string
Count uint64
}
// BrowserShare is a browser group's share of access logs.
type BrowserShare struct {
Browser string
Count uint64
}
// TopUser is an active user ranked by access count.
type TopUser struct {
UserID uint64
Count uint64
}
// NodeAccessLogRegionCount aggregates access log regions.
type NodeAccessLogRegionCount struct {
Region string
Count int64
}
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
type NodeAccessLogTrafficSummary struct {
RequestCount int64
ErrorCount int64
UniqueIPCount int64
BytesSent int64
RequestLength int64
NodeCount int64
}
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
type NodeAccessLogValueCount struct {
Value string
Count int64
}
// NodeAccessLogNodeAggregate is per-node traffic over a window.
type NodeAccessLogNodeAggregate struct {
NodeID string
RequestCount int64
ErrorCount int64
UniqueIPCount int64
}
// BatchWriterStats is a point-in-time snapshot of a batch writer queue and failure counters.
type BatchWriterStats struct {
Name string `json:"name"`
Depth int `json:"depth"`
Cap int `json:"cap"`
Drops int64 `json:"drops"`
FlushErrors int64 `json:"flush_errors"`
Running bool `json:"running"`
}
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
// and in-process batch writer queue health.
type ClickHouseOperationalStats struct {
Database string `json:"database"`
ActiveParts int64 `json:"active_parts"`
TotalRows int64 `json:"total_rows"`
PendingMutations int64 `json:"pending_mutations"`
AsyncInsertQueue int64 `json:"async_insert_queue"`
AsyncInsertBytes int64 `json:"async_insert_bytes"`
// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
BatchWriters []BatchWriterStats `json:"batch_writers,omitempty"`
}
@@ -90,6 +90,34 @@ type AccessLogHourly struct {
RequestLength int64 `gorm:"column:request_length"`
}
// NodeTrafficHourly is an hourly traffic rollup row.
//
// UniqueVisitorCount is always 0 when sourced from of_access_log_hourly
// (true UV requires raw uniqExact on access logs).
type NodeTrafficHourly struct {
NodeID string
Hour time.Time
RequestCount int64
ErrorCount int64
UniqueVisitorCount int64
}
// NodeMetricHourly is an hourly metric snapshot aggregation row.
//
// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
// lagInFrame samples per node (negative deltas after counter reset are dropped).
type NodeMetricHourly struct {
Hour time.Time
AverageCPUUsagePercent float64
AverageMemoryUsagePercent float64
NetworkRxBytes int64
NetworkTxBytes int64
DiskReadBytes int64
DiskWriteBytes int64
ReportedNodes int
}
// NodeObsFrps stores FRPS observability snapshots in ClickHouse.
type NodeObsFrps struct {
ID uint64 `gorm:"column:id"`
@@ -1,7 +1,6 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics defines ClickHouse analytics domain models.
package analytics
import (
+124
View File
@@ -0,0 +1,124 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import "strings"
// User-Agent 浏览器/OS/设备分类(纯函数,无 IO)。
// 与 internal/repository/analytics/browser.go 的判定逻辑保持一致(Task 4 复制,
// 因为 model 不得 import analyticsrepo);后续若移除旧 CH 实现,可让 analyticsrepo 改以别名复用本包。
const (
uaLabelUnknown = "Unknown"
uaLabelBot = "Bot"
uaLabelOther = "Other"
uaTokenBot = "bot"
uaTokenAndroid = "android"
uaTokenSpider = "spider"
uaTokenCrawler = "crawler"
)
type uaMatchRule struct {
label string
contains []string
allOf []string
noneOf []string
}
func matchUARules(uaLower string, rules []uaMatchRule, fallback string) string {
if uaLower == "" {
return uaLabelUnknown
}
for _, rule := range rules {
matched := false
for _, token := range rule.contains {
if strings.Contains(uaLower, token) {
matched = true
break
}
}
if !matched && len(rule.allOf) > 0 {
matched = true
for _, token := range rule.allOf {
if !strings.Contains(uaLower, token) {
matched = false
break
}
}
}
if !matched {
continue
}
excluded := false
for _, token := range rule.noneOf {
if strings.Contains(uaLower, token) {
excluded = true
break
}
}
if excluded {
continue
}
return rule.label
}
return fallback
}
var browserRules = []uaMatchRule{
{label: "WeChat", contains: []string{"micromessenger"}},
{label: "Postman", contains: []string{"postman"}},
{label: "CLI", contains: []string{"curl/", "wget/"}},
{label: "Edge", contains: []string{"edg/", "edgios/", "edga/"}},
{label: "Opera", contains: []string{"opr/", "opera"}},
{label: "Firefox", contains: []string{"firefox", "fxios"}},
{label: "Chrome", contains: []string{"crios", "chrome"}, noneOf: []string{"chromium"}},
{label: "Chromium", contains: []string{"chromium"}},
{label: "Safari", contains: []string{"safari"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp"}},
}
var osRules = []uaMatchRule{
{label: "Android", contains: []string{uaTokenAndroid}},
{label: "iOS", contains: []string{"iphone", "ipad", "ipod", "ios"}},
{label: "Windows", contains: []string{"windows"}},
{label: "macOS", contains: []string{"mac os x", "macintosh", "macos"}},
{label: "Chrome OS", contains: []string{"cros"}},
{label: "Linux", contains: []string{"linux"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler}},
}
var deviceRules = []uaMatchRule{
{
label: uaLabelBot,
contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp", "curl/", "wget/", "python-requests", "go-http-client", "postman"},
},
{
label: "Tablet",
contains: []string{"ipad", "tablet"},
},
{
label: "Tablet",
allOf: []string{uaTokenAndroid},
noneOf: []string{"mobile"},
},
{
label: "Mobile",
contains: []string{"mobi", "iphone", "ipod", uaTokenAndroid},
},
}
// ParseBrowserName performs lightweight User-Agent browser identification.
func ParseBrowserName(ua string) string {
return matchUARules(strings.ToLower(ua), browserRules, uaLabelOther)
}
// ParseOSName performs lightweight User-Agent OS identification.
func ParseOSName(ua string) string {
return matchUARules(strings.ToLower(ua), osRules, uaLabelOther)
}
// ParseDeviceType performs lightweight User-Agent device type identification.
func ParseDeviceType(ua string) string {
return matchUARules(strings.ToLower(ua), deviceRules, "Desktop")
}