mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-04 07:06:36 +08:00
feat(log): decouple log storage from ClickHouse with switchable logstore
- New internal/repository/logstore abstraction: exported domain interfaces (AccessLogStore/ObservabilityStore/UserAccessLogStore/StatusStore), config-driven provider (Active/Build/Migrating/SetConfigReader), GORM implementation for PostgreSQL/SQLite (incl. hourly rollups computed in real time, migration listers, PG partition maintenance), and a ClickHouse wrapper preserving the native batch path; repository facade delegates to logstore; import-lint test enforces apps never import analyticsrepo. - ClickHouse is now optional: the log DB is either the main DB (postgres when database.enabled, else sqlite) or clickhouse; boot validation + first-run seed; log_database / log_db_migration are protected keys. - New user task 切换日志数据库 (of_log_db_switch): freeze log writes, drain batch writers, copy all 6 raw log tables by id (preserving IDs) with target-partition pre-creation for PG, flip log_database on success, clear the freeze flag on failure. - Per-store retention (log_retention_days_*) with expiry cleanup folded into the daily system_cleanup task; legacy database_auto_cleanup_* and of_database_auto_cleanup decommissioned. - goose migrations: 6 log tables in PG (2 monthly-partitioned) + SQLite, retention config seeds, schedule cleanup; GET /api/v1/admin/status/log-database endpoint; frontend retention settings, switch-task UI and status badge; changelog and docs updated. docs(plan): log database decoupling implementation plan docs(design): log database decoupling design (ClickHouse optional)
This commit is contained in:
ryan
@@ -38,6 +38,18 @@ func CountAccessLogs(ctx context.Context, filter AccessLogFilter) (uint64, error
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// DeleteAllUserAccessLogs hard-deletes all user access logs via TRUNCATE.
|
||||
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
|
||||
if err := userAccessLogConn(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, db.ChConn, analyticsmodel.UserAccessLog{}.TableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// ListAccessLogs returns paginated access logs and the total match count.
|
||||
func ListAccessLogs(ctx context.Context, filter AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
||||
clause, args, ok := buildUserAccessLogFilterClause(filter)
|
||||
|
||||
@@ -6,21 +6,14 @@ package analytics
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
const userAccessLogFilterClauseCapacity = 4
|
||||
|
||||
// AccessLogFilter scopes ClickHouse user access log queries.
|
||||
type AccessLogFilter struct {
|
||||
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
|
||||
UserIDs []uint64
|
||||
Path string
|
||||
// StartTime filters created_at >= StartTime when non-nil.
|
||||
StartTime *time.Time
|
||||
// EndTime filters created_at <= EndTime when non-nil.
|
||||
EndTime *time.Time
|
||||
}
|
||||
type AccessLogFilter = analyticsmodel.AccessLogFilter
|
||||
|
||||
func buildUserAccessLogFilterClause(filter AccessLogFilter) (string, []any, bool) {
|
||||
if filter.UserIDs != nil && len(filter.UserIDs) == 0 {
|
||||
|
||||
@@ -16,22 +16,13 @@ import (
|
||||
const hoursInDay = 24
|
||||
|
||||
// DailyTrend is a single day's access count.
|
||||
type DailyTrend struct {
|
||||
Date string
|
||||
Count uint64
|
||||
}
|
||||
type DailyTrend = analyticsmodel.DailyTrend
|
||||
|
||||
// BrowserShare is a browser group's share of access logs.
|
||||
type BrowserShare struct {
|
||||
Browser string
|
||||
Count uint64
|
||||
}
|
||||
type BrowserShare = analyticsmodel.BrowserShare
|
||||
|
||||
// TopUser is an active user ranked by access count.
|
||||
type TopUser struct {
|
||||
UserID uint64
|
||||
Count uint64
|
||||
}
|
||||
type TopUser = analyticsmodel.TopUser
|
||||
|
||||
// GetDailyTrend returns per-day access counts for the last days days (inclusive of today).
|
||||
func GetDailyTrend(ctx context.Context, days int) ([]DailyTrend, error) {
|
||||
|
||||
@@ -9,21 +9,12 @@ import (
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
|
||||
// and in-process batch writer queue health.
|
||||
type ClickHouseOperationalStats struct {
|
||||
Database string `json:"database"`
|
||||
ActiveParts int64 `json:"active_parts"`
|
||||
TotalRows int64 `json:"total_rows"`
|
||||
PendingMutations int64 `json:"pending_mutations"`
|
||||
AsyncInsertQueue int64 `json:"async_insert_queue"`
|
||||
AsyncInsertBytes int64 `json:"async_insert_bytes"`
|
||||
// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
|
||||
BatchWriters []batchwriter.Stats `json:"batch_writers,omitempty"`
|
||||
}
|
||||
type ClickHouseOperationalStats = analyticsmodel.ClickHouseOperationalStats
|
||||
|
||||
// GetClickHouseOperationalStats returns operational metrics for the configured database.
|
||||
func GetClickHouseOperationalStats(ctx context.Context) (*ClickHouseOperationalStats, error) {
|
||||
|
||||
@@ -15,10 +15,7 @@ import (
|
||||
)
|
||||
|
||||
// NodeAccessLogRegionCount aggregates access log regions.
|
||||
type NodeAccessLogRegionCount struct {
|
||||
Region string
|
||||
Count int64
|
||||
}
|
||||
type NodeAccessLogRegionCount = analyticsmodel.NodeAccessLogRegionCount
|
||||
|
||||
func nodeAccessLogConn() (driver.Conn, error) {
|
||||
if db.ChConn == nil {
|
||||
@@ -151,28 +148,13 @@ ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
|
||||
}
|
||||
|
||||
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
|
||||
type NodeAccessLogTrafficSummary struct {
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
BytesSent int64
|
||||
RequestLength int64
|
||||
NodeCount int64
|
||||
}
|
||||
type NodeAccessLogTrafficSummary = analyticsmodel.NodeAccessLogTrafficSummary
|
||||
|
||||
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
|
||||
type NodeAccessLogValueCount struct {
|
||||
Value string
|
||||
Count int64
|
||||
}
|
||||
type NodeAccessLogValueCount = analyticsmodel.NodeAccessLogValueCount
|
||||
|
||||
// NodeAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type NodeAccessLogNodeAggregate struct {
|
||||
NodeID string
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueIPCount int64
|
||||
}
|
||||
type NodeAccessLogNodeAggregate = analyticsmodel.NodeAccessLogNodeAggregate
|
||||
|
||||
// TrafficSummaryNodeAccessLogs returns request/error/UV/bytes/node counts for the filter.
|
||||
func TrafficSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (NodeAccessLogTrafficSummary, error) {
|
||||
|
||||
@@ -6,7 +6,8 @@ package analytics
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -25,20 +26,7 @@ const (
|
||||
)
|
||||
|
||||
// NodeAccessLogFilter scopes ClickHouse node access log queries.
|
||||
type NodeAccessLogFilter struct {
|
||||
NodeID string
|
||||
RemoteAddr string
|
||||
Host string
|
||||
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
|
||||
Hosts []string
|
||||
Path string
|
||||
Since time.Time
|
||||
Until time.Time
|
||||
Page int
|
||||
PageSize int
|
||||
SortBy string
|
||||
SortOrder string
|
||||
}
|
||||
type NodeAccessLogFilter = analyticsmodel.NodeAccessLogFilter
|
||||
|
||||
func buildNodeAccessLogFilterClause(filter NodeAccessLogFilter) (string, []any) {
|
||||
parts := make([]string, 0, nodeAccessLogFilterClauseCapacity)
|
||||
|
||||
@@ -211,33 +211,11 @@ func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NodeTrafficHourly is an hourly traffic rollup row.
|
||||
//
|
||||
// UniqueVisitorCount is always 0 when sourced from of_access_log_hourly
|
||||
// (true UV requires raw uniqExact on access logs).
|
||||
type NodeTrafficHourly struct {
|
||||
NodeID string
|
||||
Hour time.Time
|
||||
RequestCount int64
|
||||
ErrorCount int64
|
||||
UniqueVisitorCount int64
|
||||
}
|
||||
// NodeTrafficHourly 为小时级流量汇总行（别名，定义见 model/analytics）。
|
||||
type NodeTrafficHourly = analyticsmodel.NodeTrafficHourly
|
||||
|
||||
// NodeMetricHourly is an hourly metric snapshot aggregation row.
|
||||
//
|
||||
// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
|
||||
// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
|
||||
// lagInFrame samples per node (negative deltas after counter reset are dropped).
|
||||
type NodeMetricHourly struct {
|
||||
Hour time.Time
|
||||
AverageCPUUsagePercent float64
|
||||
AverageMemoryUsagePercent float64
|
||||
NetworkRxBytes int64
|
||||
NetworkTxBytes int64
|
||||
DiskReadBytes int64
|
||||
DiskWriteBytes int64
|
||||
ReportedNodes int
|
||||
}
|
||||
// NodeMetricHourly 为小时级指标聚合行（别名，定义见 model/analytics）。
|
||||
type NodeMetricHourly = analyticsmodel.NodeMetricHourly
|
||||
|
||||
// ListNodeTrafficHourly returns hourly traffic from of_access_log_hourly (M5).
|
||||
// UniqueVisitorCount is always 0 here (UV requires raw uniqExact on access logs).
|
||||
|
||||
@@ -5,17 +5,14 @@ package analytics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
const nodeObservabilityFilterClauseCapacity = 3
|
||||
|
||||
// NodeObservabilityFilter scopes ClickHouse node observability queries.
|
||||
type NodeObservabilityFilter struct {
|
||||
NodeID string
|
||||
Since time.Time
|
||||
Limit int
|
||||
}
|
||||
type NodeObservabilityFilter = analyticsmodel.NodeObservabilityFilter
|
||||
|
||||
func buildNodeObservabilityFilterClause(filter NodeObservabilityFilter, sinceColumn string) (string, []any) {
|
||||
parts := make([]string, 0, nodeObservabilityFilterClauseCapacity)
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
)
|
||||
|
||||
// CleanupSummary 汇总本次清理结果。
|
||||
type CleanupSummary struct {
|
||||
ActiveDatabase string `json:"active_database"`
|
||||
RetentionDays int `json:"retention_days"`
|
||||
Deleted int64 `json:"deleted"`
|
||||
// Tables 记录本次清理的物理表简写名（去掉 of_ 前缀，如 node_access_logs 对应
|
||||
// of_node_access_logs；CH 侧物理表名相同，简写仅便于状态展示）。
|
||||
Tables []string `json:"tables"`
|
||||
}
|
||||
|
||||
// defaultLogRetentionDays 默认日志保留天数（配置缺失/非法时回退）。
|
||||
const defaultLogRetentionDays = 90
|
||||
|
||||
// partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。
|
||||
const partitionLeadMonths = 2
|
||||
|
||||
// retentionDaysForActive 按当前激活库读取保留天数（默认 90）。
|
||||
func retentionDaysForActive(ctx context.Context) int {
|
||||
key := model.ConfigKeyLogRetentionDaysPostgres
|
||||
if dbName, _ := resolveDatabase(ctx); dbName == dbNameSQLite {
|
||||
key = model.ConfigKeyLogRetentionDaysSQLite
|
||||
} else if dbName == dbNameClickHouse {
|
||||
key = model.ConfigKeyLogRetentionDaysClickHouse
|
||||
}
|
||||
v, err := getConfig(ctx, key)
|
||||
if err != nil {
|
||||
return defaultLogRetentionDays
|
||||
}
|
||||
days, perr := strconv.Atoi(v)
|
||||
if perr != nil || days <= 0 {
|
||||
return defaultLogRetentionDays
|
||||
}
|
||||
return days
|
||||
}
|
||||
|
||||
// CleanupExpired 按当前激活库保留天数清理过期日志（每日由 system_cleanup 调用）。
|
||||
func CleanupExpired(ctx context.Context) (*CleanupSummary, error) {
|
||||
s, err := Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
days := retentionDaysForActive(ctx)
|
||||
cutoff := time.Now().AddDate(0, 0, -days)
|
||||
summary := &CleanupSummary{RetentionDays: days, Tables: []string{}}
|
||||
summary.ActiveDatabase, _ = resolveDatabase(ctx)
|
||||
|
||||
// PG 分区表仅在迁移时预建「当前+2 月」分区，此处确保分区持续存在，
|
||||
// 否则跨月后新写入会报 "no partition of relation found"（SQLite/CH 为 no-op）。
|
||||
now := time.Now().UTC()
|
||||
if err := s.AccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
|
||||
return nil, fmt.Errorf("ensure partitions: %w", err)
|
||||
}
|
||||
|
||||
if err := cleanupTable("node_access_logs", func() (int64, error) {
|
||||
return s.AccessLogs.DeleteBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cleanupTable("metric_snapshots", func() (int64, error) {
|
||||
return s.Observability.DeleteMetricSnapshotsBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cleanupTable("edge_health", func() (int64, error) {
|
||||
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cleanupTable("obs_frps", func() (int64, error) {
|
||||
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cleanupTable("obs_frpc", func() (int64, error) {
|
||||
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
|
||||
}, summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func cleanupTable(name string, fn func() (int64, error), summary *CleanupSummary) error {
|
||||
n, err := fn()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup %s: %w", name, err)
|
||||
}
|
||||
summary.Deleted += n
|
||||
summary.Tables = append(summary.Tables, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// partitionStatementsRange 生成覆盖 [from, to] 全部月份的两表分区 DDL，
|
||||
// 幂等 CREATE TABLE IF NOT EXISTS ... PARTITION OF ... FOR VALUES FROM ... TO ...。
|
||||
// 入参为任意时间点：按各自所在月份生成，含 from 月与 to 月（to 常用 max+1 月兜底）。
|
||||
func partitionStatementsRange(from, to time.Time) []string {
|
||||
var out []string
|
||||
start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 1, 0)
|
||||
for ; start.Before(end); start = start.AddDate(0, 1, 0) {
|
||||
monthEnd := start.AddDate(0, 1, 0)
|
||||
suffix := start.Format("200601")
|
||||
fromDay := start.Format("2006-01-02")
|
||||
toDay := monthEnd.Format("2006-01-02")
|
||||
for _, table := range []string{"of_node_access_logs", "w_user_access_logs"} {
|
||||
out = append(out, fmt.Sprintf(
|
||||
"CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')",
|
||||
table, suffix, table, fromDay, toDay))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// cleanupTestModels 清理涉及的 5 张日志/可观测表。
|
||||
func cleanupTestModels() []any {
|
||||
return []any{
|
||||
&analyticsmodel.NodeAccessLog{},
|
||||
&analyticsmodel.NodeMetricSnapshot{},
|
||||
&analyticsmodel.NodeEdgeHealth{},
|
||||
&analyticsmodel.NodeObsFrps{},
|
||||
&analyticsmodel.NodeObsFrpc{},
|
||||
}
|
||||
}
|
||||
|
||||
// newCleanupTestDB 构造内存 sqlite 库并注入 db.DB（CleanupExpired 经 Active → buildStore 使用）。
|
||||
func newCleanupTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
dsn := fmt.Sprintf("file:logstore-cleanup-%d?mode=memory&cache=shared", atomic.AddInt64(&testGormStoreSeq, 1))
|
||||
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
if err := gdb.AutoMigrate(cleanupTestModels()...); err != nil {
|
||||
t.Fatalf("automigrate: %v", err)
|
||||
}
|
||||
db.SetDB(gdb)
|
||||
t.Cleanup(func() { db.SetDB(nil) })
|
||||
return gdb
|
||||
}
|
||||
|
||||
// TestCleanupExpiredSQLite 验证 sqlite 激活库的过期日志清理：
|
||||
// 注入 log_retention_days_sqlite=30，40 天前的 5 表记录被删、昨天的保留。
|
||||
func TestCleanupExpiredSQLite(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case logDatabaseKey:
|
||||
return "sqlite", nil
|
||||
case model.ConfigKeyLogRetentionDaysSQLite:
|
||||
return "30", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
|
||||
gdb := newCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
old := time.Now().AddDate(0, 0, -40).UTC()
|
||||
recent := time.Now().AddDate(0, 0, -1).UTC()
|
||||
|
||||
if err := gdb.Create([]analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: old, RemoteAddr: "1.1.1.1"},
|
||||
{ID: 2, NodeID: "n1", LoggedAt: recent, RemoteAddr: "2.2.2.2"},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed node access logs: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeMetricSnapshot{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed metric snapshots: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeEdgeHealth{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed edge health: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeObsFrps{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed obs frps: %v", err)
|
||||
}
|
||||
if err := gdb.Create([]analyticsmodel.NodeObsFrpc{
|
||||
{ID: 1, NodeID: "n1", CapturedAt: old},
|
||||
{ID: 2, NodeID: "n1", CapturedAt: recent},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed obs frpc: %v", err)
|
||||
}
|
||||
|
||||
summary, err := CleanupExpired(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("CleanupExpired: %v", err)
|
||||
}
|
||||
if summary.ActiveDatabase != "sqlite" {
|
||||
t.Fatalf("ActiveDatabase = %q, want sqlite", summary.ActiveDatabase)
|
||||
}
|
||||
if summary.RetentionDays != 30 {
|
||||
t.Fatalf("RetentionDays = %d, want 30", summary.RetentionDays)
|
||||
}
|
||||
if summary.Deleted != 5 {
|
||||
t.Fatalf("Deleted = %d, want 5", summary.Deleted)
|
||||
}
|
||||
if len(summary.Tables) != 5 {
|
||||
t.Fatalf("Tables = %v, want 5 tables", summary.Tables)
|
||||
}
|
||||
|
||||
assertCount := func(m any, want int64, label string) {
|
||||
t.Helper()
|
||||
var n int64
|
||||
if err := gdb.Model(m).Count(&n).Error; err != nil {
|
||||
t.Fatalf("count %s: %v", label, err)
|
||||
}
|
||||
if n != want {
|
||||
t.Fatalf("%s count = %d, want %d", label, n, want)
|
||||
}
|
||||
}
|
||||
assertCount(&analyticsmodel.NodeAccessLog{}, 1, "node_access_logs")
|
||||
assertCount(&analyticsmodel.NodeMetricSnapshot{}, 1, "metric_snapshots")
|
||||
assertCount(&analyticsmodel.NodeEdgeHealth{}, 1, "edge_health")
|
||||
assertCount(&analyticsmodel.NodeObsFrps{}, 1, "obs_frps")
|
||||
assertCount(&analyticsmodel.NodeObsFrpc{}, 1, "obs_frpc")
|
||||
|
||||
var kept analyticsmodel.NodeAccessLog
|
||||
if err := gdb.First(&kept).Error; err != nil {
|
||||
t.Fatalf("recent node access log missing: %v", err)
|
||||
}
|
||||
if kept.ID != 2 {
|
||||
t.Fatalf("kept log ID = %d, want 2 (recent)", kept.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetentionDaysForActive 覆盖保留天数读取：按激活库选 key、非法值回退默认 90。
|
||||
func TestRetentionDaysForActive(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case logDatabaseKey:
|
||||
return "sqlite", nil
|
||||
case model.ConfigKeyLogRetentionDaysSQLite:
|
||||
return "30", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if got := retentionDaysForActive(context.Background()); got != 30 {
|
||||
t.Fatalf("retentionDaysForActive = %d, want 30", got)
|
||||
}
|
||||
|
||||
// 非法值（非数字/<=0）回退默认 90。
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
switch key {
|
||||
case logDatabaseKey:
|
||||
return "postgres", nil
|
||||
case model.ConfigKeyLogRetentionDaysPostgres:
|
||||
return "abc", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if got := retentionDaysForActive(context.Background()); got != 90 {
|
||||
t.Fatalf("retentionDaysForActive invalid value = %d, want 90", got)
|
||||
}
|
||||
|
||||
// reader 报错回退默认 90。
|
||||
SetConfigReader(func(_ context.Context, _ string) (string, error) {
|
||||
return "", fmt.Errorf("boom")
|
||||
})
|
||||
if got := retentionDaysForActive(context.Background()); got != 90 {
|
||||
t.Fatalf("retentionDaysForActive reader error = %d, want 90", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPartitionStatements 验证 PG 分区 DDL 生成：当前月 + 未来 2 个月 × 2 表，
|
||||
// 幂等 PARTITION OF 语句与迁移 SQL 命名一致（含跨年）。
|
||||
func TestPartitionStatements(t *testing.T) {
|
||||
now := time.Date(2026, 8, 15, 10, 0, 0, 0, time.UTC)
|
||||
stmts := partitionStatementsRange(now, now.AddDate(0, 2, 0))
|
||||
if len(stmts) != 6 {
|
||||
t.Fatalf("partitionStatements len = %d, want 6", len(stmts))
|
||||
}
|
||||
want := []string{
|
||||
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202608 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
|
||||
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202608 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
|
||||
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202609 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
|
||||
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202609 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
|
||||
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202610 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
|
||||
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202610 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
|
||||
}
|
||||
for i, w := range want {
|
||||
if stmts[i] != w {
|
||||
t.Fatalf("stmt[%d] = %q, want %q", i, stmts[i], w)
|
||||
}
|
||||
}
|
||||
|
||||
// 跨年：2026-11 → 202611, 202612, 202701。
|
||||
nov := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
|
||||
suffixes := []string{"202611", "202612", "202701"}
|
||||
for _, stmt := range partitionStatementsRange(nov, nov.AddDate(0, 2, 0)) {
|
||||
if !hasAnySuffix(stmt, suffixes) {
|
||||
t.Fatalf("statement lacks expected month suffix: %s", stmt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func hasAnySuffix(stmt string, suffixes []string) bool {
|
||||
for _, table := range []string{"of_node_access_logs", "w_user_access_logs"} {
|
||||
for _, suf := range suffixes {
|
||||
if strings.Contains(stmt, table+"_"+suf) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,731 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
)
|
||||
|
||||
// clickhouseLogStore 实现 AccessLogStore / ObservabilityStore / StatusStore，
|
||||
// 逐方法委托 analyticsrepo（CH 原生 batch 写入，零性能损耗）。
|
||||
// UserAccessLogStore 由 clickhouseUserAccessLogStore 实现（List/Count 方法名已被
|
||||
// AccessLogStore 占用，Go 不允许同名不同签名方法）。
|
||||
type clickhouseLogStore struct {
|
||||
// skipFreeze 为 true 时跳过迁移冻结检查（仅迁移目标 store 使用）。
|
||||
skipFreeze bool
|
||||
}
|
||||
|
||||
func newClickHouseStore() *clickhouseLogStore { return &clickhouseLogStore{} }
|
||||
|
||||
// 编译期断言。
|
||||
var (
|
||||
_ AccessLogStore = (*clickhouseLogStore)(nil)
|
||||
_ ObservabilityStore = (*clickhouseLogStore)(nil)
|
||||
_ StatusStore = (*clickhouseLogStore)(nil)
|
||||
_ UserAccessLogStore = (*clickhouseUserAccessLogStore)(nil)
|
||||
)
|
||||
|
||||
func chConnErr() error {
|
||||
if !db.ChConnReady() {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureWritable 迁移冻结期拒绝写入。
|
||||
func (s *clickhouseLogStore) ensureWritable(ctx context.Context) error {
|
||||
if !s.skipFreeze && Migrating(ctx) {
|
||||
return ErrMigrating
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- AccessLogStore ----
|
||||
|
||||
// InsertBatch 节点访问日志写入入口：冻结检查后经 hook 入队（异步），不直接落库。
|
||||
func (s *clickhouseLogStore) InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
rows := make([]analyticsmodel.NodeAccessLog, 0, len(records))
|
||||
for _, r := range records {
|
||||
if r == nil {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, toAnalyticsNodeAccessLog(r))
|
||||
}
|
||||
if h := currentAccessLogHooks().QueueNodeAccessLogs; h != nil {
|
||||
h(rows)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchInsertNodeAccessLogs 是 batchwriter flush 目标：CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeAccessLogs(ctx, rows)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
|
||||
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeAccessLogs(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
|
||||
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
|
||||
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]*model.OpenFlareAccessLogRegionCount, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = &model.OpenFlareAccessLogRegionCount{Region: r.Region, Count: r.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) BucketAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error) {
|
||||
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) CountBuckets(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
|
||||
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) BucketDimensions(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error) {
|
||||
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) IPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error) {
|
||||
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), exactRemoteAddr)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) IPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error) {
|
||||
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), recentSince)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) CountIPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, error) {
|
||||
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) WAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error) {
|
||||
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) IPTrend(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error) {
|
||||
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) TrafficSummary(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
|
||||
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return model.OpenFlareAccessLogTrafficSummary{}, err
|
||||
}
|
||||
return model.OpenFlareAccessLogTrafficSummary{
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueIPCount: row.UniqueIPCount,
|
||||
BytesSent: row.BytesSent,
|
||||
RequestLength: row.RequestLength,
|
||||
NodeCount: row.NodeCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ValueCounts(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
|
||||
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]model.OpenFlareAccessLogValueCount, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = model.OpenFlareAccessLogValueCount{Value: r.Value, Count: r.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) NodeAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
|
||||
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]model.OpenFlareAccessLogNodeAggregate, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = model.OpenFlareAccessLogNodeAggregate{NodeID: r.NodeID, RequestCount: r.RequestCount, ErrorCount: r.ErrorCount, UniqueIPCount: r.UniqueIPCount}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
|
||||
}
|
||||
|
||||
// ListForMigration 按 id 升序分页读取（迁移复制用）：直接查询 CH 原生表。
|
||||
func (s *clickhouseLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error) {
|
||||
if err := chConnErr(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := db.ChConn.Query(ctx, `
|
||||
SELECT `+analyticsmodel.NodeAccessLog{}.InsertColumns()+`
|
||||
FROM `+analyticsmodel.NodeAccessLog{}.TableName()+`
|
||||
WHERE id > ?
|
||||
ORDER BY id ASC
|
||||
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node access logs for migration: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []analyticsmodel.NodeAccessLog
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeAccessLog
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.LoggedAt,
|
||||
&item.RemoteAddr,
|
||||
&item.Region,
|
||||
&item.Host,
|
||||
&item.Path,
|
||||
&item.UserAgent,
|
||||
&item.CacheStatus,
|
||||
&item.StatusCode,
|
||||
&item.BytesSent,
|
||||
&item.RequestLength,
|
||||
&item.RequestTimeMs,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node access log row: %w", err)
|
||||
}
|
||||
item.LoggedAt = item.LoggedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ---- ObservabilityStore ----
|
||||
|
||||
// InsertMetricSnapshot 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||||
func (s *clickhouseLogStore) InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueMetricSnapshot; h != nil {
|
||||
h(toAnalyticsNodeMetricSnapshot(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
rows, err := analyticsrepo.ListNodeMetricSnapshots(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeMetricSnapshots(ctx)
|
||||
}
|
||||
|
||||
// ListTrafficHourly 委托 analyticsrepo 读 of_access_log_hourly rollup（M5 口径，UV 恒 0）。
|
||||
func (s *clickhouseLogStore) ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error) {
|
||||
return analyticsrepo.ListNodeTrafficHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||||
}
|
||||
|
||||
// ListAccessLogHourly 委托 analyticsrepo 读 of_access_log_hourly rollup。
|
||||
func (s *clickhouseLogStore) ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error) {
|
||||
return analyticsrepo.ListAccessLogHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||||
}
|
||||
|
||||
// ListMetricHourly 委托 analyticsrepo ListNodeMetricHourly：rollup 覆盖窗口时读
|
||||
// of_node_metric_capacity_hourly，否则按 mergeNodeMetricHourlyPreferRollup 合并 raw 兜底。
|
||||
func (s *clickhouseLogStore) ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error) {
|
||||
return analyticsrepo.ListNodeMetricHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// BatchInsertNodeMetricSnapshots 是 batchwriter flush 目标：CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeMetricSnapshots(ctx, rows)
|
||||
}
|
||||
|
||||
// InsertEdgeHealth 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||||
func (s *clickhouseLogStore) InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueEdgeHealth; h != nil {
|
||||
h(toAnalyticsNodeEdgeHealth(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
|
||||
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeEdgeHealths(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// BatchInsertNodeEdgeHealth 是 batchwriter flush 目标：CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeEdgeHealth(ctx, rows)
|
||||
}
|
||||
|
||||
// InsertNodeObservationFrps 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||||
func (s *clickhouseLogStore) InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueNodeObsFrps; h != nil {
|
||||
h(toAnalyticsNodeObsFrps(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
|
||||
rows, err := analyticsrepo.ListNodeObsFrps(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeObsFrps(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllNodeObservationFrps(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeObsFrps(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeObsFrpsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrps 是 batchwriter flush 目标：CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeObsFrps(ctx, rows)
|
||||
}
|
||||
|
||||
// InsertNodeObservationFrpc 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||||
func (s *clickhouseLogStore) InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := currentObservabilityHooks().QueueNodeObsFrpc; h != nil {
|
||||
h(toAnalyticsNodeObsFrpc(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
|
||||
rows, err := analyticsrepo.ListNodeObsFrpc(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeObsFrpc(rows), nil
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllNodeObsFrpc(ctx)
|
||||
}
|
||||
|
||||
func (s *clickhouseLogStore) DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteNodeObsFrpcBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// MigrationRange 返回 of_node_access_logs.logged_at 的最小/最大值（空表返回零值）。
|
||||
func (s *clickhouseLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
|
||||
return chMigrationRange(ctx, analyticsmodel.NodeAccessLog{}.TableName(), "logged_at")
|
||||
}
|
||||
|
||||
// EnsurePartitions 是 CH 分支 no-op（CH 无 PG 式分区）。
|
||||
func (s *clickhouseLogStore) EnsurePartitions(_ context.Context, _, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// chMigrationRange 查询 CH 表时间列 MIN/MAX；空表（NULL）返回零值。
|
||||
func chMigrationRange(ctx context.Context, table, column string) (time.Time, time.Time, error) {
|
||||
if err := chConnErr(); err != nil {
|
||||
return time.Time{}, time.Time{}, err
|
||||
}
|
||||
var minTime, maxTime *time.Time
|
||||
if err := db.ChConn.QueryRow(ctx,
|
||||
"SELECT min("+column+"), max("+column+") FROM "+table,
|
||||
).Scan(&minTime, &maxTime); err != nil {
|
||||
return time.Time{}, time.Time{}, fmt.Errorf("query migration range %s: %w", table, err)
|
||||
}
|
||||
if minTime == nil || maxTime == nil {
|
||||
return time.Time{}, time.Time{}, nil
|
||||
}
|
||||
return minTime.UTC(), maxTime.UTC(), nil
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrpc 是 batchwriter flush 目标：CH 原生批量写入。
|
||||
func (s *clickhouseLogStore) BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsertNodeObsFrpc(ctx, rows)
|
||||
}
|
||||
|
||||
// ListMetricSnapshotsForMigration 按 id 升序分页读取（迁移复制用）。
|
||||
func (s *clickhouseLogStore) ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
return chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeMetricSnapshot{}.TableName(),
|
||||
analyticsmodel.NodeMetricSnapshot{}.InsertColumns(),
|
||||
func(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
var result []analyticsmodel.NodeMetricSnapshot
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeMetricSnapshot
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.CPUUsagePercent,
|
||||
&item.MemoryUsedBytes,
|
||||
&item.MemoryTotalBytes,
|
||||
&item.StorageUsedBytes,
|
||||
&item.StorageTotalBytes,
|
||||
&item.DiskReadBytes,
|
||||
&item.DiskWriteBytes,
|
||||
&item.NetworkRxBytes,
|
||||
&item.NetworkTxBytes,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
})
|
||||
}
|
||||
|
||||
// chObsRow 迁移读取共用的双字段观测行（字符串状态 + 数值计数）：
|
||||
// edge_health（status/connections）与 obs_frpc（tunnel_status/connected_relays_count）同形状。
|
||||
type chObsRow struct {
|
||||
ID uint64
|
||||
NodeID string
|
||||
CapturedAt time.Time
|
||||
Status string
|
||||
Count int64
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// countToInt32 将观测计数转为 int32（防御溢出；观测计数远小于 int32 上限）。
|
||||
func countToInt32(v int64) int32 {
|
||||
if v > math.MaxInt32 {
|
||||
return math.MaxInt32
|
||||
}
|
||||
if v < math.MinInt32 {
|
||||
return math.MinInt32
|
||||
}
|
||||
return int32(v)
|
||||
}
|
||||
|
||||
// scanChObsRow 扫描 chObsRow（含 UTC 归一化）。
|
||||
func scanChObsRow(rows driver.Rows) ([]chObsRow, error) {
|
||||
var result []chObsRow
|
||||
for rows.Next() {
|
||||
var item chObsRow
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.Status,
|
||||
&item.Count,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListEdgeHealthForMigration 按 id 升序分页读取（迁移复制用）。
|
||||
func (s *clickhouseLogStore) ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error) {
|
||||
rows, err := chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeEdgeHealth{}.TableName(),
|
||||
analyticsmodel.NodeEdgeHealth{}.InsertColumns(),
|
||||
scanChObsRow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.NodeEdgeHealth, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = analyticsmodel.NodeEdgeHealth{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, Status: r.Status, Connections: r.Count, CreatedAt: r.CreatedAt}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListNodeObsFrpsForMigration 按 id 升序分页读取（迁移复制用）。
|
||||
func (s *clickhouseLogStore) ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
return chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeObsFrps{}.TableName(),
|
||||
analyticsmodel.NodeObsFrps{}.InsertColumns(),
|
||||
func(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
var result []analyticsmodel.NodeObsFrps
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrps
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.FrpsConnections,
|
||||
&item.FrpsProxyCount,
|
||||
&item.FrpsClientCount,
|
||||
&item.FrpsProxies,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frps observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListNodeObsFrpcForMigration 按 id 升序分页读取（迁移复制用）。
|
||||
func (s *clickhouseLogStore) ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
rows, err := chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.NodeObsFrpc{}.TableName(),
|
||||
analyticsmodel.NodeObsFrpc{}.InsertColumns(),
|
||||
scanChObsRow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.NodeObsFrpc, len(rows))
|
||||
for i, r := range rows {
|
||||
out[i] = analyticsmodel.NodeObsFrpc{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, TunnelStatus: r.Status, ConnectedRelaysCount: countToInt32(r.Count), CreatedAt: r.CreatedAt}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// chListForMigration 执行按 id 升序分页的 CH 原生表查询，并交给 scanner 扫描。
|
||||
func chListForMigration[T any](ctx context.Context, afterID uint64, limit int, table, columns string, scanner func(driver.Rows) ([]T, error)) ([]T, error) {
|
||||
if err := chConnErr(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := db.ChConn.Query(ctx, `
|
||||
SELECT `+columns+`
|
||||
FROM `+table+`
|
||||
WHERE id > ?
|
||||
ORDER BY id ASC
|
||||
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list %s for migration: %w", table, err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanner(rows)
|
||||
}
|
||||
|
||||
// ---- StatusStore ----
|
||||
|
||||
// ActiveDatabase 返回当前日志主库名（CH 分支固定 clickhouse）。
|
||||
func (s *clickhouseLogStore) ActiveDatabase(_ context.Context) (string, error) {
|
||||
return dbNameClickHouse, nil
|
||||
}
|
||||
|
||||
// ClickHouseOperationalStats 委托 analyticsrepo 汇总 CH 运行状态。
|
||||
func (s *clickhouseLogStore) ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) {
|
||||
return analyticsrepo.GetClickHouseOperationalStats(ctx)
|
||||
}
|
||||
|
||||
// ---- UserAccessLogStore ----
|
||||
|
||||
// clickhouseUserAccessLogStore 实现 UserAccessLogStore。clickhouseLogStore 已占用
|
||||
// List/Count 方法名（AccessLogStore 接口），Go 不允许同名不同签名方法，故用户访问日志
|
||||
// 用独立类型嵌入同一 clickhouseLogStore（与 userAccessLogGormStore 同构），复用 ensureWritable。
|
||||
type clickhouseUserAccessLogStore struct {
|
||||
*clickhouseLogStore
|
||||
}
|
||||
|
||||
func newClickHouseUserAccessLogStore() *clickhouseUserAccessLogStore {
|
||||
return &clickhouseUserAccessLogStore{clickhouseLogStore: newClickHouseStore()}
|
||||
}
|
||||
|
||||
// BatchInsert 是 batchwriter flush 目标：CH 原生批量写入；冻结期拒绝写入，空批次直接返回。
|
||||
func (s *clickhouseUserAccessLogStore) BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
|
||||
if len(logs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return analyticsrepo.BatchInsert(ctx, logs)
|
||||
}
|
||||
|
||||
// DeleteAll 清空全部用户访问日志（TRUNCATE 语义，迁移「覆盖目标库已有日志」幂等前提用）。
|
||||
func (s *clickhouseUserAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||||
if err := s.ensureWritable(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return analyticsrepo.DeleteAllUserAccessLogs(ctx)
|
||||
}
|
||||
|
||||
// ListForMigration 按 id 升序分页读取（迁移复制用）。
|
||||
func (s *clickhouseUserAccessLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error) {
|
||||
return chListForMigration(ctx, afterID, limit,
|
||||
analyticsmodel.UserAccessLog{}.TableName(),
|
||||
analyticsmodel.UserAccessLog{}.InsertColumns(),
|
||||
func(rows driver.Rows) ([]analyticsmodel.UserAccessLog, error) {
|
||||
var result []analyticsmodel.UserAccessLog
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.UserAccessLog
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.UserID,
|
||||
&item.Path,
|
||||
&item.Method,
|
||||
&item.IP,
|
||||
&item.UserAgent,
|
||||
&item.Headers,
|
||||
&item.Status,
|
||||
&item.Latency,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan user access log row: %w", err)
|
||||
}
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
})
|
||||
}
|
||||
|
||||
// MigrationRange 返回 w_user_access_logs.created_at 的最小/最大值（空表返回零值）。
|
||||
func (s *clickhouseUserAccessLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
|
||||
return chMigrationRange(ctx, analyticsmodel.UserAccessLog{}.TableName(), "created_at")
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
|
||||
return analyticsrepo.CountAccessLogs(ctx, filter)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
||||
return analyticsrepo.ListAccessLogs(ctx, filter, page, pageSize)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
|
||||
return analyticsrepo.GetDailyTrend(ctx, days)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
|
||||
return analyticsrepo.GetBrowserDistribution(ctx, startTime)
|
||||
}
|
||||
|
||||
func (s *clickhouseUserAccessLogStore) GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
|
||||
return analyticsrepo.GetTopActiveUsers(ctx, startTime, limit)
|
||||
}
|
||||
|
||||
// toNodeObservabilityFilter 构造 CH 可观测查询过滤器（limit<=0 表示不限制）。
|
||||
func toNodeObservabilityFilter(nodeID string, since time.Time, limit int) analyticsmodel.NodeObservabilityFilter {
|
||||
return analyticsmodel.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
Limit: limit,
|
||||
}
|
||||
}
|
||||
|
||||
// toNodeObservabilitySince 构造不带 limit 的可观测查询过滤器
|
||||
// （小时级聚合读无需分页，避免传无意义的 0）。
|
||||
func toNodeObservabilitySince(nodeID string, since time.Time) analyticsmodel.NodeObservabilityFilter {
|
||||
return analyticsmodel.NodeObservabilityFilter{NodeID: nodeID, Since: since}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
)
|
||||
|
||||
// TestClickHouseHourlyDelegationRegression 验证 CH 后端小时级聚合读委托 analyticsrepo：
|
||||
// 未初始化 CH 连接时返回 analyticsrepo 的 "clickhouse connection is not initialized" 错误
|
||||
// （而非未实现/panic），证明 3 个方法都路由到 CH 原生查询。
|
||||
func TestClickHouseHourlyDelegationRegression(t *testing.T) {
|
||||
if db.ChConnReady() {
|
||||
t.Skip("clickhouse connection initialized; skipping delegation regression")
|
||||
}
|
||||
s := newClickHouseStore()
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
check := func(name string, err error) {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
t.Fatalf("%s: want clickhouse-not-initialized error, got nil", name)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "clickhouse connection is not initialized") {
|
||||
t.Fatalf("%s: unexpected error %v", name, err)
|
||||
}
|
||||
}
|
||||
_, err := s.ListTrafficHourly(ctx, "n1", now)
|
||||
check("ListTrafficHourly", err)
|
||||
_, err = s.ListAccessLogHourly(ctx, "n1", now)
|
||||
check("ListAccessLogHourly", err)
|
||||
_, err = s.ListMetricHourly(ctx, "n1", now)
|
||||
check("ListMetricHourly", err)
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// timeBucketSQLPostgres 返回 PG 时间分桶表达式（epoch 秒 -> 分桶起点，int64）。
|
||||
func timeBucketSQLPostgres(column string, bucketSeconds int64) string {
|
||||
return "(floor(extract(epoch from " + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")::bigint"
|
||||
}
|
||||
|
||||
// dailyTrendDateSQLPostgres 返回 PG 按日聚合的日期表达式。
|
||||
func dailyTrendDateSQLPostgres() string {
|
||||
return "to_char(created_at, 'YYYY-MM-DD')"
|
||||
}
|
||||
|
||||
// epochSQLPostgres 返回 PG epoch 秒表达式（int64）。
|
||||
func epochSQLPostgres(column string) string {
|
||||
return "extract(epoch from " + column + ")::bigint"
|
||||
}
|
||||
|
||||
// textCastSQLPostgres 返回 PG 数值列转文本表达式。
|
||||
func textCastSQLPostgres(column string) string {
|
||||
return column + "::text"
|
||||
}
|
||||
|
||||
// distinctNonEmptyCountSQLPostgres 返回 PG 排除空串的 distinct 计数表达式。
|
||||
func distinctNonEmptyCountSQLPostgres(column string) string {
|
||||
return "COUNT(DISTINCT " + column + ") FILTER (WHERE " + column + " <> '')"
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// isPostgresDialect 判断 gorm 句柄是否为 PostgreSQL 方言（否则按 SQLite 处理）。
|
||||
// Dialector 经 gorm.Config 内嵌提升，Name() 可直接在 DB 上调用。
|
||||
func isPostgresDialect(db *gorm.DB) bool {
|
||||
return db != nil && db.Dialector != nil && db.Name() == "postgres"
|
||||
}
|
||||
|
||||
// timeBucketSQLSQLite 返回 SQLite 时间分桶表达式（epoch 秒 -> 分桶起点）。
|
||||
func timeBucketSQLSQLite(column string, bucketSeconds int64) string {
|
||||
return "(floor(unixepoch(" + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")"
|
||||
}
|
||||
|
||||
// dailyTrendDateSQLSQLite 返回 SQLite 按日聚合的日期表达式。
|
||||
func dailyTrendDateSQLSQLite() string {
|
||||
return "strftime('%Y-%m-%d', created_at)"
|
||||
}
|
||||
|
||||
// epochSQLSQLite 返回 SQLite epoch 秒表达式（unixepoch 整数秒）。
|
||||
func epochSQLSQLite(column string) string {
|
||||
return "unixepoch(" + column + ")"
|
||||
}
|
||||
|
||||
// textCastSQLSQLite 返回 SQLite 数值列转文本表达式。
|
||||
func textCastSQLSQLite(column string) string {
|
||||
return "CAST(" + column + " AS TEXT)"
|
||||
}
|
||||
|
||||
// distinctNonEmptyCountSQLSQLite 返回 SQLite 排除空串的 distinct 计数表达式
|
||||
// （SQLite 无 FILTER 语法，用 CASE 等价实现）。
|
||||
func distinctNonEmptyCountSQLSQLite(column string) string {
|
||||
return "COUNT(DISTINCT CASE WHEN " + column + " <> '' THEN " + column + " END)"
|
||||
}
|
||||
|
||||
// distinctNonEmptyCountSQL 按当前方言返回排除空串的 distinct 计数表达式
|
||||
// （运行时按 Dialector 分发，默认 SQLite）。
|
||||
func distinctNonEmptyCountSQL(db *gorm.DB, column string) string {
|
||||
if isPostgresDialect(db) {
|
||||
return distinctNonEmptyCountSQLPostgres(column)
|
||||
}
|
||||
return distinctNonEmptyCountSQLSQLite(column)
|
||||
}
|
||||
|
||||
// dailyTrendDateSQL 按当前方言返回按日聚合的日期表达式（运行时按 Dialector 分发，默认 SQLite）。
|
||||
func dailyTrendDateSQL(db *gorm.DB) string {
|
||||
if isPostgresDialect(db) {
|
||||
return dailyTrendDateSQLPostgres()
|
||||
}
|
||||
return dailyTrendDateSQLSQLite()
|
||||
}
|
||||
|
||||
// epochSQL 按当前方言返回 epoch 秒表达式（运行时按 Dialector 分发，默认 SQLite）。
|
||||
func epochSQL(db *gorm.DB, column string) string {
|
||||
if isPostgresDialect(db) {
|
||||
return epochSQLPostgres(column)
|
||||
}
|
||||
return epochSQLSQLite(column)
|
||||
}
|
||||
|
||||
// textCastSQL 按当前方言返回数值列转文本表达式（运行时按 Dialector 分发，默认 SQLite）。
|
||||
func textCastSQL(db *gorm.DB, column string) string {
|
||||
if isPostgresDialect(db) {
|
||||
return textCastSQLPostgres(column)
|
||||
}
|
||||
return textCastSQLSQLite(column)
|
||||
}
|
||||
|
||||
// timeBucketSQL 按当前方言返回时间分桶表达式。
|
||||
// brief 将 PG/SQLite 两版写为同名函数，同包无法共存；log_database 为运行时配置，
|
||||
// 不能使用编译期 build tag，故按 db.Dialector.Name() 运行时分发（默认 SQLite）。
|
||||
func timeBucketSQL(db *gorm.DB, column string, bucketSeconds int64) string {
|
||||
if isPostgresDialect(db) {
|
||||
return timeBucketSQLPostgres(column, bucketSeconds)
|
||||
}
|
||||
return timeBucketSQLSQLite(column, bucketSeconds)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// AccessLogHooks 节点访问日志异步入队回调（由 chwriter 装配）。
|
||||
type AccessLogHooks struct {
|
||||
QueueNodeAccessLogs func(logs []analyticsmodel.NodeAccessLog)
|
||||
}
|
||||
|
||||
// ObservabilityHooks 可观测异步入队回调（由 chwriter 装配）。
|
||||
type ObservabilityHooks struct {
|
||||
QueueMetricSnapshot func(record analyticsmodel.NodeMetricSnapshot)
|
||||
QueueEdgeHealth func(record analyticsmodel.NodeEdgeHealth)
|
||||
QueueNodeObsFrps func(record analyticsmodel.NodeObsFrps)
|
||||
QueueNodeObsFrpc func(record analyticsmodel.NodeObsFrpc)
|
||||
}
|
||||
|
||||
var (
|
||||
hooksMu sync.RWMutex
|
||||
accessLogHooks AccessLogHooks
|
||||
observabilityHooks ObservabilityHooks
|
||||
)
|
||||
|
||||
// SetAccessLogHooks 注册节点访问日志异步入队回调。
|
||||
func SetAccessLogHooks(h AccessLogHooks) {
|
||||
hooksMu.Lock()
|
||||
accessLogHooks = h
|
||||
hooksMu.Unlock()
|
||||
}
|
||||
|
||||
// SetObservabilityHooks 注册可观测异步入队回调。
|
||||
func SetObservabilityHooks(h ObservabilityHooks) {
|
||||
hooksMu.Lock()
|
||||
observabilityHooks = h
|
||||
hooksMu.Unlock()
|
||||
}
|
||||
|
||||
// currentAccessLogHooks 返回当前 hooks 快照（未注册时为 zero value，调用方判空跳过）。
|
||||
func currentAccessLogHooks() AccessLogHooks {
|
||||
hooksMu.RLock()
|
||||
defer hooksMu.RUnlock()
|
||||
return accessLogHooks
|
||||
}
|
||||
|
||||
// currentObservabilityHooks 返回当前 hooks 快照（未注册时为 zero value，调用方判空跳过）。
|
||||
func currentObservabilityHooks() ObservabilityHooks {
|
||||
hooksMu.RLock()
|
||||
defer hooksMu.RUnlock()
|
||||
return observabilityHooks
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// forbiddenImports 上层应用禁止直接触碰的底层日志实现。
|
||||
var forbiddenImports = []string{
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/analytics",
|
||||
}
|
||||
|
||||
// allowedAnalyticsDelegation 允许直接依赖 analyticsrepo 的委托层：
|
||||
// - internal/repository：持久化门面，ListOpenFlareLatestMetricSnapshotsSince 的
|
||||
// CH 快速路径仍直连 analyticsrepo（LIMIT 1 BY node_id）；小时级聚合读已改走 logstore；
|
||||
// - internal/repository/logstore：CH 后端实现按设计委托 analyticsrepo。
|
||||
//
|
||||
// 除此之外，依赖闭包内任何包都禁止引入 analyticsrepo。
|
||||
var allowedAnalyticsDelegation = map[string]bool{
|
||||
"github.com/Rain-kl/Wavelet/internal/repository": true,
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore": true,
|
||||
}
|
||||
|
||||
// allowedInfraPersistence 允许 apps 引入的 infra/persistence 子包。
|
||||
var allowedInfraPersistence = []string{
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/batchwriter", // batchwriter 统计类型
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen", // 雪花 ID 生成（无日志依赖）
|
||||
}
|
||||
|
||||
func TestAppsMustNotImportLogBackendDirectly(t *testing.T) {
|
||||
t.Chdir("../../..") // module root，保证 ./internal/apps/... 可解析
|
||||
// -test 同时列出测试二进制依赖，覆盖仅测试文件引入的底层日志实现。
|
||||
out, err := exec.Command("go", "list", "-test", "-deps", "-f", `{{.ImportPath}} {{join .Imports " "}}`, "./internal/apps/...").Output()
|
||||
if err != nil {
|
||||
t.Fatalf("go list: %v", err)
|
||||
}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
pkg := fields[0]
|
||||
for _, forbidden := range forbiddenImports {
|
||||
for _, imp := range fields[1:] {
|
||||
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
|
||||
t.Errorf("internal/apps must not import %s (via %s)", forbidden, pkg)
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(pkg, "github.com/Rain-kl/Wavelet/internal/infra/persistence/") {
|
||||
allowed := false
|
||||
for _, a := range allowedInfraPersistence {
|
||||
if pkg == a || strings.HasPrefix(pkg, a+"/") {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
t.Errorf("internal/apps must not import infra/persistence subpackage: %s", pkg)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// TestGormAccessLogInsertBatchHooks 覆盖访问日志写入入口：
|
||||
// 冻结检查、hook 入队、不直接落库、flush 后可见（行为与旧 repository clickhouse 包装一致）。
|
||||
func TestGormAccessLogInsertBatchHooks(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
|
||||
s := newTestGormStore(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
var hooked []analyticsmodel.NodeAccessLog
|
||||
SetAccessLogHooks(AccessLogHooks{
|
||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||
hooked = append(hooked, logs...)
|
||||
},
|
||||
})
|
||||
defer SetAccessLogHooks(AccessLogHooks{})
|
||||
|
||||
records := []*model.OpenFlareAccessLog{
|
||||
{NodeID: "n1", LoggedAt: now, RemoteAddr: "1.1.1.1", StatusCode: 200, BytesSent: 100},
|
||||
{NodeID: "n1", LoggedAt: now, RemoteAddr: "2.2.2.2", StatusCode: 404},
|
||||
}
|
||||
if err := s.InsertBatch(ctx, records); err != nil {
|
||||
t.Fatalf("insert batch: %v", err)
|
||||
}
|
||||
if len(hooked) != 2 || hooked[0].RemoteAddr != "1.1.1.1" || hooked[0].BytesSent != 100 || hooked[1].StatusCode != 404 {
|
||||
t.Fatalf("hook rows mismatch: %+v", hooked)
|
||||
}
|
||||
// 写入入口只入队、不直接落库。
|
||||
rows, err := s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("entry insert must not write rows, got %d", len(rows))
|
||||
}
|
||||
// flush 后可见。
|
||||
if err := s.BatchInsertNodeAccessLogs(ctx, hooked); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
rows, err = s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
|
||||
if err != nil {
|
||||
t.Fatalf("list after flush: %v", err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("list after flush want 2, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package logstore 提供日志/分析存储抽象：上层只面向本包接口，
|
||||
// 禁止直接 import internal/repository/analytics 或触碰 db.ChConn/db.ChDB。
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// ErrMigrating 表示日志数据库正在迁移，当前禁止写入。
|
||||
var ErrMigrating = errors.New("log database is migrating, writes are disabled")
|
||||
|
||||
// AccessLogStore 节点访问日志（of_node_access_logs）。
|
||||
type AccessLogStore interface {
|
||||
// InsertBatch 为写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||||
InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error
|
||||
// BatchInsertNodeAccessLogs 为 batchwriter flush 目标：直接批量写入当前存储。
|
||||
BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error
|
||||
|
||||
List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error)
|
||||
Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error)
|
||||
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error)
|
||||
BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error)
|
||||
CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
|
||||
BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error)
|
||||
IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error)
|
||||
IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error)
|
||||
CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error)
|
||||
WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error)
|
||||
IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error)
|
||||
TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error)
|
||||
ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error)
|
||||
NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error)
|
||||
DeleteAll(ctx context.Context) (int64, error)
|
||||
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error)
|
||||
// ListForMigration 按 id 升序分页读取（迁移复制用）。
|
||||
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error)
|
||||
// MigrationRange 返回源表 logged_at 的最小/最大值（空表返回零值），迁移预建分区用。
|
||||
MigrationRange(ctx context.Context) (from, to time.Time, err error)
|
||||
// EnsurePartitions 幂等预建 PG 分区（按月），覆盖 [from, to] 月份；CH/SQLite 为 no-op。
|
||||
// 目标为 PG 的迁移在复制前调用，避免历史数据写入报 "no partition of relation found"。
|
||||
EnsurePartitions(ctx context.Context, from, to time.Time) error
|
||||
}
|
||||
|
||||
// ObservabilityStore 可观测 4 表（metric snapshots / edge health / frps / frpc）。
|
||||
type ObservabilityStore interface {
|
||||
InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error
|
||||
ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error)
|
||||
DeleteAllMetricSnapshots(ctx context.Context) (int64, error)
|
||||
DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error
|
||||
|
||||
// ListTrafficHourly 返回小时级流量汇总（按 node/hour 聚合，unique_visitor_count 恒 0）。
|
||||
// CH 后端读 of_access_log_hourly rollup；PG/SQLite 从 of_node_access_logs 实时聚合。
|
||||
ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error)
|
||||
// ListAccessLogHourly 返回按 node/hour/host 的小时级访问日志汇总。
|
||||
ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error)
|
||||
// ListMetricHourly 返回小时级指标聚合（avg cpu/memory + 计数器增量，reported_nodes 去重节点数）。
|
||||
ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error)
|
||||
|
||||
InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error
|
||||
ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error)
|
||||
DeleteAllEdgeHealth(ctx context.Context) (int64, error)
|
||||
DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error
|
||||
|
||||
InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error
|
||||
ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error)
|
||||
DeleteAllNodeObservationFrps(ctx context.Context) (int64, error)
|
||||
DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error
|
||||
|
||||
InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error
|
||||
ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error)
|
||||
DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error)
|
||||
DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error
|
||||
|
||||
// 迁移复制用：按 id 升序分页读取。
|
||||
ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error)
|
||||
ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error)
|
||||
ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error)
|
||||
ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error)
|
||||
}
|
||||
|
||||
// UserAccessLogStore 用户访问日志（w_user_access_logs）。
|
||||
type UserAccessLogStore interface {
|
||||
BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error
|
||||
// DeleteAll 清空全部用户访问日志（迁移「覆盖目标库已有日志」幂等前提用）。
|
||||
DeleteAll(ctx context.Context) (int64, error)
|
||||
Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error)
|
||||
List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error)
|
||||
GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error)
|
||||
GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error)
|
||||
GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error)
|
||||
// ListForMigration 按 id 升序分页读取（迁移复制用）。
|
||||
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error)
|
||||
// MigrationRange 返回源表 created_at 的最小/最大值（空表返回零值），迁移预建分区用。
|
||||
MigrationRange(ctx context.Context) (from, to time.Time, err error)
|
||||
}
|
||||
|
||||
// StatusStore 日志库状态（供管理端状态端点）。
|
||||
type StatusStore interface {
|
||||
ActiveDatabase(ctx context.Context) (string, error)
|
||||
ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) // 仅 CH 激活时非 nil
|
||||
}
|
||||
|
||||
// Store 聚合当前生效日志库的全部域存储。
|
||||
type Store struct {
|
||||
AccessLogs AccessLogStore
|
||||
Observability ObservabilityStore
|
||||
UserAccessLogs UserAccessLogStore
|
||||
Status StatusStore
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// TestEnsurePartitionsPostgresInsertAcrossMonths 需要 TEST_POSTGRES_DSN（未设置时跳过）：
|
||||
// 验证 EnsurePartitions 预建任意月份范围分区后，跨月历史数据可写入 PG 分区表
|
||||
// （对应迁移任务从 CH/SQLite 复制历史日志到 PG 时先预建分区的场景）。
|
||||
func TestEnsurePartitionsPostgresInsertAcrossMonths(t *testing.T) {
|
||||
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
|
||||
if dsn == "" {
|
||||
t.Skip("TEST_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open postgres: %v", err)
|
||||
}
|
||||
sqlDB, err := gdb.DB()
|
||||
if err != nil {
|
||||
t.Fatalf("sql db: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
|
||||
schema := fmt.Sprintf("logstore_partition_%d", time.Now().UnixNano())
|
||||
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
|
||||
t.Fatalf("invalid schema: %s", schema)
|
||||
}
|
||||
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
|
||||
t.Fatalf("set search_path: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = gdb.Exec("SET search_path TO public").Error
|
||||
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
|
||||
_ = sqlDB.Close()
|
||||
})
|
||||
|
||||
// 与 goose/postgres/202608080001_create_log_tables.sql 保持一致的分区父表 DDL。
|
||||
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
|
||||
if err := gdb.Exec(ddl).Error; err != nil {
|
||||
t.Fatalf("create partitioned table: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, _ string) (string, error) { return "", nil })
|
||||
defer ResetForTest()
|
||||
|
||||
ctx := context.Background()
|
||||
store := newGormStore(gdb)
|
||||
ua := newUserAccessLogGormStore(gdb)
|
||||
|
||||
// 源范围跨 3 个月：2026-01-10 ~ 2026-03-20；to+1 月兜底生成 202601..202604 分区。
|
||||
from := time.Date(2026, 1, 10, 8, 0, 0, 0, time.UTC)
|
||||
max := time.Date(2026, 3, 20, 9, 30, 0, 0, time.UTC)
|
||||
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
|
||||
t.Fatalf("EnsurePartitions: %v", err)
|
||||
}
|
||||
|
||||
// 幂等：重复调用不报错（CREATE TABLE IF NOT EXISTS ... PARTITION OF）。
|
||||
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
|
||||
t.Fatalf("EnsurePartitions idempotent: %v", err)
|
||||
}
|
||||
|
||||
var partitionCount int64
|
||||
if err := gdb.Raw(
|
||||
"SELECT count(*) FROM pg_inherits WHERE inhrelid = to_regclass('of_node_access_logs')",
|
||||
).Scan(&partitionCount).Error; err != nil {
|
||||
t.Fatalf("count partitions: %v", err)
|
||||
}
|
||||
if partitionCount != 4 {
|
||||
t.Fatalf("of_node_access_logs partitions = %d, want 4", partitionCount)
|
||||
}
|
||||
|
||||
// 跨月插入：1/2/3 月各 2 条节点访问日志 + 2 条用户访问日志，均应命中已有分区。
|
||||
nodeRows := []analyticsmodel.NodeAccessLog{
|
||||
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
|
||||
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.2"},
|
||||
{ID: 3, NodeID: "n2", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.2"},
|
||||
{ID: 4, NodeID: "n2", LoggedAt: time.Date(2026, 2, 12, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.3"},
|
||||
{ID: 5, NodeID: "n1", LoggedAt: time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
|
||||
{ID: 6, NodeID: "n1", LoggedAt: time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.4"},
|
||||
}
|
||||
if err := store.BatchInsertNodeAccessLogs(ctx, nodeRows); err != nil {
|
||||
t.Fatalf("insert node access logs across months: %v", err)
|
||||
}
|
||||
|
||||
userRows := []analyticsmodel.UserAccessLog{
|
||||
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)},
|
||||
{ID: 2, UserID: 102, Path: "/b", CreatedAt: time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
if err := ua.BatchInsert(ctx, userRows); err != nil {
|
||||
t.Fatalf("insert user access logs across months: %v", err)
|
||||
}
|
||||
|
||||
var nodeCount, userCount int64
|
||||
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
|
||||
t.Fatalf("count node access logs: %v", err)
|
||||
}
|
||||
if err := gdb.Model(&analyticsmodel.UserAccessLog{}).Count(&userCount).Error; err != nil {
|
||||
t.Fatalf("count user access logs: %v", err)
|
||||
}
|
||||
if nodeCount != 6 {
|
||||
t.Fatalf("node access log count = %d, want 6", nodeCount)
|
||||
}
|
||||
if userCount != 2 {
|
||||
t.Fatalf("user access log count = %d, want 2", userCount)
|
||||
}
|
||||
|
||||
// MigrationRange 返回跨月范围（覆盖两表）。
|
||||
gotFrom, gotTo, err := store.MigrationRange(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("node MigrationRange: %v", err)
|
||||
}
|
||||
if !gotFrom.Equal(time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)) || !gotTo.Equal(time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("node MigrationRange = %s ~ %s, want 2026-01-15 ~ 2026-03-18", gotFrom, gotTo)
|
||||
}
|
||||
uaFrom, uaTo, err := ua.MigrationRange(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("user MigrationRange: %v", err)
|
||||
}
|
||||
if !uaFrom.Equal(time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)) || !uaTo.Equal(time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("user MigrationRange = %s ~ %s", uaFrom, uaTo)
|
||||
}
|
||||
}
|
||||
|
||||
// postgresNodeAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
|
||||
const postgresNodeAccessLogsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS of_node_access_logs (
|
||||
id BIGINT NOT NULL,
|
||||
node_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
logged_at TIMESTAMPTZ NOT NULL,
|
||||
remote_addr VARCHAR(128) NOT NULL DEFAULT '',
|
||||
region VARCHAR(128) NOT NULL DEFAULT '',
|
||||
host VARCHAR(255) NOT NULL DEFAULT '',
|
||||
path VARCHAR(2048) NOT NULL DEFAULT '',
|
||||
user_agent TEXT NOT NULL DEFAULT '',
|
||||
cache_status VARCHAR(64) NOT NULL DEFAULT '',
|
||||
status_code INTEGER NOT NULL DEFAULT 0,
|
||||
bytes_sent BIGINT NOT NULL DEFAULT 0,
|
||||
request_length BIGINT NOT NULL DEFAULT 0,
|
||||
request_time_ms INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id, logged_at)
|
||||
) PARTITION BY RANGE (logged_at)`
|
||||
|
||||
// postgresUserAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
|
||||
const postgresUserAccessLogsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS w_user_access_logs (
|
||||
id BIGINT NOT NULL,
|
||||
user_id BIGINT NOT NULL DEFAULT 0,
|
||||
path VARCHAR(2048) NOT NULL DEFAULT '',
|
||||
method VARCHAR(16) NOT NULL DEFAULT '',
|
||||
ip VARCHAR(128) NOT NULL DEFAULT '',
|
||||
user_agent TEXT NOT NULL DEFAULT '',
|
||||
headers TEXT NOT NULL DEFAULT '',
|
||||
status INTEGER NOT NULL DEFAULT 0,
|
||||
latency BIGINT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id, created_at)
|
||||
) PARTITION BY RANGE (created_at)`
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,170 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
)
|
||||
|
||||
// logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。
|
||||
const (
|
||||
logDatabaseKey = model.ConfigKeyLogDatabase
|
||||
logMigrationKey = model.ConfigKeyLogDBMigration
|
||||
)
|
||||
|
||||
// 日志库名常量（与 model 配置值一致，集中避免散落字符串字面量）。
|
||||
const (
|
||||
dbNamePostgres = "postgres"
|
||||
dbNameSQLite = "sqlite"
|
||||
dbNameClickHouse = "clickhouse"
|
||||
)
|
||||
|
||||
// errConfigReaderNotWired 表示 config reader 尚未注入（首启/测试场景按 seed 规则兜底）。
|
||||
var errConfigReaderNotWired = errors.New("logstore: config reader not wired")
|
||||
|
||||
// ConfigReader 读取系统配置字符串值，由 bootstrap 注入（避免 logstore ↔ repository 循环依赖）。
|
||||
type ConfigReader func(ctx context.Context, key string) (string, error)
|
||||
|
||||
var (
|
||||
configReader ConfigReader
|
||||
|
||||
storeMu sync.RWMutex
|
||||
active *Store
|
||||
activeDB string
|
||||
)
|
||||
|
||||
// SetConfigReader 注入系统配置读取函数（bootstrap 调用，测试可注入内存实现）。
|
||||
func SetConfigReader(fn ConfigReader) { configReader = fn }
|
||||
|
||||
func getConfig(ctx context.Context, key string) (string, error) {
|
||||
if configReader == nil {
|
||||
return "", errConfigReaderNotWired
|
||||
}
|
||||
return configReader(ctx, key)
|
||||
}
|
||||
|
||||
// Active 返回当前生效的日志库 Store。按 log_database 系统配置惰性解析并缓存，
|
||||
// 配置更新（含迁移任务翻转）后自动重建。
|
||||
func Active(ctx context.Context) (*Store, error) {
|
||||
current, err := resolveDatabase(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storeMu.RLock()
|
||||
if active != nil && activeDB == current {
|
||||
s := active
|
||||
storeMu.RUnlock()
|
||||
return s, nil
|
||||
}
|
||||
storeMu.RUnlock()
|
||||
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
if active != nil && activeDB == current {
|
||||
return active, nil
|
||||
}
|
||||
s, err := buildStore(ctx, current, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
active = s
|
||||
activeDB = current
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Build 直接按目标构造 store（不经 Active 缓存）。
|
||||
func Build(ctx context.Context, database string) (*Store, error) {
|
||||
return buildStore(ctx, database, false)
|
||||
}
|
||||
|
||||
// BuildForMigration 构造迁移目标 store：与 Build 相同但不做冻结检查
|
||||
// （迁移期间 log_db_migration=migrating 已冻结源库写入，目标库的清空/复制写入必须放行）。
|
||||
func BuildForMigration(ctx context.Context, database string) (*Store, error) {
|
||||
return buildStore(ctx, database, true)
|
||||
}
|
||||
|
||||
// buildStore 按目标构造实现。skipFreeze 为 true 时该 store 跳过冻结检查
|
||||
// （仅迁移任务的目标 store 使用）。gorm 分支 UserAccessLogs 用独立包装类型
|
||||
// （gormLogStore 已占用 List/Count 方法名，无法再实现 UserAccessLogStore）。
|
||||
func buildStore(ctx context.Context, database string, skipFreeze bool) (*Store, error) {
|
||||
switch database {
|
||||
case dbNameClickHouse:
|
||||
ch := newClickHouseStore()
|
||||
ch.skipFreeze = skipFreeze
|
||||
return &Store{
|
||||
AccessLogs: ch,
|
||||
Observability: ch,
|
||||
UserAccessLogs: newClickHouseUserAccessLogStore(),
|
||||
Status: ch,
|
||||
}, nil
|
||||
case dbNamePostgres, dbNameSQLite:
|
||||
gdb := db.DB(ctx)
|
||||
g := newGormStore(gdb)
|
||||
g.skipFreeze = skipFreeze
|
||||
return &Store{
|
||||
AccessLogs: g,
|
||||
Observability: g,
|
||||
UserAccessLogs: newUserAccessLogGormStore(gdb),
|
||||
Status: g,
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported log database: %s", database)
|
||||
}
|
||||
}
|
||||
|
||||
// Migrating 返回日志库是否处于迁移冻结状态。
|
||||
func Migrating(ctx context.Context) bool {
|
||||
v, err := getConfig(ctx, logMigrationKey)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return v == "migrating"
|
||||
}
|
||||
|
||||
// Init 在 bootstrap 阶段预热一次激活 store（幂等，失败不致命——首次使用时再解析）。
|
||||
func Init(ctx context.Context) {
|
||||
_, _ = Active(ctx)
|
||||
}
|
||||
|
||||
// ResetForTest 清空缓存的激活 store 与 config reader，便于测试注入。
|
||||
func ResetForTest() {
|
||||
storeMu.Lock()
|
||||
active = nil
|
||||
activeDB = ""
|
||||
storeMu.Unlock()
|
||||
configReader = nil
|
||||
}
|
||||
|
||||
// ActiveDatabase 返回当前日志主库名（postgres|sqlite|clickhouse）。
|
||||
func ActiveDatabase(ctx context.Context) (string, error) {
|
||||
return resolveDatabase(ctx)
|
||||
}
|
||||
|
||||
// resolveDatabase 读取 log_database：值缺失或 reader 未装配（首启）时按启动规则 seed；
|
||||
// 已装配 reader 的真实读取错误直接透出，避免把读失败当首次启动。
|
||||
func resolveDatabase(ctx context.Context) (string, error) {
|
||||
v, err := getConfig(ctx, logDatabaseKey)
|
||||
if err != nil && !errors.Is(err, errConfigReaderNotWired) {
|
||||
return "", err
|
||||
}
|
||||
if v != "" {
|
||||
return v, nil
|
||||
}
|
||||
// 首次启动 seed：CH 启用 → clickhouse；否则随主库。
|
||||
defaultDB := dbNameSQLite
|
||||
if config.Config.Database.Enabled {
|
||||
defaultDB = dbNamePostgres
|
||||
}
|
||||
if config.Config.ClickHouse.Enabled {
|
||||
defaultDB = dbNameClickHouse
|
||||
}
|
||||
return defaultDB, nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package logstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
func TestMigratingReadsConfig(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logMigrationKey {
|
||||
return "migrating", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
if !Migrating(context.Background()) {
|
||||
t.Fatal("Migrating() = false, want true when key=migrating")
|
||||
}
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", nil
|
||||
})
|
||||
if Migrating(context.Background()) {
|
||||
t.Fatal("Migrating() = true, want false when key empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDatabaseDefaults(t *testing.T) {
|
||||
ResetForTest()
|
||||
// 配置缺失（reader 返回空值）时按主库规则 seed（config.Config 默认值由既有测试基建决定）。
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", nil
|
||||
})
|
||||
got, err := resolveDatabase(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("resolveDatabase: %v", err)
|
||||
}
|
||||
if got != "postgres" && got != "sqlite" && got != "clickhouse" {
|
||||
t.Fatalf("unexpected default log database: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDatabaseSurfacesReadError(t *testing.T) {
|
||||
ResetForTest()
|
||||
wantErr := errors.New("boom")
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
return "", wantErr
|
||||
})
|
||||
if _, err := resolveDatabase(context.Background()); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("resolveDatabase error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveBuildsStore(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logDatabaseKey {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
store, err := Active(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Active: %v", err)
|
||||
}
|
||||
if store == nil {
|
||||
t.Fatal("Active() returned nil store")
|
||||
}
|
||||
if store.AccessLogs == nil || store.Observability == nil || store.UserAccessLogs == nil || store.Status == nil {
|
||||
t.Fatalf("Active() store fields not fully wired: %+v", store)
|
||||
}
|
||||
// 再次调用应命中缓存。
|
||||
again, err := Active(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Active (cached): %v", err)
|
||||
}
|
||||
if again != store {
|
||||
t.Fatal("Active() did not return cached store")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseUserAccessLogBatchInsertFreeze 覆盖 CH 用户访问日志 flush 的冻结检查：
|
||||
// 冻结期非空批次返回 ErrMigrating（在触碰 CH 连接之前），空批次直接成功。
|
||||
func TestClickHouseUserAccessLogBatchInsertFreeze(t *testing.T) {
|
||||
ResetForTest()
|
||||
SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == logMigrationKey {
|
||||
return "migrating", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
defer ResetForTest()
|
||||
s := newClickHouseUserAccessLogStore()
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
if err := s.BatchInsert(ctx, []analyticsmodel.UserAccessLog{{UserID: 1, CreatedAt: now}}); !errors.Is(err, ErrMigrating) {
|
||||
t.Fatalf("BatchInsert during migration: want ErrMigrating, got %v", err)
|
||||
}
|
||||
if err := s.BatchInsert(ctx, nil); err != nil {
|
||||
t.Fatalf("BatchInsert empty batch: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -13,25 +12,21 @@ import (
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||
)
|
||||
|
||||
type openFlareAccessLogBucketAggregateRow = analyticsmodel.NodeAccessLogBucketAggregate
|
||||
type openFlareAccessLogBucketDimensionRow = analyticsmodel.NodeAccessLogBucketDimension
|
||||
type openFlareAccessLogIPAggregateRow = analyticsmodel.NodeAccessLogIPAggregate
|
||||
type openFlareAccessLogIPSummaryRow = analyticsmodel.NodeAccessLogIPSummary
|
||||
type openFlareAccessLogIPTrendRow = analyticsmodel.NodeAccessLogIPTrend
|
||||
type openFlareAccessLogWAFIPAggregateRow = analyticsmodel.NodeAccessLogWAFIPAggregate
|
||||
|
||||
const (
|
||||
sortOrderAsc = "asc"
|
||||
columnRemoteAddr = "remote_addr"
|
||||
columnHost = "host"
|
||||
secondsPerMinute = 60
|
||||
)
|
||||
|
||||
// ListOpenFlareAccessLogWAFIPAggregates returns per-IP aggregates for WAF automatic rules.
|
||||
func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLogWAFIPAggregate, error) {
|
||||
rows, err := currentAccessLogStore().WAFIPAggregates(ctx, query)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.AccessLogs.WAFIPAggregates(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -59,39 +54,67 @@ func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query model.Open
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// InsertOpenFlareAccessLogsBatch inserts access log rows into ClickHouse.
|
||||
// InsertOpenFlareAccessLogsBatch inserts access log rows into the active log store.
|
||||
func InsertOpenFlareAccessLogsBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
|
||||
return currentAccessLogStore().InsertBatch(ctx, records)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.AccessLogs.InsertBatch(ctx, records)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogs lists access logs matching the query.
|
||||
func ListOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
|
||||
return currentAccessLogStore().List(ctx, query)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.List(ctx, query)
|
||||
}
|
||||
|
||||
// CountOpenFlareAccessLogs counts access logs, distinct IPs, and total bytes sent matching the query.
|
||||
func CountOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
|
||||
return currentAccessLogStore().Count(ctx, query)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
return s.AccessLogs.Count(ctx, query)
|
||||
}
|
||||
|
||||
// TrafficSummaryOpenFlareAccessLogs returns window-level request/error/UV/bytes summary.
|
||||
func TrafficSummaryOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
|
||||
return currentAccessLogStore().TrafficSummary(ctx, query)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return model.OpenFlareAccessLogTrafficSummary{}, err
|
||||
}
|
||||
return s.AccessLogs.TrafficSummary(ctx, query)
|
||||
}
|
||||
|
||||
// ValueCountsOpenFlareAccessLogs groups logs by status_code, host, path, remote_addr, or user_agent.
|
||||
func ValueCountsOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
|
||||
return currentAccessLogStore().ValueCounts(ctx, query, column, limit)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.ValueCounts(ctx, query, column, limit)
|
||||
}
|
||||
|
||||
// NodeAggregatesOpenFlareAccessLogs returns per-node request/error/UV for the window.
|
||||
func NodeAggregatesOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
|
||||
return currentAccessLogStore().NodeAggregates(ctx, query)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.NodeAggregates(ctx, query)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogRegionCounts returns region counts for access logs.
|
||||
func ListOpenFlareAccessLogRegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
|
||||
return currentAccessLogStore().RegionCounts(ctx, nodeID, since, limit)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AccessLogs.RegionCounts(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogBuckets lists folded access log buckets.
|
||||
@@ -106,7 +129,11 @@ func CountOpenFlareAccessLogBuckets(ctx context.Context, query model.OpenFlareAc
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 180
|
||||
}
|
||||
return currentAccessLogStore().CountBuckets(ctx, filter, bucketSeconds)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.CountBuckets(ctx, filter, bucketSeconds)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogBucketIPs lists folded IP rows for a bucket window.
|
||||
@@ -139,7 +166,11 @@ func ListOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlar
|
||||
// CountOpenFlareAccessLogIPSummaries counts IP summaries.
|
||||
func CountOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery) (int64, error) {
|
||||
filter := openFlareAccessLogQueryFromIPSummary(query)
|
||||
return currentAccessLogStore().CountIPSummaries(ctx, filter)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.CountIPSummaries(ctx, filter)
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogIPTrend lists IP trend points.
|
||||
@@ -158,7 +189,11 @@ func ListOpenFlareAccessLogIPTrend(ctx context.Context, query model.OpenFlareAcc
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 1800
|
||||
}
|
||||
rows, err := currentAccessLogStore().IPTrend(ctx, filter, bucketSeconds)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.AccessLogs.IPTrend(ctx, filter, bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -174,17 +209,29 @@ func ListOpenFlareAccessLogIPTrend(ctx context.Context, query model.OpenFlareAcc
|
||||
|
||||
// DeleteAllOpenFlareAccessLogs deletes all access logs.
|
||||
func DeleteAllOpenFlareAccessLogs(ctx context.Context) (int64, error) {
|
||||
return currentAccessLogStore().DeleteAll(ctx)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.DeleteAll(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareAccessLogsBefore deletes access logs older than cutoff.
|
||||
func DeleteOpenFlareAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentAccessLogStore().DeleteBefore(ctx, cutoff)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.DeleteBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareAccessLogsByNodeBefore deletes access logs for a node older than cutoff.
|
||||
func DeleteOpenFlareAccessLogsByNodeBefore(ctx context.Context, nodeID string, cutoff time.Time) (int64, error) {
|
||||
return currentAccessLogStore().DeleteByNodeBefore(ctx, nodeID, cutoff)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.AccessLogs.DeleteByNodeBefore(ctx, nodeID, cutoff)
|
||||
}
|
||||
|
||||
func buildOpenFlareAccessLogBucketRows(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) ([]*model.OpenFlareAccessLogBucketRow, error) {
|
||||
@@ -193,8 +240,11 @@ func buildOpenFlareAccessLogBucketRows(ctx context.Context, query model.OpenFlar
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 180
|
||||
}
|
||||
|
||||
partials, err := currentAccessLogStore().BucketAggregates(ctx, filter, bucketSeconds)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partials, err := s.AccessLogs.BucketAggregates(ctx, filter, bucketSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -242,7 +292,11 @@ func buildOpenFlareAccessLogBucketIPRows(ctx context.Context, query model.OpenFl
|
||||
|
||||
func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*analyticsmodel.NodeAccessLogIPSummary, error) {
|
||||
filter := openFlareAccessLogQueryFromIPSummary(query)
|
||||
partials, err := currentAccessLogStore().IPSummaries(ctx, filter, recentSince)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partials, err := s.AccessLogs.IPSummaries(ctx, filter, recentSince)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -268,7 +322,11 @@ func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query model.OpenF
|
||||
}
|
||||
|
||||
func queryOpenFlareAccessLogIPAggregateRows(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
|
||||
partials, err := currentAccessLogStore().IPAggregates(ctx, filter, exactRemoteAddr)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
partials, err := s.AccessLogs.IPAggregates(ctx, filter, exactRemoteAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -385,92 +443,3 @@ func openFlareAccessLogCompareInt64(left int64, right int64) int {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func openFlareAccessLogStatusCodeToInt32(code int) int32 {
|
||||
switch {
|
||||
case code > math.MaxInt32:
|
||||
return math.MaxInt32
|
||||
case code < math.MinInt32:
|
||||
return math.MinInt32
|
||||
default:
|
||||
return int32(code)
|
||||
}
|
||||
}
|
||||
|
||||
func sortOpenFlareAccessLogBucketRows(items []*model.OpenFlareAccessLogBucketRow, sortBy string, sortOrder string) {
|
||||
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
var compare int
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "request_count":
|
||||
compare = openFlareAccessLogCompareInt64(left.RequestCount, right.RequestCount)
|
||||
default:
|
||||
compare = openFlareAccessLogCompareInt64(left.BucketEpoch, right.BucketEpoch)
|
||||
}
|
||||
if compare == 0 {
|
||||
compare = openFlareAccessLogCompareInt64(left.BucketEpoch, right.BucketEpoch)
|
||||
}
|
||||
if desc {
|
||||
return compare > 0
|
||||
}
|
||||
return compare < 0
|
||||
})
|
||||
}
|
||||
|
||||
func sortOpenFlareAccessLogIPSummaryRows(items []*model.OpenFlareAccessLogIPSummaryRow, sortBy string, sortOrder string) {
|
||||
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
var compare int
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "request_length", "bytes_received":
|
||||
compare = openFlareAccessLogCompareInt64(left.BytesReceived, right.BytesReceived)
|
||||
case "bytes_sent":
|
||||
compare = openFlareAccessLogCompareInt64(left.BytesSent, right.BytesSent)
|
||||
case "success_ratio":
|
||||
compare = openFlareAccessLogCompareFloat64(left.SuccessRatio, right.SuccessRatio)
|
||||
case "last_seen_at":
|
||||
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
|
||||
case "remote_addr":
|
||||
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
|
||||
default:
|
||||
compare = openFlareAccessLogCompareInt64(left.TotalRequests, right.TotalRequests)
|
||||
}
|
||||
if compare == 0 {
|
||||
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
|
||||
}
|
||||
if compare == 0 {
|
||||
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
|
||||
}
|
||||
if desc {
|
||||
return compare > 0
|
||||
}
|
||||
return compare < 0
|
||||
})
|
||||
}
|
||||
|
||||
func openFlareAccessLogCompareFloat64(left, right float64) int {
|
||||
if left < right {
|
||||
return -1
|
||||
}
|
||||
if left > right {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func openFlareAccessLogUintToInt64(value uint64) int64 {
|
||||
if value > math.MaxInt64 {
|
||||
return math.MaxInt64
|
||||
}
|
||||
return int64(value)
|
||||
}
|
||||
|
||||
@@ -1,308 +0,0 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
)
|
||||
|
||||
// AccessLogInsertHooks queues node access logs for async ClickHouse write.
|
||||
// Wired from openflare/chwriter.Init so model never imports the apps layer.
|
||||
type AccessLogInsertHooks struct {
|
||||
QueueNodeAccessLogs func(logs []analyticsmodel.NodeAccessLog)
|
||||
}
|
||||
|
||||
var (
|
||||
accessLogInsertHooksMu sync.RWMutex
|
||||
accessLogInsertHooks AccessLogInsertHooks
|
||||
)
|
||||
|
||||
// SetAccessLogInsertHooks registers async queue callbacks for access log inserts.
|
||||
func SetAccessLogInsertHooks(hooks AccessLogInsertHooks) {
|
||||
accessLogInsertHooksMu.Lock()
|
||||
accessLogInsertHooks = hooks
|
||||
accessLogInsertHooksMu.Unlock()
|
||||
}
|
||||
|
||||
func currentAccessLogInsertHooks() AccessLogInsertHooks {
|
||||
accessLogInsertHooksMu.RLock()
|
||||
defer accessLogInsertHooksMu.RUnlock()
|
||||
return accessLogInsertHooks
|
||||
}
|
||||
|
||||
type accessLogStore interface {
|
||||
InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error
|
||||
List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error)
|
||||
Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error)
|
||||
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error)
|
||||
BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error)
|
||||
CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
|
||||
BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error)
|
||||
IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error)
|
||||
WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error)
|
||||
IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error)
|
||||
CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error)
|
||||
IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error)
|
||||
TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error)
|
||||
ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error)
|
||||
NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error)
|
||||
DeleteAll(ctx context.Context) (int64, error)
|
||||
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error)
|
||||
}
|
||||
|
||||
var (
|
||||
accessLogStoreMu sync.RWMutex
|
||||
accessLogStoreHolder accessLogStore
|
||||
)
|
||||
|
||||
func currentAccessLogStore() accessLogStore {
|
||||
accessLogStoreMu.RLock()
|
||||
defer accessLogStoreMu.RUnlock()
|
||||
if accessLogStoreHolder != nil {
|
||||
return accessLogStoreHolder
|
||||
}
|
||||
return clickhouseAccessLogStore{}
|
||||
}
|
||||
|
||||
// SetAccessLogStoreForTest swaps the access log store implementation for unit tests.
|
||||
func SetAccessLogStoreForTest(store accessLogStore) func() {
|
||||
accessLogStoreMu.Lock()
|
||||
previous := accessLogStoreHolder
|
||||
accessLogStoreHolder = store
|
||||
accessLogStoreMu.Unlock()
|
||||
return func() {
|
||||
accessLogStoreMu.Lock()
|
||||
accessLogStoreHolder = previous
|
||||
accessLogStoreMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// NewMemoryAccessLogStore returns an in-memory access log store for unit tests.
|
||||
func NewMemoryAccessLogStore() accessLogStore {
|
||||
return &memoryAccessLogStore{
|
||||
records: make([]*model.OpenFlareAccessLog, 0),
|
||||
}
|
||||
}
|
||||
|
||||
type clickhouseAccessLogStore struct{}
|
||||
|
||||
func (clickhouseAccessLogStore) InsertBatch(_ context.Context, records []*model.OpenFlareAccessLog) error {
|
||||
logs := make([]analyticsmodel.NodeAccessLog, 0, len(records))
|
||||
for _, record := range records {
|
||||
if record == nil {
|
||||
continue
|
||||
}
|
||||
logs = append(logs, toAnalyticsNodeAccessLog(record))
|
||||
}
|
||||
if hook := currentAccessLogInsertHooks().QueueNodeAccessLogs; hook != nil {
|
||||
hook(logs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
|
||||
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeAccessLogs(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
|
||||
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
|
||||
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]*model.OpenFlareAccessLogRegionCount, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareAccessLogRegionCount{
|
||||
Region: row.Region,
|
||||
Count: row.Count,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error) {
|
||||
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
|
||||
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
|
||||
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, bucketSeconds)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error) {
|
||||
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), exactRemoteAddr)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error) {
|
||||
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), recentSince)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error) {
|
||||
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
|
||||
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
|
||||
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
|
||||
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
if err != nil {
|
||||
return model.OpenFlareAccessLogTrafficSummary{}, err
|
||||
}
|
||||
return model.OpenFlareAccessLogTrafficSummary{
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueIPCount: row.UniqueIPCount,
|
||||
BytesSent: row.BytesSent,
|
||||
RequestLength: row.RequestLength,
|
||||
NodeCount: row.NodeCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
|
||||
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]model.OpenFlareAccessLogValueCount, len(rows))
|
||||
for i, row := range rows {
|
||||
result[i] = model.OpenFlareAccessLogValueCount{Value: row.Value, Count: row.Count}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (clickhouseAccessLogStore) NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
|
||||
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]model.OpenFlareAccessLogNodeAggregate, len(rows))
|
||||
for i, row := range rows {
|
||||
result[i] = model.OpenFlareAccessLogNodeAggregate{
|
||||
NodeID: row.NodeID,
|
||||
RequestCount: row.RequestCount,
|
||||
ErrorCount: row.ErrorCount,
|
||||
UniqueIPCount: row.UniqueIPCount,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func toNodeAccessLogFilter(query model.OpenFlareAccessLogQuery) analyticsrepo.NodeAccessLogFilter {
|
||||
return analyticsrepo.NodeAccessLogFilter{
|
||||
NodeID: query.NodeID,
|
||||
RemoteAddr: query.RemoteAddr,
|
||||
Host: query.Host,
|
||||
Hosts: query.Hosts,
|
||||
Path: query.Path,
|
||||
Since: query.Since,
|
||||
Until: query.Until,
|
||||
Page: query.Page,
|
||||
PageSize: query.PageSize,
|
||||
SortBy: query.SortBy,
|
||||
SortOrder: query.SortOrder,
|
||||
}
|
||||
}
|
||||
|
||||
func toAnalyticsNodeAccessLog(record *model.OpenFlareAccessLog) analyticsmodel.NodeAccessLog {
|
||||
var bytesSent uint64
|
||||
if record.BytesSent > 0 {
|
||||
bytesSent = uint64(record.BytesSent)
|
||||
}
|
||||
var requestLength uint64
|
||||
if record.RequestLength > 0 {
|
||||
requestLength = uint64(record.RequestLength)
|
||||
}
|
||||
var requestTimeMs uint32
|
||||
if record.RequestTimeMs > 0 && record.RequestTimeMs <= int64(math.MaxUint32) {
|
||||
requestTimeMs = uint32(record.RequestTimeMs)
|
||||
}
|
||||
return analyticsmodel.NodeAccessLog{
|
||||
ID: record.ID,
|
||||
NodeID: record.NodeID,
|
||||
LoggedAt: record.LoggedAt,
|
||||
RemoteAddr: record.RemoteAddr,
|
||||
Region: record.Region,
|
||||
Host: record.Host,
|
||||
Path: record.Path,
|
||||
UserAgent: record.UserAgent,
|
||||
CacheStatus: record.CacheStatus,
|
||||
StatusCode: openFlareAccessLogStatusCodeToInt32(record.StatusCode),
|
||||
BytesSent: bytesSent,
|
||||
RequestLength: requestLength,
|
||||
RequestTimeMs: requestTimeMs,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func fromAnalyticsNodeAccessLogs(rows []analyticsmodel.NodeAccessLog) []*model.OpenFlareAccessLog {
|
||||
result := make([]*model.OpenFlareAccessLog, len(rows))
|
||||
for index, row := range rows {
|
||||
var bytesSent int64
|
||||
if row.BytesSent <= math.MaxInt64 {
|
||||
bytesSent = int64(row.BytesSent)
|
||||
} else {
|
||||
bytesSent = math.MaxInt64
|
||||
}
|
||||
var requestLength int64
|
||||
if row.RequestLength <= math.MaxInt64 {
|
||||
requestLength = int64(row.RequestLength)
|
||||
} else {
|
||||
requestLength = math.MaxInt64
|
||||
}
|
||||
result[index] = &model.OpenFlareAccessLog{
|
||||
ID: row.ID,
|
||||
NodeID: row.NodeID,
|
||||
LoggedAt: row.LoggedAt,
|
||||
RemoteAddr: row.RemoteAddr,
|
||||
Region: row.Region,
|
||||
Host: row.Host,
|
||||
Path: row.Path,
|
||||
UserAgent: row.UserAgent,
|
||||
CacheStatus: row.CacheStatus,
|
||||
StatusCode: int(row.StatusCode),
|
||||
BytesSent: bytesSent,
|
||||
RequestLength: requestLength,
|
||||
RequestTimeMs: int64(row.RequestTimeMs),
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,710 +0,0 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
|
||||
)
|
||||
|
||||
const (
|
||||
accessLogColumnStatusCode = "status_code"
|
||||
accessLogColumnHost = "host"
|
||||
accessLogColumnPath = "path"
|
||||
accessLogColumnRemoteAddr = "remote_addr"
|
||||
accessLogColumnUserAgent = "user_agent"
|
||||
)
|
||||
|
||||
type memoryAccessLogStore struct {
|
||||
mu sync.RWMutex
|
||||
records []*model.OpenFlareAccessLog
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) InsertBatch(_ context.Context, records []*model.OpenFlareAccessLog) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
for _, record := range records {
|
||||
if record == nil {
|
||||
continue
|
||||
}
|
||||
copyRecord := *record
|
||||
if copyRecord.ID == 0 {
|
||||
copyRecord.ID = idgen.NextUint64ID()
|
||||
}
|
||||
if copyRecord.CreatedAt.IsZero() {
|
||||
copyRecord.CreatedAt = now
|
||||
}
|
||||
copyRecord.LoggedAt = copyRecord.LoggedAt.UTC()
|
||||
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
|
||||
s.records = append(s.records, ©Record)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) List(_ context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(query)
|
||||
sortOpenFlareAccessLogRows(rows, query.SortBy, query.SortOrder)
|
||||
if query.PageSize > 0 {
|
||||
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
|
||||
return cloneAccessLogSlice(rows[start:end]), nil
|
||||
}
|
||||
return cloneAccessLogSlice(rows), nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) Count(_ context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(query)
|
||||
ips := make(map[string]struct{})
|
||||
var totalBytes int64
|
||||
for _, row := range rows {
|
||||
totalBytes += row.BytesSent
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
ips[remoteAddr] = struct{}{}
|
||||
}
|
||||
return int64(len(rows)), int64(len(ips)), totalBytes, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) RegionCounts(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(model.OpenFlareAccessLogQuery{NodeID: nodeID, Since: since})
|
||||
counts := make(map[string]int64)
|
||||
for _, row := range rows {
|
||||
region := strings.TrimSpace(row.Region)
|
||||
if region == "" {
|
||||
continue
|
||||
}
|
||||
counts[region]++
|
||||
}
|
||||
result := make([]*model.OpenFlareAccessLogRegionCount, 0, len(counts))
|
||||
for region, count := range counts {
|
||||
result = append(result, &model.OpenFlareAccessLogRegionCount{Region: region, Count: count})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].Count == result[j].Count {
|
||||
return result[i].Region < result[j].Region
|
||||
}
|
||||
return result[i].Count > result[j].Count
|
||||
})
|
||||
if limit > 0 && len(result) > limit {
|
||||
result = result[:limit]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
type bucketAccumulator struct {
|
||||
openFlareAccessLogBucketAggregateRow
|
||||
uniqueIPs map[string]struct{}
|
||||
uniqueHosts map[string]struct{}
|
||||
}
|
||||
aggregates := make(map[int64]*bucketAccumulator)
|
||||
for _, row := range rows {
|
||||
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
|
||||
item := aggregates[bucketEpoch]
|
||||
if item == nil {
|
||||
item = &bucketAccumulator{
|
||||
openFlareAccessLogBucketAggregateRow: openFlareAccessLogBucketAggregateRow{BucketEpoch: bucketEpoch},
|
||||
uniqueIPs: make(map[string]struct{}),
|
||||
uniqueHosts: make(map[string]struct{}),
|
||||
}
|
||||
aggregates[bucketEpoch] = item
|
||||
}
|
||||
item.RequestCount++
|
||||
item.BytesSent += row.BytesSent
|
||||
item.RequestLength += row.RequestLength
|
||||
switch {
|
||||
case row.StatusCode < 400:
|
||||
item.SuccessCount++
|
||||
case row.StatusCode < 500:
|
||||
item.ClientErrorCount++
|
||||
default:
|
||||
item.ServerErrorCount++
|
||||
}
|
||||
if remoteAddr := strings.TrimSpace(row.RemoteAddr); remoteAddr != "" {
|
||||
item.uniqueIPs[remoteAddr] = struct{}{}
|
||||
}
|
||||
if host := strings.TrimSpace(row.Host); host != "" {
|
||||
item.uniqueHosts[host] = struct{}{}
|
||||
}
|
||||
}
|
||||
result := make([]openFlareAccessLogBucketAggregateRow, 0, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
item.UniqueIPCount = int64(len(item.uniqueIPs))
|
||||
item.UniqueHostCount = int64(len(item.uniqueHosts))
|
||||
result = append(result, item.openFlareAccessLogBucketAggregateRow)
|
||||
}
|
||||
bucketRows := make([]*model.OpenFlareAccessLogBucketRow, len(result))
|
||||
for index := range result {
|
||||
bucketRows[index] = &model.OpenFlareAccessLogBucketRow{
|
||||
BucketEpoch: result[index].BucketEpoch,
|
||||
RequestCount: result[index].RequestCount,
|
||||
UniqueIPCount: result[index].UniqueIPCount,
|
||||
UniqueHostCount: result[index].UniqueHostCount,
|
||||
SuccessCount: result[index].SuccessCount,
|
||||
ClientErrorCount: result[index].ClientErrorCount,
|
||||
ServerErrorCount: result[index].ServerErrorCount,
|
||||
BytesSent: result[index].BytesSent,
|
||||
RequestLength: result[index].RequestLength,
|
||||
}
|
||||
}
|
||||
sortOpenFlareAccessLogBucketRows(bucketRows, filter.SortBy, filter.SortOrder)
|
||||
for index := range result {
|
||||
result[index] = openFlareAccessLogBucketAggregateRow{
|
||||
BucketEpoch: bucketRows[index].BucketEpoch,
|
||||
RequestCount: bucketRows[index].RequestCount,
|
||||
UniqueIPCount: bucketRows[index].UniqueIPCount,
|
||||
UniqueHostCount: bucketRows[index].UniqueHostCount,
|
||||
SuccessCount: bucketRows[index].SuccessCount,
|
||||
ClientErrorCount: bucketRows[index].ClientErrorCount,
|
||||
ServerErrorCount: bucketRows[index].ServerErrorCount,
|
||||
BytesSent: bucketRows[index].BytesSent,
|
||||
RequestLength: bucketRows[index].RequestLength,
|
||||
}
|
||||
}
|
||||
if filter.PageSize > 0 {
|
||||
start, end := openFlareAccessLogPaginateBounds(len(result), filter.Page, filter.PageSize)
|
||||
return result[start:end], nil
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) CountBuckets(_ context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
seen := make(map[int64]struct{})
|
||||
for _, row := range rows {
|
||||
seen[memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)] = struct{}{}
|
||||
}
|
||||
return int64(len(seen)), nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) BucketDimensions(_ context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
seen := make(map[int64]map[string]struct{})
|
||||
var result []openFlareAccessLogBucketDimensionRow
|
||||
for _, row := range rows {
|
||||
var value string
|
||||
switch column {
|
||||
case columnRemoteAddr:
|
||||
value = strings.TrimSpace(row.RemoteAddr)
|
||||
case columnHost:
|
||||
value = strings.TrimSpace(row.Host)
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
|
||||
if seen[bucketEpoch] == nil {
|
||||
seen[bucketEpoch] = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := seen[bucketEpoch][value]; ok {
|
||||
continue
|
||||
}
|
||||
seen[bucketEpoch][value] = struct{}{}
|
||||
result = append(result, openFlareAccessLogBucketDimensionRow{BucketEpoch: bucketEpoch, Value: value})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) IPAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if exactRemoteAddr && strings.TrimSpace(filter.RemoteAddr) == "" {
|
||||
return []openFlareAccessLogIPAggregateRow{}, nil
|
||||
}
|
||||
rows := s.filterRecords(filter)
|
||||
aggregates := make(map[string]*openFlareAccessLogIPAggregateRow)
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
if exactRemoteAddr && remoteAddr != strings.TrimSpace(filter.RemoteAddr) {
|
||||
continue
|
||||
}
|
||||
item := aggregates[remoteAddr]
|
||||
if item == nil {
|
||||
item = &openFlareAccessLogIPAggregateRow{RemoteAddr: remoteAddr}
|
||||
aggregates[remoteAddr] = item
|
||||
}
|
||||
item.RequestCount++
|
||||
epoch := row.LoggedAt.UTC().Unix()
|
||||
if epoch > item.LastSeenEpoch {
|
||||
item.LastSeenEpoch = epoch
|
||||
}
|
||||
switch {
|
||||
case row.StatusCode < 400:
|
||||
item.SuccessCount++
|
||||
case row.StatusCode < 500:
|
||||
item.ClientErrorCount++
|
||||
default:
|
||||
item.ServerErrorCount++
|
||||
}
|
||||
}
|
||||
result := make([]openFlareAccessLogIPAggregateRow, 0, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
result = append(result, *item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) IPSummaries(_ context.Context, filter model.OpenFlareAccessLogQuery, _ time.Time) ([]openFlareAccessLogIPSummaryRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
type aggregate struct {
|
||||
RemoteAddr string
|
||||
Region string
|
||||
RegionEpoch int64
|
||||
TotalRequests int64
|
||||
Success2xxCount int64
|
||||
BytesReceived int64
|
||||
BytesSent int64
|
||||
LastSeenEpoch int64
|
||||
}
|
||||
aggregates := make(map[string]*aggregate)
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
item := aggregates[remoteAddr]
|
||||
if item == nil {
|
||||
item = &aggregate{RemoteAddr: remoteAddr}
|
||||
aggregates[remoteAddr] = item
|
||||
}
|
||||
item.TotalRequests++
|
||||
if row.StatusCode >= 200 && row.StatusCode < 300 {
|
||||
item.Success2xxCount++
|
||||
}
|
||||
item.BytesReceived += row.RequestLength
|
||||
item.BytesSent += row.BytesSent
|
||||
epoch := row.LoggedAt.UTC().Unix()
|
||||
if epoch > item.LastSeenEpoch {
|
||||
item.LastSeenEpoch = epoch
|
||||
}
|
||||
if epoch >= item.RegionEpoch {
|
||||
item.RegionEpoch = epoch
|
||||
item.Region = strings.TrimSpace(row.Region)
|
||||
}
|
||||
}
|
||||
summaryRows := make([]*model.OpenFlareAccessLogIPSummaryRow, 0, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
ratio := 0.0
|
||||
if item.TotalRequests > 0 {
|
||||
ratio = float64(item.Success2xxCount) / float64(item.TotalRequests)
|
||||
}
|
||||
summaryRows = append(summaryRows, &model.OpenFlareAccessLogIPSummaryRow{
|
||||
RemoteAddr: item.RemoteAddr,
|
||||
Region: item.Region,
|
||||
TotalRequests: item.TotalRequests,
|
||||
Success2xxCount: item.Success2xxCount,
|
||||
SuccessRatio: ratio,
|
||||
BytesReceived: item.BytesReceived,
|
||||
BytesSent: item.BytesSent,
|
||||
RecentRequests: 0,
|
||||
LastSeenEpoch: item.LastSeenEpoch,
|
||||
})
|
||||
}
|
||||
sortOpenFlareAccessLogIPSummaryRows(summaryRows, filter.SortBy, filter.SortOrder)
|
||||
if filter.PageSize > 0 {
|
||||
start, end := openFlareAccessLogPaginateBounds(len(summaryRows), filter.Page, filter.PageSize)
|
||||
summaryRows = summaryRows[start:end]
|
||||
}
|
||||
result := make([]openFlareAccessLogIPSummaryRow, len(summaryRows))
|
||||
for index, item := range summaryRows {
|
||||
result[index] = openFlareAccessLogIPSummaryRow{
|
||||
RemoteAddr: item.RemoteAddr,
|
||||
Region: item.Region,
|
||||
TotalRequests: item.TotalRequests,
|
||||
Success2xxCount: item.Success2xxCount,
|
||||
SuccessRatio: item.SuccessRatio,
|
||||
BytesReceived: item.BytesReceived,
|
||||
BytesSent: item.BytesSent,
|
||||
RecentRequests: 0,
|
||||
LastSeenEpoch: item.LastSeenEpoch,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) CountIPSummaries(_ context.Context, filter model.OpenFlareAccessLogQuery) (int64, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
seen := make(map[string]struct{})
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
seen[remoteAddr] = struct{}{}
|
||||
}
|
||||
return int64(len(seen)), nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) WAFIPAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
aggregates := make(map[string]*openFlareAccessLogWAFIPAggregateRow)
|
||||
order := make([]string, 0)
|
||||
for _, row := range rows {
|
||||
remoteAddr := strings.TrimSpace(row.RemoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
item := aggregates[remoteAddr]
|
||||
if item == nil {
|
||||
item = &openFlareAccessLogWAFIPAggregateRow{
|
||||
RemoteAddr: remoteAddr,
|
||||
StatusCounts: make(map[int]int64),
|
||||
}
|
||||
aggregates[remoteAddr] = item
|
||||
order = append(order, remoteAddr)
|
||||
}
|
||||
item.RequestCount++
|
||||
item.StatusCounts[row.StatusCode]++
|
||||
if row.StatusCode == http.StatusNotFound {
|
||||
item.Status404Count++
|
||||
}
|
||||
if row.StatusCode >= 400 && row.StatusCode < 500 {
|
||||
item.ClientErrorCount++
|
||||
}
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
item.ServerErrorCount++
|
||||
}
|
||||
if memoryAccessLogHostIsIPLiteral(row.Host) {
|
||||
item.IPHostCount++
|
||||
}
|
||||
epoch := row.LoggedAt.UTC().Unix()
|
||||
if epoch > item.LastSeenEpoch {
|
||||
item.LastSeenEpoch = epoch
|
||||
}
|
||||
}
|
||||
result := make([]openFlareAccessLogWAFIPAggregateRow, 0, len(order))
|
||||
for _, remoteAddr := range order {
|
||||
if item := aggregates[remoteAddr]; item != nil {
|
||||
result = append(result, *item)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) IPTrend(_ context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
aggregates := make(map[int64]int64)
|
||||
for _, row := range rows {
|
||||
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
|
||||
aggregates[bucketEpoch]++
|
||||
}
|
||||
result := make([]openFlareAccessLogIPTrendRow, 0, len(aggregates))
|
||||
for bucketEpoch, count := range aggregates {
|
||||
result = append(result, openFlareAccessLogIPTrendRow{BucketEpoch: bucketEpoch, RequestCount: count})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].BucketEpoch < result[j].BucketEpoch })
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) DeleteAll(_ context.Context) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := int64(len(s.records))
|
||||
s.records = nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) DeleteBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cutoff = cutoff.UTC()
|
||||
remaining := make([]*model.OpenFlareAccessLog, 0, len(s.records))
|
||||
var deleted int64
|
||||
for _, row := range s.records {
|
||||
if row.LoggedAt.Before(cutoff) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.records = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) DeleteByNodeBefore(_ context.Context, nodeID string, before time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
before = before.UTC()
|
||||
remaining := make([]*model.OpenFlareAccessLog, 0, len(s.records))
|
||||
var deleted int64
|
||||
for _, row := range s.records {
|
||||
if row.NodeID == nodeID && row.LoggedAt.Before(before) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.records = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) TrafficSummary(_ context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
ips := make(map[string]struct{})
|
||||
nodes := make(map[string]struct{})
|
||||
var summary model.OpenFlareAccessLogTrafficSummary
|
||||
for _, row := range rows {
|
||||
summary.RequestCount++
|
||||
summary.BytesSent += row.BytesSent
|
||||
summary.RequestLength += row.RequestLength
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
summary.ErrorCount++
|
||||
}
|
||||
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
|
||||
ips[ip] = struct{}{}
|
||||
}
|
||||
if id := strings.TrimSpace(row.NodeID); id != "" {
|
||||
nodes[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
summary.UniqueIPCount = int64(len(ips))
|
||||
summary.NodeCount = int64(len(nodes))
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) ValueCounts(_ context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
col := strings.TrimSpace(strings.ToLower(column))
|
||||
switch col {
|
||||
case accessLogColumnStatusCode, accessLogColumnHost, accessLogColumnPath, accessLogColumnRemoteAddr, accessLogColumnUserAgent:
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
rows := s.filterRecords(filter)
|
||||
counts := make(map[string]int64)
|
||||
for _, row := range rows {
|
||||
var value string
|
||||
switch col {
|
||||
case accessLogColumnStatusCode:
|
||||
value = strconv.Itoa(row.StatusCode)
|
||||
case accessLogColumnHost:
|
||||
value = strings.TrimSpace(row.Host)
|
||||
case accessLogColumnPath:
|
||||
value = strings.TrimSpace(row.Path)
|
||||
case accessLogColumnRemoteAddr:
|
||||
value = strings.TrimSpace(row.RemoteAddr)
|
||||
case accessLogColumnUserAgent:
|
||||
value = strings.TrimSpace(row.UserAgent)
|
||||
}
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
counts[value]++
|
||||
}
|
||||
result := make([]model.OpenFlareAccessLogValueCount, 0, len(counts))
|
||||
for value, count := range counts {
|
||||
result = append(result, model.OpenFlareAccessLogValueCount{Value: value, Count: count})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].Count == result[j].Count {
|
||||
return result[i].Value < result[j].Value
|
||||
}
|
||||
return result[i].Count > result[j].Count
|
||||
})
|
||||
if limit > 0 && len(result) > limit {
|
||||
result = result[:limit]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) NodeAggregates(_ context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := s.filterRecords(filter)
|
||||
type acc struct {
|
||||
model.OpenFlareAccessLogNodeAggregate
|
||||
ips map[string]struct{}
|
||||
}
|
||||
byNode := make(map[string]*acc)
|
||||
for _, row := range rows {
|
||||
id := strings.TrimSpace(row.NodeID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
item := byNode[id]
|
||||
if item == nil {
|
||||
item = &acc{
|
||||
OpenFlareAccessLogNodeAggregate: model.OpenFlareAccessLogNodeAggregate{NodeID: id},
|
||||
ips: make(map[string]struct{}),
|
||||
}
|
||||
byNode[id] = item
|
||||
}
|
||||
item.RequestCount++
|
||||
if row.StatusCode >= http.StatusInternalServerError {
|
||||
item.ErrorCount++
|
||||
}
|
||||
if ip := strings.TrimSpace(row.RemoteAddr); ip != "" {
|
||||
item.ips[ip] = struct{}{}
|
||||
}
|
||||
}
|
||||
result := make([]model.OpenFlareAccessLogNodeAggregate, 0, len(byNode))
|
||||
for _, item := range byNode {
|
||||
item.UniqueIPCount = int64(len(item.ips))
|
||||
result = append(result, item.OpenFlareAccessLogNodeAggregate)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].RequestCount == result[j].RequestCount {
|
||||
return result[i].NodeID < result[j].NodeID
|
||||
}
|
||||
return result[i].RequestCount > result[j].RequestCount
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *memoryAccessLogStore) filterRecords(query model.OpenFlareAccessLogQuery) []*model.OpenFlareAccessLog {
|
||||
result := make([]*model.OpenFlareAccessLog, 0, len(s.records))
|
||||
for _, row := range s.records {
|
||||
if !memoryAccessLogMatches(row, query) {
|
||||
continue
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func memoryAccessLogMatches(row *model.OpenFlareAccessLog, query model.OpenFlareAccessLogQuery) bool {
|
||||
if row == nil {
|
||||
return false
|
||||
}
|
||||
if trimmed := strings.TrimSpace(query.NodeID); trimmed != "" && row.NodeID != trimmed {
|
||||
return false
|
||||
}
|
||||
if trimmed := strings.TrimSpace(query.RemoteAddr); trimmed != "" && !strings.HasPrefix(strings.TrimSpace(row.RemoteAddr), trimmed) {
|
||||
return false
|
||||
}
|
||||
if len(query.Hosts) > 0 {
|
||||
rowHost := strings.ToLower(strings.TrimSpace(row.Host))
|
||||
matched := false
|
||||
for _, host := range query.Hosts {
|
||||
if strings.ToLower(strings.TrimSpace(host)) == rowHost {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return false
|
||||
}
|
||||
} else if trimmed := strings.TrimSpace(query.Host); trimmed != "" && !strings.HasPrefix(strings.TrimSpace(row.Host), trimmed) {
|
||||
return false
|
||||
}
|
||||
if trimmed := strings.TrimSpace(query.Path); trimmed != "" && !strings.HasPrefix(strings.TrimSpace(row.Path), trimmed) {
|
||||
return false
|
||||
}
|
||||
if !query.Since.IsZero() && row.LoggedAt.Before(query.Since) {
|
||||
return false
|
||||
}
|
||||
if !query.Until.IsZero() && !row.LoggedAt.Before(query.Until) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func memoryAccessLogHostIsIPLiteral(value string) bool {
|
||||
host := strings.TrimSpace(value)
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
if parsedHost, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = parsedHost
|
||||
}
|
||||
host = strings.Trim(host, "[]")
|
||||
_, err := netip.ParseAddr(host)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func memoryAccessLogBucketEpoch(loggedAt time.Time, bucketSeconds int64) int64 {
|
||||
if bucketSeconds <= 0 {
|
||||
bucketSeconds = 180
|
||||
}
|
||||
epoch := loggedAt.UTC().Unix()
|
||||
return (epoch / bucketSeconds) * bucketSeconds
|
||||
}
|
||||
|
||||
func cloneAccessLogSlice(rows []*model.OpenFlareAccessLog) []*model.OpenFlareAccessLog {
|
||||
result := make([]*model.OpenFlareAccessLog, len(rows))
|
||||
for index, row := range rows {
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
copyRecord := *row
|
||||
result[index] = ©Record
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func sortOpenFlareAccessLogRows(items []*model.OpenFlareAccessLog, sortBy string, sortOrder string) {
|
||||
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
var compare int
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "status_code":
|
||||
compare = left.StatusCode - right.StatusCode
|
||||
case columnRemoteAddr:
|
||||
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
|
||||
case columnHost:
|
||||
compare = strings.Compare(left.Host, right.Host)
|
||||
case "path":
|
||||
compare = strings.Compare(left.Path, right.Path)
|
||||
default:
|
||||
compare = openFlareAccessLogCompareInt64(left.LoggedAt.Unix(), right.LoggedAt.Unix())
|
||||
}
|
||||
if compare == 0 {
|
||||
compare = openFlareAccessLogCompareInt64(left.LoggedAt.Unix(), right.LoggedAt.Unix())
|
||||
}
|
||||
if compare == 0 {
|
||||
compare = openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(left.ID), openFlareAccessLogUintToInt64(right.ID))
|
||||
}
|
||||
if desc {
|
||||
return compare > 0
|
||||
}
|
||||
return compare < 0
|
||||
})
|
||||
}
|
||||
@@ -6,21 +6,60 @@ package repository
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// accessLogTestDBSeq 保证每个测试获得独立的 sqlite 内存库（cache=shared 下同名 DSN 复用同一库）。
|
||||
var accessLogTestDBSeq int64
|
||||
|
||||
func setupOpenFlareAccessLogTestEnvironment(t *testing.T) (context.Context, func()) {
|
||||
t.Helper()
|
||||
store := NewMemoryAccessLogStore()
|
||||
reset := SetAccessLogStoreForTest(store)
|
||||
return context.Background(), func() {
|
||||
reset()
|
||||
dsn := fmt.Sprintf("file:repo-access-log-test-%d?mode=memory&cache=shared", atomic.AddInt64(&accessLogTestDBSeq, 1))
|
||||
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
|
||||
DisableForeignKeyConstraintWhenMigrating: true,
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gdb.AutoMigrate(&analyticsmodel.NodeAccessLog{}))
|
||||
db.SetDB(gdb)
|
||||
|
||||
logstore.ResetForTest()
|
||||
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
|
||||
if key == "log_database" {
|
||||
return "sqlite", nil
|
||||
}
|
||||
return "", nil
|
||||
})
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
|
||||
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{})
|
||||
|
||||
ctx := context.Background()
|
||||
store, err := logstore.Active(ctx)
|
||||
require.NoError(t, err)
|
||||
// 写入入口只入队；测试环境立即 flush，保证后续查询可见。
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
|
||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||
_ = store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs)
|
||||
},
|
||||
})
|
||||
return ctx, func() {
|
||||
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
|
||||
logstore.ResetForTest()
|
||||
db.SetDB(nil)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +92,7 @@ func TestListOpenFlareAccessLogsPaginated(t *testing.T) {
|
||||
require.NoError(t, InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{record}))
|
||||
}
|
||||
|
||||
// 0-based 分页与 CH ListNodeAccessLogs 一致：page=1 size=5 → OFFSET 5 → /path-05..09。
|
||||
query := model.OpenFlareAccessLogQuery{
|
||||
NodeID: "node-page",
|
||||
Since: now.Add(-24 * time.Hour),
|
||||
@@ -81,6 +121,7 @@ func TestCountOpenFlareAccessLogs(t *testing.T) {
|
||||
totalRecords, totalIPs, _, err := CountOpenFlareAccessLogs(ctx, query)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(5), totalRecords)
|
||||
// GORM 与 CH 一致：distinct IP 排除空 remote_addr（CH uniqExactIf(remote_addr, remote_addr != '')）。
|
||||
assert.Equal(t, int64(3), totalIPs)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// Hook setters are process-global; keep these tests serial.
|
||||
|
||||
func TestObservabilityInsertHooksAreInvoked(t *testing.T) {
|
||||
var gotSnapshot analyticsmodel.NodeMetricSnapshot
|
||||
SetObservabilityInsertHooks(ObservabilityInsertHooks{
|
||||
QueueMetricSnapshot: func(s analyticsmodel.NodeMetricSnapshot) {
|
||||
gotSnapshot = s
|
||||
},
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
SetObservabilityInsertHooks(ObservabilityInsertHooks{})
|
||||
})
|
||||
|
||||
record := &model.OpenFlareMetricSnapshot{
|
||||
NodeID: "node-1",
|
||||
CapturedAt: time.Unix(100, 0).UTC(),
|
||||
}
|
||||
if err := (clickhouseObservabilityStore{}).InsertMetricSnapshot(context.Background(), record); err != nil {
|
||||
t.Fatalf("InsertMetricSnapshot error = %v", err)
|
||||
}
|
||||
if gotSnapshot.NodeID != "node-1" {
|
||||
t.Fatalf("hook node id = %q, want node-1", gotSnapshot.NodeID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccessLogInsertHooksAreInvoked(t *testing.T) {
|
||||
var got []analyticsmodel.NodeAccessLog
|
||||
SetAccessLogInsertHooks(AccessLogInsertHooks{
|
||||
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
|
||||
got = append([]analyticsmodel.NodeAccessLog(nil), logs...)
|
||||
},
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
SetAccessLogInsertHooks(AccessLogInsertHooks{})
|
||||
})
|
||||
|
||||
records := []*model.OpenFlareAccessLog{
|
||||
{NodeID: "n1", Path: "/a"},
|
||||
{NodeID: "n1", Path: "/b"},
|
||||
}
|
||||
if err := (clickhouseAccessLogStore{}).InsertBatch(context.Background(), records); err != nil {
|
||||
t.Fatalf("InsertBatch error = %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("hook logs = %d, want 2", len(got))
|
||||
}
|
||||
if got[0].Path != "/a" || got[1].Path != "/b" {
|
||||
t.Fatalf("hook paths = %q/%q, want /a /b", got[0].Path, got[1].Path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsertHooksNoopWhenUnset(t *testing.T) {
|
||||
SetObservabilityInsertHooks(ObservabilityInsertHooks{})
|
||||
SetAccessLogInsertHooks(AccessLogInsertHooks{})
|
||||
|
||||
if err := (clickhouseObservabilityStore{}).InsertMetricSnapshot(context.Background(), &model.OpenFlareMetricSnapshot{NodeID: "x"}); err != nil {
|
||||
t.Fatalf("InsertMetricSnapshot with nil hook error = %v", err)
|
||||
}
|
||||
if err := (clickhouseAccessLogStore{}).InsertBatch(context.Background(), []*model.OpenFlareAccessLog{{NodeID: "x"}}); err != nil {
|
||||
t.Fatalf("InsertBatch with nil hook error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,9 @@ import (
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository/logstore"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -25,6 +27,9 @@ const (
|
||||
openFlareHealthSeverityWarning = "warning"
|
||||
openFlareHealthSeverityCritical = "critical"
|
||||
openFlareHealthEventMessageMaxLen = 4096
|
||||
|
||||
// logStoreNameClickHouse 与 logstore 内部 dbNameClickHouse 取值一致。
|
||||
logStoreNameClickHouse = "clickhouse"
|
||||
)
|
||||
|
||||
// OpenFlareHealthEventInput describes a desired active health event for reconciliation.
|
||||
@@ -51,43 +56,68 @@ func isMissingTableError(err error) bool {
|
||||
|
||||
// InsertOpenFlareMetricSnapshot inserts a metric snapshot into ClickHouse.
|
||||
func InsertOpenFlareMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
|
||||
return currentObservabilityStore().InsertMetricSnapshot(ctx, record)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertMetricSnapshot(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareEdgeHealth inserts an L2 edge health snapshot into ClickHouse.
|
||||
func InsertOpenFlareEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
|
||||
return currentObservabilityStore().InsertEdgeHealth(ctx, record)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertEdgeHealth(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareNodeObservationFrps inserts an FRPS observation into ClickHouse.
|
||||
func InsertOpenFlareNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
|
||||
return currentObservabilityStore().InsertNodeObservationFrps(ctx, record)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertNodeObservationFrps(ctx, record)
|
||||
}
|
||||
|
||||
// InsertOpenFlareNodeObservationFrpc inserts an FRPC observation into ClickHouse.
|
||||
func InsertOpenFlareNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
|
||||
return currentObservabilityStore().InsertNodeObservationFrpc(ctx, record)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Observability.InsertNodeObservationFrpc(ctx, record)
|
||||
}
|
||||
|
||||
// ListOpenFlareMetricSnapshotsSince returns metric snapshots since the given time.
|
||||
func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
return currentObservabilityStore().ListMetricSnapshots(ctx, nodeID, since, limit)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListMetricSnapshots(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareLatestMetricSnapshotsSince returns the latest metric snapshot per node.
|
||||
// Prefer ClickHouse LIMIT 1 BY; on CH unavailability fall back to store list + reduce.
|
||||
// The ClickHouse LIMIT 1 BY fast path is used only when ClickHouse is the ACTIVE log
|
||||
// database; otherwise the request goes straight to the active log store (PG/SQLite),
|
||||
// avoiding stale reads of the previous CH store after a migration.
|
||||
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
rows, err := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
if err == nil {
|
||||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||||
active, err := logstore.ActiveDatabase(ctx)
|
||||
if err == nil && active == logStoreNameClickHouse {
|
||||
rows, err := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
if err == nil {
|
||||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||||
}
|
||||
}
|
||||
// Fallback for unit tests (memory store) and environments without ClickHouse.
|
||||
// Routes through the active log store (CH unavailable or PG/SQLite active).
|
||||
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
|
||||
if listErr != nil {
|
||||
return nil, err
|
||||
return nil, listErr
|
||||
}
|
||||
return openFlareLatestMetricSnapshots(all), nil
|
||||
}
|
||||
@@ -110,13 +140,37 @@ func openFlareLatestMetricSnapshots(snapshots []*model.OpenFlareMetricSnapshot)
|
||||
return result
|
||||
}
|
||||
|
||||
// fromAnalyticsNodeMetricSnapshots converts analytics rows back to the business model.
|
||||
func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot) []*model.OpenFlareMetricSnapshot {
|
||||
result := make([]*model.OpenFlareMetricSnapshot, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareMetricSnapshot{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
CPUUsagePercent: row.CPUUsagePercent,
|
||||
MemoryUsedBytes: row.MemoryUsedBytes,
|
||||
MemoryTotalBytes: row.MemoryTotalBytes,
|
||||
StorageUsedBytes: row.StorageUsedBytes,
|
||||
StorageTotalBytes: row.StorageTotalBytes,
|
||||
DiskReadBytes: row.DiskReadBytes,
|
||||
DiskWriteBytes: row.DiskWriteBytes,
|
||||
NetworkRxBytes: row.NetworkRxBytes,
|
||||
NetworkTxBytes: row.NetworkTxBytes,
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ListOpenFlareTrafficHourlySince returns hourly traffic rollup rows since the given time.
|
||||
// Source: of_access_log_hourly (M5).
|
||||
// CH 读 of_access_log_hourly rollup；PG/SQLite 经 logstore 从 of_node_access_logs 实时聚合。
|
||||
func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareTrafficHourly, error) {
|
||||
rows, err := analyticsrepo.ListNodeTrafficHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.Observability.ListTrafficHourly(ctx, nodeID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -133,12 +187,15 @@ func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since t
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListOpenFlareAccessLogHourlySince returns of_access_log_hourly rows since the given time.
|
||||
// ListOpenFlareAccessLogHourlySince returns hourly access-log rollups since the given time,
|
||||
// read through logstore's active backend (ClickHouse of_access_log_hourly rollup;
|
||||
// PostgreSQL/SQLite real-time aggregation from of_node_access_logs).
|
||||
func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareAccessLogHourly, error) {
|
||||
rows, err := analyticsrepo.ListAccessLogHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.Observability.ListAccessLogHourly(ctx, nodeID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -159,10 +216,11 @@ func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since
|
||||
|
||||
// ListOpenFlareMetricHourlySince returns hourly metric aggregates since the given time.
|
||||
func ListOpenFlareMetricHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricHourly, error) {
|
||||
rows, err := analyticsrepo.ListNodeMetricHourly(ctx, analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
})
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.Observability.ListMetricHourly(ctx, nodeID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -223,42 +281,74 @@ func ListOpenFlareHealthEvents(ctx context.Context, nodeID string, activeOnly bo
|
||||
|
||||
// DeleteOpenFlareMetricSnapshotsBefore deletes metric snapshots captured before cutoff.
|
||||
func DeleteOpenFlareMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteMetricSnapshotsBefore(ctx, cutoff)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteMetricSnapshotsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareMetricSnapshots deletes all metric snapshots.
|
||||
func DeleteAllOpenFlareMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllMetricSnapshots(ctx)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllMetricSnapshots(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareEdgeHealthBefore deletes edge health rows captured before cutoff.
|
||||
func DeleteOpenFlareEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteEdgeHealthBefore(ctx, cutoff)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareEdgeHealth deletes all edge health snapshots.
|
||||
func DeleteAllOpenFlareEdgeHealth(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllEdgeHealth(ctx)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllEdgeHealth(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteNodeObservationFrpsBefore(ctx, cutoff)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationFrps deletes all FRPS observations.
|
||||
func DeleteAllOpenFlareNodeObservationFrps(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllNodeObservationFrps(ctx)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllNodeObservationFrps(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareNodeObservationFrpcBefore deletes FRPC observations captured before cutoff.
|
||||
func DeleteOpenFlareNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return currentObservabilityStore().DeleteNodeObservationFrpcBefore(ctx, cutoff)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
// DeleteAllOpenFlareNodeObservationFrpc deletes all FRPC observations.
|
||||
func DeleteAllOpenFlareNodeObservationFrpc(ctx context.Context) (int64, error) {
|
||||
return currentObservabilityStore().DeleteAllNodeObservationFrpc(ctx)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Observability.DeleteAllNodeObservationFrpc(ctx)
|
||||
}
|
||||
|
||||
// DeleteOpenFlareHealthEventsByNodeID deletes all health events for a node.
|
||||
@@ -523,15 +613,27 @@ func marshalOpenFlareHealthMetadata(value map[string]string) string {
|
||||
|
||||
// ListOpenFlareEdgeHealth returns L2 edge health snapshots.
|
||||
func ListOpenFlareEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
|
||||
return currentObservabilityStore().ListEdgeHealth(ctx, nodeID, since, limit)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListEdgeHealth(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareNodeObservationFrpc returns frpc observations.
|
||||
func ListOpenFlareNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
|
||||
return currentObservabilityStore().ListNodeObservationFrpc(ctx, nodeID, since, limit)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListNodeObservationFrpc(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
// ListOpenFlareNodeObservationFrps returns frps observations.
|
||||
func ListOpenFlareNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
|
||||
return currentObservabilityStore().ListNodeObservationFrps(ctx, nodeID, since, limit)
|
||||
s, err := logstore.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Observability.ListNodeObservationFrps(ctx, nodeID, since, limit)
|
||||
}
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||||
)
|
||||
|
||||
// ObservabilityInsertHooks queues observability rows for async ClickHouse write.
|
||||
// Wired from openflare/chwriter.Init so model never imports the apps layer.
|
||||
type ObservabilityInsertHooks struct {
|
||||
QueueMetricSnapshot func(analyticsmodel.NodeMetricSnapshot)
|
||||
QueueEdgeHealth func(analyticsmodel.NodeEdgeHealth)
|
||||
QueueFrpsObservation func(analyticsmodel.NodeObsFrps)
|
||||
QueueFrpcObservation func(analyticsmodel.NodeObsFrpc)
|
||||
}
|
||||
|
||||
var (
|
||||
observabilityInsertHooksMu sync.RWMutex
|
||||
observabilityInsertHooks ObservabilityInsertHooks
|
||||
)
|
||||
|
||||
// SetObservabilityInsertHooks registers async queue callbacks for observability inserts.
|
||||
func SetObservabilityInsertHooks(hooks ObservabilityInsertHooks) {
|
||||
observabilityInsertHooksMu.Lock()
|
||||
observabilityInsertHooks = hooks
|
||||
observabilityInsertHooksMu.Unlock()
|
||||
}
|
||||
|
||||
func currentObservabilityInsertHooks() ObservabilityInsertHooks {
|
||||
observabilityInsertHooksMu.RLock()
|
||||
defer observabilityInsertHooksMu.RUnlock()
|
||||
return observabilityInsertHooks
|
||||
}
|
||||
|
||||
type observabilityStore interface {
|
||||
InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error
|
||||
ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error)
|
||||
DeleteAllMetricSnapshots(ctx context.Context) (int64, error)
|
||||
DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
|
||||
InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error
|
||||
ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error)
|
||||
DeleteAllEdgeHealth(ctx context.Context) (int64, error)
|
||||
DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
|
||||
InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error
|
||||
ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error)
|
||||
DeleteAllNodeObservationFrps(ctx context.Context) (int64, error)
|
||||
DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
|
||||
InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error
|
||||
ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error)
|
||||
DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error)
|
||||
DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
}
|
||||
|
||||
var (
|
||||
observabilityStoreMu sync.RWMutex
|
||||
observabilityStoreHolder observabilityStore
|
||||
)
|
||||
|
||||
func currentObservabilityStore() observabilityStore {
|
||||
observabilityStoreMu.RLock()
|
||||
defer observabilityStoreMu.RUnlock()
|
||||
if observabilityStoreHolder != nil {
|
||||
return observabilityStoreHolder
|
||||
}
|
||||
return clickhouseObservabilityStore{}
|
||||
}
|
||||
|
||||
// SetObservabilityStoreForTest swaps the observability store implementation for unit tests.
|
||||
func SetObservabilityStoreForTest(store observabilityStore) func() {
|
||||
observabilityStoreMu.Lock()
|
||||
previous := observabilityStoreHolder
|
||||
observabilityStoreHolder = store
|
||||
observabilityStoreMu.Unlock()
|
||||
return func() {
|
||||
observabilityStoreMu.Lock()
|
||||
observabilityStoreHolder = previous
|
||||
observabilityStoreMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// NewMemoryObservabilityStore returns an in-memory observability store for unit tests.
|
||||
func NewMemoryObservabilityStore() observabilityStore {
|
||||
return &memoryObservabilityStore{}
|
||||
}
|
||||
|
||||
type clickhouseObservabilityStore struct{}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertMetricSnapshot(_ context.Context, record *model.OpenFlareMetricSnapshot) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if hook := currentObservabilityInsertHooks().QueueMetricSnapshot; hook != nil {
|
||||
hook(toAnalyticsNodeMetricSnapshot(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
rows, err := analyticsrepo.ListNodeMetricSnapshots(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteAllMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeMetricSnapshots(ctx)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
const edgeHealthStatusUnknown = "unknown"
|
||||
|
||||
func normalizeEdgeHealthStatus(status string) string {
|
||||
status = strings.TrimSpace(status)
|
||||
if status == "" {
|
||||
return edgeHealthStatusUnknown
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertEdgeHealth(_ context.Context, record *model.OpenFlareEdgeHealth) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if hook := currentObservabilityInsertHooks().QueueEdgeHealth; hook != nil {
|
||||
hook(toAnalyticsNodeEdgeHealth(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
|
||||
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeEdgeHealth(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertNodeObservationFrps(_ context.Context, record *model.OpenFlareNodeObservationFrps) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if hook := currentObservabilityInsertHooks().QueueFrpsObservation; hook != nil {
|
||||
hook(toAnalyticsNodeObsFrps(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
|
||||
rows, err := analyticsrepo.ListNodeObsFrps(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeObsFrps(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteAllNodeObservationFrps(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeObsFrps(ctx)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeObsFrpsBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) InsertNodeObservationFrpc(_ context.Context, record *model.OpenFlareNodeObservationFrpc) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
if hook := currentObservabilityInsertHooks().QueueFrpcObservation; hook != nil {
|
||||
hook(toAnalyticsNodeObsFrpc(record))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
|
||||
rows, err := analyticsrepo.ListNodeObsFrpc(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fromAnalyticsNodeObsFrpc(rows), nil
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error) {
|
||||
return analyticsrepo.DeleteAllNodeObsFrpc(ctx)
|
||||
}
|
||||
|
||||
func (clickhouseObservabilityStore) DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
return analyticsrepo.DeleteNodeObsFrpcBefore(ctx, cutoff)
|
||||
}
|
||||
|
||||
func toNodeObservabilityFilter(nodeID string, since time.Time, limit int) analyticsrepo.NodeObservabilityFilter {
|
||||
return analyticsrepo.NodeObservabilityFilter{
|
||||
NodeID: nodeID,
|
||||
Since: since,
|
||||
Limit: limit,
|
||||
}
|
||||
}
|
||||
|
||||
func toAnalyticsNodeMetricSnapshot(record *model.OpenFlareMetricSnapshot) analyticsmodel.NodeMetricSnapshot {
|
||||
return analyticsmodel.NodeMetricSnapshot{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
CapturedAt: record.CapturedAt,
|
||||
CPUUsagePercent: record.CPUUsagePercent,
|
||||
MemoryUsedBytes: record.MemoryUsedBytes,
|
||||
MemoryTotalBytes: record.MemoryTotalBytes,
|
||||
StorageUsedBytes: record.StorageUsedBytes,
|
||||
StorageTotalBytes: record.StorageTotalBytes,
|
||||
DiskReadBytes: record.DiskReadBytes,
|
||||
DiskWriteBytes: record.DiskWriteBytes,
|
||||
NetworkRxBytes: record.NetworkRxBytes,
|
||||
NetworkTxBytes: record.NetworkTxBytes,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot) []*model.OpenFlareMetricSnapshot {
|
||||
result := make([]*model.OpenFlareMetricSnapshot, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareMetricSnapshot{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
CPUUsagePercent: row.CPUUsagePercent,
|
||||
MemoryUsedBytes: row.MemoryUsedBytes,
|
||||
MemoryTotalBytes: row.MemoryTotalBytes,
|
||||
StorageUsedBytes: row.StorageUsedBytes,
|
||||
StorageTotalBytes: row.StorageTotalBytes,
|
||||
DiskReadBytes: row.DiskReadBytes,
|
||||
DiskWriteBytes: row.DiskWriteBytes,
|
||||
NetworkRxBytes: row.NetworkRxBytes,
|
||||
NetworkTxBytes: row.NetworkTxBytes,
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toAnalyticsNodeEdgeHealth(record *model.OpenFlareEdgeHealth) analyticsmodel.NodeEdgeHealth {
|
||||
return analyticsmodel.NodeEdgeHealth{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
CapturedAt: record.CapturedAt,
|
||||
Status: normalizeEdgeHealthStatus(record.Status),
|
||||
Connections: record.Connections,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func fromAnalyticsNodeEdgeHealth(rows []analyticsmodel.NodeEdgeHealth) []*model.OpenFlareEdgeHealth {
|
||||
result := make([]*model.OpenFlareEdgeHealth, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareEdgeHealth{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
Status: normalizeEdgeHealthStatus(row.Status),
|
||||
Connections: row.Connections,
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toAnalyticsNodeObsFrps(record *model.OpenFlareNodeObservationFrps) analyticsmodel.NodeObsFrps {
|
||||
return analyticsmodel.NodeObsFrps{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
CapturedAt: record.CapturedAt,
|
||||
FrpsConnections: openFlareObservabilityIntToInt32(record.FrpsConnections),
|
||||
FrpsProxyCount: openFlareObservabilityIntToInt32(record.FrpsProxyCount),
|
||||
FrpsClientCount: openFlareObservabilityIntToInt32(record.FrpsClientCount),
|
||||
FrpsProxies: record.FrpsProxies,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func fromAnalyticsNodeObsFrps(rows []analyticsmodel.NodeObsFrps) []*model.OpenFlareNodeObservationFrps {
|
||||
result := make([]*model.OpenFlareNodeObservationFrps, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareNodeObservationFrps{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
FrpsConnections: int(row.FrpsConnections),
|
||||
FrpsProxyCount: int(row.FrpsProxyCount),
|
||||
FrpsClientCount: int(row.FrpsClientCount),
|
||||
FrpsProxies: row.FrpsProxies,
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func toAnalyticsNodeObsFrpc(record *model.OpenFlareNodeObservationFrpc) analyticsmodel.NodeObsFrpc {
|
||||
return analyticsmodel.NodeObsFrpc{
|
||||
ID: uint64(record.ID),
|
||||
NodeID: record.NodeID,
|
||||
CapturedAt: record.CapturedAt,
|
||||
TunnelStatus: record.TunnelStatus,
|
||||
ConnectedRelaysCount: openFlareObservabilityIntToInt32(record.ConnectedRelaysCount),
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func openFlareObservabilityIntToInt32(value int) int32 {
|
||||
switch {
|
||||
case value > math.MaxInt32:
|
||||
return math.MaxInt32
|
||||
case value < math.MinInt32:
|
||||
return math.MinInt32
|
||||
default:
|
||||
return int32(value)
|
||||
}
|
||||
}
|
||||
|
||||
func fromAnalyticsNodeObsFrpc(rows []analyticsmodel.NodeObsFrpc) []*model.OpenFlareNodeObservationFrpc {
|
||||
result := make([]*model.OpenFlareNodeObservationFrpc, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = &model.OpenFlareNodeObservationFrpc{
|
||||
ID: uint(row.ID),
|
||||
NodeID: row.NodeID,
|
||||
CapturedAt: row.CapturedAt,
|
||||
TunnelStatus: row.TunnelStatus,
|
||||
ConnectedRelaysCount: int(row.ConnectedRelaysCount),
|
||||
CreatedAt: row.CreatedAt,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,409 +0,0 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/model"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/persistence/idgen"
|
||||
)
|
||||
|
||||
type memoryObservabilityStore struct {
|
||||
mu sync.RWMutex
|
||||
metricSnapshots []*model.OpenFlareMetricSnapshot
|
||||
edgeHealth []*model.OpenFlareEdgeHealth
|
||||
frpsObs []*model.OpenFlareNodeObservationFrps
|
||||
frpcObs []*model.OpenFlareNodeObservationFrpc
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) InsertMetricSnapshot(_ context.Context, record *model.OpenFlareMetricSnapshot) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
copyRecord := cloneOpenFlareMetricSnapshot(record)
|
||||
if memoryMetricSnapshotExists(s.metricSnapshots, copyRecord.NodeID, copyRecord.CapturedAt) {
|
||||
return nil
|
||||
}
|
||||
s.metricSnapshots = append(s.metricSnapshots, copyRecord)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) ListMetricSnapshots(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := memoryFilterMetricSnapshots(s.metricSnapshots, nodeID, since)
|
||||
sortOpenFlareMetricSnapshots(rows)
|
||||
return memoryLimitObservabilityRows(rows, limit), nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteAllMetricSnapshots(_ context.Context) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := int64(len(s.metricSnapshots))
|
||||
s.metricSnapshots = nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteMetricSnapshotsBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cutoff = cutoff.UTC()
|
||||
remaining := make([]*model.OpenFlareMetricSnapshot, 0, len(s.metricSnapshots))
|
||||
var deleted int64
|
||||
for _, row := range s.metricSnapshots {
|
||||
if row.CapturedAt.Before(cutoff) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.metricSnapshots = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) InsertEdgeHealth(_ context.Context, record *model.OpenFlareEdgeHealth) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.edgeHealth = append(s.edgeHealth, cloneOpenFlareEdgeHealth(record))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) ListEdgeHealth(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := memoryFilterEdgeHealth(s.edgeHealth, nodeID, since)
|
||||
sortOpenFlareEdgeHealth(rows)
|
||||
return memoryLimitObservabilityRows(rows, limit), nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteAllEdgeHealth(_ context.Context) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := int64(len(s.edgeHealth))
|
||||
s.edgeHealth = nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteEdgeHealthBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cutoff = cutoff.UTC()
|
||||
remaining := make([]*model.OpenFlareEdgeHealth, 0, len(s.edgeHealth))
|
||||
var deleted int64
|
||||
for _, row := range s.edgeHealth {
|
||||
if row.CapturedAt.Before(cutoff) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.edgeHealth = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) InsertNodeObservationFrps(_ context.Context, record *model.OpenFlareNodeObservationFrps) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.frpsObs = append(s.frpsObs, cloneOpenFlareNodeObservationFrps(record))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) ListNodeObservationFrps(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := memoryFilterFrpsObservations(s.frpsObs, nodeID, since)
|
||||
sortOpenFlareNodeObservationFrps(rows)
|
||||
return memoryLimitObservabilityRows(rows, limit), nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteAllNodeObservationFrps(_ context.Context) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := int64(len(s.frpsObs))
|
||||
s.frpsObs = nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteNodeObservationFrpsBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cutoff = cutoff.UTC()
|
||||
remaining := make([]*model.OpenFlareNodeObservationFrps, 0, len(s.frpsObs))
|
||||
var deleted int64
|
||||
for _, row := range s.frpsObs {
|
||||
if row.CapturedAt.Before(cutoff) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.frpsObs = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) InsertNodeObservationFrpc(_ context.Context, record *model.OpenFlareNodeObservationFrpc) error {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.frpcObs = append(s.frpcObs, cloneOpenFlareNodeObservationFrpc(record))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) ListNodeObservationFrpc(_ context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
rows := memoryFilterFrpcObservations(s.frpcObs, nodeID, since)
|
||||
sortOpenFlareNodeObservationFrpc(rows)
|
||||
return memoryLimitObservabilityRows(rows, limit), nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteAllNodeObservationFrpc(_ context.Context) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := int64(len(s.frpcObs))
|
||||
s.frpcObs = nil
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *memoryObservabilityStore) DeleteNodeObservationFrpcBefore(_ context.Context, cutoff time.Time) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cutoff = cutoff.UTC()
|
||||
remaining := make([]*model.OpenFlareNodeObservationFrpc, 0, len(s.frpcObs))
|
||||
var deleted int64
|
||||
for _, row := range s.frpcObs {
|
||||
if row.CapturedAt.Before(cutoff) {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
remaining = append(remaining, row)
|
||||
}
|
||||
s.frpcObs = remaining
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
func memoryFilterMetricSnapshots(rows []*model.OpenFlareMetricSnapshot, nodeID string, since time.Time) []*model.OpenFlareMetricSnapshot {
|
||||
result := make([]*model.OpenFlareMetricSnapshot, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
|
||||
continue
|
||||
}
|
||||
if !since.IsZero() && row.CapturedAt.Before(since) {
|
||||
continue
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func memoryFilterEdgeHealth(rows []*model.OpenFlareEdgeHealth, nodeID string, since time.Time) []*model.OpenFlareEdgeHealth {
|
||||
result := make([]*model.OpenFlareEdgeHealth, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
|
||||
continue
|
||||
}
|
||||
if !since.IsZero() && row.CapturedAt.Before(since) {
|
||||
continue
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func memoryFilterFrpsObservations(rows []*model.OpenFlareNodeObservationFrps, nodeID string, since time.Time) []*model.OpenFlareNodeObservationFrps {
|
||||
result := make([]*model.OpenFlareNodeObservationFrps, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
|
||||
continue
|
||||
}
|
||||
if !since.IsZero() && row.CapturedAt.Before(since) {
|
||||
continue
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func memoryFilterFrpcObservations(rows []*model.OpenFlareNodeObservationFrpc, nodeID string, since time.Time) []*model.OpenFlareNodeObservationFrpc {
|
||||
result := make([]*model.OpenFlareNodeObservationFrpc, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !memoryObservabilityMatchesNodeID(row.NodeID, nodeID) {
|
||||
continue
|
||||
}
|
||||
if !since.IsZero() && row.CapturedAt.Before(since) {
|
||||
continue
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func memoryObservabilityMatchesNodeID(rowNodeID string, nodeID string) bool {
|
||||
trimmed := strings.TrimSpace(nodeID)
|
||||
if trimmed == "" {
|
||||
return true
|
||||
}
|
||||
return rowNodeID == trimmed
|
||||
}
|
||||
|
||||
func memoryMetricSnapshotExists(rows []*model.OpenFlareMetricSnapshot, nodeID string, capturedAt time.Time) bool {
|
||||
capturedAt = capturedAt.UTC()
|
||||
for _, row := range rows {
|
||||
if row.NodeID == nodeID && row.CapturedAt.UTC().Equal(capturedAt) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sortOpenFlareMetricSnapshots(items []*model.OpenFlareMetricSnapshot) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
|
||||
return compare > 0
|
||||
}
|
||||
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
|
||||
})
|
||||
}
|
||||
|
||||
func sortOpenFlareEdgeHealth(items []*model.OpenFlareEdgeHealth) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
|
||||
return compare > 0
|
||||
}
|
||||
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
|
||||
})
|
||||
}
|
||||
|
||||
func sortOpenFlareNodeObservationFrps(items []*model.OpenFlareNodeObservationFrps) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
|
||||
return compare > 0
|
||||
}
|
||||
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
|
||||
})
|
||||
}
|
||||
|
||||
func sortOpenFlareNodeObservationFrpc(items []*model.OpenFlareNodeObservationFrpc) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
left := items[i]
|
||||
right := items[j]
|
||||
if left == nil || right == nil {
|
||||
return left != nil
|
||||
}
|
||||
if compare := openFlareAccessLogCompareInt64(left.CapturedAt.Unix(), right.CapturedAt.Unix()); compare != 0 {
|
||||
return compare > 0
|
||||
}
|
||||
return openFlareAccessLogCompareInt64(openFlareAccessLogUintToInt64(uint64(left.ID)), openFlareAccessLogUintToInt64(uint64(right.ID))) > 0
|
||||
})
|
||||
}
|
||||
|
||||
func memoryLimitObservabilityRows[T any](rows []T, limit int) []T {
|
||||
if limit <= 0 || len(rows) <= limit {
|
||||
result := make([]T, len(rows))
|
||||
copy(result, rows)
|
||||
return result
|
||||
}
|
||||
result := make([]T, limit)
|
||||
copy(result, rows[:limit])
|
||||
return result
|
||||
}
|
||||
|
||||
func cloneOpenFlareMetricSnapshot(record *model.OpenFlareMetricSnapshot) *model.OpenFlareMetricSnapshot {
|
||||
copyRecord := *record
|
||||
if copyRecord.ID == 0 {
|
||||
copyRecord.ID = uint(idgen.NextUint64ID())
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if copyRecord.CreatedAt.IsZero() {
|
||||
copyRecord.CreatedAt = now
|
||||
}
|
||||
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
|
||||
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
|
||||
return ©Record
|
||||
}
|
||||
|
||||
func cloneOpenFlareEdgeHealth(record *model.OpenFlareEdgeHealth) *model.OpenFlareEdgeHealth {
|
||||
copyRecord := *record
|
||||
if copyRecord.ID == 0 {
|
||||
copyRecord.ID = uint(idgen.NextUint64ID())
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if copyRecord.CreatedAt.IsZero() {
|
||||
copyRecord.CreatedAt = now
|
||||
}
|
||||
if copyRecord.CapturedAt.IsZero() {
|
||||
copyRecord.CapturedAt = now
|
||||
}
|
||||
if strings.TrimSpace(copyRecord.Status) == "" {
|
||||
copyRecord.Status = edgeHealthStatusUnknown
|
||||
}
|
||||
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
|
||||
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
|
||||
return ©Record
|
||||
}
|
||||
|
||||
func cloneOpenFlareNodeObservationFrps(record *model.OpenFlareNodeObservationFrps) *model.OpenFlareNodeObservationFrps {
|
||||
copyRecord := *record
|
||||
if copyRecord.ID == 0 {
|
||||
copyRecord.ID = uint(idgen.NextUint64ID())
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if copyRecord.CreatedAt.IsZero() {
|
||||
copyRecord.CreatedAt = now
|
||||
}
|
||||
if copyRecord.CapturedAt.IsZero() {
|
||||
copyRecord.CapturedAt = now
|
||||
}
|
||||
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
|
||||
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
|
||||
return ©Record
|
||||
}
|
||||
|
||||
func cloneOpenFlareNodeObservationFrpc(record *model.OpenFlareNodeObservationFrpc) *model.OpenFlareNodeObservationFrpc {
|
||||
copyRecord := *record
|
||||
if copyRecord.ID == 0 {
|
||||
copyRecord.ID = uint(idgen.NextUint64ID())
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if copyRecord.CreatedAt.IsZero() {
|
||||
copyRecord.CreatedAt = now
|
||||
}
|
||||
if copyRecord.CapturedAt.IsZero() {
|
||||
copyRecord.CapturedAt = now
|
||||
}
|
||||
copyRecord.CapturedAt = copyRecord.CapturedAt.UTC()
|
||||
copyRecord.CreatedAt = copyRecord.CreatedAt.UTC()
|
||||
return ©Record
|
||||
}
|
||||
Reference in New Issue
Block a user