mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-09-30 14:26:36 +08:00
ryan
f530cd4025
- Count/节点访问日志统计改为单次扫描聚合，WAF 按 IP 聚合由三次扫描合并为两次 - IPSummaries 归属地改为取过滤窗口内最新记录（对齐 ClickHouse argMax 口径）， 子查询带窗口条件，可分区裁剪并命中索引 - 新增 goose 迁移：of_node_access_logs (logged_at DESC, id DESC) 前导索引与 lower(trim(host)) 表达式索引，加速列表排序与主机过滤 - 过期日志清理先按数据校验直接 DROP 完全过期整月分区，再对边界月逐行删除； 启动时兜底预建当月及未来 2 个月分区，跨月停机重启后首次写入不再报 "no partition of relation found"
742 lines
28 KiB
Go
742 lines
28 KiB
Go
// Copyright 2026 Arctel.net
|
||
// SPDX-License-Identifier: Apache-2.0
|
||
|
||
package logstore
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"math"
|
||
"time"
|
||
|
||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||
"github.com/Rain-kl/Wavelet/internal/model"
|
||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||
analyticsrepo "github.com/Rain-kl/Wavelet/internal/repository/analytics"
|
||
)
|
||
|
||
// clickhouseLogStore 实现 AccessLogStore / ObservabilityStore / StatusStore，
|
||
// 逐方法委托 analyticsrepo（CH 原生 batch 写入，零性能损耗）。
|
||
// UserAccessLogStore 由 clickhouseUserAccessLogStore 实现（List/Count 方法名已被
|
||
// AccessLogStore 占用，Go 不允许同名不同签名方法）。
|
||
type clickhouseLogStore struct {
|
||
// skipFreeze 为 true 时跳过迁移冻结检查（仅迁移目标 store 使用）。
|
||
skipFreeze bool
|
||
}
|
||
|
||
func newClickHouseStore() *clickhouseLogStore { return &clickhouseLogStore{} }
|
||
|
||
// 编译期断言。
|
||
var (
|
||
_ AccessLogStore = (*clickhouseLogStore)(nil)
|
||
_ ObservabilityStore = (*clickhouseLogStore)(nil)
|
||
_ StatusStore = (*clickhouseLogStore)(nil)
|
||
_ UserAccessLogStore = (*clickhouseUserAccessLogStore)(nil)
|
||
)
|
||
|
||
func chConnErr() error {
|
||
if !db.ChConnReady() {
|
||
return errors.New("clickhouse connection is not initialized")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ensureWritable 迁移冻结期拒绝写入。
|
||
func (s *clickhouseLogStore) ensureWritable(ctx context.Context) error {
|
||
if !s.skipFreeze && Migrating(ctx) {
|
||
return ErrMigrating
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ---- AccessLogStore ----
|
||
|
||
// InsertBatch 节点访问日志写入入口：冻结检查后经 hook 入队（异步），不直接落库。
|
||
func (s *clickhouseLogStore) InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
rows := make([]analyticsmodel.NodeAccessLog, 0, len(records))
|
||
for _, r := range records {
|
||
if r == nil {
|
||
continue
|
||
}
|
||
rows = append(rows, toAnalyticsNodeAccessLog(r))
|
||
}
|
||
if h := currentAccessLogHooks().QueueNodeAccessLogs; h != nil {
|
||
h(rows)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// BatchInsertNodeAccessLogs 是 batchwriter flush 目标：CH 原生批量写入。
|
||
func (s *clickhouseLogStore) BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
return analyticsrepo.BatchInsertNodeAccessLogs(ctx, rows)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
|
||
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return fromAnalyticsNodeAccessLogs(rows), nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
|
||
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||
}
|
||
|
||
func (s *clickhouseLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
|
||
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make([]*model.OpenFlareAccessLogRegionCount, len(rows))
|
||
for i, r := range rows {
|
||
out[i] = &model.OpenFlareAccessLogRegionCount{Region: r.Region, Count: r.Count}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) BucketAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error) {
|
||
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) CountBuckets(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
|
||
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) BucketDimensions(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error) {
|
||
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, bucketSeconds)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) IPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error) {
|
||
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), exactRemoteAddr)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) IPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error) {
|
||
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), recentSince)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) CountIPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, error) {
|
||
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||
}
|
||
|
||
func (s *clickhouseLogStore) WAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error) {
|
||
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||
}
|
||
|
||
func (s *clickhouseLogStore) IPTrend(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error) {
|
||
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) TrafficSummary(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
|
||
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||
if err != nil {
|
||
return model.OpenFlareAccessLogTrafficSummary{}, err
|
||
}
|
||
return model.OpenFlareAccessLogTrafficSummary{
|
||
RequestCount: row.RequestCount,
|
||
ErrorCount: row.ErrorCount,
|
||
UniqueIPCount: row.UniqueIPCount,
|
||
BytesSent: row.BytesSent,
|
||
RequestLength: row.RequestLength,
|
||
NodeCount: row.NodeCount,
|
||
}, nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) ValueCounts(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
|
||
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make([]model.OpenFlareAccessLogValueCount, len(rows))
|
||
for i, r := range rows {
|
||
out[i] = model.OpenFlareAccessLogValueCount{Value: r.Value, Count: r.Count}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) NodeAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
|
||
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make([]model.OpenFlareAccessLogNodeAggregate, len(rows))
|
||
for i, r := range rows {
|
||
out[i] = model.OpenFlareAccessLogNodeAggregate{NodeID: r.NodeID, RequestCount: r.RequestCount, ErrorCount: r.ErrorCount, UniqueIPCount: r.UniqueIPCount}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
|
||
}
|
||
|
||
// ListForMigration 按 id 升序分页读取（迁移复制用）：直接查询 CH 原生表。
|
||
func (s *clickhouseLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error) {
|
||
if err := chConnErr(); err != nil {
|
||
return nil, err
|
||
}
|
||
rows, err := db.ChConn.Query(ctx, `
|
||
SELECT `+analyticsmodel.NodeAccessLog{}.InsertColumns()+`
|
||
FROM `+analyticsmodel.NodeAccessLog{}.TableName()+`
|
||
WHERE id > ?
|
||
ORDER BY id ASC
|
||
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list node access logs for migration: %w", err)
|
||
}
|
||
defer func() { _ = rows.Close() }()
|
||
var result []analyticsmodel.NodeAccessLog
|
||
for rows.Next() {
|
||
var item analyticsmodel.NodeAccessLog
|
||
if err := rows.Scan(
|
||
&item.ID,
|
||
&item.NodeID,
|
||
&item.LoggedAt,
|
||
&item.RemoteAddr,
|
||
&item.Region,
|
||
&item.Host,
|
||
&item.Path,
|
||
&item.UserAgent,
|
||
&item.CacheStatus,
|
||
&item.StatusCode,
|
||
&item.BytesSent,
|
||
&item.RequestLength,
|
||
&item.RequestTimeMs,
|
||
&item.CreatedAt,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scan node access log row: %w", err)
|
||
}
|
||
item.LoggedAt = item.LoggedAt.UTC()
|
||
item.CreatedAt = item.CreatedAt.UTC()
|
||
result = append(result, item)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// ---- ObservabilityStore ----
|
||
|
||
// InsertMetricSnapshot 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||
func (s *clickhouseLogStore) InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
|
||
if record == nil {
|
||
return nil
|
||
}
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
if h := currentObservabilityHooks().QueueMetricSnapshot; h != nil {
|
||
h(toAnalyticsNodeMetricSnapshot(record))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
|
||
rows, err := analyticsrepo.ListNodeMetricSnapshots(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return fromAnalyticsNodeMetricSnapshots(rows), nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteAllMetricSnapshots(ctx context.Context) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteAllNodeMetricSnapshots(ctx)
|
||
}
|
||
|
||
// ListTrafficHourly 委托 analyticsrepo 读 of_access_log_hourly rollup（M5 口径，UV 恒 0）。
|
||
func (s *clickhouseLogStore) ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error) {
|
||
return analyticsrepo.ListNodeTrafficHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||
}
|
||
|
||
// ListAccessLogHourly 委托 analyticsrepo 读 of_access_log_hourly rollup。
|
||
func (s *clickhouseLogStore) ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error) {
|
||
return analyticsrepo.ListAccessLogHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||
}
|
||
|
||
// ListMetricHourly 委托 analyticsrepo ListNodeMetricHourly：rollup 覆盖窗口时读
|
||
// of_node_metric_capacity_hourly，否则按 mergeNodeMetricHourlyPreferRollup 合并 raw 兜底。
|
||
func (s *clickhouseLogStore) ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error) {
|
||
return analyticsrepo.ListNodeMetricHourly(ctx, toNodeObservabilitySince(nodeID, since))
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
|
||
}
|
||
|
||
// BatchInsertNodeMetricSnapshots 是 batchwriter flush 目标：CH 原生批量写入。
|
||
func (s *clickhouseLogStore) BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
return analyticsrepo.BatchInsertNodeMetricSnapshots(ctx, rows)
|
||
}
|
||
|
||
// InsertEdgeHealth 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||
func (s *clickhouseLogStore) InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
|
||
if record == nil {
|
||
return nil
|
||
}
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
if h := currentObservabilityHooks().QueueEdgeHealth; h != nil {
|
||
h(toAnalyticsNodeEdgeHealth(record))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
|
||
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return fromAnalyticsNodeEdgeHealths(rows), nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
|
||
}
|
||
|
||
// BatchInsertNodeEdgeHealth 是 batchwriter flush 目标：CH 原生批量写入。
|
||
func (s *clickhouseLogStore) BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
return analyticsrepo.BatchInsertNodeEdgeHealth(ctx, rows)
|
||
}
|
||
|
||
// InsertNodeObservationFrps 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||
func (s *clickhouseLogStore) InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
|
||
if record == nil {
|
||
return nil
|
||
}
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
if h := currentObservabilityHooks().QueueNodeObsFrps; h != nil {
|
||
h(toAnalyticsNodeObsFrps(record))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
|
||
rows, err := analyticsrepo.ListNodeObsFrps(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return fromAnalyticsNodeObsFrps(rows), nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteAllNodeObservationFrps(ctx context.Context) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteAllNodeObsFrps(ctx)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteNodeObsFrpsBefore(ctx, cutoff)
|
||
}
|
||
|
||
// BatchInsertNodeObsFrps 是 batchwriter flush 目标：CH 原生批量写入。
|
||
func (s *clickhouseLogStore) BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
return analyticsrepo.BatchInsertNodeObsFrps(ctx, rows)
|
||
}
|
||
|
||
// InsertNodeObservationFrpc 写入入口：冻结检查 + 经 hook 入队（异步），不直接落库。
|
||
func (s *clickhouseLogStore) InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
|
||
if record == nil {
|
||
return nil
|
||
}
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
if h := currentObservabilityHooks().QueueNodeObsFrpc; h != nil {
|
||
h(toAnalyticsNodeObsFrpc(record))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
|
||
rows, err := analyticsrepo.ListNodeObsFrpc(ctx, toNodeObservabilityFilter(nodeID, since, limit))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return fromAnalyticsNodeObsFrpc(rows), nil
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteAllNodeObsFrpc(ctx)
|
||
}
|
||
|
||
func (s *clickhouseLogStore) DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteNodeObsFrpcBefore(ctx, cutoff)
|
||
}
|
||
|
||
// MigrationRange 返回 of_node_access_logs.logged_at 的最小/最大值（空表返回零值）。
|
||
func (s *clickhouseLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
|
||
return chMigrationRange(ctx, analyticsmodel.NodeAccessLog{}.TableName(), "logged_at")
|
||
}
|
||
|
||
// EnsurePartitions 是 CH 分支 no-op（CH 无 PG 式分区）。
|
||
func (s *clickhouseLogStore) EnsurePartitions(_ context.Context, _, _ time.Time) error {
|
||
return nil
|
||
}
|
||
|
||
// DropEmptyPartitions 是 CH 分支 no-op（CH 分区随数据删除自动消失，无独立分区表）。
|
||
func (s *clickhouseLogStore) DropEmptyPartitions(_ context.Context, _ time.Time) error {
|
||
return nil
|
||
}
|
||
|
||
// DropExpiredPartitions 是 CH 分支 no-op（CH 无 PG 式分区，retention 仍走 DeleteBefore）。
|
||
func (s *clickhouseLogStore) DropExpiredPartitions(_ context.Context, _ time.Time) error {
|
||
return nil
|
||
}
|
||
|
||
// chMigrationRange 查询 CH 表时间列 MIN/MAX；空表（NULL）返回零值。
|
||
func chMigrationRange(ctx context.Context, table, column string) (time.Time, time.Time, error) {
|
||
if err := chConnErr(); err != nil {
|
||
return time.Time{}, time.Time{}, err
|
||
}
|
||
var minTime, maxTime *time.Time
|
||
if err := db.ChConn.QueryRow(ctx,
|
||
"SELECT min("+column+"), max("+column+") FROM "+table,
|
||
).Scan(&minTime, &maxTime); err != nil {
|
||
return time.Time{}, time.Time{}, fmt.Errorf("query migration range %s: %w", table, err)
|
||
}
|
||
if minTime == nil || maxTime == nil {
|
||
return time.Time{}, time.Time{}, nil
|
||
}
|
||
return minTime.UTC(), maxTime.UTC(), nil
|
||
}
|
||
|
||
// BatchInsertNodeObsFrpc 是 batchwriter flush 目标：CH 原生批量写入。
|
||
func (s *clickhouseLogStore) BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
return analyticsrepo.BatchInsertNodeObsFrpc(ctx, rows)
|
||
}
|
||
|
||
// ListMetricSnapshotsForMigration 按 id 升序分页读取（迁移复制用）。
|
||
func (s *clickhouseLogStore) ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||
return chListForMigration(ctx, afterID, limit,
|
||
analyticsmodel.NodeMetricSnapshot{}.TableName(),
|
||
analyticsmodel.NodeMetricSnapshot{}.InsertColumns(),
|
||
func(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||
var result []analyticsmodel.NodeMetricSnapshot
|
||
for rows.Next() {
|
||
var item analyticsmodel.NodeMetricSnapshot
|
||
if err := rows.Scan(
|
||
&item.ID,
|
||
&item.NodeID,
|
||
&item.CapturedAt,
|
||
&item.CPUUsagePercent,
|
||
&item.MemoryUsedBytes,
|
||
&item.MemoryTotalBytes,
|
||
&item.StorageUsedBytes,
|
||
&item.StorageTotalBytes,
|
||
&item.DiskReadBytes,
|
||
&item.DiskWriteBytes,
|
||
&item.NetworkRxBytes,
|
||
&item.NetworkTxBytes,
|
||
&item.CreatedAt,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
|
||
}
|
||
item.CapturedAt = item.CapturedAt.UTC()
|
||
item.CreatedAt = item.CreatedAt.UTC()
|
||
result = append(result, item)
|
||
}
|
||
return result, nil
|
||
})
|
||
}
|
||
|
||
// chObsRow 迁移读取共用的双字段观测行（字符串状态 + 数值计数）：
|
||
// edge_health（status/connections）与 obs_frpc（tunnel_status/connected_relays_count）同形状。
|
||
type chObsRow struct {
|
||
ID uint64
|
||
NodeID string
|
||
CapturedAt time.Time
|
||
Status string
|
||
Count int64
|
||
CreatedAt time.Time
|
||
}
|
||
|
||
// countToInt32 将观测计数转为 int32（防御溢出；观测计数远小于 int32 上限）。
|
||
func countToInt32(v int64) int32 {
|
||
if v > math.MaxInt32 {
|
||
return math.MaxInt32
|
||
}
|
||
if v < math.MinInt32 {
|
||
return math.MinInt32
|
||
}
|
||
return int32(v)
|
||
}
|
||
|
||
// scanChObsRow 扫描 chObsRow（含 UTC 归一化）。
|
||
func scanChObsRow(rows driver.Rows) ([]chObsRow, error) {
|
||
var result []chObsRow
|
||
for rows.Next() {
|
||
var item chObsRow
|
||
if err := rows.Scan(
|
||
&item.ID,
|
||
&item.NodeID,
|
||
&item.CapturedAt,
|
||
&item.Status,
|
||
&item.Count,
|
||
&item.CreatedAt,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scan observation row: %w", err)
|
||
}
|
||
item.CapturedAt = item.CapturedAt.UTC()
|
||
item.CreatedAt = item.CreatedAt.UTC()
|
||
result = append(result, item)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// ListEdgeHealthForMigration 按 id 升序分页读取（迁移复制用）。
|
||
func (s *clickhouseLogStore) ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error) {
|
||
rows, err := chListForMigration(ctx, afterID, limit,
|
||
analyticsmodel.NodeEdgeHealth{}.TableName(),
|
||
analyticsmodel.NodeEdgeHealth{}.InsertColumns(),
|
||
scanChObsRow)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make([]analyticsmodel.NodeEdgeHealth, len(rows))
|
||
for i, r := range rows {
|
||
out[i] = analyticsmodel.NodeEdgeHealth{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, Status: r.Status, Connections: r.Count, CreatedAt: r.CreatedAt}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// ListNodeObsFrpsForMigration 按 id 升序分页读取（迁移复制用）。
|
||
func (s *clickhouseLogStore) ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error) {
|
||
return chListForMigration(ctx, afterID, limit,
|
||
analyticsmodel.NodeObsFrps{}.TableName(),
|
||
analyticsmodel.NodeObsFrps{}.InsertColumns(),
|
||
func(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
|
||
var result []analyticsmodel.NodeObsFrps
|
||
for rows.Next() {
|
||
var item analyticsmodel.NodeObsFrps
|
||
if err := rows.Scan(
|
||
&item.ID,
|
||
&item.NodeID,
|
||
&item.CapturedAt,
|
||
&item.FrpsConnections,
|
||
&item.FrpsProxyCount,
|
||
&item.FrpsClientCount,
|
||
&item.FrpsProxies,
|
||
&item.CreatedAt,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scan node frps observation row: %w", err)
|
||
}
|
||
item.CapturedAt = item.CapturedAt.UTC()
|
||
item.CreatedAt = item.CreatedAt.UTC()
|
||
result = append(result, item)
|
||
}
|
||
return result, nil
|
||
})
|
||
}
|
||
|
||
// ListNodeObsFrpcForMigration 按 id 升序分页读取（迁移复制用）。
|
||
func (s *clickhouseLogStore) ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error) {
|
||
rows, err := chListForMigration(ctx, afterID, limit,
|
||
analyticsmodel.NodeObsFrpc{}.TableName(),
|
||
analyticsmodel.NodeObsFrpc{}.InsertColumns(),
|
||
scanChObsRow)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make([]analyticsmodel.NodeObsFrpc, len(rows))
|
||
for i, r := range rows {
|
||
out[i] = analyticsmodel.NodeObsFrpc{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, TunnelStatus: r.Status, ConnectedRelaysCount: countToInt32(r.Count), CreatedAt: r.CreatedAt}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// chListForMigration 执行按 id 升序分页的 CH 原生表查询，并交给 scanner 扫描。
|
||
func chListForMigration[T any](ctx context.Context, afterID uint64, limit int, table, columns string, scanner func(driver.Rows) ([]T, error)) ([]T, error) {
|
||
if err := chConnErr(); err != nil {
|
||
return nil, err
|
||
}
|
||
rows, err := db.ChConn.Query(ctx, `
|
||
SELECT `+columns+`
|
||
FROM `+table+`
|
||
WHERE id > ?
|
||
ORDER BY id ASC
|
||
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("list %s for migration: %w", table, err)
|
||
}
|
||
defer func() { _ = rows.Close() }()
|
||
return scanner(rows)
|
||
}
|
||
|
||
// ---- StatusStore ----
|
||
|
||
// ActiveDatabase 返回当前日志主库名（CH 分支固定 clickhouse）。
|
||
func (s *clickhouseLogStore) ActiveDatabase(_ context.Context) (string, error) {
|
||
return dbNameClickHouse, nil
|
||
}
|
||
|
||
// ClickHouseOperationalStats 委托 analyticsrepo 汇总 CH 运行状态。
|
||
func (s *clickhouseLogStore) ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) {
|
||
return analyticsrepo.GetClickHouseOperationalStats(ctx)
|
||
}
|
||
|
||
// ---- UserAccessLogStore ----
|
||
|
||
// clickhouseUserAccessLogStore 实现 UserAccessLogStore。clickhouseLogStore 已占用
|
||
// List/Count 方法名（AccessLogStore 接口），Go 不允许同名不同签名方法，故用户访问日志
|
||
// 用独立类型嵌入同一 clickhouseLogStore（与 userAccessLogGormStore 同构），复用 ensureWritable。
|
||
type clickhouseUserAccessLogStore struct {
|
||
*clickhouseLogStore
|
||
}
|
||
|
||
func newClickHouseUserAccessLogStore() *clickhouseUserAccessLogStore {
|
||
return &clickhouseUserAccessLogStore{clickhouseLogStore: newClickHouseStore()}
|
||
}
|
||
|
||
// BatchInsert 是 batchwriter flush 目标：CH 原生批量写入；冻结期拒绝写入，空批次直接返回。
|
||
func (s *clickhouseUserAccessLogStore) BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
|
||
if len(logs) == 0 {
|
||
return nil
|
||
}
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return err
|
||
}
|
||
return analyticsrepo.BatchInsert(ctx, logs)
|
||
}
|
||
|
||
// DeleteAll 清空全部用户访问日志（TRUNCATE 语义，迁移「覆盖目标库已有日志」幂等前提用）。
|
||
func (s *clickhouseUserAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
|
||
if err := s.ensureWritable(ctx); err != nil {
|
||
return 0, err
|
||
}
|
||
return analyticsrepo.DeleteAllUserAccessLogs(ctx)
|
||
}
|
||
|
||
// ListForMigration 按 id 升序分页读取（迁移复制用）。
|
||
func (s *clickhouseUserAccessLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error) {
|
||
return chListForMigration(ctx, afterID, limit,
|
||
analyticsmodel.UserAccessLog{}.TableName(),
|
||
analyticsmodel.UserAccessLog{}.InsertColumns(),
|
||
func(rows driver.Rows) ([]analyticsmodel.UserAccessLog, error) {
|
||
var result []analyticsmodel.UserAccessLog
|
||
for rows.Next() {
|
||
var item analyticsmodel.UserAccessLog
|
||
if err := rows.Scan(
|
||
&item.ID,
|
||
&item.UserID,
|
||
&item.Path,
|
||
&item.Method,
|
||
&item.IP,
|
||
&item.UserAgent,
|
||
&item.Headers,
|
||
&item.Status,
|
||
&item.Latency,
|
||
&item.CreatedAt,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scan user access log row: %w", err)
|
||
}
|
||
item.CreatedAt = item.CreatedAt.UTC()
|
||
result = append(result, item)
|
||
}
|
||
return result, nil
|
||
})
|
||
}
|
||
|
||
// MigrationRange 返回 w_user_access_logs.created_at 的最小/最大值（空表返回零值）。
|
||
func (s *clickhouseUserAccessLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
|
||
return chMigrationRange(ctx, analyticsmodel.UserAccessLog{}.TableName(), "created_at")
|
||
}
|
||
|
||
func (s *clickhouseUserAccessLogStore) Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
|
||
return analyticsrepo.CountAccessLogs(ctx, filter)
|
||
}
|
||
|
||
func (s *clickhouseUserAccessLogStore) List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
||
return analyticsrepo.ListAccessLogs(ctx, filter, page, pageSize)
|
||
}
|
||
|
||
func (s *clickhouseUserAccessLogStore) GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
|
||
return analyticsrepo.GetDailyTrend(ctx, days)
|
||
}
|
||
|
||
func (s *clickhouseUserAccessLogStore) GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
|
||
return analyticsrepo.GetBrowserDistribution(ctx, startTime)
|
||
}
|
||
|
||
func (s *clickhouseUserAccessLogStore) GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
|
||
return analyticsrepo.GetTopActiveUsers(ctx, startTime, limit)
|
||
}
|
||
|
||
// toNodeObservabilityFilter 构造 CH 可观测查询过滤器（limit<=0 表示不限制）。
|
||
func toNodeObservabilityFilter(nodeID string, since time.Time, limit int) analyticsmodel.NodeObservabilityFilter {
|
||
return analyticsmodel.NodeObservabilityFilter{
|
||
NodeID: nodeID,
|
||
Since: since,
|
||
Limit: limit,
|
||
}
|
||
}
|
||
|
||
// toNodeObservabilitySince 构造不带 limit 的可观测查询过滤器
|
||
// （小时级聚合读无需分页，避免传无意义的 0）。
|
||
func toNodeObservabilitySince(nodeID string, since time.Time) analyticsmodel.NodeObservabilityFilter {
|
||
return analyticsmodel.NodeObservabilityFilter{NodeID: nodeID, Since: since}
|
||
}
|