mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 15:26:36 +08:00
refactor(backend): rename OpenFlare directory to lowercase openflare
This commit is contained in:
ryan
@@ -0,0 +1,185 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/column"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
type mockConn struct {
|
||||
batch driver.Batch
|
||||
batchQuery string
|
||||
prepareCalled bool
|
||||
preparedQuery string
|
||||
queries []string
|
||||
queryArgs [][]any
|
||||
queryFn func(ctx context.Context, query string, args ...any) (driver.Rows, error)
|
||||
}
|
||||
|
||||
func (m *mockConn) Contributors() []string { return nil }
|
||||
|
||||
func (m *mockConn) ServerVersion() (*driver.ServerVersion, error) { return nil, nil }
|
||||
|
||||
func (m *mockConn) Select(_ context.Context, _ any, _ string, _ ...any) error { return nil }
|
||||
|
||||
func (m *mockConn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) {
|
||||
m.queries = append(m.queries, query)
|
||||
m.queryArgs = append(m.queryArgs, args)
|
||||
if m.queryFn != nil {
|
||||
return m.queryFn(ctx, query, args...)
|
||||
}
|
||||
return &mockRows{}, nil
|
||||
}
|
||||
|
||||
func (m *mockConn) QueryRow(_ context.Context, _ string, _ ...any) driver.Row { return nil }
|
||||
|
||||
func (m *mockConn) PrepareBatch(_ context.Context, query string, _ ...driver.PrepareBatchOption) (driver.Batch, error) {
|
||||
m.prepareCalled = true
|
||||
m.preparedQuery = query
|
||||
return m.batch, nil
|
||||
}
|
||||
|
||||
func (m *mockConn) Exec(_ context.Context, _ string, _ ...any) error { return nil }
|
||||
|
||||
func (m *mockConn) AsyncInsert(_ context.Context, _ string, _ bool, _ ...any) error { return nil }
|
||||
|
||||
func (m *mockConn) InsertFormat(_ context.Context, _ string, _ string, _ io.Reader) error { return nil }
|
||||
|
||||
func (m *mockConn) QueryFormat(_ context.Context, _ string, _ string, _ ...any) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockConn) Ping(_ context.Context) error { return nil }
|
||||
|
||||
func (m *mockConn) Stats() driver.Stats { return driver.Stats{} }
|
||||
|
||||
func (m *mockConn) Close() error { return nil }
|
||||
|
||||
type mockBatch struct {
|
||||
rows [][]any
|
||||
sendCalled bool
|
||||
}
|
||||
|
||||
func (m *mockBatch) Abort() error { return nil }
|
||||
|
||||
func (m *mockBatch) Append(v ...any) error {
|
||||
m.rows = append(m.rows, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBatch) AppendStruct(_ any) error { return nil }
|
||||
|
||||
func (m *mockBatch) Column(_ int) driver.BatchColumn { return nil }
|
||||
|
||||
func (m *mockBatch) Flush() error { return nil }
|
||||
|
||||
func (m *mockBatch) Send() error {
|
||||
m.sendCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockBatch) IsSent() bool { return m.sendCalled }
|
||||
|
||||
func (m *mockBatch) Rows() int { return len(m.rows) }
|
||||
|
||||
func (m *mockBatch) Columns() []column.Interface { return nil }
|
||||
|
||||
func (m *mockBatch) Close() error { return nil }
|
||||
|
||||
// mockRows is an empty driver.Rows implementation for query-path unit tests.
|
||||
type mockRows struct {
|
||||
index int
|
||||
data [][]any
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *mockRows) Next() bool {
|
||||
if m.err != nil {
|
||||
return false
|
||||
}
|
||||
if m.index >= len(m.data) {
|
||||
return false
|
||||
}
|
||||
m.index++
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *mockRows) Scan(dest ...any) error {
|
||||
if m.err != nil {
|
||||
return m.err
|
||||
}
|
||||
if m.index == 0 || m.index > len(m.data) {
|
||||
return nil
|
||||
}
|
||||
row := m.data[m.index-1]
|
||||
for i := range dest {
|
||||
if i >= len(row) {
|
||||
break
|
||||
}
|
||||
if err := assignMockScanValue(dest[i], row[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockRows) ScanStruct(_ any) error { return nil }
|
||||
|
||||
func (m *mockRows) ColumnTypes() []driver.ColumnType { return nil }
|
||||
|
||||
func (m *mockRows) Totals(_ ...any) error { return nil }
|
||||
|
||||
func (m *mockRows) Columns() []string { return nil }
|
||||
|
||||
func (m *mockRows) Close() error { return nil }
|
||||
|
||||
func (m *mockRows) Err() error { return m.err }
|
||||
|
||||
func (m *mockRows) HasData() bool { return len(m.data) > 0 }
|
||||
|
||||
func assignMockScanValue(dest any, value any) error {
|
||||
switch d := dest.(type) {
|
||||
case *string:
|
||||
if v, ok := value.(string); ok {
|
||||
*d = v
|
||||
}
|
||||
case *uint64:
|
||||
switch v := value.(type) {
|
||||
case uint64:
|
||||
*d = v
|
||||
case int:
|
||||
*d = uint64(v)
|
||||
case int64:
|
||||
*d = uint64(v)
|
||||
}
|
||||
case *int64:
|
||||
switch v := value.(type) {
|
||||
case int64:
|
||||
*d = v
|
||||
case int:
|
||||
*d = int64(v)
|
||||
case uint64:
|
||||
*d = int64(v)
|
||||
}
|
||||
case *float64:
|
||||
switch v := value.(type) {
|
||||
case float64:
|
||||
*d = v
|
||||
case float32:
|
||||
*d = float64(v)
|
||||
case int:
|
||||
*d = float64(v)
|
||||
}
|
||||
case *time.Time:
|
||||
if v, ok := value.(time.Time); ok {
|
||||
*d = v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import "math"
|
||||
|
||||
func safeInt64Count(count uint64) int64 {
|
||||
if count > math.MaxInt64 {
|
||||
return math.MaxInt64
|
||||
}
|
||||
return int64(count)
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSafeInt64Count(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
count uint64
|
||||
want int64
|
||||
}{
|
||||
{name: "zero", count: 0, want: 0},
|
||||
{name: "small", count: 42, want: 42},
|
||||
{name: "max int64", count: math.MaxInt64, want: math.MaxInt64},
|
||||
{name: "overflow clamps", count: math.MaxUint64, want: math.MaxInt64},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := safeInt64Count(tt.count); got != tt.want {
|
||||
t.Fatalf("safeInt64Count(%d) = %d, want %d", tt.count, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// DDL TTL days for analytics tables (must match goose ClickHouse migrations).
|
||||
const (
|
||||
// TableTTLDaysNodeAccessLogs is the of_node_access_logs TTL (90 days).
|
||||
TableTTLDaysNodeAccessLogs = 90
|
||||
// TableTTLDaysNodeMetricSnapshots is the of_node_metric_snapshots TTL (30 days).
|
||||
TableTTLDaysNodeMetricSnapshots = 30
|
||||
// TableTTLDaysNodeObs is the of_node_edge_health / of_node_obs_frps / of_node_obs_frpc TTL (30 days).
|
||||
TableTTLDaysNodeObs = 30
|
||||
// TableTTLDaysUserAccessLogs is the w_user_access_logs TTL (180 days).
|
||||
TableTTLDaysUserAccessLogs = 180
|
||||
)
|
||||
|
||||
const (
|
||||
// CleanupModeTTLMaterialize expires rows via table TTL instead of ALTER DELETE mutations.
|
||||
// This is not a hard delete: deleted_count must stay 0; use EligibleCount as an estimate.
|
||||
CleanupModeTTLMaterialize = "ttl_materialize"
|
||||
// CleanupModeTruncate removes all rows via TRUNCATE TABLE (hard delete).
|
||||
CleanupModeTruncate = "truncate"
|
||||
)
|
||||
|
||||
// CleanupOutcome describes a non-mutation ClickHouse cleanup operation.
|
||||
//
|
||||
// For CleanupModeTruncate:
|
||||
// - DeletedCount and EligibleCount are the rows removed by TRUNCATE.
|
||||
//
|
||||
// For CleanupModeTTLMaterialize:
|
||||
// - DeletedCount is always 0 (MATERIALIZE TTL is async / not a counted hard delete).
|
||||
// - EligibleCount is an estimate of rows already past the table TTL policy (not an
|
||||
// arbitrary user cutoff younger than the DDL TTL).
|
||||
// - TableTTLDays is the DDL TTL used for the estimate and materialize.
|
||||
type CleanupOutcome struct {
|
||||
EligibleCount int64
|
||||
DeletedCount int64
|
||||
Mode string
|
||||
TableTTLDays int
|
||||
}
|
||||
|
||||
func countClickHouseRows(ctx context.Context, conn driver.Conn, countSQL string, countArgs []any) (int64, error) {
|
||||
var count uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count clickhouse rows: %w", err)
|
||||
}
|
||||
return safeInt64Count(count), nil
|
||||
}
|
||||
|
||||
func materializeTableTTL(ctx context.Context, conn driver.Conn, tableName string) error {
|
||||
sql := fmt.Sprintf("ALTER TABLE %s MATERIALIZE TTL", tableName)
|
||||
if err := conn.Exec(ctx, sql); err != nil {
|
||||
return fmt.Errorf("materialize ttl on %s: %w", tableName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tableTTLCutoff returns the UTC instant at which rows become eligible under a fixed day TTL.
|
||||
func tableTTLCutoff(tableTTLDays int, now time.Time) time.Time {
|
||||
if tableTTLDays < 1 {
|
||||
tableTTLDays = 1
|
||||
}
|
||||
return now.UTC().Add(-time.Duration(tableTTLDays) * 24 * time.Hour)
|
||||
}
|
||||
|
||||
// materializeExpiredByTableTTL force-materializes table TTL and estimates rows past that policy.
|
||||
//
|
||||
// countSQL must count only rows older than the table TTL (callers pass tableTTLCutoff args).
|
||||
// Node-scoped filters may be used for the estimate only; MATERIALIZE is always table-global.
|
||||
func materializeExpiredByTableTTL(
|
||||
ctx context.Context,
|
||||
conn driver.Conn,
|
||||
tableName string,
|
||||
tableTTLDays int,
|
||||
countSQL string,
|
||||
countArgs []any,
|
||||
) (CleanupOutcome, error) {
|
||||
outcome := CleanupOutcome{
|
||||
Mode: CleanupModeTTLMaterialize,
|
||||
TableTTLDays: tableTTLDays,
|
||||
}
|
||||
count, err := countClickHouseRows(ctx, conn, countSQL, countArgs)
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
outcome.EligibleCount = count
|
||||
// Always force materialize so ClickHouse applies the DDL TTL policy promptly.
|
||||
// EligibleCount is informational only; MATERIALIZE does not return a deleted row count.
|
||||
if err := materializeTableTTL(ctx, conn, tableName); err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
func truncateClickHouseTable(ctx context.Context, conn driver.Conn, tableName string) (CleanupOutcome, error) {
|
||||
count, err := countClickHouseRows(ctx, conn, "SELECT count() FROM "+tableName, nil)
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
if count == 0 {
|
||||
return CleanupOutcome{Mode: CleanupModeTruncate}, nil
|
||||
}
|
||||
if err := conn.Exec(ctx, "TRUNCATE TABLE "+tableName); err != nil {
|
||||
return CleanupOutcome{}, fmt.Errorf("truncate %s: %w", tableName, err)
|
||||
}
|
||||
return CleanupOutcome{
|
||||
EligibleCount: count,
|
||||
DeletedCount: count,
|
||||
Mode: CleanupModeTruncate,
|
||||
}, nil
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestTableTTLCutoff(t *testing.T) {
|
||||
now := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
|
||||
got := tableTTLCutoff(30, now)
|
||||
assert.Equal(t, now.Add(-30*24*time.Hour), got)
|
||||
|
||||
got = tableTTLCutoff(90, now)
|
||||
assert.Equal(t, now.Add(-90*24*time.Hour), got)
|
||||
|
||||
// Invalid TTL floors to 1 day.
|
||||
got = tableTTLCutoff(0, now)
|
||||
assert.Equal(t, now.Add(-24*time.Hour), got)
|
||||
}
|
||||
|
||||
func TestCleanupModeConstants(t *testing.T) {
|
||||
assert.Equal(t, "ttl_materialize", CleanupModeTTLMaterialize)
|
||||
assert.Equal(t, "truncate", CleanupModeTruncate)
|
||||
}
|
||||
|
||||
func TestTableTTLDaysMatchDDL(t *testing.T) {
|
||||
assert.Equal(t, 90, TableTTLDaysNodeAccessLogs)
|
||||
assert.Equal(t, 30, TableTTLDaysNodeMetricSnapshots)
|
||||
assert.Equal(t, 30, TableTTLDaysNodeObs)
|
||||
assert.Equal(t, 180, TableTTLDaysUserAccessLogs)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/openflare/plugins/server/kernel/runtimeconfig"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
|
||||
// and in-process batch writer queue health.
|
||||
type ClickHouseOperationalStats = analyticsmodel.ClickHouseOperationalStats
|
||||
|
||||
// GetClickHouseOperationalStats returns operational metrics for the configured database.
|
||||
func GetClickHouseOperationalStats(ctx context.Context) (*ClickHouseOperationalStats, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, errors.New("clickhouse native connection is not initialized")
|
||||
}
|
||||
database := runtimeconfig.Get().ClickHouse.Database
|
||||
stats := &ClickHouseOperationalStats{Database: database}
|
||||
|
||||
partsSQL := `
|
||||
SELECT
|
||||
count() AS active_parts,
|
||||
ifNull(sum(rows), 0) AS total_rows
|
||||
FROM system.parts
|
||||
WHERE active AND database = ?`
|
||||
var activeParts, totalRows uint64
|
||||
if err := db.ChConn.QueryRow(ctx, partsSQL, database).Scan(&activeParts, &totalRows); err != nil {
|
||||
return nil, fmt.Errorf("query system.parts: %w", err)
|
||||
}
|
||||
stats.ActiveParts = safeInt64Count(activeParts)
|
||||
stats.TotalRows = safeInt64Count(totalRows)
|
||||
|
||||
mutationsSQL := `
|
||||
SELECT count()
|
||||
FROM system.mutations
|
||||
WHERE is_done = 0 AND database = ?`
|
||||
if err := db.ChConn.QueryRow(ctx, mutationsSQL, database).Scan(&stats.PendingMutations); err != nil {
|
||||
return nil, fmt.Errorf("query system.mutations: %w", err)
|
||||
}
|
||||
|
||||
asyncSQL := `
|
||||
SELECT
|
||||
count() AS queue_entries,
|
||||
ifNull(sum(bytes), 0) AS queue_bytes
|
||||
FROM system.asynchronous_inserts
|
||||
WHERE database = ?`
|
||||
var queueEntries, queueBytes uint64
|
||||
if err := db.ChConn.QueryRow(ctx, asyncSQL, database).Scan(&queueEntries, &queueBytes); err != nil {
|
||||
// Older ClickHouse versions may not expose asynchronous_inserts; treat as optional.
|
||||
stats.AsyncInsertQueue = 0
|
||||
stats.AsyncInsertBytes = 0
|
||||
} else {
|
||||
stats.AsyncInsertQueue = safeInt64Count(queueEntries)
|
||||
stats.AsyncInsertBytes = safeInt64Count(queueBytes)
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// NodeAccessLogRegionCount aggregates access log regions.
|
||||
type NodeAccessLogRegionCount = analyticsmodel.NodeAccessLogRegionCount
|
||||
|
||||
func nodeAccessLogConn() (driver.Conn, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
return db.ChConn, nil
|
||||
}
|
||||
|
||||
// ListNodeAccessLogs returns access logs matching filter.
|
||||
func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]analyticsmodel.NodeAccessLog, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, logged_at, remote_addr, region, host, path, user_agent, cache_status, status_code, bytes_sent, request_length, request_time_ms, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeAccessLogRows(rows)
|
||||
}
|
||||
|
||||
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
|
||||
func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, error) {
|
||||
var result []analyticsmodel.NodeAccessLog
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeAccessLog
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.LoggedAt,
|
||||
&item.RemoteAddr,
|
||||
&item.Region,
|
||||
&item.Host,
|
||||
&item.Path,
|
||||
&item.UserAgent,
|
||||
&item.CacheStatus,
|
||||
&item.StatusCode,
|
||||
&item.BytesSent,
|
||||
&item.RequestLength,
|
||||
&item.RequestTimeMs,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node access log row: %w", err)
|
||||
}
|
||||
item.LoggedAt = item.LoggedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountNodeAccessLogs returns total records, distinct IPs, and total bytes sent matching filter.
|
||||
func CountNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, int64, int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
|
||||
countSQL := fmt.Sprintf(`
|
||||
SELECT
|
||||
count() AS total_records,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS total_ips,
|
||||
sum(bytes_sent) AS total_bytes
|
||||
FROM %s
|
||||
WHERE %s`, tableName, clause)
|
||||
var totalRecords, totalIPs, totalBytes uint64
|
||||
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords, &totalIPs, &totalBytes); err != nil {
|
||||
return 0, 0, 0, fmt.Errorf("count node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalRecords), safeInt64Count(totalIPs), safeInt64Count(totalBytes), nil
|
||||
}
|
||||
|
||||
// RegionCountsNodeAccessLogs returns region counts for a node since a time.
|
||||
func RegionCountsNodeAccessLogs(ctx context.Context, nodeID string, since time.Time, limit int) ([]NodeAccessLogRegionCount, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter := NodeAccessLogFilter{NodeID: nodeID, Since: since}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT trim(region) AS trimmed_region, count() AS count
|
||||
FROM %s
|
||||
WHERE %s AND trim(region) != ''
|
||||
GROUP BY trimmed_region
|
||||
ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
|
||||
if limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("region counts node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogRegionCount
|
||||
for rows.Next() {
|
||||
var (
|
||||
region string
|
||||
count uint64
|
||||
)
|
||||
if err := rows.Scan(®ion, &count); err != nil {
|
||||
return nil, fmt.Errorf("scan region count row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogRegionCount{
|
||||
Region: region,
|
||||
Count: safeInt64Count(count),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
|
||||
type NodeAccessLogTrafficSummary = analyticsmodel.NodeAccessLogTrafficSummary
|
||||
|
||||
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
|
||||
type NodeAccessLogValueCount = analyticsmodel.NodeAccessLogValueCount
|
||||
|
||||
// NodeAccessLogNodeAggregate is per-node traffic over a window.
|
||||
type NodeAccessLogNodeAggregate = analyticsmodel.NodeAccessLogNodeAggregate
|
||||
|
||||
// TrafficSummaryNodeAccessLogs returns request/error/UV/bytes/node counts for the filter.
|
||||
func TrafficSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (NodeAccessLogTrafficSummary, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return NodeAccessLogTrafficSummary{}, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
count() AS request_count,
|
||||
countIf(status_code >= 500) AS error_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length,
|
||||
uniqExactIf(node_id, node_id != '') AS node_count
|
||||
FROM %s
|
||||
WHERE %s`, tableName, clause)
|
||||
var requestCount, errorCount, uniqueIPs, bytesSent, requestLength, nodeCount uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(
|
||||
&requestCount, &errorCount, &uniqueIPs, &bytesSent, &requestLength, &nodeCount,
|
||||
); err != nil {
|
||||
return NodeAccessLogTrafficSummary{}, fmt.Errorf("traffic summary node access logs: %w", err)
|
||||
}
|
||||
return NodeAccessLogTrafficSummary{
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
ErrorCount: safeInt64Count(errorCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPs),
|
||||
BytesSent: safeInt64Count(bytesSent),
|
||||
RequestLength: safeInt64Count(requestLength),
|
||||
NodeCount: safeInt64Count(nodeCount),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
|
||||
// Allowed columns: status_code, host, path, remote_addr, user_agent.
|
||||
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
col := strings.TrimSpace(strings.ToLower(column))
|
||||
valueExpr, ok := nodeAccessLogValueCountExpr(col)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported value count column: %s", column)
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
filterExpr := valueExpr + " != ''"
|
||||
if col == nodeAccessLogColumnStatusCode {
|
||||
filterExpr = "status_code >= 0"
|
||||
}
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT %s AS value, count() AS count
|
||||
FROM %s
|
||||
WHERE %s AND %s
|
||||
GROUP BY value
|
||||
ORDER BY count DESC, value ASC`, valueExpr, tableName, clause, filterExpr)
|
||||
if limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("value counts node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []NodeAccessLogValueCount
|
||||
for rows.Next() {
|
||||
var (
|
||||
value string
|
||||
count uint64
|
||||
)
|
||||
if err := rows.Scan(&value, &count); err != nil {
|
||||
return nil, fmt.Errorf("scan value count row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogValueCount{
|
||||
Value: value,
|
||||
Count: safeInt64Count(count),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func nodeAccessLogValueCountExpr(column string) (string, bool) {
|
||||
switch column {
|
||||
case nodeAccessLogColumnStatusCode:
|
||||
return "toString(" + nodeAccessLogColumnStatusCode + ")", true
|
||||
case nodeAccessLogColumnHost:
|
||||
return "trim(" + nodeAccessLogColumnHost + ")", true
|
||||
case nodeAccessLogColumnPath:
|
||||
return "trim(" + nodeAccessLogColumnPath + ")", true
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
return "trim(" + nodeAccessLogColumnRemoteAddr + ")", true
|
||||
case nodeAccessLogColumnUserAgent:
|
||||
return "trim(" + nodeAccessLogColumnUserAgent + ")", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// NodeAggregatesNodeAccessLogs returns per-node request/error/UV aggregates.
|
||||
func NodeAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogNodeAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
node_id,
|
||||
count() AS request_count,
|
||||
countIf(status_code >= 500) AS error_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips
|
||||
FROM %s
|
||||
WHERE %s AND node_id != ''
|
||||
GROUP BY node_id
|
||||
ORDER BY request_count DESC, node_id ASC`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("node aggregates node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []NodeAccessLogNodeAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
nodeID string
|
||||
requestCount, errorCount, uniqueIPs uint64
|
||||
)
|
||||
if err := rows.Scan(&nodeID, &requestCount, &errorCount, &uniqueIPs); err != nil {
|
||||
return nil, fmt.Errorf("scan node aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogNodeAggregate{
|
||||
NodeID: nodeID,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
ErrorCount: safeInt64Count(errorCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPs),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeleteAllNodeAccessLogs hard-deletes all node access logs via TRUNCATE.
|
||||
func DeleteAllNodeAccessLogs(ctx context.Context) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeAccessLogTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsBefore force-materializes of_node_access_logs table TTL.
|
||||
//
|
||||
// The cutoff argument is kept for call-site compatibility and is not used to select rows:
|
||||
// ClickHouse MATERIALIZE TTL only enforces the DDL policy (TableTTLDaysNodeAccessLogs).
|
||||
// Returns an estimate of rows past table TTL as the int64 (not a hard-deleted count).
|
||||
// Callers that need honest API fields should prefer MaterializeNodeAccessLogsTTL.
|
||||
func DeleteNodeAccessLogsBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeAccessLogsTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeAccessLogsTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeAccessLogsTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeAccessLogTableName()
|
||||
ttlDays := TableTTLDaysNodeAccessLogs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteNodeAccessLogsByNodeBefore force-materializes table-global TTL.
|
||||
//
|
||||
// Node-scoped hard delete is not supported: MATERIALIZE TTL is table-global.
|
||||
// The returned count is an estimate of rows for nodeID past table TTL only.
|
||||
func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeAccessLogsTTLByNode(ctx, nodeID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeAccessLogsTTLByNode materializes table-global TTL and estimates node-scoped rows past TTL.
|
||||
func MaterializeNodeAccessLogsTTLByNode(ctx context.Context, nodeID string) (CleanupOutcome, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeAccessLogTableName()
|
||||
ttlDays := TableTTLDaysNodeAccessLogs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
|
||||
[]any{nodeID, cutoff},
|
||||
)
|
||||
}
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
nodeAccessLogFilterClauseCapacity = 7
|
||||
|
||||
nodeAccessLogSortDesc = "DESC"
|
||||
nodeAccessLogSortAsc = "ASC"
|
||||
nodeAccessLogSortAscInput = "asc"
|
||||
|
||||
nodeAccessLogColumnRemoteAddr = "remote_addr"
|
||||
nodeAccessLogColumnStatusCode = "status_code"
|
||||
nodeAccessLogColumnHost = "host"
|
||||
nodeAccessLogColumnPath = "path"
|
||||
nodeAccessLogColumnUserAgent = "user_agent"
|
||||
nodeAccessLogColumnLoggedAt = "logged_at"
|
||||
)
|
||||
|
||||
// NodeAccessLogFilter scopes ClickHouse node access log queries.
|
||||
type NodeAccessLogFilter = analyticsmodel.NodeAccessLogFilter
|
||||
|
||||
func buildNodeAccessLogFilterClause(filter NodeAccessLogFilter) (string, []any) {
|
||||
parts := make([]string, 0, nodeAccessLogFilterClauseCapacity)
|
||||
args := make([]any, 0, nodeAccessLogFilterClauseCapacity)
|
||||
if trimmed := strings.TrimSpace(filter.NodeID); trimmed != "" {
|
||||
parts = append(parts, "node_id = ?")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr); trimmed != "" {
|
||||
parts = append(parts, "remote_addr LIKE ?")
|
||||
args = append(args, util.EscapeLike(trimmed)+"%")
|
||||
}
|
||||
hosts := normalizeNodeAccessLogHosts(filter.Hosts)
|
||||
if len(hosts) > 0 {
|
||||
placeholders := make([]string, 0, len(hosts))
|
||||
for _, host := range hosts {
|
||||
placeholders = append(placeholders, "?")
|
||||
args = append(args, host)
|
||||
}
|
||||
parts = append(parts, "lowerUTF8(trim(host)) IN ("+strings.Join(placeholders, ", ")+")")
|
||||
} else if trimmed := strings.TrimSpace(filter.Host); trimmed != "" {
|
||||
parts = append(parts, "host LIKE ?")
|
||||
args = append(args, util.EscapeLike(trimmed)+"%")
|
||||
}
|
||||
if trimmed := strings.TrimSpace(filter.Path); trimmed != "" {
|
||||
parts = append(parts, "path LIKE ?")
|
||||
args = append(args, util.EscapeLike(trimmed)+"%")
|
||||
}
|
||||
if filter.StatusCode > 0 {
|
||||
parts = append(parts, "status_code = ?")
|
||||
args = append(args, filter.StatusCode)
|
||||
}
|
||||
if !filter.Since.IsZero() {
|
||||
parts = append(parts, "logged_at >= ?")
|
||||
args = append(args, filter.Since.UTC())
|
||||
}
|
||||
if !filter.Until.IsZero() {
|
||||
parts = append(parts, "logged_at < ?")
|
||||
args = append(args, filter.Until.UTC())
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "1", nil
|
||||
}
|
||||
return strings.Join(parts, " AND "), args
|
||||
}
|
||||
|
||||
func combineNodeAccessLogSQLClauses(left string, right string) string {
|
||||
if strings.TrimSpace(left) == "" || left == "TRUE" || left == "1" {
|
||||
return right
|
||||
}
|
||||
return left + " AND " + right
|
||||
}
|
||||
|
||||
func nodeAccessLogOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
column := nodeAccessLogColumnLoggedAt
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case nodeAccessLogColumnStatusCode:
|
||||
column = nodeAccessLogColumnStatusCode
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
column = nodeAccessLogColumnRemoteAddr
|
||||
case nodeAccessLogColumnHost:
|
||||
column = nodeAccessLogColumnHost
|
||||
case nodeAccessLogColumnPath:
|
||||
column = nodeAccessLogColumnPath
|
||||
}
|
||||
if column == nodeAccessLogColumnLoggedAt {
|
||||
return column + " " + direction + ", id " + direction
|
||||
}
|
||||
return column + " " + direction + ", " + nodeAccessLogColumnLoggedAt + " " + direction + ", id " + direction
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogRemoteAddr(value string) string {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogHosts(hosts []string) []string {
|
||||
if len(hosts) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(hosts))
|
||||
result := make([]string, 0, len(hosts))
|
||||
for _, host := range hosts {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(host))
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[trimmed]; ok {
|
||||
continue
|
||||
}
|
||||
seen[trimmed] = struct{}{}
|
||||
result = append(result, trimmed)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeNodeAccessLogSortOrder(sortOrder string) string {
|
||||
if strings.EqualFold(strings.TrimSpace(sortOrder), "asc") {
|
||||
return "asc"
|
||||
}
|
||||
return "desc"
|
||||
}
|
||||
|
||||
func nodeAccessLogBucketEpochExpr(bucketSeconds int64) string {
|
||||
return fmt.Sprintf("toInt64(intDiv(toUnixTimestamp(logged_at), %d) * %d)", bucketSeconds, bucketSeconds)
|
||||
}
|
||||
|
||||
func nodeAccessLogEpochExpr() string {
|
||||
return "toInt64(toUnixTimestamp(logged_at))"
|
||||
}
|
||||
|
||||
func nodeAccessLogHostIsIPLiteralExpr() string {
|
||||
return `(
|
||||
toIPv4OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
|
||||
OR toIPv6OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
|
||||
)`
|
||||
}
|
||||
|
||||
func nodeAccessLogBucketOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "request_count":
|
||||
return "request_count " + direction + ", bucket_epoch DESC"
|
||||
default:
|
||||
return "bucket_epoch " + direction
|
||||
}
|
||||
}
|
||||
|
||||
func nodeAccessLogIPSummaryOrderClause(sortBy string, sortOrder string) string {
|
||||
direction := nodeAccessLogSortDesc
|
||||
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
|
||||
direction = nodeAccessLogSortAsc
|
||||
}
|
||||
column := "total_requests"
|
||||
switch strings.TrimSpace(sortBy) {
|
||||
case "request_length", "bytes_received":
|
||||
column = "request_length"
|
||||
case "bytes_sent":
|
||||
column = "bytes_sent"
|
||||
case "success_ratio":
|
||||
column = "success_ratio"
|
||||
case "last_seen_at":
|
||||
column = "last_seen_epoch"
|
||||
case "recent_requests":
|
||||
// Deprecated sort key; fall back to total_requests.
|
||||
column = "total_requests"
|
||||
case nodeAccessLogColumnRemoteAddr:
|
||||
column = nodeAccessLogColumnRemoteAddr
|
||||
}
|
||||
return column + " " + direction + ", last_seen_epoch DESC, remote_addr ASC"
|
||||
}
|
||||
|
||||
func nodeAccessLogTableName() string {
|
||||
return "of_node_access_logs"
|
||||
}
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
// NodeAccessLogBucketAggregate is a folded bucket aggregate row.
|
||||
type NodeAccessLogBucketAggregate = analyticsmodel.NodeAccessLogBucketAggregate
|
||||
|
||||
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
|
||||
type NodeAccessLogWAFIPAggregate = analyticsmodel.NodeAccessLogWAFIPAggregate
|
||||
|
||||
// NodeAccessLogBucketDimension is a bucket dimension value.
|
||||
type NodeAccessLogBucketDimension = analyticsmodel.NodeAccessLogBucketDimension
|
||||
|
||||
// NodeAccessLogIPAggregate is an IP aggregate row.
|
||||
type NodeAccessLogIPAggregate = analyticsmodel.NodeAccessLogIPAggregate
|
||||
|
||||
// NodeAccessLogIPSummary is an IP summary row.
|
||||
type NodeAccessLogIPSummary = analyticsmodel.NodeAccessLogIPSummary
|
||||
|
||||
// NodeAccessLogIPTrend is an IP trend bucket row.
|
||||
type NodeAccessLogIPTrend = analyticsmodel.NodeAccessLogIPTrend
|
||||
|
||||
// BucketAggregatesNodeAccessLogs returns folded bucket aggregates with unique IP/host counts.
|
||||
func BucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogBucketAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
countIf(status_code >= 200 AND status_code < 300) AS status_2xx_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS status_4xx_count,
|
||||
countIf(status_code >= 500) AS status_5xx_count,
|
||||
uniqExactIf(remote_addr, remote_addr != '') AS unique_ip_count,
|
||||
uniqExactIf(host, host != '') AS unique_host_count,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY bucket_epoch
|
||||
ORDER BY %s`, bucketExpr, tableName, clause, nodeAccessLogBucketOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bucket aggregates node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogBucketAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
bucketEpoch int64
|
||||
requestCount, successCount, clientErrorCount, serverErrorCount, status2xxCount, status4xxCount, status5xxCount, uniqueIPCount, uniqueHostCount, bytesSent, requestLength uint64
|
||||
)
|
||||
if err := rows.Scan(&bucketEpoch, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &status2xxCount, &status4xxCount, &status5xxCount, &uniqueIPCount, &uniqueHostCount, &bytesSent, &requestLength); err != nil {
|
||||
return nil, fmt.Errorf("scan bucket aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogBucketAggregate{
|
||||
BucketEpoch: bucketEpoch,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
SuccessCount: safeInt64Count(successCount),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
Status2xxCount: safeInt64Count(status2xxCount),
|
||||
Status4xxCount: safeInt64Count(status4xxCount),
|
||||
Status5xxCount: safeInt64Count(status5xxCount),
|
||||
UniqueIPCount: safeInt64Count(uniqueIPCount),
|
||||
UniqueHostCount: safeInt64Count(uniqueHostCount),
|
||||
BytesSent: safeInt64Count(bytesSent),
|
||||
RequestLength: safeInt64Count(requestLength),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountBucketAggregatesNodeAccessLogs returns the number of folded buckets matching filter.
|
||||
func CountBucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT 1
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY %s
|
||||
)`, tableName, clause, bucketExpr)
|
||||
var totalBuckets uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalBuckets); err != nil {
|
||||
return 0, fmt.Errorf("count bucket aggregates node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalBuckets), nil
|
||||
}
|
||||
|
||||
// BucketDimensionsNodeAccessLogs returns bucket dimension values.
|
||||
func BucketDimensionsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, bucketSeconds int64) ([]NodeAccessLogBucketDimension, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
trim(%s) AS value
|
||||
FROM %s
|
||||
WHERE %s AND trim(%s) != ''
|
||||
GROUP BY bucket_epoch, trim(%s)`, bucketExpr, column, tableName, clause, column, column)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bucket dimensions node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogBucketDimension
|
||||
for rows.Next() {
|
||||
var item NodeAccessLogBucketDimension
|
||||
if err := rows.Scan(&item.BucketEpoch, &item.Value); err != nil {
|
||||
return nil, fmt.Errorf("scan bucket dimension row: %w", err)
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// IPAggregatesNodeAccessLogs returns IP aggregate rows.
|
||||
func IPAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, exactRemoteAddr bool) ([]NodeAccessLogIPAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
queryClause := clause
|
||||
queryArgs := append([]any{}, args...)
|
||||
if exactRemoteAddr {
|
||||
trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr)
|
||||
if trimmed == "" {
|
||||
return []NodeAccessLogIPAggregate{}, nil
|
||||
}
|
||||
queryClause = combineNodeAccessLogSQLClauses(queryClause, "remote_addr = ?")
|
||||
queryArgs = append(queryArgs, trimmed)
|
||||
}
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
count() AS request_count,
|
||||
countIf(status_code < 400) AS success_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, lastSeenExpr, tableName, queryClause)
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip aggregates node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogIPAggregate
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
lastSeenEpoch int64
|
||||
requestCount, successCount, clientErrorCount, serverErrorCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &lastSeenEpoch); err != nil {
|
||||
return nil, fmt.Errorf("scan ip aggregate row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogIPAggregate{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
SuccessCount: safeInt64Count(successCount),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
LastSeenEpoch: lastSeenEpoch,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// IPSummariesNodeAccessLogs returns paginated IP summary rows for the filter window.
|
||||
// recentSince is ignored (kept for call-site compatibility); recent_requests is always 0.
|
||||
func IPSummariesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, _ time.Time) ([]NodeAccessLogIPSummary, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
queryArgs := append([]any{}, args...)
|
||||
tableName := nodeAccessLogTableName()
|
||||
// Outer query allows ORDER BY success_ratio without repeating countIf.
|
||||
innerSQL := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
argMax(region, logged_at) AS region,
|
||||
count() AS total_requests,
|
||||
countIf(status_code >= 200 AND status_code < 300) AS success_2xx_count,
|
||||
sum(request_length) AS request_length,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, lastSeenExpr, tableName, clause)
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
region,
|
||||
total_requests,
|
||||
success_2xx_count,
|
||||
if(total_requests = 0, 0., toFloat64(success_2xx_count) / toFloat64(total_requests)) AS success_ratio,
|
||||
request_length,
|
||||
bytes_sent,
|
||||
last_seen_epoch
|
||||
FROM (%s)
|
||||
ORDER BY %s`, innerSQL, nodeAccessLogIPSummaryOrderClause(filter.SortBy, filter.SortOrder))
|
||||
if filter.PageSize > 0 {
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += clickHouseLimitOffsetClause
|
||||
queryArgs = append(queryArgs, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip summaries node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogIPSummary
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr, region string
|
||||
lastSeenEpoch int64
|
||||
successRatio float64
|
||||
totalRequests, success2xx, bytesReceived, bytes uint64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&remoteAddr,
|
||||
®ion,
|
||||
&totalRequests,
|
||||
&success2xx,
|
||||
&successRatio,
|
||||
&bytesReceived,
|
||||
&bytes,
|
||||
&lastSeenEpoch,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan ip summary row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogIPSummary{
|
||||
RemoteAddr: remoteAddr,
|
||||
Region: region,
|
||||
TotalRequests: safeInt64Count(totalRequests),
|
||||
Success2xxCount: safeInt64Count(success2xx),
|
||||
SuccessRatio: successRatio,
|
||||
BytesReceived: safeInt64Count(bytesReceived),
|
||||
BytesSent: safeInt64Count(bytes),
|
||||
RecentRequests: 0,
|
||||
LastSeenEpoch: lastSeenEpoch,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CountIPSummaryNodeAccessLogs returns the number of distinct IPs matching filter.
|
||||
func CountIPSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT count() FROM (
|
||||
SELECT 1
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr
|
||||
)`, tableName, clause)
|
||||
var totalIPs uint64
|
||||
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalIPs); err != nil {
|
||||
return 0, fmt.Errorf("count ip summary node access logs: %w", err)
|
||||
}
|
||||
return safeInt64Count(totalIPs), nil
|
||||
}
|
||||
|
||||
// IPAggregatesForWAFNodeAccessLogs returns per-IP aggregates for WAF automatic rules.
|
||||
func IPAggregatesForWAFNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogWAFIPAggregate, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
lastSeenExpr := nodeAccessLogEpochExpr()
|
||||
hostIsIPExpr := nodeAccessLogHostIsIPLiteralExpr()
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
count() AS request_count,
|
||||
countIf(status_code = 404) AS status_404_count,
|
||||
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
|
||||
countIf(status_code >= 500) AS server_error_count,
|
||||
countIf(%s) AS ip_host_count,
|
||||
max(%s) AS last_seen_epoch
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr`, hostIsIPExpr, lastSeenExpr, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip aggregates for waf node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
aggregates := make(map[string]*NodeAccessLogWAFIPAggregate)
|
||||
order := make([]string, 0)
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
lastSeenEpoch int64
|
||||
requestCount, status404Count, clientErrorCount, serverErrorCount, ipHostCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &requestCount, &status404Count, &clientErrorCount, &serverErrorCount, &ipHostCount, &lastSeenEpoch); err != nil {
|
||||
return nil, fmt.Errorf("scan waf ip aggregate row: %w", err)
|
||||
}
|
||||
remoteAddr = strings.TrimSpace(remoteAddr)
|
||||
if remoteAddr == "" {
|
||||
continue
|
||||
}
|
||||
aggregates[remoteAddr] = &NodeAccessLogWAFIPAggregate{
|
||||
RemoteAddr: remoteAddr,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
Status404Count: safeInt64Count(status404Count),
|
||||
ClientErrorCount: safeInt64Count(clientErrorCount),
|
||||
ServerErrorCount: safeInt64Count(serverErrorCount),
|
||||
IPHostCount: safeInt64Count(ipHostCount),
|
||||
LastSeenEpoch: lastSeenEpoch,
|
||||
StatusCounts: make(map[int]int64),
|
||||
}
|
||||
order = append(order, remoteAddr)
|
||||
}
|
||||
if err := mergeWAFIPStatusCodeCounts(ctx, filter, aggregates); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]NodeAccessLogWAFIPAggregate, 0, len(order))
|
||||
for _, remoteAddr := range order {
|
||||
if aggregate := aggregates[remoteAddr]; aggregate != nil {
|
||||
result = append(result, *aggregate)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func mergeWAFIPStatusCodeCounts(ctx context.Context, filter NodeAccessLogFilter, aggregates map[string]*NodeAccessLogWAFIPAggregate) error {
|
||||
if len(aggregates) == 0 {
|
||||
return nil
|
||||
}
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
remote_addr,
|
||||
status_code,
|
||||
count() AS status_count
|
||||
FROM %s
|
||||
WHERE %s AND remote_addr != ''
|
||||
GROUP BY remote_addr, status_code`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("waf ip status code counts: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
for rows.Next() {
|
||||
var (
|
||||
remoteAddr string
|
||||
statusCode int32
|
||||
statusCount uint64
|
||||
)
|
||||
if err := rows.Scan(&remoteAddr, &statusCode, &statusCount); err != nil {
|
||||
return fmt.Errorf("scan waf ip status code row: %w", err)
|
||||
}
|
||||
remoteAddr = strings.TrimSpace(remoteAddr)
|
||||
aggregate := aggregates[remoteAddr]
|
||||
if aggregate == nil {
|
||||
continue
|
||||
}
|
||||
if aggregate.StatusCounts == nil {
|
||||
aggregate.StatusCounts = make(map[int]int64)
|
||||
}
|
||||
aggregate.StatusCounts[int(statusCode)] = safeInt64Count(statusCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IPTrendNodeAccessLogs returns IP trend bucket rows.
|
||||
func IPTrendNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogIPTrend, error) {
|
||||
conn, err := nodeAccessLogConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeAccessLogFilterClause(filter)
|
||||
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
|
||||
tableName := nodeAccessLogTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS bucket_epoch,
|
||||
count() AS request_count
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY bucket_epoch
|
||||
ORDER BY bucket_epoch ASC`, bucketExpr, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ip trend node access logs: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var result []NodeAccessLogIPTrend
|
||||
for rows.Next() {
|
||||
var (
|
||||
bucketEpoch int64
|
||||
requestCount uint64
|
||||
)
|
||||
if err := rows.Scan(&bucketEpoch, &requestCount); err != nil {
|
||||
return nil, fmt.Errorf("scan ip trend row: %w", err)
|
||||
}
|
||||
result = append(result, NodeAccessLogIPTrend{
|
||||
BucketEpoch: bucketEpoch,
|
||||
RequestCount: safeInt64Count(requestCount),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/idgen"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBatchInsertNodeAccessLogs_Empty(t *testing.T) {
|
||||
err := BatchInsertNodeAccessLogs(context.Background(), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestBatchInsertNodeAccessLogs_UsesModelBatchSQL(t *testing.T) {
|
||||
require.NoError(t, idgen.Init(1))
|
||||
ctx := context.Background()
|
||||
mockBatch := &mockBatch{}
|
||||
mockConn := &mockConn{
|
||||
batch: mockBatch,
|
||||
batchQuery: analyticsmodel.NodeAccessLog{}.BatchInsertSQL(),
|
||||
}
|
||||
db.SetChConnForTest(mockConn)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
loggedAt := time.Now().UTC()
|
||||
err := BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
|
||||
{
|
||||
NodeID: "node-a",
|
||||
LoggedAt: loggedAt,
|
||||
RemoteAddr: "1.1.1.1",
|
||||
Region: "US",
|
||||
Host: "example.com",
|
||||
Path: "/alpha",
|
||||
StatusCode: 200,
|
||||
BytesSent: 2048,
|
||||
CreatedAt: loggedAt,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, mockConn.prepareCalled)
|
||||
assert.Equal(t, analyticsmodel.NodeAccessLog{}.BatchInsertSQL(), mockConn.preparedQuery)
|
||||
assert.True(t, mockBatch.sendCalled)
|
||||
require.Len(t, mockBatch.rows, 1)
|
||||
assert.Equal(t, "node-a", mockBatch.rows[0][1])
|
||||
require.Len(t, mockBatch.rows[0], 14)
|
||||
assert.Empty(t, mockBatch.rows[0][7]) // user_agent
|
||||
assert.Empty(t, mockBatch.rows[0][8]) // cache_status
|
||||
assert.Equal(t, uint64(2048), mockBatch.rows[0][10]) // bytes_sent
|
||||
assert.Equal(t, uint64(0), mockBatch.rows[0][11]) // request_length
|
||||
assert.Equal(t, uint32(0), mockBatch.rows[0][12]) // request_time_ms
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/idgen"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
// BatchInsertNodeAccessLogs writes node access logs to ClickHouse using the native batch API.
|
||||
func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAccessLog) error {
|
||||
if len(logs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeAccessLog{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, logItem := range logs {
|
||||
id := logItem.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := logItem.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
logItem.NodeID,
|
||||
logItem.LoggedAt.UTC(),
|
||||
strings.TrimSpace(logItem.RemoteAddr),
|
||||
logItem.Region,
|
||||
logItem.Host,
|
||||
logItem.Path,
|
||||
strings.TrimSpace(logItem.UserAgent),
|
||||
strings.TrimSpace(logItem.CacheStatus),
|
||||
logItem.StatusCode,
|
||||
logItem.BytesSent,
|
||||
logItem.RequestLength,
|
||||
logItem.RequestTimeMs,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node access log to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
func observabilityConn() (driver.Conn, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
return db.ChConn, nil
|
||||
}
|
||||
|
||||
// ListNodeMetricSnapshots returns metric snapshots matching filter.
|
||||
func ListNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node metric snapshots: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// ListLatestNodeMetricSnapshots returns the latest metric snapshot per node_id.
|
||||
// Uses ClickHouse LIMIT 1 BY so dashboard health does not depend on a global raw LIMIT.
|
||||
func ListLatestNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s%s`, nodeMetricSnapshotTableName(), clause, nodeObservabilityCapturedAtOrderClause(), clickHouseLimit1ByNodeIDClause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list latest node metric snapshots: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeEdgeHealth returns L2 OpenResty health snapshots.
|
||||
func ListNodeEdgeHealth(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeEdgeHealth, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeEdgeHealthTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, status, connections, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node edge health: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []analyticsmodel.NodeEdgeHealth
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeEdgeHealth
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.Status,
|
||||
&item.Connections,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node edge health row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListNodeObsFrps returns FRPS observations matching filter.
|
||||
func ListNodeObsFrps(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeObsFrpsTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, frps_connections, frps_proxy_count, frps_client_count, frps_proxies, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node frps observations: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeObsFrpsRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeObsFrpc returns FRPC observations matching filter.
|
||||
func ListNodeObsFrpc(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeObsFrpcTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, tunnel_status, connected_relays_count, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node frpc observations: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeObsFrpcRows(rows)
|
||||
}
|
||||
|
||||
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
|
||||
func scanNodeMetricSnapshotRows(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
var result []analyticsmodel.NodeMetricSnapshot
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeMetricSnapshot
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.CPUUsagePercent,
|
||||
&item.MemoryUsedBytes,
|
||||
&item.MemoryTotalBytes,
|
||||
&item.StorageUsedBytes,
|
||||
&item.StorageTotalBytes,
|
||||
&item.DiskReadBytes,
|
||||
&item.DiskWriteBytes,
|
||||
&item.NetworkRxBytes,
|
||||
&item.NetworkTxBytes,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
var result []analyticsmodel.NodeObsFrps
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrps
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.FrpsConnections,
|
||||
&item.FrpsProxyCount,
|
||||
&item.FrpsClientCount,
|
||||
&item.FrpsProxies,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frps observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NodeTrafficHourly 为小时级流量汇总行（别名，定义见 model/analytics）。
|
||||
type NodeTrafficHourly = analyticsmodel.NodeTrafficHourly
|
||||
|
||||
// NodeMetricHourly 为小时级指标聚合行（别名，定义见 model/analytics）。
|
||||
type NodeMetricHourly = analyticsmodel.NodeMetricHourly
|
||||
|
||||
// ListNodeTrafficHourly returns hourly traffic from of_access_log_hourly (M5).
|
||||
// UniqueVisitorCount is always 0 here (UV requires raw uniqExact on access logs).
|
||||
func ListNodeTrafficHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeTrafficHourly, error) {
|
||||
rows, err := ListAccessLogHourly(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Aggregate across hosts per node/hour.
|
||||
type key struct {
|
||||
node string
|
||||
hour int64
|
||||
}
|
||||
merged := make(map[key]*NodeTrafficHourly)
|
||||
order := make([]key, 0)
|
||||
for _, row := range rows {
|
||||
k := key{node: row.NodeID, hour: row.Hour.UTC().Unix()}
|
||||
item := merged[k]
|
||||
if item == nil {
|
||||
item = &NodeTrafficHourly{NodeID: row.NodeID, Hour: row.Hour.UTC()}
|
||||
merged[k] = item
|
||||
order = append(order, k)
|
||||
}
|
||||
item.RequestCount += row.RequestCount
|
||||
item.ErrorCount += row.ErrorCount
|
||||
}
|
||||
result := make([]NodeTrafficHourly, 0, len(order))
|
||||
for _, k := range order {
|
||||
result = append(result, *merged[k])
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListAccessLogHourly returns Server-side access log hourly rollups.
|
||||
func ListAccessLogHourly(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.AccessLogHourly, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "hour")
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
node_id,
|
||||
hour,
|
||||
host,
|
||||
sum(request_count) AS request_count,
|
||||
sum(error_count) AS error_count,
|
||||
sum(bytes_sent) AS bytes_sent,
|
||||
sum(request_length) AS request_length
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY node_id, hour, host
|
||||
ORDER BY hour ASC, node_id ASC, host ASC`, accessLogHourlyTableName(), clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list access log hourly: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var result []analyticsmodel.AccessLogHourly
|
||||
for rows.Next() {
|
||||
var (
|
||||
item analyticsmodel.AccessLogHourly
|
||||
requestCount, errorCount, bytesSent, requestLength uint64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&item.NodeID,
|
||||
&item.Hour,
|
||||
&item.Host,
|
||||
&requestCount,
|
||||
&errorCount,
|
||||
&bytesSent,
|
||||
&requestLength,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan access log hourly row: %w", err)
|
||||
}
|
||||
item.Hour = item.Hour.UTC()
|
||||
item.RequestCount = safeInt64Count(requestCount)
|
||||
item.ErrorCount = safeInt64Count(errorCount)
|
||||
item.BytesSent = safeInt64Count(bytesSent)
|
||||
item.RequestLength = safeInt64Count(requestLength)
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// hourlyRollupMaxLead is how far after filter.Since the earliest rollup bucket may start
|
||||
// while still treating pre-aggregated tables as a complete window (skip raw query).
|
||||
const hourlyRollupMaxLead = 2 * time.Hour
|
||||
|
||||
// hourlyRollupCoversWindow reports whether rollup coverage starts near the requested window.
|
||||
// rows must be ordered by hour ascending.
|
||||
func hourlyRollupCoversWindow(earliestHour time.Time, since time.Time) bool {
|
||||
if since.IsZero() {
|
||||
return true
|
||||
}
|
||||
sinceHour := since.UTC().Truncate(time.Hour)
|
||||
earliest := earliestHour.UTC().Truncate(time.Hour)
|
||||
return !earliest.After(sinceHour.Add(hourlyRollupMaxLead))
|
||||
}
|
||||
|
||||
// ListNodeMetricHourly returns hourly metric snapshot aggregates matching filter.
|
||||
//
|
||||
// Strategy (optimal for correctness + cost):
|
||||
// 1. Load of_node_metric_capacity_hourly rollup.
|
||||
// 2. If rollup spans the window from filter.Since, return it alone (cheap path).
|
||||
// 3. Otherwise load raw lagInFrame aggregates and merge by hour: rollup wins on
|
||||
// overlap, raw fills historical gaps (MV never backfills pre-creation data).
|
||||
func ListNodeMetricHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
|
||||
rollup, rollupErr := listNodeMetricHourlyFromRollup(ctx, filter)
|
||||
if rollupErr == nil && len(rollup) > 0 && hourlyRollupCoversWindow(rollup[0].Hour, filter.Since) {
|
||||
return rollup, nil
|
||||
}
|
||||
|
||||
raw, rawErr := listNodeMetricHourlyFromRaw(ctx, filter)
|
||||
if rawErr != nil {
|
||||
if rollupErr == nil && len(rollup) > 0 {
|
||||
return rollup, nil
|
||||
}
|
||||
return nil, rawErr
|
||||
}
|
||||
if len(rollup) == 0 {
|
||||
return raw, nil
|
||||
}
|
||||
// Partial rollup (or rollupErr with empty slice): merge; raw fills historical gaps.
|
||||
return mergeNodeMetricHourlyPreferRollup(rollup, raw), nil
|
||||
}
|
||||
|
||||
// mergeNodeMetricHourlyPreferRollup unions two hour series (both ASC by Hour).
|
||||
// Rollup values replace raw for the same hour; raw supplies missing hours.
|
||||
func mergeNodeMetricHourlyPreferRollup(rollup, raw []NodeMetricHourly) []NodeMetricHourly {
|
||||
byHour := make(map[int64]NodeMetricHourly, len(raw)+len(rollup))
|
||||
order := make([]int64, 0, len(raw)+len(rollup))
|
||||
add := func(row NodeMetricHourly, overwrite bool) {
|
||||
key := row.Hour.UTC().Truncate(time.Hour).Unix()
|
||||
if _, exists := byHour[key]; !exists {
|
||||
order = append(order, key)
|
||||
byHour[key] = row
|
||||
return
|
||||
}
|
||||
if overwrite {
|
||||
byHour[key] = row
|
||||
}
|
||||
}
|
||||
for _, row := range raw {
|
||||
add(row, false)
|
||||
}
|
||||
for _, row := range rollup {
|
||||
add(row, true)
|
||||
}
|
||||
result := make([]NodeMetricHourly, 0, len(order))
|
||||
// Keep chronological order of first-seen keys; re-sort by hour for stability.
|
||||
slices.Sort(order)
|
||||
for _, key := range order {
|
||||
result = append(result, byHour[key])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func listNodeMetricHourlyFromRollup(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "hour")
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
hour,
|
||||
if(sum(cpu_usage_count) > 0, sum(cpu_usage_sum) / sum(cpu_usage_count), 0) AS average_cpu_usage_percent,
|
||||
if(sum(memory_usage_count) > 0, sum(memory_usage_sum) / sum(memory_usage_count), 0) AS average_memory_usage_percent,
|
||||
sum(greatest(network_rx_max - network_rx_min, 0)) AS network_rx_bytes,
|
||||
sum(greatest(network_tx_max - network_tx_min, 0)) AS network_tx_bytes,
|
||||
sum(greatest(disk_read_max - disk_read_min, 0)) AS disk_read_bytes,
|
||||
sum(greatest(disk_write_max - disk_write_min, 0)) AS disk_write_bytes,
|
||||
toUInt64(uniqExact(node_id)) AS reported_nodes
|
||||
FROM %s
|
||||
WHERE %s
|
||||
GROUP BY hour
|
||||
ORDER BY hour ASC`, nodeMetricCapacityHourlyTableName(), clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node metric hourly from rollup: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricHourlyRows(rows)
|
||||
}
|
||||
|
||||
func listNodeMetricHourlyFromRaw(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
hour,
|
||||
avg(cpu_usage_percent) AS average_cpu_usage_percent,
|
||||
avg(memory_usage_percent) AS average_memory_usage_percent,
|
||||
sum(if(network_rx_delta >= 0, network_rx_delta, 0)) AS network_rx_bytes,
|
||||
sum(if(network_tx_delta >= 0, network_tx_delta, 0)) AS network_tx_bytes,
|
||||
sum(if(disk_read_delta >= 0, disk_read_delta, 0)) AS disk_read_bytes,
|
||||
sum(if(disk_write_delta >= 0, disk_write_delta, 0)) AS disk_write_bytes,
|
||||
toUInt64(uniqExact(node_id)) AS reported_nodes
|
||||
FROM (
|
||||
SELECT
|
||||
node_id,
|
||||
toStartOfHour(captured_at) AS hour,
|
||||
cpu_usage_percent,
|
||||
if(memory_total_bytes > 0, (memory_used_bytes * 100.0) / memory_total_bytes, 0) AS memory_usage_percent,
|
||||
network_rx_bytes - lagInFrame(network_rx_bytes, 1, network_rx_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS network_rx_delta,
|
||||
network_tx_bytes - lagInFrame(network_tx_bytes, 1, network_tx_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS network_tx_delta,
|
||||
disk_read_bytes - lagInFrame(disk_read_bytes, 1, disk_read_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS disk_read_delta,
|
||||
disk_write_bytes - lagInFrame(disk_write_bytes, 1, disk_write_bytes) OVER (
|
||||
PARTITION BY node_id ORDER BY captured_at, id
|
||||
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS disk_write_delta
|
||||
FROM %s
|
||||
WHERE %s
|
||||
)
|
||||
GROUP BY hour
|
||||
ORDER BY hour ASC`, tableName, clause)
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node metric hourly: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricHourlyRows(rows)
|
||||
}
|
||||
|
||||
func scanNodeMetricHourlyRows(rows driver.Rows) ([]NodeMetricHourly, error) {
|
||||
result := make([]NodeMetricHourly, 0)
|
||||
for rows.Next() {
|
||||
var (
|
||||
item NodeMetricHourly
|
||||
reportedNodes uint64
|
||||
networkRx int64
|
||||
networkTx int64
|
||||
diskRead int64
|
||||
diskWrite int64
|
||||
)
|
||||
if err := rows.Scan(
|
||||
&item.Hour,
|
||||
&item.AverageCPUUsagePercent,
|
||||
&item.AverageMemoryUsagePercent,
|
||||
&networkRx,
|
||||
&networkTx,
|
||||
&diskRead,
|
||||
&diskWrite,
|
||||
&reportedNodes,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node metric hourly row: %w", err)
|
||||
}
|
||||
item.Hour = item.Hour.UTC()
|
||||
item.NetworkRxBytes = networkRx
|
||||
item.NetworkTxBytes = networkTx
|
||||
item.DiskReadBytes = diskRead
|
||||
item.DiskWriteBytes = diskWrite
|
||||
item.ReportedNodes = int(safeInt64Count(reportedNodes))
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsFrpcRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
var result []analyticsmodel.NodeObsFrpc
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrpc
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.TunnelStatus,
|
||||
&item.ConnectedRelaysCount,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frpc observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeleteAllNodeMetricSnapshots hard-deletes all node metric snapshots via TRUNCATE.
|
||||
func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeMetricSnapshotTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeMetricSnapshotsBefore force-materializes of_node_metric_snapshots table TTL.
|
||||
// cutoff is ignored; see MaterializeNodeMetricSnapshotsTTL.
|
||||
func DeleteNodeMetricSnapshotsBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeMetricSnapshotsTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeMetricSnapshotsTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeMetricSnapshotsTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
ttlDays := TableTTLDaysNodeMetricSnapshots
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeEdgeHealth truncates of_node_edge_health.
|
||||
func DeleteAllNodeEdgeHealth(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeEdgeHealthTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeEdgeHealthBefore force-materializes of_node_edge_health TTL.
|
||||
func DeleteNodeEdgeHealthBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeEdgeHealthTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeEdgeHealthTTL force-materializes of_node_edge_health table TTL.
|
||||
func MaterializeNodeEdgeHealthTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeEdgeHealthTableName()
|
||||
ttlDays := TableTTLDaysNodeObs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrps hard-deletes all FRPS observations via TRUNCATE.
|
||||
func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpsTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpsBefore force-materializes of_node_obs_frps table TTL.
|
||||
// cutoff is ignored; see MaterializeNodeObsFrpsTTL.
|
||||
func DeleteNodeObsFrpsBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeObsFrpsTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeObsFrpsTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeObsFrpsTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeObsFrpsTableName()
|
||||
ttlDays := TableTTLDaysNodeObs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrpc hard-deletes all FRPC observations via TRUNCATE.
|
||||
func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpcTableName())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.DeletedCount, nil
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpcBefore force-materializes of_node_obs_frpc table TTL.
|
||||
// cutoff is ignored; see MaterializeNodeObsFrpcTTL.
|
||||
func DeleteNodeObsFrpcBefore(ctx context.Context, _ time.Time) (int64, error) {
|
||||
outcome, err := MaterializeNodeObsFrpcTTL(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return outcome.EligibleCount, nil
|
||||
}
|
||||
|
||||
// MaterializeNodeObsFrpcTTL force-materializes table TTL and reports an honest outcome.
|
||||
func MaterializeNodeObsFrpcTTL(ctx context.Context) (CleanupOutcome, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return CleanupOutcome{}, err
|
||||
}
|
||||
tableName := nodeObsFrpcTableName()
|
||||
ttlDays := TableTTLDaysNodeObs
|
||||
cutoff := tableTTLCutoff(ttlDays, time.Now())
|
||||
return materializeExpiredByTableTTL(
|
||||
ctx,
|
||||
conn,
|
||||
tableName,
|
||||
ttlDays,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
)
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
)
|
||||
|
||||
const nodeObservabilityFilterClauseCapacity = 3
|
||||
|
||||
// NodeObservabilityFilter scopes ClickHouse node observability queries.
|
||||
type NodeObservabilityFilter = analyticsmodel.NodeObservabilityFilter
|
||||
|
||||
func buildNodeObservabilityFilterClause(filter NodeObservabilityFilter, sinceColumn string) (string, []any) {
|
||||
parts := make([]string, 0, nodeObservabilityFilterClauseCapacity)
|
||||
args := make([]any, 0, nodeObservabilityFilterClauseCapacity)
|
||||
if trimmed := strings.TrimSpace(filter.NodeID); trimmed != "" {
|
||||
parts = append(parts, "node_id = ?")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if !filter.Since.IsZero() {
|
||||
parts = append(parts, sinceColumn+" >= ?")
|
||||
args = append(args, filter.Since.UTC())
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "1", nil
|
||||
}
|
||||
return strings.Join(parts, " AND "), args
|
||||
}
|
||||
|
||||
func nodeObservabilityCapturedAtOrderClause() string {
|
||||
return "captured_at DESC, id DESC"
|
||||
}
|
||||
|
||||
func nodeMetricSnapshotTableName() string {
|
||||
return "of_node_metric_snapshots"
|
||||
}
|
||||
|
||||
func nodeEdgeHealthTableName() string {
|
||||
return "of_node_edge_health"
|
||||
}
|
||||
|
||||
func accessLogHourlyTableName() string {
|
||||
return "of_access_log_hourly"
|
||||
}
|
||||
|
||||
func nodeObsFrpsTableName() string {
|
||||
return "of_node_obs_frps"
|
||||
}
|
||||
|
||||
func nodeObsFrpcTableName() string {
|
||||
return "of_node_obs_frpc"
|
||||
}
|
||||
|
||||
func nodeMetricCapacityHourlyTableName() string {
|
||||
return "of_node_metric_capacity_hourly"
|
||||
}
|
||||
|
||||
// clickHouseLimit1ByNodeIDClause selects the first row per node_id after ORDER BY.
|
||||
const clickHouseLimit1ByNodeIDClause = " LIMIT 1 BY node_id"
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestListLatestNodeMetricSnapshots_UsesLimit1ByNodeID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mock := &mockConn{}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
since := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
|
||||
_, err := ListLatestNodeMetricSnapshots(ctx, NodeObservabilityFilter{Since: since})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, mock.queries, 1)
|
||||
assert.Contains(t, mock.queries[0], "LIMIT 1 BY node_id")
|
||||
assert.Contains(t, mock.queries[0], nodeMetricSnapshotTableName())
|
||||
assert.Contains(t, mock.queries[0], "captured_at DESC")
|
||||
assert.NotContains(t, mock.queries[0], "LIMIT ?")
|
||||
require.Len(t, mock.queryArgs, 1)
|
||||
require.Len(t, mock.queryArgs[0], 1)
|
||||
assert.Equal(t, since, mock.queryArgs[0][0])
|
||||
}
|
||||
|
||||
func TestListNodeMetricHourly_PrefersRollup(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
hour := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
|
||||
since := hour.Add(-1 * time.Hour)
|
||||
mock := &mockConn{
|
||||
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
|
||||
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
|
||||
return &mockRows{data: [][]any{{
|
||||
hour, 42.5, 60.0, int64(100), int64(200), int64(10), int64(20), uint64(2),
|
||||
}}}, nil
|
||||
}
|
||||
return nil, errors.New("raw path should not be used when rollup covers the window")
|
||||
},
|
||||
}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{Since: since})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.InDelta(t, 42.5, rows[0].AverageCPUUsagePercent, 1e-9)
|
||||
assert.InDelta(t, 60.0, rows[0].AverageMemoryUsagePercent, 1e-9)
|
||||
assert.Equal(t, int64(100), rows[0].NetworkRxBytes)
|
||||
assert.Equal(t, 2, rows[0].ReportedNodes)
|
||||
require.Len(t, mock.queries, 1)
|
||||
assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
|
||||
}
|
||||
|
||||
func TestListNodeMetricHourly_MergesRawGapsWithPartialRollup(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// 24h window starts far before the only rollup bucket (last hour).
|
||||
since := time.Date(2026, 7, 9, 12, 0, 0, 0, time.UTC)
|
||||
rollupHour := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
|
||||
rawHour := time.Date(2026, 7, 9, 15, 0, 0, 0, time.UTC)
|
||||
mock := &mockConn{
|
||||
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
|
||||
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
|
||||
return &mockRows{data: [][]any{{
|
||||
rollupHour, 99.0, 99.0, int64(1), int64(1), int64(1), int64(1), uint64(1),
|
||||
}}}, nil
|
||||
}
|
||||
if strings.Contains(query, nodeMetricSnapshotTableName()) {
|
||||
return &mockRows{data: [][]any{
|
||||
{rawHour, 12.0, 34.0, int64(5), int64(6), int64(7), int64(8), uint64(1)},
|
||||
{rollupHour, 50.0, 50.0, int64(9), int64(9), int64(9), int64(9), uint64(1)},
|
||||
}}, nil
|
||||
}
|
||||
return &mockRows{}, nil
|
||||
},
|
||||
}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{Since: since})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 2)
|
||||
assert.Equal(t, rawHour, rows[0].Hour)
|
||||
assert.InDelta(t, 12.0, rows[0].AverageCPUUsagePercent, 1e-9)
|
||||
// Overlapping hour prefers rollup (99) over raw (50).
|
||||
assert.Equal(t, rollupHour, rows[1].Hour)
|
||||
assert.InDelta(t, 99.0, rows[1].AverageCPUUsagePercent, 1e-9)
|
||||
require.GreaterOrEqual(t, len(mock.queries), 2)
|
||||
assert.Contains(t, mock.queries[1], "lagInFrame")
|
||||
}
|
||||
|
||||
func TestMergeNodeMetricHourlyPreferRollup(t *testing.T) {
|
||||
h1 := time.Date(2026, 7, 10, 10, 0, 0, 0, time.UTC)
|
||||
h2 := time.Date(2026, 7, 10, 11, 0, 0, 0, time.UTC)
|
||||
merged := mergeNodeMetricHourlyPreferRollup(
|
||||
[]NodeMetricHourly{{Hour: h2, AverageCPUUsagePercent: 80}},
|
||||
[]NodeMetricHourly{
|
||||
{Hour: h1, AverageCPUUsagePercent: 10},
|
||||
{Hour: h2, AverageCPUUsagePercent: 20},
|
||||
},
|
||||
)
|
||||
require.Len(t, merged, 2)
|
||||
assert.Equal(t, h1, merged[0].Hour)
|
||||
assert.InDelta(t, 10.0, merged[0].AverageCPUUsagePercent, 1e-9)
|
||||
assert.Equal(t, h2, merged[1].Hour)
|
||||
assert.InDelta(t, 80.0, merged[1].AverageCPUUsagePercent, 1e-9)
|
||||
}
|
||||
|
||||
func TestHourlyRollupCoversWindow(t *testing.T) {
|
||||
since := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
|
||||
assert.True(t, hourlyRollupCoversWindow(since, since))
|
||||
assert.True(t, hourlyRollupCoversWindow(since.Add(2*time.Hour), since))
|
||||
assert.False(t, hourlyRollupCoversWindow(since.Add(3*time.Hour), since))
|
||||
assert.True(t, hourlyRollupCoversWindow(time.Date(2026, 7, 11, 0, 0, 0, 0, time.UTC), time.Time{}))
|
||||
}
|
||||
|
||||
func TestListNodeMetricHourly_FallsBackToRawOnRollupError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
hour := time.Date(2026, 7, 10, 13, 0, 0, 0, time.UTC)
|
||||
mock := &mockConn{
|
||||
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
|
||||
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
|
||||
return nil, errors.New("rollup missing")
|
||||
}
|
||||
if strings.Contains(query, nodeMetricSnapshotTableName()) {
|
||||
return &mockRows{data: [][]any{{
|
||||
hour, 10.0, 20.0, int64(1), int64(2), int64(3), int64(4), uint64(1),
|
||||
}}}, nil
|
||||
}
|
||||
return &mockRows{}, nil
|
||||
},
|
||||
}
|
||||
db.SetChConnForTest(mock)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.InDelta(t, 10.0, rows[0].AverageCPUUsagePercent, 1e-9)
|
||||
assert.Equal(t, int64(3), rows[0].DiskReadBytes)
|
||||
require.GreaterOrEqual(t, len(mock.queries), 2)
|
||||
assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
|
||||
assert.Contains(t, mock.queries[1], "lagInFrame")
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestInsertNodeEdgeHealth_EmptyNodeID(t *testing.T) {
|
||||
err := InsertNodeEdgeHealth(context.Background(), analyticsmodel.NodeEdgeHealth{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestInsertNodeEdgeHealth_UsesEdgeHealthBatchSQL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mockBatch := &mockBatch{}
|
||||
mockConn := &mockConn{
|
||||
batch: mockBatch,
|
||||
batchQuery: analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL(),
|
||||
}
|
||||
db.SetChConnForTest(mockConn)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
capturedAt := time.Now().UTC()
|
||||
err := InsertNodeEdgeHealth(ctx, analyticsmodel.NodeEdgeHealth{
|
||||
NodeID: "node-a",
|
||||
CapturedAt: capturedAt,
|
||||
Status: "",
|
||||
Connections: 3,
|
||||
CreatedAt: capturedAt,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, mockConn.prepareCalled)
|
||||
assert.Equal(t, analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL(), mockConn.preparedQuery)
|
||||
assert.True(t, mockBatch.sendCalled)
|
||||
require.Len(t, mockBatch.rows, 1)
|
||||
assert.Equal(t, "node-a", mockBatch.rows[0][1])
|
||||
assert.Equal(t, "unknown", mockBatch.rows[0][3]) // status default
|
||||
assert.Equal(t, int64(3), mockBatch.rows[0][4]) // connections
|
||||
}
|
||||
+272
@@ -0,0 +1,272 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
"Wavelet/pkg/idgen"
|
||||
db "Wavelet/plugins/infra/database"
|
||||
)
|
||||
|
||||
const edgeHealthStatusUnknown = "unknown"
|
||||
|
||||
// InsertNodeMetricSnapshot writes a single metric snapshot via the batch API.
|
||||
func InsertNodeMetricSnapshot(ctx context.Context, snapshot analyticsmodel.NodeMetricSnapshot) error {
|
||||
if strings.TrimSpace(snapshot.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeMetricSnapshots(ctx, []analyticsmodel.NodeMetricSnapshot{snapshot})
|
||||
}
|
||||
|
||||
// BatchInsertNodeMetricSnapshots writes metric snapshots to ClickHouse.
|
||||
func BatchInsertNodeMetricSnapshots(ctx context.Context, snapshots []analyticsmodel.NodeMetricSnapshot) error {
|
||||
if len(snapshots) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeMetricSnapshot{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, snapshot := range snapshots {
|
||||
nodeID := strings.TrimSpace(snapshot.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := snapshot.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := snapshot.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
snapshot.CapturedAt.UTC(),
|
||||
snapshot.CPUUsagePercent,
|
||||
snapshot.MemoryUsedBytes,
|
||||
snapshot.MemoryTotalBytes,
|
||||
snapshot.StorageUsedBytes,
|
||||
snapshot.StorageTotalBytes,
|
||||
snapshot.DiskReadBytes,
|
||||
snapshot.DiskWriteBytes,
|
||||
snapshot.NetworkRxBytes,
|
||||
snapshot.NetworkTxBytes,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node metric snapshot to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeEdgeHealthStatus(status string) string {
|
||||
status = strings.TrimSpace(status)
|
||||
if status == "" {
|
||||
return edgeHealthStatusUnknown
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// InsertNodeEdgeHealth writes a single edge health snapshot.
|
||||
func InsertNodeEdgeHealth(ctx context.Context, row analyticsmodel.NodeEdgeHealth) error {
|
||||
if strings.TrimSpace(row.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeEdgeHealth(ctx, []analyticsmodel.NodeEdgeHealth{row})
|
||||
}
|
||||
|
||||
// BatchInsertNodeEdgeHealth writes L2 OpenResty health snapshots to ClickHouse.
|
||||
func BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, row := range rows {
|
||||
nodeID := strings.TrimSpace(row.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := row.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := row.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := row.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
normalizeEdgeHealthStatus(row.Status),
|
||||
row.Connections,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node edge health to batch: %w", err)
|
||||
}
|
||||
}
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertNodeObsFrps writes a single FRPS observation via the batch API.
|
||||
func InsertNodeObsFrps(ctx context.Context, obs analyticsmodel.NodeObsFrps) error {
|
||||
if strings.TrimSpace(obs.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeObsFrps(ctx, []analyticsmodel.NodeObsFrps{obs})
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrps writes FRPS observations to ClickHouse.
|
||||
func BatchInsertNodeObsFrps(ctx context.Context, observations []analyticsmodel.NodeObsFrps) error {
|
||||
if len(observations) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrps{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, obs := range observations {
|
||||
nodeID := strings.TrimSpace(obs.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := obs.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := obs.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := obs.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
obs.FrpsConnections,
|
||||
obs.FrpsProxyCount,
|
||||
obs.FrpsClientCount,
|
||||
obs.FrpsProxies,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node frps observation to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertNodeObsFrpc writes a single FRPC observation via the batch API.
|
||||
func InsertNodeObsFrpc(ctx context.Context, obs analyticsmodel.NodeObsFrpc) error {
|
||||
if strings.TrimSpace(obs.NodeID) == "" {
|
||||
return nil
|
||||
}
|
||||
return BatchInsertNodeObsFrpc(ctx, []analyticsmodel.NodeObsFrpc{obs})
|
||||
}
|
||||
|
||||
// BatchInsertNodeObsFrpc writes FRPC observations to ClickHouse.
|
||||
func BatchInsertNodeObsFrpc(ctx context.Context, observations []analyticsmodel.NodeObsFrpc) error {
|
||||
if len(observations) == 0 {
|
||||
return nil
|
||||
}
|
||||
if db.ChConn == nil {
|
||||
return errors.New("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrpc{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
for _, obs := range observations {
|
||||
nodeID := strings.TrimSpace(obs.NodeID)
|
||||
if nodeID == "" {
|
||||
continue
|
||||
}
|
||||
id := obs.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := obs.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := obs.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
obs.TunnelStatus,
|
||||
obs.ConnectedRelaysCount,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node frpc observation to batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if batch.Rows() == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
const (
|
||||
clickHouseLimitClause = " LIMIT ?"
|
||||
clickHouseLimitOffsetClause = " LIMIT ? OFFSET ?"
|
||||
)
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
|
||||
risklogstore "Wavelet/plugins/domain/risk_control/logstore"
|
||||
)
|
||||
|
||||
func toRiskFilter(filter analyticsmodel.AccessLogFilter) risklogstore.AccessLogFilter {
|
||||
return risklogstore.AccessLogFilter{
|
||||
UserIDs: filter.UserIDs,
|
||||
Path: filter.Path,
|
||||
StartTime: filter.StartTime,
|
||||
EndTime: filter.EndTime,
|
||||
}
|
||||
}
|
||||
|
||||
// BatchInsert writes user access logs via Wavelet risk_control.
|
||||
func BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
|
||||
return risklogstore.BatchInsert(ctx, logs)
|
||||
}
|
||||
|
||||
// DeleteAllUserAccessLogs truncates user access logs via Wavelet risk_control.
|
||||
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
|
||||
return risklogstore.DeleteAllUserAccessLogs(ctx)
|
||||
}
|
||||
|
||||
// CountAccessLogs counts user access logs via Wavelet risk_control.
|
||||
func CountAccessLogs(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
|
||||
return risklogstore.CountAccessLogs(ctx, toRiskFilter(filter))
|
||||
}
|
||||
|
||||
// ListAccessLogs lists user access logs via Wavelet risk_control.
|
||||
func ListAccessLogs(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
|
||||
return risklogstore.ListAccessLogs(ctx, toRiskFilter(filter), page, pageSize)
|
||||
}
|
||||
|
||||
// GetDailyTrend returns the daily trend via Wavelet risk_control.
|
||||
func GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
|
||||
src, err := risklogstore.GetDailyTrend(ctx, days)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.DailyTrend, len(src))
|
||||
for i, v := range src {
|
||||
out[i] = analyticsmodel.DailyTrend{Date: v.Date, Count: v.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetBrowserDistribution returns browser share via Wavelet risk_control.
|
||||
func GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
|
||||
src, err := risklogstore.GetBrowserDistribution(ctx, startTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.BrowserShare, len(src))
|
||||
for i, v := range src {
|
||||
out[i] = analyticsmodel.BrowserShare{Browser: v.Browser, Count: v.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetTopActiveUsers returns top users via Wavelet risk_control.
|
||||
func GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
|
||||
src, err := risklogstore.GetTopActiveUsers(ctx, startTime, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analyticsmodel.TopUser, len(src))
|
||||
for i, v := range src {
|
||||
out[i] = analyticsmodel.TopUser{UserID: v.UserID, Count: v.Count}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user