mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-05 23:26:38 +08:00
observation clickhouse
This commit is contained in:
@@ -43,7 +43,7 @@ ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.
|
||||
if filter.Page < 0 {
|
||||
filter.Page = 0
|
||||
}
|
||||
sql += " LIMIT ? OFFSET ?"
|
||||
sql += clickHouseLimitOffsetClause
|
||||
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
@@ -123,7 +123,7 @@ WHERE %s AND trim(region) != ''
|
||||
GROUP BY trimmed_region
|
||||
ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
|
||||
if limit > 0 {
|
||||
sql += " LIMIT ?"
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
func observabilityConn() (driver.Conn, error) {
|
||||
if db.ChConn == nil {
|
||||
return nil, fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
return db.ChConn, nil
|
||||
}
|
||||
|
||||
// ListNodeMetricSnapshots returns metric snapshots matching filter.
|
||||
func ListNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node metric snapshots: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeMetricSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeRequestReports returns request reports matching filter.
|
||||
func ListNodeRequestReports(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeRequestReport, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "window_ended_at")
|
||||
tableName := nodeRequestReportTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, window_started_at, window_ended_at, request_count, error_count, unique_visitor_count, status_codes_json, top_domains_json, source_countries_json, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityWindowEndedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node request reports: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeRequestReportRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeObsOpenresty returns OpenResty observations matching filter.
|
||||
func ListNodeObsOpenresty(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsOpenresty, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeObsOpenrestyTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, openresty_rx_bytes, openresty_tx_bytes, openresty_connections, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node openresty observations: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeObsOpenrestyRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeObsFrps returns FRPS observations matching filter.
|
||||
func ListNodeObsFrps(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeObsFrpsTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, frps_connections, frps_proxy_count, frps_client_count, frps_proxies, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node frps observations: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeObsFrpsRows(rows)
|
||||
}
|
||||
|
||||
// ListNodeObsFrpc returns FRPC observations matching filter.
|
||||
func ListNodeObsFrpc(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
|
||||
tableName := nodeObsFrpcTableName()
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT id, node_id, captured_at, tunnel_status, connected_relays_count, created_at
|
||||
FROM %s
|
||||
WHERE %s
|
||||
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
|
||||
if filter.Limit > 0 {
|
||||
sql += clickHouseLimitClause
|
||||
args = append(args, filter.Limit)
|
||||
}
|
||||
rows, err := conn.Query(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list node frpc observations: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
return scanNodeObsFrpcRows(rows)
|
||||
}
|
||||
|
||||
func scanNodeMetricSnapshotRows(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
|
||||
var result []analyticsmodel.NodeMetricSnapshot
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeMetricSnapshot
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.CPUUsagePercent,
|
||||
&item.MemoryUsedBytes,
|
||||
&item.MemoryTotalBytes,
|
||||
&item.StorageUsedBytes,
|
||||
&item.StorageTotalBytes,
|
||||
&item.DiskReadBytes,
|
||||
&item.DiskWriteBytes,
|
||||
&item.NetworkRxBytes,
|
||||
&item.NetworkTxBytes,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeRequestReportRows(rows driver.Rows) ([]analyticsmodel.NodeRequestReport, error) {
|
||||
var result []analyticsmodel.NodeRequestReport
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeRequestReport
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.WindowStartedAt,
|
||||
&item.WindowEndedAt,
|
||||
&item.RequestCount,
|
||||
&item.ErrorCount,
|
||||
&item.UniqueVisitorCount,
|
||||
&item.StatusCodesJSON,
|
||||
&item.TopDomainsJSON,
|
||||
&item.SourceCountriesJSON,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node request report row: %w", err)
|
||||
}
|
||||
item.WindowStartedAt = item.WindowStartedAt.UTC()
|
||||
item.WindowEndedAt = item.WindowEndedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsOpenrestyRows(rows driver.Rows) ([]analyticsmodel.NodeObsOpenresty, error) {
|
||||
var result []analyticsmodel.NodeObsOpenresty
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsOpenresty
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.OpenrestyRxBytes,
|
||||
&item.OpenrestyTxBytes,
|
||||
&item.OpenrestyConnections,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node openresty observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
|
||||
var result []analyticsmodel.NodeObsFrps
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrps
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.FrpsConnections,
|
||||
&item.FrpsProxyCount,
|
||||
&item.FrpsClientCount,
|
||||
&item.FrpsProxies,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frps observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func scanNodeObsFrpcRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrpc, error) {
|
||||
var result []analyticsmodel.NodeObsFrpc
|
||||
for rows.Next() {
|
||||
var item analyticsmodel.NodeObsFrpc
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.NodeID,
|
||||
&item.CapturedAt,
|
||||
&item.TunnelStatus,
|
||||
&item.ConnectedRelaysCount,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan node frpc observation row: %w", err)
|
||||
}
|
||||
item.CapturedAt = item.CapturedAt.UTC()
|
||||
item.CreatedAt = item.CreatedAt.UTC()
|
||||
result = append(result, item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeleteAllNodeMetricSnapshots deletes all node metric snapshots.
|
||||
func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeMetricSnapshotsBefore deletes metric snapshots captured before cutoff.
|
||||
func DeleteNodeMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeMetricSnapshotTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeRequestReports deletes all node request reports.
|
||||
func DeleteAllNodeRequestReports(ctx context.Context) (int64, error) {
|
||||
tableName := nodeRequestReportTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeRequestReportsBefore deletes request reports ending before cutoff.
|
||||
func DeleteNodeRequestReportsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeRequestReportTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE window_ended_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE window_ended_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsOpenresty deletes all OpenResty observations.
|
||||
func DeleteAllNodeObsOpenresty(ctx context.Context) (int64, error) {
|
||||
tableName := nodeObsOpenrestyTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeObsOpenrestyBefore deletes OpenResty observations captured before cutoff.
|
||||
func DeleteNodeObsOpenrestyBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeObsOpenrestyTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrps deletes all FRPS observations.
|
||||
func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
|
||||
tableName := nodeObsFrpsTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpsBefore deletes FRPS observations captured before cutoff.
|
||||
func DeleteNodeObsFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeObsFrpsTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
// DeleteAllNodeObsFrpc deletes all FRPC observations.
|
||||
func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
|
||||
tableName := nodeObsFrpcTableName()
|
||||
return deleteNodeObservabilityWithCount(ctx, "SELECT count() FROM "+tableName, nil, "ALTER TABLE "+tableName+" DELETE WHERE 1")
|
||||
}
|
||||
|
||||
// DeleteNodeObsFrpcBefore deletes FRPC observations captured before cutoff.
|
||||
func DeleteNodeObsFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
|
||||
tableName := nodeObsFrpcTableName()
|
||||
cutoff = cutoff.UTC()
|
||||
return deleteNodeObservabilityWithCount(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
|
||||
[]any{cutoff},
|
||||
fmt.Sprintf("ALTER TABLE %s DELETE WHERE captured_at < ?", tableName),
|
||||
cutoff,
|
||||
)
|
||||
}
|
||||
|
||||
func deleteNodeObservabilityWithCount(ctx context.Context, countSQL string, countArgs []any, deleteSQL string, deleteArgs ...any) (int64, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var count int64
|
||||
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count node observability rows for delete: %w", err)
|
||||
}
|
||||
if count == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := conn.Exec(ctx, deleteSQL, deleteArgs...); err != nil {
|
||||
return 0, fmt.Errorf("delete node observability rows: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const nodeObservabilityFilterClauseCapacity = 3
|
||||
|
||||
// NodeObservabilityFilter scopes ClickHouse node observability queries.
|
||||
type NodeObservabilityFilter struct {
|
||||
NodeID string
|
||||
Since time.Time
|
||||
Limit int
|
||||
}
|
||||
|
||||
func buildNodeObservabilityFilterClause(filter NodeObservabilityFilter, sinceColumn string) (string, []any) {
|
||||
parts := make([]string, 0, nodeObservabilityFilterClauseCapacity)
|
||||
args := make([]any, 0, nodeObservabilityFilterClauseCapacity)
|
||||
if trimmed := strings.TrimSpace(filter.NodeID); trimmed != "" {
|
||||
parts = append(parts, "node_id = ?")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if !filter.Since.IsZero() {
|
||||
parts = append(parts, sinceColumn+" >= ?")
|
||||
args = append(args, filter.Since.UTC())
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "1", nil
|
||||
}
|
||||
return strings.Join(parts, " AND "), args
|
||||
}
|
||||
|
||||
func nodeObservabilityCapturedAtOrderClause() string {
|
||||
return "captured_at DESC, id DESC"
|
||||
}
|
||||
|
||||
func nodeObservabilityWindowEndedAtOrderClause() string {
|
||||
return "window_ended_at DESC, id DESC"
|
||||
}
|
||||
|
||||
func nodeMetricSnapshotTableName() string {
|
||||
return "of_node_metric_snapshots"
|
||||
}
|
||||
|
||||
func nodeRequestReportTableName() string {
|
||||
return "of_node_request_reports"
|
||||
}
|
||||
|
||||
func nodeObsOpenrestyTableName() string {
|
||||
return "of_node_obs_openresty"
|
||||
}
|
||||
|
||||
func nodeObsFrpsTableName() string {
|
||||
return "of_node_obs_frps"
|
||||
}
|
||||
|
||||
func nodeObsFrpcTableName() string {
|
||||
return "of_node_obs_frpc"
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestInsertNodeObsOpenresty_EmptyNodeID(t *testing.T) {
|
||||
err := InsertNodeObsOpenresty(context.Background(), analyticsmodel.NodeObsOpenresty{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestInsertNodeObsOpenresty_UsesModelBatchSQL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mockBatch := &mockBatch{}
|
||||
mockConn := &mockConn{
|
||||
batch: mockBatch,
|
||||
batchQuery: analyticsmodel.NodeObsOpenresty{}.BatchInsertSQL(),
|
||||
}
|
||||
db.SetChConnForTest(mockConn)
|
||||
t.Cleanup(func() { db.SetChConnForTest(nil) })
|
||||
|
||||
capturedAt := time.Now().UTC()
|
||||
err := InsertNodeObsOpenresty(ctx, analyticsmodel.NodeObsOpenresty{
|
||||
NodeID: "node-a",
|
||||
CapturedAt: capturedAt,
|
||||
OpenrestyRxBytes: 100,
|
||||
OpenrestyTxBytes: 200,
|
||||
OpenrestyConnections: 3,
|
||||
CreatedAt: capturedAt,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, mockConn.prepareCalled)
|
||||
assert.Equal(t, analyticsmodel.NodeObsOpenresty{}.BatchInsertSQL(), mockConn.preparedQuery)
|
||||
assert.True(t, mockBatch.sendCalled)
|
||||
require.Len(t, mockBatch.rows, 1)
|
||||
assert.Equal(t, "node-a", mockBatch.rows[0][1])
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/db"
|
||||
"github.com/Rain-kl/Wavelet/internal/db/idgen"
|
||||
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
|
||||
)
|
||||
|
||||
// InsertNodeMetricSnapshot writes a metric snapshot when no row exists for node_id+captured_at.
|
||||
func InsertNodeMetricSnapshot(ctx context.Context, snapshot analyticsmodel.NodeMetricSnapshot) error {
|
||||
nodeID := strings.TrimSpace(snapshot.NodeID)
|
||||
if nodeID == "" {
|
||||
return nil
|
||||
}
|
||||
capturedAt := snapshot.CapturedAt.UTC()
|
||||
|
||||
exists, err := nodeObservabilityRowExists(
|
||||
ctx,
|
||||
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND captured_at = ?", nodeMetricSnapshotTableName()),
|
||||
nodeID, capturedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
if db.ChConn == nil {
|
||||
return fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeMetricSnapshot{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := snapshot.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := snapshot.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
snapshot.CPUUsagePercent,
|
||||
snapshot.MemoryUsedBytes,
|
||||
snapshot.MemoryTotalBytes,
|
||||
snapshot.StorageUsedBytes,
|
||||
snapshot.StorageTotalBytes,
|
||||
snapshot.DiskReadBytes,
|
||||
snapshot.DiskWriteBytes,
|
||||
snapshot.NetworkRxBytes,
|
||||
snapshot.NetworkTxBytes,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node metric snapshot to batch: %w", err)
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertNodeRequestReport writes a request report when no row exists for node_id+window bounds.
|
||||
func InsertNodeRequestReport(ctx context.Context, report analyticsmodel.NodeRequestReport) error {
|
||||
nodeID := strings.TrimSpace(report.NodeID)
|
||||
if nodeID == "" {
|
||||
return nil
|
||||
}
|
||||
windowStartedAt := report.WindowStartedAt.UTC()
|
||||
windowEndedAt := report.WindowEndedAt.UTC()
|
||||
|
||||
exists, err := nodeObservabilityRowExists(
|
||||
ctx,
|
||||
fmt.Sprintf(
|
||||
"SELECT count() FROM %s WHERE node_id = ? AND window_started_at = ? AND window_ended_at = ?",
|
||||
nodeRequestReportTableName(),
|
||||
),
|
||||
nodeID, windowStartedAt, windowEndedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
if db.ChConn == nil {
|
||||
return fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeRequestReport{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := report.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := report.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
windowStartedAt,
|
||||
windowEndedAt,
|
||||
report.RequestCount,
|
||||
report.ErrorCount,
|
||||
report.UniqueVisitorCount,
|
||||
report.StatusCodesJSON,
|
||||
report.TopDomainsJSON,
|
||||
report.SourceCountriesJSON,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node request report to batch: %w", err)
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertNodeObsOpenresty writes an OpenResty observability snapshot.
|
||||
func InsertNodeObsOpenresty(ctx context.Context, obs analyticsmodel.NodeObsOpenresty) error {
|
||||
nodeID := strings.TrimSpace(obs.NodeID)
|
||||
if nodeID == "" {
|
||||
return nil
|
||||
}
|
||||
return insertNodeObsOpenrestyBatch(ctx, obs, nodeID)
|
||||
}
|
||||
|
||||
// InsertNodeObsFrps writes an FRPS observability snapshot.
|
||||
func InsertNodeObsFrps(ctx context.Context, obs analyticsmodel.NodeObsFrps) error {
|
||||
nodeID := strings.TrimSpace(obs.NodeID)
|
||||
if nodeID == "" {
|
||||
return nil
|
||||
}
|
||||
return insertNodeObsFrpsBatch(ctx, obs, nodeID)
|
||||
}
|
||||
|
||||
// InsertNodeObsFrpc writes an FRPC observability snapshot.
|
||||
func InsertNodeObsFrpc(ctx context.Context, obs analyticsmodel.NodeObsFrpc) error {
|
||||
nodeID := strings.TrimSpace(obs.NodeID)
|
||||
if nodeID == "" {
|
||||
return nil
|
||||
}
|
||||
return insertNodeObsFrpcBatch(ctx, obs, nodeID)
|
||||
}
|
||||
|
||||
func insertNodeObsOpenrestyBatch(ctx context.Context, obs analyticsmodel.NodeObsOpenresty, nodeID string) error {
|
||||
if db.ChConn == nil {
|
||||
return fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsOpenresty{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := obs.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := obs.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := obs.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
obs.OpenrestyRxBytes,
|
||||
obs.OpenrestyTxBytes,
|
||||
obs.OpenrestyConnections,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node openresty observation to batch: %w", err)
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertNodeObsFrpsBatch(ctx context.Context, obs analyticsmodel.NodeObsFrps, nodeID string) error {
|
||||
if db.ChConn == nil {
|
||||
return fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrps{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := obs.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := obs.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := obs.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
obs.FrpsConnections,
|
||||
obs.FrpsProxyCount,
|
||||
obs.FrpsClientCount,
|
||||
obs.FrpsProxies,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node frps observation to batch: %w", err)
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertNodeObsFrpcBatch(ctx context.Context, obs analyticsmodel.NodeObsFrpc, nodeID string) error {
|
||||
if db.ChConn == nil {
|
||||
return fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrpc{}.BatchInsertSQL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare clickhouse batch: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := obs.ID
|
||||
if id == 0 {
|
||||
id = idgen.NextUint64ID()
|
||||
}
|
||||
createdAt := obs.CreatedAt
|
||||
if createdAt.IsZero() {
|
||||
createdAt = now
|
||||
}
|
||||
capturedAt := obs.CapturedAt.UTC()
|
||||
if capturedAt.IsZero() {
|
||||
capturedAt = now
|
||||
}
|
||||
if err := batch.Append(
|
||||
id,
|
||||
nodeID,
|
||||
capturedAt,
|
||||
obs.TunnelStatus,
|
||||
obs.ConnectedRelaysCount,
|
||||
createdAt.UTC(),
|
||||
); err != nil {
|
||||
return fmt.Errorf("append node frpc observation to batch: %w", err)
|
||||
}
|
||||
if err := batch.Send(); err != nil {
|
||||
return fmt.Errorf("send clickhouse batch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func nodeObservabilityRowExists(ctx context.Context, countSQL string, args ...any) (bool, error) {
|
||||
conn, err := observabilityConn()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var count int64
|
||||
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&count); err != nil {
|
||||
return false, fmt.Errorf("check observability row exists: %w", err)
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package analytics
|
||||
|
||||
const (
|
||||
clickHouseLimitClause = " LIMIT ?"
|
||||
clickHouseLimitOffsetClause = " LIMIT ? OFFSET ?"
|
||||
)
|
||||
Reference in New Issue
Block a user