refactor(backend): rename OpenFlare directory to lowercase openflare

This commit is contained in:
ryan
2026-08-30 17:43:23 +08:00
parent 06d5fedbfc
commit c93ff6674f
543 changed files with 819 additions and 819 deletions
@@ -0,0 +1,185 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"io"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/column"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)
type mockConn struct {
batch driver.Batch
batchQuery string
prepareCalled bool
preparedQuery string
queries []string
queryArgs [][]any
queryFn func(ctx context.Context, query string, args ...any) (driver.Rows, error)
}
func (m *mockConn) Contributors() []string { return nil }
func (m *mockConn) ServerVersion() (*driver.ServerVersion, error) { return nil, nil }
func (m *mockConn) Select(_ context.Context, _ any, _ string, _ ...any) error { return nil }
func (m *mockConn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) {
m.queries = append(m.queries, query)
m.queryArgs = append(m.queryArgs, args)
if m.queryFn != nil {
return m.queryFn(ctx, query, args...)
}
return &mockRows{}, nil
}
func (m *mockConn) QueryRow(_ context.Context, _ string, _ ...any) driver.Row { return nil }
func (m *mockConn) PrepareBatch(_ context.Context, query string, _ ...driver.PrepareBatchOption) (driver.Batch, error) {
m.prepareCalled = true
m.preparedQuery = query
return m.batch, nil
}
func (m *mockConn) Exec(_ context.Context, _ string, _ ...any) error { return nil }
func (m *mockConn) AsyncInsert(_ context.Context, _ string, _ bool, _ ...any) error { return nil }
func (m *mockConn) InsertFormat(_ context.Context, _ string, _ string, _ io.Reader) error { return nil }
func (m *mockConn) QueryFormat(_ context.Context, _ string, _ string, _ ...any) (io.ReadCloser, error) {
return nil, nil
}
func (m *mockConn) Ping(_ context.Context) error { return nil }
func (m *mockConn) Stats() driver.Stats { return driver.Stats{} }
func (m *mockConn) Close() error { return nil }
type mockBatch struct {
rows [][]any
sendCalled bool
}
func (m *mockBatch) Abort() error { return nil }
func (m *mockBatch) Append(v ...any) error {
m.rows = append(m.rows, v)
return nil
}
func (m *mockBatch) AppendStruct(_ any) error { return nil }
func (m *mockBatch) Column(_ int) driver.BatchColumn { return nil }
func (m *mockBatch) Flush() error { return nil }
func (m *mockBatch) Send() error {
m.sendCalled = true
return nil
}
func (m *mockBatch) IsSent() bool { return m.sendCalled }
func (m *mockBatch) Rows() int { return len(m.rows) }
func (m *mockBatch) Columns() []column.Interface { return nil }
func (m *mockBatch) Close() error { return nil }
// mockRows is an empty driver.Rows implementation for query-path unit tests.
type mockRows struct {
index int
data [][]any
err error
}
func (m *mockRows) Next() bool {
if m.err != nil {
return false
}
if m.index >= len(m.data) {
return false
}
m.index++
return true
}
func (m *mockRows) Scan(dest ...any) error {
if m.err != nil {
return m.err
}
if m.index == 0 || m.index > len(m.data) {
return nil
}
row := m.data[m.index-1]
for i := range dest {
if i >= len(row) {
break
}
if err := assignMockScanValue(dest[i], row[i]); err != nil {
return err
}
}
return nil
}
func (m *mockRows) ScanStruct(_ any) error { return nil }
func (m *mockRows) ColumnTypes() []driver.ColumnType { return nil }
func (m *mockRows) Totals(_ ...any) error { return nil }
func (m *mockRows) Columns() []string { return nil }
func (m *mockRows) Close() error { return nil }
func (m *mockRows) Err() error { return m.err }
func (m *mockRows) HasData() bool { return len(m.data) > 0 }
func assignMockScanValue(dest any, value any) error {
switch d := dest.(type) {
case *string:
if v, ok := value.(string); ok {
*d = v
}
case *uint64:
switch v := value.(type) {
case uint64:
*d = v
case int:
*d = uint64(v)
case int64:
*d = uint64(v)
}
case *int64:
switch v := value.(type) {
case int64:
*d = v
case int:
*d = int64(v)
case uint64:
*d = int64(v)
}
case *float64:
switch v := value.(type) {
case float64:
*d = v
case float32:
*d = float64(v)
case int:
*d = float64(v)
}
case *time.Time:
if v, ok := value.(time.Time); ok {
*d = v
}
}
return nil
}
@@ -0,0 +1,13 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import "math"
func safeInt64Count(count uint64) int64 {
if count > math.MaxInt64 {
return math.MaxInt64
}
return int64(count)
}
@@ -0,0 +1,32 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"math"
"testing"
)
func TestSafeInt64Count(t *testing.T) {
t.Parallel()
tests := []struct {
name string
count uint64
want int64
}{
{name: "zero", count: 0, want: 0},
{name: "small", count: 42, want: 42},
{name: "max int64", count: math.MaxInt64, want: math.MaxInt64},
{name: "overflow clamps", count: math.MaxUint64, want: math.MaxInt64},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := safeInt64Count(tt.count); got != tt.want {
t.Fatalf("safeInt64Count(%d) = %d, want %d", tt.count, got, tt.want)
}
})
}
}
@@ -0,0 +1,120 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"fmt"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)
// DDL TTL days for analytics tables (must match goose ClickHouse migrations).
const (
// TableTTLDaysNodeAccessLogs is the of_node_access_logs TTL (90 days).
TableTTLDaysNodeAccessLogs = 90
// TableTTLDaysNodeMetricSnapshots is the of_node_metric_snapshots TTL (30 days).
TableTTLDaysNodeMetricSnapshots = 30
// TableTTLDaysNodeObs is the of_node_edge_health / of_node_obs_frps / of_node_obs_frpc TTL (30 days).
TableTTLDaysNodeObs = 30
// TableTTLDaysUserAccessLogs is the w_user_access_logs TTL (180 days).
TableTTLDaysUserAccessLogs = 180
)
const (
// CleanupModeTTLMaterialize expires rows via table TTL instead of ALTER DELETE mutations.
// This is not a hard delete: deleted_count must stay 0; use EligibleCount as an estimate.
CleanupModeTTLMaterialize = "ttl_materialize"
// CleanupModeTruncate removes all rows via TRUNCATE TABLE (hard delete).
CleanupModeTruncate = "truncate"
)
// CleanupOutcome describes a non-mutation ClickHouse cleanup operation.
//
// For CleanupModeTruncate:
// - DeletedCount and EligibleCount are the rows removed by TRUNCATE.
//
// For CleanupModeTTLMaterialize:
// - DeletedCount is always 0 (MATERIALIZE TTL is async / not a counted hard delete).
// - EligibleCount is an estimate of rows already past the table TTL policy (not an
// arbitrary user cutoff younger than the DDL TTL).
// - TableTTLDays is the DDL TTL used for the estimate and materialize.
type CleanupOutcome struct {
EligibleCount int64
DeletedCount int64
Mode string
TableTTLDays int
}
func countClickHouseRows(ctx context.Context, conn driver.Conn, countSQL string, countArgs []any) (int64, error) {
var count uint64
if err := conn.QueryRow(ctx, countSQL, countArgs...).Scan(&count); err != nil {
return 0, fmt.Errorf("count clickhouse rows: %w", err)
}
return safeInt64Count(count), nil
}
func materializeTableTTL(ctx context.Context, conn driver.Conn, tableName string) error {
sql := fmt.Sprintf("ALTER TABLE %s MATERIALIZE TTL", tableName)
if err := conn.Exec(ctx, sql); err != nil {
return fmt.Errorf("materialize ttl on %s: %w", tableName, err)
}
return nil
}
// tableTTLCutoff returns the UTC instant at which rows become eligible under a fixed day TTL.
func tableTTLCutoff(tableTTLDays int, now time.Time) time.Time {
if tableTTLDays < 1 {
tableTTLDays = 1
}
return now.UTC().Add(-time.Duration(tableTTLDays) * 24 * time.Hour)
}
// materializeExpiredByTableTTL force-materializes table TTL and estimates rows past that policy.
//
// countSQL must count only rows older than the table TTL (callers pass tableTTLCutoff args).
// Node-scoped filters may be used for the estimate only; MATERIALIZE is always table-global.
func materializeExpiredByTableTTL(
ctx context.Context,
conn driver.Conn,
tableName string,
tableTTLDays int,
countSQL string,
countArgs []any,
) (CleanupOutcome, error) {
outcome := CleanupOutcome{
Mode: CleanupModeTTLMaterialize,
TableTTLDays: tableTTLDays,
}
count, err := countClickHouseRows(ctx, conn, countSQL, countArgs)
if err != nil {
return CleanupOutcome{}, err
}
outcome.EligibleCount = count
// Always force materialize so ClickHouse applies the DDL TTL policy promptly.
// EligibleCount is informational only; MATERIALIZE does not return a deleted row count.
if err := materializeTableTTL(ctx, conn, tableName); err != nil {
return CleanupOutcome{}, err
}
return outcome, nil
}
func truncateClickHouseTable(ctx context.Context, conn driver.Conn, tableName string) (CleanupOutcome, error) {
count, err := countClickHouseRows(ctx, conn, "SELECT count() FROM "+tableName, nil)
if err != nil {
return CleanupOutcome{}, err
}
if count == 0 {
return CleanupOutcome{Mode: CleanupModeTruncate}, nil
}
if err := conn.Exec(ctx, "TRUNCATE TABLE "+tableName); err != nil {
return CleanupOutcome{}, fmt.Errorf("truncate %s: %w", tableName, err)
}
return CleanupOutcome{
EligibleCount: count,
DeletedCount: count,
Mode: CleanupModeTruncate,
}, nil
}
@@ -0,0 +1,36 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestTableTTLCutoff(t *testing.T) {
now := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
got := tableTTLCutoff(30, now)
assert.Equal(t, now.Add(-30*24*time.Hour), got)
got = tableTTLCutoff(90, now)
assert.Equal(t, now.Add(-90*24*time.Hour), got)
// Invalid TTL floors to 1 day.
got = tableTTLCutoff(0, now)
assert.Equal(t, now.Add(-24*time.Hour), got)
}
func TestCleanupModeConstants(t *testing.T) {
assert.Equal(t, "ttl_materialize", CleanupModeTTLMaterialize)
assert.Equal(t, "truncate", CleanupModeTruncate)
}
func TestTableTTLDaysMatchDDL(t *testing.T) {
assert.Equal(t, 90, TableTTLDaysNodeAccessLogs)
assert.Equal(t, 30, TableTTLDaysNodeMetricSnapshots)
assert.Equal(t, 30, TableTTLDaysNodeObs)
assert.Equal(t, 180, TableTTLDaysUserAccessLogs)
}
@@ -0,0 +1,66 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"errors"
"fmt"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
"Wavelet/openflare/plugins/server/kernel/runtimeconfig"
db "Wavelet/plugins/infra/database"
)
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
// and in-process batch writer queue health.
type ClickHouseOperationalStats = analyticsmodel.ClickHouseOperationalStats
// GetClickHouseOperationalStats returns operational metrics for the configured database.
func GetClickHouseOperationalStats(ctx context.Context) (*ClickHouseOperationalStats, error) {
if db.ChConn == nil {
return nil, errors.New("clickhouse native connection is not initialized")
}
database := runtimeconfig.Get().ClickHouse.Database
stats := &ClickHouseOperationalStats{Database: database}
partsSQL := `
SELECT
count() AS active_parts,
ifNull(sum(rows), 0) AS total_rows
FROM system.parts
WHERE active AND database = ?`
var activeParts, totalRows uint64
if err := db.ChConn.QueryRow(ctx, partsSQL, database).Scan(&activeParts, &totalRows); err != nil {
return nil, fmt.Errorf("query system.parts: %w", err)
}
stats.ActiveParts = safeInt64Count(activeParts)
stats.TotalRows = safeInt64Count(totalRows)
mutationsSQL := `
SELECT count()
FROM system.mutations
WHERE is_done = 0 AND database = ?`
if err := db.ChConn.QueryRow(ctx, mutationsSQL, database).Scan(&stats.PendingMutations); err != nil {
return nil, fmt.Errorf("query system.mutations: %w", err)
}
asyncSQL := `
SELECT
count() AS queue_entries,
ifNull(sum(bytes), 0) AS queue_bytes
FROM system.asynchronous_inserts
WHERE database = ?`
var queueEntries, queueBytes uint64
if err := db.ChConn.QueryRow(ctx, asyncSQL, database).Scan(&queueEntries, &queueBytes); err != nil {
// Older ClickHouse versions may not expose asynchronous_inserts; treat as optional.
stats.AsyncInsertQueue = 0
stats.AsyncInsertBytes = 0
} else {
stats.AsyncInsertQueue = safeInt64Count(queueEntries)
stats.AsyncInsertBytes = safeInt64Count(queueBytes)
}
return stats, nil
}
@@ -0,0 +1,302 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"errors"
"fmt"
"strings"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
db "Wavelet/plugins/infra/database"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)
// NodeAccessLogRegionCount aggregates access log regions.
type NodeAccessLogRegionCount = analyticsmodel.NodeAccessLogRegionCount
func nodeAccessLogConn() (driver.Conn, error) {
if db.ChConn == nil {
return nil, errors.New("clickhouse connection is not initialized")
}
return db.ChConn, nil
}
// ListNodeAccessLogs returns access logs matching filter.
func ListNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]analyticsmodel.NodeAccessLog, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, logged_at, remote_addr, region, host, path, user_agent, cache_status, status_code, bytes_sent, request_length, request_time_ms, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeAccessLogOrderClause(filter.SortBy, filter.SortOrder))
if filter.PageSize > 0 {
if filter.Page < 0 {
filter.Page = 0
}
sql += clickHouseLimitOffsetClause
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
return scanNodeAccessLogRows(rows)
}
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
func scanNodeAccessLogRows(rows driver.Rows) ([]analyticsmodel.NodeAccessLog, error) {
var result []analyticsmodel.NodeAccessLog
for rows.Next() {
var item analyticsmodel.NodeAccessLog
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.LoggedAt,
&item.RemoteAddr,
&item.Region,
&item.Host,
&item.Path,
&item.UserAgent,
&item.CacheStatus,
&item.StatusCode,
&item.BytesSent,
&item.RequestLength,
&item.RequestTimeMs,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node access log row: %w", err)
}
item.LoggedAt = item.LoggedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
// CountNodeAccessLogs returns total records, distinct IPs, and total bytes sent matching filter.
func CountNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, int64, int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, 0, 0, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
countSQL := fmt.Sprintf(`
SELECT
count() AS total_records,
uniqExactIf(remote_addr, remote_addr != '') AS total_ips,
sum(bytes_sent) AS total_bytes
FROM %s
WHERE %s`, tableName, clause)
var totalRecords, totalIPs, totalBytes uint64
if err := conn.QueryRow(ctx, countSQL, args...).Scan(&totalRecords, &totalIPs, &totalBytes); err != nil {
return 0, 0, 0, fmt.Errorf("count node access logs: %w", err)
}
return safeInt64Count(totalRecords), safeInt64Count(totalIPs), safeInt64Count(totalBytes), nil
}
// RegionCountsNodeAccessLogs returns region counts for a node since a time.
func RegionCountsNodeAccessLogs(ctx context.Context, nodeID string, since time.Time, limit int) ([]NodeAccessLogRegionCount, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
filter := NodeAccessLogFilter{NodeID: nodeID, Since: since}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT trim(region) AS trimmed_region, count() AS count
FROM %s
WHERE %s AND trim(region) != ''
GROUP BY trimmed_region
ORDER BY count DESC, trimmed_region ASC`, tableName, clause)
if limit > 0 {
sql += clickHouseLimitClause
args = append(args, limit)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("region counts node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogRegionCount
for rows.Next() {
var (
region string
count uint64
)
if err := rows.Scan(&region, &count); err != nil {
return nil, fmt.Errorf("scan region count row: %w", err)
}
result = append(result, NodeAccessLogRegionCount{
Region: region,
Count: safeInt64Count(count),
})
}
return result, nil
}
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
type NodeAccessLogTrafficSummary = analyticsmodel.NodeAccessLogTrafficSummary
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
type NodeAccessLogValueCount = analyticsmodel.NodeAccessLogValueCount
// NodeAccessLogNodeAggregate is per-node traffic over a window.
type NodeAccessLogNodeAggregate = analyticsmodel.NodeAccessLogNodeAggregate
// TrafficSummaryNodeAccessLogs returns request/error/UV/bytes/node counts for the filter.
func TrafficSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (NodeAccessLogTrafficSummary, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return NodeAccessLogTrafficSummary{}, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
count() AS request_count,
countIf(status_code >= 500) AS error_count,
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips,
sum(bytes_sent) AS bytes_sent,
sum(request_length) AS request_length,
uniqExactIf(node_id, node_id != '') AS node_count
FROM %s
WHERE %s`, tableName, clause)
var requestCount, errorCount, uniqueIPs, bytesSent, requestLength, nodeCount uint64
if err := conn.QueryRow(ctx, sql, args...).Scan(
&requestCount, &errorCount, &uniqueIPs, &bytesSent, &requestLength, &nodeCount,
); err != nil {
return NodeAccessLogTrafficSummary{}, fmt.Errorf("traffic summary node access logs: %w", err)
}
return NodeAccessLogTrafficSummary{
RequestCount: safeInt64Count(requestCount),
ErrorCount: safeInt64Count(errorCount),
UniqueIPCount: safeInt64Count(uniqueIPs),
BytesSent: safeInt64Count(bytesSent),
RequestLength: safeInt64Count(requestLength),
NodeCount: safeInt64Count(nodeCount),
}, nil
}
// ValueCountsNodeAccessLogs groups logs by a single dimension column.
// Allowed columns: status_code, host, path, remote_addr, user_agent.
func ValueCountsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, limit int) ([]NodeAccessLogValueCount, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
col := strings.TrimSpace(strings.ToLower(column))
valueExpr, ok := nodeAccessLogValueCountExpr(col)
if !ok {
return nil, fmt.Errorf("unsupported value count column: %s", column)
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
filterExpr := valueExpr + " != ''"
if col == nodeAccessLogColumnStatusCode {
filterExpr = "status_code >= 0"
}
sql := fmt.Sprintf(`
SELECT %s AS value, count() AS count
FROM %s
WHERE %s AND %s
GROUP BY value
ORDER BY count DESC, value ASC`, valueExpr, tableName, clause, filterExpr)
if limit > 0 {
sql += clickHouseLimitClause
args = append(args, limit)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("value counts node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogValueCount
for rows.Next() {
var (
value string
count uint64
)
if err := rows.Scan(&value, &count); err != nil {
return nil, fmt.Errorf("scan value count row: %w", err)
}
result = append(result, NodeAccessLogValueCount{
Value: value,
Count: safeInt64Count(count),
})
}
return result, nil
}
func nodeAccessLogValueCountExpr(column string) (string, bool) {
switch column {
case nodeAccessLogColumnStatusCode:
return "toString(" + nodeAccessLogColumnStatusCode + ")", true
case nodeAccessLogColumnHost:
return "trim(" + nodeAccessLogColumnHost + ")", true
case nodeAccessLogColumnPath:
return "trim(" + nodeAccessLogColumnPath + ")", true
case nodeAccessLogColumnRemoteAddr:
return "trim(" + nodeAccessLogColumnRemoteAddr + ")", true
case nodeAccessLogColumnUserAgent:
return "trim(" + nodeAccessLogColumnUserAgent + ")", true
default:
return "", false
}
}
// NodeAggregatesNodeAccessLogs returns per-node request/error/UV aggregates.
func NodeAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogNodeAggregate, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
node_id,
count() AS request_count,
countIf(status_code >= 500) AS error_count,
uniqExactIf(remote_addr, remote_addr != '') AS unique_ips
FROM %s
WHERE %s AND node_id != ''
GROUP BY node_id
ORDER BY request_count DESC, node_id ASC`, tableName, clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("node aggregates node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogNodeAggregate
for rows.Next() {
var (
nodeID string
requestCount, errorCount, uniqueIPs uint64
)
if err := rows.Scan(&nodeID, &requestCount, &errorCount, &uniqueIPs); err != nil {
return nil, fmt.Errorf("scan node aggregate row: %w", err)
}
result = append(result, NodeAccessLogNodeAggregate{
NodeID: nodeID,
RequestCount: safeInt64Count(requestCount),
ErrorCount: safeInt64Count(errorCount),
UniqueIPCount: safeInt64Count(uniqueIPs),
})
}
return result, nil
}
@@ -0,0 +1,87 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"fmt"
"time"
)
// DeleteAllNodeAccessLogs hard-deletes all node access logs via TRUNCATE.
func DeleteAllNodeAccessLogs(ctx context.Context) (int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, err
}
outcome, err := truncateClickHouseTable(ctx, conn, nodeAccessLogTableName())
if err != nil {
return 0, err
}
return outcome.DeletedCount, nil
}
// DeleteNodeAccessLogsBefore force-materializes of_node_access_logs table TTL.
//
// The cutoff argument is kept for call-site compatibility and is not used to select rows:
// ClickHouse MATERIALIZE TTL only enforces the DDL policy (TableTTLDaysNodeAccessLogs).
// Returns an estimate of rows past table TTL as the int64 (not a hard-deleted count).
// Callers that need honest API fields should prefer MaterializeNodeAccessLogsTTL.
func DeleteNodeAccessLogsBefore(ctx context.Context, _ time.Time) (int64, error) {
outcome, err := MaterializeNodeAccessLogsTTL(ctx)
if err != nil {
return 0, err
}
return outcome.EligibleCount, nil
}
// MaterializeNodeAccessLogsTTL force-materializes table TTL and reports an honest outcome.
func MaterializeNodeAccessLogsTTL(ctx context.Context) (CleanupOutcome, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return CleanupOutcome{}, err
}
tableName := nodeAccessLogTableName()
ttlDays := TableTTLDaysNodeAccessLogs
cutoff := tableTTLCutoff(ttlDays, time.Now())
return materializeExpiredByTableTTL(
ctx,
conn,
tableName,
ttlDays,
fmt.Sprintf("SELECT count() FROM %s WHERE logged_at < ?", tableName),
[]any{cutoff},
)
}
// DeleteNodeAccessLogsByNodeBefore force-materializes table-global TTL.
//
// Node-scoped hard delete is not supported: MATERIALIZE TTL is table-global.
// The returned count is an estimate of rows for nodeID past table TTL only.
func DeleteNodeAccessLogsByNodeBefore(ctx context.Context, nodeID string, _ time.Time) (int64, error) {
outcome, err := MaterializeNodeAccessLogsTTLByNode(ctx, nodeID)
if err != nil {
return 0, err
}
return outcome.EligibleCount, nil
}
// MaterializeNodeAccessLogsTTLByNode materializes table-global TTL and estimates node-scoped rows past TTL.
func MaterializeNodeAccessLogsTTLByNode(ctx context.Context, nodeID string) (CleanupOutcome, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return CleanupOutcome{}, err
}
tableName := nodeAccessLogTableName()
ttlDays := TableTTLDaysNodeAccessLogs
cutoff := tableTTLCutoff(ttlDays, time.Now())
return materializeExpiredByTableTTL(
ctx,
conn,
tableName,
ttlDays,
fmt.Sprintf("SELECT count() FROM %s WHERE node_id = ? AND logged_at < ?", tableName),
[]any{nodeID, cutoff},
)
}
@@ -0,0 +1,191 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"fmt"
"strings"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
"Wavelet/pkg/util"
)
const (
nodeAccessLogFilterClauseCapacity = 7
nodeAccessLogSortDesc = "DESC"
nodeAccessLogSortAsc = "ASC"
nodeAccessLogSortAscInput = "asc"
nodeAccessLogColumnRemoteAddr = "remote_addr"
nodeAccessLogColumnStatusCode = "status_code"
nodeAccessLogColumnHost = "host"
nodeAccessLogColumnPath = "path"
nodeAccessLogColumnUserAgent = "user_agent"
nodeAccessLogColumnLoggedAt = "logged_at"
)
// NodeAccessLogFilter scopes ClickHouse node access log queries.
type NodeAccessLogFilter = analyticsmodel.NodeAccessLogFilter
func buildNodeAccessLogFilterClause(filter NodeAccessLogFilter) (string, []any) {
parts := make([]string, 0, nodeAccessLogFilterClauseCapacity)
args := make([]any, 0, nodeAccessLogFilterClauseCapacity)
if trimmed := strings.TrimSpace(filter.NodeID); trimmed != "" {
parts = append(parts, "node_id = ?")
args = append(args, trimmed)
}
if trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr); trimmed != "" {
parts = append(parts, "remote_addr LIKE ?")
args = append(args, util.EscapeLike(trimmed)+"%")
}
hosts := normalizeNodeAccessLogHosts(filter.Hosts)
if len(hosts) > 0 {
placeholders := make([]string, 0, len(hosts))
for _, host := range hosts {
placeholders = append(placeholders, "?")
args = append(args, host)
}
parts = append(parts, "lowerUTF8(trim(host)) IN ("+strings.Join(placeholders, ", ")+")")
} else if trimmed := strings.TrimSpace(filter.Host); trimmed != "" {
parts = append(parts, "host LIKE ?")
args = append(args, util.EscapeLike(trimmed)+"%")
}
if trimmed := strings.TrimSpace(filter.Path); trimmed != "" {
parts = append(parts, "path LIKE ?")
args = append(args, util.EscapeLike(trimmed)+"%")
}
if filter.StatusCode > 0 {
parts = append(parts, "status_code = ?")
args = append(args, filter.StatusCode)
}
if !filter.Since.IsZero() {
parts = append(parts, "logged_at >= ?")
args = append(args, filter.Since.UTC())
}
if !filter.Until.IsZero() {
parts = append(parts, "logged_at < ?")
args = append(args, filter.Until.UTC())
}
if len(parts) == 0 {
return "1", nil
}
return strings.Join(parts, " AND "), args
}
func combineNodeAccessLogSQLClauses(left string, right string) string {
if strings.TrimSpace(left) == "" || left == "TRUE" || left == "1" {
return right
}
return left + " AND " + right
}
func nodeAccessLogOrderClause(sortBy string, sortOrder string) string {
direction := nodeAccessLogSortDesc
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
direction = nodeAccessLogSortAsc
}
column := nodeAccessLogColumnLoggedAt
switch strings.TrimSpace(sortBy) {
case nodeAccessLogColumnStatusCode:
column = nodeAccessLogColumnStatusCode
case nodeAccessLogColumnRemoteAddr:
column = nodeAccessLogColumnRemoteAddr
case nodeAccessLogColumnHost:
column = nodeAccessLogColumnHost
case nodeAccessLogColumnPath:
column = nodeAccessLogColumnPath
}
if column == nodeAccessLogColumnLoggedAt {
return column + " " + direction + ", id " + direction
}
return column + " " + direction + ", " + nodeAccessLogColumnLoggedAt + " " + direction + ", id " + direction
}
func normalizeNodeAccessLogRemoteAddr(value string) string {
return strings.TrimSpace(value)
}
func normalizeNodeAccessLogHosts(hosts []string) []string {
if len(hosts) == 0 {
return nil
}
seen := make(map[string]struct{}, len(hosts))
result := make([]string, 0, len(hosts))
for _, host := range hosts {
trimmed := strings.ToLower(strings.TrimSpace(host))
if trimmed == "" {
continue
}
if _, ok := seen[trimmed]; ok {
continue
}
seen[trimmed] = struct{}{}
result = append(result, trimmed)
}
return result
}
func normalizeNodeAccessLogSortOrder(sortOrder string) string {
if strings.EqualFold(strings.TrimSpace(sortOrder), "asc") {
return "asc"
}
return "desc"
}
func nodeAccessLogBucketEpochExpr(bucketSeconds int64) string {
return fmt.Sprintf("toInt64(intDiv(toUnixTimestamp(logged_at), %d) * %d)", bucketSeconds, bucketSeconds)
}
func nodeAccessLogEpochExpr() string {
return "toInt64(toUnixTimestamp(logged_at))"
}
func nodeAccessLogHostIsIPLiteralExpr() string {
return `(
toIPv4OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
OR toIPv6OrNull(trim(if(position(trim(host), ':') > 0 AND NOT startsWith(trim(host), '['), splitByChar(':', trim(host))[1], replaceRegexpAll(trim(host), '\\[|\\]', '')))) IS NOT NULL
)`
}
func nodeAccessLogBucketOrderClause(sortBy string, sortOrder string) string {
direction := nodeAccessLogSortDesc
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
direction = nodeAccessLogSortAsc
}
switch strings.TrimSpace(sortBy) {
case "request_count":
return "request_count " + direction + ", bucket_epoch DESC"
default:
return "bucket_epoch " + direction
}
}
func nodeAccessLogIPSummaryOrderClause(sortBy string, sortOrder string) string {
direction := nodeAccessLogSortDesc
if normalizeNodeAccessLogSortOrder(sortOrder) == nodeAccessLogSortAscInput {
direction = nodeAccessLogSortAsc
}
column := "total_requests"
switch strings.TrimSpace(sortBy) {
case "request_length", "bytes_received":
column = "request_length"
case "bytes_sent":
column = "bytes_sent"
case "success_ratio":
column = "success_ratio"
case "last_seen_at":
column = "last_seen_epoch"
case "recent_requests":
// Deprecated sort key; fall back to total_requests.
column = "total_requests"
case nodeAccessLogColumnRemoteAddr:
column = nodeAccessLogColumnRemoteAddr
}
return column + " " + direction + ", last_seen_epoch DESC, remote_addr ASC"
}
func nodeAccessLogTableName() string {
return "of_node_access_logs"
}
@@ -0,0 +1,470 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"fmt"
"strings"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
)
// NodeAccessLogBucketAggregate is a folded bucket aggregate row.
type NodeAccessLogBucketAggregate = analyticsmodel.NodeAccessLogBucketAggregate
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
type NodeAccessLogWAFIPAggregate = analyticsmodel.NodeAccessLogWAFIPAggregate
// NodeAccessLogBucketDimension is a bucket dimension value.
type NodeAccessLogBucketDimension = analyticsmodel.NodeAccessLogBucketDimension
// NodeAccessLogIPAggregate is an IP aggregate row.
type NodeAccessLogIPAggregate = analyticsmodel.NodeAccessLogIPAggregate
// NodeAccessLogIPSummary is an IP summary row.
type NodeAccessLogIPSummary = analyticsmodel.NodeAccessLogIPSummary
// NodeAccessLogIPTrend is an IP trend bucket row.
type NodeAccessLogIPTrend = analyticsmodel.NodeAccessLogIPTrend
// BucketAggregatesNodeAccessLogs returns folded bucket aggregates with unique IP/host counts.
func BucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogBucketAggregate, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
%s AS bucket_epoch,
count() AS request_count,
countIf(status_code < 400) AS success_count,
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
countIf(status_code >= 500) AS server_error_count,
countIf(status_code >= 200 AND status_code < 300) AS status_2xx_count,
countIf(status_code >= 400 AND status_code < 500) AS status_4xx_count,
countIf(status_code >= 500) AS status_5xx_count,
uniqExactIf(remote_addr, remote_addr != '') AS unique_ip_count,
uniqExactIf(host, host != '') AS unique_host_count,
sum(bytes_sent) AS bytes_sent,
sum(request_length) AS request_length
FROM %s
WHERE %s
GROUP BY bucket_epoch
ORDER BY %s`, bucketExpr, tableName, clause, nodeAccessLogBucketOrderClause(filter.SortBy, filter.SortOrder))
if filter.PageSize > 0 {
if filter.Page < 0 {
filter.Page = 0
}
sql += clickHouseLimitOffsetClause
args = append(args, filter.PageSize, filter.Page*filter.PageSize)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("bucket aggregates node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogBucketAggregate
for rows.Next() {
var (
bucketEpoch int64
requestCount, successCount, clientErrorCount, serverErrorCount, status2xxCount, status4xxCount, status5xxCount, uniqueIPCount, uniqueHostCount, bytesSent, requestLength uint64
)
if err := rows.Scan(&bucketEpoch, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &status2xxCount, &status4xxCount, &status5xxCount, &uniqueIPCount, &uniqueHostCount, &bytesSent, &requestLength); err != nil {
return nil, fmt.Errorf("scan bucket aggregate row: %w", err)
}
result = append(result, NodeAccessLogBucketAggregate{
BucketEpoch: bucketEpoch,
RequestCount: safeInt64Count(requestCount),
SuccessCount: safeInt64Count(successCount),
ClientErrorCount: safeInt64Count(clientErrorCount),
ServerErrorCount: safeInt64Count(serverErrorCount),
Status2xxCount: safeInt64Count(status2xxCount),
Status4xxCount: safeInt64Count(status4xxCount),
Status5xxCount: safeInt64Count(status5xxCount),
UniqueIPCount: safeInt64Count(uniqueIPCount),
UniqueHostCount: safeInt64Count(uniqueHostCount),
BytesSent: safeInt64Count(bytesSent),
RequestLength: safeInt64Count(requestLength),
})
}
return result, nil
}
// CountBucketAggregatesNodeAccessLogs returns the number of folded buckets matching filter.
func CountBucketAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) (int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT count() FROM (
SELECT 1
FROM %s
WHERE %s
GROUP BY %s
)`, tableName, clause, bucketExpr)
var totalBuckets uint64
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalBuckets); err != nil {
return 0, fmt.Errorf("count bucket aggregates node access logs: %w", err)
}
return safeInt64Count(totalBuckets), nil
}
// BucketDimensionsNodeAccessLogs returns bucket dimension values.
func BucketDimensionsNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, column string, bucketSeconds int64) ([]NodeAccessLogBucketDimension, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
%s AS bucket_epoch,
trim(%s) AS value
FROM %s
WHERE %s AND trim(%s) != ''
GROUP BY bucket_epoch, trim(%s)`, bucketExpr, column, tableName, clause, column, column)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("bucket dimensions node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogBucketDimension
for rows.Next() {
var item NodeAccessLogBucketDimension
if err := rows.Scan(&item.BucketEpoch, &item.Value); err != nil {
return nil, fmt.Errorf("scan bucket dimension row: %w", err)
}
result = append(result, item)
}
return result, nil
}
// IPAggregatesNodeAccessLogs returns IP aggregate rows.
func IPAggregatesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, exactRemoteAddr bool) ([]NodeAccessLogIPAggregate, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
queryClause := clause
queryArgs := append([]any{}, args...)
if exactRemoteAddr {
trimmed := normalizeNodeAccessLogRemoteAddr(filter.RemoteAddr)
if trimmed == "" {
return []NodeAccessLogIPAggregate{}, nil
}
queryClause = combineNodeAccessLogSQLClauses(queryClause, "remote_addr = ?")
queryArgs = append(queryArgs, trimmed)
}
lastSeenExpr := nodeAccessLogEpochExpr()
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
remote_addr,
count() AS request_count,
countIf(status_code < 400) AS success_count,
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
countIf(status_code >= 500) AS server_error_count,
max(%s) AS last_seen_epoch
FROM %s
WHERE %s AND remote_addr != ''
GROUP BY remote_addr`, lastSeenExpr, tableName, queryClause)
rows, err := conn.Query(ctx, sql, queryArgs...)
if err != nil {
return nil, fmt.Errorf("ip aggregates node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogIPAggregate
for rows.Next() {
var (
remoteAddr string
lastSeenEpoch int64
requestCount, successCount, clientErrorCount, serverErrorCount uint64
)
if err := rows.Scan(&remoteAddr, &requestCount, &successCount, &clientErrorCount, &serverErrorCount, &lastSeenEpoch); err != nil {
return nil, fmt.Errorf("scan ip aggregate row: %w", err)
}
result = append(result, NodeAccessLogIPAggregate{
RemoteAddr: remoteAddr,
RequestCount: safeInt64Count(requestCount),
SuccessCount: safeInt64Count(successCount),
ClientErrorCount: safeInt64Count(clientErrorCount),
ServerErrorCount: safeInt64Count(serverErrorCount),
LastSeenEpoch: lastSeenEpoch,
})
}
return result, nil
}
// IPSummariesNodeAccessLogs returns paginated IP summary rows for the filter window.
// recentSince is ignored (kept for call-site compatibility); recent_requests is always 0.
func IPSummariesNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, _ time.Time) ([]NodeAccessLogIPSummary, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
lastSeenExpr := nodeAccessLogEpochExpr()
queryArgs := append([]any{}, args...)
tableName := nodeAccessLogTableName()
// Outer query allows ORDER BY success_ratio without repeating countIf.
innerSQL := fmt.Sprintf(`
SELECT
remote_addr,
argMax(region, logged_at) AS region,
count() AS total_requests,
countIf(status_code >= 200 AND status_code < 300) AS success_2xx_count,
sum(request_length) AS request_length,
sum(bytes_sent) AS bytes_sent,
max(%s) AS last_seen_epoch
FROM %s
WHERE %s AND remote_addr != ''
GROUP BY remote_addr`, lastSeenExpr, tableName, clause)
sql := fmt.Sprintf(`
SELECT
remote_addr,
region,
total_requests,
success_2xx_count,
if(total_requests = 0, 0., toFloat64(success_2xx_count) / toFloat64(total_requests)) AS success_ratio,
request_length,
bytes_sent,
last_seen_epoch
FROM (%s)
ORDER BY %s`, innerSQL, nodeAccessLogIPSummaryOrderClause(filter.SortBy, filter.SortOrder))
if filter.PageSize > 0 {
if filter.Page < 0 {
filter.Page = 0
}
sql += clickHouseLimitOffsetClause
queryArgs = append(queryArgs, filter.PageSize, filter.Page*filter.PageSize)
}
rows, err := conn.Query(ctx, sql, queryArgs...)
if err != nil {
return nil, fmt.Errorf("ip summaries node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogIPSummary
for rows.Next() {
var (
remoteAddr, region string
lastSeenEpoch int64
successRatio float64
totalRequests, success2xx, bytesReceived, bytes uint64
)
if err := rows.Scan(
&remoteAddr,
&region,
&totalRequests,
&success2xx,
&successRatio,
&bytesReceived,
&bytes,
&lastSeenEpoch,
); err != nil {
return nil, fmt.Errorf("scan ip summary row: %w", err)
}
result = append(result, NodeAccessLogIPSummary{
RemoteAddr: remoteAddr,
Region: region,
TotalRequests: safeInt64Count(totalRequests),
Success2xxCount: safeInt64Count(success2xx),
SuccessRatio: successRatio,
BytesReceived: safeInt64Count(bytesReceived),
BytesSent: safeInt64Count(bytes),
RecentRequests: 0,
LastSeenEpoch: lastSeenEpoch,
})
}
return result, nil
}
// CountIPSummaryNodeAccessLogs returns the number of distinct IPs matching filter.
func CountIPSummaryNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) (int64, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return 0, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT count() FROM (
SELECT 1
FROM %s
WHERE %s AND remote_addr != ''
GROUP BY remote_addr
)`, tableName, clause)
var totalIPs uint64
if err := conn.QueryRow(ctx, sql, args...).Scan(&totalIPs); err != nil {
return 0, fmt.Errorf("count ip summary node access logs: %w", err)
}
return safeInt64Count(totalIPs), nil
}
// IPAggregatesForWAFNodeAccessLogs returns per-IP aggregates for WAF automatic rules.
func IPAggregatesForWAFNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter) ([]NodeAccessLogWAFIPAggregate, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
lastSeenExpr := nodeAccessLogEpochExpr()
hostIsIPExpr := nodeAccessLogHostIsIPLiteralExpr()
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
remote_addr,
count() AS request_count,
countIf(status_code = 404) AS status_404_count,
countIf(status_code >= 400 AND status_code < 500) AS client_error_count,
countIf(status_code >= 500) AS server_error_count,
countIf(%s) AS ip_host_count,
max(%s) AS last_seen_epoch
FROM %s
WHERE %s AND remote_addr != ''
GROUP BY remote_addr`, hostIsIPExpr, lastSeenExpr, tableName, clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("ip aggregates for waf node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
aggregates := make(map[string]*NodeAccessLogWAFIPAggregate)
order := make([]string, 0)
for rows.Next() {
var (
remoteAddr string
lastSeenEpoch int64
requestCount, status404Count, clientErrorCount, serverErrorCount, ipHostCount uint64
)
if err := rows.Scan(&remoteAddr, &requestCount, &status404Count, &clientErrorCount, &serverErrorCount, &ipHostCount, &lastSeenEpoch); err != nil {
return nil, fmt.Errorf("scan waf ip aggregate row: %w", err)
}
remoteAddr = strings.TrimSpace(remoteAddr)
if remoteAddr == "" {
continue
}
aggregates[remoteAddr] = &NodeAccessLogWAFIPAggregate{
RemoteAddr: remoteAddr,
RequestCount: safeInt64Count(requestCount),
Status404Count: safeInt64Count(status404Count),
ClientErrorCount: safeInt64Count(clientErrorCount),
ServerErrorCount: safeInt64Count(serverErrorCount),
IPHostCount: safeInt64Count(ipHostCount),
LastSeenEpoch: lastSeenEpoch,
StatusCounts: make(map[int]int64),
}
order = append(order, remoteAddr)
}
if err := mergeWAFIPStatusCodeCounts(ctx, filter, aggregates); err != nil {
return nil, err
}
result := make([]NodeAccessLogWAFIPAggregate, 0, len(order))
for _, remoteAddr := range order {
if aggregate := aggregates[remoteAddr]; aggregate != nil {
result = append(result, *aggregate)
}
}
return result, nil
}
func mergeWAFIPStatusCodeCounts(ctx context.Context, filter NodeAccessLogFilter, aggregates map[string]*NodeAccessLogWAFIPAggregate) error {
if len(aggregates) == 0 {
return nil
}
conn, err := nodeAccessLogConn()
if err != nil {
return err
}
clause, args := buildNodeAccessLogFilterClause(filter)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
remote_addr,
status_code,
count() AS status_count
FROM %s
WHERE %s AND remote_addr != ''
GROUP BY remote_addr, status_code`, tableName, clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return fmt.Errorf("waf ip status code counts: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var (
remoteAddr string
statusCode int32
statusCount uint64
)
if err := rows.Scan(&remoteAddr, &statusCode, &statusCount); err != nil {
return fmt.Errorf("scan waf ip status code row: %w", err)
}
remoteAddr = strings.TrimSpace(remoteAddr)
aggregate := aggregates[remoteAddr]
if aggregate == nil {
continue
}
if aggregate.StatusCounts == nil {
aggregate.StatusCounts = make(map[int]int64)
}
aggregate.StatusCounts[int(statusCode)] = safeInt64Count(statusCount)
}
return nil
}
// IPTrendNodeAccessLogs returns IP trend bucket rows.
func IPTrendNodeAccessLogs(ctx context.Context, filter NodeAccessLogFilter, bucketSeconds int64) ([]NodeAccessLogIPTrend, error) {
conn, err := nodeAccessLogConn()
if err != nil {
return nil, err
}
clause, args := buildNodeAccessLogFilterClause(filter)
bucketExpr := nodeAccessLogBucketEpochExpr(bucketSeconds)
tableName := nodeAccessLogTableName()
sql := fmt.Sprintf(`
SELECT
%s AS bucket_epoch,
count() AS request_count
FROM %s
WHERE %s
GROUP BY bucket_epoch
ORDER BY bucket_epoch ASC`, bucketExpr, tableName, clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("ip trend node access logs: %w", err)
}
defer func() { _ = rows.Close() }()
var result []NodeAccessLogIPTrend
for rows.Next() {
var (
bucketEpoch int64
requestCount uint64
)
if err := rows.Scan(&bucketEpoch, &requestCount); err != nil {
return nil, fmt.Errorf("scan ip trend row: %w", err)
}
result = append(result, NodeAccessLogIPTrend{
BucketEpoch: bucketEpoch,
RequestCount: safeInt64Count(requestCount),
})
}
return result, nil
}
@@ -0,0 +1,61 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"testing"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
"Wavelet/pkg/idgen"
db "Wavelet/plugins/infra/database"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBatchInsertNodeAccessLogs_Empty(t *testing.T) {
err := BatchInsertNodeAccessLogs(context.Background(), nil)
require.NoError(t, err)
}
func TestBatchInsertNodeAccessLogs_UsesModelBatchSQL(t *testing.T) {
require.NoError(t, idgen.Init(1))
ctx := context.Background()
mockBatch := &mockBatch{}
mockConn := &mockConn{
batch: mockBatch,
batchQuery: analyticsmodel.NodeAccessLog{}.BatchInsertSQL(),
}
db.SetChConnForTest(mockConn)
t.Cleanup(func() { db.SetChConnForTest(nil) })
loggedAt := time.Now().UTC()
err := BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
{
NodeID: "node-a",
LoggedAt: loggedAt,
RemoteAddr: "1.1.1.1",
Region: "US",
Host: "example.com",
Path: "/alpha",
StatusCode: 200,
BytesSent: 2048,
CreatedAt: loggedAt,
},
})
require.NoError(t, err)
assert.True(t, mockConn.prepareCalled)
assert.Equal(t, analyticsmodel.NodeAccessLog{}.BatchInsertSQL(), mockConn.preparedQuery)
assert.True(t, mockBatch.sendCalled)
require.Len(t, mockBatch.rows, 1)
assert.Equal(t, "node-a", mockBatch.rows[0][1])
require.Len(t, mockBatch.rows[0], 14)
assert.Empty(t, mockBatch.rows[0][7]) // user_agent
assert.Empty(t, mockBatch.rows[0][8]) // cache_status
assert.Equal(t, uint64(2048), mockBatch.rows[0][10]) // bytes_sent
assert.Equal(t, uint64(0), mockBatch.rows[0][11]) // request_length
assert.Equal(t, uint32(0), mockBatch.rows[0][12]) // request_time_ms
}
@@ -0,0 +1,66 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"errors"
"fmt"
"strings"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
"Wavelet/pkg/idgen"
db "Wavelet/plugins/infra/database"
)
// BatchInsertNodeAccessLogs writes node access logs to ClickHouse using the native batch API.
func BatchInsertNodeAccessLogs(ctx context.Context, logs []analyticsmodel.NodeAccessLog) error {
if len(logs) == 0 {
return nil
}
if db.ChConn == nil {
return errors.New("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeAccessLog{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
now := time.Now().UTC()
for _, logItem := range logs {
id := logItem.ID
if id == 0 {
id = idgen.NextUint64ID()
}
createdAt := logItem.CreatedAt
if createdAt.IsZero() {
createdAt = now
}
if err := batch.Append(
id,
logItem.NodeID,
logItem.LoggedAt.UTC(),
strings.TrimSpace(logItem.RemoteAddr),
logItem.Region,
logItem.Host,
logItem.Path,
strings.TrimSpace(logItem.UserAgent),
strings.TrimSpace(logItem.CacheStatus),
logItem.StatusCode,
logItem.BytesSent,
logItem.RequestLength,
logItem.RequestTimeMs,
createdAt.UTC(),
); err != nil {
return fmt.Errorf("append node access log to batch: %w", err)
}
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send clickhouse batch: %w", err)
}
return nil
}
@@ -0,0 +1,512 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"errors"
"fmt"
"slices"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
db "Wavelet/plugins/infra/database"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)
func observabilityConn() (driver.Conn, error) {
if db.ChConn == nil {
return nil, errors.New("clickhouse connection is not initialized")
}
return db.ChConn, nil
}
// ListNodeMetricSnapshots returns metric snapshots matching filter.
func ListNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
tableName := nodeMetricSnapshotTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
if filter.Limit > 0 {
sql += clickHouseLimitClause
args = append(args, filter.Limit)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list node metric snapshots: %w", err)
}
defer func() { _ = rows.Close() }()
return scanNodeMetricSnapshotRows(rows)
}
// ListLatestNodeMetricSnapshots returns the latest metric snapshot per node_id.
// Uses ClickHouse LIMIT 1 BY so dashboard health does not depend on a global raw LIMIT.
func ListLatestNodeMetricSnapshots(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeMetricSnapshot, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
sql := fmt.Sprintf(`
SELECT id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at
FROM %s
WHERE %s
ORDER BY %s%s`, nodeMetricSnapshotTableName(), clause, nodeObservabilityCapturedAtOrderClause(), clickHouseLimit1ByNodeIDClause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list latest node metric snapshots: %w", err)
}
defer func() { _ = rows.Close() }()
return scanNodeMetricSnapshotRows(rows)
}
// ListNodeEdgeHealth returns L2 OpenResty health snapshots.
func ListNodeEdgeHealth(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeEdgeHealth, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
tableName := nodeEdgeHealthTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, captured_at, status, connections, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
if filter.Limit > 0 {
sql += clickHouseLimitClause
args = append(args, filter.Limit)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list node edge health: %w", err)
}
defer func() { _ = rows.Close() }()
var result []analyticsmodel.NodeEdgeHealth
for rows.Next() {
var item analyticsmodel.NodeEdgeHealth
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.Status,
&item.Connections,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node edge health row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
// ListNodeObsFrps returns FRPS observations matching filter.
func ListNodeObsFrps(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrps, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
tableName := nodeObsFrpsTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, captured_at, frps_connections, frps_proxy_count, frps_client_count, frps_proxies, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
if filter.Limit > 0 {
sql += clickHouseLimitClause
args = append(args, filter.Limit)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list node frps observations: %w", err)
}
defer func() { _ = rows.Close() }()
return scanNodeObsFrpsRows(rows)
}
// ListNodeObsFrpc returns FRPC observations matching filter.
func ListNodeObsFrpc(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.NodeObsFrpc, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
tableName := nodeObsFrpcTableName()
sql := fmt.Sprintf(`
SELECT id, node_id, captured_at, tunnel_status, connected_relays_count, created_at
FROM %s
WHERE %s
ORDER BY %s`, tableName, clause, nodeObservabilityCapturedAtOrderClause())
if filter.Limit > 0 {
sql += clickHouseLimitClause
args = append(args, filter.Limit)
}
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list node frpc observations: %w", err)
}
defer func() { _ = rows.Close() }()
return scanNodeObsFrpcRows(rows)
}
//nolint:dupl // scan shapes differ by model fields; shared helper would obscure CH column mapping
func scanNodeMetricSnapshotRows(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
var result []analyticsmodel.NodeMetricSnapshot
for rows.Next() {
var item analyticsmodel.NodeMetricSnapshot
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.CPUUsagePercent,
&item.MemoryUsedBytes,
&item.MemoryTotalBytes,
&item.StorageUsedBytes,
&item.StorageTotalBytes,
&item.DiskReadBytes,
&item.DiskWriteBytes,
&item.NetworkRxBytes,
&item.NetworkTxBytes,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
func scanNodeObsFrpsRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
var result []analyticsmodel.NodeObsFrps
for rows.Next() {
var item analyticsmodel.NodeObsFrps
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.FrpsConnections,
&item.FrpsProxyCount,
&item.FrpsClientCount,
&item.FrpsProxies,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node frps observation row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
// NodeTrafficHourly 为小时级流量汇总行(别名,定义见 model/analytics)。
type NodeTrafficHourly = analyticsmodel.NodeTrafficHourly
// NodeMetricHourly 为小时级指标聚合行(别名,定义见 model/analytics)。
type NodeMetricHourly = analyticsmodel.NodeMetricHourly
// ListNodeTrafficHourly returns hourly traffic from of_access_log_hourly (M5).
// UniqueVisitorCount is always 0 here (UV requires raw uniqExact on access logs).
func ListNodeTrafficHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeTrafficHourly, error) {
rows, err := ListAccessLogHourly(ctx, filter)
if err != nil {
return nil, err
}
// Aggregate across hosts per node/hour.
type key struct {
node string
hour int64
}
merged := make(map[key]*NodeTrafficHourly)
order := make([]key, 0)
for _, row := range rows {
k := key{node: row.NodeID, hour: row.Hour.UTC().Unix()}
item := merged[k]
if item == nil {
item = &NodeTrafficHourly{NodeID: row.NodeID, Hour: row.Hour.UTC()}
merged[k] = item
order = append(order, k)
}
item.RequestCount += row.RequestCount
item.ErrorCount += row.ErrorCount
}
result := make([]NodeTrafficHourly, 0, len(order))
for _, k := range order {
result = append(result, *merged[k])
}
return result, nil
}
// ListAccessLogHourly returns Server-side access log hourly rollups.
func ListAccessLogHourly(ctx context.Context, filter NodeObservabilityFilter) ([]analyticsmodel.AccessLogHourly, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "hour")
sql := fmt.Sprintf(`
SELECT
node_id,
hour,
host,
sum(request_count) AS request_count,
sum(error_count) AS error_count,
sum(bytes_sent) AS bytes_sent,
sum(request_length) AS request_length
FROM %s
WHERE %s
GROUP BY node_id, hour, host
ORDER BY hour ASC, node_id ASC, host ASC`, accessLogHourlyTableName(), clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list access log hourly: %w", err)
}
defer func() { _ = rows.Close() }()
var result []analyticsmodel.AccessLogHourly
for rows.Next() {
var (
item analyticsmodel.AccessLogHourly
requestCount, errorCount, bytesSent, requestLength uint64
)
if err := rows.Scan(
&item.NodeID,
&item.Hour,
&item.Host,
&requestCount,
&errorCount,
&bytesSent,
&requestLength,
); err != nil {
return nil, fmt.Errorf("scan access log hourly row: %w", err)
}
item.Hour = item.Hour.UTC()
item.RequestCount = safeInt64Count(requestCount)
item.ErrorCount = safeInt64Count(errorCount)
item.BytesSent = safeInt64Count(bytesSent)
item.RequestLength = safeInt64Count(requestLength)
result = append(result, item)
}
return result, nil
}
// hourlyRollupMaxLead is how far after filter.Since the earliest rollup bucket may start
// while still treating pre-aggregated tables as a complete window (skip raw query).
const hourlyRollupMaxLead = 2 * time.Hour
// hourlyRollupCoversWindow reports whether rollup coverage starts near the requested window.
// rows must be ordered by hour ascending.
func hourlyRollupCoversWindow(earliestHour time.Time, since time.Time) bool {
if since.IsZero() {
return true
}
sinceHour := since.UTC().Truncate(time.Hour)
earliest := earliestHour.UTC().Truncate(time.Hour)
return !earliest.After(sinceHour.Add(hourlyRollupMaxLead))
}
// ListNodeMetricHourly returns hourly metric snapshot aggregates matching filter.
//
// Strategy (optimal for correctness + cost):
// 1. Load of_node_metric_capacity_hourly rollup.
// 2. If rollup spans the window from filter.Since, return it alone (cheap path).
// 3. Otherwise load raw lagInFrame aggregates and merge by hour: rollup wins on
// overlap, raw fills historical gaps (MV never backfills pre-creation data).
func ListNodeMetricHourly(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
rollup, rollupErr := listNodeMetricHourlyFromRollup(ctx, filter)
if rollupErr == nil && len(rollup) > 0 && hourlyRollupCoversWindow(rollup[0].Hour, filter.Since) {
return rollup, nil
}
raw, rawErr := listNodeMetricHourlyFromRaw(ctx, filter)
if rawErr != nil {
if rollupErr == nil && len(rollup) > 0 {
return rollup, nil
}
return nil, rawErr
}
if len(rollup) == 0 {
return raw, nil
}
// Partial rollup (or rollupErr with empty slice): merge; raw fills historical gaps.
return mergeNodeMetricHourlyPreferRollup(rollup, raw), nil
}
// mergeNodeMetricHourlyPreferRollup unions two hour series (both ASC by Hour).
// Rollup values replace raw for the same hour; raw supplies missing hours.
func mergeNodeMetricHourlyPreferRollup(rollup, raw []NodeMetricHourly) []NodeMetricHourly {
byHour := make(map[int64]NodeMetricHourly, len(raw)+len(rollup))
order := make([]int64, 0, len(raw)+len(rollup))
add := func(row NodeMetricHourly, overwrite bool) {
key := row.Hour.UTC().Truncate(time.Hour).Unix()
if _, exists := byHour[key]; !exists {
order = append(order, key)
byHour[key] = row
return
}
if overwrite {
byHour[key] = row
}
}
for _, row := range raw {
add(row, false)
}
for _, row := range rollup {
add(row, true)
}
result := make([]NodeMetricHourly, 0, len(order))
// Keep chronological order of first-seen keys; re-sort by hour for stability.
slices.Sort(order)
for _, key := range order {
result = append(result, byHour[key])
}
return result
}
func listNodeMetricHourlyFromRollup(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "hour")
sql := fmt.Sprintf(`
SELECT
hour,
if(sum(cpu_usage_count) > 0, sum(cpu_usage_sum) / sum(cpu_usage_count), 0) AS average_cpu_usage_percent,
if(sum(memory_usage_count) > 0, sum(memory_usage_sum) / sum(memory_usage_count), 0) AS average_memory_usage_percent,
sum(greatest(network_rx_max - network_rx_min, 0)) AS network_rx_bytes,
sum(greatest(network_tx_max - network_tx_min, 0)) AS network_tx_bytes,
sum(greatest(disk_read_max - disk_read_min, 0)) AS disk_read_bytes,
sum(greatest(disk_write_max - disk_write_min, 0)) AS disk_write_bytes,
toUInt64(uniqExact(node_id)) AS reported_nodes
FROM %s
WHERE %s
GROUP BY hour
ORDER BY hour ASC`, nodeMetricCapacityHourlyTableName(), clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list node metric hourly from rollup: %w", err)
}
defer func() { _ = rows.Close() }()
return scanNodeMetricHourlyRows(rows)
}
func listNodeMetricHourlyFromRaw(ctx context.Context, filter NodeObservabilityFilter) ([]NodeMetricHourly, error) {
conn, err := observabilityConn()
if err != nil {
return nil, err
}
clause, args := buildNodeObservabilityFilterClause(filter, "captured_at")
tableName := nodeMetricSnapshotTableName()
sql := fmt.Sprintf(`
SELECT
hour,
avg(cpu_usage_percent) AS average_cpu_usage_percent,
avg(memory_usage_percent) AS average_memory_usage_percent,
sum(if(network_rx_delta >= 0, network_rx_delta, 0)) AS network_rx_bytes,
sum(if(network_tx_delta >= 0, network_tx_delta, 0)) AS network_tx_bytes,
sum(if(disk_read_delta >= 0, disk_read_delta, 0)) AS disk_read_bytes,
sum(if(disk_write_delta >= 0, disk_write_delta, 0)) AS disk_write_bytes,
toUInt64(uniqExact(node_id)) AS reported_nodes
FROM (
SELECT
node_id,
toStartOfHour(captured_at) AS hour,
cpu_usage_percent,
if(memory_total_bytes > 0, (memory_used_bytes * 100.0) / memory_total_bytes, 0) AS memory_usage_percent,
network_rx_bytes - lagInFrame(network_rx_bytes, 1, network_rx_bytes) OVER (
PARTITION BY node_id ORDER BY captured_at, id
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
) AS network_rx_delta,
network_tx_bytes - lagInFrame(network_tx_bytes, 1, network_tx_bytes) OVER (
PARTITION BY node_id ORDER BY captured_at, id
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
) AS network_tx_delta,
disk_read_bytes - lagInFrame(disk_read_bytes, 1, disk_read_bytes) OVER (
PARTITION BY node_id ORDER BY captured_at, id
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
) AS disk_read_delta,
disk_write_bytes - lagInFrame(disk_write_bytes, 1, disk_write_bytes) OVER (
PARTITION BY node_id ORDER BY captured_at, id
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
) AS disk_write_delta
FROM %s
WHERE %s
)
GROUP BY hour
ORDER BY hour ASC`, tableName, clause)
rows, err := conn.Query(ctx, sql, args...)
if err != nil {
return nil, fmt.Errorf("list node metric hourly: %w", err)
}
defer func() { _ = rows.Close() }()
return scanNodeMetricHourlyRows(rows)
}
func scanNodeMetricHourlyRows(rows driver.Rows) ([]NodeMetricHourly, error) {
result := make([]NodeMetricHourly, 0)
for rows.Next() {
var (
item NodeMetricHourly
reportedNodes uint64
networkRx int64
networkTx int64
diskRead int64
diskWrite int64
)
if err := rows.Scan(
&item.Hour,
&item.AverageCPUUsagePercent,
&item.AverageMemoryUsagePercent,
&networkRx,
&networkTx,
&diskRead,
&diskWrite,
&reportedNodes,
); err != nil {
return nil, fmt.Errorf("scan node metric hourly row: %w", err)
}
item.Hour = item.Hour.UTC()
item.NetworkRxBytes = networkRx
item.NetworkTxBytes = networkTx
item.DiskReadBytes = diskRead
item.DiskWriteBytes = diskWrite
item.ReportedNodes = int(safeInt64Count(reportedNodes))
result = append(result, item)
}
return result, nil
}
func scanNodeObsFrpcRows(rows driver.Rows) ([]analyticsmodel.NodeObsFrpc, error) {
var result []analyticsmodel.NodeObsFrpc
for rows.Next() {
var item analyticsmodel.NodeObsFrpc
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.TunnelStatus,
&item.ConnectedRelaysCount,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node frpc observation row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
@@ -0,0 +1,177 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"fmt"
"time"
)
// DeleteAllNodeMetricSnapshots hard-deletes all node metric snapshots via TRUNCATE.
func DeleteAllNodeMetricSnapshots(ctx context.Context) (int64, error) {
conn, err := observabilityConn()
if err != nil {
return 0, err
}
outcome, err := truncateClickHouseTable(ctx, conn, nodeMetricSnapshotTableName())
if err != nil {
return 0, err
}
return outcome.DeletedCount, nil
}
// DeleteNodeMetricSnapshotsBefore force-materializes of_node_metric_snapshots table TTL.
// cutoff is ignored; see MaterializeNodeMetricSnapshotsTTL.
func DeleteNodeMetricSnapshotsBefore(ctx context.Context, _ time.Time) (int64, error) {
outcome, err := MaterializeNodeMetricSnapshotsTTL(ctx)
if err != nil {
return 0, err
}
return outcome.EligibleCount, nil
}
// MaterializeNodeMetricSnapshotsTTL force-materializes table TTL and reports an honest outcome.
func MaterializeNodeMetricSnapshotsTTL(ctx context.Context) (CleanupOutcome, error) {
conn, err := observabilityConn()
if err != nil {
return CleanupOutcome{}, err
}
tableName := nodeMetricSnapshotTableName()
ttlDays := TableTTLDaysNodeMetricSnapshots
cutoff := tableTTLCutoff(ttlDays, time.Now())
return materializeExpiredByTableTTL(
ctx,
conn,
tableName,
ttlDays,
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
[]any{cutoff},
)
}
// DeleteAllNodeEdgeHealth truncates of_node_edge_health.
func DeleteAllNodeEdgeHealth(ctx context.Context) (int64, error) {
conn, err := observabilityConn()
if err != nil {
return 0, err
}
outcome, err := truncateClickHouseTable(ctx, conn, nodeEdgeHealthTableName())
if err != nil {
return 0, err
}
return outcome.DeletedCount, nil
}
// DeleteNodeEdgeHealthBefore force-materializes of_node_edge_health TTL.
func DeleteNodeEdgeHealthBefore(ctx context.Context, _ time.Time) (int64, error) {
outcome, err := MaterializeNodeEdgeHealthTTL(ctx)
if err != nil {
return 0, err
}
return outcome.EligibleCount, nil
}
// MaterializeNodeEdgeHealthTTL force-materializes of_node_edge_health table TTL.
func MaterializeNodeEdgeHealthTTL(ctx context.Context) (CleanupOutcome, error) {
conn, err := observabilityConn()
if err != nil {
return CleanupOutcome{}, err
}
tableName := nodeEdgeHealthTableName()
ttlDays := TableTTLDaysNodeObs
cutoff := tableTTLCutoff(ttlDays, time.Now())
return materializeExpiredByTableTTL(
ctx,
conn,
tableName,
ttlDays,
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
[]any{cutoff},
)
}
// DeleteAllNodeObsFrps hard-deletes all FRPS observations via TRUNCATE.
func DeleteAllNodeObsFrps(ctx context.Context) (int64, error) {
conn, err := observabilityConn()
if err != nil {
return 0, err
}
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpsTableName())
if err != nil {
return 0, err
}
return outcome.DeletedCount, nil
}
// DeleteNodeObsFrpsBefore force-materializes of_node_obs_frps table TTL.
// cutoff is ignored; see MaterializeNodeObsFrpsTTL.
func DeleteNodeObsFrpsBefore(ctx context.Context, _ time.Time) (int64, error) {
outcome, err := MaterializeNodeObsFrpsTTL(ctx)
if err != nil {
return 0, err
}
return outcome.EligibleCount, nil
}
// MaterializeNodeObsFrpsTTL force-materializes table TTL and reports an honest outcome.
func MaterializeNodeObsFrpsTTL(ctx context.Context) (CleanupOutcome, error) {
conn, err := observabilityConn()
if err != nil {
return CleanupOutcome{}, err
}
tableName := nodeObsFrpsTableName()
ttlDays := TableTTLDaysNodeObs
cutoff := tableTTLCutoff(ttlDays, time.Now())
return materializeExpiredByTableTTL(
ctx,
conn,
tableName,
ttlDays,
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
[]any{cutoff},
)
}
// DeleteAllNodeObsFrpc hard-deletes all FRPC observations via TRUNCATE.
func DeleteAllNodeObsFrpc(ctx context.Context) (int64, error) {
conn, err := observabilityConn()
if err != nil {
return 0, err
}
outcome, err := truncateClickHouseTable(ctx, conn, nodeObsFrpcTableName())
if err != nil {
return 0, err
}
return outcome.DeletedCount, nil
}
// DeleteNodeObsFrpcBefore force-materializes of_node_obs_frpc table TTL.
// cutoff is ignored; see MaterializeNodeObsFrpcTTL.
func DeleteNodeObsFrpcBefore(ctx context.Context, _ time.Time) (int64, error) {
outcome, err := MaterializeNodeObsFrpcTTL(ctx)
if err != nil {
return 0, err
}
return outcome.EligibleCount, nil
}
// MaterializeNodeObsFrpcTTL force-materializes table TTL and reports an honest outcome.
func MaterializeNodeObsFrpcTTL(ctx context.Context) (CleanupOutcome, error) {
conn, err := observabilityConn()
if err != nil {
return CleanupOutcome{}, err
}
tableName := nodeObsFrpcTableName()
ttlDays := TableTTLDaysNodeObs
cutoff := tableTTLCutoff(ttlDays, time.Now())
return materializeExpiredByTableTTL(
ctx,
conn,
tableName,
ttlDays,
fmt.Sprintf("SELECT count() FROM %s WHERE captured_at < ?", tableName),
[]any{cutoff},
)
}
@@ -0,0 +1,63 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"strings"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
)
const nodeObservabilityFilterClauseCapacity = 3
// NodeObservabilityFilter scopes ClickHouse node observability queries.
type NodeObservabilityFilter = analyticsmodel.NodeObservabilityFilter
func buildNodeObservabilityFilterClause(filter NodeObservabilityFilter, sinceColumn string) (string, []any) {
parts := make([]string, 0, nodeObservabilityFilterClauseCapacity)
args := make([]any, 0, nodeObservabilityFilterClauseCapacity)
if trimmed := strings.TrimSpace(filter.NodeID); trimmed != "" {
parts = append(parts, "node_id = ?")
args = append(args, trimmed)
}
if !filter.Since.IsZero() {
parts = append(parts, sinceColumn+" >= ?")
args = append(args, filter.Since.UTC())
}
if len(parts) == 0 {
return "1", nil
}
return strings.Join(parts, " AND "), args
}
func nodeObservabilityCapturedAtOrderClause() string {
return "captured_at DESC, id DESC"
}
func nodeMetricSnapshotTableName() string {
return "of_node_metric_snapshots"
}
func nodeEdgeHealthTableName() string {
return "of_node_edge_health"
}
func accessLogHourlyTableName() string {
return "of_access_log_hourly"
}
func nodeObsFrpsTableName() string {
return "of_node_obs_frps"
}
func nodeObsFrpcTableName() string {
return "of_node_obs_frpc"
}
func nodeMetricCapacityHourlyTableName() string {
return "of_node_metric_capacity_hourly"
}
// clickHouseLimit1ByNodeIDClause selects the first row per node_id after ORDER BY.
const clickHouseLimit1ByNodeIDClause = " LIMIT 1 BY node_id"
@@ -0,0 +1,156 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"errors"
"strings"
"testing"
"time"
db "Wavelet/plugins/infra/database"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestListLatestNodeMetricSnapshots_UsesLimit1ByNodeID(t *testing.T) {
ctx := context.Background()
mock := &mockConn{}
db.SetChConnForTest(mock)
t.Cleanup(func() { db.SetChConnForTest(nil) })
since := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
_, err := ListLatestNodeMetricSnapshots(ctx, NodeObservabilityFilter{Since: since})
require.NoError(t, err)
require.Len(t, mock.queries, 1)
assert.Contains(t, mock.queries[0], "LIMIT 1 BY node_id")
assert.Contains(t, mock.queries[0], nodeMetricSnapshotTableName())
assert.Contains(t, mock.queries[0], "captured_at DESC")
assert.NotContains(t, mock.queries[0], "LIMIT ?")
require.Len(t, mock.queryArgs, 1)
require.Len(t, mock.queryArgs[0], 1)
assert.Equal(t, since, mock.queryArgs[0][0])
}
func TestListNodeMetricHourly_PrefersRollup(t *testing.T) {
ctx := context.Background()
hour := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
since := hour.Add(-1 * time.Hour)
mock := &mockConn{
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
return &mockRows{data: [][]any{{
hour, 42.5, 60.0, int64(100), int64(200), int64(10), int64(20), uint64(2),
}}}, nil
}
return nil, errors.New("raw path should not be used when rollup covers the window")
},
}
db.SetChConnForTest(mock)
t.Cleanup(func() { db.SetChConnForTest(nil) })
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{Since: since})
require.NoError(t, err)
require.Len(t, rows, 1)
assert.InDelta(t, 42.5, rows[0].AverageCPUUsagePercent, 1e-9)
assert.InDelta(t, 60.0, rows[0].AverageMemoryUsagePercent, 1e-9)
assert.Equal(t, int64(100), rows[0].NetworkRxBytes)
assert.Equal(t, 2, rows[0].ReportedNodes)
require.Len(t, mock.queries, 1)
assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
}
func TestListNodeMetricHourly_MergesRawGapsWithPartialRollup(t *testing.T) {
ctx := context.Background()
// 24h window starts far before the only rollup bucket (last hour).
since := time.Date(2026, 7, 9, 12, 0, 0, 0, time.UTC)
rollupHour := time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC)
rawHour := time.Date(2026, 7, 9, 15, 0, 0, 0, time.UTC)
mock := &mockConn{
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
return &mockRows{data: [][]any{{
rollupHour, 99.0, 99.0, int64(1), int64(1), int64(1), int64(1), uint64(1),
}}}, nil
}
if strings.Contains(query, nodeMetricSnapshotTableName()) {
return &mockRows{data: [][]any{
{rawHour, 12.0, 34.0, int64(5), int64(6), int64(7), int64(8), uint64(1)},
{rollupHour, 50.0, 50.0, int64(9), int64(9), int64(9), int64(9), uint64(1)},
}}, nil
}
return &mockRows{}, nil
},
}
db.SetChConnForTest(mock)
t.Cleanup(func() { db.SetChConnForTest(nil) })
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{Since: since})
require.NoError(t, err)
require.Len(t, rows, 2)
assert.Equal(t, rawHour, rows[0].Hour)
assert.InDelta(t, 12.0, rows[0].AverageCPUUsagePercent, 1e-9)
// Overlapping hour prefers rollup (99) over raw (50).
assert.Equal(t, rollupHour, rows[1].Hour)
assert.InDelta(t, 99.0, rows[1].AverageCPUUsagePercent, 1e-9)
require.GreaterOrEqual(t, len(mock.queries), 2)
assert.Contains(t, mock.queries[1], "lagInFrame")
}
func TestMergeNodeMetricHourlyPreferRollup(t *testing.T) {
h1 := time.Date(2026, 7, 10, 10, 0, 0, 0, time.UTC)
h2 := time.Date(2026, 7, 10, 11, 0, 0, 0, time.UTC)
merged := mergeNodeMetricHourlyPreferRollup(
[]NodeMetricHourly{{Hour: h2, AverageCPUUsagePercent: 80}},
[]NodeMetricHourly{
{Hour: h1, AverageCPUUsagePercent: 10},
{Hour: h2, AverageCPUUsagePercent: 20},
},
)
require.Len(t, merged, 2)
assert.Equal(t, h1, merged[0].Hour)
assert.InDelta(t, 10.0, merged[0].AverageCPUUsagePercent, 1e-9)
assert.Equal(t, h2, merged[1].Hour)
assert.InDelta(t, 80.0, merged[1].AverageCPUUsagePercent, 1e-9)
}
func TestHourlyRollupCoversWindow(t *testing.T) {
since := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
assert.True(t, hourlyRollupCoversWindow(since, since))
assert.True(t, hourlyRollupCoversWindow(since.Add(2*time.Hour), since))
assert.False(t, hourlyRollupCoversWindow(since.Add(3*time.Hour), since))
assert.True(t, hourlyRollupCoversWindow(time.Date(2026, 7, 11, 0, 0, 0, 0, time.UTC), time.Time{}))
}
func TestListNodeMetricHourly_FallsBackToRawOnRollupError(t *testing.T) {
ctx := context.Background()
hour := time.Date(2026, 7, 10, 13, 0, 0, 0, time.UTC)
mock := &mockConn{
queryFn: func(_ context.Context, query string, _ ...any) (driver.Rows, error) {
if strings.Contains(query, nodeMetricCapacityHourlyTableName()) {
return nil, errors.New("rollup missing")
}
if strings.Contains(query, nodeMetricSnapshotTableName()) {
return &mockRows{data: [][]any{{
hour, 10.0, 20.0, int64(1), int64(2), int64(3), int64(4), uint64(1),
}}}, nil
}
return &mockRows{}, nil
},
}
db.SetChConnForTest(mock)
t.Cleanup(func() { db.SetChConnForTest(nil) })
rows, err := ListNodeMetricHourly(ctx, NodeObservabilityFilter{})
require.NoError(t, err)
require.Len(t, rows, 1)
assert.InDelta(t, 10.0, rows[0].AverageCPUUsagePercent, 1e-9)
assert.Equal(t, int64(3), rows[0].DiskReadBytes)
require.GreaterOrEqual(t, len(mock.queries), 2)
assert.Contains(t, mock.queries[0], nodeMetricCapacityHourlyTableName())
assert.Contains(t, mock.queries[1], "lagInFrame")
}
@@ -0,0 +1,49 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"testing"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
db "Wavelet/plugins/infra/database"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestInsertNodeEdgeHealth_EmptyNodeID(t *testing.T) {
err := InsertNodeEdgeHealth(context.Background(), analyticsmodel.NodeEdgeHealth{})
require.NoError(t, err)
}
func TestInsertNodeEdgeHealth_UsesEdgeHealthBatchSQL(t *testing.T) {
ctx := context.Background()
mockBatch := &mockBatch{}
mockConn := &mockConn{
batch: mockBatch,
batchQuery: analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL(),
}
db.SetChConnForTest(mockConn)
t.Cleanup(func() { db.SetChConnForTest(nil) })
capturedAt := time.Now().UTC()
err := InsertNodeEdgeHealth(ctx, analyticsmodel.NodeEdgeHealth{
NodeID: "node-a",
CapturedAt: capturedAt,
Status: "",
Connections: 3,
CreatedAt: capturedAt,
})
require.NoError(t, err)
assert.True(t, mockConn.prepareCalled)
assert.Equal(t, analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL(), mockConn.preparedQuery)
assert.True(t, mockBatch.sendCalled)
require.Len(t, mockBatch.rows, 1)
assert.Equal(t, "node-a", mockBatch.rows[0][1])
assert.Equal(t, "unknown", mockBatch.rows[0][3]) // status default
assert.Equal(t, int64(3), mockBatch.rows[0][4]) // connections
}
@@ -0,0 +1,272 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"errors"
"fmt"
"strings"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
"Wavelet/pkg/idgen"
db "Wavelet/plugins/infra/database"
)
const edgeHealthStatusUnknown = "unknown"
// InsertNodeMetricSnapshot writes a single metric snapshot via the batch API.
func InsertNodeMetricSnapshot(ctx context.Context, snapshot analyticsmodel.NodeMetricSnapshot) error {
if strings.TrimSpace(snapshot.NodeID) == "" {
return nil
}
return BatchInsertNodeMetricSnapshots(ctx, []analyticsmodel.NodeMetricSnapshot{snapshot})
}
// BatchInsertNodeMetricSnapshots writes metric snapshots to ClickHouse.
func BatchInsertNodeMetricSnapshots(ctx context.Context, snapshots []analyticsmodel.NodeMetricSnapshot) error {
if len(snapshots) == 0 {
return nil
}
if db.ChConn == nil {
return errors.New("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeMetricSnapshot{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
now := time.Now().UTC()
for _, snapshot := range snapshots {
nodeID := strings.TrimSpace(snapshot.NodeID)
if nodeID == "" {
continue
}
id := snapshot.ID
if id == 0 {
id = idgen.NextUint64ID()
}
createdAt := snapshot.CreatedAt
if createdAt.IsZero() {
createdAt = now
}
if err := batch.Append(
id,
nodeID,
snapshot.CapturedAt.UTC(),
snapshot.CPUUsagePercent,
snapshot.MemoryUsedBytes,
snapshot.MemoryTotalBytes,
snapshot.StorageUsedBytes,
snapshot.StorageTotalBytes,
snapshot.DiskReadBytes,
snapshot.DiskWriteBytes,
snapshot.NetworkRxBytes,
snapshot.NetworkTxBytes,
createdAt.UTC(),
); err != nil {
return fmt.Errorf("append node metric snapshot to batch: %w", err)
}
}
if batch.Rows() == 0 {
return nil
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send clickhouse batch: %w", err)
}
return nil
}
func normalizeEdgeHealthStatus(status string) string {
status = strings.TrimSpace(status)
if status == "" {
return edgeHealthStatusUnknown
}
return status
}
// InsertNodeEdgeHealth writes a single edge health snapshot.
func InsertNodeEdgeHealth(ctx context.Context, row analyticsmodel.NodeEdgeHealth) error {
if strings.TrimSpace(row.NodeID) == "" {
return nil
}
return BatchInsertNodeEdgeHealth(ctx, []analyticsmodel.NodeEdgeHealth{row})
}
// BatchInsertNodeEdgeHealth writes L2 OpenResty health snapshots to ClickHouse.
func BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
if len(rows) == 0 {
return nil
}
if db.ChConn == nil {
return errors.New("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeEdgeHealth{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
now := time.Now().UTC()
for _, row := range rows {
nodeID := strings.TrimSpace(row.NodeID)
if nodeID == "" {
continue
}
id := row.ID
if id == 0 {
id = idgen.NextUint64ID()
}
createdAt := row.CreatedAt
if createdAt.IsZero() {
createdAt = now
}
capturedAt := row.CapturedAt.UTC()
if capturedAt.IsZero() {
capturedAt = now
}
if err := batch.Append(
id,
nodeID,
capturedAt,
normalizeEdgeHealthStatus(row.Status),
row.Connections,
createdAt.UTC(),
); err != nil {
return fmt.Errorf("append node edge health to batch: %w", err)
}
}
if batch.Rows() == 0 {
return nil
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send clickhouse batch: %w", err)
}
return nil
}
// InsertNodeObsFrps writes a single FRPS observation via the batch API.
func InsertNodeObsFrps(ctx context.Context, obs analyticsmodel.NodeObsFrps) error {
if strings.TrimSpace(obs.NodeID) == "" {
return nil
}
return BatchInsertNodeObsFrps(ctx, []analyticsmodel.NodeObsFrps{obs})
}
// BatchInsertNodeObsFrps writes FRPS observations to ClickHouse.
func BatchInsertNodeObsFrps(ctx context.Context, observations []analyticsmodel.NodeObsFrps) error {
if len(observations) == 0 {
return nil
}
if db.ChConn == nil {
return errors.New("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrps{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
now := time.Now().UTC()
for _, obs := range observations {
nodeID := strings.TrimSpace(obs.NodeID)
if nodeID == "" {
continue
}
id := obs.ID
if id == 0 {
id = idgen.NextUint64ID()
}
createdAt := obs.CreatedAt
if createdAt.IsZero() {
createdAt = now
}
capturedAt := obs.CapturedAt.UTC()
if capturedAt.IsZero() {
capturedAt = now
}
if err := batch.Append(
id,
nodeID,
capturedAt,
obs.FrpsConnections,
obs.FrpsProxyCount,
obs.FrpsClientCount,
obs.FrpsProxies,
createdAt.UTC(),
); err != nil {
return fmt.Errorf("append node frps observation to batch: %w", err)
}
}
if batch.Rows() == 0 {
return nil
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send clickhouse batch: %w", err)
}
return nil
}
// InsertNodeObsFrpc writes a single FRPC observation via the batch API.
func InsertNodeObsFrpc(ctx context.Context, obs analyticsmodel.NodeObsFrpc) error {
if strings.TrimSpace(obs.NodeID) == "" {
return nil
}
return BatchInsertNodeObsFrpc(ctx, []analyticsmodel.NodeObsFrpc{obs})
}
// BatchInsertNodeObsFrpc writes FRPC observations to ClickHouse.
func BatchInsertNodeObsFrpc(ctx context.Context, observations []analyticsmodel.NodeObsFrpc) error {
if len(observations) == 0 {
return nil
}
if db.ChConn == nil {
return errors.New("clickhouse connection is not initialized")
}
batch, err := db.ChConn.PrepareBatch(ctx, analyticsmodel.NodeObsFrpc{}.BatchInsertSQL())
if err != nil {
return fmt.Errorf("prepare clickhouse batch: %w", err)
}
now := time.Now().UTC()
for _, obs := range observations {
nodeID := strings.TrimSpace(obs.NodeID)
if nodeID == "" {
continue
}
id := obs.ID
if id == 0 {
id = idgen.NextUint64ID()
}
createdAt := obs.CreatedAt
if createdAt.IsZero() {
createdAt = now
}
capturedAt := obs.CapturedAt.UTC()
if capturedAt.IsZero() {
capturedAt = now
}
if err := batch.Append(
id,
nodeID,
capturedAt,
obs.TunnelStatus,
obs.ConnectedRelaysCount,
createdAt.UTC(),
); err != nil {
return fmt.Errorf("append node frpc observation to batch: %w", err)
}
}
if batch.Rows() == 0 {
return nil
}
if err := batch.Send(); err != nil {
return fmt.Errorf("send clickhouse batch: %w", err)
}
return nil
}
@@ -0,0 +1,9 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
const (
clickHouseLimitClause = " LIMIT ?"
clickHouseLimitOffsetClause = " LIMIT ? OFFSET ?"
)
@@ -0,0 +1,80 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
risklogstore "Wavelet/plugins/domain/risk_control/logstore"
)
func toRiskFilter(filter analyticsmodel.AccessLogFilter) risklogstore.AccessLogFilter {
return risklogstore.AccessLogFilter{
UserIDs: filter.UserIDs,
Path: filter.Path,
StartTime: filter.StartTime,
EndTime: filter.EndTime,
}
}
// BatchInsert writes user access logs via Wavelet risk_control.
func BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
return risklogstore.BatchInsert(ctx, logs)
}
// DeleteAllUserAccessLogs truncates user access logs via Wavelet risk_control.
func DeleteAllUserAccessLogs(ctx context.Context) (int64, error) {
return risklogstore.DeleteAllUserAccessLogs(ctx)
}
// CountAccessLogs counts user access logs via Wavelet risk_control.
func CountAccessLogs(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
return risklogstore.CountAccessLogs(ctx, toRiskFilter(filter))
}
// ListAccessLogs lists user access logs via Wavelet risk_control.
func ListAccessLogs(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
return risklogstore.ListAccessLogs(ctx, toRiskFilter(filter), page, pageSize)
}
// GetDailyTrend returns the daily trend via Wavelet risk_control.
func GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
src, err := risklogstore.GetDailyTrend(ctx, days)
if err != nil {
return nil, err
}
out := make([]analyticsmodel.DailyTrend, len(src))
for i, v := range src {
out[i] = analyticsmodel.DailyTrend{Date: v.Date, Count: v.Count}
}
return out, nil
}
// GetBrowserDistribution returns browser share via Wavelet risk_control.
func GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
src, err := risklogstore.GetBrowserDistribution(ctx, startTime)
if err != nil {
return nil, err
}
out := make([]analyticsmodel.BrowserShare, len(src))
for i, v := range src {
out[i] = analyticsmodel.BrowserShare{Browser: v.Browser, Count: v.Count}
}
return out, nil
}
// GetTopActiveUsers returns top users via Wavelet risk_control.
func GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
src, err := risklogstore.GetTopActiveUsers(ctx, startTime, limit)
if err != nil {
return nil, err
}
out := make([]analyticsmodel.TopUser, len(src))
for i, v := range src {
out[i] = analyticsmodel.TopUser{UserID: v.UserID, Count: v.Count}
}
return out, nil
}
@@ -0,0 +1,27 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
// Persistence and repository-layer parameter messages live here (unexported).
// Domain field validation used by model.Validate stays in internal/model/errs.go;
// repository may call model.Validate and return those errors as-is.
// Keep wording aligned with model where the same user-facing phrase applies,
// but do not import or re-export model unexported consts (would require exporting).
const (
errDatabaseNotInitialized = "database not initialized"
errConfigIntParseFailed = "配置 %s 的值 '%s' 无法转换为整数: %w"
errConfigDecimalParseFailed = "配置 %s 的值 '%s' 无法转换为decimal: %w"
errConfigBoolParseFailed = "配置 %s 的值 '%s' 无法转换为布尔值: %w"
errParseMenuDisplayConfigFailed = "解析目录显示配置失败: %w"
errAuthSourceNameRequired = "认证源名称不能为空"
errAuthSourceIDRequired = "认证源 ID 不能为空"
errExternalAccountBindingIncomplete = "外部账号绑定信息不完整"
errExternalAccountAlreadyBoundToAnother = "该外部账号已绑定到其他用户"
errUserIDRequired = "用户 ID 不能为空"
errExternalAccountBindingIDRequired = "绑定记录 ID 不能为空"
)
const colName = "name"
const colEnabled = "enabled"
@@ -0,0 +1,207 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"time"
"Wavelet/openflare/plugins/server/kernel/model"
"Wavelet/pkg/logger"
)
// CleanupSummary 汇总本次清理结果。
type CleanupSummary struct {
ActiveDatabase string `json:"active_database"`
// RetentionDays 访问日志(节点访问/用户访问)保留天数,按日志库读取。
RetentionDays int `json:"retention_days"`
// MetricRetentionDays 性能指标(CPU/内存/磁盘/网络)保留天数,三库共用短留存。
MetricRetentionDays int `json:"metric_retention_days"`
Deleted int64 `json:"deleted"`
// Tables 记录本次清理的物理表简写名(去掉 of_ 前缀,如 node_access_logs 对应
// of_node_access_logs;CH 侧物理表名相同,简写仅便于状态展示)。
Tables []string `json:"tables"`
}
// defaultLogRetentionDays 默认日志保留天数(配置缺失/非法时回退)。
const defaultLogRetentionDays = 90
// defaultMetricRetentionDays 默认性能指标保留天数(配置缺失/非法时回退)。
// 性能数据价值衰减快,默认短留存(3 天)。
const defaultMetricRetentionDays = 3
// partitionLeadMonths 清理时确保「当前月 + 未来 2 个月」分区持续存在。
const partitionLeadMonths = 2
// accessLogPartitionTables 按月分区的访问日志表(分区预建/空分区清理共用)。
var accessLogPartitionTables = []string{"of_node_access_logs", "w_user_access_logs"}
// retentionDaysForDatabase 按给定日志库读取保留天数(默认 90)。
func retentionDaysForDatabase(ctx context.Context, dbName string) int {
key := model.ConfigKeyLogRetentionDaysPostgres
switch dbName {
case dbNameSQLite:
key = model.ConfigKeyLogRetentionDaysSQLite
case dbNameClickHouse:
key = model.ConfigKeyLogRetentionDaysClickHouse
}
v, err := getConfig(ctx, key)
if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "读取日志保留天数配置失败(key=%s),回退默认 %d 天: %v", key, defaultLogRetentionDays, err)
}
return defaultLogRetentionDays
}
days, perr := strconv.Atoi(v)
if perr != nil || days <= 0 {
logger.ErrorF(ctx, "日志保留天数配置非法(key=%s, value=%q),回退默认 %d 天", key, v, defaultLogRetentionDays)
return defaultLogRetentionDays
}
return days
}
// metricRetentionDays 读取性能指标保留天数(三库共用,默认 3 天)。
func metricRetentionDays(ctx context.Context) int {
v, err := getConfig(ctx, model.ConfigKeyMetricRetentionDays)
if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "读取性能指标保留天数配置失败(key=%s),回退默认 %d 天: %v", model.ConfigKeyMetricRetentionDays, defaultMetricRetentionDays, err)
}
return defaultMetricRetentionDays
}
days, perr := strconv.Atoi(v)
if perr != nil || days <= 0 {
logger.ErrorF(ctx, "性能指标保留天数配置非法(key=%s, value=%q),回退默认 %d 天", model.ConfigKeyMetricRetentionDays, v, defaultMetricRetentionDays)
return defaultMetricRetentionDays
}
return days
}
// CleanupExpired 按当前激活库保留天数清理过期日志(每日由 system_cleanup 调用):
// 访问日志(节点访问/用户访问)按 log_retention_days_* 清理;
// 性能指标(CPU/内存/磁盘/网络)按三库共用的短留存 metric_retention_days 清理。
func CleanupExpired(ctx context.Context) (*CleanupSummary, error) {
dbName, err := resolveDatabase(ctx)
if err != nil {
return nil, fmt.Errorf("resolve active database: %w", err)
}
s, err := Active(ctx)
if err != nil {
return nil, err
}
days := retentionDaysForDatabase(ctx, dbName)
metricDays := metricRetentionDays(ctx)
cutoff := time.Now().AddDate(0, 0, -days)
metricCutoff := time.Now().AddDate(0, 0, -metricDays)
summary := &CleanupSummary{ActiveDatabase: dbName, RetentionDays: days, MetricRetentionDays: metricDays, Tables: []string{}}
// PG 分区表仅在迁移时预建「当前+2 月」分区,此处确保分区持续存在,
// 否则跨月后新写入会报 "no partition of relation found"(SQLite/CH 为 no-op)。
now := time.Now().UTC()
if err := s.AccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
return nil, fmt.Errorf("ensure partitions: %w", err)
}
// 先直接删除完全过期的整月分区(比逐行 DELETE 快几个数量级、无 MVCC/WAL 负担),
// 再对边界月份执行 DeleteBefore(边界月仍可能含未过期数据,不可整表删)。
if err := s.AccessLogs.DropExpiredPartitions(ctx, cutoff); err != nil {
return nil, fmt.Errorf("drop expired partitions: %w", err)
}
if err := cleanupTable("node_access_logs", func() (int64, error) {
return s.AccessLogs.DeleteBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
// 过期数据删除后清理旧月份空分区表,避免分区表无限累积;
// 仅删「当前月之前」且无数据的分区(best-effort,失败不阻断数据保留清理)。
if err := s.AccessLogs.DropEmptyPartitions(ctx, now); err != nil {
logger.WarnF(ctx, "drop empty log partitions failed: %v", err)
}
if err := cleanupTable("metric_snapshots", func() (int64, error) {
return s.Observability.DeleteMetricSnapshotsBefore(ctx, metricCutoff)
}, summary); err != nil {
return nil, err
}
if err := cleanupTable("edge_health", func() (int64, error) {
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
if err := cleanupTable("obs_frps", func() (int64, error) {
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
if err := cleanupTable("obs_frpc", func() (int64, error) {
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
}, summary); err != nil {
return nil, err
}
return summary, nil
}
func cleanupTable(name string, fn func() (int64, error), summary *CleanupSummary) error {
n, err := fn()
if err != nil {
return fmt.Errorf("cleanup %s: %w", name, err)
}
summary.Deleted += n
summary.Tables = append(summary.Tables, name)
return nil
}
// partitionStatementsRange 生成覆盖 [from, to] 全部月份的两表分区 DDL,
// 幂等 CREATE TABLE IF NOT EXISTS ... PARTITION OF ... FOR VALUES FROM ... TO ...。
// 入参为任意时间点:按各自所在月份生成,含 from 月与 to 月(to 常用 max+1 月兜底)。
func partitionStatementsRange(from, to time.Time) []string {
var out []string
start := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC)
end := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 1, 0)
for ; start.Before(end); start = start.AddDate(0, 1, 0) {
monthEnd := start.AddDate(0, 1, 0)
suffix := start.Format("200601")
fromDay := start.Format("2006-01-02")
toDay := monthEnd.Format("2006-01-02")
for _, table := range accessLogPartitionTables {
out = append(out, fmt.Sprintf(
"CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')",
table, suffix, table, fromDay, toDay))
}
}
return out
}
// partitionNameMonth 解析按月分区表名 <table>_YYYYMM 的所属月份;命名不匹配返回 (零值, false)。
func partitionNameMonth(table, name string) (time.Time, bool) {
suffix, ok := strings.CutPrefix(name, table+"_")
if !ok || len(suffix) != 6 {
return time.Time{}, false
}
m, err := time.Parse("200601", suffix)
if err != nil {
return time.Time{}, false
}
return m, true
}
// dropEligiblePartitionNames 返回 before 月份之前、命名合法的分区表名(是否为空由调用方校验)。
func dropEligiblePartitionNames(table string, names []string, before time.Time) []string {
beforeMonth := time.Date(before.Year(), before.Month(), 1, 0, 0, 0, 0, time.UTC)
out := make([]string, 0, len(names))
for _, name := range names {
month, ok := partitionNameMonth(table, name)
if !ok || !month.Before(beforeMonth) {
continue // 非法命名或当月/未来月分区,必须保留
}
out = append(out, name)
}
return out
}
@@ -0,0 +1,403 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"Wavelet/openflare/plugins/server/kernel/model"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
db "Wavelet/plugins/infra/database"
)
// cleanupTestModels 清理涉及的 5 张日志/可观测表。
func cleanupTestModels() []any {
return []any{
&analyticsmodel.NodeAccessLog{},
&analyticsmodel.NodeMetricSnapshot{},
&analyticsmodel.NodeEdgeHealth{},
&analyticsmodel.NodeObsFrps{},
&analyticsmodel.NodeObsFrpc{},
}
}
// newCleanupTestDB 构造内存 sqlite 库并注入 db.DB(CleanupExpired 经 Active → buildStore 使用)。
func newCleanupTestDB(t *testing.T) *gorm.DB {
t.Helper()
dsn := fmt.Sprintf("file:logstore-cleanup-%d?mode=memory&cache=shared", atomic.AddInt64(&testGormStoreSeq, 1))
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := gdb.AutoMigrate(cleanupTestModels()...); err != nil {
t.Fatalf("automigrate: %v", err)
}
db.SetDB(gdb)
t.Cleanup(func() { db.SetDB(nil) })
return gdb
}
// TestCleanupExpiredSQLite 验证 sqlite 激活库的过期日志清理:
// 注入 log_retention_days_sqlite=30,40 天前的 5 表记录被删、昨天的保留。
func TestCleanupExpiredSQLite(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case logDatabaseKey:
return "sqlite", nil
case model.ConfigKeyLogRetentionDaysSQLite:
return "30", nil
case model.ConfigKeyMetricRetentionDays:
return "3", nil
}
return "", nil
})
defer ResetForTest()
gdb := newCleanupTestDB(t)
ctx := context.Background()
old := time.Now().AddDate(0, 0, -40).UTC()
recent := time.Now().AddDate(0, 0, -1).UTC()
if err := gdb.Create([]analyticsmodel.NodeAccessLog{
{ID: 1, NodeID: "n1", LoggedAt: old, RemoteAddr: "1.1.1.1"},
{ID: 2, NodeID: "n1", LoggedAt: recent, RemoteAddr: "2.2.2.2"},
}).Error; err != nil {
t.Fatalf("seed node access logs: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeMetricSnapshot{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed metric snapshots: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeEdgeHealth{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed edge health: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeObsFrps{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed obs frps: %v", err)
}
if err := gdb.Create([]analyticsmodel.NodeObsFrpc{
{ID: 1, NodeID: "n1", CapturedAt: old},
{ID: 2, NodeID: "n1", CapturedAt: recent},
}).Error; err != nil {
t.Fatalf("seed obs frpc: %v", err)
}
summary, err := CleanupExpired(ctx)
if err != nil {
t.Fatalf("CleanupExpired: %v", err)
}
if summary.ActiveDatabase != "sqlite" {
t.Fatalf("ActiveDatabase = %q, want sqlite", summary.ActiveDatabase)
}
if summary.RetentionDays != 30 {
t.Fatalf("RetentionDays = %d, want 30", summary.RetentionDays)
}
if summary.MetricRetentionDays != 3 {
t.Fatalf("MetricRetentionDays = %d, want 3", summary.MetricRetentionDays)
}
if summary.Deleted != 5 {
t.Fatalf("Deleted = %d, want 5", summary.Deleted)
}
if len(summary.Tables) != 5 {
t.Fatalf("Tables = %v, want 5 tables", summary.Tables)
}
assertCount := func(m any, want int64, label string) {
t.Helper()
var n int64
if err := gdb.Model(m).Count(&n).Error; err != nil {
t.Fatalf("count %s: %v", label, err)
}
if n != want {
t.Fatalf("%s count = %d, want %d", label, n, want)
}
}
assertCount(&analyticsmodel.NodeAccessLog{}, 1, "node_access_logs")
assertCount(&analyticsmodel.NodeMetricSnapshot{}, 1, "metric_snapshots")
assertCount(&analyticsmodel.NodeEdgeHealth{}, 1, "edge_health")
assertCount(&analyticsmodel.NodeObsFrps{}, 1, "obs_frps")
assertCount(&analyticsmodel.NodeObsFrpc{}, 1, "obs_frpc")
var kept analyticsmodel.NodeAccessLog
if err := gdb.First(&kept).Error; err != nil {
t.Fatalf("recent node access log missing: %v", err)
}
if kept.ID != 2 {
t.Fatalf("kept log ID = %d, want 2 (recent)", kept.ID)
}
}
// TestCleanupExpiredMetricShortRetention 回归:性能指标(CPU/内存/磁盘/网络)按三库共用
// 的短留存(默认 3 天)清理,与访问日志保留天数(log_retention_days_*)解耦。
// 10 天前的指标快照被删(> 3 天),同日期的访问日志保留(< 30 天)。
func TestCleanupExpiredMetricShortRetention(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case logDatabaseKey:
return "sqlite", nil
case model.ConfigKeyLogRetentionDaysSQLite:
return "30", nil
case model.ConfigKeyMetricRetentionDays:
return "3", nil
}
return "", nil
})
defer ResetForTest()
gdb := newCleanupTestDB(t)
ctx := context.Background()
mid := time.Now().AddDate(0, 0, -10).UTC() // 10 天前:超指标留存、未超日志留存
if err := gdb.Create(&analyticsmodel.NodeAccessLog{ID: 1, NodeID: "n1", LoggedAt: mid, RemoteAddr: "1.1.1.1"}).Error; err != nil {
t.Fatalf("seed node access log: %v", err)
}
if err := gdb.Create(&analyticsmodel.NodeMetricSnapshot{ID: 1, NodeID: "n1", CapturedAt: mid}).Error; err != nil {
t.Fatalf("seed metric snapshot: %v", err)
}
summary, err := CleanupExpired(ctx)
if err != nil {
t.Fatalf("CleanupExpired: %v", err)
}
if summary.RetentionDays != 30 || summary.MetricRetentionDays != 3 {
t.Fatalf("retention = (%d, %d), want (30, 3)", summary.RetentionDays, summary.MetricRetentionDays)
}
var accessCount, metricCount int64
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&accessCount).Error; err != nil {
t.Fatalf("count access logs: %v", err)
}
if err := gdb.Model(&analyticsmodel.NodeMetricSnapshot{}).Count(&metricCount).Error; err != nil {
t.Fatalf("count metric snapshots: %v", err)
}
if accessCount != 1 {
t.Fatalf("node_access_logs count = %d, want 1 (10 天在 30 天日志留存内)", accessCount)
}
if metricCount != 0 {
t.Fatalf("metric_snapshots count = %d, want 0 (10 天超 3 天指标留存)", metricCount)
}
}
// TestMetricRetentionDays 覆盖性能指标保留天数读取:合法值、非法值回退默认 3。
func TestMetricRetentionDays(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case model.ConfigKeyMetricRetentionDays:
return "5", nil
}
return "", nil
})
if got := metricRetentionDays(context.Background()); got != 5 {
t.Fatalf("metricRetentionDays = %d, want 5", got)
}
// 非法值(非数字/<=0)回退默认 3。
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case model.ConfigKeyMetricRetentionDays:
return "abc", nil
}
return "", nil
})
if got := metricRetentionDays(context.Background()); got != defaultMetricRetentionDays {
t.Fatalf("metricRetentionDays invalid value = %d, want %d", got, defaultMetricRetentionDays)
}
// reader 报错回退默认 3。
SetConfigReader(func(_ context.Context, _ string) (string, error) {
return "", fmt.Errorf("boom")
})
if got := metricRetentionDays(context.Background()); got != defaultMetricRetentionDays {
t.Fatalf("metricRetentionDays reader error = %d, want %d", got, defaultMetricRetentionDays)
}
}
// TestRetentionDaysForDatabase 覆盖保留天数读取:按激活库选 key、非法值回退默认 90。
func TestRetentionDaysForDatabase(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case logDatabaseKey:
return "sqlite", nil
case model.ConfigKeyLogRetentionDaysSQLite:
return "30", nil
}
return "", nil
})
if got := retentionDaysForDatabase(context.Background(), "sqlite"); got != 30 {
t.Fatalf("retentionDaysForDatabase = %d, want 30", got)
}
// 非法值(非数字/<=0)回退默认 90。
SetConfigReader(func(_ context.Context, key string) (string, error) {
switch key {
case logDatabaseKey:
return "postgres", nil
case model.ConfigKeyLogRetentionDaysPostgres:
return "abc", nil
}
return "", nil
})
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
t.Fatalf("retentionDaysForDatabase invalid value = %d, want 90", got)
}
// reader 报错回退默认 90。
SetConfigReader(func(_ context.Context, _ string) (string, error) {
return "", fmt.Errorf("boom")
})
if got := retentionDaysForDatabase(context.Background(), "postgres"); got != 90 {
t.Fatalf("retentionDaysForDatabase reader error = %d, want 90", got)
}
}
// TestPartitionStatements 验证 PG 分区 DDL 生成:当前月 + 未来 2 个月 × 2 表,
// 幂等 PARTITION OF 语句与迁移 SQL 命名一致(含跨年)。
func TestPartitionStatements(t *testing.T) {
now := time.Date(2026, 8, 15, 10, 0, 0, 0, time.UTC)
stmts := partitionStatementsRange(now, now.AddDate(0, 2, 0))
if len(stmts) != 6 {
t.Fatalf("partitionStatements len = %d, want 6", len(stmts))
}
want := []string{
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202608 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202608 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-08-01') TO ('2026-09-01')",
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202609 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202609 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-09-01') TO ('2026-10-01')",
"CREATE TABLE IF NOT EXISTS of_node_access_logs_202610 PARTITION OF of_node_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
"CREATE TABLE IF NOT EXISTS w_user_access_logs_202610 PARTITION OF w_user_access_logs FOR VALUES FROM ('2026-10-01') TO ('2026-11-01')",
}
for i, w := range want {
if stmts[i] != w {
t.Fatalf("stmt[%d] = %q, want %q", i, stmts[i], w)
}
}
// 跨年:2026-11 → 202611, 202612, 202701。
nov := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
suffixes := []string{"202611", "202612", "202701"}
for _, stmt := range partitionStatementsRange(nov, nov.AddDate(0, 2, 0)) {
if !hasAnySuffix(stmt, suffixes) {
t.Fatalf("statement lacks expected month suffix: %s", stmt)
}
}
}
func hasAnySuffix(stmt string, suffixes []string) bool {
for _, table := range []string{"of_node_access_logs", "w_user_access_logs"} {
for _, suf := range suffixes {
if strings.Contains(stmt, table+"_"+suf) {
return true
}
}
}
return false
}
// TestPartitionNameMonth 覆盖按月分区表名解析:合法命名返回所属月份,非法/其它表前缀返回 false。
func TestPartitionNameMonth(t *testing.T) {
cases := []struct {
table string
name string
want string // 期望 "YYYY-MM";空串表示应解析失败
}{
{"of_node_access_logs", "of_node_access_logs_202608", "2026-08"},
{"w_user_access_logs", "w_user_access_logs_202612", "2026-12"},
{"of_node_access_logs", "w_user_access_logs_202608", ""}, // 其它表前缀
{"of_node_access_logs", "of_node_access_logs_20268", ""}, // 位数不足
{"of_node_access_logs", "of_node_access_logs_202613", ""}, // 非法月份
{"of_node_access_logs", "of_node_access_logs_default", ""}, // 非数字后缀
}
for _, c := range cases {
got, ok := partitionNameMonth(c.table, c.name)
if c.want == "" {
if ok {
t.Fatalf("partitionNameMonth(%q, %q) ok = true, want false", c.table, c.name)
}
continue
}
if !ok || got.Format("2006-01") != c.want {
t.Fatalf("partitionNameMonth(%q, %q) = %v, want %s", c.table, c.name, got, c.want)
}
}
}
// TestDropEligiblePartitionNames 覆盖空分区清理筛选:只保留 before 月份之前、命名合法的分区。
func TestDropEligiblePartitionNames(t *testing.T) {
before := time.Date(2026, 10, 15, 0, 0, 0, 0, time.UTC)
names := []string{
"of_node_access_logs_202608",
"of_node_access_logs_202609",
"of_node_access_logs_202610", // 当月:保留
"of_node_access_logs_202611", // 未来:保留
"of_node_access_logs_default", // 非法命名:忽略
}
got := dropEligiblePartitionNames("of_node_access_logs", names, before)
want := []string{"of_node_access_logs_202608", "of_node_access_logs_202609"}
if len(got) != len(want) {
t.Fatalf("eligible = %v, want %v", got, want)
}
for i, w := range want {
if got[i] != w {
t.Fatalf("eligible[%d] = %q, want %q", i, got[i], w)
}
}
// 月初边界:before 恰为当月 1 日 0 点,当月分区仍保留。
first := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
if got := dropEligiblePartitionNames("of_node_access_logs", []string{"of_node_access_logs_202610"}, first); len(got) != 0 {
t.Fatalf("eligible at month boundary = %v, want empty", got)
}
}
// TestDropExpiredPartitionsSQLiteNoop 验证 SQLite 下 DropExpiredPartitions 为 no-op:
// 直接返回 nil、不触碰任何分区 SQL(SQLite 无分区),数据不受影响。
func TestDropExpiredPartitionsSQLiteNoop(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logDatabaseKey {
return "sqlite", nil
}
return "", nil
})
defer ResetForTest()
gdb := newCleanupTestDB(t)
ctx := context.Background()
if err := gdb.Create(&analyticsmodel.NodeAccessLog{ID: 1, NodeID: "n1", LoggedAt: time.Now().AddDate(0, 0, -100).UTC(), RemoteAddr: "1.1.1.1"}).Error; err != nil {
t.Fatalf("seed node access log: %v", err)
}
store := newGormStore(gdb)
if err := store.DropExpiredPartitions(ctx, time.Now().AddDate(0, 0, -90)); err != nil {
t.Fatalf("DropExpiredPartitions on sqlite: %v", err)
}
var n int64
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&n).Error; err != nil {
t.Fatalf("count node access logs: %v", err)
}
if n != 1 {
t.Fatalf("node access log count = %d, want 1(no-op 不应删除任何行)", n)
}
}
@@ -0,0 +1,742 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"math"
"time"
"Wavelet/openflare/plugins/server/kernel/model"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
analyticsrepo "Wavelet/openflare/plugins/server/kernel/repository/analytics"
db "Wavelet/plugins/infra/database"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)
// clickhouseLogStore 实现 AccessLogStore / ObservabilityStore / StatusStore,
// 逐方法委托 analyticsrepo(CH 原生 batch 写入,零性能损耗)。
// UserAccessLogStore 由 clickhouseUserAccessLogStore 实现(List/Count 方法名已被
// AccessLogStore 占用,Go 不允许同名不同签名方法)。
type clickhouseLogStore struct {
// skipFreeze 为 true 时跳过迁移冻结检查(仅迁移目标 store 使用)。
skipFreeze bool
}
func newClickHouseStore() *clickhouseLogStore { return &clickhouseLogStore{} }
// 编译期断言。
var (
_ AccessLogStore = (*clickhouseLogStore)(nil)
_ ObservabilityStore = (*clickhouseLogStore)(nil)
_ StatusStore = (*clickhouseLogStore)(nil)
_ UserAccessLogStore = (*clickhouseUserAccessLogStore)(nil)
)
func chConnErr() error {
if db.ChConn == nil {
return errors.New("clickhouse connection is not initialized")
}
return nil
}
// ensureWritable 迁移冻结期拒绝写入。
func (s *clickhouseLogStore) ensureWritable(ctx context.Context) error {
if !s.skipFreeze && Migrating(ctx) {
return ErrMigrating
}
return nil
}
// ---- AccessLogStore ----
// InsertBatch 节点访问日志写入入口:冻结检查后经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
rows := make([]analyticsmodel.NodeAccessLog, 0, len(records))
for _, r := range records {
if r == nil {
continue
}
rows = append(rows, toAnalyticsNodeAccessLog(r))
}
if h := currentAccessLogHooks().QueueNodeAccessLogs; h != nil {
h(rows)
}
return nil
}
// BatchInsertNodeAccessLogs 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeAccessLogs(ctx, rows)
}
func (s *clickhouseLogStore) List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
rows, err := analyticsrepo.ListNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
if err != nil {
return nil, err
}
return fromAnalyticsNodeAccessLogs(rows), nil
}
func (s *clickhouseLogStore) Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
return analyticsrepo.CountNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
func (s *clickhouseLogStore) RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
rows, err := analyticsrepo.RegionCountsNodeAccessLogs(ctx, nodeID, since, limit)
if err != nil {
return nil, err
}
out := make([]*model.OpenFlareAccessLogRegionCount, len(rows))
for i, r := range rows {
out[i] = &model.OpenFlareAccessLogRegionCount{Region: r.Region, Count: r.Count}
}
return out, nil
}
func (s *clickhouseLogStore) BucketAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error) {
return analyticsrepo.BucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
}
func (s *clickhouseLogStore) CountBuckets(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
}
func (s *clickhouseLogStore) BucketDimensions(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error) {
return analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, bucketSeconds)
}
func (s *clickhouseLogStore) IPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error) {
return analyticsrepo.IPAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), exactRemoteAddr)
}
func (s *clickhouseLogStore) IPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error) {
return analyticsrepo.IPSummariesNodeAccessLogs(ctx, toNodeAccessLogFilter(query), recentSince)
}
func (s *clickhouseLogStore) CountIPSummaries(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, error) {
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
func (s *clickhouseLogStore) WAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error) {
return analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
}
func (s *clickhouseLogStore) IPTrend(ctx context.Context, query model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error) {
return analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(query), bucketSeconds)
}
func (s *clickhouseLogStore) TrafficSummary(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
row, err := analyticsrepo.TrafficSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
if err != nil {
return model.OpenFlareAccessLogTrafficSummary{}, err
}
return model.OpenFlareAccessLogTrafficSummary{
RequestCount: row.RequestCount,
ErrorCount: row.ErrorCount,
UniqueIPCount: row.UniqueIPCount,
BytesSent: row.BytesSent,
RequestLength: row.RequestLength,
NodeCount: row.NodeCount,
}, nil
}
func (s *clickhouseLogStore) ValueCounts(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
rows, err := analyticsrepo.ValueCountsNodeAccessLogs(ctx, toNodeAccessLogFilter(query), column, limit)
if err != nil {
return nil, err
}
out := make([]model.OpenFlareAccessLogValueCount, len(rows))
for i, r := range rows {
out[i] = model.OpenFlareAccessLogValueCount{Value: r.Value, Count: r.Count}
}
return out, nil
}
func (s *clickhouseLogStore) NodeAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
rows, err := analyticsrepo.NodeAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(query))
if err != nil {
return nil, err
}
out := make([]model.OpenFlareAccessLogNodeAggregate, len(rows))
for i, r := range rows {
out[i] = model.OpenFlareAccessLogNodeAggregate{NodeID: r.NodeID, RequestCount: r.RequestCount, ErrorCount: r.ErrorCount, UniqueIPCount: r.UniqueIPCount}
}
return out, nil
}
func (s *clickhouseLogStore) DeleteAll(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeAccessLogs(ctx)
}
func (s *clickhouseLogStore) DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeAccessLogsBefore(ctx, cutoff)
}
func (s *clickhouseLogStore) DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeAccessLogsByNodeBefore(ctx, nodeID, before)
}
// ListForMigration 按 id 升序分页读取(迁移复制用):直接查询 CH 原生表。
func (s *clickhouseLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error) {
if err := chConnErr(); err != nil {
return nil, err
}
rows, err := db.ChConn.Query(ctx, `
SELECT `+analyticsmodel.NodeAccessLog{}.InsertColumns()+`
FROM `+analyticsmodel.NodeAccessLog{}.TableName()+`
WHERE id > ?
ORDER BY id ASC
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
if err != nil {
return nil, fmt.Errorf("list node access logs for migration: %w", err)
}
defer func() { _ = rows.Close() }()
var result []analyticsmodel.NodeAccessLog
for rows.Next() {
var item analyticsmodel.NodeAccessLog
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.LoggedAt,
&item.RemoteAddr,
&item.Region,
&item.Host,
&item.Path,
&item.UserAgent,
&item.CacheStatus,
&item.StatusCode,
&item.BytesSent,
&item.RequestLength,
&item.RequestTimeMs,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node access log row: %w", err)
}
item.LoggedAt = item.LoggedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
// ---- ObservabilityStore ----
// InsertMetricSnapshot 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueMetricSnapshot; h != nil {
h(toAnalyticsNodeMetricSnapshot(record))
}
return nil
}
func (s *clickhouseLogStore) ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
rows, err := analyticsrepo.ListNodeMetricSnapshots(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeMetricSnapshots(rows), nil
}
func (s *clickhouseLogStore) DeleteAllMetricSnapshots(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeMetricSnapshots(ctx)
}
// ListTrafficHourly 委托 analyticsrepo 读 of_access_log_hourly rollup(M5 口径,UV 恒 0)。
func (s *clickhouseLogStore) ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error) {
return analyticsrepo.ListNodeTrafficHourly(ctx, toNodeObservabilitySince(nodeID, since))
}
// ListAccessLogHourly 委托 analyticsrepo 读 of_access_log_hourly rollup。
func (s *clickhouseLogStore) ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error) {
return analyticsrepo.ListAccessLogHourly(ctx, toNodeObservabilitySince(nodeID, since))
}
// ListMetricHourly 委托 analyticsrepo ListNodeMetricHourly:rollup 覆盖窗口时读
// of_node_metric_capacity_hourly,否则按 mergeNodeMetricHourlyPreferRollup 合并 raw 兜底。
func (s *clickhouseLogStore) ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error) {
return analyticsrepo.ListNodeMetricHourly(ctx, toNodeObservabilitySince(nodeID, since))
}
func (s *clickhouseLogStore) DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeMetricSnapshotsBefore(ctx, cutoff)
}
// BatchInsertNodeMetricSnapshots 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeMetricSnapshots(ctx, rows)
}
// InsertEdgeHealth 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueEdgeHealth; h != nil {
h(toAnalyticsNodeEdgeHealth(record))
}
return nil
}
func (s *clickhouseLogStore) ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
rows, err := analyticsrepo.ListNodeEdgeHealth(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeEdgeHealths(rows), nil
}
func (s *clickhouseLogStore) DeleteAllEdgeHealth(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeEdgeHealth(ctx)
}
func (s *clickhouseLogStore) DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeEdgeHealthBefore(ctx, cutoff)
}
// BatchInsertNodeEdgeHealth 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeEdgeHealth(ctx, rows)
}
// InsertNodeObservationFrps 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueNodeObsFrps; h != nil {
h(toAnalyticsNodeObsFrps(record))
}
return nil
}
func (s *clickhouseLogStore) ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
rows, err := analyticsrepo.ListNodeObsFrps(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeObsFrps(rows), nil
}
func (s *clickhouseLogStore) DeleteAllNodeObservationFrps(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeObsFrps(ctx)
}
func (s *clickhouseLogStore) DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeObsFrpsBefore(ctx, cutoff)
}
// BatchInsertNodeObsFrps 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeObsFrps(ctx, rows)
}
// InsertNodeObservationFrpc 写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
func (s *clickhouseLogStore) InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
if record == nil {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
if h := currentObservabilityHooks().QueueNodeObsFrpc; h != nil {
h(toAnalyticsNodeObsFrpc(record))
}
return nil
}
func (s *clickhouseLogStore) ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
rows, err := analyticsrepo.ListNodeObsFrpc(ctx, toNodeObservabilityFilter(nodeID, since, limit))
if err != nil {
return nil, err
}
return fromAnalyticsNodeObsFrpc(rows), nil
}
func (s *clickhouseLogStore) DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllNodeObsFrpc(ctx)
}
func (s *clickhouseLogStore) DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteNodeObsFrpcBefore(ctx, cutoff)
}
// MigrationRange 返回 of_node_access_logs.logged_at 的最小/最大值(空表返回零值)。
func (s *clickhouseLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
return chMigrationRange(ctx, analyticsmodel.NodeAccessLog{}.TableName(), "logged_at")
}
// EnsurePartitions 是 CH 分支 no-op(CH 无 PG 式分区)。
func (s *clickhouseLogStore) EnsurePartitions(_ context.Context, _, _ time.Time) error {
return nil
}
// DropEmptyPartitions 是 CH 分支 no-op(CH 分区随数据删除自动消失,无独立分区表)。
func (s *clickhouseLogStore) DropEmptyPartitions(_ context.Context, _ time.Time) error {
return nil
}
// DropExpiredPartitions 是 CH 分支 no-op(CH 无 PG 式分区,retention 仍走 DeleteBefore)。
func (s *clickhouseLogStore) DropExpiredPartitions(_ context.Context, _ time.Time) error {
return nil
}
// chMigrationRange 查询 CH 表时间列 MIN/MAX;空表(NULL)返回零值。
func chMigrationRange(ctx context.Context, table, column string) (time.Time, time.Time, error) {
if err := chConnErr(); err != nil {
return time.Time{}, time.Time{}, err
}
var minTime, maxTime *time.Time
if err := db.ChConn.QueryRow(ctx,
"SELECT min("+column+"), max("+column+") FROM "+table,
).Scan(&minTime, &maxTime); err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("query migration range %s: %w", table, err)
}
if minTime == nil || maxTime == nil {
return time.Time{}, time.Time{}, nil
}
return minTime.UTC(), maxTime.UTC(), nil
}
// BatchInsertNodeObsFrpc 是 batchwriter flush 目标:CH 原生批量写入。
func (s *clickhouseLogStore) BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error {
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsertNodeObsFrpc(ctx, rows)
}
// ListMetricSnapshotsForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error) {
return chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeMetricSnapshot{}.TableName(),
analyticsmodel.NodeMetricSnapshot{}.InsertColumns(),
func(rows driver.Rows) ([]analyticsmodel.NodeMetricSnapshot, error) {
var result []analyticsmodel.NodeMetricSnapshot
for rows.Next() {
var item analyticsmodel.NodeMetricSnapshot
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.CPUUsagePercent,
&item.MemoryUsedBytes,
&item.MemoryTotalBytes,
&item.StorageUsedBytes,
&item.StorageTotalBytes,
&item.DiskReadBytes,
&item.DiskWriteBytes,
&item.NetworkRxBytes,
&item.NetworkTxBytes,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node metric snapshot row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
})
}
// chObsRow 迁移读取共用的双字段观测行(字符串状态 + 数值计数):
// edge_health(status/connections)与 obs_frpc(tunnel_status/connected_relays_count)同形状。
type chObsRow struct {
ID uint64
NodeID string
CapturedAt time.Time
Status string
Count int64
CreatedAt time.Time
}
// countToInt32 将观测计数转为 int32(防御溢出;观测计数远小于 int32 上限)。
func countToInt32(v int64) int32 {
if v > math.MaxInt32 {
return math.MaxInt32
}
if v < math.MinInt32 {
return math.MinInt32
}
return int32(v)
}
// scanChObsRow 扫描 chObsRow(含 UTC 归一化)。
func scanChObsRow(rows driver.Rows) ([]chObsRow, error) {
var result []chObsRow
for rows.Next() {
var item chObsRow
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.Status,
&item.Count,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan observation row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
}
// ListEdgeHealthForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error) {
rows, err := chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeEdgeHealth{}.TableName(),
analyticsmodel.NodeEdgeHealth{}.InsertColumns(),
scanChObsRow)
if err != nil {
return nil, err
}
out := make([]analyticsmodel.NodeEdgeHealth, len(rows))
for i, r := range rows {
out[i] = analyticsmodel.NodeEdgeHealth{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, Status: r.Status, Connections: r.Count, CreatedAt: r.CreatedAt}
}
return out, nil
}
// ListNodeObsFrpsForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error) {
return chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeObsFrps{}.TableName(),
analyticsmodel.NodeObsFrps{}.InsertColumns(),
func(rows driver.Rows) ([]analyticsmodel.NodeObsFrps, error) {
var result []analyticsmodel.NodeObsFrps
for rows.Next() {
var item analyticsmodel.NodeObsFrps
if err := rows.Scan(
&item.ID,
&item.NodeID,
&item.CapturedAt,
&item.FrpsConnections,
&item.FrpsProxyCount,
&item.FrpsClientCount,
&item.FrpsProxies,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan node frps observation row: %w", err)
}
item.CapturedAt = item.CapturedAt.UTC()
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
})
}
// ListNodeObsFrpcForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseLogStore) ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error) {
rows, err := chListForMigration(ctx, afterID, limit,
analyticsmodel.NodeObsFrpc{}.TableName(),
analyticsmodel.NodeObsFrpc{}.InsertColumns(),
scanChObsRow)
if err != nil {
return nil, err
}
out := make([]analyticsmodel.NodeObsFrpc, len(rows))
for i, r := range rows {
out[i] = analyticsmodel.NodeObsFrpc{ID: r.ID, NodeID: r.NodeID, CapturedAt: r.CapturedAt, TunnelStatus: r.Status, ConnectedRelaysCount: countToInt32(r.Count), CreatedAt: r.CreatedAt}
}
return out, nil
}
// chListForMigration 执行按 id 升序分页的 CH 原生表查询,并交给 scanner 扫描。
func chListForMigration[T any](ctx context.Context, afterID uint64, limit int, table, columns string, scanner func(driver.Rows) ([]T, error)) ([]T, error) {
if err := chConnErr(); err != nil {
return nil, err
}
rows, err := db.ChConn.Query(ctx, `
SELECT `+columns+`
FROM `+table+`
WHERE id > ?
ORDER BY id ASC
LIMIT ?`, afterID, limitOr(limit, migrationPageSize))
if err != nil {
return nil, fmt.Errorf("list %s for migration: %w", table, err)
}
defer func() { _ = rows.Close() }()
return scanner(rows)
}
// ---- StatusStore ----
// ActiveDatabase 返回当前日志主库名(CH 分支固定 clickhouse)。
func (s *clickhouseLogStore) ActiveDatabase(_ context.Context) (string, error) {
return dbNameClickHouse, nil
}
// ClickHouseOperationalStats 委托 analyticsrepo 汇总 CH 运行状态。
func (s *clickhouseLogStore) ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) {
return analyticsrepo.GetClickHouseOperationalStats(ctx)
}
// ---- UserAccessLogStore ----
// clickhouseUserAccessLogStore 实现 UserAccessLogStore。clickhouseLogStore 已占用
// List/Count 方法名(AccessLogStore 接口),Go 不允许同名不同签名方法,故用户访问日志
// 用独立类型嵌入同一 clickhouseLogStore(与 userAccessLogGormStore 同构),复用 ensureWritable。
type clickhouseUserAccessLogStore struct {
*clickhouseLogStore
}
func newClickHouseUserAccessLogStore() *clickhouseUserAccessLogStore {
return &clickhouseUserAccessLogStore{clickhouseLogStore: newClickHouseStore()}
}
// BatchInsert 是 batchwriter flush 目标:CH 原生批量写入;冻结期拒绝写入,空批次直接返回。
func (s *clickhouseUserAccessLogStore) BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error {
if len(logs) == 0 {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
return analyticsrepo.BatchInsert(ctx, logs)
}
// DeleteAll 清空全部用户访问日志(TRUNCATE 语义,迁移「覆盖目标库已有日志」幂等前提用)。
func (s *clickhouseUserAccessLogStore) DeleteAll(ctx context.Context) (int64, error) {
if err := s.ensureWritable(ctx); err != nil {
return 0, err
}
return analyticsrepo.DeleteAllUserAccessLogs(ctx)
}
// ListForMigration 按 id 升序分页读取(迁移复制用)。
func (s *clickhouseUserAccessLogStore) ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error) {
return chListForMigration(ctx, afterID, limit,
analyticsmodel.UserAccessLog{}.TableName(),
analyticsmodel.UserAccessLog{}.InsertColumns(),
func(rows driver.Rows) ([]analyticsmodel.UserAccessLog, error) {
var result []analyticsmodel.UserAccessLog
for rows.Next() {
var item analyticsmodel.UserAccessLog
if err := rows.Scan(
&item.ID,
&item.UserID,
&item.Path,
&item.Method,
&item.IP,
&item.UserAgent,
&item.Headers,
&item.Status,
&item.Latency,
&item.CreatedAt,
); err != nil {
return nil, fmt.Errorf("scan user access log row: %w", err)
}
item.CreatedAt = item.CreatedAt.UTC()
result = append(result, item)
}
return result, nil
})
}
// MigrationRange 返回 w_user_access_logs.created_at 的最小/最大值(空表返回零值)。
func (s *clickhouseUserAccessLogStore) MigrationRange(ctx context.Context) (time.Time, time.Time, error) {
return chMigrationRange(ctx, analyticsmodel.UserAccessLog{}.TableName(), "created_at")
}
func (s *clickhouseUserAccessLogStore) Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error) {
return analyticsrepo.CountAccessLogs(ctx, filter)
}
func (s *clickhouseUserAccessLogStore) List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error) {
return analyticsrepo.ListAccessLogs(ctx, filter, page, pageSize)
}
func (s *clickhouseUserAccessLogStore) GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error) {
return analyticsrepo.GetDailyTrend(ctx, days)
}
func (s *clickhouseUserAccessLogStore) GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error) {
return analyticsrepo.GetBrowserDistribution(ctx, startTime)
}
func (s *clickhouseUserAccessLogStore) GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error) {
return analyticsrepo.GetTopActiveUsers(ctx, startTime, limit)
}
// toNodeObservabilityFilter 构造 CH 可观测查询过滤器(limit<=0 表示不限制)。
func toNodeObservabilityFilter(nodeID string, since time.Time, limit int) analyticsmodel.NodeObservabilityFilter {
return analyticsmodel.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
Limit: limit,
}
}
// toNodeObservabilitySince 构造不带 limit 的可观测查询过滤器
// (小时级聚合读无需分页,避免传无意义的 0)。
func toNodeObservabilitySince(nodeID string, since time.Time) analyticsmodel.NodeObservabilityFilter {
return analyticsmodel.NodeObservabilityFilter{NodeID: nodeID, Since: since}
}
@@ -0,0 +1,40 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"strings"
"testing"
"time"
db "Wavelet/plugins/infra/database"
)
// TestClickHouseHourlyDelegationRegression 验证 CH 后端小时级聚合读委托 analyticsrepo:
// 未初始化 CH 连接时返回 analyticsrepo 的 "clickhouse connection is not initialized" 错误
// (而非未实现/panic),证明 3 个方法都路由到 CH 原生查询。
func TestClickHouseHourlyDelegationRegression(t *testing.T) {
if db.ChConn != nil {
t.Skip("clickhouse connection initialized; skipping delegation regression")
}
s := newClickHouseStore()
ctx := context.Background()
now := time.Now()
check := func(name string, err error) {
t.Helper()
if err == nil {
t.Fatalf("%s: want clickhouse-not-initialized error, got nil", name)
}
if !strings.Contains(err.Error(), "clickhouse connection is not initialized") {
t.Fatalf("%s: unexpected error %v", name, err)
}
}
_, err := s.ListTrafficHourly(ctx, "n1", now)
check("ListTrafficHourly", err)
_, err = s.ListAccessLogHourly(ctx, "n1", now)
check("ListAccessLogHourly", err)
_, err = s.ListMetricHourly(ctx, "n1", now)
check("ListMetricHourly", err)
}
@@ -0,0 +1,33 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"strconv"
)
// timeBucketSQLPostgres 返回 PG 时间分桶表达式(epoch 秒 -> 分桶起点,int64)。
func timeBucketSQLPostgres(column string, bucketSeconds int64) string {
return "(floor(extract(epoch from " + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")::bigint"
}
// dailyTrendDateSQLPostgres 返回 PG 按日聚合的日期表达式。
func dailyTrendDateSQLPostgres() string {
return "to_char(created_at, 'YYYY-MM-DD')"
}
// epochSQLPostgres 返回 PG epoch 秒表达式(int64)。
func epochSQLPostgres(column string) string {
return "extract(epoch from " + column + ")::bigint"
}
// textCastSQLPostgres 返回 PG 数值列转文本表达式。
func textCastSQLPostgres(column string) string {
return column + "::text"
}
// distinctNonEmptyCountSQLPostgres 返回 PG 排除空串的 distinct 计数表达式。
func distinctNonEmptyCountSQLPostgres(column string) string {
return "COUNT(DISTINCT " + column + ") FILTER (WHERE " + column + " <> '')"
}
@@ -0,0 +1,85 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"strconv"
"gorm.io/gorm"
)
// isPostgresDialect 判断 gorm 句柄是否为 PostgreSQL 方言(否则按 SQLite 处理)。
// Dialector 经 gorm.Config 内嵌提升,Name() 可直接在 DB 上调用。
func isPostgresDialect(db *gorm.DB) bool {
return db != nil && db.Dialector != nil && db.Name() == "postgres"
}
// timeBucketSQLSQLite 返回 SQLite 时间分桶表达式(epoch 秒 -> 分桶起点)。
func timeBucketSQLSQLite(column string, bucketSeconds int64) string {
return "(floor(unixepoch(" + column + ")/" + strconv.FormatInt(bucketSeconds, 10) + ")*" + strconv.FormatInt(bucketSeconds, 10) + ")"
}
// dailyTrendDateSQLSQLite 返回 SQLite 按日聚合的日期表达式。
func dailyTrendDateSQLSQLite() string {
return "strftime('%Y-%m-%d', created_at)"
}
// epochSQLSQLite 返回 SQLite epoch 秒表达式(unixepoch 整数秒)。
func epochSQLSQLite(column string) string {
return "unixepoch(" + column + ")"
}
// textCastSQLSQLite 返回 SQLite 数值列转文本表达式。
func textCastSQLSQLite(column string) string {
return "CAST(" + column + " AS TEXT)"
}
// distinctNonEmptyCountSQLSQLite 返回 SQLite 排除空串的 distinct 计数表达式
// (SQLite 无 FILTER 语法,用 CASE 等价实现)。
func distinctNonEmptyCountSQLSQLite(column string) string {
return "COUNT(DISTINCT CASE WHEN " + column + " <> '' THEN " + column + " END)"
}
// distinctNonEmptyCountSQL 按当前方言返回排除空串的 distinct 计数表达式
// (运行时按 Dialector 分发,默认 SQLite)。
func distinctNonEmptyCountSQL(db *gorm.DB, column string) string {
if isPostgresDialect(db) {
return distinctNonEmptyCountSQLPostgres(column)
}
return distinctNonEmptyCountSQLSQLite(column)
}
// dailyTrendDateSQL 按当前方言返回按日聚合的日期表达式(运行时按 Dialector 分发,默认 SQLite)。
func dailyTrendDateSQL(db *gorm.DB) string {
if isPostgresDialect(db) {
return dailyTrendDateSQLPostgres()
}
return dailyTrendDateSQLSQLite()
}
// epochSQL 按当前方言返回 epoch 秒表达式(运行时按 Dialector 分发,默认 SQLite)。
func epochSQL(db *gorm.DB, column string) string {
if isPostgresDialect(db) {
return epochSQLPostgres(column)
}
return epochSQLSQLite(column)
}
// textCastSQL 按当前方言返回数值列转文本表达式(运行时按 Dialector 分发,默认 SQLite)。
func textCastSQL(db *gorm.DB, column string) string {
if isPostgresDialect(db) {
return textCastSQLPostgres(column)
}
return textCastSQLSQLite(column)
}
// timeBucketSQL 按当前方言返回时间分桶表达式。
// brief 将 PG/SQLite 两版写为同名函数,同包无法共存;log_database 为运行时配置,
// 不能使用编译期 build tag,故按 db.Dialector.Name() 运行时分发(默认 SQLite)。
func timeBucketSQL(db *gorm.DB, column string, bucketSeconds int64) string {
if isPostgresDialect(db) {
return timeBucketSQLPostgres(column, bucketSeconds)
}
return timeBucketSQLSQLite(column, bucketSeconds)
}
@@ -0,0 +1,57 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"sync"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
)
// AccessLogHooks 节点访问日志异步入队回调(由 chwriter 装配)。
type AccessLogHooks struct {
QueueNodeAccessLogs func(logs []analyticsmodel.NodeAccessLog)
}
// ObservabilityHooks 可观测异步入队回调(由 chwriter 装配)。
type ObservabilityHooks struct {
QueueMetricSnapshot func(record analyticsmodel.NodeMetricSnapshot)
QueueEdgeHealth func(record analyticsmodel.NodeEdgeHealth)
QueueNodeObsFrps func(record analyticsmodel.NodeObsFrps)
QueueNodeObsFrpc func(record analyticsmodel.NodeObsFrpc)
}
var (
hooksMu sync.RWMutex
accessLogHooks AccessLogHooks
observabilityHooks ObservabilityHooks
)
// SetAccessLogHooks 注册节点访问日志异步入队回调。
func SetAccessLogHooks(h AccessLogHooks) {
hooksMu.Lock()
accessLogHooks = h
hooksMu.Unlock()
}
// SetObservabilityHooks 注册可观测异步入队回调。
func SetObservabilityHooks(h ObservabilityHooks) {
hooksMu.Lock()
observabilityHooks = h
hooksMu.Unlock()
}
// currentAccessLogHooks 返回当前 hooks 快照(未注册时为 zero value,调用方判空跳过)。
func currentAccessLogHooks() AccessLogHooks {
hooksMu.RLock()
defer hooksMu.RUnlock()
return accessLogHooks
}
// currentObservabilityHooks 返回当前 hooks 快照(未注册时为 zero value,调用方判空跳过)。
func currentObservabilityHooks() ObservabilityHooks {
hooksMu.RLock()
defer hooksMu.RUnlock()
return observabilityHooks
}
@@ -0,0 +1,117 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"os/exec"
"strings"
"testing"
)
// serverPkg 是下游 server 插件的包路径前缀。
const serverPkg = "Wavelet/openflare/plugins/server"
// forbiddenImports 业务域禁止直接触碰的底层日志实现。
var forbiddenImports = []string{
serverPkg + "/kernel/repository/analytics",
}
// allowedAnalyticsDelegation 允许直接依赖 analytics 仓储的委托层:
// - repository:持久化门面,ListOpenFlareLatestMetricSnapshotsSince 的
// CH 快速路径仍直连 analytics(LIMIT 1 BY node_id);小时级聚合读已改走 logstore;
// - repository/logstore:CH 后端实现按设计委托 analytics。
//
// 除此之外,依赖闭包内任何包都禁止引入 analytics 仓储。
var allowedAnalyticsDelegation = map[string]bool{
serverPkg + "/kernel/repository": true,
serverPkg + "/kernel/repository/logstore": true,
}
// allowedInfraPersistence 允许业务域包引入的 infra/persistence 子包。
var allowedInfraPersistence = []string{
serverPkg + "/infra/persistence/batchwriter", // batchwriter 统计类型
serverPkg + "/infra/persistence/idgen", // 雪花 ID 生成(无日志依赖)
}
// domainScopes 是 server 插件内的业务域包(等价于改造前的 internal/apps/...)。
// 持久化与基础设施层(repository/infra/model/…)不受本门禁约束。
var domainScopes = []string{
"domain/site", "domain/fleet", "domain/pages", "domain/waf", "domain/tls",
"domain/cloudflare", "domain/observability", "domain/dashboard", "domain/option",
"updater",
}
func TestDomainsMustNotImportLogBackendDirectly(t *testing.T) {
t.Chdir(moduleRoot(t))
wanted := make([]string, 0, len(domainScopes))
patterns := make([]string, 0, len(domainScopes))
for _, d := range domainScopes {
wanted = append(wanted, serverPkg+"/"+d)
patterns = append(patterns, "./openflare/plugins/server/"+d+"/...")
}
args := append([]string{"list", "-test", "-f", `{{.ImportPath}} {{join .Imports " "}}`}, patterns...)
//nolint:gosec // 固定参数,无外部输入
out, err := exec.Command("go", args...).Output()
if err != nil {
t.Fatalf("go list: %v", err)
}
scanned := 0
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
pkg := fields[0]
if !hasAnyPrefix(pkg, wanted) {
continue
}
scanned++
for _, imp := range fields[1:] {
for _, forbidden := range forbiddenImports {
if imp == forbidden && !allowedAnalyticsDelegation[pkg] {
t.Errorf("%s must not import forbidden log backend %s", pkg, forbidden)
}
}
if strings.HasPrefix(imp, serverPkg+"/infra/persistence/") {
allowed := false
for _, a := range allowedInfraPersistence {
if imp == a || strings.HasPrefix(imp, a+"/") {
allowed = true
break
}
}
if !allowed {
t.Errorf("%s must not import infra/persistence subpackage directly: %s", pkg, imp)
}
}
}
}
// 扫描到 0 个包说明包路径已漂移,门禁会静默失效——必须报错而非给绿灯。
if scanned == 0 {
t.Fatalf("no domain package scanned; domainScopes is stale: %v", wanted)
}
}
func hasAnyPrefix(s string, prefixes []string) bool {
for _, p := range prefixes {
if s == p || strings.HasPrefix(s, p+"/") {
return true
}
}
return false
}
// moduleRoot 向 go 查询模块根目录,避免依赖测试文件所在深度的相对路径。
func moduleRoot(t *testing.T) string {
t.Helper()
cmd := exec.Command("go", "list", "-m", "-f", "{{.Dir}}")
out, err := cmd.Output()
if err != nil {
t.Fatalf("resolve module root: %v", err)
}
return strings.TrimSpace(string(out))
}
@@ -0,0 +1,65 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"testing"
"time"
"Wavelet/openflare/plugins/server/kernel/model"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
)
// TestGormAccessLogInsertBatchHooks 覆盖访问日志写入入口:
// 冻结检查、hook 入队、不直接落库、flush 后可见(行为与旧 repository clickhouse 包装一致)。
func TestGormAccessLogInsertBatchHooks(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", nil
})
defer ResetForTest()
s := newTestGormStore(t)
ctx := context.Background()
now := time.Now().UTC()
var hooked []analyticsmodel.NodeAccessLog
SetAccessLogHooks(AccessLogHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
hooked = append(hooked, logs...)
},
})
defer SetAccessLogHooks(AccessLogHooks{})
records := []*model.OpenFlareAccessLog{
{NodeID: "n1", LoggedAt: now, RemoteAddr: "1.1.1.1", StatusCode: 200, BytesSent: 100},
{NodeID: "n1", LoggedAt: now, RemoteAddr: "2.2.2.2", StatusCode: 404},
}
if err := s.InsertBatch(ctx, records); err != nil {
t.Fatalf("insert batch: %v", err)
}
if len(hooked) != 2 || hooked[0].RemoteAddr != "1.1.1.1" || hooked[0].BytesSent != 100 || hooked[1].StatusCode != 404 {
t.Fatalf("hook rows mismatch: %+v", hooked)
}
// 写入入口只入队、不直接落库。
rows, err := s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
if err != nil {
t.Fatalf("list: %v", err)
}
if len(rows) != 0 {
t.Fatalf("entry insert must not write rows, got %d", len(rows))
}
// flush 后可见。
if err := s.BatchInsertNodeAccessLogs(ctx, hooked); err != nil {
t.Fatalf("flush: %v", err)
}
rows, err = s.List(ctx, model.OpenFlareAccessLogQuery{NodeID: "n1"})
if err != nil {
t.Fatalf("list after flush: %v", err)
}
if len(rows) != 2 {
t.Fatalf("list after flush want 2, got %d", len(rows))
}
}
@@ -0,0 +1,128 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package logstore 提供日志/分析存储抽象:上层只面向本包接口,
// 禁止直接 import internal/repository/analytics 或触碰 db.ChConn/db.ChDB。
package logstore
import (
"context"
"errors"
"time"
"Wavelet/openflare/plugins/server/kernel/model"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
)
// ErrMigrating 表示日志数据库正在迁移,当前禁止写入。
var ErrMigrating = errors.New("log database is migrating, writes are disabled")
// AccessLogStore 节点访问日志(of_node_access_logs)。
type AccessLogStore interface {
// InsertBatch 为写入入口:冻结检查 + 经 hook 入队(异步),不直接落库。
InsertBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error
// BatchInsertNodeAccessLogs 为 batchwriter flush 目标:直接批量写入当前存储。
BatchInsertNodeAccessLogs(ctx context.Context, rows []analyticsmodel.NodeAccessLog) error
List(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error)
Count(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error)
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error)
BucketAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketAggregate, error)
CountBuckets(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
BucketDimensions(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogBucketDimension, error)
IPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]analyticsmodel.NodeAccessLogIPAggregate, error)
IPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery, recentSince time.Time) ([]analyticsmodel.NodeAccessLogIPSummary, error)
CountIPSummaries(ctx context.Context, filter model.OpenFlareAccessLogQuery) (int64, error)
WAFIPAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]analyticsmodel.NodeAccessLogWAFIPAggregate, error)
IPTrend(ctx context.Context, filter model.OpenFlareAccessLogQuery, bucketSeconds int64) ([]analyticsmodel.NodeAccessLogIPTrend, error)
TrafficSummary(ctx context.Context, filter model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error)
ValueCounts(ctx context.Context, filter model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error)
NodeAggregates(ctx context.Context, filter model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error)
DeleteAll(ctx context.Context) (int64, error)
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
DeleteByNodeBefore(ctx context.Context, nodeID string, before time.Time) (int64, error)
// ListForMigration 按 id 升序分页读取(迁移复制用)。
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeAccessLog, error)
// MigrationRange 返回源表 logged_at 的最小/最大值(空表返回零值),迁移预建分区用。
MigrationRange(ctx context.Context) (from, to time.Time, err error)
// EnsurePartitions 幂等预建 PG 分区(按月),覆盖 [from, to] 月份;CH/SQLite 为 no-op。
// 目标为 PG 的迁移在复制前调用,避免历史数据写入报 "no partition of relation found"。
EnsurePartitions(ctx context.Context, from, to time.Time) error
// DropEmptyPartitions 幂等清理 PG 空分区表:删除 before 月份之前、且无任何数据的按月分区;
// CH/SQLite 为 no-op(CH 分区随数据删除自动消失、SQLite 无分区)。
DropEmptyPartitions(ctx context.Context, before time.Time) error
// DropExpiredPartitions 直接删除完全过期的 PG 整月分区(候选为月份早于 cutoff 月的分区,
// 删除前校验分区内无保留期内数据,避免时区偏移下误删;迁移冻结期间拒绝执行);CH/SQLite 为 no-op。
DropExpiredPartitions(ctx context.Context, cutoff time.Time) error
}
// ObservabilityStore 可观测 4 表(metric snapshots / edge health / frps / frpc)。
type ObservabilityStore interface {
InsertMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error
ListMetricSnapshots(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error)
DeleteAllMetricSnapshots(ctx context.Context) (int64, error)
DeleteMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeMetricSnapshots(ctx context.Context, rows []analyticsmodel.NodeMetricSnapshot) error
// ListTrafficHourly 返回小时级流量汇总(按 node/hour 聚合,unique_visitor_count 恒 0)。
// CH 后端读 of_access_log_hourly rollup;PG/SQLite 从 of_node_access_logs 实时聚合。
ListTrafficHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeTrafficHourly, error)
// ListAccessLogHourly 返回按 node/hour/host 的小时级访问日志汇总。
ListAccessLogHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.AccessLogHourly, error)
// ListMetricHourly 返回小时级指标聚合(avg cpu/memory + 计数器增量,reported_nodes 去重节点数)。
ListMetricHourly(ctx context.Context, nodeID string, since time.Time) ([]analyticsmodel.NodeMetricHourly, error)
InsertEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error
ListEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error)
DeleteAllEdgeHealth(ctx context.Context) (int64, error)
DeleteEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeEdgeHealth(ctx context.Context, rows []analyticsmodel.NodeEdgeHealth) error
InsertNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error
ListNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error)
DeleteAllNodeObservationFrps(ctx context.Context) (int64, error)
DeleteNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeObsFrps(ctx context.Context, rows []analyticsmodel.NodeObsFrps) error
InsertNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error
ListNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error)
DeleteAllNodeObservationFrpc(ctx context.Context) (int64, error)
DeleteNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error)
BatchInsertNodeObsFrpc(ctx context.Context, rows []analyticsmodel.NodeObsFrpc) error
// 迁移复制用:按 id 升序分页读取。
ListMetricSnapshotsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeMetricSnapshot, error)
ListEdgeHealthForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeEdgeHealth, error)
ListNodeObsFrpsForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrps, error)
ListNodeObsFrpcForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.NodeObsFrpc, error)
}
// UserAccessLogStore 用户访问日志(w_user_access_logs)。
type UserAccessLogStore interface {
BatchInsert(ctx context.Context, logs []analyticsmodel.UserAccessLog) error
// DeleteAll 清空全部用户访问日志(迁移「覆盖目标库已有日志」幂等前提用)。
DeleteAll(ctx context.Context) (int64, error)
Count(ctx context.Context, filter analyticsmodel.AccessLogFilter) (uint64, error)
List(ctx context.Context, filter analyticsmodel.AccessLogFilter, page, pageSize int) ([]analyticsmodel.UserAccessLog, uint64, error)
GetDailyTrend(ctx context.Context, days int) ([]analyticsmodel.DailyTrend, error)
GetBrowserDistribution(ctx context.Context, startTime time.Time) ([]analyticsmodel.BrowserShare, error)
GetTopActiveUsers(ctx context.Context, startTime time.Time, limit int) ([]analyticsmodel.TopUser, error)
// ListForMigration 按 id 升序分页读取(迁移复制用)。
ListForMigration(ctx context.Context, afterID uint64, limit int) ([]analyticsmodel.UserAccessLog, error)
// MigrationRange 返回源表 created_at 的最小/最大值(空表返回零值),迁移预建分区用。
MigrationRange(ctx context.Context) (from, to time.Time, err error)
}
// StatusStore 日志库状态(供管理端状态端点)。
type StatusStore interface {
ActiveDatabase(ctx context.Context) (string, error)
ClickHouseOperationalStats(ctx context.Context) (*analyticsmodel.ClickHouseOperationalStats, error) // 仅 CH 激活时非 nil
}
// Store 聚合当前生效日志库的全部域存储。
type Store struct {
AccessLogs AccessLogStore
Observability ObservabilityStore
UserAccessLogs UserAccessLogStore
Status StatusStore
}
@@ -0,0 +1,66 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"fmt"
"time"
"gorm.io/gorm"
)
// listPartitionNames 列出 table 在当前 schema 下的全部直接分区表名(pg_inherits)。
func listPartitionNames(ctx context.Context, gdb *gorm.DB, table string) ([]string, error) {
var names []string
if err := gdb.WithContext(ctx).Raw(`
SELECT c.relname
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_class p ON p.oid = i.inhparent
JOIN pg_namespace n ON n.oid = p.relnamespace AND n.nspname = current_schema()
WHERE p.relname = ?`, table).Scan(&names).Error; err != nil {
return nil, fmt.Errorf("list partitions of %s: %w", table, err)
}
return names, nil
}
// DropExpiredPartitions 直接删除完全过期的 PG 整月分区(避免 retention 清理逐行 DELETE):
// 候选 = 月份早于 cutoff 月(按 cutoff 的 UTC 时刻取月,避免本地时区偏移超前误删)的分区,
// 且删除前校验分区内不存在 logged_at >= cutoff 的行(分区边界随会话时区偏移,
// 名称月份只能粗筛,必须以数据为准);仅处理 of_node_access_logs
// (w_user_access_logs 无 retention 清理,刻意不删其分区);迁移冻结期间(ensureWritable)
// 直接返回 ErrMigrating,避免对冻结源库整月 DROP 丢数据;CH/SQLite 为 no-op。
func (s *gormLogStore) DropExpiredPartitions(ctx context.Context, cutoff time.Time) error {
if !isPostgresDialect(s.db) {
return nil
}
if err := s.ensureWritable(ctx); err != nil {
return err
}
names, err := listPartitionNames(ctx, s.db, "of_node_access_logs")
if err != nil {
return err
}
cu := cutoff.UTC()
cutoffMonth := time.Date(cu.Year(), cu.Month(), 1, 0, 0, 0, 0, time.UTC)
for _, name := range names {
month, ok := partitionNameMonth("of_node_access_logs", name)
if !ok || !month.Before(cutoffMonth) {
continue // 非法命名或当月/未来月分区,必须保留
}
// 数据校验:分区内仍有 logged_at >= cutoff 的行则保留(时区偏移下名称月份可能超前于真实边界)。
var hasRetained int
if err := s.db.WithContext(ctx).Raw("SELECT 1 FROM "+name+" WHERE logged_at >= ? LIMIT 1", cu).Scan(&hasRetained).Error; err != nil {
return fmt.Errorf("check partition %s retained rows: %w", name, err)
}
if hasRetained == 1 {
continue
}
if err := s.db.WithContext(ctx).Exec("DROP TABLE IF EXISTS " + name).Error; err != nil {
return fmt.Errorf("drop expired partition %s: %w", name, err)
}
}
return nil
}
@@ -0,0 +1,700 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"fmt"
"os"
"regexp"
"strings"
"testing"
"time"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/logger"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
)
// TestEnsurePartitionsPostgresInsertAcrossMonths 需要 TEST_POSTGRES_DSN(未设置时跳过):
// 验证 EnsurePartitions 预建任意月份范围分区后,跨月历史数据可写入 PG 分区表
// (对应迁移任务从 CH/SQLite 复制历史日志到 PG 时先预建分区的场景)。
func TestEnsurePartitionsPostgresInsertAcrossMonths(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("TEST_POSTGRES_DSN is not set")
}
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open postgres: %v", err)
}
sqlDB, err := gdb.DB()
if err != nil {
t.Fatalf("sql db: %v", err)
}
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("logstore_partition_%d", time.Now().UnixNano())
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
t.Fatalf("invalid schema: %s", schema)
}
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
t.Fatalf("create schema: %v", err)
}
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
t.Fatalf("set search_path: %v", err)
}
t.Cleanup(func() {
_ = gdb.Exec("SET search_path TO public").Error
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
_ = sqlDB.Close()
})
// 与 goose/postgres/202608080001_create_log_tables.sql 保持一致的分区父表 DDL。
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
if err := gdb.Exec(ddl).Error; err != nil {
t.Fatalf("create partitioned table: %v", err)
}
}
ResetForTest()
SetConfigReader(func(_ context.Context, _ string) (string, error) { return "", nil })
defer ResetForTest()
ctx := context.Background()
store := newGormStore(gdb)
ua := newUserAccessLogGormStore(gdb)
// 源范围跨 3 个月:2026-01-10 ~ 2026-03-20;to+1 月兜底生成 202601..202604 分区。
from := time.Date(2026, 1, 10, 8, 0, 0, 0, time.UTC)
max := time.Date(2026, 3, 20, 9, 30, 0, 0, time.UTC)
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
t.Fatalf("EnsurePartitions: %v", err)
}
// 幂等:重复调用不报错(CREATE TABLE IF NOT EXISTS ... PARTITION OF)。
if err := store.EnsurePartitions(ctx, from, max.AddDate(0, 1, 0)); err != nil {
t.Fatalf("EnsurePartitions idempotent: %v", err)
}
var partitionCount int64
if err := gdb.Raw(
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass('of_node_access_logs')",
).Scan(&partitionCount).Error; err != nil {
t.Fatalf("count partitions: %v", err)
}
if partitionCount != 4 {
t.Fatalf("of_node_access_logs partitions = %d, want 4", partitionCount)
}
// 跨月插入:1/2/3 月各 2 条节点访问日志 + 2 条用户访问日志,均应命中已有分区。
nodeRows := []analyticsmodel.NodeAccessLog{
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.2"},
{ID: 3, NodeID: "n2", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.2"},
{ID: 4, NodeID: "n2", LoggedAt: time.Date(2026, 2, 12, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.3"},
{ID: 5, NodeID: "n1", LoggedAt: time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
{ID: 6, NodeID: "n1", LoggedAt: time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.4"},
}
if err := store.BatchInsertNodeAccessLogs(ctx, nodeRows); err != nil {
t.Fatalf("insert node access logs across months: %v", err)
}
userRows := []analyticsmodel.UserAccessLog{
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)},
{ID: 2, UserID: 102, Path: "/b", CreatedAt: time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)},
}
if err := ua.BatchInsert(ctx, userRows); err != nil {
t.Fatalf("insert user access logs across months: %v", err)
}
var nodeCount, userCount int64
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
t.Fatalf("count node access logs: %v", err)
}
if err := gdb.Model(&analyticsmodel.UserAccessLog{}).Count(&userCount).Error; err != nil {
t.Fatalf("count user access logs: %v", err)
}
if nodeCount != 6 {
t.Fatalf("node access log count = %d, want 6", nodeCount)
}
if userCount != 2 {
t.Fatalf("user access log count = %d, want 2", userCount)
}
// MigrationRange 返回跨月范围(覆盖两表)。
gotFrom, gotTo, err := store.MigrationRange(ctx)
if err != nil {
t.Fatalf("node MigrationRange: %v", err)
}
if !gotFrom.Equal(time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)) || !gotTo.Equal(time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC)) {
t.Fatalf("node MigrationRange = %s ~ %s, want 2026-01-15 ~ 2026-03-18", gotFrom, gotTo)
}
uaFrom, uaTo, err := ua.MigrationRange(ctx)
if err != nil {
t.Fatalf("user MigrationRange: %v", err)
}
if !uaFrom.Equal(time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)) || !uaTo.Equal(time.Date(2026, 3, 17, 0, 0, 0, 0, time.UTC)) {
t.Fatalf("user MigrationRange = %s ~ %s", uaFrom, uaTo)
}
}
// TestDropEmptyPartitionsPostgres 需要 TEST_POSTGRES_DSN(未设置时跳过):
// 验证空分区清理只删除 before 月份之前且无数据的分区:空旧月删除、有数据旧月保留、
// 当月/未来月保留;用户访问日志分区同步清理。
func TestDropEmptyPartitionsPostgres(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("TEST_POSTGRES_DSN is not set")
}
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open postgres: %v", err)
}
sqlDB, err := gdb.DB()
if err != nil {
t.Fatalf("sql db: %v", err)
}
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("logstore_drop_partition_%d", time.Now().UnixNano())
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
t.Fatalf("invalid schema: %s", schema)
}
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
t.Fatalf("create schema: %v", err)
}
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
t.Fatalf("set search_path: %v", err)
}
t.Cleanup(func() {
_ = gdb.Exec("SET search_path TO public").Error
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
_ = sqlDB.Close()
})
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
if err := gdb.Exec(ddl).Error; err != nil {
t.Fatalf("create partitioned table: %v", err)
}
}
ctx := context.Background()
store := newGormStore(gdb)
ua := newUserAccessLogGormStore(gdb)
// 预建 202601..202603 分区,仅 202602 有数据(节点+用户各 1 条),202601/202603 为空。
if err := store.EnsurePartitions(ctx,
time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
time.Date(2026, 3, 20, 0, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("EnsurePartitions: %v", err)
}
if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
}); err != nil {
t.Fatalf("insert node access log: %v", err)
}
if err := ua.BatchInsert(ctx, []analyticsmodel.UserAccessLog{
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 2, 11, 0, 0, 0, 0, time.UTC)},
}); err != nil {
t.Fatalf("insert user access log: %v", err)
}
// before=2026-03:202601(空)应删,202602(有数据)与 202603(当月)保留。
if err := store.DropEmptyPartitions(ctx, time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("DropEmptyPartitions: %v", err)
}
assertPartitions := func(parent string, want int64) {
t.Helper()
var n int64
if err := gdb.Raw(
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
parent,
).Scan(&n).Error; err != nil {
t.Fatalf("count partitions of %s: %v", parent, err)
}
if n != want {
t.Fatalf("%s partitions = %d, want %d", parent, n, want)
}
}
assertPartitions("of_node_access_logs", 2)
assertPartitions("w_user_access_logs", 2)
// 数据未受影响。
var nodeCount, userCount int64
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
t.Fatalf("count node access logs: %v", err)
}
if err := gdb.Model(&analyticsmodel.UserAccessLog{}).Count(&userCount).Error; err != nil {
t.Fatalf("count user access logs: %v", err)
}
if nodeCount != 1 || userCount != 1 {
t.Fatalf("data counts = (%d, %d), want (1, 1)", nodeCount, userCount)
}
}
// TestDropExpiredPartitionsPostgres 需要 TEST_POSTGRES_DSN(未设置时跳过):
// 验证直接删除完全早于 cutoff 月份的整月分区:早于 cutoff 月的分区(含其中全部数据)被整表 DROP、
// 边界月分区保留且数据仍在;重复调用幂等;w_user_access_logs 分区不受影响(无 retention 清理)。
func TestDropExpiredPartitionsPostgres(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("TEST_POSTGRES_DSN is not set")
}
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open postgres: %v", err)
}
sqlDB, err := gdb.DB()
if err != nil {
t.Fatalf("sql db: %v", err)
}
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("logstore_drop_expired_%d", time.Now().UnixNano())
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
t.Fatalf("invalid schema: %s", schema)
}
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
t.Fatalf("create schema: %v", err)
}
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
t.Fatalf("set search_path: %v", err)
}
t.Cleanup(func() {
_ = gdb.Exec("SET search_path TO public").Error
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
_ = sqlDB.Close()
})
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
if err := gdb.Exec(ddl).Error; err != nil {
t.Fatalf("create partitioned table: %v", err)
}
}
ctx := context.Background()
store := newGormStore(gdb)
ua := newUserAccessLogGormStore(gdb)
// 预建 202601..202604 分区;1/3 月有数据、2/4 月为空。
if err := store.EnsurePartitions(ctx,
time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
time.Date(2026, 4, 20, 0, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("EnsurePartitions: %v", err)
}
if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.2"},
{ID: 3, NodeID: "n2", LoggedAt: time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
{ID: 4, NodeID: "n2", LoggedAt: time.Date(2026, 3, 18, 0, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.4"},
}); err != nil {
t.Fatalf("insert node access logs: %v", err)
}
if err := ua.BatchInsert(ctx, []analyticsmodel.UserAccessLog{
{ID: 1, UserID: 101, Path: "/a", CreatedAt: time.Date(2026, 1, 16, 0, 0, 0, 0, time.UTC)},
}); err != nil {
t.Fatalf("insert user access log: %v", err)
}
// cutoff=2026-03-10:分区月份早于 2026-03 的(202601、202602)整表 DROP;
// 202603(边界月,可能含未过期数据)与 202604(未来月)保留。
if err := store.DropExpiredPartitions(ctx, time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("DropExpiredPartitions: %v", err)
}
// 幂等:重复调用不报错、不额外删除。
if err := store.DropExpiredPartitions(ctx, time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("DropExpiredPartitions idempotent: %v", err)
}
assertPartitions := func(parent string, want int64) {
t.Helper()
var n int64
if err := gdb.Raw(
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
parent,
).Scan(&n).Error; err != nil {
t.Fatalf("count partitions of %s: %v", parent, err)
}
if n != want {
t.Fatalf("%s partitions = %d, want %d", parent, n, want)
}
}
// of_node_access_logs 只剩边界月+未来月 2 个分区;w_user_access_logs 不受影响(仍 4 个)。
assertPartitions("of_node_access_logs", 2)
assertPartitions("w_user_access_logs", 4)
// 202601/202602 分区被整表 DROP:1 月数据随之消失,3 月数据保留。
var nodeCount int64
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
t.Fatalf("count node access logs: %v", err)
}
if nodeCount != 2 {
t.Fatalf("node access log count = %d, want 2(仅剩 3 月数据)", nodeCount)
}
}
// TestDropExpiredPartitionsTimezoneSafety 需要 TEST_POSTGRES_DSN(未设置时跳过):
// 覆盖本地时区偏移下 DropExpiredPartitions 的时区安全性:cutoff 为 UTC+8 本地时刻
// (其实刻 = 2026-02-28T21:00Z),名称月份早于 cutoff 月但分区内仍含保留期行的
// 202602 不得被误删(旧实现按本地月份取 cutoffMonth=2026-03 会整表 DROP 丢数据);
// 完全过期的 202601 正常整表 DROP;保留期行仍可查询到。
func TestDropExpiredPartitionsTimezoneSafety(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("TEST_POSTGRES_DSN is not set")
}
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open postgres: %v", err)
}
sqlDB, err := gdb.DB()
if err != nil {
t.Fatalf("sql db: %v", err)
}
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("logstore_drop_expired_tz_%d", time.Now().UnixNano())
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
t.Fatalf("invalid schema: %s", schema)
}
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
t.Fatalf("create schema: %v", err)
}
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
t.Fatalf("set search_path: %v", err)
}
t.Cleanup(func() {
_ = gdb.Exec("SET search_path TO public").Error
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
_ = sqlDB.Close()
})
for _, ddl := range []string{postgresNodeAccessLogsDDL, postgresUserAccessLogsDDL} {
if err := gdb.Exec(ddl).Error; err != nil {
t.Fatalf("create partitioned table: %v", err)
}
}
ctx := context.Background()
store := newGormStore(gdb)
// 预建 202601..202602 分区。
if err := store.EnsurePartitions(ctx,
time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC),
time.Date(2026, 2, 20, 0, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("EnsurePartitions: %v", err)
}
// 202601 仅含完全过期行;202602 含一条过期行(2026-02-10)与一条保留期行
// (2026-02-28T21:00Z,恰等于 cutoff 其实刻,>= 语义下必须保留)。
if err := store.BatchInsertNodeAccessLogs(ctx, []analyticsmodel.NodeAccessLog{
{ID: 1, NodeID: "n1", LoggedAt: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC), RemoteAddr: "1.1.1.1"},
{ID: 2, NodeID: "n1", LoggedAt: time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC), RemoteAddr: "2.2.2.2"},
{ID: 3, NodeID: "n1", LoggedAt: time.Date(2026, 2, 28, 21, 0, 0, 0, time.UTC), RemoteAddr: "3.3.3.3"},
}); err != nil {
t.Fatalf("insert node access logs: %v", err)
}
// cutoff 为 UTC+8 本地时刻 2026-03-01 05:00,其实刻 = 2026-02-28T21:00Z:
// 旧实现按本地月份取 cutoffMonth=2026-03 会把 202602 误判为完全过期整表 DROP。
cutoff := time.Date(2026, 3, 1, 5, 0, 0, 0, time.FixedZone("UTC+8", 8*3600))
if err := store.DropExpiredPartitions(ctx, cutoff); err != nil {
t.Fatalf("DropExpiredPartitions: %v", err)
}
assertPartitions := func(parent string, want int64) {
t.Helper()
var n int64
if err := gdb.Raw(
"SELECT count(*) FROM pg_inherits WHERE inhparent = to_regclass(?)",
parent,
).Scan(&n).Error; err != nil {
t.Fatalf("count partitions of %s: %v", parent, err)
}
if n != want {
t.Fatalf("%s partitions = %d, want %d", parent, n, want)
}
}
// 202601 已整表 DROP,202602 保留;w_user_access_logs 不受影响(仍 2 个)。
assertPartitions("of_node_access_logs", 1)
assertPartitions("w_user_access_logs", 2)
// 202601 数据随之消失,202602 内保留期行(2026-02-28T21:00Z)仍可查询到。
var nodeCount int64
if err := gdb.Model(&analyticsmodel.NodeAccessLog{}).Count(&nodeCount).Error; err != nil {
t.Fatalf("count node access logs: %v", err)
}
if nodeCount != 2 {
t.Fatalf("node access log count = %d, want 2(仅剩 202602 两行)", nodeCount)
}
var retained int64
if err := gdb.Raw(
"SELECT count(*) FROM of_node_access_logs WHERE logged_at >= ?",
cutoff.UTC(),
).Scan(&retained).Error; err != nil {
t.Fatalf("count retained rows: %v", err)
}
if retained != 1 {
t.Fatalf("retained rows (logged_at >= cutoff) = %d, want 1", retained)
}
}
// TestBatchInsertGeneratesIDsPostgres 回归:PG 日志表 id BIGINT NOT NULL 且无默认值;
// GORM 把零值 uint64 主键视为自增并省略 id 列,直接插入会报 23502 not-null 违例。
// 验证 6 张日志表 BatchInsert* 为零 ID 行生成雪花 ID 后正常落库(修复前本测试失败)。
func TestBatchInsertGeneratesIDsPostgres(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("TEST_POSTGRES_DSN"))
if dsn == "" {
t.Skip("TEST_POSTGRES_DSN is not set")
}
gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
t.Fatalf("open postgres: %v", err)
}
sqlDB, err := gdb.DB()
if err != nil {
t.Fatalf("sql db: %v", err)
}
sqlDB.SetMaxOpenConns(1)
schema := fmt.Sprintf("logstore_ids_%d", time.Now().UnixNano())
if !regexp.MustCompile(`^[a-z0-9_]+$`).MatchString(schema) {
t.Fatalf("invalid schema: %s", schema)
}
if err := gdb.Exec(`CREATE SCHEMA "` + schema + `"`).Error; err != nil {
t.Fatalf("create schema: %v", err)
}
if err := gdb.Exec(`SET search_path TO "` + schema + `"`).Error; err != nil {
t.Fatalf("set search_path: %v", err)
}
t.Cleanup(func() {
_ = gdb.Exec("SET search_path TO public").Error
_ = gdb.Exec(`DROP SCHEMA IF EXISTS "` + schema + `" CASCADE`).Error
_ = sqlDB.Close()
})
for _, ddl := range []string{
postgresNodeAccessLogsDDL,
postgresUserAccessLogsDDL,
postgresMetricSnapshotsDDL,
postgresEdgeHealthDDL,
postgresObsFrpsDDL,
postgresObsFrpcDDL,
} {
if err := gdb.Exec(ddl).Error; err != nil {
t.Fatalf("create table: %v", err)
}
}
ResetForTest()
SetConfigReader(func(_ context.Context, _ string) (string, error) { return "", nil })
defer ResetForTest()
ctx := context.Background()
store := newGormStore(gdb)
ua := newUserAccessLogGormStore(gdb)
now := time.Now().UTC()
if err := store.EnsurePartitions(ctx, now, now.AddDate(0, 1, 0)); err != nil {
t.Fatalf("EnsurePartitions: %v", err)
}
nodeRows := []analyticsmodel.NodeAccessLog{
{NodeID: "n1", LoggedAt: now, RemoteAddr: "1.1.1.1", StatusCode: 200},
{NodeID: "n1", LoggedAt: now.Add(time.Second), RemoteAddr: "2.2.2.2", StatusCode: 500},
}
if err := store.BatchInsertNodeAccessLogs(ctx, nodeRows); err != nil {
t.Fatalf("insert node access logs with zero ids: %v", err)
}
if nodeRows[0].ID == 0 || nodeRows[1].ID == 0 || nodeRows[0].ID == nodeRows[1].ID {
t.Fatalf("node access log ids not generated: %+v", nodeRows)
}
metricRows := []analyticsmodel.NodeMetricSnapshot{
{NodeID: "n1", CapturedAt: now},
{NodeID: "n2", CapturedAt: now},
}
if err := store.BatchInsertNodeMetricSnapshots(ctx, metricRows); err != nil {
t.Fatalf("insert metric snapshots with zero ids: %v", err)
}
if metricRows[0].ID == 0 || metricRows[1].ID == 0 || metricRows[0].ID == metricRows[1].ID {
t.Fatalf("metric snapshot ids not generated: %+v", metricRows)
}
edgeRows := []analyticsmodel.NodeEdgeHealth{
{NodeID: "n1", CapturedAt: now, Status: "ok"},
{NodeID: "n2", CapturedAt: now, Status: "ok"},
}
if err := store.BatchInsertNodeEdgeHealth(ctx, edgeRows); err != nil {
t.Fatalf("insert edge health with zero ids: %v", err)
}
if edgeRows[0].ID == 0 || edgeRows[1].ID == 0 || edgeRows[0].ID == edgeRows[1].ID {
t.Fatalf("edge health ids not generated: %+v", edgeRows)
}
frpsRows := []analyticsmodel.NodeObsFrps{
{NodeID: "n1", CapturedAt: now, FrpsConnections: 1},
{NodeID: "n2", CapturedAt: now, FrpsConnections: 2},
}
if err := store.BatchInsertNodeObsFrps(ctx, frpsRows); err != nil {
t.Fatalf("insert obs frps with zero ids: %v", err)
}
if frpsRows[0].ID == 0 || frpsRows[1].ID == 0 || frpsRows[0].ID == frpsRows[1].ID {
t.Fatalf("obs frps ids not generated: %+v", frpsRows)
}
frpcRows := []analyticsmodel.NodeObsFrpc{
{NodeID: "n1", CapturedAt: now, TunnelStatus: "online"},
{NodeID: "n2", CapturedAt: now, TunnelStatus: "online"},
}
if err := store.BatchInsertNodeObsFrpc(ctx, frpcRows); err != nil {
t.Fatalf("insert obs frpc with zero ids: %v", err)
}
if frpcRows[0].ID == 0 || frpcRows[1].ID == 0 || frpcRows[0].ID == frpcRows[1].ID {
t.Fatalf("obs frpc ids not generated: %+v", frpcRows)
}
userRows := []analyticsmodel.UserAccessLog{
{UserID: 101, Path: "/a", CreatedAt: now},
{UserID: 102, Path: "/b", CreatedAt: now},
}
if err := ua.BatchInsert(ctx, userRows); err != nil {
t.Fatalf("insert user access logs with zero ids: %v", err)
}
if userRows[0].ID == 0 || userRows[1].ID == 0 || userRows[0].ID == userRows[1].ID {
t.Fatalf("user access log ids not generated: %+v", userRows)
}
expect := []struct {
name string
model any
want int64
}{
{"of_node_access_logs", &analyticsmodel.NodeAccessLog{}, 2},
{"of_node_metric_snapshots", &analyticsmodel.NodeMetricSnapshot{}, 2},
{"of_node_edge_health", &analyticsmodel.NodeEdgeHealth{}, 2},
{"of_node_obs_frps", &analyticsmodel.NodeObsFrps{}, 2},
{"of_node_obs_frpc", &analyticsmodel.NodeObsFrpc{}, 2},
{"w_user_access_logs", &analyticsmodel.UserAccessLog{}, 2},
}
for _, e := range expect {
var got int64
if err := gdb.Model(e.model).Count(&got).Error; err != nil {
t.Fatalf("count %s: %v", e.name, err)
}
if got != e.want {
t.Fatalf("%s count = %d, want %d", e.name, got, e.want)
}
}
}
// postgresNodeAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
const postgresNodeAccessLogsDDL = `
CREATE TABLE IF NOT EXISTS of_node_access_logs (
id BIGINT NOT NULL,
node_id VARCHAR(64) NOT NULL DEFAULT '',
logged_at TIMESTAMPTZ NOT NULL,
remote_addr VARCHAR(128) NOT NULL DEFAULT '',
region VARCHAR(128) NOT NULL DEFAULT '',
host VARCHAR(255) NOT NULL DEFAULT '',
path VARCHAR(2048) NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
cache_status VARCHAR(64) NOT NULL DEFAULT '',
status_code INTEGER NOT NULL DEFAULT 0,
bytes_sent BIGINT NOT NULL DEFAULT 0,
request_length BIGINT NOT NULL DEFAULT 0,
request_time_ms INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, logged_at)
) PARTITION BY RANGE (logged_at)`
// postgresUserAccessLogsDDL 与 goose/postgres/202608080001_create_log_tables.sql 对齐。
const postgresUserAccessLogsDDL = `
CREATE TABLE IF NOT EXISTS w_user_access_logs (
id BIGINT NOT NULL,
user_id BIGINT NOT NULL DEFAULT 0,
path VARCHAR(2048) NOT NULL DEFAULT '',
method VARCHAR(16) NOT NULL DEFAULT '',
ip VARCHAR(128) NOT NULL DEFAULT '',
user_agent TEXT NOT NULL DEFAULT '',
headers TEXT NOT NULL DEFAULT '',
status INTEGER NOT NULL DEFAULT 0,
latency BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id, created_at)
) PARTITION BY RANGE (created_at)`
// postgresMetricSnapshotsDDL / postgresEdgeHealthDDL / postgresObsFrpsDDL / postgresObsFrpcDDL
// 与 goose/postgres/202608080001_create_log_tables.sql 对齐(普通表,无分区)。
const postgresMetricSnapshotsDDL = `
CREATE TABLE IF NOT EXISTS of_node_metric_snapshots (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
cpu_usage_percent DOUBLE PRECISION NOT NULL DEFAULT 0,
memory_used_bytes BIGINT NOT NULL DEFAULT 0,
memory_total_bytes BIGINT NOT NULL DEFAULT 0,
storage_used_bytes BIGINT NOT NULL DEFAULT 0,
storage_total_bytes BIGINT NOT NULL DEFAULT 0,
disk_read_bytes BIGINT NOT NULL DEFAULT 0,
disk_write_bytes BIGINT NOT NULL DEFAULT 0,
network_rx_bytes BIGINT NOT NULL DEFAULT 0,
network_tx_bytes BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)`
const postgresEdgeHealthDDL = `
CREATE TABLE IF NOT EXISTS of_node_edge_health (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
status VARCHAR(64) NOT NULL DEFAULT '',
connections BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)`
const postgresObsFrpsDDL = `
CREATE TABLE IF NOT EXISTS of_node_obs_frps (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
frps_connections INTEGER NOT NULL DEFAULT 0,
frps_proxy_count INTEGER NOT NULL DEFAULT 0,
frps_client_count INTEGER NOT NULL DEFAULT 0,
frps_proxies TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)`
const postgresObsFrpcDDL = `
CREATE TABLE IF NOT EXISTS of_node_obs_frpc (
id BIGINT NOT NULL PRIMARY KEY,
node_id VARCHAR(64) NOT NULL DEFAULT '',
captured_at TIMESTAMPTZ NOT NULL,
tunnel_status VARCHAR(16) NOT NULL DEFAULT '',
connected_relays_count INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)`
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,217 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"fmt"
"sync"
"time"
"Wavelet/openflare/plugins/server/kernel/model"
"Wavelet/openflare/plugins/server/kernel/runtimeconfig"
"Wavelet/pkg/logger"
db "Wavelet/plugins/infra/database"
)
// logDatabaseKey / logMigrationKey 对应 model.ConfigKeyLogDatabase / ConfigKeyLogDBMigration。
const (
logDatabaseKey = model.ConfigKeyLogDatabase
logMigrationKey = model.ConfigKeyLogDBMigration
)
// 日志库名常量(与 model 配置值一致,集中避免散落字符串字面量)。
const (
dbNamePostgres = "postgres"
dbNameSQLite = "sqlite"
dbNameClickHouse = "clickhouse"
)
// errConfigReaderNotWired 表示 config reader 尚未注入(首启/测试场景按 seed 规则兜底)。
var errConfigReaderNotWired = errors.New("logstore: config reader not wired")
// ConfigReader 读取系统配置字符串值,由 bootstrap 注入(避免 logstore ↔ repository 循环依赖)。
type ConfigReader func(ctx context.Context, key string) (string, error)
const resolveCacheTTL = 1 * time.Second
var (
configReader ConfigReader
storeMu sync.RWMutex
active *Store
activeDB string
lastResolveDB string
lastResolveTime time.Time
)
// SetConfigReader 注入系统配置读取函数(bootstrap 调用,测试可注入内存实现)。
func SetConfigReader(fn ConfigReader) { configReader = fn }
func getConfig(ctx context.Context, key string) (string, error) {
if configReader == nil {
return "", errConfigReaderNotWired
}
return configReader(ctx, key)
}
// Active 返回当前生效的日志库 Store。按 log_database 系统配置惰性解析并缓存,
// 配置更新(含迁移任务翻转)后自动重建。
func Active(ctx context.Context) (*Store, error) {
current, err := resolveDatabase(ctx)
if err != nil {
return nil, err
}
storeMu.RLock()
if active != nil && activeDB == current {
s := active
storeMu.RUnlock()
return s, nil
}
storeMu.RUnlock()
storeMu.Lock()
defer storeMu.Unlock()
if active != nil && activeDB == current {
return active, nil
}
s, err := buildStore(ctx, current, false)
if err != nil {
return nil, err
}
active = s
activeDB = current
return s, nil
}
// Build 直接按目标构造 store(不经 Active 缓存)。
func Build(ctx context.Context, database string) (*Store, error) {
return buildStore(ctx, database, false)
}
// BuildForMigration 构造迁移目标 store:与 Build 相同但不做冻结检查
// (迁移期间 log_db_migration=migrating 已冻结源库写入,目标库的清空/复制写入必须放行)。
func BuildForMigration(ctx context.Context, database string) (*Store, error) {
return buildStore(ctx, database, true)
}
// buildStore 按目标构造实现。skipFreeze 为 true 时该 store 跳过冻结检查
// (仅迁移任务的目标 store 使用)。gorm 分支 UserAccessLogs 用独立包装类型
// (gormLogStore 已占用 List/Count 方法名,无法再实现 UserAccessLogStore)。
func buildStore(ctx context.Context, database string, skipFreeze bool) (*Store, error) {
switch database {
case dbNameClickHouse:
ch := newClickHouseStore()
ch.skipFreeze = skipFreeze
ual := newClickHouseUserAccessLogStore()
ual.skipFreeze = skipFreeze
return &Store{
AccessLogs: ch,
Observability: ch,
UserAccessLogs: ual,
Status: ch,
}, nil
case dbNamePostgres, dbNameSQLite:
gdb := db.DB(ctx)
g := newGormStore(gdb)
g.skipFreeze = skipFreeze
ual := newUserAccessLogGormStore(gdb)
ual.skipFreeze = skipFreeze
return &Store{
AccessLogs: g,
Observability: g,
UserAccessLogs: ual,
Status: g,
}, nil
default:
return nil, fmt.Errorf("unsupported log database: %s", database)
}
}
// Migrating 返回日志库是否处于迁移冻结状态。
func Migrating(ctx context.Context) bool {
v, err := getConfig(ctx, logMigrationKey)
if err != nil {
if !errors.Is(err, errConfigReaderNotWired) {
logger.ErrorF(ctx, "read log migration config failed: %v", err)
}
return false
}
return v == "migrating"
}
// Init 在 bootstrap 阶段预热一次激活 store(幂等,失败不致命——首次使用时再解析),
// 并兜底预建「当前月 + 未来 2 个月」分区:进程停机跨月边界、重启后每日 cleanup 之前
// 首次写入不会报 "no partition of relation found"(CH/SQLite 分支 EnsurePartitions 为 no-op)。
func Init(ctx context.Context) {
s, err := Active(ctx)
if err != nil {
return
}
now := time.Now().UTC()
if err := s.AccessLogs.EnsurePartitions(ctx, now, now.AddDate(0, partitionLeadMonths, 0)); err != nil {
logger.WarnF(ctx, "logstore: ensure startup partitions failed: %v", err)
}
}
// InvalidateCache 清空日志库解析缓存(在修改 log_database 配置后显式调用)。
func InvalidateCache() {
storeMu.Lock()
defer storeMu.Unlock()
lastResolveTime = time.Time{}
lastResolveDB = ""
}
// ResetForTest 清空缓存的激活 store 与 config reader,便于测试注入。
func ResetForTest() {
storeMu.Lock()
active = nil
activeDB = ""
lastResolveDB = ""
lastResolveTime = time.Time{}
storeMu.Unlock()
configReader = nil
}
// ActiveDatabase 返回当前日志主库名(postgres|sqlite|clickhouse)。
func ActiveDatabase(ctx context.Context) (string, error) {
return resolveDatabase(ctx)
}
// resolveDatabase 读取 log_database:值缺失或 reader 未装配(首启)时按启动规则 seed;
// 已装配 reader 的真实读取错误直接透出,避免把读失败当首次启动。
func resolveDatabase(ctx context.Context) (string, error) {
storeMu.RLock()
if active != nil && time.Since(lastResolveTime) < resolveCacheTTL {
db := lastResolveDB
storeMu.RUnlock()
return db, nil
}
storeMu.RUnlock()
v, err := getConfig(ctx, logDatabaseKey)
if err != nil && !errors.Is(err, errConfigReaderNotWired) {
return "", err
}
resolved := v
if resolved == "" {
// 首次启动 seed:CH 启用 → clickhouse;否则随主库。
resolved = dbNameSQLite
if runtimeconfig.DatabaseEnabled() {
resolved = dbNamePostgres
}
if runtimeconfig.ClickHouseEnabled() {
resolved = dbNameClickHouse
}
}
storeMu.Lock()
lastResolveDB = resolved
lastResolveTime = time.Now()
storeMu.Unlock()
return resolved, nil
}
@@ -0,0 +1,109 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"errors"
"testing"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
)
func TestMigratingReadsConfig(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logMigrationKey {
return "migrating", nil
}
return "", nil
})
if !Migrating(context.Background()) {
t.Fatal("Migrating() = false, want true when key=migrating")
}
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", nil
})
if Migrating(context.Background()) {
t.Fatal("Migrating() = true, want false when key empty")
}
}
func TestResolveDatabaseDefaults(t *testing.T) {
ResetForTest()
// 配置缺失(reader 返回空值)时按主库规则 seed(config.Config 默认值由既有测试基建决定)。
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", nil
})
got, err := resolveDatabase(context.Background())
if err != nil {
t.Fatalf("resolveDatabase: %v", err)
}
if got != "postgres" && got != "sqlite" && got != "clickhouse" {
t.Fatalf("unexpected default log database: %s", got)
}
}
func TestResolveDatabaseSurfacesReadError(t *testing.T) {
ResetForTest()
wantErr := errors.New("boom")
SetConfigReader(func(_ context.Context, key string) (string, error) {
return "", wantErr
})
if _, err := resolveDatabase(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("resolveDatabase error = %v, want %v", err, wantErr)
}
}
func TestActiveBuildsStore(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logDatabaseKey {
return "sqlite", nil
}
return "", nil
})
store, err := Active(context.Background())
if err != nil {
t.Fatalf("Active: %v", err)
}
if store == nil {
t.Fatal("Active() returned nil store")
}
if store.AccessLogs == nil || store.Observability == nil || store.UserAccessLogs == nil || store.Status == nil {
t.Fatalf("Active() store fields not fully wired: %+v", store)
}
// 再次调用应命中缓存。
again, err := Active(context.Background())
if err != nil {
t.Fatalf("Active (cached): %v", err)
}
if again != store {
t.Fatal("Active() did not return cached store")
}
}
// TestClickHouseUserAccessLogBatchInsertFreeze 覆盖 CH 用户访问日志 flush 的冻结检查:
// 冻结期非空批次返回 ErrMigrating(在触碰 CH 连接之前),空批次直接成功。
func TestClickHouseUserAccessLogBatchInsertFreeze(t *testing.T) {
ResetForTest()
SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == logMigrationKey {
return "migrating", nil
}
return "", nil
})
defer ResetForTest()
s := newClickHouseUserAccessLogStore()
ctx := context.Background()
now := time.Now()
if err := s.BatchInsert(ctx, []analyticsmodel.UserAccessLog{{UserID: 1, CreatedAt: now}}); !errors.Is(err, ErrMigrating) {
t.Fatalf("BatchInsert during migration: want ErrMigrating, got %v", err)
}
if err := s.BatchInsert(ctx, nil); err != nil {
t.Fatalf("BatchInsert empty batch: %v", err)
}
}
@@ -0,0 +1,442 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"sort"
"strings"
"time"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
"Wavelet/openflare/plugins/server/kernel/model"
"Wavelet/openflare/plugins/server/kernel/repository/logstore"
)
const (
sortOrderAsc = "asc"
secondsPerMinute = 60
)
// ListOpenFlareAccessLogWAFIPAggregates returns per-IP aggregates for WAF automatic rules.
func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLogWAFIPAggregate, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.AccessLogs.WAFIPAggregates(ctx, query)
if err != nil {
return nil, err
}
result := make([]*model.OpenFlareAccessLogWAFIPAggregate, 0, len(rows))
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
statusCounts := make(map[int]int, len(row.StatusCounts))
for code, count := range row.StatusCounts {
statusCounts[code] = int(count)
}
result = append(result, &model.OpenFlareAccessLogWAFIPAggregate{
RemoteAddr: remoteAddr,
RequestCount: int(row.RequestCount),
Status404Count: int(row.Status404Count),
ClientErrorCount: int(row.ClientErrorCount),
ServerErrorCount: int(row.ServerErrorCount),
IPHostCount: int(row.IPHostCount),
LastSeenEpoch: row.LastSeenEpoch,
StatusCounts: statusCounts,
})
}
return result, nil
}
// InsertOpenFlareAccessLogsBatch inserts access log rows into the active log store.
func InsertOpenFlareAccessLogsBatch(ctx context.Context, records []*model.OpenFlareAccessLog) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.AccessLogs.InsertBatch(ctx, records)
}
// ListOpenFlareAccessLogs lists access logs matching the query.
func ListOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]*model.OpenFlareAccessLog, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.List(ctx, query)
}
// CountOpenFlareAccessLogs counts access logs, distinct IPs, and total bytes sent matching the query.
func CountOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (int64, int64, int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, 0, 0, err
}
return s.AccessLogs.Count(ctx, query)
}
// TrafficSummaryOpenFlareAccessLogs returns window-level request/error/UV/bytes summary.
func TrafficSummaryOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) (model.OpenFlareAccessLogTrafficSummary, error) {
s, err := logstore.Active(ctx)
if err != nil {
return model.OpenFlareAccessLogTrafficSummary{}, err
}
return s.AccessLogs.TrafficSummary(ctx, query)
}
// ValueCountsOpenFlareAccessLogs groups logs by status_code, host, path, remote_addr, or user_agent.
func ValueCountsOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery, column string, limit int) ([]model.OpenFlareAccessLogValueCount, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.ValueCounts(ctx, query, column, limit)
}
// NodeAggregatesOpenFlareAccessLogs returns per-node request/error/UV for the window.
func NodeAggregatesOpenFlareAccessLogs(ctx context.Context, query model.OpenFlareAccessLogQuery) ([]model.OpenFlareAccessLogNodeAggregate, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.NodeAggregates(ctx, query)
}
// ListOpenFlareAccessLogRegionCounts returns region counts for access logs.
func ListOpenFlareAccessLogRegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareAccessLogRegionCount, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.AccessLogs.RegionCounts(ctx, nodeID, since, limit)
}
// ListOpenFlareAccessLogBuckets lists folded access log buckets.
func ListOpenFlareAccessLogBuckets(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) ([]*model.OpenFlareAccessLogBucketRow, error) {
return buildOpenFlareAccessLogBucketRows(ctx, query)
}
// CountOpenFlareAccessLogBuckets counts folded access log buckets.
func CountOpenFlareAccessLogBuckets(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) (int64, error) {
filter := openFlareAccessLogQueryFromBucket(query)
bucketSeconds := int64(query.FoldMinutes * secondsPerMinute)
if bucketSeconds <= 0 {
bucketSeconds = 180
}
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.CountBuckets(ctx, filter, bucketSeconds)
}
// ListOpenFlareAccessLogBucketIPs lists folded IP rows for a bucket window.
func ListOpenFlareAccessLogBucketIPs(ctx context.Context, query model.OpenFlareAccessLogBucketIPQuery) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
rows, err := buildOpenFlareAccessLogBucketIPRows(ctx, query)
if err != nil {
return nil, err
}
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
if start >= len(rows) {
return []*model.OpenFlareAccessLogBucketIPRow{}, nil
}
return rows[start:end], nil
}
// CountOpenFlareAccessLogBucketIPs counts folded IP rows for a bucket window.
func CountOpenFlareAccessLogBucketIPs(ctx context.Context, query model.OpenFlareAccessLogBucketIPQuery) (int64, error) {
rows, err := buildOpenFlareAccessLogBucketIPRows(ctx, query)
if err != nil {
return 0, err
}
return int64(len(rows)), nil
}
// ListOpenFlareAccessLogIPSummaries lists IP summaries.
func ListOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*analyticsmodel.NodeAccessLogIPSummary, error) {
return buildOpenFlareAccessLogIPSummaryRows(ctx, query, recentSince)
}
// CountOpenFlareAccessLogIPSummaries counts IP summaries.
func CountOpenFlareAccessLogIPSummaries(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery) (int64, error) {
filter := openFlareAccessLogQueryFromIPSummary(query)
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.CountIPSummaries(ctx, filter)
}
// ListOpenFlareAccessLogIPTrend lists IP trend points.
func ListOpenFlareAccessLogIPTrend(ctx context.Context, query model.OpenFlareAccessLogIPTrendQuery) ([]*analyticsmodel.NodeAccessLogIPTrend, error) {
remoteAddr := strings.TrimSpace(query.RemoteAddr)
if remoteAddr == "" {
return []*analyticsmodel.NodeAccessLogIPTrend{}, nil
}
filter := model.OpenFlareAccessLogQuery{
NodeID: query.NodeID,
RemoteAddr: remoteAddr,
Host: query.Host,
Since: query.Since,
}
bucketSeconds := int64(query.BucketMinutes * secondsPerMinute)
if bucketSeconds <= 0 {
bucketSeconds = 1800
}
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.AccessLogs.IPTrend(ctx, filter, bucketSeconds)
if err != nil {
return nil, err
}
result := make([]*analyticsmodel.NodeAccessLogIPTrend, len(rows))
for index, row := range rows {
result[index] = &analyticsmodel.NodeAccessLogIPTrend{
BucketEpoch: row.BucketEpoch,
RequestCount: row.RequestCount,
}
}
return result, nil
}
// DeleteAllOpenFlareAccessLogs deletes all access logs.
func DeleteAllOpenFlareAccessLogs(ctx context.Context) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.DeleteAll(ctx)
}
// DeleteOpenFlareAccessLogsBefore deletes access logs older than cutoff.
func DeleteOpenFlareAccessLogsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.DeleteBefore(ctx, cutoff)
}
// DeleteOpenFlareAccessLogsByNodeBefore deletes access logs for a node older than cutoff.
func DeleteOpenFlareAccessLogsByNodeBefore(ctx context.Context, nodeID string, cutoff time.Time) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.AccessLogs.DeleteByNodeBefore(ctx, nodeID, cutoff)
}
func buildOpenFlareAccessLogBucketRows(ctx context.Context, query model.OpenFlareAccessLogBucketQuery) ([]*model.OpenFlareAccessLogBucketRow, error) {
filter := openFlareAccessLogQueryFromBucket(query)
bucketSeconds := int64(query.FoldMinutes * secondsPerMinute)
if bucketSeconds <= 0 {
bucketSeconds = 180
}
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
partials, err := s.AccessLogs.BucketAggregates(ctx, filter, bucketSeconds)
if err != nil {
return nil, err
}
rows := make([]*model.OpenFlareAccessLogBucketRow, 0, len(partials))
for _, partial := range partials {
rows = append(rows, &model.OpenFlareAccessLogBucketRow{
BucketEpoch: partial.BucketEpoch,
RequestCount: partial.RequestCount,
UniqueIPCount: partial.UniqueIPCount,
UniqueHostCount: partial.UniqueHostCount,
SuccessCount: partial.SuccessCount,
ClientErrorCount: partial.ClientErrorCount,
ServerErrorCount: partial.ServerErrorCount,
Status2xxCount: partial.Status2xxCount,
Status4xxCount: partial.Status4xxCount,
Status5xxCount: partial.Status5xxCount,
BytesSent: partial.BytesSent,
RequestLength: partial.RequestLength,
})
}
return rows, nil
}
func buildOpenFlareAccessLogBucketIPRows(ctx context.Context, query model.OpenFlareAccessLogBucketIPQuery) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
if query.BucketStartedAt.IsZero() {
return []*model.OpenFlareAccessLogBucketIPRow{}, nil
}
foldMinutes := query.FoldMinutes
if foldMinutes <= 0 {
foldMinutes = 3
}
bucketStartedAt := query.BucketStartedAt.UTC()
filter := model.OpenFlareAccessLogQuery{
NodeID: query.NodeID,
RemoteAddr: query.RemoteAddr,
Host: query.Host,
Path: query.Path,
Since: bucketStartedAt,
Until: bucketStartedAt.Add(time.Duration(foldMinutes) * time.Minute),
}
rows, err := queryOpenFlareAccessLogIPAggregateRows(ctx, filter, false)
if err != nil {
return nil, err
}
sortOpenFlareAccessLogBucketIPRows(rows, query.SortBy, query.SortOrder)
return rows, nil
}
func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query model.OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*analyticsmodel.NodeAccessLogIPSummary, error) {
filter := openFlareAccessLogQueryFromIPSummary(query)
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
partials, err := s.AccessLogs.IPSummaries(ctx, filter, recentSince)
if err != nil {
return nil, err
}
rows := make([]*analyticsmodel.NodeAccessLogIPSummary, 0, len(partials))
for _, partial := range partials {
remoteAddr := strings.TrimSpace(partial.RemoteAddr)
if remoteAddr == "" {
continue
}
rows = append(rows, &analyticsmodel.NodeAccessLogIPSummary{
RemoteAddr: remoteAddr,
Region: strings.TrimSpace(partial.Region),
TotalRequests: partial.TotalRequests,
Success2xxCount: partial.Success2xxCount,
SuccessRatio: partial.SuccessRatio,
BytesReceived: partial.BytesReceived,
BytesSent: partial.BytesSent,
RecentRequests: 0,
LastSeenEpoch: partial.LastSeenEpoch,
})
}
return rows, nil
}
func queryOpenFlareAccessLogIPAggregateRows(ctx context.Context, filter model.OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]*model.OpenFlareAccessLogBucketIPRow, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
partials, err := s.AccessLogs.IPAggregates(ctx, filter, exactRemoteAddr)
if err != nil {
return nil, err
}
rows := make([]*model.OpenFlareAccessLogBucketIPRow, 0, len(partials))
for _, partial := range partials {
remoteAddr := strings.TrimSpace(partial.RemoteAddr)
if remoteAddr == "" {
continue
}
rows = append(rows, &model.OpenFlareAccessLogBucketIPRow{
RemoteAddr: remoteAddr,
RequestCount: partial.RequestCount,
SuccessCount: partial.SuccessCount,
ClientErrorCount: partial.ClientErrorCount,
ServerErrorCount: partial.ServerErrorCount,
LastSeenEpoch: partial.LastSeenEpoch,
})
}
return rows, nil
}
func openFlareAccessLogQueryFromBucket(query model.OpenFlareAccessLogBucketQuery) model.OpenFlareAccessLogQuery {
return model.OpenFlareAccessLogQuery{
NodeID: query.NodeID,
RemoteAddr: query.RemoteAddr,
Host: query.Host,
Hosts: query.Hosts,
Path: query.Path,
Since: query.Since,
Until: query.Until,
Page: query.Page,
PageSize: query.PageSize,
SortBy: query.SortBy,
SortOrder: query.SortOrder,
}
}
func openFlareAccessLogQueryFromIPSummary(query model.OpenFlareAccessLogIPSummaryQuery) model.OpenFlareAccessLogQuery {
return model.OpenFlareAccessLogQuery{
NodeID: query.NodeID,
RemoteAddr: query.RemoteAddr,
Host: query.Host,
Since: query.Since,
Until: query.Until,
Page: query.Page,
PageSize: query.PageSize,
SortBy: query.SortBy,
SortOrder: query.SortOrder,
}
}
func sortOpenFlareAccessLogBucketIPRows(items []*model.OpenFlareAccessLogBucketIPRow, sortBy string, sortOrder string) {
desc := openFlareAccessLogNormalizeSortOrder(sortOrder) != sortOrderAsc
sort.Slice(items, func(i, j int) bool {
left := items[i]
right := items[j]
if left == nil || right == nil {
return left != nil
}
var compare int
switch strings.TrimSpace(sortBy) {
case "last_seen_at":
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
case "remote_addr":
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
default:
compare = openFlareAccessLogCompareInt64(left.RequestCount, right.RequestCount)
}
if compare == 0 {
compare = openFlareAccessLogCompareInt64(left.LastSeenEpoch, right.LastSeenEpoch)
}
if compare == 0 {
compare = strings.Compare(left.RemoteAddr, right.RemoteAddr)
}
if desc {
return compare > 0
}
return compare < 0
})
}
func openFlareAccessLogPaginateBounds(total int, page int, pageSize int) (int, int) {
if page < 0 {
page = 0
}
if pageSize <= 0 {
return 0, total
}
start := min(page*pageSize, total)
end := min(start+pageSize, total)
return start, end
}
func openFlareAccessLogNormalizeSortOrder(sortOrder string) string {
if strings.EqualFold(strings.TrimSpace(sortOrder), sortOrderAsc) {
return sortOrderAsc
}
return "desc"
}
func openFlareAccessLogCompareInt64(left int64, right int64) int {
switch {
case left > right:
return 1
case left < right:
return -1
default:
return 0
}
}
@@ -0,0 +1,164 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"fmt"
"sync/atomic"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"Wavelet/openflare/plugins/server/kernel/model"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
"Wavelet/openflare/plugins/server/kernel/repository/logstore"
"Wavelet/pkg/idgen"
db "Wavelet/plugins/infra/database"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// accessLogTestDBSeq 保证每个测试获得独立的 sqlite 内存库(cache=shared 下同名 DSN 复用同一库)。
var accessLogTestDBSeq int64
func setupOpenFlareAccessLogTestEnvironment(t *testing.T) (context.Context, func()) {
t.Helper()
dsn := fmt.Sprintf("file:repo-access-log-test-%d?mode=memory&cache=shared", atomic.AddInt64(&accessLogTestDBSeq, 1))
gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: logger.Default.LogMode(logger.Silent),
})
require.NoError(t, err)
require.NoError(t, gdb.AutoMigrate(&analyticsmodel.NodeAccessLog{}))
db.SetDB(gdb)
require.NoError(t, idgen.Init(1))
logstore.ResetForTest()
logstore.SetConfigReader(func(_ context.Context, key string) (string, error) {
if key == model.ConfigKeyLogDatabase {
return "sqlite", nil
}
return "", nil
})
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
logstore.SetObservabilityHooks(logstore.ObservabilityHooks{})
ctx := context.Background()
store, err := logstore.Active(ctx)
require.NoError(t, err)
// 写入入口只入队;测试环境立即 flush,保证后续查询可见。
logstore.SetAccessLogHooks(logstore.AccessLogHooks{
QueueNodeAccessLogs: func(logs []analyticsmodel.NodeAccessLog) {
require.NoError(t, store.AccessLogs.BatchInsertNodeAccessLogs(context.Background(), logs))
},
})
return ctx, func() {
logstore.SetAccessLogHooks(logstore.AccessLogHooks{})
logstore.ResetForTest()
db.SetDB(nil)
}
}
func seedOpenFlareAccessLogs(t *testing.T, ctx context.Context, now time.Time) {
t.Helper()
records := []*model.OpenFlareAccessLog{
{NodeID: "node-a", LoggedAt: now.Add(-5 * time.Minute), RemoteAddr: "1.1.1.1", Region: "US", Host: "a.example.com", Path: "/alpha", StatusCode: 200},
{NodeID: "node-a", LoggedAt: now.Add(-4 * time.Minute), RemoteAddr: "2.2.2.2", Region: "US", Host: "a.example.com", Path: "/beta", StatusCode: 404},
{NodeID: "node-b", LoggedAt: now.Add(-3 * time.Minute), RemoteAddr: "1.1.1.1", Region: "EU", Host: "b.example.com", Path: "/gamma", StatusCode: 502},
{NodeID: "node-b", LoggedAt: now.Add(-2 * time.Minute), RemoteAddr: "3.3.3.3", Region: "EU", Host: "b.example.com", Path: "/delta", StatusCode: 200},
{NodeID: "node-b", LoggedAt: now.Add(-1 * time.Minute), RemoteAddr: "", Region: "", Host: "b.example.com", Path: "/empty-ip", StatusCode: 200},
}
require.NoError(t, InsertOpenFlareAccessLogsBatch(ctx, records))
}
func TestListOpenFlareAccessLogsPaginated(t *testing.T) {
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
defer cleanup()
now := time.Now().UTC()
for index := range 15 {
record := &model.OpenFlareAccessLog{
NodeID: "node-page",
LoggedAt: now.Add(-time.Duration(index) * time.Minute),
RemoteAddr: fmt.Sprintf("203.0.113.%d", (index%5)+1),
Host: "example.com",
Path: fmt.Sprintf("/path-%02d", index),
StatusCode: 200,
}
require.NoError(t, InsertOpenFlareAccessLogsBatch(ctx, []*model.OpenFlareAccessLog{record}))
}
// 0-based 分页与 CH ListNodeAccessLogs 一致:page=1 size=5 → OFFSET 5 → /path-05..09。
query := model.OpenFlareAccessLogQuery{
NodeID: "node-page",
Since: now.Add(-24 * time.Hour),
Page: 1,
PageSize: 5,
SortBy: "logged_at",
SortOrder: "desc",
}
page, err := ListOpenFlareAccessLogs(ctx, query)
require.NoError(t, err)
require.Len(t, page, 5)
assert.Equal(t, "/path-05", page[0].Path)
assert.Equal(t, "/path-09", page[4].Path)
}
func TestCountOpenFlareAccessLogs(t *testing.T) {
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
defer cleanup()
now := time.Now().UTC()
seedOpenFlareAccessLogs(t, ctx, now)
query := model.OpenFlareAccessLogQuery{
Since: now.Add(-10 * time.Minute),
}
totalRecords, totalIPs, _, err := CountOpenFlareAccessLogs(ctx, query)
require.NoError(t, err)
assert.Equal(t, int64(5), totalRecords)
// GORM 与 CH 一致:distinct IP 排除空 remote_addr(CH uniqExactIf(remote_addr, remote_addr != ''))。
assert.Equal(t, int64(3), totalIPs)
}
func TestListOpenFlareAccessLogsFiltersAndSort(t *testing.T) {
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
defer cleanup()
now := time.Now().UTC()
seedOpenFlareAccessLogs(t, ctx, now)
query := model.OpenFlareAccessLogQuery{
NodeID: "node-a",
Since: now.Add(-10 * time.Minute),
SortBy: "status_code",
SortOrder: "desc",
}
rows, err := ListOpenFlareAccessLogs(ctx, query)
require.NoError(t, err)
require.Len(t, rows, 2)
assert.Equal(t, 404, rows[0].StatusCode)
assert.Equal(t, 200, rows[1].StatusCode)
}
func TestDeleteOpenFlareAccessLogsBefore(t *testing.T) {
ctx, cleanup := setupOpenFlareAccessLogTestEnvironment(t)
defer cleanup()
now := time.Now().UTC()
seedOpenFlareAccessLogs(t, ctx, now)
deleted, err := DeleteOpenFlareAccessLogsBefore(ctx, now.Add(-2*time.Minute))
require.NoError(t, err)
assert.Equal(t, int64(3), deleted)
totalRecords, _, _, err := CountOpenFlareAccessLogs(ctx, model.OpenFlareAccessLogQuery{Since: now.Add(-10 * time.Minute)})
require.NoError(t, err)
assert.Equal(t, int64(2), totalRecords)
}
@@ -0,0 +1,68 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"gorm.io/gorm"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// GetAcmeAccountByID 按 ID 查询 ACME 账号。
func GetAcmeAccountByID(ctx context.Context, id uint) (*model.AcmeAccount, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var account model.AcmeAccount
if err := conn.First(&account, id).Error; err != nil {
return nil, err
}
return &account, nil
}
// CreateAcmeAccountRecord 创建 ACME 账号。
func CreateAcmeAccountRecord(ctx context.Context, account *model.AcmeAccount) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Create(account).Error
}
// SaveAcmeAccount 保存 ACME 账号。
func SaveAcmeAccount(ctx context.Context, account *model.AcmeAccount) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Save(account).Error
}
// GetDefaultAcmeAccount 获取默认 ACME 账号,不存在时创建占位记录。
func GetDefaultAcmeAccount(ctx context.Context) (*model.AcmeAccount, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var account model.AcmeAccount
err := conn.Order("id asc").First(&account).Error
if err == nil {
return &account, nil
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
account = model.AcmeAccount{
Email: "admin@openflare.dev",
}
if err = conn.Create(&account).Error; err != nil {
return nil, err
}
return &account, nil
}
@@ -0,0 +1,162 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"strings"
"time"
"gorm.io/gorm"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// ListOpenFlareApplyLogs returns apply logs ordered by id desc with optional pagination.
func ListOpenFlareApplyLogs(ctx context.Context, query model.OpenFlareApplyLogQuery) ([]*model.OpenFlareApplyLog, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
dbQuery := conn.Model(&model.OpenFlareApplyLog{}).Order("id desc")
if query.NodeID != "" {
dbQuery = dbQuery.Where("node_id = ?", query.NodeID)
}
if query.PageSize > 0 {
offset := 0
if query.PageNo > 1 {
offset = (query.PageNo - 1) * query.PageSize
}
dbQuery = dbQuery.Limit(query.PageSize).Offset(offset)
}
var logs []*model.OpenFlareApplyLog
if err := dbQuery.Find(&logs).Error; err != nil {
return nil, err
}
return logs, nil
}
// CountOpenFlareApplyLogs returns total apply logs, optionally filtered by node_id.
func CountOpenFlareApplyLogs(ctx context.Context, nodeID string) (int64, error) {
conn := db.DB(ctx)
if conn == nil {
return 0, errors.New(errDatabaseNotInitialized)
}
query := conn.Model(&model.OpenFlareApplyLog{})
if nodeID != "" {
query = query.Where("node_id = ?", nodeID)
}
var total int64
if err := query.Count(&total).Error; err != nil {
return 0, err
}
return total, nil
}
// GetLatestOpenFlareApplyLogByNodeID returns the most recent apply log for a node.
func GetLatestOpenFlareApplyLogByNodeID(ctx context.Context, nodeID string) (*model.OpenFlareApplyLog, error) {
nodeID = strings.TrimSpace(nodeID)
if nodeID == "" {
return nil, errors.New("node_id is required")
}
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var log model.OpenFlareApplyLog
err := conn.Where("node_id = ?", nodeID).Order("id desc").First(&log).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &log, nil
}
// GetLatestOpenFlareApplyLogsByNodeIDs returns the latest apply log per node id.
func GetLatestOpenFlareApplyLogsByNodeIDs(ctx context.Context, nodeIDs []string) (map[string]*model.OpenFlareApplyLog, error) {
result := make(map[string]*model.OpenFlareApplyLog)
if len(nodeIDs) == 0 {
return result, nil
}
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var logs []*model.OpenFlareApplyLog
subQuery := conn.Model(&model.OpenFlareApplyLog{}).
Select("MAX(id) AS id").
Where("node_id IN ?", nodeIDs).
Group("node_id")
if err := conn.Where("id IN (?)", subQuery).Find(&logs).Error; err != nil {
return nil, err
}
for _, log := range logs {
result[log.NodeID] = log
}
return result, nil
}
// CreateOpenFlareApplyLog inserts an apply log row.
func CreateOpenFlareApplyLog(ctx context.Context, log *model.OpenFlareApplyLog) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Create(log).Error
}
// CreateOpenFlareApplyLogAndUpdateNode creates an apply log and updates the node from the apply result in one transaction.
func CreateOpenFlareApplyLogAndUpdateNode(ctx context.Context, log *model.OpenFlareApplyLog, applyResult, version, message string) error {
if log == nil {
return errors.New("apply log is required")
}
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
now := log.CreatedAt
if now.IsZero() {
now = time.Now()
}
return conn.Transaction(func(tx *gorm.DB) error {
if err := tx.Create(log).Error; err != nil {
return err
}
return updateOpenFlareNodeFromApplyResultTx(tx, log.NodeID, applyResult, version, message, now)
})
}
// DeleteAllOpenFlareApplyLogs removes every apply log record.
func DeleteAllOpenFlareApplyLogs(ctx context.Context) (int64, error) {
conn := db.DB(ctx)
if conn == nil {
return 0, errors.New(errDatabaseNotInitialized)
}
result := conn.Session(&gorm.Session{AllowGlobalUpdate: true}).Delete(&model.OpenFlareApplyLog{})
return result.RowsAffected, result.Error
}
// DeleteOpenFlareApplyLogsBefore removes apply logs created before the cutoff time.
func DeleteOpenFlareApplyLogsBefore(ctx context.Context, before time.Time) (int64, error) {
conn := db.DB(ctx)
if conn == nil {
return 0, errors.New(errDatabaseNotInitialized)
}
result := conn.Where("created_at < ?", before).Delete(&model.OpenFlareApplyLog{})
return result.RowsAffected, result.Error
}
@@ -0,0 +1,79 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"testing"
"time"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func setupApplyLogModelTestDB(t *testing.T) func() {
t.Helper()
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, sqliteDB.AutoMigrate(&model.OpenFlareApplyLog{}))
db.SetDB(sqliteDB)
return func() {
db.SetDB(nil)
}
}
func TestIsRepeatSuccessApplyLog(t *testing.T) {
latest := &model.OpenFlareApplyLog{
Version: "20260615-001",
Checksum: "checksum-a",
Result: "success",
}
assert.True(t, model.IsRepeatSuccessApplyLog(latest, "20260615-001", "checksum-a", "success"))
assert.False(t, model.IsRepeatSuccessApplyLog(latest, "20260615-002", "checksum-a", "success"))
assert.False(t, model.IsRepeatSuccessApplyLog(latest, "20260615-001", "checksum-b", "success"))
assert.False(t, model.IsRepeatSuccessApplyLog(latest, "20260615-001", "checksum-a", "failed"))
assert.False(t, model.IsRepeatSuccessApplyLog(nil, "20260615-001", "checksum-a", "success"))
}
func TestGetLatestOpenFlareApplyLogByNodeID(t *testing.T) {
cleanup := setupApplyLogModelTestDB(t)
defer cleanup()
ctx := context.Background()
now := time.Now().UTC()
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareApplyLog{
NodeID: "node-1",
Version: "v1",
Result: "success",
Checksum: "checksum-1",
CreatedAt: now.Add(-time.Hour),
}).Error)
require.NoError(t, db.DB(ctx).Create(&model.OpenFlareApplyLog{
NodeID: "node-1",
Version: "v2",
Result: "success",
Checksum: "checksum-2",
CreatedAt: now,
}).Error)
latest, err := GetLatestOpenFlareApplyLogByNodeID(ctx, "node-1")
require.NoError(t, err)
require.NotNil(t, latest)
assert.Equal(t, "v2", latest.Version)
missing, err := GetLatestOpenFlareApplyLogByNodeID(ctx, "node-missing")
require.NoError(t, err)
assert.Nil(t, missing)
}
@@ -0,0 +1,226 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
"gorm.io/gorm"
)
const singletonCFConnectionID uint = 1
// CFPointingMemberContext contains all local state needed to reconcile one member.
type CFPointingMemberContext struct {
Member model.CFPointingMember
Group model.CFPointingGroup
Domain model.ZoneDomain
Zone model.Zone
Node model.OpenFlareNode
}
// GetCFConnection returns the global Cloudflare connection.
func GetCFConnection(ctx context.Context) (*model.CFConnection, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var item model.CFConnection
if err := conn.First(&item, singletonCFConnectionID).Error; err != nil {
return nil, err
}
return &item, nil
}
// UpsertCFConnection creates or replaces the global Cloudflare connection.
func UpsertCFConnection(ctx context.Context, item *model.CFConnection) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
item.ID = singletonCFConnectionID
return conn.Save(item).Error
}
// DeleteCFConnection clears the global Cloudflare connection.
func DeleteCFConnection(ctx context.Context) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Delete(&model.CFConnection{}, singletonCFConnectionID).Error
}
// ListCFPointingGroups lists Cloudflare pointing groups newest first.
func ListCFPointingGroups(ctx context.Context) ([]model.CFPointingGroup, error) {
var items []model.CFPointingGroup
if err := db.DB(ctx).Order("id desc").Find(&items).Error; err != nil {
return nil, err
}
return items, nil
}
// GetCFPointingGroup returns a group by ID.
func GetCFPointingGroup(ctx context.Context, id uint) (*model.CFPointingGroup, error) {
var item model.CFPointingGroup
if err := db.DB(ctx).First(&item, id).Error; err != nil {
return nil, err
}
return &item, nil
}
// CreateCFPointingGroup creates a group.
func CreateCFPointingGroup(ctx context.Context, item *model.CFPointingGroup) error {
return db.DB(ctx).Create(item).Error
}
// SaveCFPointingGroup persists a group.
func SaveCFPointingGroup(ctx context.Context, item *model.CFPointingGroup) error {
return db.DB(ctx).Save(item).Error
}
// DeleteCFPointingGroup deletes an empty group.
func DeleteCFPointingGroup(ctx context.Context, id uint) error {
return db.DB(ctx).Delete(&model.CFPointingGroup{}, id).Error
}
// CountCFPointingMembersByGroupID counts members in a group.
func CountCFPointingMembersByGroupID(ctx context.Context, groupID uint) (int64, error) {
var count int64
err := db.DB(ctx).Table("of_cf_pointing_members AS members").
Joins("JOIN of_zone_domains AS domains ON domains.id = members.zone_domain_id").
Where("members.group_id = ?", groupID).Count(&count).Error
return count, err
}
// ListCFPointingMembersByGroupID lists members by group.
func ListCFPointingMembersByGroupID(ctx context.Context, groupID uint) ([]model.CFPointingMember, error) {
var items []model.CFPointingMember
if err := db.DB(ctx).Where("group_id = ?", groupID).Order("id asc").Find(&items).Error; err != nil {
return nil, err
}
return items, nil
}
// ListCFPointingMembersByActiveNodeID lists members whose group currently targets a node.
func ListCFPointingMembersByActiveNodeID(ctx context.Context, nodeID uint) ([]model.CFPointingMember, error) {
var items []model.CFPointingMember
err := db.DB(ctx).Table("of_cf_pointing_members AS members").
Select("members.*").
Joins("JOIN of_cf_pointing_groups AS groups ON groups.id = members.group_id").
Where("groups.active_node_id = ? AND groups.enabled = ?", nodeID, true).
Order("members.id asc").Scan(&items).Error
return items, err
}
// GetCFPointingMember returns a member scoped to its group.
func GetCFPointingMember(ctx context.Context, groupID, memberID uint) (*model.CFPointingMember, error) {
var item model.CFPointingMember
if err := db.DB(ctx).Where("id = ? AND group_id = ?", memberID, groupID).First(&item).Error; err != nil {
return nil, err
}
return &item, nil
}
// GetCFPointingMemberByID returns a member by ID.
func GetCFPointingMemberByID(ctx context.Context, id uint) (*model.CFPointingMember, error) {
var item model.CFPointingMember
if err := db.DB(ctx).First(&item, id).Error; err != nil {
return nil, err
}
return &item, nil
}
// GetCFPointingMemberByZoneDomainID returns the member managing a ZoneDomain.
func GetCFPointingMemberByZoneDomainID(ctx context.Context, zoneDomainID uint) (*model.CFPointingMember, error) {
var item model.CFPointingMember
if err := db.DB(ctx).Where("zone_domain_id = ?", zoneDomainID).First(&item).Error; err != nil {
return nil, err
}
return &item, nil
}
// CreateCFPointingMember creates a member.
func CreateCFPointingMember(ctx context.Context, item *model.CFPointingMember) error {
return db.DB(ctx).Create(item).Error
}
// SaveCFPointingMember persists a member.
func SaveCFPointingMember(ctx context.Context, item *model.CFPointingMember) error {
return db.DB(ctx).Save(item).Error
}
// UpdateCFPointingMemberColumns updates selected member fields.
func UpdateCFPointingMemberColumns(ctx context.Context, id uint, changes map[string]any) error {
return db.DB(ctx).Model(&model.CFPointingMember{}).Where("id = ?", id).Updates(changes).Error
}
// DeleteCFPointingMember deletes a member.
func DeleteCFPointingMember(ctx context.Context, item *model.CFPointingMember) error {
return db.DB(ctx).Delete(item).Error
}
// ListAvailableCFZoneDomains returns ZoneDomains not already managed by Cloudflare pointing.
func ListAvailableCFZoneDomains(ctx context.Context) ([]model.ZoneDomain, error) {
var items []model.ZoneDomain
err := db.DB(ctx).Where(`NOT EXISTS (
SELECT 1 FROM of_cf_pointing_members AS members
WHERE members.zone_domain_id = of_zone_domains.id
)`).Order("domain asc").Find(&items).Error
return items, err
}
// GetCFPointingMemberContext loads one member and all referenced local objects.
func GetCFPointingMemberContext(ctx context.Context, memberID uint) (*CFPointingMemberContext, error) {
member, err := GetCFPointingMemberByID(ctx, memberID)
if err != nil {
return nil, err
}
group, err := GetCFPointingGroup(ctx, member.GroupID)
if err != nil {
return nil, err
}
domain, err := GetZoneDomainByID(ctx, member.ZoneDomainID)
if err != nil {
return nil, err
}
zone, err := GetZoneByID(ctx, domain.ZoneID)
if err != nil {
return nil, err
}
node, err := GetOpenFlareNodeByID(ctx, group.ActiveNodeID)
if err != nil {
return nil, err
}
return &CFPointingMemberContext{Member: *member, Group: *group, Domain: *domain, Zone: *zone, Node: *node}, nil
}
// GetZoneDomainByID returns a ZoneDomain by primary key.
func GetZoneDomainByID(ctx context.Context, id uint) (*model.ZoneDomain, error) {
var item model.ZoneDomain
if err := db.DB(ctx).First(&item, id).Error; err != nil {
return nil, err
}
return &item, nil
}
// MarkCFPointingGroupMembersPending resets every member after target changes.
func MarkCFPointingGroupMembersPending(ctx context.Context, groupID uint) error {
return db.DB(ctx).Model(&model.CFPointingMember{}).Where("group_id = ?", groupID).
Updates(map[string]any{"sync_status": model.CFMemberSyncPending, "last_error": ""}).Error
}
// DeleteCFPointingGroupAndMembers removes a group after its remote records are deleted.
func DeleteCFPointingGroupAndMembers(ctx context.Context, groupID uint) error {
return db.DB(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Where("group_id = ?", groupID).Delete(&model.CFPointingMember{}).Error; err != nil {
return err
}
return tx.Delete(&model.CFPointingGroup{}, groupID).Error
})
}
@@ -0,0 +1,87 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"testing"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func setupCloudflareRepositoryDB(t *testing.T) *gorm.DB {
t.Helper()
conn, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{DisableForeignKeyConstraintWhenMigrating: true})
if err != nil {
t.Fatalf("gorm.Open() error = %v", err)
}
if err := conn.AutoMigrate(
&model.CFConnection{}, &model.CFPointingGroup{}, &model.CFPointingMember{},
&model.Zone{}, &model.ZoneDomain{}, &model.OpenFlareNode{}, &model.DNSAccount{},
); err != nil {
t.Fatalf("AutoMigrate() error = %v", err)
}
db.SetDB(conn)
t.Cleanup(func() { db.SetDB(nil) })
return conn
}
func TestUpsertCFConnectionKeepsSingleRow(t *testing.T) {
setupCloudflareRepositoryDB(t)
ctx := context.Background()
first := &model.CFConnection{Source: model.CFConnectionSourceStandalone, Authorization: "one"}
if err := UpsertCFConnection(ctx, first); err != nil {
t.Fatalf("UpsertCFConnection(first) error = %v", err)
}
accountID := uint(9)
second := &model.CFConnection{Source: model.CFConnectionSourceDNSAccount, DNSAccountID: &accountID}
if err := UpsertCFConnection(ctx, second); err != nil {
t.Fatalf("UpsertCFConnection(second) error = %v", err)
}
got, err := GetCFConnection(ctx)
if err != nil {
t.Fatalf("GetCFConnection() error = %v", err)
}
if got.ID != first.ID || got.Source != model.CFConnectionSourceDNSAccount {
t.Errorf("GetCFConnection() = %+v, want same row with dns_account source", got)
}
}
func TestListAvailableCFZoneDomainsExcludesMembers(t *testing.T) {
conn := setupCloudflareRepositoryDB(t)
ctx := context.Background()
zone := model.Zone{Domain: "example.com"}
if err := conn.Create(&zone).Error; err != nil {
t.Fatalf("Create(zone) error = %v", err)
}
domains := []model.ZoneDomain{
{ZoneID: zone.ID, Domain: "api.example.com"},
{ZoneID: zone.ID, Domain: "www.example.com"},
}
if err := conn.Create(&domains).Error; err != nil {
t.Fatalf("Create(domains) error = %v", err)
}
group := model.CFPointingGroup{Name: "edge", PrimaryNodeID: 1, ActiveNodeID: 1, Enabled: true}
if err := CreateCFPointingGroup(ctx, &group); err != nil {
t.Fatalf("CreateCFPointingGroup() error = %v", err)
}
member := model.CFPointingMember{GroupID: group.ID, ZoneDomainID: domains[0].ID}
if err := CreateCFPointingMember(ctx, &member); err != nil {
t.Fatalf("CreateCFPointingMember() error = %v", err)
}
got, err := ListAvailableCFZoneDomains(ctx)
if err != nil {
t.Fatalf("ListAvailableCFZoneDomains() error = %v", err)
}
if len(got) != 1 || got[0].ID != domains[1].ID {
t.Errorf("ListAvailableCFZoneDomains() = %+v, want only %d", got, domains[1].ID)
}
}
@@ -0,0 +1,135 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"gorm.io/gorm"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// ListConfigVersionSummaries returns config version summaries ordered by created_at desc.
func ListConfigVersionSummaries(ctx context.Context) ([]*model.ConfigVersionSummary, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var versions []*model.ConfigVersionSummary
err := conn.Model(&model.ConfigVersion{}).
Select("version", "checksum", "is_active", "created_by", "created_at").
Order("created_at desc, version desc").
Find(&versions).Error
return versions, err
}
// GetConfigVersionByVersion returns a config version by version string.
func GetConfigVersionByVersion(ctx context.Context, version string) (*model.ConfigVersion, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var cv model.ConfigVersion
if err := conn.First(&cv, "version = ?", version).Error; err != nil {
return nil, err
}
return &cv, nil
}
// GetActiveConfigVersion returns the currently active config version.
func GetActiveConfigVersion(ctx context.Context) (*model.ConfigVersion, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var version model.ConfigVersion
if err := conn.Where("is_active = ?", true).Order("version desc").First(&version).Error; err != nil {
return nil, err
}
return &version, nil
}
// GetLatestConfigVersionByPrefix returns the latest version string matching a date prefix.
func GetLatestConfigVersionByPrefix(ctx context.Context, prefix string) (string, error) {
conn := db.DB(ctx)
if conn == nil {
return "", errors.New(errDatabaseNotInitialized)
}
var version model.ConfigVersion
err := conn.Model(&model.ConfigVersion{}).
Select("version").
Where("version LIKE ?", prefix+"-%").
Order("version desc").
First(&version).Error
if err != nil {
return "", err
}
return version.Version, nil
}
// CreateConfigVersion inserts a new config version record.
func CreateConfigVersion(ctx context.Context, version *model.ConfigVersion) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Create(version).Error
}
// PublishConfigVersionTx deactivates all versions and creates a new active version.
func PublishConfigVersionTx(ctx context.Context, version *model.ConfigVersion) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&model.ConfigVersion{}).Where("is_active = ?", true).Update("is_active", false).Error; err != nil {
return err
}
return tx.Create(version).Error
})
}
// ActivateConfigVersionTx marks the given version active and deactivates others.
func ActivateConfigVersionTx(ctx context.Context, version string) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&model.ConfigVersion{}).Where("is_active = ?", true).Update("is_active", false).Error; err != nil {
return err
}
return tx.Model(&model.ConfigVersion{}).Where("version = ?", version).Update("is_active", true).Error
})
}
// DeleteConfigVersionsByVersions removes config versions by versions.
func DeleteConfigVersionsByVersions(ctx context.Context, versions []string) (int64, error) {
if len(versions) == 0 {
return 0, nil
}
conn := db.DB(ctx)
if conn == nil {
return 0, errors.New(errDatabaseNotInitialized)
}
result := conn.Where("version IN ?", versions).Delete(&model.ConfigVersion{})
return result.RowsAffected, result.Error
}
// ListEnabledProxyRoutes returns enabled proxy routes ordered by id asc.
func ListEnabledProxyRoutes(ctx context.Context) ([]*model.ProxyRoute, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var routes []*model.ProxyRoute
if err := conn.Where("enabled = ?", true).Order("id asc").Find(&routes).Error; err != nil {
return nil, err
}
return routes, nil
}
@@ -0,0 +1,65 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// ListDNSAccounts 列出全部 DNS 账号(授权信息不通过 JSON 暴露)。
func ListDNSAccounts(ctx context.Context) ([]model.DNSAccount, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var accounts []model.DNSAccount
if err := conn.Order("id desc").Find(&accounts).Error; err != nil {
return nil, err
}
return accounts, nil
}
// GetDNSAccountByID 按 ID 查询 DNS 账号。
func GetDNSAccountByID(ctx context.Context, id uint) (*model.DNSAccount, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var account model.DNSAccount
if err := conn.First(&account, id).Error; err != nil {
return nil, err
}
return &account, nil
}
// CreateDNSAccountRecord 创建 DNS 账号。
func CreateDNSAccountRecord(ctx context.Context, account *model.DNSAccount) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Create(account).Error
}
// SaveDNSAccount 保存 DNS 账号。
func SaveDNSAccount(ctx context.Context, account *model.DNSAccount) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Save(account).Error
}
// DeleteDNSAccountRecord 删除 DNS 账号。
func DeleteDNSAccountRecord(ctx context.Context, id uint) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Delete(&model.DNSAccount{}, id).Error
}
@@ -0,0 +1,167 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"time"
"gorm.io/gorm"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
const (
openFlareNodeStatusOnline = "online"
openFlareApplyResultSuccess = "success"
)
// ListOpenFlareNodes returns all nodes ordered by id desc.
func ListOpenFlareNodes(ctx context.Context) ([]model.OpenFlareNode, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var nodes []model.OpenFlareNode
if err := conn.Order("id desc").Find(&nodes).Error; err != nil {
return nil, err
}
return nodes, nil
}
// ListOpenFlareNodesByNodeIDs returns nodes matching the given node ids.
func ListOpenFlareNodesByNodeIDs(ctx context.Context, nodeIDs []string) ([]model.OpenFlareNode, error) {
if len(nodeIDs) == 0 {
return []model.OpenFlareNode{}, nil
}
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var nodes []model.OpenFlareNode
if err := conn.Where("node_id IN ?", nodeIDs).Find(&nodes).Error; err != nil {
return nil, err
}
return nodes, nil
}
// GetOpenFlareNodeByID returns a node by primary key.
func GetOpenFlareNodeByID(ctx context.Context, id uint) (*model.OpenFlareNode, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var node model.OpenFlareNode
if err := conn.First(&node, id).Error; err != nil {
return nil, err
}
return &node, nil
}
// GetOpenFlareNodeByNodeID returns a node by node_id.
func GetOpenFlareNodeByNodeID(ctx context.Context, nodeID string) (*model.OpenFlareNode, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var node model.OpenFlareNode
if err := conn.Where("node_id = ?", nodeID).First(&node).Error; err != nil {
return nil, err
}
return &node, nil
}
// GetOpenFlareNodeByAccessToken returns a node by access token.
func GetOpenFlareNodeByAccessToken(ctx context.Context, token string) (*model.OpenFlareNode, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var node model.OpenFlareNode
if err := conn.Where("access_token = ?", token).First(&node).Error; err != nil {
return nil, err
}
return &node, nil
}
// CreateOpenFlareNode inserts a new node.
func CreateOpenFlareNode(ctx context.Context, node *model.OpenFlareNode) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Create(node).Error
}
// SaveOpenFlareNode persists node changes.
func SaveOpenFlareNode(ctx context.Context, node *model.OpenFlareNode) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Save(node).Error
}
// UpdateOpenFlareNodeFields updates selected columns for a node.
func UpdateOpenFlareNodeFields(ctx context.Context, node *model.OpenFlareNode, fields ...string) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
if len(fields) == 0 {
return conn.Save(node).Error
}
return conn.Model(node).Select(fields).Updates(node).Error
}
// UpdateOpenFlareNodeColumns updates node columns from a map of column values.
// Empty maps are no-ops.
func UpdateOpenFlareNodeColumns(ctx context.Context, node *model.OpenFlareNode, changes map[string]any) error {
if node == nil || len(changes) == 0 {
return nil
}
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Model(node).Updates(changes).Error
}
// UpdateOpenFlareNodeFromApplyResult updates node status, last_seen, version and last_error after an apply report.
// When applyResult is "success", current_version is set and last_error is cleared; otherwise last_error is set to message.
func UpdateOpenFlareNodeFromApplyResult(ctx context.Context, nodeID, applyResult, version, message string, now time.Time) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return updateOpenFlareNodeFromApplyResultTx(conn, nodeID, applyResult, version, message, now)
}
func updateOpenFlareNodeFromApplyResultTx(tx *gorm.DB, nodeID, applyResult, version, message string, now time.Time) error {
record := &model.OpenFlareNode{}
if err := tx.Where("node_id = ?", nodeID).First(record).Error; err != nil {
return err
}
record.Status = openFlareNodeStatusOnline
lastSeen := now
record.LastSeenAt = &lastSeen
if applyResult == openFlareApplyResultSuccess {
record.CurrentVersion = version
record.LastError = ""
} else {
record.LastError = message
}
return tx.Model(record).Select("status", "last_seen_at", "current_version", "last_error").Updates(record).Error
}
// DeleteOpenFlareNode removes a node by primary key.
func DeleteOpenFlareNode(ctx context.Context, id uint) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Delete(&model.OpenFlareNode{}, id).Error
}
@@ -0,0 +1,644 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"Wavelet/openflare/plugins/server/kernel/model"
analyticsmodel "Wavelet/openflare/plugins/server/kernel/model/analytics"
analyticsrepo "Wavelet/openflare/plugins/server/kernel/repository/analytics"
"Wavelet/openflare/plugins/server/kernel/repository/logstore"
"Wavelet/pkg/logger"
db "Wavelet/plugins/infra/database"
)
const (
openFlareHealthEventStatusActive = "active"
openFlareHealthEventStatusResolved = "resolved"
openFlareHealthSeverityInfo = "info"
openFlareHealthSeverityWarning = "warning"
openFlareHealthSeverityCritical = "critical"
openFlareHealthEventMessageMaxLen = 4096
// logStoreNameClickHouse 与 logstore 内部 dbNameClickHouse 取值一致。
logStoreNameClickHouse = "clickhouse"
)
// OpenFlareHealthEventInput describes a desired active health event for reconciliation.
type OpenFlareHealthEventInput struct {
EventType string
Severity string
Message string
TriggeredAtUnix int64
Metadata map[string]string
}
func isMissingTableError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, gorm.ErrRecordNotFound) {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "no such table") ||
strings.Contains(msg, "doesn't exist") ||
strings.Contains(msg, "does not exist")
}
// InsertOpenFlareMetricSnapshot inserts a metric snapshot into ClickHouse.
func InsertOpenFlareMetricSnapshot(ctx context.Context, record *model.OpenFlareMetricSnapshot) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertMetricSnapshot(ctx, record)
}
// InsertOpenFlareEdgeHealth inserts an L2 edge health snapshot into ClickHouse.
func InsertOpenFlareEdgeHealth(ctx context.Context, record *model.OpenFlareEdgeHealth) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertEdgeHealth(ctx, record)
}
// InsertOpenFlareNodeObservationFrps inserts an FRPS observation into ClickHouse.
func InsertOpenFlareNodeObservationFrps(ctx context.Context, record *model.OpenFlareNodeObservationFrps) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertNodeObservationFrps(ctx, record)
}
// InsertOpenFlareNodeObservationFrpc inserts an FRPC observation into ClickHouse.
func InsertOpenFlareNodeObservationFrpc(ctx context.Context, record *model.OpenFlareNodeObservationFrpc) error {
s, err := logstore.Active(ctx)
if err != nil {
return err
}
return s.Observability.InsertNodeObservationFrpc(ctx, record)
}
// ListOpenFlareMetricSnapshotsSince returns metric snapshots since the given time.
func ListOpenFlareMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareMetricSnapshot, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListMetricSnapshots(ctx, nodeID, since, limit)
}
// ListOpenFlareLatestMetricSnapshotsSince returns the latest metric snapshot per node.
// The ClickHouse LIMIT 1 BY fast path is used only when ClickHouse is the ACTIVE log
// database; otherwise the request goes straight to the active log store (PG/SQLite),
// avoiding stale reads of the previous CH store after a migration.
func ListOpenFlareLatestMetricSnapshotsSince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricSnapshot, error) {
active, err := logstore.ActiveDatabase(ctx)
if err != nil {
logger.ErrorF(ctx, "failed to resolve active log database for latest metric snapshots: %v", err)
} else if active == logStoreNameClickHouse {
rows, chErr := analyticsrepo.ListLatestNodeMetricSnapshots(ctx, analyticsrepo.NodeObservabilityFilter{
NodeID: nodeID,
Since: since,
})
if chErr == nil {
return fromAnalyticsNodeMetricSnapshots(rows), nil
}
logger.ErrorF(ctx, "clickhouse fast-path ListLatestNodeMetricSnapshots failed: %v", chErr)
return nil, chErr
}
// Routes through the active log store (PG/SQLite active).
all, listErr := ListOpenFlareMetricSnapshotsSince(ctx, nodeID, since, 0)
if listErr != nil {
return nil, listErr
}
return openFlareLatestMetricSnapshots(all), nil
}
func openFlareLatestMetricSnapshots(snapshots []*model.OpenFlareMetricSnapshot) []*model.OpenFlareMetricSnapshot {
latestByNode := make(map[string]*model.OpenFlareMetricSnapshot, len(snapshots))
for _, snapshot := range snapshots {
if snapshot == nil || snapshot.NodeID == "" {
continue
}
if existing, ok := latestByNode[snapshot.NodeID]; ok && !snapshot.CapturedAt.After(existing.CapturedAt) {
continue
}
latestByNode[snapshot.NodeID] = snapshot
}
result := make([]*model.OpenFlareMetricSnapshot, 0, len(latestByNode))
for _, snapshot := range latestByNode {
result = append(result, snapshot)
}
return result
}
// fromAnalyticsNodeMetricSnapshots converts analytics rows back to the business model.
func fromAnalyticsNodeMetricSnapshots(rows []analyticsmodel.NodeMetricSnapshot) []*model.OpenFlareMetricSnapshot {
result := make([]*model.OpenFlareMetricSnapshot, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareMetricSnapshot{
ID: uint(row.ID),
NodeID: row.NodeID,
CapturedAt: row.CapturedAt,
CPUUsagePercent: row.CPUUsagePercent,
MemoryUsedBytes: row.MemoryUsedBytes,
MemoryTotalBytes: row.MemoryTotalBytes,
StorageUsedBytes: row.StorageUsedBytes,
StorageTotalBytes: row.StorageTotalBytes,
DiskReadBytes: row.DiskReadBytes,
DiskWriteBytes: row.DiskWriteBytes,
NetworkRxBytes: row.NetworkRxBytes,
NetworkTxBytes: row.NetworkTxBytes,
CreatedAt: row.CreatedAt,
}
}
return result
}
// ListOpenFlareTrafficHourlySince returns hourly traffic rollup rows since the given time.
// CH 读 of_access_log_hourly rollup;PG/SQLite 经 logstore 从 of_node_access_logs 实时聚合。
func ListOpenFlareTrafficHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareTrafficHourly, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.Observability.ListTrafficHourly(ctx, nodeID, since)
if err != nil {
return nil, err
}
result := make([]*model.OpenFlareTrafficHourly, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareTrafficHourly{
NodeID: row.NodeID,
Hour: row.Hour,
RequestCount: row.RequestCount,
ErrorCount: row.ErrorCount,
UniqueVisitorCount: row.UniqueVisitorCount,
}
}
return result, nil
}
// ListOpenFlareAccessLogHourlySince returns hourly access-log rollups since the given time,
// read through logstore's active backend (ClickHouse of_access_log_hourly rollup;
// PostgreSQL/SQLite real-time aggregation from of_node_access_logs).
func ListOpenFlareAccessLogHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareAccessLogHourly, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.Observability.ListAccessLogHourly(ctx, nodeID, since)
if err != nil {
return nil, err
}
result := make([]*model.OpenFlareAccessLogHourly, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareAccessLogHourly{
NodeID: row.NodeID,
Hour: row.Hour,
Host: row.Host,
RequestCount: row.RequestCount,
ErrorCount: row.ErrorCount,
BytesSent: row.BytesSent,
RequestLength: row.RequestLength,
}
}
return result, nil
}
// ListOpenFlareMetricHourlySince returns hourly metric aggregates since the given time.
func ListOpenFlareMetricHourlySince(ctx context.Context, nodeID string, since time.Time) ([]*model.OpenFlareMetricHourly, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
rows, err := s.Observability.ListMetricHourly(ctx, nodeID, since)
if err != nil {
return nil, err
}
result := make([]*model.OpenFlareMetricHourly, len(rows))
for index, row := range rows {
result[index] = &model.OpenFlareMetricHourly{
Hour: row.Hour,
AverageCPUUsagePercent: row.AverageCPUUsagePercent,
AverageMemoryUsagePercent: row.AverageMemoryUsagePercent,
NetworkRxBytes: row.NetworkRxBytes,
NetworkTxBytes: row.NetworkTxBytes,
DiskReadBytes: row.DiskReadBytes,
DiskWriteBytes: row.DiskWriteBytes,
ReportedNodes: row.ReportedNodes,
}
}
return result, nil
}
// ListOpenFlareActiveHealthEvents returns active health events across all nodes.
func ListOpenFlareActiveHealthEvents(ctx context.Context) ([]*model.OpenFlareHealthEvent, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var rows []*model.OpenFlareHealthEvent
if err := conn.Where("status = ?", "active").Order("last_triggered_at desc").Find(&rows).Error; err != nil {
if isMissingTableError(err) {
return []*model.OpenFlareHealthEvent{}, nil
}
return nil, err
}
return rows, nil
}
// ListOpenFlareHealthEvents returns health events for a node.
func ListOpenFlareHealthEvents(ctx context.Context, nodeID string, activeOnly bool, limit int) ([]*model.OpenFlareHealthEvent, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
query := conn.Model(&model.OpenFlareHealthEvent{}).Where("node_id = ?", nodeID).Order("last_triggered_at desc")
if activeOnly {
query = query.Where("status = ?", "active")
}
if limit > 0 {
query = query.Limit(limit)
}
var rows []*model.OpenFlareHealthEvent
if err := query.Find(&rows).Error; err != nil {
if isMissingTableError(err) {
return []*model.OpenFlareHealthEvent{}, nil
}
return nil, err
}
return rows, nil
}
// DeleteOpenFlareMetricSnapshotsBefore deletes metric snapshots captured before cutoff.
func DeleteOpenFlareMetricSnapshotsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteMetricSnapshotsBefore(ctx, cutoff)
}
// DeleteAllOpenFlareMetricSnapshots deletes all metric snapshots.
func DeleteAllOpenFlareMetricSnapshots(ctx context.Context) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllMetricSnapshots(ctx)
}
// DeleteOpenFlareEdgeHealthBefore deletes edge health rows captured before cutoff.
func DeleteOpenFlareEdgeHealthBefore(ctx context.Context, cutoff time.Time) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteEdgeHealthBefore(ctx, cutoff)
}
// DeleteAllOpenFlareEdgeHealth deletes all edge health snapshots.
func DeleteAllOpenFlareEdgeHealth(ctx context.Context) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllEdgeHealth(ctx)
}
// DeleteOpenFlareNodeObservationFrpsBefore deletes FRPS observations captured before cutoff.
func DeleteOpenFlareNodeObservationFrpsBefore(ctx context.Context, cutoff time.Time) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteNodeObservationFrpsBefore(ctx, cutoff)
}
// DeleteAllOpenFlareNodeObservationFrps deletes all FRPS observations.
func DeleteAllOpenFlareNodeObservationFrps(ctx context.Context) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllNodeObservationFrps(ctx)
}
// DeleteOpenFlareNodeObservationFrpcBefore deletes FRPC observations captured before cutoff.
func DeleteOpenFlareNodeObservationFrpcBefore(ctx context.Context, cutoff time.Time) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteNodeObservationFrpcBefore(ctx, cutoff)
}
// DeleteAllOpenFlareNodeObservationFrpc deletes all FRPC observations.
func DeleteAllOpenFlareNodeObservationFrpc(ctx context.Context) (int64, error) {
s, err := logstore.Active(ctx)
if err != nil {
return 0, err
}
return s.Observability.DeleteAllNodeObservationFrpc(ctx)
}
// DeleteOpenFlareHealthEventsByNodeID deletes all health events for a node.
func DeleteOpenFlareHealthEventsByNodeID(ctx context.Context, nodeID string) (int64, error) {
conn := db.DB(ctx)
if conn == nil {
return 0, errors.New(errDatabaseNotInitialized)
}
result := conn.Where("node_id = ?", nodeID).Delete(&model.OpenFlareHealthEvent{})
if result.Error != nil {
if isMissingTableError(result.Error) {
return 0, nil
}
return 0, result.Error
}
return result.RowsAffected, nil
}
// GetOpenFlareNodeSystemProfile returns the system profile for a node.
func GetOpenFlareNodeSystemProfile(ctx context.Context, nodeID string) (*model.OpenFlareNodeSystemProfile, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var profile model.OpenFlareNodeSystemProfile
if err := conn.Where("node_id = ?", nodeID).First(&profile).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) || isMissingTableError(err) {
return nil, gorm.ErrRecordNotFound
}
return nil, err
}
return &profile, nil
}
// UpsertOpenFlareNodeSystemProfile inserts or updates the latest system profile for a node.
func UpsertOpenFlareNodeSystemProfile(ctx context.Context, record *model.OpenFlareNodeSystemProfile) error {
if record == nil {
return nil
}
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return upsertOpenFlareNodeSystemProfileTx(conn, record)
}
func upsertOpenFlareNodeSystemProfileTx(tx *gorm.DB, record *model.OpenFlareNodeSystemProfile) error {
return tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "node_id"}},
DoUpdates: clause.AssignmentColumns([]string{
"hostname",
"os_name",
"os_version",
"kernel_version",
"architecture",
"cpu_model",
"cpu_cores",
"total_memory_bytes",
"total_disk_bytes",
"uptime_seconds",
"reported_at",
"updated_at",
}),
}).Create(record).Error
}
// ReconcileOpenFlareHealthEvents reconciles active health events for a node.
// Desired active events are created or updated; previously active types not present are resolved.
// When managedEventTypes is non-empty, only those event types are considered.
// Runs inside a transaction so multi-row create/update/resolve stays atomic.
func ReconcileOpenFlareHealthEvents(
ctx context.Context,
nodeID string,
events []OpenFlareHealthEventInput,
reportedAt time.Time,
managedEventTypes map[string]struct{},
) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Transaction(func(tx *gorm.DB) error {
return reconcileOpenFlareHealthEventsTx(tx, nodeID, events, reportedAt, managedEventTypes)
})
}
// PersistOpenFlareNodePGObservability upserts an optional system profile and optionally reconciles
// health events in a single transaction (Postgres-side heartbeat observability).
// When reconcileHealth is false, health events are left untouched.
func PersistOpenFlareNodePGObservability(
ctx context.Context,
profile *model.OpenFlareNodeSystemProfile,
nodeID string,
events []OpenFlareHealthEventInput,
reconcileHealth bool,
reportedAt time.Time,
managedEventTypes map[string]struct{},
) error {
if profile == nil && !reconcileHealth {
return nil
}
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Transaction(func(tx *gorm.DB) error {
if profile != nil {
if err := upsertOpenFlareNodeSystemProfileTx(tx, profile); err != nil {
return err
}
}
if reconcileHealth {
if err := reconcileOpenFlareHealthEventsTx(tx, nodeID, events, reportedAt, managedEventTypes); err != nil {
return err
}
}
return nil
})
}
func reconcileOpenFlareHealthEventsTx(
tx *gorm.DB,
nodeID string,
events []OpenFlareHealthEventInput,
reportedAt time.Time,
managedEventTypes map[string]struct{},
) error {
activeTypes := make(map[string]OpenFlareHealthEventInput, len(events))
for _, event := range events {
eventType := normalizeOpenFlareHealthEventType(event.EventType)
if eventType == "" {
continue
}
if len(managedEventTypes) > 0 {
if _, ok := managedEventTypes[eventType]; !ok {
continue
}
}
event.EventType = eventType
event.Severity = normalizeOpenFlareHealthSeverity(event.Severity)
if event.TriggeredAtUnix <= 0 {
event.TriggeredAtUnix = reportedAt.Unix()
}
activeTypes[eventType] = event
}
var activeEvents []*model.OpenFlareHealthEvent
query := tx.Where("node_id = ? AND status = ?", nodeID, openFlareHealthEventStatusActive)
if len(managedEventTypes) > 0 {
scopedTypes := make([]string, 0, len(managedEventTypes))
for eventType := range managedEventTypes {
eventType = normalizeOpenFlareHealthEventType(eventType)
if eventType != "" {
scopedTypes = append(scopedTypes, eventType)
}
}
if len(scopedTypes) == 0 {
return nil
}
query = query.Where("event_type IN ?", scopedTypes)
}
if err := query.Find(&activeEvents).Error; err != nil {
return err
}
activeByType := make(map[string]*model.OpenFlareHealthEvent, len(activeEvents))
for _, event := range activeEvents {
activeByType[event.EventType] = event
}
for eventType, event := range activeTypes {
triggeredAt := timeFromUnixSeconds(event.TriggeredAtUnix, reportedAt)
if existing, ok := activeByType[eventType]; ok {
existing.Severity = event.Severity
existing.Message = normalizeOpenFlareHealthEventMessage(event.Message)
existing.LastTriggeredAt = triggeredAt
existing.ReportedAt = reportedAt
existing.MetadataJSON = marshalOpenFlareHealthMetadata(event.Metadata)
existing.ResolvedAt = nil
if err := tx.Save(existing).Error; err != nil {
return err
}
continue
}
record := &model.OpenFlareHealthEvent{
NodeID: nodeID,
EventType: eventType,
Severity: event.Severity,
Status: openFlareHealthEventStatusActive,
Message: normalizeOpenFlareHealthEventMessage(event.Message),
FirstTriggeredAt: triggeredAt,
LastTriggeredAt: triggeredAt,
ReportedAt: reportedAt,
MetadataJSON: marshalOpenFlareHealthMetadata(event.Metadata),
}
if err := tx.Create(record).Error; err != nil {
return err
}
}
for _, existing := range activeEvents {
if _, ok := activeTypes[existing.EventType]; ok {
continue
}
resolvedAt := reportedAt
existing.Status = openFlareHealthEventStatusResolved
existing.ReportedAt = reportedAt
existing.ResolvedAt = &resolvedAt
if err := tx.Save(existing).Error; err != nil {
return err
}
}
return nil
}
func normalizeOpenFlareHealthEventType(eventType string) string {
eventType = strings.TrimSpace(strings.ToLower(eventType))
eventType = strings.ReplaceAll(eventType, " ", "_")
return eventType
}
func normalizeOpenFlareHealthSeverity(severity string) string {
switch strings.ToLower(strings.TrimSpace(severity)) {
case openFlareHealthSeverityCritical:
return openFlareHealthSeverityCritical
case openFlareHealthSeverityInfo:
return openFlareHealthSeverityInfo
default:
return openFlareHealthSeverityWarning
}
}
func normalizeOpenFlareHealthEventMessage(message string) string {
if openFlareHealthEventMessageMaxLen <= 0 {
return ""
}
runes := []rune(strings.TrimSpace(message))
if len(runes) <= openFlareHealthEventMessageMaxLen {
return string(runes)
}
return string(runes[:openFlareHealthEventMessageMaxLen])
}
func timeFromUnixSeconds(unixSeconds int64, fallback time.Time) time.Time {
if unixSeconds <= 0 {
return fallback
}
return time.Unix(unixSeconds, 0).UTC()
}
func marshalOpenFlareHealthMetadata(value map[string]string) string {
if value == nil {
return ""
}
raw, err := json.Marshal(value)
if err != nil {
return ""
}
return string(raw)
}
// ListOpenFlareEdgeHealth returns L2 edge health snapshots.
func ListOpenFlareEdgeHealth(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareEdgeHealth, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListEdgeHealth(ctx, nodeID, since, limit)
}
// ListOpenFlareNodeObservationFrpc returns frpc observations.
func ListOpenFlareNodeObservationFrpc(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrpc, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListNodeObservationFrpc(ctx, nodeID, since, limit)
}
// ListOpenFlareNodeObservationFrps returns frps observations.
func ListOpenFlareNodeObservationFrps(ctx context.Context, nodeID string, since time.Time, limit int) ([]*model.OpenFlareNodeObservationFrps, error) {
s, err := logstore.Active(ctx)
if err != nil {
return nil, err
}
return s.Observability.ListNodeObservationFrps(ctx, nodeID, since, limit)
}
@@ -0,0 +1,127 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"gorm.io/gorm"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// WithOriginTx runs fn inside a database transaction for origin multi-step work.
func WithOriginTx(ctx context.Context, fn func(tx *gorm.DB) error) error {
return db.DB(ctx).Transaction(fn)
}
// HasProxyRoutesTable 判断代理规则表是否已迁移。
func HasProxyRoutesTable(ctx context.Context) bool {
return db.DB(ctx).Migrator().HasTable(&model.OriginProxyRoute{})
}
// ListOrigins 列出全部源站。
func ListOrigins(ctx context.Context) ([]model.Origin, error) {
var origins []model.Origin
if err := db.DB(ctx).Order("id desc").Find(&origins).Error; err != nil {
return nil, err
}
return origins, nil
}
// GetOriginByID 按 ID 查询源站。
func GetOriginByID(ctx context.Context, id uint) (*model.Origin, error) {
var origin model.Origin
if err := db.DB(ctx).First(&origin, id).Error; err != nil {
return nil, err
}
return &origin, nil
}
// GetOriginByAddress 按地址查询源站。
func GetOriginByAddress(ctx context.Context, address string) (*model.Origin, error) {
var origin model.Origin
if err := db.DB(ctx).Where("address = ?", address).First(&origin).Error; err != nil {
return nil, err
}
return &origin, nil
}
// CreateOriginRecord 创建源站。
func CreateOriginRecord(ctx context.Context, origin *model.Origin) error {
return db.DB(ctx).Create(origin).Error
}
// SaveOrigin 保存源站。
func SaveOrigin(ctx context.Context, origin *model.Origin) error {
return SaveOriginTx(db.DB(ctx), origin)
}
// SaveOriginTx saves an origin within an existing transaction.
func SaveOriginTx(tx *gorm.DB, origin *model.Origin) error {
return tx.Save(origin).Error
}
// DeleteOriginRecord 删除源站。
func DeleteOriginRecord(ctx context.Context, id uint) error {
return db.DB(ctx).Delete(&model.Origin{}, id).Error
}
// ListOriginRouteCounts 统计各源站关联的代理规则数量。
func ListOriginRouteCounts(ctx context.Context) ([]model.OriginRouteCount, error) {
if !HasProxyRoutesTable(ctx) {
return nil, nil
}
result := make([]model.OriginRouteCount, 0)
err := db.DB(ctx).Model(&model.OriginProxyRoute{}).
Select("origin_id, COUNT(*) AS route_count").
Where("origin_id IS NOT NULL").
Group("origin_id").
Scan(&result).Error
return result, err
}
// ListProxyRoutesByOriginID 列出源站关联的代理规则。
func ListProxyRoutesByOriginID(ctx context.Context, originID uint) ([]model.OriginProxyRoute, error) {
if !HasProxyRoutesTable(ctx) {
return nil, nil
}
var routes []model.OriginProxyRoute
if err := db.DB(ctx).Where("origin_id = ?", originID).Order("id desc").Find(&routes).Error; err != nil {
return nil, err
}
return routes, nil
}
// ListProxyRoutesByOriginIDAscTx lists origin-linked proxy routes ordered by id asc within a transaction.
func ListProxyRoutesByOriginIDAscTx(tx *gorm.DB, originID uint) ([]model.OriginProxyRoute, error) {
var routes []model.OriginProxyRoute
if err := tx.Where("origin_id = ?", originID).Order("id asc").Find(&routes).Error; err != nil {
return nil, err
}
return routes, nil
}
// UpdateProxyRouteOriginAddressTx updates a proxy route's origin_url and upstreams within a transaction.
func UpdateProxyRouteOriginAddressTx(tx *gorm.DB, routeID uint, originURL, upstreamsJSON string) error {
return tx.Model(&model.OriginProxyRoute{}).
Where("id = ?", routeID).
Updates(map[string]any{
"origin_url": originURL,
"upstreams": upstreamsJSON,
}).Error
}
// CountProxyRoutesByOriginID 统计源站关联的代理规则数量。
func CountProxyRoutesByOriginID(ctx context.Context, originID uint) (int64, error) {
if !HasProxyRoutesTable(ctx) {
return 0, nil
}
var count int64
if err := db.DB(ctx).Model(&model.OriginProxyRoute{}).Where("origin_id = ?", originID).Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
@@ -0,0 +1,96 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// HasPagesProjectsTable 判断 Pages 项目表是否已迁移。
func HasPagesProjectsTable(ctx context.Context) bool {
return db.DB(ctx).Migrator().HasTable(&model.PagesProject{})
}
// ListPagesProjects 列出全部 Pages 项目。
func ListPagesProjects(ctx context.Context) ([]model.PagesProject, error) {
var projects []model.PagesProject
if err := db.DB(ctx).Order("id desc").Find(&projects).Error; err != nil {
return nil, err
}
return projects, nil
}
// GetPagesProjectByID 按 ID 查询 Pages 项目。
func GetPagesProjectByID(ctx context.Context, id uint) (*model.PagesProject, error) {
var project model.PagesProject
if err := db.DB(ctx).First(&project, id).Error; err != nil {
return nil, err
}
return &project, nil
}
// GetPagesProjectBySlug 按 slug 查询 Pages 项目。
func GetPagesProjectBySlug(ctx context.Context, slug string) (*model.PagesProject, error) {
var project model.PagesProject
if err := db.DB(ctx).Where("slug = ?", slug).First(&project).Error; err != nil {
return nil, err
}
return &project, nil
}
// CreatePagesProjectRecord 创建 Pages 项目。
func CreatePagesProjectRecord(ctx context.Context, project *model.PagesProject) error {
return db.DB(ctx).Create(project).Error
}
// ListPagesDeployments 列出项目的全部部署。
func ListPagesDeployments(ctx context.Context, projectID uint) ([]model.PagesDeployment, error) {
var deployments []model.PagesDeployment
if err := db.DB(ctx).Where("project_id = ?", projectID).Order("id desc").Find(&deployments).Error; err != nil {
return nil, err
}
return deployments, nil
}
// GetPagesDeploymentByID 按 ID 查询 Pages 部署。
func GetPagesDeploymentByID(ctx context.Context, id uint) (*model.PagesDeployment, error) {
var deployment model.PagesDeployment
if err := db.DB(ctx).First(&deployment, id).Error; err != nil {
return nil, err
}
return &deployment, nil
}
// ListPagesDeploymentFiles 列出部署文件清单。
func ListPagesDeploymentFiles(ctx context.Context, deploymentID uint) ([]model.PagesDeploymentFile, error) {
var files []model.PagesDeploymentFile
if err := db.DB(ctx).Where("deployment_id = ?", deploymentID).Order("path asc").Find(&files).Error; err != nil {
return nil, err
}
return files, nil
}
// CountPagesDeploymentsByProjectID 统计项目部署数量。
func CountPagesDeploymentsByProjectID(ctx context.Context, projectID uint) (int64, error) {
var count int64
if err := db.DB(ctx).Model(&model.PagesDeployment{}).Where("project_id = ?", projectID).Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
// CountProxyRoutesByPagesProjectID 统计引用 Pages 项目的代理规则数量。
func CountProxyRoutesByPagesProjectID(ctx context.Context, projectID uint) (int64, error) {
if !HasProxyRoutesTable(ctx) {
return 0, nil
}
var count int64
if err := db.DB(ctx).Model(&model.ProxyRoute{}).Where("pages_project_id = ?", projectID).Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
@@ -0,0 +1,58 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// ListPagesOrphanUploadCandidates returns at most 100 unreferenced, isolated
// Pages V2 upload records. Callers must still lock and recheck every condition
// before deleting a candidate.
func ListPagesOrphanUploadCandidates(
ctx context.Context,
input model.PagesOrphanUploadCandidateQuery,
) ([]model.Upload, error) {
if input.SystemUserID == 0 || input.UploadType == "" || input.Marker == "" || input.CreatedBefore.IsZero() {
return nil, errors.New("invalid pages orphan upload candidate query")
}
markerPredicate, err := pagesOrphanMarkerPredicate(db.DB(ctx).Name())
if err != nil {
return nil, err
}
deploymentTable := (model.PagesDeployment{}).TableName()
uploadTable := (model.Upload{}).TableName()
var candidates []model.Upload
err = db.DB(ctx).
Model(&model.Upload{}).
Where(uploadTable+".status = ?", model.UploadStatusUsed).
Where(uploadTable+".user_id = ?", input.SystemUserID).
Where(uploadTable+".type = ?", input.UploadType).
Where(uploadTable+".created_at < ?", input.CreatedBefore).
Where(markerPredicate, input.Marker).
Where("NOT EXISTS (SELECT 1 FROM " + deploymentTable + " WHERE " + deploymentTable + ".upload_id = " + uploadTable + ".id)").
Order(uploadTable + ".id ASC").
Limit(model.PagesOrphanUploadCandidateLimit).
Find(&candidates).Error
if err != nil {
return nil, err
}
return candidates, nil
}
func pagesOrphanMarkerPredicate(dialect string) (string, error) {
switch dialect {
case "postgres":
return model.PagesOrphanMarkerPredicatePostgres, nil
case "sqlite":
return model.PagesOrphanMarkerPredicateSQLite, nil
default:
return "", errors.New("unsupported database dialect for Pages orphan cleanup")
}
}
@@ -0,0 +1,196 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"strings"
"testing"
"time"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func TestPagesOrphanMarkerPredicate(t *testing.T) {
tests := []struct {
name string
dialect string
want string
wantErr bool
}{
{
name: "postgres jsonb path",
dialect: "postgres",
want: "metadata #>> '{extra,pages_ingest_marker}'",
},
{
name: "sqlite guarded json extract",
dialect: "sqlite",
want: "CASE WHEN json_valid(w_uploads.metadata) THEN json_extract",
},
{
name: "unknown dialect rejected",
dialect: "mysql",
wantErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := pagesOrphanMarkerPredicate(test.dialect)
if gotErr := err != nil; gotErr != test.wantErr {
t.Fatalf("pagesOrphanMarkerPredicate(%q) error = %v, want error presence = %t", test.dialect, err, test.wantErr)
}
if test.want != "" && !strings.Contains(got, test.want) {
t.Errorf("pagesOrphanMarkerPredicate(%q) = %q, want substring %q", test.dialect, got, test.want)
}
})
}
}
func TestListPagesOrphanUploadCandidatesFiltersAndLimits(t *testing.T) {
ctx := context.Background()
gormDB := setupPagesCleanupModelTestDB(t)
cutoff := time.Now().UTC().Add(-2 * time.Hour)
old := cutoff.Add(-time.Minute)
marker := model.UploadMetadata{Extra: map[string]any{
"pages_ingest_marker": "pages_deployment_v2",
"pages_project_id": "1",
}}
valid := make([]model.Upload, 0, model.PagesOrphanUploadCandidateLimit+1)
for index := 0; index < model.PagesOrphanUploadCandidateLimit+1; index++ {
valid = append(valid, pagesCleanupModelUpload(uint64(index+100), 999, "openflare_pages_deployment", model.UploadStatusUsed, old, marker))
}
if err := gormDB.Create(&valid).Error; err != nil {
t.Fatalf("create valid candidates error = %v, want nil", err)
}
referenced := pagesCleanupModelUpload(1, 999, "openflare_pages_deployment", model.UploadStatusUsed, old, marker)
wrongOwner := pagesCleanupModelUpload(2, 1000, "openflare_pages_deployment", model.UploadStatusUsed, old, marker)
wrongType := pagesCleanupModelUpload(3, 999, "generic", model.UploadStatusUsed, old, marker)
wrongStatus := pagesCleanupModelUpload(4, 999, "openflare_pages_deployment", model.UploadStatusPending, old, marker)
fresh := pagesCleanupModelUpload(5, 999, "openflare_pages_deployment", model.UploadStatusUsed, cutoff, marker)
wrongMarker := pagesCleanupModelUpload(6, 999, "openflare_pages_deployment", model.UploadStatusUsed, old, model.UploadMetadata{Extra: map[string]any{
"pages_ingest_marker": "pages_deployment_v1",
"pages_project_id": "1",
}})
for _, upload := range []model.Upload{referenced, wrongOwner, wrongType, wrongStatus, fresh, wrongMarker} {
if err := gormDB.Create(&upload).Error; err != nil {
t.Fatalf("create filtered upload %d error = %v, want nil", upload.ID, err)
}
}
if err := gormDB.Create(&model.PagesDeployment{
ProjectID: 1,
DeploymentNumber: 1,
Checksum: "referenced",
Status: model.PagesDeploymentStatusUploaded,
UploadID: referenced.ID,
}).Error; err != nil {
t.Fatalf("create referenced deployment error = %v, want nil", err)
}
invalidJSON := pagesCleanupModelUpload(7, 999, "openflare_pages_deployment", model.UploadStatusUsed, old, marker)
if err := gormDB.Create(&invalidJSON).Error; err != nil {
t.Fatalf("create invalid JSON upload error = %v, want nil", err)
}
if err := gormDB.Table((model.Upload{}).TableName()).Where("id = ?", invalidJSON.ID).
UpdateColumn("metadata", "{invalid").Error; err != nil {
t.Fatalf("corrupt upload metadata error = %v, want nil", err)
}
got, err := ListPagesOrphanUploadCandidates(ctx, model.PagesOrphanUploadCandidateQuery{
SystemUserID: 999,
UploadType: "openflare_pages_deployment",
Marker: "pages_deployment_v2",
CreatedBefore: cutoff,
})
if err != nil {
t.Fatalf("ListPagesOrphanUploadCandidates() error = %v, want nil", err)
}
if len(got) != model.PagesOrphanUploadCandidateLimit {
t.Fatalf("ListPagesOrphanUploadCandidates() count = %d, want %d", len(got), model.PagesOrphanUploadCandidateLimit)
}
for index, candidate := range got {
wantID := uint64(index + 100)
if candidate.ID != wantID {
t.Errorf("ListPagesOrphanUploadCandidates()[%d].ID = %d, want %d", index, candidate.ID, wantID)
}
}
}
func TestListPagesOrphanUploadCandidatesSkipsInvalidSQLiteJSON(t *testing.T) {
ctx := context.Background()
gormDB := setupPagesCleanupModelTestDB(t)
cutoff := time.Now().UTC().Add(-2 * time.Hour)
upload := pagesCleanupModelUpload(1, 999, "openflare_pages_deployment", model.UploadStatusUsed, cutoff.Add(-time.Minute), model.UploadMetadata{})
if err := gormDB.Create(&upload).Error; err != nil {
t.Fatalf("create invalid JSON candidate error = %v, want nil", err)
}
if err := gormDB.Table((model.Upload{}).TableName()).Where("id = ?", upload.ID).
UpdateColumn("metadata", "{invalid").Error; err != nil {
t.Fatalf("corrupt upload metadata error = %v, want nil", err)
}
got, err := ListPagesOrphanUploadCandidates(ctx, model.PagesOrphanUploadCandidateQuery{
SystemUserID: 999,
UploadType: "openflare_pages_deployment",
Marker: "pages_deployment_v2",
CreatedBefore: cutoff,
})
if err != nil {
t.Fatalf("ListPagesOrphanUploadCandidates(invalid JSON) error = %v, want nil", err)
}
if len(got) != 0 {
t.Errorf("ListPagesOrphanUploadCandidates(invalid JSON) count = %d, want 0", len(got))
}
}
func setupPagesCleanupModelTestDB(t *testing.T) *gorm.DB {
t.Helper()
gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
if err != nil {
t.Fatalf("open Pages cleanup model test database error = %v, want nil", err)
}
if err := gormDB.AutoMigrate(&model.Upload{}, &model.PagesDeployment{}); err != nil {
t.Fatalf("migrate Pages cleanup model test database error = %v, want nil", err)
}
db.SetDB(gormDB)
t.Cleanup(func() { db.SetDB(nil) })
return gormDB
}
func pagesCleanupModelUpload(
id uint64,
userID uint64,
uploadType string,
status model.UploadStatus,
createdAt time.Time,
metadata model.UploadMetadata,
) model.Upload {
return model.Upload{
ID: id,
UserID: userID,
FileName: "site.zip",
FilePath: "pages/site.zip",
FileSize: 10,
MimeType: "application/zip",
Extension: "zip",
Hash: "checksum",
Type: uploadType,
Status: status,
AccessMode: 0,
Metadata: metadata,
CreatedAt: createdAt,
UpdatedAt: createdAt,
}
}
@@ -0,0 +1,417 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
const pagesRowLockStrength = "UPDATE"
// WithPagesTx runs fn inside a database transaction for Pages multi-step work.
func WithPagesTx(ctx context.Context, fn func(tx *gorm.DB) error) error {
return db.DB(ctx).Transaction(fn)
}
// GetPagesProjectSourceByID loads a project source by primary key.
func GetPagesProjectSourceByID(ctx context.Context, id uint) (*model.PagesProjectSource, error) {
var source model.PagesProjectSource
if err := db.DB(ctx).Where("id = ?", id).First(&source).Error; err != nil {
return nil, err
}
return &source, nil
}
// GetPagesProjectSourceByProjectID loads the unique source for a project.
func GetPagesProjectSourceByProjectID(ctx context.Context, projectID uint) (*model.PagesProjectSource, error) {
var source model.PagesProjectSource
if err := db.DB(ctx).Where("project_id = ?", projectID).First(&source).Error; err != nil {
return nil, err
}
return &source, nil
}
// GetPagesProjectSourceByIDAndConfigVersion loads a source matching both id and config version.
func GetPagesProjectSourceByIDAndConfigVersion(
ctx context.Context,
id uint,
configVersion int,
) (*model.PagesProjectSource, error) {
var source model.PagesProjectSource
if err := db.DB(ctx).Where("id = ? AND config_version = ?", id, configVersion).First(&source).Error; err != nil {
return nil, err
}
return &source, nil
}
// GetPagesProjectSourceRuntimeBySourceID loads runtime for a source.
func GetPagesProjectSourceRuntimeBySourceID(
ctx context.Context,
sourceID uint,
) (*model.PagesProjectSourceRuntime, error) {
var runtime model.PagesProjectSourceRuntime
if err := db.DB(ctx).Where("source_id = ?", sourceID).First(&runtime).Error; err != nil {
return nil, err
}
return &runtime, nil
}
// GetPagesProjectSourceAndRuntimeByProjectID loads source and its runtime for a project.
func GetPagesProjectSourceAndRuntimeByProjectID(
ctx context.Context,
projectID uint,
) (*model.PagesProjectSource, *model.PagesProjectSourceRuntime, error) {
source, err := GetPagesProjectSourceByProjectID(ctx, projectID)
if err != nil {
return nil, nil, err
}
runtime, err := GetPagesProjectSourceRuntimeBySourceID(ctx, source.ID)
if err != nil {
return nil, nil, err
}
return source, runtime, nil
}
// CreatePagesProjectSourceTx creates a source row inside an existing transaction.
func CreatePagesProjectSourceTx(tx *gorm.DB, source *model.PagesProjectSource) error {
return tx.Create(source).Error
}
// CreatePagesProjectSourceRuntimeTx creates a runtime row inside an existing transaction.
func CreatePagesProjectSourceRuntimeTx(tx *gorm.DB, runtime *model.PagesProjectSourceRuntime) error {
return tx.Create(runtime).Error
}
// UpdatePagesProjectSourceTx applies partial updates to a source inside a transaction.
func UpdatePagesProjectSourceTx(tx *gorm.DB, source *model.PagesProjectSource, updates map[string]any) error {
if len(updates) == 0 {
return nil
}
return tx.Model(source).Updates(updates).Error
}
// UpdatePagesProjectSourceRuntimeTx applies partial updates to a runtime inside a transaction.
func UpdatePagesProjectSourceRuntimeTx(
tx *gorm.DB,
runtime *model.PagesProjectSourceRuntime,
updates map[string]any,
) error {
if len(updates) == 0 {
return nil
}
return tx.Model(runtime).Updates(updates).Error
}
// UpdatePagesProjectSourceRuntimeFieldTx updates a single column on a runtime row.
func UpdatePagesProjectSourceRuntimeFieldTx(
tx *gorm.DB,
runtime *model.PagesProjectSourceRuntime,
column string,
value any,
) error {
return tx.Model(runtime).Update(column, value).Error
}
// DeletePagesProjectSourceRuntimeBySourceIDTx deletes runtime rows for a source.
func DeletePagesProjectSourceRuntimeBySourceIDTx(tx *gorm.DB, sourceID uint) error {
return tx.Where("source_id = ?", sourceID).Delete(&model.PagesProjectSourceRuntime{}).Error
}
// DeletePagesProjectSourceTx deletes a source row inside a transaction.
func DeletePagesProjectSourceTx(tx *gorm.DB, source *model.PagesProjectSource) error {
return tx.Delete(source).Error
}
// LockPagesProjectByIDTx locks a project row for update.
func LockPagesProjectByIDTx(tx *gorm.DB, id uint) (*model.PagesProject, error) {
var project model.PagesProject
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).First(&project, id).Error; err != nil {
return nil, err
}
return &project, nil
}
// LockPagesProjectSourceByProjectIDTx locks the source for a project.
func LockPagesProjectSourceByProjectIDTx(tx *gorm.DB, projectID uint) (*model.PagesProjectSource, error) {
var source model.PagesProjectSource
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("project_id = ?", projectID).
First(&source).Error; err != nil {
return nil, err
}
return &source, nil
}
// LockPagesProjectSourceByIDTx locks a source by id.
func LockPagesProjectSourceByIDTx(tx *gorm.DB, sourceID uint) (*model.PagesProjectSource, error) {
var source model.PagesProjectSource
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id = ?", sourceID).
First(&source).Error; err != nil {
return nil, err
}
return &source, nil
}
// LockPagesProjectSourceByIDAndProjectIDTx locks a source matching both identifiers.
func LockPagesProjectSourceByIDAndProjectIDTx(
tx *gorm.DB,
sourceID uint,
projectID uint,
) (*model.PagesProjectSource, error) {
var source model.PagesProjectSource
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("id = ? AND project_id = ?", sourceID, projectID).
First(&source).Error; err != nil {
return nil, err
}
return &source, nil
}
// LockPagesProjectSourceRuntimeBySourceIDTx locks runtime for a source.
func LockPagesProjectSourceRuntimeBySourceIDTx(
tx *gorm.DB,
sourceID uint,
) (*model.PagesProjectSourceRuntime, error) {
var runtime model.PagesProjectSourceRuntime
if err := tx.Clauses(clause.Locking{Strength: pagesRowLockStrength}).
Where("source_id = ?", sourceID).
First(&runtime).Error; err != nil {
return nil, err
}
return &runtime, nil
}
// GetPagesProjectSourceByIDTx loads a source by id without locking.
func GetPagesProjectSourceByIDTx(tx *gorm.DB, sourceID uint) (*model.PagesProjectSource, error) {
var source model.PagesProjectSource
if err := tx.Where("id = ?", sourceID).First(&source).Error; err != nil {
return nil, err
}
return &source, nil
}
// TryAcquirePagesSourceRuntimeLease conditionally claims an idle/expired lease when config matches.
func TryAcquirePagesSourceRuntimeLease(
ctx context.Context,
sourceID uint,
expectedConfigVersion int,
now time.Time,
updates map[string]any,
) (int64, error) {
result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", sourceID).
Where("lease_expires_at IS NULL OR lease_expires_at <= ?", now).
Where(
"EXISTS (SELECT 1 FROM of_pages_project_sources source WHERE source.id = ? AND source.config_version = ?)",
sourceID,
expectedConfigVersion,
).
Updates(updates)
return result.RowsAffected, result.Error
}
// RenewPagesSourceRuntimeLease extends an active lease held by the given token.
func RenewPagesSourceRuntimeLease(
ctx context.Context,
sourceID uint,
token string,
now time.Time,
expiresAt time.Time,
) (int64, error) {
result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?", sourceID, token, now).
Updates(map[string]any{"lease_expires_at": expiresAt})
return result.RowsAffected, result.Error
}
// UpdatePagesSourceRuntimeByActiveLease updates runtime while the caller still owns the lease.
func UpdatePagesSourceRuntimeByActiveLease(
ctx context.Context,
sourceID uint,
token string,
now time.Time,
updates map[string]any,
) (int64, error) {
return UpdatePagesSourceRuntimeByActiveLeaseTx(db.DB(ctx), sourceID, token, now, updates)
}
// UpdatePagesSourceRuntimeByActiveLeaseTx updates runtime under an active lease inside a transaction.
func UpdatePagesSourceRuntimeByActiveLeaseTx(
tx *gorm.DB,
sourceID uint,
token string,
now time.Time,
updates map[string]any,
) (int64, error) {
result := tx.Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ? AND lease_token = ? AND lease_expires_at > ?", sourceID, token, now).
Updates(updates)
return result.RowsAffected, result.Error
}
// RecoverExpiredPagesSourceRuntimeLease clears one exact expired lease owner.
func RecoverExpiredPagesSourceRuntimeLease(
ctx context.Context,
sourceID uint,
token string,
expiresAt time.Time,
status string,
now time.Time,
updates map[string]any,
) (int64, error) {
result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", sourceID).
Where("lease_token = ?", token).
Where("lease_expires_at = ? AND lease_expires_at <= ?", expiresAt, now).
Where("sync_status = ?", status).
Updates(updates)
return result.RowsAffected, result.Error
}
// MarkPagesSourceInitialCheckDispatchFailed marks runtime failed when config still matches and lease is free.
func MarkPagesSourceInitialCheckDispatchFailed(
ctx context.Context,
sourceID uint,
configVersion int,
now time.Time,
updates map[string]any,
) (int64, error) {
result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", sourceID).
Where("lease_expires_at IS NULL OR lease_expires_at <= ?", now).
Where(
"EXISTS (SELECT 1 FROM of_pages_project_sources source WHERE source.id = ? AND source.config_version = ?)",
sourceID,
configVersion,
).
Updates(updates)
return result.RowsAffected, result.Error
}
// RecordPagesSourceAutoDispatchFailure records a failed auto-sync dispatch while status still matches.
func RecordPagesSourceAutoDispatchFailure(
ctx context.Context,
sourceID uint,
configVersion int,
sourceType string,
releaseSelector string,
revision string,
updateAvailableStatus string,
now time.Time,
updates map[string]any,
) (int64, error) {
result := db.DB(ctx).Model(&model.PagesProjectSourceRuntime{}).
Where("source_id = ?", sourceID).
Where("sync_status = ? AND last_seen_revision = ?", updateAvailableStatus, revision).
Where("lease_expires_at IS NULL OR lease_expires_at <= ?", now).
Where(`EXISTS (
SELECT 1 FROM of_pages_project_sources AS source
WHERE source.id = ? AND source.config_version = ?
AND source.source_type = ? AND source.release_selector = ?
AND source.auto_update_enabled = ?
)`,
sourceID,
configVersion,
sourceType,
releaseSelector,
true,
).
Updates(updates)
return result.RowsAffected, result.Error
}
// ListExpiredPagesSourceLeaseCandidates returns expired checking/syncing leases for recovery.
func ListExpiredPagesSourceLeaseCandidates(
ctx context.Context,
now time.Time,
syncStatuses []string,
) ([]model.PagesExpiredSourceLeaseCandidate, error) {
var candidates []model.PagesExpiredSourceLeaseCandidate
err := db.DB(ctx).
Table("of_pages_project_source_runtime AS runtime").
Select(`runtime.source_id, runtime.lease_token, runtime.lease_expires_at,
runtime.sync_status, source.source_type, source.release_selector`).
Joins("JOIN of_pages_project_sources AS source ON source.id = runtime.source_id").
Where("runtime.lease_token <> ''").
Where("runtime.lease_expires_at IS NOT NULL AND runtime.lease_expires_at <= ?", now).
Where("runtime.sync_status IN ?", syncStatuses).
Order("runtime.source_id ASC").
Scan(&candidates).Error
if err != nil {
return nil, err
}
return candidates, nil
}
// CountDueGitHubPagesSourceChecks counts due latest GitHub sources.
func CountDueGitHubPagesSourceChecks(
ctx context.Context,
now time.Time,
sourceType string,
releaseSelector string,
) (int64, error) {
var count int64
err := dueGitHubPagesSourceQuery(ctx, now, sourceType, releaseSelector).Count(&count).Error
return count, err
}
// ListDueGitHubPagesSourceChecks lists a batch of due latest GitHub sources in stable order.
func ListDueGitHubPagesSourceChecks(
ctx context.Context,
now time.Time,
sourceType string,
releaseSelector string,
limit int,
) ([]model.PagesDueGitHubSourceCandidate, error) {
var candidates []model.PagesDueGitHubSourceCandidate
err := dueGitHubPagesSourceQuery(ctx, now, sourceType, releaseSelector).
Select("source.id AS source_id, source.config_version").
Order("runtime.next_check_at ASC").
Order("source.id ASC").
Limit(limit).
Scan(&candidates).Error
if err != nil {
return nil, err
}
return candidates, nil
}
func dueGitHubPagesSourceQuery(
ctx context.Context,
now time.Time,
sourceType string,
releaseSelector string,
) *gorm.DB {
return db.DB(ctx).
Table("of_pages_project_source_runtime AS runtime").
Joins("JOIN of_pages_project_sources AS source ON source.id = runtime.source_id").
Where("source.source_type = ?", sourceType).
Where("source.release_selector = ?", releaseSelector).
Where("runtime.next_check_at IS NOT NULL AND runtime.next_check_at <= ?", now)
}
// GetPagesDeploymentBySourceRevision loads a deployment by project source identity and revision.
func GetPagesDeploymentBySourceRevision(
ctx context.Context,
projectID uint,
sourceIdentity string,
revision string,
) (*model.PagesDeployment, error) {
var deployment model.PagesDeployment
err := db.DB(ctx).
Where("project_id = ? AND source_identity = ? AND source_revision = ?", projectID, sourceIdentity, revision).
First(&deployment).Error
if err != nil {
return nil, err
}
return &deployment, nil
}
@@ -0,0 +1,154 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// Zone domain binding sentinel errors (shared by proxy_route and zone binding helpers).
var (
// ErrZoneDomainBoundToAnotherRoute is returned when a domain is already bound to a different route.
ErrZoneDomainBoundToAnotherRoute = errors.New("zone domain is already bound to another proxy route")
// ErrZoneDomainNotFound is returned when one or more requested domain IDs do not exist.
ErrZoneDomainNotFound = errors.New("one or more zone domains do not exist")
)
// WithProxyRouteTx runs fn inside a database transaction for proxy-route multi-step work.
func WithProxyRouteTx(ctx context.Context, fn func(tx *gorm.DB) error) error {
return db.DB(ctx).Transaction(fn)
}
// ListProxyRoutes 列出全部代理规则。
func ListProxyRoutes(ctx context.Context) ([]*model.ProxyRoute, error) {
var routes []*model.ProxyRoute
if err := db.DB(ctx).Order("id desc").Find(&routes).Error; err != nil {
return nil, err
}
return routes, nil
}
// GetProxyRouteByID 按 ID 查询代理规则。
func GetProxyRouteByID(ctx context.Context, id uint) (*model.ProxyRoute, error) {
var route model.ProxyRoute
if err := db.DB(ctx).First(&route, id).Error; err != nil {
return nil, err
}
return &route, nil
}
// CreateProxyRouteRecord 创建代理规则。
func CreateProxyRouteRecord(ctx context.Context, route *model.ProxyRoute) error {
return CreateProxyRouteRecordTx(db.DB(ctx), route)
}
// CreateProxyRouteRecordTx creates a proxy route within an existing transaction.
func CreateProxyRouteRecordTx(tx *gorm.DB, route *model.ProxyRoute) error {
return tx.Create(route).Error
}
// UpdateProxyRouteRecord 更新代理规则。
func UpdateProxyRouteRecord(ctx context.Context, route *model.ProxyRoute) error {
return UpdateProxyRouteRecordTx(db.DB(ctx), route)
}
// UpdateProxyRouteRecordTx updates a proxy route within an existing transaction.
func UpdateProxyRouteRecordTx(tx *gorm.DB, route *model.ProxyRoute) error {
return tx.Model(&model.ProxyRoute{}).Where("id = ?", route.ID).Updates(proxyRouteUpdateMap(route)).Error
}
func proxyRouteUpdateMap(route *model.ProxyRoute) map[string]any {
return map[string]any{
"site_name": route.SiteName,
"origin_id": route.OriginID,
"origin_url": route.OriginURL,
"origin_host": route.OriginHost,
"upstreams": route.Upstreams,
colEnabled: route.Enabled,
"enable_https": route.EnableHTTPS,
"redirect_http": route.RedirectHTTP,
"limit_conn_per_server": route.LimitConnPerServer,
"limit_conn_per_ip": route.LimitConnPerIP,
"limit_rate": route.LimitRate,
"limit_req_per_ip": route.LimitReqPerIP,
"cache_enabled": route.CacheEnabled,
"cache_policy": route.CachePolicy,
"cache_rules": route.CacheRules,
"custom_headers": route.CustomHeaders,
"basic_auth_enabled": route.BasicAuthEnabled,
"basic_auth_username": route.BasicAuthUsername,
"basic_auth_password": route.BasicAuthPassword,
"upstream_type": route.UpstreamType,
"tunnel_node_id": route.TunnelNodeID,
"tunnel_target_addr": route.TunnelTargetAddr,
"tunnel_target_protocol": route.TunnelTargetProtocol,
"pages_project_id": route.PagesProjectID,
}
}
// DeleteProxyRouteRecord 删除代理规则。
func DeleteProxyRouteRecord(ctx context.Context, id uint) error {
return DeleteProxyRouteRecordTx(db.DB(ctx), id)
}
// DeleteProxyRouteRecordTx deletes a proxy route within an existing transaction.
func DeleteProxyRouteRecordTx(tx *gorm.DB, id uint) error {
return tx.Delete(&model.ProxyRoute{}, id).Error
}
// ClearZoneDomainProxyRouteBindingsTx unbinds every zone domain from a proxy route.
func ClearZoneDomainProxyRouteBindingsTx(tx *gorm.DB, routeID uint) error {
return tx.Model(&model.ZoneDomain{}).Where("proxy_route_id = ?", routeID).Update("proxy_route_id", nil).Error
}
// ReplaceZoneDomainRouteBindingsTx replaces every ZoneDomain binding for a proxy route
// inside the caller's transaction (with row locks on requested domains).
func ReplaceZoneDomainRouteBindingsTx(tx *gorm.DB, routeID uint, domainIDs []uint) error {
var requested []model.ZoneDomain
if len(domainIDs) > 0 {
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
Where("id IN ?", domainIDs).
Find(&requested).Error; err != nil {
return err
}
if len(requested) != len(uniqueZoneDomainIDs(domainIDs)) {
return ErrZoneDomainNotFound
}
for _, domain := range requested {
if domain.ProxyRouteID != nil && *domain.ProxyRouteID != routeID {
return ErrZoneDomainBoundToAnotherRoute
}
}
}
current := tx.Model(&model.ZoneDomain{}).Where("proxy_route_id = ?", routeID)
if len(domainIDs) > 0 {
current = current.Where("id NOT IN ?", domainIDs)
}
if err := current.Update("proxy_route_id", nil).Error; err != nil {
return err
}
if len(domainIDs) == 0 {
return nil
}
return tx.Model(&model.ZoneDomain{}).Where("id IN ?", domainIDs).Update("proxy_route_id", routeID).Error
}
// DeleteProxyRouteAndUnbind clears domain bindings then deletes the proxy route in one transaction.
func DeleteProxyRouteAndUnbind(ctx context.Context, id uint) error {
return WithProxyRouteTx(ctx, func(tx *gorm.DB) error {
if err := ClearZoneDomainProxyRouteBindingsTx(tx, id); err != nil {
return err
}
return DeleteProxyRouteRecordTx(tx, id)
})
}
@@ -0,0 +1,95 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// HasTLSProxyRoutesTable 判断代理规则表是否已迁移。
func HasTLSProxyRoutesTable(ctx context.Context) bool {
return db.DB(ctx).Migrator().HasTable(&model.TLSProxyRouteRef{})
}
// ListTLSCertificates 列出全部证书(不含 PEM 敏感字段的 JSON 暴露由 struct tag 控制)。
func ListTLSCertificates(ctx context.Context) ([]model.TLSCertificate, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var certificates []model.TLSCertificate
if err := conn.Order("id desc").Find(&certificates).Error; err != nil {
return nil, err
}
return certificates, nil
}
// GetTLSCertificateByID 按 ID 查询证书。
func GetTLSCertificateByID(ctx context.Context, id uint) (*model.TLSCertificate, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
var certificate model.TLSCertificate
if err := conn.First(&certificate, id).Error; err != nil {
return nil, err
}
return &certificate, nil
}
// CreateTLSCertificateRecord 创建证书记录。
func CreateTLSCertificateRecord(ctx context.Context, certificate *model.TLSCertificate) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Create(certificate).Error
}
// SaveTLSCertificate 保存证书记录。
func SaveTLSCertificate(ctx context.Context, certificate *model.TLSCertificate) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Save(certificate).Error
}
// DeleteTLSCertificateRecord 删除证书记录。
func DeleteTLSCertificateRecord(ctx context.Context, id uint) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Delete(&model.TLSCertificate{}, id).Error
}
// CountTLSCertificatesByDNSAccountID 统计引用指定 DNS 账号的证书数量。
func CountTLSCertificatesByDNSAccountID(ctx context.Context, dnsAccountID uint) (int64, error) {
conn := db.DB(ctx)
if conn == nil {
return 0, errors.New(errDatabaseNotInitialized)
}
var count int64
if err := conn.Model(&model.TLSCertificate{}).Where("dns_account_id = ?", dnsAccountID).Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
// ListTLSProxyRouteRefs 列出代理规则证书引用字段。
func ListTLSProxyRouteRefs(ctx context.Context) ([]model.TLSProxyRouteRef, error) {
if !HasTLSProxyRoutesTable(ctx) {
return nil, nil
}
var routes []model.TLSProxyRouteRef
if err := db.DB(ctx).Order("id asc").Find(&routes).Error; err != nil {
return nil, err
}
return routes, nil
}
@@ -0,0 +1,357 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"time"
"gorm.io/gorm"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
func wafDB(ctx context.Context) (*gorm.DB, error) {
conn := db.DB(ctx)
if conn == nil {
return nil, errors.New(errDatabaseNotInitialized)
}
return conn, nil
}
// ListOpenFlareWAFRuleGroups returns all rule groups.
func ListOpenFlareWAFRuleGroups(ctx context.Context) ([]*model.OpenFlareWAFRuleGroup, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var groups []*model.OpenFlareWAFRuleGroup
if err = conn.Order("is_global desc").Order("id asc").Find(&groups).Error; err != nil {
return nil, err
}
return groups, nil
}
// GetOpenFlareWAFRuleGroupByID returns a rule group by id.
func GetOpenFlareWAFRuleGroupByID(ctx context.Context, id uint) (*model.OpenFlareWAFRuleGroup, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var group model.OpenFlareWAFRuleGroup
if err = conn.First(&group, id).Error; err != nil {
return nil, err
}
return &group, nil
}
// GetGlobalOpenFlareWAFRuleGroup returns the global rule group if present.
func GetGlobalOpenFlareWAFRuleGroup(ctx context.Context) (*model.OpenFlareWAFRuleGroup, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var group model.OpenFlareWAFRuleGroup
if err = conn.Where("is_global = ?", true).Order("id asc").First(&group).Error; err != nil {
return nil, err
}
return &group, nil
}
// CreateOpenFlareWAFRuleGroup inserts a rule group.
func CreateOpenFlareWAFRuleGroup(ctx context.Context, group *model.OpenFlareWAFRuleGroup) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Create(group).Error
}
// UpdateOpenFlareWAFRuleGroup updates mutable rule group fields.
func UpdateOpenFlareWAFRuleGroup(ctx context.Context, group *model.OpenFlareWAFRuleGroup) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Model(&model.OpenFlareWAFRuleGroup{}).Where("id = ?", group.ID).Updates(map[string]any{
"name": group.Name,
colEnabled: group.Enabled,
"is_global": group.IsGlobal,
}).Error
}
// UpdateOpenFlareWAFRuleGraph atomically replaces a graph when revision is current.
func UpdateOpenFlareWAFRuleGraph(ctx context.Context, id uint, revision uint64, graph string) (uint64, error) {
conn, err := wafDB(ctx)
if err != nil {
return 0, err
}
result := conn.Model(&model.OpenFlareWAFRuleGroup{}).
Where("id = ? AND revision = ?", id, revision).
Updates(map[string]any{"graph": graph, "revision": gorm.Expr("revision + 1")})
if result.Error != nil {
return 0, result.Error
}
if result.RowsAffected != 1 {
return 0, model.ErrWAFRuleRevisionConflict
}
return revision + 1, nil
}
// DeleteOpenFlareWAFRuleGroup removes a rule group.
func DeleteOpenFlareWAFRuleGroup(ctx context.Context, id uint) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Delete(&model.OpenFlareWAFRuleGroup{}, id).Error
}
// ListOpenFlareWAFIPGroups returns all IP groups.
func ListOpenFlareWAFIPGroups(ctx context.Context) ([]*model.OpenFlareWAFIPGroup, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var groups []*model.OpenFlareWAFIPGroup
if err = conn.Order("type asc").Order("id asc").Find(&groups).Error; err != nil {
return nil, err
}
return groups, nil
}
// ListOpenFlareWAFIPGroupsByIDs returns IP groups for the given ids.
func ListOpenFlareWAFIPGroupsByIDs(ctx context.Context, ids []uint) ([]*model.OpenFlareWAFIPGroup, error) {
if len(ids) == 0 {
return []*model.OpenFlareWAFIPGroup{}, nil
}
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var groups []*model.OpenFlareWAFIPGroup
if err = conn.Where("id IN ?", ids).Find(&groups).Error; err != nil {
return nil, err
}
return groups, nil
}
// GetOpenFlareWAFIPGroupByID returns an IP group by id.
func GetOpenFlareWAFIPGroupByID(ctx context.Context, id uint) (*model.OpenFlareWAFIPGroup, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var group model.OpenFlareWAFIPGroup
if err = conn.First(&group, id).Error; err != nil {
return nil, err
}
return &group, nil
}
// CreateOpenFlareWAFIPGroup inserts an IP group.
func CreateOpenFlareWAFIPGroup(ctx context.Context, group *model.OpenFlareWAFIPGroup) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Create(group).Error
}
// UpdateOpenFlareWAFIPGroup updates mutable IP group fields.
func UpdateOpenFlareWAFIPGroup(ctx context.Context, group *model.OpenFlareWAFIPGroup) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Model(&model.OpenFlareWAFIPGroup{}).Where("id = ?", group.ID).Updates(map[string]any{
"name": group.Name,
"type": group.Type,
colEnabled: group.Enabled,
"ip_list": group.IPList,
"auto_config": group.AutoConfig,
"ext_ips": group.ExtIPs,
"subscription_url": group.SubscriptionURL,
"subscription_format": group.SubscriptionFormat,
"subscription_mapping_rule": group.SubscriptionMappingRule,
"sync_interval_minutes": group.SyncIntervalMinutes,
"next_sync_at": group.NextSyncAt,
"last_sync_status": group.LastSyncStatus,
"last_sync_message": group.LastSyncMessage,
}).Error
}
// ListDueOpenFlareWAFIPGroups returns enabled automatic/subscription groups due for sync.
func ListDueOpenFlareWAFIPGroups(ctx context.Context, now time.Time) ([]*model.OpenFlareWAFIPGroup, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var groups []*model.OpenFlareWAFIPGroup
err = conn.Where(
"enabled = ? AND (type = ? OR (type = ? AND subscription_url <> '')) AND (next_sync_at IS NULL OR next_sync_at <= ?)",
true, "automatic", "subscription", now,
).Order("id asc").Find(&groups).Error
return groups, err
}
// UpdateOpenFlareWAFIPGroupSyncResult persists IP group sync outcome fields.
func UpdateOpenFlareWAFIPGroupSyncResult(ctx context.Context, group *model.OpenFlareWAFIPGroup) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Model(&model.OpenFlareWAFIPGroup{}).Where("id = ?", group.ID).Updates(map[string]any{
"ip_list": group.IPList,
"ext_ips": group.ExtIPs,
"last_synced_at": group.LastSyncedAt,
"next_sync_at": group.NextSyncAt,
"last_sync_status": group.LastSyncStatus,
"last_sync_message": group.LastSyncMessage,
"subscription_format": group.SubscriptionFormat,
}).Error
}
// DeleteOpenFlareWAFIPGroup removes an IP group.
func DeleteOpenFlareWAFIPGroup(ctx context.Context, id uint) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Delete(&model.OpenFlareWAFIPGroup{}, id).Error
}
// ListOpenFlareWAFRuleGroupBindings returns all bindings.
func ListOpenFlareWAFRuleGroupBindings(ctx context.Context) ([]model.OpenFlareWAFRuleGroupBinding, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var bindings []model.OpenFlareWAFRuleGroupBinding
if err = conn.Order("sequence asc").Order("id asc").Find(&bindings).Error; err != nil {
return nil, err
}
return bindings, nil
}
// ListOpenFlareWAFRuleGroupBindingsByRouteID returns bindings for a proxy route.
func ListOpenFlareWAFRuleGroupBindingsByRouteID(ctx context.Context, routeID uint) ([]model.OpenFlareWAFRuleGroupBinding, error) {
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var bindings []model.OpenFlareWAFRuleGroupBinding
if err = conn.Where("proxy_route_id = ?", routeID).Order("sequence asc").Order("id asc").Find(&bindings).Error; err != nil {
return nil, err
}
return bindings, nil
}
func syncWAFBindingIDSequence(tx *gorm.DB) error {
if tx == nil || tx.Dialector.Name() != "postgres" { //nolint:staticcheck // QF1008: keep explicit Dialector field access
return nil
}
return tx.Exec(`
SELECT setval(
pg_get_serial_sequence('of_waf_rule_group_bindings', 'id'),
GREATEST(COALESCE((SELECT MAX(id) FROM of_waf_rule_group_bindings), 0), 1),
COALESCE((SELECT MAX(id) FROM of_waf_rule_group_bindings), 0) > 0
)
`).Error
}
func insertOpenFlareWAFRuleGroupBindings(tx *gorm.DB, bindings []model.OpenFlareWAFRuleGroupBinding) error {
if len(bindings) == 0 {
return nil
}
if err := syncWAFBindingIDSequence(tx); err != nil {
return err
}
return tx.Create(&bindings).Error
}
// ReplaceOpenFlareWAFRuleGroupBindings replaces bindings for a rule group.
func ReplaceOpenFlareWAFRuleGroupBindings(ctx context.Context, groupID uint, routeIDs []uint) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Transaction(func(tx *gorm.DB) error {
if err = tx.Where("rule_group_id = ?", groupID).Delete(&model.OpenFlareWAFRuleGroupBinding{}).Error; err != nil {
return err
}
bindings := make([]model.OpenFlareWAFRuleGroupBinding, 0, len(routeIDs))
for index, routeID := range routeIDs {
bindings = append(bindings, model.OpenFlareWAFRuleGroupBinding{
RuleGroupID: groupID,
ProxyRouteID: routeID,
Sequence: index,
})
}
return insertOpenFlareWAFRuleGroupBindings(tx, bindings)
})
}
// ReplaceOpenFlareWAFSiteRuleGroupBindings replaces bindings for a proxy route.
func ReplaceOpenFlareWAFSiteRuleGroupBindings(ctx context.Context, routeID uint, groupIDs []uint) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Transaction(func(tx *gorm.DB) error {
if err = tx.Where("proxy_route_id = ?", routeID).Delete(&model.OpenFlareWAFRuleGroupBinding{}).Error; err != nil {
return err
}
bindings := make([]model.OpenFlareWAFRuleGroupBinding, 0, len(groupIDs))
for index, groupID := range groupIDs {
bindings = append(bindings, model.OpenFlareWAFRuleGroupBinding{
RuleGroupID: groupID,
ProxyRouteID: routeID,
Sequence: index,
})
}
return insertOpenFlareWAFRuleGroupBindings(tx, bindings)
})
}
// DeleteOpenFlareWAFRuleGroupBindingsByGroupID removes bindings for a rule group.
func DeleteOpenFlareWAFRuleGroupBindingsByGroupID(ctx context.Context, groupID uint) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Where("rule_group_id = ?", groupID).Delete(&model.OpenFlareWAFRuleGroupBinding{}).Error
}
// DeleteOpenFlareWAFRuleGroupWithBindings removes a rule group and its bindings.
func DeleteOpenFlareWAFRuleGroupWithBindings(ctx context.Context, groupID uint) error {
conn, err := wafDB(ctx)
if err != nil {
return err
}
return conn.Transaction(func(tx *gorm.DB) error {
if err = tx.Where("rule_group_id = ?", groupID).Delete(&model.OpenFlareWAFRuleGroupBinding{}).Error; err != nil {
return err
}
return tx.Delete(&model.OpenFlareWAFRuleGroup{}, groupID).Error
})
}
// GetOpenFlareProxyRouteByID returns a proxy route by id when the table exists.
func GetOpenFlareProxyRouteByID(ctx context.Context, id uint) (*model.OriginProxyRoute, error) {
if !HasProxyRoutesTable(ctx) {
return nil, gorm.ErrRecordNotFound
}
conn, err := wafDB(ctx)
if err != nil {
return nil, err
}
var route model.OriginProxyRoute
if err = conn.First(&route, id).Error; err != nil {
return nil, err
}
return &route, nil
}
@@ -0,0 +1,57 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"testing"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func setupWAFBindingsTestDB(t *testing.T) func() {
t.Helper()
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, sqliteDB.AutoMigrate(&model.OpenFlareWAFRuleGroupBinding{}))
db.SetDB(sqliteDB)
return func() {
db.SetDB(nil)
}
}
func TestReplaceOpenFlareWAFRuleGroupBindingsAfterExplicitHighID(t *testing.T) {
cleanup := setupWAFBindingsTestDB(t)
defer cleanup()
ctx := context.Background()
conn := db.DB(ctx)
require.NotNil(t, conn)
require.NoError(t, conn.Create(&model.OpenFlareWAFRuleGroupBinding{
ID: 50,
RuleGroupID: 1,
ProxyRouteID: 1,
}).Error)
require.NoError(t, ReplaceOpenFlareWAFRuleGroupBindings(ctx, 2, []uint{2, 3}))
var bindings []model.OpenFlareWAFRuleGroupBinding
require.NoError(t, conn.Where("rule_group_id = ?", 2).Order("proxy_route_id asc").Find(&bindings).Error)
require.Len(t, bindings, 2)
assert.Equal(t, uint(2), bindings[0].ProxyRouteID)
assert.Equal(t, uint(3), bindings[1].ProxyRouteID)
assert.Greater(t, bindings[0].ID, uint(50))
assert.Greater(t, bindings[1].ID, bindings[0].ID)
}
@@ -0,0 +1,63 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"testing"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
const defaultWAFRuleGraph = `{"schema_version":1,"nodes":[{"id":"start","type":"start","position":{"x":0,"y":0},"config":{}},{"id":"allow","type":"allow","position":{"x":320,"y":0},"config":{}}],"edges":[{"id":"start-allow","source":"start","source_handle":"next","target":"allow"}]}`
func TestOpenFlareWAFGraphOptimisticUpdate(t *testing.T) {
conn, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, conn.AutoMigrate(&model.OpenFlareWAFRuleGroup{}))
db.SetDB(conn)
t.Cleanup(func() { db.SetDB(nil) })
group := model.OpenFlareWAFRuleGroup{Name: "rule", Graph: defaultWAFRuleGraph, Revision: 1}
require.NoError(t, conn.Create(&group).Error)
nextRevision, err := UpdateOpenFlareWAFRuleGraph(context.Background(), group.ID, 1, `{"schema_version":1}`)
require.NoError(t, err)
assert.Equal(t, uint64(2), nextRevision)
_, err = UpdateOpenFlareWAFRuleGraph(context.Background(), group.ID, 1, defaultWAFRuleGraph)
assert.ErrorIs(t, err, model.ErrWAFRuleRevisionConflict)
}
func TestReplaceOpenFlareWAFRuleGroupBindingsPreservesInputOrder(t *testing.T) {
cleanup := setupWAFBindingsTestDB(t)
defer cleanup()
ctx := context.Background()
require.NoError(t, ReplaceOpenFlareWAFSiteRuleGroupBindings(ctx, 7, []uint{30, 10, 20}))
bindings, err := ListOpenFlareWAFRuleGroupBindingsByRouteID(ctx, 7)
require.NoError(t, err)
require.Len(t, bindings, 3)
assert.Equal(t, []uint{30, 10, 20}, []uint{bindings[0].RuleGroupID, bindings[1].RuleGroupID, bindings[2].RuleGroupID})
assert.Equal(t, []int{0, 1, 2}, []int{bindings[0].Sequence, bindings[1].Sequence, bindings[2].Sequence})
}
func TestLegacyWAFColumnsRemoved(t *testing.T) {
conn, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, conn.AutoMigrate(&model.OpenFlareWAFRuleGroup{}))
legacy := []string{"block_status_code", "block_response_body", "ip_whitelist", "ip_blacklist", "ip_whitelist_groups", "ip_blacklist_groups", "country_whitelist", "country_blacklist", "region_whitelist", "region_blacklist", "pow_enabled", "pow_config"}
for _, column := range legacy {
if conn.Migrator().HasColumn(&model.OpenFlareWAFRuleGroup{}, column) {
t.Fatalf("legacy WAF column %s still exists", column)
}
}
}
@@ -0,0 +1,175 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"gorm.io/gorm"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
)
// ListZones returns all zones ordered by domain ascending.
func ListZones(ctx context.Context) ([]model.Zone, error) {
var zones []model.Zone
if err := db.DB(ctx).Order("domain asc").Find(&zones).Error; err != nil {
return nil, err
}
return zones, nil
}
// GetZoneByID returns a zone by primary key.
func GetZoneByID(ctx context.Context, id uint) (*model.Zone, error) {
var zone model.Zone
if err := db.DB(ctx).First(&zone, id).Error; err != nil {
return nil, err
}
return &zone, nil
}
// CreateZone creates a zone record.
func CreateZone(ctx context.Context, zone *model.Zone) error {
return db.DB(ctx).Create(zone).Error
}
// SaveZone persists zone updates.
func SaveZone(ctx context.Context, zone *model.Zone) error {
return db.DB(ctx).Save(zone).Error
}
// DeleteZone deletes a zone by primary key.
func DeleteZone(ctx context.Context, id uint) error {
return db.DB(ctx).Delete(&model.Zone{}, id).Error
}
// ListZoneDomainCounts returns per-zone domain counts for list cards.
func ListZoneDomainCounts(ctx context.Context) ([]model.ZoneDomainCount, error) {
var rows []model.ZoneDomainCount
if err := db.DB(ctx).Model(&model.ZoneDomain{}).
Select("zone_id, count(*) as count").
Group("zone_id").
Scan(&rows).Error; err != nil {
return nil, err
}
return rows, nil
}
// ListZoneDomainsByZoneID returns domains under a zone ordered by domain ascending.
func ListZoneDomainsByZoneID(ctx context.Context, zoneID uint) ([]model.ZoneDomain, error) {
var domains []model.ZoneDomain
if err := db.DB(ctx).Where("zone_id = ?", zoneID).Order("domain asc").Find(&domains).Error; err != nil {
return nil, err
}
return domains, nil
}
// CountZoneDomainsByZoneID counts domains under a zone.
func CountZoneDomainsByZoneID(ctx context.Context, zoneID uint) (int64, error) {
var count int64
if err := db.DB(ctx).Model(&model.ZoneDomain{}).Where("zone_id = ?", zoneID).Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}
// GetZoneDomainByZoneAndID returns a domain scoped to a zone.
func GetZoneDomainByZoneAndID(ctx context.Context, zoneID, id uint) (*model.ZoneDomain, error) {
var item model.ZoneDomain
if err := db.DB(ctx).Where("id = ? AND zone_id = ?", id, zoneID).First(&item).Error; err != nil {
return nil, err
}
return &item, nil
}
// CreateZoneDomain creates a zone domain record.
func CreateZoneDomain(ctx context.Context, domain *model.ZoneDomain) error {
return db.DB(ctx).Create(domain).Error
}
// SaveZoneDomain persists zone domain updates.
func SaveZoneDomain(ctx context.Context, domain *model.ZoneDomain) error {
return db.DB(ctx).Save(domain).Error
}
// DeleteZoneDomain deletes a zone domain record.
func DeleteZoneDomain(ctx context.Context, domain *model.ZoneDomain) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Transaction(func(tx *gorm.DB) error {
if err := tx.Where("zone_domain_id = ?", domain.ID).Delete(&model.CFPointingMember{}).Error; err != nil {
return err
}
return tx.Delete(domain).Error
})
}
// ListZoneDomainsByRouteID returns the domains bound to a proxy route.
func ListZoneDomainsByRouteID(ctx context.Context, routeID uint) ([]model.ZoneDomain, error) {
var domains []model.ZoneDomain
if err := db.DB(ctx).Where("proxy_route_id = ?", routeID).Order("id asc").Find(&domains).Error; err != nil {
return nil, err
}
return domains, nil
}
// ListZoneDomainsByIDs returns explicit domains in the requested ID order.
func ListZoneDomainsByIDs(ctx context.Context, domainIDs []uint) ([]model.ZoneDomain, error) {
if len(domainIDs) == 0 {
return []model.ZoneDomain{}, nil
}
var domains []model.ZoneDomain
if err := db.DB(ctx).Where("id IN ?", domainIDs).Find(&domains).Error; err != nil {
return nil, err
}
byID := make(map[uint]model.ZoneDomain, len(domains))
for _, domain := range domains {
byID[domain.ID] = domain
}
ordered := make([]model.ZoneDomain, 0, len(domainIDs))
for _, id := range domainIDs {
domain, ok := byID[id]
if !ok {
return nil, errors.New("one or more zone domains do not exist")
}
ordered = append(ordered, domain)
}
return ordered, nil
}
// CountZoneDomainsByCertificateID reports whether a certificate is assigned to a model.Zone domain.
func CountZoneDomainsByCertificateID(ctx context.Context, certificateID uint) (int64, error) {
var count int64
err := db.DB(ctx).Model(&model.ZoneDomain{}).Where("cert_id = ?", certificateID).Count(&count).Error
return count, err
}
// ReplaceZoneDomainRouteBindings replaces every model.ZoneDomain binding for a proxy route.
func ReplaceZoneDomainRouteBindings(ctx context.Context, routeID uint, domainIDs []uint) error {
conn := db.DB(ctx)
if conn == nil {
return errors.New(errDatabaseNotInitialized)
}
return conn.Transaction(func(tx *gorm.DB) error {
return ReplaceZoneDomainRouteBindingsTx(tx, routeID, domainIDs)
})
}
func uniqueZoneDomainIDs(domainIDs []uint) []uint {
seen := make(map[uint]struct{}, len(domainIDs))
ids := make([]uint, 0, len(domainIDs))
for _, id := range domainIDs {
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
return ids
}
@@ -0,0 +1,91 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"testing"
"Wavelet/openflare/plugins/server/kernel/model"
db "Wavelet/plugins/infra/database"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func setupZoneTestDB(t *testing.T) *gorm.DB {
t.Helper()
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
require.NoError(t, err)
require.NoError(t, sqliteDB.AutoMigrate(&model.Zone{}, &model.ZoneDomain{}))
db.SetDB(sqliteDB)
t.Cleanup(func() { db.SetDB(nil) })
return sqliteDB
}
func TestReplaceZoneDomainRouteBindingsRejectsForeignDomain(t *testing.T) {
conn := setupZoneTestDB(t)
ctx := context.Background()
zone := model.Zone{Domain: "example.com"}
require.NoError(t, conn.Create(&zone).Error)
foreignRouteID := uint(11)
domain := model.ZoneDomain{
ZoneID: zone.ID,
ProxyRouteID: &foreignRouteID,
Domain: "api.example.com",
}
require.NoError(t, conn.Create(&domain).Error)
err := ReplaceZoneDomainRouteBindings(ctx, 12, []uint{domain.ID})
require.Error(t, err)
var got model.ZoneDomain
require.NoError(t, conn.First(&got, domain.ID).Error)
require.Equal(t, &foreignRouteID, got.ProxyRouteID)
}
func TestReplaceZoneDomainRouteBindingsReplacesCurrentRouteBindings(t *testing.T) {
conn := setupZoneTestDB(t)
ctx := context.Background()
zone := model.Zone{Domain: "example.com"}
require.NoError(t, conn.Create(&zone).Error)
routeID := uint(21)
boundDomain := model.ZoneDomain{ZoneID: zone.ID, ProxyRouteID: &routeID, Domain: "old.example.com"}
requestedDomain := model.ZoneDomain{ZoneID: zone.ID, Domain: "new.example.com"}
require.NoError(t, conn.Create(&boundDomain).Error)
require.NoError(t, conn.Create(&requestedDomain).Error)
require.NoError(t, ReplaceZoneDomainRouteBindings(ctx, routeID, []uint{requestedDomain.ID}))
var domains []model.ZoneDomain
require.NoError(t, conn.Order("id asc").Find(&domains).Error)
require.Len(t, domains, 2)
require.Nil(t, domains[0].ProxyRouteID)
require.Equal(t, &routeID, domains[1].ProxyRouteID)
}
func TestListZoneDomainsByRouteID(t *testing.T) {
conn := setupZoneTestDB(t)
ctx := context.Background()
zone := model.Zone{Domain: "example.com"}
require.NoError(t, conn.Create(&zone).Error)
routeID := uint(31)
boundDomain := model.ZoneDomain{ZoneID: zone.ID, ProxyRouteID: &routeID, Domain: "api.example.com"}
unboundDomain := model.ZoneDomain{ZoneID: zone.ID, Domain: "www.example.com"}
require.NoError(t, conn.Create(&boundDomain).Error)
require.NoError(t, conn.Create(&unboundDomain).Error)
domains, err := ListZoneDomainsByRouteID(ctx, routeID)
require.NoError(t, err)
require.Len(t, domains, 1)
require.Equal(t, boundDomain.ID, domains[0].ID)
}
@@ -0,0 +1,42 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"sync"
"Wavelet/core/contracts"
)
var (
svcMu sync.RWMutex
authSvc contracts.AuthService
userSvc contracts.UserService
)
// SetAuthService injects the platform AuthService used by GetActiveAuthSources.
func SetAuthService(s contracts.AuthService) {
svcMu.Lock()
defer svcMu.Unlock()
authSvc = s
}
// SetUserService injects the platform UserService used by GetSystemUser.
func SetUserService(s contracts.UserService) {
svcMu.Lock()
defer svcMu.Unlock()
userSvc = s
}
func currentAuthService() contracts.AuthService {
svcMu.RLock()
defer svcMu.RUnlock()
return authSvc
}
func currentUserService() contracts.UserService {
svcMu.RLock()
defer svcMu.RUnlock()
return userSvc
}
@@ -0,0 +1,117 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"Wavelet/openflare/plugins/server/kernel/model"
adminrepo "Wavelet/plugins/domain/admin/repository"
db "Wavelet/plugins/infra/database"
)
const configTypeSystem = "system"
// ensureAdminStore points OF config access at Wavelet's admin repository so
// reads hit the same cache that SaveOrUpdateSystemConfig invalidates.
func ensureAdminStore(ctx context.Context) error {
if conn := db.DB(ctx); conn != nil {
adminrepo.SetDBService(db.NewService(conn))
}
if adminrepo.GetDB(ctx) == nil {
return errors.New(errDatabaseNotInitialized)
}
return nil
}
// GetSystemConfigByKey loads a config row by key through the admin store cache.
func GetSystemConfigByKey(ctx context.Context, key string) (model.SystemConfig, error) {
if err := ensureAdminStore(ctx); err != nil {
return model.SystemConfig{}, err
}
return adminrepo.GetSystemConfigByKey(ctx, key)
}
// ListSystemConfigsByKeys loads multiple config keys through the admin store cache.
func ListSystemConfigsByKeys(ctx context.Context, keys []string) (map[string]model.SystemConfig, error) {
if err := ensureAdminStore(ctx); err != nil {
return nil, err
}
return adminrepo.ListSystemConfigsByKeys(ctx, keys)
}
// ListVisibleSystemConfigs returns visibility=1 configs from the admin store cache.
func ListVisibleSystemConfigs(ctx context.Context) ([]model.SystemConfig, error) {
if err := ensureAdminStore(ctx); err != nil {
return nil, err
}
return adminrepo.ListVisibleSystemConfigs(ctx)
}
// GetIntByKey queries config and converts to int.
func GetIntByKey(ctx context.Context, key string) (int, error) {
if err := ensureAdminStore(ctx); err != nil {
return 0, err
}
return adminrepo.GetIntByKey(ctx, key)
}
// GetBoolByKey queries config and converts to bool.
func GetBoolByKey(ctx context.Context, key string) (bool, error) {
if err := ensureAdminStore(ctx); err != nil {
return false, err
}
return adminrepo.GetBoolByKey(ctx, key)
}
// CreateSystemConfig persists a new system config row.
func CreateSystemConfig(ctx context.Context, config *model.SystemConfig) error {
if err := ensureAdminStore(ctx); err != nil {
return err
}
return adminrepo.CreateSystemConfigRecord(ctx, config)
}
// SaveOrUpdateSystemConfig creates or updates a config row and invalidates the admin cache.
func SaveOrUpdateSystemConfig(ctx context.Context, key, value string) error {
if err := ensureAdminStore(ctx); err != nil {
return err
}
return adminrepo.SaveOrUpdateSystemConfig(ctx, key, value)
}
// InvalidateSystemConfigCache evicts one key from Wavelet's system-config cache.
func InvalidateSystemConfigCache(ctx context.Context, key string) error {
if err := ensureAdminStore(ctx); err != nil {
return err
}
return adminrepo.InvalidateSystemConfigCache(ctx, key)
}
// InvalidateAllSystemConfigCaches evicts the whole Wavelet system-config cache.
func InvalidateAllSystemConfigCaches(ctx context.Context) error {
if err := ensureAdminStore(ctx); err != nil {
return err
}
return adminrepo.InvalidateAllSystemConfigCaches(ctx)
}
// StopSystemConfigCacheListener is retained for existing tests.
func StopSystemConfigCacheListener() {
adminrepo.StopSystemConfigCacheListener()
}
// ResetSystemConfigRAMCacheForTest clears the process-local admin config cache.
func ResetSystemConfigRAMCacheForTest() {
adminrepo.ResetSystemConfigRAMCacheForTest()
}
// ListAdminSystemConfigs returns configs, optionally filtered by type.
func ListAdminSystemConfigs(ctx context.Context, configType string) ([]model.SystemConfig, error) {
if err := ensureAdminStore(ctx); err != nil {
return nil, err
}
return adminrepo.ListAdminSystemConfigs(ctx, configType)
}
@@ -0,0 +1,68 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
"Wavelet/openflare/plugins/server/kernel/model"
adminrepo "Wavelet/plugins/domain/admin/repository"
)
const fallbackSystemUserID uint64 = 999
// GetActiveAuthSources lists enabled Wavelet auth sources via AuthService.
func GetActiveAuthSources(ctx context.Context) ([]model.AuthSource, error) {
svc := currentAuthService()
if svc == nil {
return nil, errors.New("auth service not initialized")
}
views, err := svc.ListAuthSources(ctx)
if err != nil {
return nil, err
}
sources := make([]model.AuthSource, 0, len(views))
for _, view := range views {
if !view.IsActive {
continue
}
sources = append(sources, model.AuthSource{
ID: view.ID,
Name: view.Name,
Type: view.Type,
DisplayName: view.DisplayName,
IconURL: view.IconURL,
IsActive: true,
})
}
return sources, nil
}
// GetTaskExecutionByTaskID loads a task execution by public task ID.
func GetTaskExecutionByTaskID(ctx context.Context, taskID string) (*model.TaskExecution, error) {
if err := ensureAdminStore(ctx); err != nil {
return nil, err
}
return adminrepo.GetTaskExecutionByTaskID(ctx, taskID)
}
// GetSystemUser loads the built-in system user via UserService, or a synthetic fallback.
func GetSystemUser(ctx context.Context) model.User {
if svc := currentUserService(); svc != nil {
if user, err := svc.GetUserByUsername(ctx, configTypeSystem); err == nil && user != nil {
return model.User{
ID: user.ID,
Username: user.Username,
Nickname: user.Nickname,
IsActive: user.IsActive,
}
}
}
return model.User{
ID: fallbackSystemUserID,
Username: configTypeSystem,
Nickname: "系统",
}
}
@@ -0,0 +1,121 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"testing"
"Wavelet/core/contracts"
"Wavelet/pkg/idgen"
adminmodel "Wavelet/plugins/domain/admin/model"
"Wavelet/plugins/infra/database"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
type stubUserService struct {
contracts.UserService
user *contracts.UserDTO
}
func (s stubUserService) GetUserByUsername(context.Context, string) (*contracts.UserDTO, error) {
return s.user, nil
}
type stubAuthService struct {
contracts.AuthService
sources []contracts.AuthSourceViewDTO
}
func (s stubAuthService) ListAuthSources(context.Context) ([]contracts.AuthSourceViewDTO, error) {
return s.sources, nil
}
func setupRepoTestDB(t *testing.T) (*gorm.DB, func()) {
t.Helper()
sqliteDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
})
if err != nil {
t.Fatalf("gorm.Open() error = %v", err)
}
if err := sqliteDB.AutoMigrate(&adminmodel.TaskExecution{}); err != nil {
t.Fatalf("AutoMigrate(TaskExecution) error = %v", err)
}
if err := idgen.Init(1); err != nil {
t.Fatalf("idgen.Init() error = %v", err)
}
database.SetDB(sqliteDB)
return sqliteDB, func() { database.SetDB(nil) }
}
func TestGetActiveAuthSourcesUsesAuthService(t *testing.T) {
SetAuthService(stubAuthService{})
t.Cleanup(func() { SetAuthService(nil) })
got, err := GetActiveAuthSources(context.Background())
if err != nil {
t.Fatalf("GetActiveAuthSources() error = %v", err)
}
if len(got) != 0 {
t.Fatalf("GetActiveAuthSources() len = %d, want 0", len(got))
}
SetAuthService(stubAuthService{sources: []contracts.AuthSourceViewDTO{
{ID: 1, Name: "inactive", Type: "oidc", DisplayName: "Off", IsActive: false},
{ID: 2, Name: "github", Type: "oidc", DisplayName: "GitHub", IconURL: "/i.png", IsActive: true},
}})
got, err = GetActiveAuthSources(context.Background())
if err != nil {
t.Fatalf("GetActiveAuthSources() error = %v", err)
}
if len(got) != 1 {
t.Fatalf("GetActiveAuthSources() len = %d, want 1", len(got))
}
if got[0].ID != 2 || got[0].Name != "github" || !got[0].IsActive {
t.Fatalf("GetActiveAuthSources()[0] = %+v, want active github id=2", got[0])
}
}
func TestGetSystemUserUsesUserService(t *testing.T) {
SetUserService(stubUserService{user: &contracts.UserDTO{
ID: 42,
Username: "system",
Nickname: "System User",
IsActive: true,
}})
t.Cleanup(func() { SetUserService(nil) })
got := GetSystemUser(context.Background())
if got.ID != 42 || got.Username != "system" || got.Nickname != "System User" {
t.Fatalf("GetSystemUser() = %+v, want id=42 username=system", got)
}
}
func TestGetTaskExecutionByTaskIDUsesAdminStore(t *testing.T) {
_, cleanup := setupRepoTestDB(t)
t.Cleanup(cleanup)
ctx := context.Background()
row := &adminmodel.TaskExecution{
ID: 7,
TaskID: "task-public-id",
TaskType: "pages_source_action",
Status: adminmodel.TaskExecutionStatusPending,
}
if err := database.DB(ctx).Create(row).Error; err != nil {
t.Fatalf("Create(TaskExecution) error = %v", err)
}
got, err := GetTaskExecutionByTaskID(ctx, "task-public-id")
if err != nil {
t.Fatalf("GetTaskExecutionByTaskID(%q) error = %v", "task-public-id", err)
}
if got.ID != 7 || got.TaskType != "pages_source_action" {
t.Fatalf("GetTaskExecutionByTaskID() = %+v, want id=7 type=pages_source_action", got)
}
}