refactor(backend): rename OpenFlare directory to lowercase openflare

This commit is contained in:
ryan
2026-08-30 17:43:23 +08:00
parent 06d5fedbfc
commit c93ff6674f
543 changed files with 819 additions and 819 deletions
@@ -0,0 +1,116 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
// Package analytics defines ClickHouse analytics domain models and query DTOs
// (pure data, no IO).
package analytics
import "time"
// AccessLogFilter scopes user access log queries.
// 单一权威字段集(CH 原字段,Task 1 迁入):禁止追加仅某实现使用的字段(避免双字段集分叉)。
type AccessLogFilter struct {
// UserIDs filters by user IDs. nil means no user filter; an empty slice means no matches.
UserIDs []uint64
Path string
// StartTime filters created_at >= StartTime when non-nil.
StartTime *time.Time
// EndTime filters created_at <= EndTime when non-nil(闭区间,与 CH/GORM 实现一致)。
EndTime *time.Time
}
// NodeAccessLogFilter scopes ClickHouse node access log queries.
type NodeAccessLogFilter struct {
NodeID string
RemoteAddr string
Host string
// Hosts exact-matches any host (case-insensitive). Prefer over Host for multi-domain scopes.
Hosts []string
Path string
// StatusCode filters by exact HTTP status code when > 0.
StatusCode int
Since time.Time
Until time.Time
Page int
PageSize int
SortBy string
SortOrder string
}
// NodeObservabilityFilter scopes ClickHouse node observability queries.
type NodeObservabilityFilter struct {
NodeID string
Since time.Time
Limit int
}
// DailyTrend is a single day's access count.
type DailyTrend struct {
Date string
Count uint64
}
// BrowserShare is a browser group's share of access logs.
type BrowserShare struct {
Browser string
Count uint64
}
// TopUser is an active user ranked by access count.
type TopUser struct {
UserID uint64
Count uint64
}
// NodeAccessLogRegionCount aggregates access log regions.
type NodeAccessLogRegionCount struct {
Region string
Count int64
}
// NodeAccessLogTrafficSummary is a window-level access log traffic summary.
type NodeAccessLogTrafficSummary struct {
RequestCount int64
ErrorCount int64
UniqueIPCount int64
BytesSent int64
RequestLength int64
NodeCount int64
}
// NodeAccessLogValueCount is a grouped value count (status_code, host, ...).
type NodeAccessLogValueCount struct {
Value string
Count int64
}
// NodeAccessLogNodeAggregate is per-node traffic over a window.
type NodeAccessLogNodeAggregate struct {
NodeID string
RequestCount int64
ErrorCount int64
UniqueIPCount int64
}
// BatchWriterStats is a point-in-time snapshot of a batch writer queue and failure counters.
type BatchWriterStats struct {
Name string `json:"name"`
Depth int `json:"depth"`
Cap int `json:"cap"`
Drops int64 `json:"drops"`
FlushErrors int64 `json:"flush_errors"`
Running bool `json:"running"`
}
// ClickHouseOperationalStats summarizes ClickHouse merge/mutation pressure
// and in-process batch writer queue health.
type ClickHouseOperationalStats struct {
Database string `json:"database"`
ActiveParts int64 `json:"active_parts"`
TotalRows int64 `json:"total_rows"`
PendingMutations int64 `json:"pending_mutations"`
AsyncInsertQueue int64 `json:"async_insert_queue"`
AsyncInsertBytes int64 `json:"async_insert_bytes"`
// BatchWriters reports in-process queue depth/drops/flush errors for CH writers.
BatchWriters []BatchWriterStats `json:"batch_writers,omitempty"`
}
@@ -0,0 +1,47 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"fmt"
"time"
)
const (
nodeAccessLogTableName = "of_node_access_logs"
nodeAccessLogInsertColumns = "id, node_id, logged_at, remote_addr, region, host, path, user_agent, cache_status, status_code, bytes_sent, request_length, request_time_ms, created_at"
)
// NodeAccessLog stores OpenFlare edge node access records in ClickHouse.
type NodeAccessLog struct {
ID uint64 `gorm:"column:id"`
NodeID string `gorm:"column:node_id"`
LoggedAt time.Time `gorm:"column:logged_at"`
RemoteAddr string `gorm:"column:remote_addr"`
Region string `gorm:"column:region"`
Host string `gorm:"column:host"`
Path string `gorm:"column:path"`
UserAgent string `gorm:"column:user_agent"`
CacheStatus string `gorm:"column:cache_status"`
StatusCode int32 `gorm:"column:status_code"`
BytesSent uint64 `gorm:"column:bytes_sent"`
RequestLength uint64 `gorm:"column:request_length"`
RequestTimeMs uint32 `gorm:"column:request_time_ms"`
CreatedAt time.Time `gorm:"column:created_at"`
}
// TableName returns the ClickHouse table name.
func (NodeAccessLog) TableName() string {
return nodeAccessLogTableName
}
// InsertColumns returns comma-separated column names for batch insert.
func (NodeAccessLog) InsertColumns() string {
return nodeAccessLogInsertColumns
}
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
func (NodeAccessLog) BatchInsertSQL() string {
return fmt.Sprintf("INSERT INTO %s (%s)", nodeAccessLogTableName, nodeAccessLogInsertColumns)
}
@@ -0,0 +1,68 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
// NodeAccessLogBucketAggregate is a folded bucket aggregate row.
type NodeAccessLogBucketAggregate struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
RequestCount int64 `gorm:"column:request_count"`
SuccessCount int64 `gorm:"column:success_count"`
ClientErrorCount int64 `gorm:"column:client_error_count"`
ServerErrorCount int64 `gorm:"column:server_error_count"`
Status2xxCount int64 `gorm:"column:status_2xx_count"`
Status4xxCount int64 `gorm:"column:status_4xx_count"`
Status5xxCount int64 `gorm:"column:status_5xx_count"`
UniqueIPCount int64 `gorm:"column:unique_ip_count"`
UniqueHostCount int64 `gorm:"column:unique_host_count"`
BytesSent int64 `gorm:"column:bytes_sent"`
RequestLength int64 `gorm:"column:request_length"`
}
// NodeAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
type NodeAccessLogWAFIPAggregate struct {
RemoteAddr string
RequestCount int64
Status404Count int64
ClientErrorCount int64
ServerErrorCount int64
IPHostCount int64
LastSeenEpoch int64
StatusCounts map[int]int64
}
// NodeAccessLogBucketDimension is a bucket dimension value.
type NodeAccessLogBucketDimension struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
Value string `gorm:"column:value"`
}
// NodeAccessLogIPAggregate is an IP aggregate row.
type NodeAccessLogIPAggregate struct {
RemoteAddr string `gorm:"column:remote_addr"`
RequestCount int64 `gorm:"column:request_count"`
SuccessCount int64 `gorm:"column:success_count"`
ClientErrorCount int64 `gorm:"column:client_error_count"`
ServerErrorCount int64 `gorm:"column:server_error_count"`
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
}
// NodeAccessLogIPSummary is an IP summary row.
type NodeAccessLogIPSummary struct {
RemoteAddr string `gorm:"column:remote_addr"`
Region string `gorm:"column:region"`
TotalRequests int64 `gorm:"column:total_requests"`
Success2xxCount int64 `gorm:"column:success_2xx_count"`
SuccessRatio float64 `gorm:"column:success_ratio"`
BytesReceived int64 `gorm:"column:request_length"`
BytesSent int64 `gorm:"column:bytes_sent"`
// RecentRequests is deprecated (always 0); kept for wire compatibility.
RecentRequests int64 `gorm:"column:recent_requests"`
LastSeenEpoch int64 `gorm:"column:last_seen_epoch"`
}
// NodeAccessLogIPTrend is an IP trend bucket row.
type NodeAccessLogIPTrend struct {
BucketEpoch int64 `gorm:"column:bucket_epoch"`
RequestCount int64 `gorm:"column:request_count"`
}
@@ -0,0 +1,171 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"fmt"
"time"
)
const (
nodeMetricSnapshotTableName = "of_node_metric_snapshots"
nodeMetricSnapshotInsertColumns = "id, node_id, captured_at, cpu_usage_percent, memory_used_bytes, memory_total_bytes, storage_used_bytes, storage_total_bytes, disk_read_bytes, disk_write_bytes, network_rx_bytes, network_tx_bytes, created_at"
nodeEdgeHealthTableName = "of_node_edge_health"
nodeEdgeHealthInsertColumns = "id, node_id, captured_at, status, connections, created_at"
nodeObsFrpsTableName = "of_node_obs_frps"
nodeObsFrpsInsertColumns = "id, node_id, captured_at, frps_connections, frps_proxy_count, frps_client_count, frps_proxies, created_at"
nodeObsFrpcTableName = "of_node_obs_frpc"
nodeObsFrpcInsertColumns = "id, node_id, captured_at, tunnel_status, connected_relays_count, created_at"
)
// NodeMetricSnapshot stores periodic node resource utilization metrics in ClickHouse.
type NodeMetricSnapshot struct {
ID uint64 `gorm:"column:id"`
NodeID string `gorm:"column:node_id"`
CapturedAt time.Time `gorm:"column:captured_at"`
CPUUsagePercent float64 `gorm:"column:cpu_usage_percent"`
MemoryUsedBytes int64 `gorm:"column:memory_used_bytes"`
MemoryTotalBytes int64 `gorm:"column:memory_total_bytes"`
StorageUsedBytes int64 `gorm:"column:storage_used_bytes"`
StorageTotalBytes int64 `gorm:"column:storage_total_bytes"`
DiskReadBytes int64 `gorm:"column:disk_read_bytes"`
DiskWriteBytes int64 `gorm:"column:disk_write_bytes"`
NetworkRxBytes int64 `gorm:"column:network_rx_bytes"`
NetworkTxBytes int64 `gorm:"column:network_tx_bytes"`
CreatedAt time.Time `gorm:"column:created_at"`
}
// TableName returns the ClickHouse table name.
func (NodeMetricSnapshot) TableName() string {
return nodeMetricSnapshotTableName
}
// InsertColumns returns comma-separated column names for batch insert.
func (NodeMetricSnapshot) InsertColumns() string {
return nodeMetricSnapshotInsertColumns
}
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
func (NodeMetricSnapshot) BatchInsertSQL() string {
return fmt.Sprintf("INSERT INTO %s (%s)", nodeMetricSnapshotTableName, nodeMetricSnapshotInsertColumns)
}
// NodeEdgeHealth stores L2 OpenResty health snapshots (connections + status).
type NodeEdgeHealth struct {
ID uint64 `gorm:"column:id"`
NodeID string `gorm:"column:node_id"`
CapturedAt time.Time `gorm:"column:captured_at"`
Status string `gorm:"column:status"`
Connections int64 `gorm:"column:connections"`
CreatedAt time.Time `gorm:"column:created_at"`
}
// TableName returns the ClickHouse table name.
func (NodeEdgeHealth) TableName() string {
return nodeEdgeHealthTableName
}
// InsertColumns returns comma-separated column names for batch insert.
func (NodeEdgeHealth) InsertColumns() string {
return nodeEdgeHealthInsertColumns
}
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
func (NodeEdgeHealth) BatchInsertSQL() string {
return fmt.Sprintf("INSERT INTO %s (%s)", nodeEdgeHealthTableName, nodeEdgeHealthInsertColumns)
}
// AccessLogHourly is a Server-side hourly rollup of access logs.
type AccessLogHourly struct {
NodeID string `gorm:"column:node_id"`
Hour time.Time `gorm:"column:hour"`
Host string `gorm:"column:host"`
RequestCount int64 `gorm:"column:request_count"`
ErrorCount int64 `gorm:"column:error_count"`
BytesSent int64 `gorm:"column:bytes_sent"`
RequestLength int64 `gorm:"column:request_length"`
}
// NodeTrafficHourly is an hourly traffic rollup row.
//
// UniqueVisitorCount is always 0 when sourced from of_access_log_hourly
// (true UV requires raw uniqExact on access logs).
type NodeTrafficHourly struct {
NodeID string
Hour time.Time
RequestCount int64
ErrorCount int64
UniqueVisitorCount int64
}
// NodeMetricHourly is an hourly metric snapshot aggregation row.
//
// Disk and host network counters are cumulative. Prefer pre-aggregated min/max
// deltas from of_node_metric_capacity_hourly; raw fallback uses consecutive
// lagInFrame samples per node (negative deltas after counter reset are dropped).
type NodeMetricHourly struct {
Hour time.Time
AverageCPUUsagePercent float64
AverageMemoryUsagePercent float64
NetworkRxBytes int64
NetworkTxBytes int64
DiskReadBytes int64
DiskWriteBytes int64
ReportedNodes int
}
// NodeObsFrps stores FRPS observability snapshots in ClickHouse.
type NodeObsFrps struct {
ID uint64 `gorm:"column:id"`
NodeID string `gorm:"column:node_id"`
CapturedAt time.Time `gorm:"column:captured_at"`
FrpsConnections int32 `gorm:"column:frps_connections"`
FrpsProxyCount int32 `gorm:"column:frps_proxy_count"`
FrpsClientCount int32 `gorm:"column:frps_client_count"`
FrpsProxies string `gorm:"column:frps_proxies"`
CreatedAt time.Time `gorm:"column:created_at"`
}
// TableName returns the ClickHouse table name.
func (NodeObsFrps) TableName() string {
return nodeObsFrpsTableName
}
// InsertColumns returns comma-separated column names for batch insert.
func (NodeObsFrps) InsertColumns() string {
return nodeObsFrpsInsertColumns
}
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
func (NodeObsFrps) BatchInsertSQL() string {
return fmt.Sprintf("INSERT INTO %s (%s)", nodeObsFrpsTableName, nodeObsFrpsInsertColumns)
}
// NodeObsFrpc stores FRPC observability snapshots in ClickHouse.
type NodeObsFrpc struct {
ID uint64 `gorm:"column:id"`
NodeID string `gorm:"column:node_id"`
CapturedAt time.Time `gorm:"column:captured_at"`
TunnelStatus string `gorm:"column:tunnel_status"`
ConnectedRelaysCount int32 `gorm:"column:connected_relays_count"`
CreatedAt time.Time `gorm:"column:created_at"`
}
// TableName returns the ClickHouse table name.
func (NodeObsFrpc) TableName() string {
return nodeObsFrpcTableName
}
// InsertColumns returns comma-separated column names for batch insert.
func (NodeObsFrpc) InsertColumns() string {
return nodeObsFrpcInsertColumns
}
// BatchInsertSQL returns the INSERT prefix used by native batch writers.
func (NodeObsFrpc) BatchInsertSQL() string {
return fmt.Sprintf("INSERT INTO %s (%s)", nodeObsFrpcTableName, nodeObsFrpcInsertColumns)
}
@@ -0,0 +1,11 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
risklogstore "Wavelet/plugins/domain/risk_control/logstore"
)
// UserAccessLog is Wavelet risk_control's w_user_access_logs entity.
type UserAccessLog = risklogstore.UserAccessLog
@@ -0,0 +1,124 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import "strings"
// User-Agent 浏览器/OS/设备分类(纯函数,无 IO)。
// 与 internal/repository/analytics/browser.go 的判定逻辑保持一致(Task 4 复制,
// 因为 model 不得 import analyticsrepo);后续若移除旧 CH 实现,可让 analyticsrepo 改以别名复用本包。
const (
uaLabelUnknown = "Unknown"
uaLabelBot = "Bot"
uaLabelOther = "Other"
uaTokenBot = "bot"
uaTokenAndroid = "android"
uaTokenSpider = "spider"
uaTokenCrawler = "crawler"
)
type uaMatchRule struct {
label string
contains []string
allOf []string
noneOf []string
}
func matchUARules(uaLower string, rules []uaMatchRule, fallback string) string {
if uaLower == "" {
return uaLabelUnknown
}
for _, rule := range rules {
matched := false
for _, token := range rule.contains {
if strings.Contains(uaLower, token) {
matched = true
break
}
}
if !matched && len(rule.allOf) > 0 {
matched = true
for _, token := range rule.allOf {
if !strings.Contains(uaLower, token) {
matched = false
break
}
}
}
if !matched {
continue
}
excluded := false
for _, token := range rule.noneOf {
if strings.Contains(uaLower, token) {
excluded = true
break
}
}
if excluded {
continue
}
return rule.label
}
return fallback
}
var browserRules = []uaMatchRule{
{label: "WeChat", contains: []string{"micromessenger"}},
{label: "Postman", contains: []string{"postman"}},
{label: "CLI", contains: []string{"curl/", "wget/"}},
{label: "Edge", contains: []string{"edg/", "edgios/", "edga/"}},
{label: "Opera", contains: []string{"opr/", "opera"}},
{label: "Firefox", contains: []string{"firefox", "fxios"}},
{label: "Chrome", contains: []string{"crios", "chrome"}, noneOf: []string{"chromium"}},
{label: "Chromium", contains: []string{"chromium"}},
{label: "Safari", contains: []string{"safari"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp"}},
}
var osRules = []uaMatchRule{
{label: "Android", contains: []string{uaTokenAndroid}},
{label: "iOS", contains: []string{"iphone", "ipad", "ipod", "ios"}},
{label: "Windows", contains: []string{"windows"}},
{label: "macOS", contains: []string{"mac os x", "macintosh", "macos"}},
{label: "Chrome OS", contains: []string{"cros"}},
{label: "Linux", contains: []string{"linux"}},
{label: uaLabelBot, contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler}},
}
var deviceRules = []uaMatchRule{
{
label: uaLabelBot,
contains: []string{uaTokenBot, uaTokenSpider, uaTokenCrawler, "slurp", "curl/", "wget/", "python-requests", "go-http-client", "postman"},
},
{
label: "Tablet",
contains: []string{"ipad", "tablet"},
},
{
label: "Tablet",
allOf: []string{uaTokenAndroid},
noneOf: []string{"mobile"},
},
{
label: "Mobile",
contains: []string{"mobi", "iphone", "ipod", uaTokenAndroid},
},
}
// ParseBrowserName performs lightweight User-Agent browser identification.
func ParseBrowserName(ua string) string {
return matchUARules(strings.ToLower(ua), browserRules, uaLabelOther)
}
// ParseOSName performs lightweight User-Agent OS identification.
func ParseOSName(ua string) string {
return matchUARules(strings.ToLower(ua), osRules, uaLabelOther)
}
// ParseDeviceType performs lightweight User-Agent device type identification.
func ParseDeviceType(ua string) string {
return matchUARules(strings.ToLower(ua), deviceRules, "Desktop")
}