tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-04 07:06:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(clickhouse): P0/P1 access log and WAF query aggregation and SQL pagination

Browse Source
This commit is contained in:
ryan
2026-07-02 15:22:36 +08:00
parent 3e910f1961
commit 6525bef15d
10 changed files with 555 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/model/openflare_access_log.go
+76 -98
View File
@@ -24,6 +24,8 @@ type openFlareAccessLogBucketAggregateRow struct {
SuccessCount int64 `gorm:"column:success_count"`
ClientErrorCount int64 `gorm:"column:client_error_count"`
ServerErrorCount int64 `gorm:"column:server_error_count"`
UniqueIPCount int64 `gorm:"column:unique_ip_count"`
UniqueHostCount int64 `gorm:"column:unique_host_count"`
}
type openFlareAccessLogBucketDimensionRow struct {
@@ -52,9 +54,45 @@ type openFlareAccessLogIPTrendRow struct {
RequestCount int64 `gorm:"column:request_count"`
}
// ListOpenFlareAccessLogsForWAFIPGroup lists access logs in a time window for automatic IP group rules.
func ListOpenFlareAccessLogsForWAFIPGroup(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLog, error) {
return ListOpenFlareAccessLogs(ctx, query)
type openFlareAccessLogWAFIPAggregateRow struct {
RemoteAddr string
RequestCount int64
Status404Count int64
ClientErrorCount int64
ServerErrorCount int64
IPHostCount int64
LastSeenEpoch int64
StatusCounts map[int]int64
}
// ListOpenFlareAccessLogWAFIPAggregates returns per-IP aggregates for WAF automatic rules.
func ListOpenFlareAccessLogWAFIPAggregates(ctx context.Context, query OpenFlareAccessLogQuery) ([]*OpenFlareAccessLogWAFIPAggregate, error) {
rows, err := currentAccessLogStore().WAFIPAggregates(ctx, query)
if err != nil {
return nil, err
}
result := make([]*OpenFlareAccessLogWAFIPAggregate, 0, len(rows))
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
statusCounts := make(map[int]int, len(row.StatusCounts))
for code, count := range row.StatusCounts {
statusCounts[code] = int(count)
}
result = append(result, &OpenFlareAccessLogWAFIPAggregate{
RemoteAddr: remoteAddr,
RequestCount: int(row.RequestCount),
Status404Count: int(row.Status404Count),
ClientErrorCount: int(row.ClientErrorCount),
ServerErrorCount: int(row.ServerErrorCount),
IPHostCount: int(row.IPHostCount),
LastSeenEpoch: row.LastSeenEpoch,
StatusCounts: statusCounts,
})
}
return result, nil
}
// InsertOpenFlareAccessLogsBatch inserts access log rows into ClickHouse.
@@ -79,24 +117,17 @@ func ListOpenFlareAccessLogRegionCounts(ctx context.Context, nodeID string, sinc
// ListOpenFlareAccessLogBuckets lists folded access log buckets.
func ListOpenFlareAccessLogBuckets(ctx context.Context, query OpenFlareAccessLogBucketQuery) ([]*OpenFlareAccessLogBucketRow, error) {
rows, err := buildOpenFlareAccessLogBucketRows(ctx, query)
if err != nil {
return nil, err
}
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
if start >= len(rows) {
return []*OpenFlareAccessLogBucketRow{}, nil
}
return rows[start:end], nil
return buildOpenFlareAccessLogBucketRows(ctx, query)
}
// CountOpenFlareAccessLogBuckets counts folded access log buckets.
func CountOpenFlareAccessLogBuckets(ctx context.Context, query OpenFlareAccessLogBucketQuery) (int64, error) {
rows, err := buildOpenFlareAccessLogBucketRows(ctx, query)
if err != nil {
return 0, err
filter := openFlareAccessLogQueryFromBucket(query)
bucketSeconds := int64(query.FoldMinutes * secondsPerMinute)
if bucketSeconds <= 0 {
bucketSeconds = 180
}
return int64(len(rows)), nil
return currentAccessLogStore().CountBuckets(ctx, filter, bucketSeconds)
}
// ListOpenFlareAccessLogBucketIPs lists folded IP rows for a bucket window.
@@ -123,24 +154,13 @@ func CountOpenFlareAccessLogBucketIPs(ctx context.Context, query OpenFlareAccess
// ListOpenFlareAccessLogIPSummaries lists IP summaries.
func ListOpenFlareAccessLogIPSummaries(ctx context.Context, query OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*OpenFlareAccessLogIPSummaryRow, error) {
rows, err := buildOpenFlareAccessLogIPSummaryRows(ctx, query, recentSince)
if err != nil {
return nil, err
}
start, end := openFlareAccessLogPaginateBounds(len(rows), query.Page, query.PageSize)
if start >= len(rows) {
return []*OpenFlareAccessLogIPSummaryRow{}, nil
}
return rows[start:end], nil
return buildOpenFlareAccessLogIPSummaryRows(ctx, query, recentSince)
}
// CountOpenFlareAccessLogIPSummaries counts IP summaries.
func CountOpenFlareAccessLogIPSummaries(ctx context.Context, query OpenFlareAccessLogIPSummaryQuery) (int64, error) {
rows, err := buildOpenFlareAccessLogIPSummaryRows(ctx, query, time.Time{})
if err != nil {
return 0, err
}
return int64(len(rows)), nil
filter := openFlareAccessLogQueryFromIPSummary(query)
return currentAccessLogStore().CountIPSummaries(ctx, filter)
}
// ListOpenFlareAccessLogIPTrend lists IP trend points.
@@ -195,75 +215,22 @@ func buildOpenFlareAccessLogBucketRows(ctx context.Context, query OpenFlareAcces
bucketSeconds = 180
}
type bucketAccumulator struct {
requestCount int64
uniqueIPs map[string]struct{}
uniqueHosts map[string]struct{}
successCount int64
clientErrorCount int64
serverErrorCount int64
}
accumulators := make(map[int64]*bucketAccumulator)
partials, err := currentAccessLogStore().BucketAggregates(ctx, filter, bucketSeconds)
if err != nil {
return nil, err
}
rows := make([]*OpenFlareAccessLogBucketRow, 0, len(partials))
for _, partial := range partials {
accumulator := accumulators[partial.BucketEpoch]
if accumulator == nil {
accumulator = &bucketAccumulator{
uniqueIPs: make(map[string]struct{}),
uniqueHosts: make(map[string]struct{}),
}
accumulators[partial.BucketEpoch] = accumulator
}
accumulator.requestCount += partial.RequestCount
accumulator.successCount += partial.SuccessCount
accumulator.clientErrorCount += partial.ClientErrorCount
accumulator.serverErrorCount += partial.ServerErrorCount
}
for _, column := range []string{columnRemoteAddr, columnHost} {
dimensions, err := currentAccessLogStore().BucketDimensions(ctx, filter, column, bucketSeconds)
if err != nil {
return nil, err
}
for _, item := range dimensions {
accumulator := accumulators[item.BucketEpoch]
if accumulator == nil {
accumulator = &bucketAccumulator{
uniqueIPs: make(map[string]struct{}),
uniqueHosts: make(map[string]struct{}),
}
accumulators[item.BucketEpoch] = accumulator
}
trimmed := strings.TrimSpace(item.Value)
if trimmed == "" {
continue
}
switch column {
case columnRemoteAddr:
accumulator.uniqueIPs[trimmed] = struct{}{}
case columnHost:
accumulator.uniqueHosts[trimmed] = struct{}{}
}
}
}
rows := make([]*OpenFlareAccessLogBucketRow, 0, len(accumulators))
for bucketEpoch, accumulator := range accumulators {
rows = append(rows, &OpenFlareAccessLogBucketRow{
BucketEpoch: bucketEpoch,
RequestCount: accumulator.requestCount,
UniqueIPCount: int64(len(accumulator.uniqueIPs)),
UniqueHostCount: int64(len(accumulator.uniqueHosts)),
SuccessCount: accumulator.successCount,
ClientErrorCount: accumulator.clientErrorCount,
ServerErrorCount: accumulator.serverErrorCount,
BucketEpoch: partial.BucketEpoch,
RequestCount: partial.RequestCount,
UniqueIPCount: partial.UniqueIPCount,
UniqueHostCount: partial.UniqueHostCount,
SuccessCount: partial.SuccessCount,
ClientErrorCount: partial.ClientErrorCount,
ServerErrorCount: partial.ServerErrorCount,
})
}
sortOpenFlareAccessLogBucketRows(rows, query.SortBy, query.SortOrder)
return rows, nil
}
@@ -293,12 +260,7 @@ func buildOpenFlareAccessLogBucketIPRows(ctx context.Context, query OpenFlareAcc
}
func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query OpenFlareAccessLogIPSummaryQuery, recentSince time.Time) ([]*OpenFlareAccessLogIPSummaryRow, error) {
filter := OpenFlareAccessLogQuery{
NodeID: query.NodeID,
RemoteAddr: query.RemoteAddr,
Host: query.Host,
Since: query.Since,
}
filter := openFlareAccessLogQueryFromIPSummary(query)
partials, err := currentAccessLogStore().IPSummaries(ctx, filter, recentSince)
if err != nil {
return nil, err
@@ -316,7 +278,6 @@ func buildOpenFlareAccessLogIPSummaryRows(ctx context.Context, query OpenFlareAc
LastSeenEpoch: partial.LastSeenEpoch,
})
}
sortOpenFlareAccessLogIPSummaryRows(rows, query.SortBy, query.SortOrder)
return rows, nil
}
@@ -350,6 +311,23 @@ func openFlareAccessLogQueryFromBucket(query OpenFlareAccessLogBucketQuery) Open
Host: query.Host,
Path: query.Path,
Since: query.Since,
Page: query.Page,
PageSize: query.PageSize,
SortBy: query.SortBy,
SortOrder: query.SortOrder,
}
}
func openFlareAccessLogQueryFromIPSummary(query OpenFlareAccessLogIPSummaryQuery) OpenFlareAccessLogQuery {
return OpenFlareAccessLogQuery{
NodeID: query.NodeID,
RemoteAddr: query.RemoteAddr,
Host: query.Host,
Since: query.Since,
Page: query.Page,
PageSize: query.PageSize,
SortBy: query.SortBy,
SortOrder: query.SortOrder,
}
}
internal/model/openflare_access_log_store.go
+34
View File
@@ -19,9 +19,12 @@ type accessLogStore interface {
Count(ctx context.Context, query OpenFlareAccessLogQuery) (int64, int64, error)
RegionCounts(ctx context.Context, nodeID string, since time.Time, limit int) ([]*OpenFlareAccessLogRegionCount, error)
BucketAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogBucketAggregateRow, error)
CountBuckets(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error)
BucketDimensions(ctx context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error)
IPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery, exactRemoteAddr bool) ([]openFlareAccessLogIPAggregateRow, error)
WAFIPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error)
IPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery, recentSince time.Time) ([]openFlareAccessLogIPSummaryRow, error)
CountIPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery) (int64, error)
IPTrend(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error)
DeleteAll(ctx context.Context) (int64, error)
DeleteBefore(ctx context.Context, cutoff time.Time) (int64, error)
@@ -116,11 +119,17 @@ func (clickhouseAccessLogStore) BucketAggregates(ctx context.Context, filter Ope
SuccessCount: row.SuccessCount,
ClientErrorCount: row.ClientErrorCount,
ServerErrorCount: row.ServerErrorCount,
UniqueIPCount: row.UniqueIPCount,
UniqueHostCount: row.UniqueHostCount,
}
}
return result, nil
}
func (clickhouseAccessLogStore) CountBuckets(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
return analyticsrepo.CountBucketAggregatesNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
}
func (clickhouseAccessLogStore) BucketDimensions(ctx context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
rows, err := analyticsrepo.BucketDimensionsNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), column, bucketSeconds)
if err != nil {
@@ -172,6 +181,31 @@ func (clickhouseAccessLogStore) IPSummaries(ctx context.Context, filter OpenFlar
return result, nil
}
func (clickhouseAccessLogStore) CountIPSummaries(ctx context.Context, filter OpenFlareAccessLogQuery) (int64, error) {
return analyticsrepo.CountIPSummaryNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
}
func (clickhouseAccessLogStore) WAFIPAggregates(ctx context.Context, filter OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
rows, err := analyticsrepo.IPAggregatesForWAFNodeAccessLogs(ctx, toNodeAccessLogFilter(filter))
if err != nil {
return nil, err
}
result := make([]openFlareAccessLogWAFIPAggregateRow, len(rows))
for index, row := range rows {
result[index] = openFlareAccessLogWAFIPAggregateRow{
RemoteAddr: row.RemoteAddr,
RequestCount: row.RequestCount,
Status404Count: row.Status404Count,
ClientErrorCount: row.ClientErrorCount,
ServerErrorCount: row.ServerErrorCount,
IPHostCount: row.IPHostCount,
LastSeenEpoch: row.LastSeenEpoch,
StatusCounts: row.StatusCounts,
}
}
return result, nil
}
func (clickhouseAccessLogStore) IPTrend(ctx context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) ([]openFlareAccessLogIPTrendRow, error) {
rows, err := analyticsrepo.IPTrendNodeAccessLogs(ctx, toNodeAccessLogFilter(filter), bucketSeconds)
if err != nil {
internal/model/openflare_access_log_store_memory.go
+158 -5
View File
@@ -5,6 +5,8 @@ package model
import (
"context"
"net"
"net/netip"
"sort"
"strings"
"sync"
@@ -99,12 +101,21 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
aggregates := make(map[int64]*openFlareAccessLogBucketAggregateRow)
type bucketAccumulator struct {
openFlareAccessLogBucketAggregateRow
uniqueIPs map[string]struct{}
uniqueHosts map[string]struct{}
}
aggregates := make(map[int64]*bucketAccumulator)
for _, row := range rows {
bucketEpoch := memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)
item := aggregates[bucketEpoch]
if item == nil {
item = &openFlareAccessLogBucketAggregateRow{BucketEpoch: bucketEpoch}
item = &bucketAccumulator{
openFlareAccessLogBucketAggregateRow: openFlareAccessLogBucketAggregateRow{BucketEpoch: bucketEpoch},
uniqueIPs: make(map[string]struct{}),
uniqueHosts: make(map[string]struct{}),
}
aggregates[bucketEpoch] = item
}
item.RequestCount++
@@ -116,14 +127,61 @@ func (s *memoryAccessLogStore) BucketAggregates(_ context.Context, filter OpenFl
default:
item.ServerErrorCount++
}
if remoteAddr := strings.TrimSpace(row.RemoteAddr); remoteAddr != "" {
item.uniqueIPs[remoteAddr] = struct{}{}
}
if host := strings.TrimSpace(row.Host); host != "" {
item.uniqueHosts[host] = struct{}{}
}
}
result := make([]openFlareAccessLogBucketAggregateRow, 0, len(aggregates))
for _, item := range aggregates {
result = append(result, *item)
item.UniqueIPCount = int64(len(item.uniqueIPs))
item.UniqueHostCount = int64(len(item.uniqueHosts))
result = append(result, item.openFlareAccessLogBucketAggregateRow)
}
bucketRows := make([]*OpenFlareAccessLogBucketRow, len(result))
for index := range result {
bucketRows[index] = &OpenFlareAccessLogBucketRow{
BucketEpoch: result[index].BucketEpoch,
RequestCount: result[index].RequestCount,
UniqueIPCount: result[index].UniqueIPCount,
UniqueHostCount: result[index].UniqueHostCount,
SuccessCount: result[index].SuccessCount,
ClientErrorCount: result[index].ClientErrorCount,
ServerErrorCount: result[index].ServerErrorCount,
}
}
sortOpenFlareAccessLogBucketRows(bucketRows, filter.SortBy, filter.SortOrder)
for index := range result {
result[index] = openFlareAccessLogBucketAggregateRow{
BucketEpoch: bucketRows[index].BucketEpoch,
RequestCount: bucketRows[index].RequestCount,
UniqueIPCount: bucketRows[index].UniqueIPCount,
UniqueHostCount: bucketRows[index].UniqueHostCount,
SuccessCount: bucketRows[index].SuccessCount,
ClientErrorCount: bucketRows[index].ClientErrorCount,
ServerErrorCount: bucketRows[index].ServerErrorCount,
}
}
if filter.PageSize > 0 {
start, end := openFlareAccessLogPaginateBounds(len(result), filter.Page, filter.PageSize)
return result[start:end], nil
}
return result, nil
}
func (s *memoryAccessLogStore) CountBuckets(_ context.Context, filter OpenFlareAccessLogQuery, bucketSeconds int64) (int64, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
seen := make(map[int64]struct{})
for _, row := range rows {
seen[memoryAccessLogBucketEpoch(row.LoggedAt, bucketSeconds)] = struct{}{}
}
return int64(len(seen)), nil
}
func (s *memoryAccessLogStore) BucketDimensions(_ context.Context, filter OpenFlareAccessLogQuery, column string, bucketSeconds int64) ([]openFlareAccessLogBucketDimensionRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
@@ -222,9 +280,91 @@ func (s *memoryAccessLogStore) IPSummaries(_ context.Context, filter OpenFlareAc
item.LastSeenEpoch = epoch
}
}
result := make([]openFlareAccessLogIPSummaryRow, 0, len(aggregates))
summaryRows := make([]*OpenFlareAccessLogIPSummaryRow, 0, len(aggregates))
for _, item := range aggregates {
result = append(result, *item)
summaryRows = append(summaryRows, &OpenFlareAccessLogIPSummaryRow{
RemoteAddr: item.RemoteAddr,
TotalRequests: item.TotalRequests,
RecentRequests: item.RecentRequests,
LastSeenEpoch: item.LastSeenEpoch,
})
}
sortOpenFlareAccessLogIPSummaryRows(summaryRows, filter.SortBy, filter.SortOrder)
if filter.PageSize > 0 {
start, end := openFlareAccessLogPaginateBounds(len(summaryRows), filter.Page, filter.PageSize)
summaryRows = summaryRows[start:end]
}
result := make([]openFlareAccessLogIPSummaryRow, len(summaryRows))
for index, item := range summaryRows {
result[index] = openFlareAccessLogIPSummaryRow{
RemoteAddr: item.RemoteAddr,
TotalRequests: item.TotalRequests,
RecentRequests: item.RecentRequests,
LastSeenEpoch: item.LastSeenEpoch,
}
}
return result, nil
}
func (s *memoryAccessLogStore) CountIPSummaries(_ context.Context, filter OpenFlareAccessLogQuery) (int64, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
seen := make(map[string]struct{})
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
seen[remoteAddr] = struct{}{}
}
return int64(len(seen)), nil
}
func (s *memoryAccessLogStore) WAFIPAggregates(_ context.Context, filter OpenFlareAccessLogQuery) ([]openFlareAccessLogWAFIPAggregateRow, error) {
s.mu.RLock()
defer s.mu.RUnlock()
rows := s.filterRecords(filter)
aggregates := make(map[string]*openFlareAccessLogWAFIPAggregateRow)
order := make([]string, 0)
for _, row := range rows {
remoteAddr := strings.TrimSpace(row.RemoteAddr)
if remoteAddr == "" {
continue
}
item := aggregates[remoteAddr]
if item == nil {
item = &openFlareAccessLogWAFIPAggregateRow{
RemoteAddr: remoteAddr,
StatusCounts: make(map[int]int64),
}
aggregates[remoteAddr] = item
order = append(order, remoteAddr)
}
item.RequestCount++
item.StatusCounts[row.StatusCode]++
if row.StatusCode == 404 {
item.Status404Count++
}
if row.StatusCode >= 400 && row.StatusCode < 500 {
item.ClientErrorCount++
}
if row.StatusCode >= 500 {
item.ServerErrorCount++
}
if memoryAccessLogHostIsIPLiteral(row.Host) {
item.IPHostCount++
}
epoch := row.LoggedAt.UTC().Unix()
if epoch > item.LastSeenEpoch {
item.LastSeenEpoch = epoch
}
}
result := make([]openFlareAccessLogWAFIPAggregateRow, 0, len(order))
for _, remoteAddr := range order {
if item := aggregates[remoteAddr]; item != nil {
result = append(result, *item)
}
}
return result, nil
}
@@ -324,6 +464,19 @@ func memoryAccessLogMatches(row *OpenFlareAccessLog, query OpenFlareAccessLogQue
return true
}
func memoryAccessLogHostIsIPLiteral(value string) bool {
host := strings.TrimSpace(value)
if host == "" {
return false
}
if parsedHost, _, err := net.SplitHostPort(host); err == nil {
host = parsedHost
}
host = strings.Trim(host, "[]")
_, err := netip.ParseAddr(host)
return err == nil
}
func memoryAccessLogBucketEpoch(loggedAt time.Time, bucketSeconds int64) int64 {
if bucketSeconds <= 0 {
bucketSeconds = 180
internal/model/openflare_observability.go
+12
View File
@@ -277,6 +277,18 @@ type OpenFlareAccessLogIPTrendRow struct {
RequestCount int64 `json:"request_count"`
}
// OpenFlareAccessLogWAFIPAggregate is a per-IP aggregate row for WAF automatic rules.
type OpenFlareAccessLogWAFIPAggregate struct {
RemoteAddr string
RequestCount int
Status404Count int
ClientErrorCount int
ServerErrorCount int
IPHostCount int
LastSeenEpoch int64
StatusCounts map[int]int
}
func isMissingTableError(err error) bool {
if err == nil {
return false