perf(clickhouse): P0/P1 access log and WAF query aggregation and SQL pagination

internal/apps/openflare/observability/access_log_logics.go

@@ -21,11 +21,15 @@ const (
 	defaultIPTrendBucketMinute = 30
 	maxIPTrendHours            = 168
 	nodeAccessLogRetentionDays = 90
+	defaultAccessLogQueryDays  = 7
 	accessLogFieldRemoteAddr   = "remote_addr"
 	accessLogFieldRequestCount = "request_count"
 )
 
-var nodeAccessLogRetentionWindow = nodeAccessLogRetentionDays * 24 * time.Hour
+var (
+	nodeAccessLogRetentionWindow = nodeAccessLogRetentionDays * 24 * time.Hour
+	defaultAccessLogQueryWindow  = defaultAccessLogQueryDays * 24 * time.Hour
+)
 
 // AccessLogQuery filters access log list queries.
 type AccessLogQuery struct {
@@ -346,7 +350,7 @@ func ListFoldedAccessLogIPs(ctx context.Context, input FoldedAccessLogIPQuery) (
 // ListAccessLogIPSummaries returns paginated IP summaries.
 func ListAccessLogIPSummaries(ctx context.Context, input AccessLogIPSummaryQuery) (*AccessLogIPSummaryList, error) {
 	normalized := normalizeAccessLogIPSummaryQuery(input)
-	since := time.Now().UTC().Add(-nodeAccessLogRetentionWindow)
+	since := defaultAccessLogSince()
 	recentSince := time.Now().UTC().Add(-3 * time.Hour)
 	query := model.OpenFlareAccessLogIPSummaryQuery{
 		NodeID:     strings.TrimSpace(normalized.NodeID),
@@ -453,7 +457,7 @@ func buildModelAccessLogQuery(input AccessLogQuery) model.OpenFlareAccessLogQuer
 		RemoteAddr: strings.TrimSpace(input.RemoteAddr),
 		Host:       strings.TrimSpace(input.Host),
 		Path:       strings.TrimSpace(input.Path),
-		Since:      time.Now().UTC().Add(-nodeAccessLogRetentionWindow),
+		Since:      defaultAccessLogSince(),
 		Page:       input.Page,
 		PageSize:   input.PageSize,
 		SortBy:     input.SortBy,
@@ -461,6 +465,10 @@ func buildModelAccessLogQuery(input AccessLogQuery) model.OpenFlareAccessLogQuer
 	}
 }
 
+func defaultAccessLogSince() time.Time {
+	return time.Now().UTC().Add(-defaultAccessLogQueryWindow)
+}
+
 func listNodeNameMap(ctx context.Context, logs []*model.OpenFlareAccessLog) (map[string]string, error) {
 	nodeIDs := make([]string, 0, len(logs))
 	seen := make(map[string]struct{}, len(logs))

internal/apps/openflare/waf/ip_group_sync.go

@@ -234,15 +234,15 @@ func evaluateParsedIPGroupAutoConfig(ctx context.Context, config ipGroupAutoConf
 		}
 		programs = append(programs, program)
 	}
-	logs, err := model.ListOpenFlareAccessLogsForWAFIPGroup(ctx, model.OpenFlareAccessLogQuery{
+	aggregates, err := model.ListOpenFlareAccessLogWAFIPAggregates(ctx, model.OpenFlareAccessLogQuery{
 		Since: now.Add(-time.Duration(config.LookbackMinutes) * time.Minute),
 		Until: now,
 	})
 	if err != nil {
 		return nil, err
 	}
-	accumulators := make(map[string]*ipGroupAutoAccumulator)
-	for _, item := range logs {
+	accumulators := make(map[string]*ipGroupAutoAccumulator, len(aggregates))
+	for _, item := range aggregates {
 		if item == nil {
 			continue
 		}
@@ -250,30 +250,23 @@ func evaluateParsedIPGroupAutoConfig(ctx context.Context, config ipGroupAutoConf
 		if !ok {
 			continue
 		}
-		acc := accumulators[ip]
-		if acc == nil {
-			acc = &ipGroupAutoAccumulator{
-				ip:           ip,
-				statusCounts: make(map[int]int),
-			}
-			accumulators[ip] = acc
+		lastSeen := time.Time{}
+		if item.LastSeenEpoch > 0 {
+			lastSeen = time.Unix(item.LastSeenEpoch, 0).UTC()
 		}
-		acc.requestCount++
-		acc.statusCounts[item.StatusCode]++
-		if item.StatusCode == http.StatusNotFound {
-			acc.status404Count++
+		statusCounts := make(map[int]int, len(item.StatusCounts))
+		for code, count := range item.StatusCounts {
+			statusCounts[code] = count
 		}
-		if item.StatusCode >= 400 && item.StatusCode < 500 {
-			acc.clientErrorCount++
-		}
-		if item.StatusCode >= http.StatusInternalServerError {
-			acc.serverErrorCount++
-		}
-		if hostIsIPLiteral(item.Host) {
-			acc.ipHostCount++
-		}
-		if item.LoggedAt.After(acc.lastSeen) {
-			acc.lastSeen = item.LoggedAt
+		accumulators[ip] = &ipGroupAutoAccumulator{
+			ip:               ip,
+			requestCount:     item.RequestCount,
+			status404Count:   item.Status404Count,
+			ipHostCount:      item.IPHostCount,
+			clientErrorCount: item.ClientErrorCount,
+			serverErrorCount: item.ServerErrorCount,
+			lastSeen:         lastSeen,
+			statusCounts:     statusCounts,
 		}
 	}
 	matched := make([]string, 0)