[新增] 添加自动 IP 组规则的抓取记录功能,支持查看已抓取 IP 列表及其到期状态

This commit is contained in:
ryan
2026-06-01 10:30:00 +08:00
parent 2dc7e72621
commit bd7f008322
7 changed files with 352 additions and 24 deletions
+48
View File
@@ -0,0 +1,48 @@
package migrate
import (
"fmt"
"gorm.io/gorm"
)
// WAF IP Group schema changes for v18:
// We introduce the `ext_ips` column to keep track of captured IPs with their capture timestamps.
// If more information needs to be saved for captured IPs, it can be added directly inside this JSON structure.
type wafIPGroupV18 struct {
ExtIPs string `gorm:"column:ext_ips;type:text;not null;default:'[]'"`
}
func init() {
Register(V18())
}
func V18() Migration {
return Migration{
FromVersion: 17,
ToVersion: 18,
Migrate: migrateV18,
Validate: validateV18,
}
}
func (wafIPGroupV18) TableName() string {
return "waf_ip_groups"
}
func migrateV18(ctx Context, db *gorm.DB, backend string) error {
return ctx.ApplyCurrentSchema(db, backend)
}
func validateV18(ctx Context, db *gorm.DB, backend string) error {
if err := ctx.ValidateDatabaseSchemaVersion(db, backend, 17); err != nil {
return err
}
if db == nil || !db.Migrator().HasTable(&wafIPGroupV18{}) {
return fmt.Errorf("table waf_ip_groups is missing")
}
if !db.Migrator().HasColumn(&wafIPGroupV18{}, "ext_ips") {
return fmt.Errorf("column waf_ip_groups.ext_ips is missing")
}
return nil
}
+12
View File
@@ -80,6 +80,8 @@ func (databaseSchemaMigrationContext) ValidateDatabaseSchemaVersion(db *gorm.DB,
return validateDatabaseSchemaV16(db, backend)
case 17:
return validateDatabaseSchemaV17(db, backend)
case 18:
return validateDatabaseSchemaV18(db, backend)
default:
return fmt.Errorf("database schema validation for v%d is not defined", version)
}
@@ -1190,6 +1192,16 @@ func validateDatabaseSchemaV17(db *gorm.DB, backend string) error {
return nil
}
func validateDatabaseSchemaV18(db *gorm.DB, backend string) error {
if err := validateDatabaseSchemaV17(db, backend); err != nil {
return err
}
if !db.Migrator().HasColumn(&WAFIPGroup{}, "ext_ips") {
return fmt.Errorf("column waf_ip_groups.ext_ips is missing")
}
return nil
}
func databaseSchemaMigrations() []databaseSchemaMigration {
ctx := databaseSchemaMigrationContext{}
migrations := []databaseSchemaMigration{}
+3
View File
@@ -31,6 +31,7 @@ type WAFIPGroup struct {
Enabled bool `json:"enabled" gorm:"not null;default:true"`
IPList string `json:"ip_list" gorm:"type:text;not null;default:'[]'"`
AutoConfig string `json:"auto_config" gorm:"type:text;not null;default:'{}'"`
ExtIPs string `json:"ext_ips" gorm:"type:text;not null;default:'[]'"`
SubscriptionURL string `json:"subscription_url" gorm:"size:2048;not null;default:''"`
SubscriptionFormat string `json:"subscription_format" gorm:"size:32;not null;default:'text'"`
SubscriptionMappingRule string `json:"subscription_mapping_rule" gorm:"size:255;not null;default:''"`
@@ -138,6 +139,7 @@ func (group *WAFIPGroup) Update() error {
"enabled": group.Enabled,
"ip_list": group.IPList,
"auto_config": group.AutoConfig,
"ext_ips": group.ExtIPs,
"subscription_url": group.SubscriptionURL,
"subscription_format": group.SubscriptionFormat,
"subscription_mapping_rule": group.SubscriptionMappingRule,
@@ -152,6 +154,7 @@ func (group *WAFIPGroup) Update() error {
func (group *WAFIPGroup) UpdateSyncResult() error {
return DB.Model(&WAFIPGroup{}).Where("id = ?", group.ID).Updates(map[string]any{
"ip_list": group.IPList,
"ext_ips": group.ExtIPs,
"last_synced_at": group.LastSyncedAt,
"next_sync_at": group.NextSyncAt,
"last_sync_status": group.LastSyncStatus,
+102 -22
View File
@@ -37,9 +37,20 @@ const (
type wafIPGroupAutoConfig struct {
LookbackMinutes int `json:"lookback_minutes"`
TTL int `json:"ttl"` // in seconds, default -1 (permanent)
Rules []wafIPGroupAutoRule `json:"rules"`
}
type WAFIPGroupExtIP struct {
IP string `json:"ip"`
CapturedAt time.Time `json:"captured_at"`
}
type WAFIPGroupExtIPView struct {
IP string `json:"ip"`
CapturedAt string `json:"captured_at"`
}
type wafIPGroupAutoRule struct {
Name string `json:"name"`
Expr string `json:"expr"`
@@ -81,24 +92,25 @@ type WAFIPGroupInput struct {
}
type WAFIPGroupView struct {
ID uint `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Enabled bool `json:"enabled"`
IPList []string `json:"ip_list"`
AutoConfig json.RawMessage `json:"auto_config"`
SubscriptionURL string `json:"subscription_url"`
SubscriptionFormat string `json:"subscription_format"`
SubscriptionMappingRule string `json:"subscription_mapping_rule"`
SyncIntervalMinutes int `json:"sync_interval_minutes"`
LastSyncedAt string `json:"last_synced_at,omitempty"`
NextSyncAt string `json:"next_sync_at,omitempty"`
LastSyncStatus string `json:"last_sync_status"`
LastSyncMessage string `json:"last_sync_message"`
Remark string `json:"remark"`
ReferencedByRuleCount int `json:"referenced_by_rule_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ID uint `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Enabled bool `json:"enabled"`
IPList []string `json:"ip_list"`
AutoConfig json.RawMessage `json:"auto_config"`
ExtIPs []WAFIPGroupExtIPView `json:"ext_ips"`
SubscriptionURL string `json:"subscription_url"`
SubscriptionFormat string `json:"subscription_format"`
SubscriptionMappingRule string `json:"subscription_mapping_rule"`
SyncIntervalMinutes int `json:"sync_interval_minutes"`
LastSyncedAt string `json:"last_synced_at,omitempty"`
NextSyncAt string `json:"next_sync_at,omitempty"`
LastSyncStatus string `json:"last_sync_status"`
LastSyncMessage string `json:"last_sync_message"`
Remark string `json:"remark"`
ReferencedByRuleCount int `json:"referenced_by_rule_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type WAFIPGroupSyncResult struct {
@@ -285,6 +297,7 @@ func buildWAFIPGroup(group *model.WAFIPGroup, input WAFIPGroupInput) (*model.WAF
ipListJSON, _ := json.Marshal(normalizedIPs)
if group == nil {
group = &model.WAFIPGroup{}
group.ExtIPs = "[]"
}
group.Name = name
group.Type = groupType
@@ -312,6 +325,17 @@ func buildWAFIPGroupView(group *model.WAFIPGroup, referenceCount int) (WAFIPGrou
if len(autoConfig) == 0 {
autoConfig = json.RawMessage("{}")
}
var extIPs []WAFIPGroupExtIP
if group.ExtIPs != "" && group.ExtIPs != "[]" {
_ = json.Unmarshal([]byte(group.ExtIPs), &extIPs)
}
viewExtIPs := make([]WAFIPGroupExtIPView, 0, len(extIPs))
for _, extIP := range extIPs {
viewExtIPs = append(viewExtIPs, WAFIPGroupExtIPView{
IP: extIP.IP,
CapturedAt: extIP.CapturedAt.Format(time.RFC3339),
})
}
view := WAFIPGroupView{
ID: group.ID,
Name: group.Name,
@@ -319,6 +343,7 @@ func buildWAFIPGroupView(group *model.WAFIPGroup, referenceCount int) (WAFIPGrou
Enabled: group.Enabled,
IPList: ips,
AutoConfig: autoConfig,
ExtIPs: viewExtIPs,
SubscriptionURL: group.SubscriptionURL,
SubscriptionFormat: group.SubscriptionFormat,
SubscriptionMappingRule: group.SubscriptionMappingRule,
@@ -389,18 +414,70 @@ func syncWAFIPGroupSubscription(group *model.WAFIPGroup, now time.Time) (*WAFIPG
}
func syncWAFIPGroupAutomatic(group *model.WAFIPGroup, now time.Time) (*WAFIPGroupSyncResult, error) {
ips, err := evaluateWAFIPGroupAutoConfig(group.AutoConfig, now)
config, err := parseWAFIPGroupAutoConfig(json.RawMessage(group.AutoConfig))
if err != nil {
recordWAFIPGroupSyncFailure(group, now, err)
return nil, err
}
ipListJSON, _ := json.Marshal(ips)
var existingExtIPs []WAFIPGroupExtIP
if group.ExtIPs != "" && group.ExtIPs != "[]" {
_ = json.Unmarshal([]byte(group.ExtIPs), &existingExtIPs)
}
activeExtIPs := make([]WAFIPGroupExtIP, 0, len(existingExtIPs))
for _, extIP := range existingExtIPs {
if config.TTL > 0 {
expirationTime := extIP.CapturedAt.Add(time.Duration(config.TTL) * time.Second)
if expirationTime.Before(now) {
continue
}
}
activeExtIPs = append(activeExtIPs, extIP)
}
ips, err := evaluateParsedWAFIPGroupAutoConfig(config, now)
if err != nil {
recordWAFIPGroupSyncFailure(group, now, err)
return nil, err
}
extIPMap := make(map[string]int)
for idx, extIP := range activeExtIPs {
extIPMap[extIP.IP] = idx
}
for _, ip := range ips {
if idx, ok := extIPMap[ip]; ok {
activeExtIPs[idx].CapturedAt = now
} else {
activeExtIPs = append(activeExtIPs, WAFIPGroupExtIP{
IP: ip,
CapturedAt: now,
})
}
}
finalIPs := make([]string, 0, len(activeExtIPs))
for _, extIP := range activeExtIPs {
finalIPs = append(finalIPs, extIP.IP)
}
finalIPs, err = normalizeWAFIPList(finalIPs)
if err != nil {
recordWAFIPGroupSyncFailure(group, now, err)
return nil, err
}
extIPsJSON, _ := json.Marshal(activeExtIPs)
ipListJSON, _ := json.Marshal(finalIPs)
nextSyncAt := now.Add(time.Duration(normalizeWAFIPGroupSyncInterval(group.SyncIntervalMinutes)) * time.Minute)
group.IPList = string(ipListJSON)
group.ExtIPs = string(extIPsJSON)
group.LastSyncedAt = &now
group.NextSyncAt = &nextSyncAt
group.LastSyncStatus = "success"
group.LastSyncMessage = fmt.Sprintf("自动规则执行成功,共命中 %d 个 IP", len(ips))
group.LastSyncMessage = fmt.Sprintf("自动规则执行成功,共命中 %d 个 IP,当前生效 %d 个 IP", len(ips), len(finalIPs))
if err := group.UpdateSyncResult(); err != nil {
return nil, err
}
@@ -410,7 +487,7 @@ func syncWAFIPGroupAutomatic(group *model.WAFIPGroup, now time.Time) (*WAFIPGrou
}
return &WAFIPGroupSyncResult{
Group: *view,
IPCount: len(ips),
IPCount: len(finalIPs),
SyncedAt: now.Format(time.RFC3339),
NextSyncAt: nextSyncAt.Format(time.RFC3339),
Status: group.LastSyncStatus,
@@ -470,6 +547,9 @@ func parseWAFIPGroupAutoConfig(raw json.RawMessage) (wafIPGroupAutoConfig, error
if config.LookbackMinutes > 43200 {
config.LookbackMinutes = 43200
}
if config.TTL == 0 {
config.TTL = -1
}
if config.Rules == nil {
config.Rules = []wafIPGroupAutoRule{}
}
+77
View File
@@ -360,6 +360,83 @@ func TestPublishConfigVersionExpandsWAFIPGroupReferences(t *testing.T) {
}
}
func TestWAFIPGroupAutomaticTTLExpiration(t *testing.T) {
setupServiceTestDB(t)
now := time.Now().UTC()
// Seed access logs at now
seedWAFNodeAccessLogs(t, now, "203.0.113.10", "app.example.com", 120, 100)
group, err := CreateWAFIPGroup(WAFIPGroupInput{
Name: "auto ttl blacklist",
Type: WAFIPGroupTypeAutomatic,
Enabled: true,
AutoConfig: json.RawMessage(`{
"lookback_minutes": 60,
"ttl": 10,
"rules": [
{"name":"404 Scan","expr":"request_count > 100 && status_404_ratio >= 0.8"}
]
}`),
})
if err != nil {
t.Fatalf("CreateWAFIPGroup failed: %v", err)
}
groupModel, err := model.GetWAFIPGroupByID(group.ID)
if err != nil {
t.Fatalf("GetWAFIPGroupByID failed: %v", err)
}
// First Sync (at now): should match 203.0.113.10
res1, err := syncWAFIPGroup(groupModel, now)
if err != nil {
t.Fatalf("First Sync failed: %v", err)
}
if res1.IPCount != 1 || res1.Group.IPList[0] != "203.0.113.10" {
t.Fatalf("expected 203.0.113.10 to be blacklisted, got: %#v", res1.Group.IPList)
}
if len(res1.Group.ExtIPs) != 1 || res1.Group.ExtIPs[0].IP != "203.0.113.10" {
t.Fatalf("expected 203.0.113.10 to be in ExtIPs, got: %#v", res1.Group.ExtIPs)
}
// Second Sync (65 minutes later):
// Since 65 minutes is outside the 60 minutes lookback window, the original logs won't match.
// And since 65 minutes > 10s TTL, it should be expired and removed!
futureTime := now.Add(65 * time.Minute)
res2, err := syncWAFIPGroup(groupModel, futureTime)
if err != nil {
t.Fatalf("Second Sync failed: %v", err)
}
if res2.IPCount != 0 {
t.Fatalf("expected IP to be expired and removed, got: %#v", res2.Group.IPList)
}
if len(res2.Group.ExtIPs) != 0 {
t.Fatalf("expected ExtIPs to be empty after expiration, got: %#v", res2.Group.ExtIPs)
}
// Third Sync: test lease refresh / extension!
// Re-run sync at now to get it captured again first
_, err = syncWAFIPGroup(groupModel, now)
if err != nil {
t.Fatalf("Re-sync at now failed: %v", err)
}
// Now run sync at now + 5 seconds (5s < 10s TTL, so not expired, but matched again!):
// Since it matches again, it should keep the IP active and extend CapturedAt to now + 5s!
futureTime2 := now.Add(5 * time.Second)
res3, err := syncWAFIPGroup(groupModel, futureTime2)
if err != nil {
t.Fatalf("Third Sync failed: %v", err)
}
if res3.IPCount != 1 || res3.Group.IPList[0] != "203.0.113.10" {
t.Fatalf("expected IP to remain active, got: %#v", res3.Group.IPList)
}
if len(res3.Group.ExtIPs) != 1 || res3.Group.ExtIPs[0].CapturedAt != futureTime2.Format(time.RFC3339) {
t.Fatalf("expected CapturedAt to be updated to %v, got %v", futureTime2.Format(time.RFC3339), res3.Group.ExtIPs[0].CapturedAt)
}
}
func seedWAFNodeAccessLogs(t *testing.T, loggedAt time.Time, remoteAddr string, host string, total int, notFound int) {
t.Helper()
for i := 0; i < total; i++ {
@@ -1,7 +1,7 @@
'use client';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { ArrowLeft, Download, Play, Plus, Save, Trash2 } from 'lucide-react';
import { ArrowLeft, Download, Play, Plus, Save, Trash2, Eye } from 'lucide-react';
import { useRouter } from 'next/navigation';
import { useEffect, useMemo, useState } from 'react';
@@ -11,6 +11,7 @@ import { InlineMessage } from '@/components/feedback/inline-message';
import { LoadingState } from '@/components/feedback/loading-state';
import { PageHeader } from '@/components/layout/page-header';
import { AppCard } from '@/components/ui/app-card';
import { AppModal } from '@/components/ui/app-modal';
import {
createWAFIPGroup,
deleteWAFIPGroup,
@@ -169,6 +170,7 @@ export function WAFIPGroupsPage() {
const [feedback, setFeedback] = useState<FeedbackState | null>(null);
const [autoTestResult, setAutoTestResult] =
useState<WAFIPGroupAutoTestResult | null>(null);
const [isCapturedIPsModalOpen, setIsCapturedIPsModalOpen] = useState(false);
const groupsQuery = useQuery({
queryKey: ['waf', 'ip-groups'],
@@ -394,6 +396,15 @@ export function WAFIPGroupsPage() {
: '立即同步'}
</SecondaryButton>
) : null}
{selectedGroup?.type === 'automatic' ? (
<SecondaryButton
type="button"
onClick={() => setIsCapturedIPsModalOpen(true)}
>
<Eye className="mr-2 h-4 w-4" />
查看已抓取 IP
</SecondaryButton>
) : null}
<PrimaryButton
type="button"
disabled={saveMutation.isPending}
@@ -519,6 +530,24 @@ export function WAFIPGroupsPage() {
{draft.type === 'automatic' ? (
<div className="space-y-4">
<div className="grid gap-5 xl:grid-cols-2">
<ResourceField
label="同步间隔(分钟)"
hint="定时从请求日志挖掘恶意 IP 的周期。最小 5 分钟,默认 1440 分钟。"
>
<ResourceInput
type="number"
min={5}
value={draft.sync_interval_minutes}
onChange={(event) =>
setDraft((current) => ({
...current,
sync_interval_minutes: Number(event.target.value),
}))
}
/>
</ResourceField>
</div>
<ResourceField
label="预设规则"
hint="表达式按单个 IP 的请求日志聚合指标计算。"
@@ -554,7 +583,7 @@ export function WAFIPGroupsPage() {
</ResourceField>
<ResourceField
label="自动配置 JSON"
hint="可用字段:request_count、status_404_count、status_404_ratio、ip_host_count、ip_host_ratio。"
hint="可用字段:request_count、status_404_count、status_404_ratio、ip_host_count、ip_host_ratio。支持 ttl(秒,默认 -1 永久拉黑)。"
>
<ResourceTextarea
value={draft.auto_config_text}
@@ -652,6 +681,79 @@ export function WAFIPGroupsPage() {
</div>
</AppCard>
</div>
{selectedGroup?.type === 'automatic' ? (
<AppModal
isOpen={isCapturedIPsModalOpen}
title={`已抓取 IP 列表 - ${selectedGroup.name}`}
description="展示当前自动挖掘并放入黑名单的 IP 列表及其到期状态。"
size="lg"
onClose={() => setIsCapturedIPsModalOpen(false)}
footer={
<div className="flex justify-end">
<SecondaryButton type="button" onClick={() => setIsCapturedIPsModalOpen(false)}>
关闭
</SecondaryButton>
</div>
}
>
<div className="space-y-4">
{!selectedGroup.ext_ips || selectedGroup.ext_ips.length === 0 ? (
<EmptyState
title="暂无抓取记录"
description="该自动规则暂未抓取或命中任何恶意 IP,点击「立即执行」手动触发抓取。"
/>
) : (
<div className="overflow-hidden rounded-2xl border border-[var(--border-default)]">
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-[var(--border-default)]">
<thead className="bg-[var(--surface-muted)]">
<tr>
<th className="px-4 py-3 text-left text-xs font-semibold text-[var(--foreground-secondary)]">IP 地址</th>
<th className="px-4 py-3 text-left text-xs font-semibold text-[var(--foreground-secondary)]">抓取时间</th>
<th className="px-4 py-3 text-left text-xs font-semibold text-[var(--foreground-secondary)]">封禁剩余时间</th>
</tr>
</thead>
<tbody className="divide-y divide-[var(--border-default)] bg-[var(--surface-elevated)]">
{selectedGroup.ext_ips.map((item) => {
const autoConfig = selectedGroup.auto_config as { ttl?: number } | undefined;
const ttl = autoConfig?.ttl ?? -1;
let expireText = '永久';
if (ttl > 0) {
const capturedDate = new Date(item.captured_at);
const expireDate = new Date(capturedDate.getTime() + ttl * 1000);
const now = new Date();
if (expireDate.getTime() <= now.getTime()) {
expireText = '已过期';
} else {
const diffMs = expireDate.getTime() - now.getTime();
const diffMins = Math.round(diffMs / (60 * 1000));
if (diffMins < 60) {
expireText = `${diffMins} 分钟后`;
} else {
const diffHours = Math.round(diffMins / 60);
expireText = `${diffHours} 小时后`;
}
}
}
return (
<tr key={item.ip}>
<td className="whitespace-nowrap px-4 py-3 text-sm font-medium text-[var(--foreground-primary)] font-mono">{item.ip}</td>
<td className="whitespace-nowrap px-4 py-3 text-sm text-[var(--foreground-secondary)]">{new Date(item.captured_at).toLocaleString()}</td>
<td className={cn("whitespace-nowrap px-4 py-3 text-sm font-medium", ttl > 0 ? "text-amber-500" : "text-emerald-500")}>{expireText}</td>
</tr>
);
})}
</tbody>
</table>
</div>
</div>
)}
<div className="rounded-2xl bg-[var(--surface-muted)] p-4 text-xs text-[var(--foreground-secondary)] leading-relaxed">
💡 提示:抓取记录持久化存储在 IP 组的 <code>ext_ips</code> 扩展字段中。未来如需为已抓取到的 IP 记录更多维度的扩展元数据,可直接在此 JSON 对象中添加字段以供扩展。
</div>
</div>
</AppModal>
) : null}
</div>
);
}
@@ -53,6 +53,11 @@ export interface WAFSiteRuleGroups {
export type WAFIPGroupType = 'manual' | 'automatic' | 'subscription';
export type WAFIPGroupSubscriptionFormat = 'text' | 'json';
export interface WAFIPGroupExtIP {
ip: string;
captured_at: string;
}
export interface WAFIPGroup {
id: number;
name: string;
@@ -60,6 +65,7 @@ export interface WAFIPGroup {
enabled: boolean;
ip_list: string[];
auto_config: Record<string, unknown>;
ext_ips?: WAFIPGroupExtIP[];
subscription_url: string;
subscription_format: WAFIPGroupSubscriptionFormat;
subscription_mapping_rule: string;