diff --git a/openflare_server/model/migrate/v18.go b/openflare_server/model/migrate/v18.go new file mode 100644 index 00000000..09e45ab6 --- /dev/null +++ b/openflare_server/model/migrate/v18.go @@ -0,0 +1,48 @@ +package migrate + +import ( + "fmt" + + "gorm.io/gorm" +) + +// WAF IP Group schema changes for v18: +// We introduce the `ext_ips` column to keep track of captured IPs with their capture timestamps. +// If more information needs to be saved for captured IPs, it can be added directly inside this JSON structure. +type wafIPGroupV18 struct { + ExtIPs string `gorm:"column:ext_ips;type:text;not null;default:'[]'"` +} + +func init() { + Register(V18()) +} + +func V18() Migration { + return Migration{ + FromVersion: 17, + ToVersion: 18, + Migrate: migrateV18, + Validate: validateV18, + } +} + +func (wafIPGroupV18) TableName() string { + return "waf_ip_groups" +} + +func migrateV18(ctx Context, db *gorm.DB, backend string) error { + return ctx.ApplyCurrentSchema(db, backend) +} + +func validateV18(ctx Context, db *gorm.DB, backend string) error { + if err := ctx.ValidateDatabaseSchemaVersion(db, backend, 17); err != nil { + return err + } + if db == nil || !db.Migrator().HasTable(&wafIPGroupV18{}) { + return fmt.Errorf("table waf_ip_groups is missing") + } + if !db.Migrator().HasColumn(&wafIPGroupV18{}, "ext_ips") { + return fmt.Errorf("column waf_ip_groups.ext_ips is missing") + } + return nil +} diff --git a/openflare_server/model/migrations.go b/openflare_server/model/migrations.go index 8cc3c2da..70b2f7cd 100644 --- a/openflare_server/model/migrations.go +++ b/openflare_server/model/migrations.go @@ -80,6 +80,8 @@ func (databaseSchemaMigrationContext) ValidateDatabaseSchemaVersion(db *gorm.DB, return validateDatabaseSchemaV16(db, backend) case 17: return validateDatabaseSchemaV17(db, backend) + case 18: + return validateDatabaseSchemaV18(db, backend) default: return fmt.Errorf("database schema validation for v%d is not defined", version) } @@ -1190,6 +1192,16 @@ func validateDatabaseSchemaV17(db *gorm.DB, backend string) error { return nil } +func validateDatabaseSchemaV18(db *gorm.DB, backend string) error { + if err := validateDatabaseSchemaV17(db, backend); err != nil { + return err + } + if !db.Migrator().HasColumn(&WAFIPGroup{}, "ext_ips") { + return fmt.Errorf("column waf_ip_groups.ext_ips is missing") + } + return nil +} + func databaseSchemaMigrations() []databaseSchemaMigration { ctx := databaseSchemaMigrationContext{} migrations := []databaseSchemaMigration{} diff --git a/openflare_server/model/waf.go b/openflare_server/model/waf.go index 44e84da4..f72bd595 100644 --- a/openflare_server/model/waf.go +++ b/openflare_server/model/waf.go @@ -31,6 +31,7 @@ type WAFIPGroup struct { Enabled bool `json:"enabled" gorm:"not null;default:true"` IPList string `json:"ip_list" gorm:"type:text;not null;default:'[]'"` AutoConfig string `json:"auto_config" gorm:"type:text;not null;default:'{}'"` + ExtIPs string `json:"ext_ips" gorm:"type:text;not null;default:'[]'"` SubscriptionURL string `json:"subscription_url" gorm:"size:2048;not null;default:''"` SubscriptionFormat string `json:"subscription_format" gorm:"size:32;not null;default:'text'"` SubscriptionMappingRule string `json:"subscription_mapping_rule" gorm:"size:255;not null;default:''"` @@ -138,6 +139,7 @@ func (group *WAFIPGroup) Update() error { "enabled": group.Enabled, "ip_list": group.IPList, "auto_config": group.AutoConfig, + "ext_ips": group.ExtIPs, "subscription_url": group.SubscriptionURL, "subscription_format": group.SubscriptionFormat, "subscription_mapping_rule": group.SubscriptionMappingRule, @@ -152,6 +154,7 @@ func (group *WAFIPGroup) Update() error { func (group *WAFIPGroup) UpdateSyncResult() error { return DB.Model(&WAFIPGroup{}).Where("id = ?", group.ID).Updates(map[string]any{ "ip_list": group.IPList, + "ext_ips": group.ExtIPs, "last_synced_at": group.LastSyncedAt, "next_sync_at": group.NextSyncAt, "last_sync_status": group.LastSyncStatus, diff --git a/openflare_server/service/waf_ip_group.go b/openflare_server/service/waf_ip_group.go index f3459f3e..9cd0d447 100644 --- a/openflare_server/service/waf_ip_group.go +++ b/openflare_server/service/waf_ip_group.go @@ -37,9 +37,20 @@ const ( type wafIPGroupAutoConfig struct { LookbackMinutes int `json:"lookback_minutes"` + TTL int `json:"ttl"` // in seconds, default -1 (permanent) Rules []wafIPGroupAutoRule `json:"rules"` } +type WAFIPGroupExtIP struct { + IP string `json:"ip"` + CapturedAt time.Time `json:"captured_at"` +} + +type WAFIPGroupExtIPView struct { + IP string `json:"ip"` + CapturedAt string `json:"captured_at"` +} + type wafIPGroupAutoRule struct { Name string `json:"name"` Expr string `json:"expr"` @@ -81,24 +92,25 @@ type WAFIPGroupInput struct { } type WAFIPGroupView struct { - ID uint `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - Enabled bool `json:"enabled"` - IPList []string `json:"ip_list"` - AutoConfig json.RawMessage `json:"auto_config"` - SubscriptionURL string `json:"subscription_url"` - SubscriptionFormat string `json:"subscription_format"` - SubscriptionMappingRule string `json:"subscription_mapping_rule"` - SyncIntervalMinutes int `json:"sync_interval_minutes"` - LastSyncedAt string `json:"last_synced_at,omitempty"` - NextSyncAt string `json:"next_sync_at,omitempty"` - LastSyncStatus string `json:"last_sync_status"` - LastSyncMessage string `json:"last_sync_message"` - Remark string `json:"remark"` - ReferencedByRuleCount int `json:"referenced_by_rule_count"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` + ID uint `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Enabled bool `json:"enabled"` + IPList []string `json:"ip_list"` + AutoConfig json.RawMessage `json:"auto_config"` + ExtIPs []WAFIPGroupExtIPView `json:"ext_ips"` + SubscriptionURL string `json:"subscription_url"` + SubscriptionFormat string `json:"subscription_format"` + SubscriptionMappingRule string `json:"subscription_mapping_rule"` + SyncIntervalMinutes int `json:"sync_interval_minutes"` + LastSyncedAt string `json:"last_synced_at,omitempty"` + NextSyncAt string `json:"next_sync_at,omitempty"` + LastSyncStatus string `json:"last_sync_status"` + LastSyncMessage string `json:"last_sync_message"` + Remark string `json:"remark"` + ReferencedByRuleCount int `json:"referenced_by_rule_count"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } type WAFIPGroupSyncResult struct { @@ -285,6 +297,7 @@ func buildWAFIPGroup(group *model.WAFIPGroup, input WAFIPGroupInput) (*model.WAF ipListJSON, _ := json.Marshal(normalizedIPs) if group == nil { group = &model.WAFIPGroup{} + group.ExtIPs = "[]" } group.Name = name group.Type = groupType @@ -312,6 +325,17 @@ func buildWAFIPGroupView(group *model.WAFIPGroup, referenceCount int) (WAFIPGrou if len(autoConfig) == 0 { autoConfig = json.RawMessage("{}") } + var extIPs []WAFIPGroupExtIP + if group.ExtIPs != "" && group.ExtIPs != "[]" { + _ = json.Unmarshal([]byte(group.ExtIPs), &extIPs) + } + viewExtIPs := make([]WAFIPGroupExtIPView, 0, len(extIPs)) + for _, extIP := range extIPs { + viewExtIPs = append(viewExtIPs, WAFIPGroupExtIPView{ + IP: extIP.IP, + CapturedAt: extIP.CapturedAt.Format(time.RFC3339), + }) + } view := WAFIPGroupView{ ID: group.ID, Name: group.Name, @@ -319,6 +343,7 @@ func buildWAFIPGroupView(group *model.WAFIPGroup, referenceCount int) (WAFIPGrou Enabled: group.Enabled, IPList: ips, AutoConfig: autoConfig, + ExtIPs: viewExtIPs, SubscriptionURL: group.SubscriptionURL, SubscriptionFormat: group.SubscriptionFormat, SubscriptionMappingRule: group.SubscriptionMappingRule, @@ -389,18 +414,70 @@ func syncWAFIPGroupSubscription(group *model.WAFIPGroup, now time.Time) (*WAFIPG } func syncWAFIPGroupAutomatic(group *model.WAFIPGroup, now time.Time) (*WAFIPGroupSyncResult, error) { - ips, err := evaluateWAFIPGroupAutoConfig(group.AutoConfig, now) + config, err := parseWAFIPGroupAutoConfig(json.RawMessage(group.AutoConfig)) if err != nil { recordWAFIPGroupSyncFailure(group, now, err) return nil, err } - ipListJSON, _ := json.Marshal(ips) + + var existingExtIPs []WAFIPGroupExtIP + if group.ExtIPs != "" && group.ExtIPs != "[]" { + _ = json.Unmarshal([]byte(group.ExtIPs), &existingExtIPs) + } + + activeExtIPs := make([]WAFIPGroupExtIP, 0, len(existingExtIPs)) + for _, extIP := range existingExtIPs { + if config.TTL > 0 { + expirationTime := extIP.CapturedAt.Add(time.Duration(config.TTL) * time.Second) + if expirationTime.Before(now) { + continue + } + } + activeExtIPs = append(activeExtIPs, extIP) + } + + ips, err := evaluateParsedWAFIPGroupAutoConfig(config, now) + if err != nil { + recordWAFIPGroupSyncFailure(group, now, err) + return nil, err + } + + extIPMap := make(map[string]int) + for idx, extIP := range activeExtIPs { + extIPMap[extIP.IP] = idx + } + + for _, ip := range ips { + if idx, ok := extIPMap[ip]; ok { + activeExtIPs[idx].CapturedAt = now + } else { + activeExtIPs = append(activeExtIPs, WAFIPGroupExtIP{ + IP: ip, + CapturedAt: now, + }) + } + } + + finalIPs := make([]string, 0, len(activeExtIPs)) + for _, extIP := range activeExtIPs { + finalIPs = append(finalIPs, extIP.IP) + } + finalIPs, err = normalizeWAFIPList(finalIPs) + if err != nil { + recordWAFIPGroupSyncFailure(group, now, err) + return nil, err + } + + extIPsJSON, _ := json.Marshal(activeExtIPs) + ipListJSON, _ := json.Marshal(finalIPs) + nextSyncAt := now.Add(time.Duration(normalizeWAFIPGroupSyncInterval(group.SyncIntervalMinutes)) * time.Minute) group.IPList = string(ipListJSON) + group.ExtIPs = string(extIPsJSON) group.LastSyncedAt = &now group.NextSyncAt = &nextSyncAt group.LastSyncStatus = "success" - group.LastSyncMessage = fmt.Sprintf("自动规则执行成功，共命中 %d 个 IP", len(ips)) + group.LastSyncMessage = fmt.Sprintf("自动规则执行成功，共命中 %d 个 IP，当前生效 %d 个 IP", len(ips), len(finalIPs)) if err := group.UpdateSyncResult(); err != nil { return nil, err } @@ -410,7 +487,7 @@ func syncWAFIPGroupAutomatic(group *model.WAFIPGroup, now time.Time) (*WAFIPGrou } return &WAFIPGroupSyncResult{ Group: *view, - IPCount: len(ips), + IPCount: len(finalIPs), SyncedAt: now.Format(time.RFC3339), NextSyncAt: nextSyncAt.Format(time.RFC3339), Status: group.LastSyncStatus, @@ -470,6 +547,9 @@ func parseWAFIPGroupAutoConfig(raw json.RawMessage) (wafIPGroupAutoConfig, error if config.LookbackMinutes > 43200 { config.LookbackMinutes = 43200 } + if config.TTL == 0 { + config.TTL = -1 + } if config.Rules == nil { config.Rules = []wafIPGroupAutoRule{} } diff --git a/openflare_server/service/waf_test.go b/openflare_server/service/waf_test.go index 5a8a19b6..88e29340 100644 --- a/openflare_server/service/waf_test.go +++ b/openflare_server/service/waf_test.go @@ -360,6 +360,83 @@ func TestPublishConfigVersionExpandsWAFIPGroupReferences(t *testing.T) { } } +func TestWAFIPGroupAutomaticTTLExpiration(t *testing.T) { + setupServiceTestDB(t) + + now := time.Now().UTC() + // Seed access logs at now + seedWAFNodeAccessLogs(t, now, "203.0.113.10", "app.example.com", 120, 100) + + group, err := CreateWAFIPGroup(WAFIPGroupInput{ + Name: "auto ttl blacklist", + Type: WAFIPGroupTypeAutomatic, + Enabled: true, + AutoConfig: json.RawMessage(`{ + "lookback_minutes": 60, + "ttl": 10, + "rules": [ + {"name":"404 Scan","expr":"request_count > 100 && status_404_ratio >= 0.8"} + ] + }`), + }) + if err != nil { + t.Fatalf("CreateWAFIPGroup failed: %v", err) + } + + groupModel, err := model.GetWAFIPGroupByID(group.ID) + if err != nil { + t.Fatalf("GetWAFIPGroupByID failed: %v", err) + } + + // First Sync (at now): should match 203.0.113.10 + res1, err := syncWAFIPGroup(groupModel, now) + if err != nil { + t.Fatalf("First Sync failed: %v", err) + } + if res1.IPCount != 1 || res1.Group.IPList[0] != "203.0.113.10" { + t.Fatalf("expected 203.0.113.10 to be blacklisted, got: %#v", res1.Group.IPList) + } + if len(res1.Group.ExtIPs) != 1 || res1.Group.ExtIPs[0].IP != "203.0.113.10" { + t.Fatalf("expected 203.0.113.10 to be in ExtIPs, got: %#v", res1.Group.ExtIPs) + } + + // Second Sync (65 minutes later): + // Since 65 minutes is outside the 60 minutes lookback window, the original logs won't match. + // And since 65 minutes > 10s TTL, it should be expired and removed! + futureTime := now.Add(65 * time.Minute) + res2, err := syncWAFIPGroup(groupModel, futureTime) + if err != nil { + t.Fatalf("Second Sync failed: %v", err) + } + if res2.IPCount != 0 { + t.Fatalf("expected IP to be expired and removed, got: %#v", res2.Group.IPList) + } + if len(res2.Group.ExtIPs) != 0 { + t.Fatalf("expected ExtIPs to be empty after expiration, got: %#v", res2.Group.ExtIPs) + } + + // Third Sync: test lease refresh / extension! + // Re-run sync at now to get it captured again first + _, err = syncWAFIPGroup(groupModel, now) + if err != nil { + t.Fatalf("Re-sync at now failed: %v", err) + } + + // Now run sync at now + 5 seconds (5s < 10s TTL, so not expired, but matched again!): + // Since it matches again, it should keep the IP active and extend CapturedAt to now + 5s! + futureTime2 := now.Add(5 * time.Second) + res3, err := syncWAFIPGroup(groupModel, futureTime2) + if err != nil { + t.Fatalf("Third Sync failed: %v", err) + } + if res3.IPCount != 1 || res3.Group.IPList[0] != "203.0.113.10" { + t.Fatalf("expected IP to remain active, got: %#v", res3.Group.IPList) + } + if len(res3.Group.ExtIPs) != 1 || res3.Group.ExtIPs[0].CapturedAt != futureTime2.Format(time.RFC3339) { + t.Fatalf("expected CapturedAt to be updated to %v, got %v", futureTime2.Format(time.RFC3339), res3.Group.ExtIPs[0].CapturedAt) + } +} + func seedWAFNodeAccessLogs(t *testing.T, loggedAt time.Time, remoteAddr string, host string, total int, notFound int) { t.Helper() for i := 0; i < total; i++ { diff --git a/openflare_server/web/features/waf/components/ip-groups-page.tsx b/openflare_server/web/features/waf/components/ip-groups-page.tsx index d2c63b13..a25a71d8 100644 --- a/openflare_server/web/features/waf/components/ip-groups-page.tsx +++ b/openflare_server/web/features/waf/components/ip-groups-page.tsx @@ -1,7 +1,7 @@ 'use client'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; -import { ArrowLeft, Download, Play, Plus, Save, Trash2 } from 'lucide-react'; +import { ArrowLeft, Download, Play, Plus, Save, Trash2, Eye } from 'lucide-react'; import { useRouter } from 'next/navigation'; import { useEffect, useMemo, useState } from 'react'; @@ -11,6 +11,7 @@ import { InlineMessage } from '@/components/feedback/inline-message'; import { LoadingState } from '@/components/feedback/loading-state'; import { PageHeader } from '@/components/layout/page-header'; import { AppCard } from '@/components/ui/app-card'; +import { AppModal } from '@/components/ui/app-modal'; import { createWAFIPGroup, deleteWAFIPGroup, @@ -169,6 +170,7 @@ export function WAFIPGroupsPage() { const [feedback, setFeedback] = useState(null); const [autoTestResult, setAutoTestResult] = useState(null); + const [isCapturedIPsModalOpen, setIsCapturedIPsModalOpen] = useState(false); const groupsQuery = useQuery({ queryKey: ['waf', 'ip-groups'], @@ -394,6 +396,15 @@ export function WAFIPGroupsPage() { : '立即同步'} ) : null} + {selectedGroup?.type === 'automatic' ? ( + setIsCapturedIPsModalOpen(true)} + > + + 查看已抓取 IP + + ) : null} +
+ + + setDraft((current) => ({ + ...current, + sync_interval_minutes: Number(event.target.value), + })) + } + /> + +
+ {selectedGroup?.type === 'automatic' ? ( + setIsCapturedIPsModalOpen(false)} + footer={ +
+ setIsCapturedIPsModalOpen(false)}> + 关闭 + +
+ } + > +
+ {!selectedGroup.ext_ips || selectedGroup.ext_ips.length === 0 ? ( + + ) : ( +
+
+ + + + + + + + + + {selectedGroup.ext_ips.map((item) => { + const autoConfig = selectedGroup.auto_config as { ttl?: number } | undefined; + const ttl = autoConfig?.ttl ?? -1; + let expireText = '永久'; + if (ttl > 0) { + const capturedDate = new Date(item.captured_at); + const expireDate = new Date(capturedDate.getTime() + ttl * 1000); + const now = new Date(); + if (expireDate.getTime() <= now.getTime()) { + expireText = '已过期'; + } else { + const diffMs = expireDate.getTime() - now.getTime(); + const diffMins = Math.round(diffMs / (60 * 1000)); + if (diffMins < 60) { + expireText = `${diffMins} 分钟后`; + } else { + const diffHours = Math.round(diffMins / 60); + expireText = `${diffHours} 小时后`; + } + } + } + return ( + + + + + + ); + })} + +
IP 地址抓取时间封禁剩余时间
{item.ip}{new Date(item.captured_at).toLocaleString()} 0 ? "text-amber-500" : "text-emerald-500")}>{expireText}
+
+
+ )} +
+ 💡 提示：抓取记录持久化存储在 IP 组的 ext_ips 扩展字段中。未来如需为已抓取到的 IP 记录更多维度的扩展元数据，可直接在此 JSON 对象中添加字段以供扩展。 +
+
+ + ) : null} ); } diff --git a/openflare_server/web/features/waf/types.ts b/openflare_server/web/features/waf/types.ts index 1f121914..6c033353 100644 --- a/openflare_server/web/features/waf/types.ts +++ b/openflare_server/web/features/waf/types.ts @@ -53,6 +53,11 @@ export interface WAFSiteRuleGroups { export type WAFIPGroupType = 'manual' | 'automatic' | 'subscription'; export type WAFIPGroupSubscriptionFormat = 'text' | 'json'; +export interface WAFIPGroupExtIP { + ip: string; + captured_at: string; +} + export interface WAFIPGroup { id: number; name: string; @@ -60,6 +65,7 @@ export interface WAFIPGroup { enabled: boolean; ip_list: string[]; auto_config: Record; + ext_ips?: WAFIPGroupExtIP[]; subscription_url: string; subscription_format: WAFIPGroupSubscriptionFormat; subscription_mapping_rule: string;