tcjlj/OpenFlare
1
0
Fork 0
mirror of https://github.com/Rain-kl/OpenFlare.git synced 2026-10-05 07:26:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

[新增] 添加自动 IP 组规则的抓取记录功能，支持查看已抓取 IP 列表及其到期状态

Browse Source
This commit is contained in:
ryan
2026-06-01 10:30:00 +08:00
parent 2dc7e72621
commit bd7f008322
7 changed files with 352 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openflare_server/service/waf_ip_group.go
+102 -22
View File
@@ -37,9 +37,20 @@ const (
type wafIPGroupAutoConfig struct {
LookbackMinutes int `json:"lookback_minutes"`
TTL int `json:"ttl"` // in seconds, default -1 (permanent)
Rules []wafIPGroupAutoRule `json:"rules"`
}
type WAFIPGroupExtIP struct {
IP string `json:"ip"`
CapturedAt time.Time `json:"captured_at"`
}
type WAFIPGroupExtIPView struct {
IP string `json:"ip"`
CapturedAt string `json:"captured_at"`
}
type wafIPGroupAutoRule struct {
Name string `json:"name"`
Expr string `json:"expr"`
@@ -81,24 +92,25 @@ type WAFIPGroupInput struct {
}
type WAFIPGroupView struct {
ID uint `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Enabled bool `json:"enabled"`
IPList []string `json:"ip_list"`
AutoConfig json.RawMessage `json:"auto_config"`
SubscriptionURL string `json:"subscription_url"`
SubscriptionFormat string `json:"subscription_format"`
SubscriptionMappingRule string `json:"subscription_mapping_rule"`
SyncIntervalMinutes int `json:"sync_interval_minutes"`
LastSyncedAt string `json:"last_synced_at,omitempty"`
NextSyncAt string `json:"next_sync_at,omitempty"`
LastSyncStatus string `json:"last_sync_status"`
LastSyncMessage string `json:"last_sync_message"`
Remark string `json:"remark"`
ReferencedByRuleCount int `json:"referenced_by_rule_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ID uint `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Enabled bool `json:"enabled"`
IPList []string `json:"ip_list"`
AutoConfig json.RawMessage `json:"auto_config"`
ExtIPs []WAFIPGroupExtIPView `json:"ext_ips"`
SubscriptionURL string `json:"subscription_url"`
SubscriptionFormat string `json:"subscription_format"`
SubscriptionMappingRule string `json:"subscription_mapping_rule"`
SyncIntervalMinutes int `json:"sync_interval_minutes"`
LastSyncedAt string `json:"last_synced_at,omitempty"`
NextSyncAt string `json:"next_sync_at,omitempty"`
LastSyncStatus string `json:"last_sync_status"`
LastSyncMessage string `json:"last_sync_message"`
Remark string `json:"remark"`
ReferencedByRuleCount int `json:"referenced_by_rule_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type WAFIPGroupSyncResult struct {
@@ -285,6 +297,7 @@ func buildWAFIPGroup(group *model.WAFIPGroup, input WAFIPGroupInput) (*model.WAF
ipListJSON, _ := json.Marshal(normalizedIPs)
if group == nil {
group = &model.WAFIPGroup{}
group.ExtIPs = "[]"
}
group.Name = name
group.Type = groupType
@@ -312,6 +325,17 @@ func buildWAFIPGroupView(group *model.WAFIPGroup, referenceCount int) (WAFIPGrou
if len(autoConfig) == 0 {
autoConfig = json.RawMessage("{}")
}
var extIPs []WAFIPGroupExtIP
if group.ExtIPs != "" && group.ExtIPs != "[]" {
_ = json.Unmarshal([]byte(group.ExtIPs), &extIPs)
}
viewExtIPs := make([]WAFIPGroupExtIPView, 0, len(extIPs))
for _, extIP := range extIPs {
viewExtIPs = append(viewExtIPs, WAFIPGroupExtIPView{
IP: extIP.IP,
CapturedAt: extIP.CapturedAt.Format(time.RFC3339),
})
}
view := WAFIPGroupView{
ID: group.ID,
Name: group.Name,
@@ -319,6 +343,7 @@ func buildWAFIPGroupView(group *model.WAFIPGroup, referenceCount int) (WAFIPGrou
Enabled: group.Enabled,
IPList: ips,
AutoConfig: autoConfig,
ExtIPs: viewExtIPs,
SubscriptionURL: group.SubscriptionURL,
SubscriptionFormat: group.SubscriptionFormat,
SubscriptionMappingRule: group.SubscriptionMappingRule,
@@ -389,18 +414,70 @@ func syncWAFIPGroupSubscription(group *model.WAFIPGroup, now time.Time) (*WAFIPG
}
func syncWAFIPGroupAutomatic(group *model.WAFIPGroup, now time.Time) (*WAFIPGroupSyncResult, error) {
ips, err := evaluateWAFIPGroupAutoConfig(group.AutoConfig, now)
config, err := parseWAFIPGroupAutoConfig(json.RawMessage(group.AutoConfig))
if err != nil {
recordWAFIPGroupSyncFailure(group, now, err)
return nil, err
}
ipListJSON, _ := json.Marshal(ips)
var existingExtIPs []WAFIPGroupExtIP
if group.ExtIPs != "" && group.ExtIPs != "[]" {
_ = json.Unmarshal([]byte(group.ExtIPs), &existingExtIPs)
}
activeExtIPs := make([]WAFIPGroupExtIP, 0, len(existingExtIPs))
for _, extIP := range existingExtIPs {
if config.TTL > 0 {
expirationTime := extIP.CapturedAt.Add(time.Duration(config.TTL) * time.Second)
if expirationTime.Before(now) {
continue
}
}
activeExtIPs = append(activeExtIPs, extIP)
}
ips, err := evaluateParsedWAFIPGroupAutoConfig(config, now)
if err != nil {
recordWAFIPGroupSyncFailure(group, now, err)
return nil, err
}
extIPMap := make(map[string]int)
for idx, extIP := range activeExtIPs {
extIPMap[extIP.IP] = idx
}
for _, ip := range ips {
if idx, ok := extIPMap[ip]; ok {
activeExtIPs[idx].CapturedAt = now
} else {
activeExtIPs = append(activeExtIPs, WAFIPGroupExtIP{
IP: ip,
CapturedAt: now,
})
}
}
finalIPs := make([]string, 0, len(activeExtIPs))
for _, extIP := range activeExtIPs {
finalIPs = append(finalIPs, extIP.IP)
}
finalIPs, err = normalizeWAFIPList(finalIPs)
if err != nil {
recordWAFIPGroupSyncFailure(group, now, err)
return nil, err
}
extIPsJSON, _ := json.Marshal(activeExtIPs)
ipListJSON, _ := json.Marshal(finalIPs)
nextSyncAt := now.Add(time.Duration(normalizeWAFIPGroupSyncInterval(group.SyncIntervalMinutes)) * time.Minute)
group.IPList = string(ipListJSON)
group.ExtIPs = string(extIPsJSON)
group.LastSyncedAt = &now
group.NextSyncAt = &nextSyncAt
group.LastSyncStatus = "success"
group.LastSyncMessage = fmt.Sprintf("自动规则执行成功，共命中 %d 个 IP", len(ips))
group.LastSyncMessage = fmt.Sprintf("自动规则执行成功，共命中 %d 个 IP，当前生效 %d 个 IP", len(ips), len(finalIPs))
if err := group.UpdateSyncResult(); err != nil {
return nil, err
}
@@ -410,7 +487,7 @@ func syncWAFIPGroupAutomatic(group *model.WAFIPGroup, now time.Time) (*WAFIPGrou
}
return &WAFIPGroupSyncResult{
Group: *view,
IPCount: len(ips),
IPCount: len(finalIPs),
SyncedAt: now.Format(time.RFC3339),
NextSyncAt: nextSyncAt.Format(time.RFC3339),
Status: group.LastSyncStatus,
@@ -470,6 +547,9 @@ func parseWAFIPGroupAutoConfig(raw json.RawMessage) (wafIPGroupAutoConfig, error
if config.LookbackMinutes > 43200 {
config.LookbackMinutes = 43200
}
if config.TTL == 0 {
config.TTL = -1
}
if config.Rules == nil {
config.Rules = []wafIPGroupAutoRule{}
}
openflare_server/service/waf_test.go
+77
View File
@@ -360,6 +360,83 @@ func TestPublishConfigVersionExpandsWAFIPGroupReferences(t *testing.T) {
}
}
func TestWAFIPGroupAutomaticTTLExpiration(t *testing.T) {
setupServiceTestDB(t)
now := time.Now().UTC()
// Seed access logs at now
seedWAFNodeAccessLogs(t, now, "203.0.113.10", "app.example.com", 120, 100)
group, err := CreateWAFIPGroup(WAFIPGroupInput{
Name: "auto ttl blacklist",
Type: WAFIPGroupTypeAutomatic,
Enabled: true,
AutoConfig: json.RawMessage(`{
"lookback_minutes": 60,
"ttl": 10,
"rules": [
{"name":"404 Scan","expr":"request_count > 100 && status_404_ratio >= 0.8"}
]
}`),
})
if err != nil {
t.Fatalf("CreateWAFIPGroup failed: %v", err)
}
groupModel, err := model.GetWAFIPGroupByID(group.ID)
if err != nil {
t.Fatalf("GetWAFIPGroupByID failed: %v", err)
}
// First Sync (at now): should match 203.0.113.10
res1, err := syncWAFIPGroup(groupModel, now)
if err != nil {
t.Fatalf("First Sync failed: %v", err)
}
if res1.IPCount != 1 || res1.Group.IPList[0] != "203.0.113.10" {
t.Fatalf("expected 203.0.113.10 to be blacklisted, got: %#v", res1.Group.IPList)
}
if len(res1.Group.ExtIPs) != 1 || res1.Group.ExtIPs[0].IP != "203.0.113.10" {
t.Fatalf("expected 203.0.113.10 to be in ExtIPs, got: %#v", res1.Group.ExtIPs)
}
// Second Sync (65 minutes later):
// Since 65 minutes is outside the 60 minutes lookback window, the original logs won't match.
// And since 65 minutes > 10s TTL, it should be expired and removed!
futureTime := now.Add(65 * time.Minute)
res2, err := syncWAFIPGroup(groupModel, futureTime)
if err != nil {
t.Fatalf("Second Sync failed: %v", err)
}
if res2.IPCount != 0 {
t.Fatalf("expected IP to be expired and removed, got: %#v", res2.Group.IPList)
}
if len(res2.Group.ExtIPs) != 0 {
t.Fatalf("expected ExtIPs to be empty after expiration, got: %#v", res2.Group.ExtIPs)
}
// Third Sync: test lease refresh / extension!
// Re-run sync at now to get it captured again first
_, err = syncWAFIPGroup(groupModel, now)
if err != nil {
t.Fatalf("Re-sync at now failed: %v", err)
}
// Now run sync at now + 5 seconds (5s < 10s TTL, so not expired, but matched again!):
// Since it matches again, it should keep the IP active and extend CapturedAt to now + 5s!
futureTime2 := now.Add(5 * time.Second)
res3, err := syncWAFIPGroup(groupModel, futureTime2)
if err != nil {
t.Fatalf("Third Sync failed: %v", err)
}
if res3.IPCount != 1 || res3.Group.IPList[0] != "203.0.113.10" {
t.Fatalf("expected IP to remain active, got: %#v", res3.Group.IPList)
}
if len(res3.Group.ExtIPs) != 1 || res3.Group.ExtIPs[0].CapturedAt != futureTime2.Format(time.RFC3339) {
t.Fatalf("expected CapturedAt to be updated to %v, got %v", futureTime2.Format(time.RFC3339), res3.Group.ExtIPs[0].CapturedAt)
}
}
func seedWAFNodeAccessLogs(t *testing.T, loggedAt time.Time, remoteAddr string, host string, total int, notFound int) {
t.Helper()
for i := 0; i < total; i++ {