diff --git a/docs/changelog/index.md b/docs/changelog/index.md index 0d8c43e7..5293b1e9 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -15,6 +15,12 @@ sidebar: false >3.5.1 版本解耦了日志存储，ClickHouse 变为可选项，如果想切换数据库, 点击 「任务管理」 -> 「切换日志数据库」任务，按提示迁移数据并切换主库。 > +## [Unreleased] + +### 🛠 修复 + +- 修复 IP 组按规则自动抓取时会收录内网、回环及链路本地等地址的问题；规则测试同步过滤这些地址，历史抓取结果会在下次同步时清理。 + ## [v3.5.8] - 2026-10-08 ### ✨ 新功能 diff --git a/docs/guide/waf-ip-group-expr.md b/docs/guide/waf-ip-group-expr.md index 3a4f6573..8edc47b6 100644 --- a/docs/guide/waf-ip-group-expr.md +++ b/docs/guide/waf-ip-group-expr.md @@ -32,11 +32,13 @@ 自动 IP 组先按单个客户端 IP 聚合指标，再对每个 IP 执行规则表达式： 1. Server 读取最近 `lookback` 时长内的请求日志。 -2. 按 `remote_addr` 归一化后的 IP 分组。 +2. 按 `remote_addr` 归一化后的 IP 分组，跳过 IPv4 私网、IPv6 唯一本地地址（ULA）、回环、链路本地、未指定、组播及广播地址；IPv4 映射 IPv6 地址按对应的 IPv4 地址判断。 3. 为每个 IP 计算请求数、404 数、直连 IP Host 次数等指标。 4. 逐个 IP 执行 `rules[].expr`。 5. 只要某个 IP 命中任意规则，就写入该自动 IP 组的 `IP / IP 段` 列表。 +**测试规则**与实际自动抓取使用相同的地址过滤逻辑。自动 IP 组下次同步时，也会清理历史抓取结果中的上述地址，不受存活时间（TTL）限制。手动 IP 组和订阅 IP 组不受此过滤影响。 + Host 是否为“通过 IP 访问”按请求日志中的 `Host` 字段判断：如果 Host 是 IPv4 或 IPv6 字面量，例如 `203.0.113.10`、`[2001:db8::10]`、`203.0.113.10:443`，就计入 `ip_host_count`。 ## 可用关键字 diff --git a/internal/apps/openflare/waf/ip_group_sync.go b/internal/apps/openflare/waf/ip_group_sync.go index 5738b6b4..53d55a14 100644 --- a/internal/apps/openflare/waf/ip_group_sync.go +++ b/internal/apps/openflare/waf/ip_group_sync.go @@ -19,6 +19,7 @@ import ( "time" "github.com/Rain-kl/Wavelet/internal/repository" + "github.com/Rain-kl/Wavelet/pkg/geoip/iputil" "github.com/Rain-kl/Wavelet/internal/apps/openflare/agent" "github.com/Rain-kl/Wavelet/internal/apps/openflare/websocket" @@ -211,6 +212,9 @@ func syncIPGroupAutomatic(ctx context.Context, group *model.OpenFlareWAFIPGroup, activeExtIPs := make([]ipGroupExtIP, 0, len(existingExtIPs)) for _, extIP := range existingExtIPs { + if !iputil.IsPublicString(extIP.IP) { + continue + } if config.TTL > 0 { expirationTime := extIP.CapturedAt.Add(time.Duration(config.TTL) * time.Second) if expirationTime.Before(now) { @@ -318,7 +322,7 @@ func evaluateParsedIPGroupAutoConfig(ctx context.Context, config ipGroupAutoConf continue } ip, ok := normalizeIPLiteral(item.RemoteAddr) - if !ok { + if !ok || !iputil.IsPublicString(ip) { continue } lastSeen := time.Time{} diff --git a/internal/apps/openflare/waf/ip_group_sync_test.go b/internal/apps/openflare/waf/ip_group_sync_test.go index a01544d2..0d2ee9cd 100644 --- a/internal/apps/openflare/waf/ip_group_sync_test.go +++ b/internal/apps/openflare/waf/ip_group_sync_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "slices" "testing" "time" @@ -151,6 +152,79 @@ func TestTestIPGroupAutoConfigReturnsMatchedIPs(t *testing.T) { assert.Empty(t, want) } +func TestIPGroupAutomaticExcludesInternalIPs(t *testing.T) { + t.Cleanup(setupIPGroupSyncTestDB(t)) + ctx := context.Background() + now := time.Now().UTC() + internalIPs := []string{ + "10.0.0.1", "172.16.0.1", "172.31.255.254", "192.168.1.1", + "127.0.0.1", "169.254.1.1", "0.0.0.0", "224.0.0.1", "255.255.255.255", + "fc00::1", "fd12:3456::1", "fe80::1", "::1", "::", "ff02::1", + "::ffff:192.168.1.1", "::ffff:127.0.0.1", + } + for _, ip := range internalIPs { + seedWAFAccessLogs(t, ctx, now, ip, "app.example.com", 1, 0) + } + seedWAFAccessLogs(t, ctx, now, "192.168.1.2:1234", "app.example.com", 1, 0) + seedWAFAccessLogs(t, ctx, now, "[fd00::2]:1234", "app.example.com", 1, 0) + publicIPs := []string{"1.1.1.1", "172.32.0.1", "2606:4700:4700::1111"} + for _, ip := range publicIPs { + seedWAFAccessLogs(t, ctx, now, ip, "app.example.com", 1, 0) + } + + autoConfig := json.RawMessage(`{"lookback":"1h","ttl":-1,"rules":[{"expr":"request_count > 0"}]}`) + preview, err := TestIPGroupAutoConfig(ctx, IPGroupAutoTestInput{AutoConfig: autoConfig}) + if err != nil { + t.Fatalf("TestIPGroupAutoConfig(%s) error = %v", autoConfig, err) + } + if !slices.Equal(preview.MatchedIPs, publicIPs) || preview.MatchedCount != len(publicIPs) { + t.Errorf("TestIPGroupAutoConfig(%s) matched = %v (count %d), want %v", autoConfig, preview.MatchedIPs, preview.MatchedCount, publicIPs) + } + + view, err := CreateIPGroup(ctx, IPGroupInput{ + Name: "automatic public IPs", Type: wafIPGroupTypeAutomatic, Enabled: true, AutoConfig: autoConfig, + }) + if err != nil { + t.Fatalf("CreateIPGroup(automatic public IPs) error = %v", err) + } + group, err := repository.GetOpenFlareWAFIPGroupByID(ctx, view.ID) + if err != nil { + t.Fatalf("GetOpenFlareWAFIPGroupByID(%d) error = %v", view.ID, err) + } + capturedAt := now.Add(-24 * time.Hour) + extIPs := make([]ipGroupExtIP, 0, len(internalIPs)+1) + for _, ip := range internalIPs { + extIPs = append(extIPs, ipGroupExtIP{IP: ip, CapturedAt: capturedAt}) + } + extIPs = append(extIPs, ipGroupExtIP{IP: "8.8.8.8", CapturedAt: capturedAt}) + extJSON, err := json.Marshal(extIPs) + if err != nil { + t.Fatalf("json.Marshal(extIPs) error = %v", err) + } + group.ExtIPs = string(extJSON) + if err := repository.UpdateOpenFlareWAFIPGroup(ctx, group); err != nil { + t.Fatalf("UpdateOpenFlareWAFIPGroup(%d) error = %v", group.ID, err) + } + + result, err := SyncIPGroup(ctx, group.ID) + if err != nil { + t.Fatalf("SyncIPGroup(%d) error = %v", group.ID, err) + } + wantIPs := append(slices.Clone(publicIPs), "8.8.8.8") + slices.Sort(wantIPs) + if !slices.Equal(result.Group.IPList, wantIPs) || result.IPCount != len(wantIPs) { + t.Errorf("SyncIPGroup(%d) IPs = %v (count %d), want %v", group.ID, result.Group.IPList, result.IPCount, wantIPs) + } + gotExtIPs := make([]string, 0, len(result.Group.ExtIPs)) + for _, item := range result.Group.ExtIPs { + gotExtIPs = append(gotExtIPs, item.IP) + } + slices.Sort(gotExtIPs) + if !slices.Equal(gotExtIPs, wantIPs) { + t.Errorf("SyncIPGroup(%d) ExtIPs = %v, want %v", group.ID, gotExtIPs, wantIPs) + } +} + func TestListDueOpenFlareWAFIPGroups(t *testing.T) { cleanup := setupIPGroupSyncTestDB(t) defer cleanup()