feat(waf): 自动 IP 组 lookback 支持 60m/1h 时长写法

将 lookback_minutes 替换为 lookback,移除最小 5 分钟回看限制,并兼容旧字段。
This commit is contained in:
ryan
2026-07-20 15:48:16 +08:00
parent 866f1df5e3
commit d58b4b6b0e
12 changed files with 242 additions and 58 deletions
+1
View File
@@ -30,6 +30,7 @@ sidebar: false
- IP 组自动规则中的 `StatusCount` / `StatusRatio` 支持状态码类写法(如 `"2xx"`、`"4xx"`、`"5xx"`),便于按整类错误率匹配。
- IP 组同步间隔下限由 5 分钟调整为 1 分钟,便于更频繁同步自动/订阅名单。
- 自动 IP 组回看窗口字段由 `lookback_minutes` 调整为 `lookback`,支持 `60m`、`1h` 等时长写法,并移除最小 5 分钟限制(兼容旧字段)。
### 修复
+2 -2
View File
@@ -19675,8 +19675,8 @@ const docTemplate = `{
"waf.IPGroupAutoTestResult": {
"type": "object",
"properties": {
"lookback_minutes": {
"type": "integer"
"lookback": {
"type": "string"
},
"matched_count": {
"type": "integer"
+8 -8
View File
@@ -8,7 +8,7 @@
```json
{
"lookback_minutes": 60,
"lookback": "1h",
"rules": [
{
"name": "单 IP 404 高频扫描",
@@ -22,7 +22,7 @@
| 字段 | 类型 | 作用 |
| --- | --- | --- |
| `lookback_minutes` | number | 每次执行时回看多少分钟内的请求日志。未填写时默认 60 分钟,最小 5 分钟,最大 43200 分钟。 |
| `lookback` | string | 回看窗口时长,使用 Go Duration 写法,例如 `30m`、`1h`、`90m`。未填写时默认 `1h`,最大 30 天。兼容旧字段 `lookback_minutes`(整数分钟)。 |
| `rules` | array | 自动规则列表。任意一条规则命中时,该 IP 会进入自动 IP 组名单。 |
| `rules[].name` | string | 规则名称,只用于界面展示和错误提示。 |
| `rules[].expr` | string | Expr 表达式,必须返回布尔值。 |
@@ -31,7 +31,7 @@
自动规则不是逐条请求判断,而是先按单个客户端 IP 聚合:
1. Server 读取最近 `lookback_minutes` 分钟内的请求日志。
1. Server 读取最近 `lookback` 时长内的请求日志。
2. 按 `remote_addr` 归一化后的 IP 分组。
3. 为每个 IP 计算请求数、404 数、直连 IP Host 次数等指标。
4. 逐个 IP 执行 `rules[].expr`。
@@ -115,7 +115,7 @@ Host 是否为“通过 IP 访问”按请求日志中的 `Host` 字段判断:
```json
{
"lookback_minutes": 60,
"lookback": "1h",
"rules": [
{
"name": "高频 404 扫描",
@@ -129,7 +129,7 @@ IP 直连访问异常:
```json
{
"lookback_minutes": 30,
"lookback": "30m",
"rules": [
{
"name": "IP 直连访问异常",
@@ -143,7 +143,7 @@ IP 直连访问异常:
```json
{
"lookback_minutes": 120,
"lookback": "2h",
"rules": [
{
"name": "异常错误率",
@@ -157,7 +157,7 @@ IP 直连访问异常:
```json
{
"lookback_minutes": 120,
"lookback": "2h",
"rules": [
{
"name": "高 4xx 或 5xx 占比",
@@ -171,7 +171,7 @@ IP 直连访问异常:
```json
{
"lookback_minutes": 60,
"lookback": "1h",
"rules": [
{
"name": "排除可信 IP 的 404 扫描",
+2 -2
View File
@@ -19668,8 +19668,8 @@
"waf.IPGroupAutoTestResult": {
"type": "object",
"properties": {
"lookback_minutes": {
"type": "integer"
"lookback": {
"type": "string"
},
"matched_count": {
"type": "integer"
+2 -2
View File
@@ -3872,8 +3872,8 @@ definitions:
type: object
waf.IPGroupAutoTestResult:
properties:
lookback_minutes:
type: integer
lookback:
type: string
matched_count:
type: integer
matched_ips:
@@ -89,7 +89,7 @@ const defaultValues: IPGroupFormValues = {
enabled: true,
ip_list_text: '',
auto_config_text: JSON.stringify(
{ lookback_minutes: 60, ttl: -1, rules: [] },
{ lookback: '1h', ttl: -1, rules: [] },
null,
2,
),
@@ -146,14 +146,20 @@ function appendAutomaticPresetRule(
(item as { expr?: unknown }).expr === rule.expr,
);
const nextRules = exists ? rules : [...rules, rule];
const lookback =
typeof config.lookback === 'string' && config.lookback.trim()
? config.lookback
: typeof config.lookback_minutes === 'number'
? `${config.lookback_minutes}m`
: '1h';
// strip legacy field so saved JSON only keeps lookback duration string
const { lookback_minutes: _legacyLookbackMinutes, ...rest } = config;
return JSON.stringify(
{
lookback_minutes:
typeof config.lookback_minutes === 'number'
? config.lookback_minutes
: 60,
ttl: typeof config.ttl === 'number' ? config.ttl : -1,
...config,
lookback,
ttl: typeof rest.ttl === 'number' ? rest.ttl : -1,
...rest,
lookback,
rules: nextRules,
},
null,
@@ -384,8 +390,8 @@ export function IPGroupDialog({
/>
</FormControl>
<FormDescription>
定时从请求日志挖掘恶意 IP 的周期。最小 1 分钟,默认
1440 分钟。
定时从请求日志挖掘恶意 IP 的周期。最小 1 分钟,默认 1440
分钟。
</FormDescription>
<FormMessage />
</FormItem>
@@ -42,8 +42,8 @@ export function IPGroupTestDialog({
<div className='space-y-4'>
<div className='rounded-lg border border-dashed p-4 text-sm'>
<p>
回看 {result.lookback_minutes} 分钟 · 规则 {result.rule_count}{' '}
条 · 命中 {result.matched_count} 个 IP
回看 {result.lookback} · 规则 {result.rule_count} 条 · 命中{' '}
{result.matched_count} 个 IP
</p>
<p className='text-xs text-muted-foreground mt-1'>
测试时间:{new Date(result.tested_at).toLocaleString()}
+1 -1
View File
@@ -1116,7 +1116,7 @@ export interface WAFIPGroupAutoTestPayload {
export interface WAFIPGroupAutoTestResult {
matched_ips: string[];
matched_count: number;
lookback_minutes: number;
lookback: string;
rule_count: number;
tested_at: string;
}
+5 -1
View File
@@ -318,8 +318,12 @@ func evaluateParsedIPGroupAutoConfig(ctx context.Context, config ipGroupAutoConf
}
programs = append(programs, program)
}
lookback := config.lookbackDuration
if lookback <= 0 {
lookback = defaultWAFIPGroupAutoLookbackDur
}
aggregates, err := model.ListOpenFlareAccessLogWAFIPAggregates(ctx, model.OpenFlareAccessLogQuery{
Since: now.Add(-time.Duration(config.LookbackMinutes) * time.Minute),
Since: now.Add(-lookback),
Until: now,
})
if err != nil {
@@ -96,7 +96,7 @@ func TestSyncIPGroupAutomaticExprRules(t *testing.T) {
Type: wafIPGroupTypeAutomatic,
Enabled: true,
AutoConfig: json.RawMessage(`{
"lookback_minutes": 60,
"lookback": "60m",
"rules": [
{"name":"单 IP 404 高频扫描","expr":"request_count > 100 && StatusRatio(404) >= 0.8"},
{"name":"单 IP 直连访问异常","expr":"ip_host_count > 50 && ip_host_ratio > 0.5"}
@@ -129,7 +129,7 @@ func TestTestIPGroupAutoConfigReturnsMatchedIPs(t *testing.T) {
result, err := TestIPGroupAutoConfig(ctx, IPGroupAutoTestInput{
AutoConfig: json.RawMessage(`{
"lookback_minutes": 60,
"lookback": "1h",
"rules": [
{"name":"单 IP 404 高频扫描","expr":"request_count > 100 && StatusRatio(404) >= 0.8"},
{"name":"单 IP 直连访问异常","expr":"ip_host_count > 50 && ip_host_ratio > 0.5"}
@@ -139,7 +139,7 @@ func TestTestIPGroupAutoConfigReturnsMatchedIPs(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, 2, result.MatchedCount)
assert.Equal(t, 2, result.RuleCount)
assert.Equal(t, 60, result.LookbackMinutes)
assert.Equal(t, "1h", result.Lookback)
want := map[string]bool{"203.0.113.10": true, "203.0.113.11": true}
for _, item := range result.MatchedIPs {
@@ -210,6 +210,45 @@ func seedWAFAccessLogs(t *testing.T, ctx context.Context, loggedAt time.Time, re
require.NoError(t, model.InsertOpenFlareAccessLogsBatch(ctx, records))
}
func TestParseIPGroupAutoConfigLookback(t *testing.T) {
cases := []struct {
name string
raw string
want string
wantDur time.Duration
wantErr bool
}{
{name: "duration 60m", raw: `{"lookback":"60m","rules":[]}`, want: "1h", wantDur: time.Hour},
{name: "duration 1h", raw: `{"lookback":"1h","rules":[]}`, want: "1h", wantDur: time.Hour},
{name: "duration 30m", raw: `{"lookback":"30m","rules":[]}`, want: "30m", wantDur: 30 * time.Minute},
{name: "duration 1m no min floor", raw: `{"lookback":"1m","rules":[]}`, want: "1m", wantDur: time.Minute},
{name: "legacy minutes", raw: `{"lookback_minutes":45,"rules":[]}`, want: "45m", wantDur: 45 * time.Minute},
{name: "default empty", raw: `{"rules":[]}`, want: "1h", wantDur: time.Hour},
{name: "invalid", raw: `{"lookback":"abc","rules":[]}`, wantErr: true},
{name: "zero lookback uses default", raw: `{"lookback":"","rules":[]}`, want: "1h", wantDur: time.Hour},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, err := parseIPGroupAutoConfig(json.RawMessage(tc.raw))
if tc.wantErr {
if err == nil {
t.Fatalf("parseIPGroupAutoConfig(%s) error = nil, want error", tc.raw)
}
return
}
if err != nil {
t.Fatalf("parseIPGroupAutoConfig(%s) error = %v", tc.raw, err)
}
if cfg.Lookback != tc.want {
t.Errorf("Lookback = %q, want %q", cfg.Lookback, tc.want)
}
if cfg.lookbackDuration != tc.wantDur {
t.Errorf("lookbackDuration = %v, want %v", cfg.lookbackDuration, tc.wantDur)
}
})
}
}
func TestCountStatusMatchesSupportsClassTokens(t *testing.T) {
counts := map[int]int{
200: 10,
@@ -257,7 +296,7 @@ func TestStatusRatioClassTokenInExpr(t *testing.T) {
result, err := TestIPGroupAutoConfig(ctx, IPGroupAutoTestInput{
AutoConfig: json.RawMessage(`{
"lookback_minutes": 60,
"lookback": "60m",
"rules": [
{"name":"高 4xx 占比","expr":"request_count >= 100 && StatusRatio(\"4xx\") >= 0.8"}
]
+160 -26
View File
@@ -31,10 +31,11 @@ const (
wafIPGroupSubscriptionFormatJSON = "json"
defaultWAFIPGroupSyncIntervalMinutes = 1440
defaultWAFIPGroupAutoLookbackMinutes = 60
defaultWAFIPGroupAutoLookback = "1h"
defaultWAFIPGroupAutoLookbackDur = time.Hour
minWAFIPGroupSyncIntervalMinutes = 1
minWAFIPGroupAutoLookbackMinutes = 5
maxWAFIPGroupSyncIntervalMinutes = 43200
maxWAFIPGroupAutoLookback = 30 * 24 * time.Hour
minPoWSessionTTLSeconds = 60
minPoWChallengeTTLSeconds = 30
powAlgorithmFast = "fast"
@@ -113,17 +114,19 @@ type IPGroupAutoTestInput struct {
// IPGroupAutoTestResult is the response for automatic IP group test.
type IPGroupAutoTestResult struct {
MatchedIPs []string `json:"matched_ips"`
MatchedCount int `json:"matched_count"`
LookbackMinutes int `json:"lookback_minutes"`
RuleCount int `json:"rule_count"`
TestedAt string `json:"tested_at"`
MatchedIPs []string `json:"matched_ips"`
MatchedCount int `json:"matched_count"`
Lookback string `json:"lookback"`
RuleCount int `json:"rule_count"`
TestedAt string `json:"tested_at"`
}
type ipGroupAutoConfig struct {
LookbackMinutes int `json:"lookback_minutes"`
TTL int `json:"ttl"`
Rules []ipGroupAutoRule `json:"rules"`
Lookback string `json:"lookback"`
TTL int `json:"ttl"`
Rules []ipGroupAutoRule `json:"rules"`
// lookbackDuration is resolved from Lookback (and legacy lookback_minutes) for runtime queries.
lookbackDuration time.Duration `json:"-"`
}
type ipGroupAutoRule struct {
@@ -329,11 +332,11 @@ func TestIPGroupAutoConfig(ctx context.Context, input IPGroupAutoTestInput) (*IP
return nil, err
}
return &IPGroupAutoTestResult{
MatchedIPs: ips,
MatchedCount: len(ips),
LookbackMinutes: config.LookbackMinutes,
RuleCount: len(config.Rules),
TestedAt: now.Format(time.RFC3339),
MatchedIPs: ips,
MatchedCount: len(ips),
Lookback: config.Lookback,
RuleCount: len(config.Rules),
TestedAt: now.Format(time.RFC3339),
}, nil
}
@@ -497,23 +500,20 @@ func parseIPGroupAutoConfig(raw json.RawMessage) (ipGroupAutoConfig, error) {
if text == "" {
text = "{}"
}
var config ipGroupAutoConfig
if err := json.Unmarshal([]byte(text), &config); err != nil {
return ipGroupAutoConfig{}, errors.New("自动 IP 组配置必须是 JSON 对象")
}
var object map[string]any
if err := json.Unmarshal([]byte(text), &object); err != nil || object == nil {
return ipGroupAutoConfig{}, errors.New("自动 IP 组配置必须是 JSON 对象")
}
if config.LookbackMinutes <= 0 {
config.LookbackMinutes = defaultWAFIPGroupAutoLookbackMinutes
var config ipGroupAutoConfig
if err := json.Unmarshal([]byte(text), &config); err != nil {
return ipGroupAutoConfig{}, errors.New("自动 IP 组配置必须是 JSON 对象")
}
if config.LookbackMinutes < minWAFIPGroupAutoLookbackMinutes {
config.LookbackMinutes = minWAFIPGroupAutoLookbackMinutes
}
if config.LookbackMinutes > maxWAFIPGroupSyncIntervalMinutes {
config.LookbackMinutes = maxWAFIPGroupSyncIntervalMinutes
lookbackDur, lookbackText, err := resolveIPGroupAutoLookback(object)
if err != nil {
return ipGroupAutoConfig{}, err
}
config.Lookback = lookbackText
config.lookbackDuration = lookbackDur
if config.TTL == 0 {
config.TTL = -1
}
@@ -534,6 +534,140 @@ func parseIPGroupAutoConfig(raw json.RawMessage) (ipGroupAutoConfig, error) {
return config, nil
}
// resolveIPGroupAutoLookback accepts lookback as duration string (60m/1h) or legacy lookback_minutes number.
func resolveIPGroupAutoLookback(object map[string]any) (time.Duration, string, error) {
if raw, ok := object["lookback"]; ok && raw != nil {
dur, text, err := parseIPGroupLookbackValue(raw)
if err != nil {
return 0, "", err
}
return dur, text, nil
}
if raw, ok := object["lookback_minutes"]; ok && raw != nil {
// legacy: minutes as number or numeric string
minutes, err := parsePositiveNumber(raw)
if err != nil {
return 0, "", fmt.Errorf("lookback_minutes 无效: %w", err)
}
if minutes <= 0 {
return defaultWAFIPGroupAutoLookbackDur, defaultWAFIPGroupAutoLookback, nil
}
dur := time.Duration(minutes) * time.Minute
if dur > maxWAFIPGroupAutoLookback {
return 0, "", fmt.Errorf("回看窗口不能超过 %s", formatLookbackDuration(maxWAFIPGroupAutoLookback))
}
return dur, formatLookbackDuration(dur), nil
}
return defaultWAFIPGroupAutoLookbackDur, defaultWAFIPGroupAutoLookback, nil
}
func parseIPGroupLookbackValue(raw any) (time.Duration, string, error) {
switch v := raw.(type) {
case string:
trimmed := strings.TrimSpace(v)
if trimmed == "" {
return defaultWAFIPGroupAutoLookbackDur, defaultWAFIPGroupAutoLookback, nil
}
// bare integer string → minutes
if isAllDigits(trimmed) {
minutes, err := parsePositiveNumber(trimmed)
if err != nil || minutes <= 0 {
return 0, "", errors.New("lookback 格式不合法,请使用 60m、1h 等时长")
}
dur := time.Duration(minutes) * time.Minute
if dur > maxWAFIPGroupAutoLookback {
return 0, "", fmt.Errorf("回看窗口不能超过 %s", formatLookbackDuration(maxWAFIPGroupAutoLookback))
}
return dur, formatLookbackDuration(dur), nil
}
dur, err := time.ParseDuration(strings.ToLower(trimmed))
if err != nil || dur <= 0 {
return 0, "", errors.New("lookback 格式不合法,请使用 60m、1h 等时长")
}
if dur > maxWAFIPGroupAutoLookback {
return 0, "", fmt.Errorf("回看窗口不能超过 %s", formatLookbackDuration(maxWAFIPGroupAutoLookback))
}
return dur, formatLookbackDuration(dur), nil
case float64:
if v <= 0 {
return defaultWAFIPGroupAutoLookbackDur, defaultWAFIPGroupAutoLookback, nil
}
if v != float64(int64(v)) {
return 0, "", errors.New("lookback 数值必须为整数分钟")
}
dur := time.Duration(int64(v)) * time.Minute
if dur > maxWAFIPGroupAutoLookback {
return 0, "", fmt.Errorf("回看窗口不能超过 %s", formatLookbackDuration(maxWAFIPGroupAutoLookback))
}
return dur, formatLookbackDuration(dur), nil
case json.Number:
return parseIPGroupLookbackValue(string(v))
default:
return 0, "", errors.New("lookback 格式不合法,请使用 60m、1h 等时长")
}
}
func parsePositiveNumber(raw any) (int, error) {
switch v := raw.(type) {
case float64:
if v != float64(int(v)) {
return 0, errors.New("必须为整数")
}
return int(v), nil
case int:
return v, nil
case int64:
return int(v), nil
case json.Number:
i, err := v.Int64()
if err != nil {
return 0, err
}
return int(i), nil
case string:
trimmed := strings.TrimSpace(v)
if trimmed == "" || !isAllDigits(trimmed) {
return 0, errors.New("必须为整数")
}
n := 0
for _, ch := range trimmed {
n = n*10 + int(ch-'0')
}
return n, nil
default:
return 0, errors.New("必须为整数")
}
}
func isAllDigits(s string) bool {
if s == "" {
return false
}
for _, ch := range s {
if ch < '0' || ch > '9' {
return false
}
}
return true
}
func formatLookbackDuration(d time.Duration) string {
if d <= 0 {
return defaultWAFIPGroupAutoLookback
}
// Prefer compact human units used in config examples.
if d%time.Hour == 0 {
return fmt.Sprintf("%dh", int(d/time.Hour))
}
if d%time.Minute == 0 {
return fmt.Sprintf("%dm", int(d/time.Minute))
}
if d%time.Second == 0 {
return fmt.Sprintf("%ds", int(d/time.Second))
}
return d.String()
}
func validateSubscriptionURL(rawURL string) error {
parsed, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil || parsed.Host == "" {
+1 -1
View File
@@ -53,7 +53,7 @@ func TestUpdateIPGroupPrunesAutomaticExtIPs(t *testing.T) {
Name: "auto group",
Type: wafIPGroupTypeAutomatic,
Enabled: true,
AutoConfig: []byte(`{"lookback_minutes":60,"ttl":-1,"rules":[{"name":"scan","expr":"request_count > 1"}]}`),
AutoConfig: []byte(`{"lookback":"60m","ttl":-1,"rules":[{"name":"scan","expr":"request_count > 1"}]}`),
})
require.NoError(t, err)