mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-10-06 23:56:37 +08:00
feat(waf): 新增 UA 检查节点 ua_check
支持要求携带 UA、浏览器/OS 白名单 and-or 匹配,以及优先屏蔽爬虫与非正常 UA。
This commit is contained in:
@@ -782,6 +782,9 @@ func TestManagedWAFLuaExecutesCompiledGraphWithoutRequestIO(t *testing.T) {
|
||||
if !strings.Contains(openRestyWAFRuntimeLua, `node.type == "ip_match"`) {
|
||||
t.Fatal("expected WAF runtime to execute compiled IP match nodes")
|
||||
}
|
||||
if !strings.Contains(openRestyWAFRuntimeLua, `node.type == "ua_check"`) {
|
||||
t.Fatal("expected WAF runtime to execute compiled UA check nodes")
|
||||
}
|
||||
checkStart := strings.Index(openRestyWAFRuntimeLua, "function _M.check()")
|
||||
if checkStart < 0 || strings.Contains(openRestyWAFRuntimeLua[checkStart:], "io.open") {
|
||||
t.Fatal("expected WAF request path not to perform file I/O")
|
||||
|
||||
@@ -294,6 +294,108 @@ local function matches_ip_values(config, ip)
|
||||
return false
|
||||
end
|
||||
|
||||
local function ua_trim(value)
|
||||
return (string.gsub(value or "", "^%s*(.-)%s*$", "%1"))
|
||||
end
|
||||
|
||||
local function ua_label_in(items, value)
|
||||
if type(items) ~= "table" or not value then return false end
|
||||
for _, item in ipairs(items) do
|
||||
if tostring(item) == value then return true end
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
local function match_ua_rules(ua_lower, rules, fallback)
|
||||
if ua_lower == "" then return "Unknown" end
|
||||
for _, rule in ipairs(rules) do
|
||||
local matched = false
|
||||
for _, token in ipairs(rule.contains or {}) do
|
||||
if string.find(ua_lower, token, 1, true) then
|
||||
matched = true
|
||||
break
|
||||
end
|
||||
end
|
||||
if not matched and type(rule.all_of) == "table" and #rule.all_of > 0 then
|
||||
matched = true
|
||||
for _, token in ipairs(rule.all_of) do
|
||||
if not string.find(ua_lower, token, 1, true) then
|
||||
matched = false
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
if matched then
|
||||
local excluded = false
|
||||
for _, token in ipairs(rule.none_of or {}) do
|
||||
if string.find(ua_lower, token, 1, true) then
|
||||
excluded = true
|
||||
break
|
||||
end
|
||||
end
|
||||
if not excluded then return rule.label end
|
||||
end
|
||||
end
|
||||
return fallback
|
||||
end
|
||||
|
||||
-- Mirrors internal/repository/analytics/browser.go browserRules / osRules.
|
||||
local browser_rules = {
|
||||
{ label = "WeChat", contains = { "micromessenger" } },
|
||||
{ label = "Postman", contains = { "postman" } },
|
||||
{ label = "CLI", contains = { "curl/", "wget/" } },
|
||||
{ label = "Edge", contains = { "edg/", "edgios/", "edga/" } },
|
||||
{ label = "Opera", contains = { "opr/", "opera" } },
|
||||
{ label = "Firefox", contains = { "firefox", "fxios" } },
|
||||
{ label = "Chrome", contains = { "crios", "chrome" }, none_of = { "chromium" } },
|
||||
{ label = "Chromium", contains = { "chromium" } },
|
||||
{ label = "Safari", contains = { "safari" } },
|
||||
{ label = "Bot", contains = { "bot", "spider", "crawler", "slurp" } },
|
||||
}
|
||||
|
||||
local os_rules = {
|
||||
{ label = "Android", contains = { "android" } },
|
||||
{ label = "iOS", contains = { "iphone", "ipad", "ipod", "ios" } },
|
||||
{ label = "Windows", contains = { "windows" } },
|
||||
{ label = "macOS", contains = { "mac os x", "macintosh", "macos" } },
|
||||
{ label = "Chrome OS", contains = { "cros" } },
|
||||
{ label = "Linux", contains = { "linux" } },
|
||||
{ label = "Bot", contains = { "bot", "spider", "crawler" } },
|
||||
}
|
||||
|
||||
local function parse_browser_name(ua)
|
||||
return match_ua_rules(string.lower(ua or ""), browser_rules, "Other")
|
||||
end
|
||||
|
||||
local function parse_os_name(ua)
|
||||
return match_ua_rules(string.lower(ua or ""), os_rules, "Other")
|
||||
end
|
||||
|
||||
local function matches_ua_check(config)
|
||||
config = config or {}
|
||||
local ua = ua_trim(ngx.var.http_user_agent or "")
|
||||
if config.require_ua and ua == "" then return false end
|
||||
local browser = parse_browser_name(ua)
|
||||
local os_name = parse_os_name(ua)
|
||||
if config.block_common_bots and (browser == "Bot" or os_name == "Bot") then return false end
|
||||
if config.block_abnormal_ua and (browser == "Bot" or browser == "Other" or browser == "Unknown") then
|
||||
return false
|
||||
end
|
||||
local browsers = array_or_empty(config.browsers)
|
||||
local operating_systems = array_or_empty(config.operating_systems)
|
||||
local has_browsers = #browsers > 0
|
||||
local has_os = #operating_systems > 0
|
||||
if not has_browsers and not has_os then return true end
|
||||
local browser_ok = ua_label_in(browsers, browser)
|
||||
local os_ok = ua_label_in(operating_systems, os_name)
|
||||
if has_browsers and not has_os then return browser_ok end
|
||||
if has_os and not has_browsers then return os_ok end
|
||||
local mode = config.match_mode
|
||||
if mode ~= "and" and mode ~= "or" then mode = "or" end
|
||||
if mode == "and" then return browser_ok and os_ok end
|
||||
return browser_ok or os_ok
|
||||
end
|
||||
|
||||
local function fail_closed(reason)
|
||||
local dict = ngx.shared and ngx.shared.openflare_waf_config
|
||||
if not dict or not dict.add or dict:add("_damaged_graph_logged", true, 60) then
|
||||
@@ -347,6 +449,8 @@ local function execute_graph(graph)
|
||||
local region_required = type(config.regions) == "table" and #config.regions > 0
|
||||
local country, region = geo_lookup(ngx.var.remote_addr or "", region_required)
|
||||
handle = (list_contains(config.countries, country) or list_contains(config.regions, region)) and "true" or "false"
|
||||
elseif node.type == "ua_check" then
|
||||
handle = matches_ua_check(node.config or {}) and "true" or "false"
|
||||
elseif node.type == "pow" then
|
||||
if pow_runtime.evaluate(node.config or {}) ~= true then
|
||||
return { kind = "takeover" }
|
||||
|
||||
@@ -79,8 +79,15 @@ local function load_runtime(config, options)
|
||||
return runtime
|
||||
end
|
||||
|
||||
local function reset_request(site, ip, uri, is_internal)
|
||||
ngx.var = { openflare_waf_site = site, remote_addr = ip or "192.0.2.1", uri = uri or "/", request_id = "request-1", openflare_internal = is_internal == true }
|
||||
local function reset_request(site, ip, uri, is_internal, user_agent)
|
||||
ngx.var = {
|
||||
openflare_waf_site = site,
|
||||
remote_addr = ip or "192.0.2.1",
|
||||
uri = uri or "/",
|
||||
request_id = "request-1",
|
||||
openflare_internal = is_internal == true,
|
||||
http_user_agent = user_agent,
|
||||
}
|
||||
ngx.ctx = {}
|
||||
ngx.header = {}
|
||||
output = {}
|
||||
@@ -531,6 +538,98 @@ local function test_request_path_has_no_file_io()
|
||||
io.open = original_open
|
||||
end
|
||||
|
||||
local function test_ua_check_require_block_and_whitelist()
|
||||
local chrome_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
local safari_ios_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
|
||||
local bot_ua = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
local weird_ua = "TotallyUnknownClient/1.0"
|
||||
|
||||
local function ua_graph(config)
|
||||
return graph({
|
||||
start = start_to("ua"),
|
||||
ua = node("ua_check", config, { ["true"] = "allow", ["false"] = "blocked" }),
|
||||
blocked = node("block", { status_code = 403, response_body = "ua blocked" }),
|
||||
allow = node("allow"),
|
||||
})
|
||||
end
|
||||
|
||||
local runtime = load_runtime({
|
||||
rule_groups = { rule(1, false, ua_graph({ require_ua = true })) },
|
||||
bindings = { binding("ua-site", { 1 }) },
|
||||
})
|
||||
reset_request("ua-site", nil, nil, nil, nil)
|
||||
runtime.check()
|
||||
assert_equal(output.exit, 403, "missing UA with require_ua should block")
|
||||
reset_request("ua-site", nil, nil, nil, chrome_ua)
|
||||
output = {}
|
||||
runtime.check()
|
||||
assert_equal(output.exit, nil, "present UA with require_ua should allow")
|
||||
|
||||
runtime = load_runtime({
|
||||
rule_groups = { rule(1, false, ua_graph({ block_common_bots = true })) },
|
||||
bindings = { binding("ua-site", { 1 }) },
|
||||
})
|
||||
reset_request("ua-site", nil, nil, nil, bot_ua)
|
||||
runtime.check()
|
||||
assert_equal(output.exit, 403, "common bot should be blocked")
|
||||
|
||||
runtime = load_runtime({
|
||||
rule_groups = { rule(1, false, ua_graph({ block_abnormal_ua = true })) },
|
||||
bindings = { binding("ua-site", { 1 }) },
|
||||
})
|
||||
reset_request("ua-site", nil, nil, nil, weird_ua)
|
||||
runtime.check()
|
||||
assert_equal(output.exit, 403, "abnormal UA should be blocked")
|
||||
reset_request("ua-site", nil, nil, nil, chrome_ua)
|
||||
output = {}
|
||||
runtime.check()
|
||||
assert_equal(output.exit, nil, "normal browser should pass abnormal check")
|
||||
|
||||
runtime = load_runtime({
|
||||
rule_groups = { rule(1, false, ua_graph({ browsers = { "Chrome" }, match_mode = "or" })) },
|
||||
bindings = { binding("ua-site", { 1 }) },
|
||||
})
|
||||
reset_request("ua-site", nil, nil, nil, safari_ios_ua)
|
||||
runtime.check()
|
||||
assert_equal(output.exit, 403, "Safari should miss Chrome whitelist")
|
||||
reset_request("ua-site", nil, nil, nil, chrome_ua)
|
||||
output = {}
|
||||
runtime.check()
|
||||
assert_equal(output.exit, nil, "Chrome should hit whitelist")
|
||||
|
||||
runtime = load_runtime({
|
||||
rule_groups = { rule(1, false, ua_graph({
|
||||
browsers = { "Chrome" },
|
||||
operating_systems = { "iOS" },
|
||||
match_mode = "and",
|
||||
})) },
|
||||
bindings = { binding("ua-site", { 1 }) },
|
||||
})
|
||||
reset_request("ua-site", nil, nil, nil, chrome_ua)
|
||||
runtime.check()
|
||||
assert_equal(output.exit, 403, "Chrome desktop should fail Chrome+iOS and")
|
||||
reset_request("ua-site", nil, nil, nil, safari_ios_ua)
|
||||
output = {}
|
||||
runtime.check()
|
||||
assert_equal(output.exit, 403, "Safari iOS should fail Chrome+iOS and")
|
||||
|
||||
runtime = load_runtime({
|
||||
rule_groups = { rule(1, false, ua_graph({
|
||||
browsers = { "Chrome" },
|
||||
operating_systems = { "iOS" },
|
||||
match_mode = "or",
|
||||
})) },
|
||||
bindings = { binding("ua-site", { 1 }) },
|
||||
})
|
||||
reset_request("ua-site", nil, nil, nil, chrome_ua)
|
||||
runtime.check()
|
||||
assert_equal(output.exit, nil, "Chrome desktop should pass Chrome|iOS or")
|
||||
reset_request("ua-site", nil, nil, nil, safari_ios_ua)
|
||||
output = {}
|
||||
runtime.check()
|
||||
assert_equal(output.exit, nil, "Safari iOS should pass Chrome|iOS or")
|
||||
end
|
||||
|
||||
test_ip_true_and_false()
|
||||
test_ipv6_exact_cidr_and_group()
|
||||
test_geo_true_and_false()
|
||||
@@ -546,5 +645,6 @@ test_block_config_and_rule_order()
|
||||
test_damaged_graphs_fail_closed()
|
||||
test_null_binding_ids_are_treated_as_empty()
|
||||
test_request_path_has_no_file_io()
|
||||
test_ua_check_require_block_and_whitelist()
|
||||
|
||||
return true
|
||||
|
||||
@@ -88,6 +88,17 @@ func compileRuleNodeConfig(node RuleNode) (any, error) {
|
||||
case RuleNodePoW:
|
||||
var config PoWNodeConfig
|
||||
return config, decodeStrictConfig(node.Config, &config)
|
||||
case RuleNodeUACheck:
|
||||
var config UACheckConfig
|
||||
if err := decodeStrictConfig(node.Config, &config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config.Browsers = sortedUniqueStrings(config.Browsers)
|
||||
config.OperatingSystems = sortedUniqueStrings(config.OperatingSystems)
|
||||
if config.MatchMode == "" {
|
||||
config.MatchMode = UACheckMatchModeOr
|
||||
}
|
||||
return config, nil
|
||||
case RuleNodeBlock:
|
||||
var config BlockNodeConfig
|
||||
return config, decodeStrictConfig(node.Config, &config)
|
||||
|
||||
@@ -42,6 +42,39 @@ func TestCompileRuleGraph(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileUACheckConfigNormalizesListsAndMatchMode(t *testing.T) {
|
||||
graph := RuleGraph{SchemaVersion: RuleGraphSchemaVersion, Nodes: []RuleNode{
|
||||
{ID: "start", Type: RuleNodeStart, Config: rawConfig(`{}`)},
|
||||
{ID: "ua", Type: RuleNodeUACheck, Config: rawConfig(`{"browsers":["Safari","Chrome","Chrome"],"operating_systems":["iOS","Android"],"require_ua":true,"block_common_bots":true}`)},
|
||||
{ID: "allow", Type: RuleNodeAllow, Config: rawConfig(`{}`)},
|
||||
{ID: "block", Type: RuleNodeBlock, Config: rawConfig(`{"status_code":403}`)},
|
||||
}, Edges: []RuleEdge{
|
||||
{ID: "e1", Source: "start", SourceHandle: "next", Target: "ua"},
|
||||
{ID: "e2", Source: "ua", SourceHandle: "true", Target: "allow"},
|
||||
{ID: "e3", Source: "ua", SourceHandle: "false", Target: "block"},
|
||||
}}
|
||||
compiled, err := CompileRuleGraph(graph)
|
||||
if err != nil {
|
||||
t.Fatalf("CompileRuleGraph() error = %v", err)
|
||||
}
|
||||
cfg, ok := compiled.Nodes["ua"].Config.(UACheckConfig)
|
||||
if !ok {
|
||||
t.Fatalf("config type = %T", compiled.Nodes["ua"].Config)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Browsers, []string{"Chrome", "Safari"}) {
|
||||
t.Fatalf("browsers = %#v", cfg.Browsers)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.OperatingSystems, []string{"Android", "iOS"}) {
|
||||
t.Fatalf("os = %#v", cfg.OperatingSystems)
|
||||
}
|
||||
if cfg.MatchMode != UACheckMatchModeOr {
|
||||
t.Fatalf("match_mode = %q, want or", cfg.MatchMode)
|
||||
}
|
||||
if !cfg.RequireUA || !cfg.BlockCommonBots || cfg.BlockAbnormalUA {
|
||||
t.Fatalf("flags = %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileRuleGraphIsDeterministicForNodeAndEdgeOrder(t *testing.T) {
|
||||
first := RuleGraph{SchemaVersion: RuleGraphSchemaVersion, Nodes: []RuleNode{
|
||||
{ID: "start", Type: RuleNodeStart, Config: rawConfig(`{}`)},
|
||||
|
||||
@@ -24,6 +24,8 @@ const (
|
||||
RuleNodeGeoMatch RuleNodeType = "geo_match"
|
||||
// RuleNodePoW runs a proof-of-work challenge before continuing.
|
||||
RuleNodePoW RuleNodeType = "pow"
|
||||
// RuleNodeUACheck branches on User-Agent presence, classification, and lists.
|
||||
RuleNodeUACheck RuleNodeType = "ua_check"
|
||||
)
|
||||
|
||||
// RuleGraph is the editor-facing representation of an executable WAF graph.
|
||||
@@ -83,6 +85,22 @@ type BlockNodeConfig struct {
|
||||
ResponseBody string `json:"response_body,omitempty"`
|
||||
}
|
||||
|
||||
// UACheckConfig configures User-Agent presence, whitelist, and block switches.
|
||||
type UACheckConfig struct {
|
||||
RequireUA bool `json:"require_ua"`
|
||||
Browsers []string `json:"browsers,omitempty"`
|
||||
OperatingSystems []string `json:"operating_systems,omitempty"`
|
||||
MatchMode string `json:"match_mode,omitempty"`
|
||||
BlockCommonBots bool `json:"block_common_bots"`
|
||||
BlockAbnormalUA bool `json:"block_abnormal_ua"`
|
||||
}
|
||||
|
||||
// UA check match modes.
|
||||
const (
|
||||
UACheckMatchModeAnd = "and"
|
||||
UACheckMatchModeOr = "or"
|
||||
)
|
||||
|
||||
// DefaultRuleGraph returns the minimal start-to-allow graph.
|
||||
func DefaultRuleGraph() RuleGraph {
|
||||
return RuleGraph{SchemaVersion: RuleGraphSchemaVersion, Nodes: []RuleNode{
|
||||
|
||||
@@ -87,7 +87,7 @@ func validateRuleGraphNodes(ctx context.Context, graphNodes []RuleNode, ipGroupE
|
||||
startID = node.ID
|
||||
case RuleNodeAllow:
|
||||
allowCount++
|
||||
case RuleNodeBlock, RuleNodeIPMatch, RuleNodeGeoMatch, RuleNodePoW:
|
||||
case RuleNodeBlock, RuleNodeIPMatch, RuleNodeGeoMatch, RuleNodePoW, RuleNodeUACheck:
|
||||
default:
|
||||
return nil, "", fmt.Errorf("节点 %s 的类型 %s 未知", node.ID, node.Type)
|
||||
}
|
||||
@@ -180,6 +180,8 @@ func validateRuleNodeConfig(ctx context.Context, node RuleNode, exists func(cont
|
||||
return validateGeoMatchNodeConfig(node)
|
||||
case RuleNodePoW:
|
||||
return validatePoWNodeConfig(node)
|
||||
case RuleNodeUACheck:
|
||||
return validateUACheckNodeConfig(node)
|
||||
case RuleNodeBlock:
|
||||
return validateBlockNodeConfig(node)
|
||||
}
|
||||
@@ -283,6 +285,42 @@ func validateBlockNodeConfig(node RuleNode) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateUACheckNodeConfig(node RuleNode) error {
|
||||
var cfg UACheckConfig
|
||||
if err := decodeNodeConfig(node, &cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
mode := cfg.MatchMode
|
||||
if mode == "" {
|
||||
mode = UACheckMatchModeOr
|
||||
}
|
||||
if mode != UACheckMatchModeAnd && mode != UACheckMatchModeOr {
|
||||
return fmt.Errorf("节点 %s 的匹配模式必须为 and 或 or", node.ID)
|
||||
}
|
||||
for _, label := range cfg.Browsers {
|
||||
if !uaBrowserLabels[label] {
|
||||
return fmt.Errorf("节点 %s 的浏览器标签 %s 无效", node.ID, label)
|
||||
}
|
||||
}
|
||||
for _, label := range cfg.OperatingSystems {
|
||||
if !uaOSLabels[label] {
|
||||
return fmt.Errorf("节点 %s 的操作系统标签 %s 无效", node.ID, label)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var uaBrowserLabels = map[string]bool{
|
||||
"Chrome": true, "Safari": true, "Firefox": true, "Edge": true, "Opera": true,
|
||||
"Chromium": true, "WeChat": true, "Postman": true, "CLI": true, "Bot": true,
|
||||
"Unknown": true, "Other": true,
|
||||
}
|
||||
|
||||
var uaOSLabels = map[string]bool{
|
||||
"Android": true, "iOS": true, "Windows": true, "macOS": true, "Chrome OS": true,
|
||||
"Linux": true, "Bot": true, "Unknown": true, "Other": true,
|
||||
}
|
||||
|
||||
func decodeNodeConfig(node RuleNode, dst any) error {
|
||||
if err := decodeStrictConfig(node.Config, dst); err != nil {
|
||||
return fmt.Errorf("节点 %s 的配置无效: %w", node.ID, err)
|
||||
@@ -321,7 +359,7 @@ func requiredHandles(t RuleNodeType) []string {
|
||||
switch t {
|
||||
case RuleNodeStart, RuleNodePoW:
|
||||
return []string{"next"}
|
||||
case RuleNodeIPMatch, RuleNodeGeoMatch:
|
||||
case RuleNodeIPMatch, RuleNodeGeoMatch, RuleNodeUACheck:
|
||||
return []string{"true", "false"}
|
||||
default:
|
||||
return nil
|
||||
|
||||
@@ -77,6 +77,14 @@ func TestValidateRuleGraph(t *testing.T) {
|
||||
g.Edges[1].SourceHandle = "next"
|
||||
g.Edges = g.Edges[:2]
|
||||
}, "节点 match-1 的 PoW 难度必须在 1-16 之间"},
|
||||
{"invalid ua browser", func(g *RuleGraph) {
|
||||
g.Nodes[1].Type = RuleNodeUACheck
|
||||
g.Nodes[1].Config = rawConfig(`{"browsers":["NotABrowser"],"match_mode":"or"}`)
|
||||
}, "节点 match-1 的浏览器标签 NotABrowser 无效"},
|
||||
{"invalid ua match mode", func(g *RuleGraph) {
|
||||
g.Nodes[1].Type = RuleNodeUACheck
|
||||
g.Nodes[1].Config = rawConfig(`{"match_mode":"xor"}`)
|
||||
}, "节点 match-1 的匹配模式必须为 and 或 or"},
|
||||
{"unknown config field", func(g *RuleGraph) { g.Nodes[1].Config = rawConfig(`{"ips":[],"surprise":true}`) }, "节点 match-1 的配置无效"},
|
||||
{"null config", func(g *RuleGraph) { g.Nodes[1].Config = rawConfig(`null`) }, "节点 match-1 的配置无效"},
|
||||
{"too many nodes", func(g *RuleGraph) {
|
||||
|
||||
Reference in New Issue
Block a user