feat(domain): 支持通配符域名及关联功能适配 (#63)

* feat(domain): 支持通配符域名及关联功能适配

* fix(domain): preserve legacy Unicode imports and address review findings

* fix(domain): canonicalize legacy IDNs and exclude empty host labels

* fix(domain): normalize equivalent Unicode IDN spellings

---------

Co-authored-by: Ryan <63696351+Rain-kl@users.noreply.github.com>
Co-authored-by: ryan <ryan@arctel.net>
This commit is contained in:
Deqiying
2026-11-09 10:31:03 +08:00
committed by GitHub
parent 8d8b3e1ea8
commit 9a51a68206
37 changed files with 1614 additions and 113 deletions
@@ -42,12 +42,28 @@ func buildNodeAccessLogFilterClause(filter NodeAccessLogFilter) (string, []any)
}
hosts := normalizeNodeAccessLogHosts(filter.Hosts)
if len(hosts) > 0 {
hostParts := make([]string, 0, len(hosts))
placeholders := make([]string, 0, len(hosts))
var exactHosts []any
for _, host := range hosts {
if suffix, wildcard := strings.CutPrefix(host, "*."); wildcard && suffix != "" {
// _% 要求非空子域名,并保留 Nginx 通配符可匹配多层子域名的语义。
hostParts = append(hostParts, "(lowerUTF8(trim(host)) LIKE ? AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%')")
args = append(args, "_%."+util.EscapeLike(suffix))
continue
}
placeholders = append(placeholders, "?")
args = append(args, host)
exactHosts = append(exactHosts, host)
}
parts = append(parts, "lowerUTF8(trim(host)) IN ("+strings.Join(placeholders, ", ")+")")
if len(exactHosts) > 0 {
hostParts = append(hostParts, "lowerUTF8(trim(host)) IN ("+strings.Join(placeholders, ", ")+")")
args = append(args, exactHosts...)
}
hostClause := strings.Join(hostParts, " OR ")
if len(hostParts) > 1 {
hostClause = "(" + hostClause + ")"
}
parts = append(parts, hostClause)
} else if trimmed := strings.TrimSpace(filter.Host); trimmed != "" {
parts = append(parts, "host LIKE ?")
args = append(args, util.EscapeLike(trimmed)+"%")
@@ -0,0 +1,132 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package analytics
import (
"context"
"strings"
"testing"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/google/go-cmp/cmp"
)
func TestNodeAccessLogHostsWildcardSQL(t *testing.T) {
since := time.Date(2026, time.June, 1, 0, 0, 0, 0, time.UTC)
until := since.Add(time.Hour)
tests := []struct {
name string
filter NodeAccessLogFilter
clause string
args []any
}{
{
name: "exact_hosts_remain_exact_and_normalized",
filter: NodeAccessLogFilter{Hosts: []string{" API.Example.com ", "api.example.com", "", "www.example.com"}},
clause: "lowerUTF8(trim(host)) IN (?, ?)",
args: []any{"api.example.com", "www.example.com"},
},
{
name: "wildcard_requires_nonempty_prefix",
filter: NodeAccessLogFilter{Hosts: []string{" *.Example.com ", "*.example.com"}},
clause: "(lowerUTF8(trim(host)) LIKE ? AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%')",
args: []any{"_%.example.com"},
},
{
name: "nested_wildcard",
filter: NodeAccessLogFilter{Hosts: []string{"*.s3.example.com"}},
clause: "(lowerUTF8(trim(host)) LIKE ? AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%')",
args: []any{"_%.s3.example.com"},
},
{
name: "overlapping_scopes_are_one_group_with_other_filters",
filter: NodeAccessLogFilter{
NodeID: " n1 ", Host: "ignored", Hosts: []string{"api.example.com", "*.example.com", "*.s3.example.com"},
StatusCode: 200, Since: since, Until: until,
},
clause: "node_id = ? AND ((lowerUTF8(trim(host)) LIKE ? AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%') OR (lowerUTF8(trim(host)) LIKE ? AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%') OR lowerUTF8(trim(host)) IN (?)) AND status_code = ? AND logged_at >= ? AND logged_at < ?",
args: []any{"n1", "_%.example.com", "_%.s3.example.com", "api.example.com", 200, since, until},
},
{
name: "wildcard_suffix_like_metacharacters_are_literal",
filter: NodeAccessLogFilter{Hosts: []string{`*.a%_\'?.example.com`}},
clause: "(lowerUTF8(trim(host)) LIKE ? AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%')",
args: []any{`_%.a\%\_\\'?.example.com`},
},
{
name: "single_host_keeps_prefix_search",
filter: NodeAccessLogFilter{Host: " *.ex%_ ", Hosts: []string{" "}},
clause: "host LIKE ?",
args: []any{`*.ex\%\_%`},
},
{
name: "non_leading_wildcards_are_exact",
filter: NodeAccessLogFilter{Hosts: []string{"api.*.example.com", "*", "*."}},
clause: "lowerUTF8(trim(host)) IN (?, ?, ?)",
args: []any{"api.*.example.com", "*", "*."},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clause, args := buildNodeAccessLogFilterClause(tt.filter)
if clause != tt.clause {
t.Errorf("buildNodeAccessLogFilterClause(%+v) SQL = %q, want %q", tt.filter, clause, tt.clause)
}
if diff := cmp.Diff(tt.args, args); diff != "" {
t.Errorf("buildNodeAccessLogFilterClause(%+v) args mismatch (-want +got):\n%s", tt.filter, diff)
}
})
}
}
// wildcardQueryConn captures both single-row and multi-row query entry points.
type wildcardQueryConn struct{ mockConn }
func (m *wildcardQueryConn) QueryRow(_ context.Context, query string, args ...any) driver.Row {
m.queries = append(m.queries, query)
m.queryArgs = append(m.queryArgs, args)
return wildcardCountRow{}
}
type wildcardCountRow struct{ driver.Row }
func (wildcardCountRow) Scan(dest ...any) error {
for _, value := range dest {
*value.(*uint64) = 0
}
return nil
}
// TestWildcardQueryEntryPoints protects host filtering in count and bucket queries.
func TestWildcardQueryEntryPoints(t *testing.T) {
conn := &wildcardQueryConn{}
previous := db.ChConn
db.SetChConnForTest(conn)
t.Cleanup(func() { db.SetChConnForTest(previous) })
ctx := context.Background()
filter := NodeAccessLogFilter{Hosts: []string{"*.Example.com", "api.example.com"}}
if _, _, _, err := CountNodeAccessLogs(ctx, filter); err != nil {
t.Fatal(err)
}
if _, err := BucketAggregatesNodeAccessLogs(ctx, filter, 300); err != nil {
t.Fatal(err)
}
if _, err := CountBucketAggregatesNodeAccessLogs(ctx, filter, 300); err != nil {
t.Fatal(err)
}
if len(conn.queries) != 3 {
t.Fatalf("count/bucket query calls = %d, want 3", len(conn.queries))
}
for i, query := range conn.queries {
if !strings.Contains(query, "WHERE ((lowerUTF8(trim(host)) LIKE ? AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%') OR lowerUTF8(trim(host)) IN (?))") {
t.Errorf("count/bucket SQL = %q, want grouped wildcard and exact host predicates", query)
}
if diff := cmp.Diff([]any{"_%.example.com", "api.example.com"}, conn.queryArgs[i]); diff != "" {
t.Errorf("count/bucket args mismatch (-want +got):\n%s", diff)
}
}
}
@@ -0,0 +1,135 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package logstore
import (
"context"
"slices"
"testing"
"time"
"github.com/Rain-kl/Wavelet/internal/model"
analyticsmodel "github.com/Rain-kl/Wavelet/internal/model/analytics"
"github.com/google/go-cmp/cmp"
)
func TestGormNodeAccessLogWildcardHosts(t *testing.T) {
ResetForTest()
t.Cleanup(ResetForTest)
SetConfigReader(func(context.Context, string) (string, error) { return "", nil })
s := newTestGormStore(t)
ctx := context.Background()
base := time.Date(2026, time.June, 1, 0, 0, 0, 0, time.UTC)
hosts := []string{
"api.example.com", " A.B.Example.COM ", "bucket.s3.example.com", "deep.bucket.s3.example.com",
"example.com", "s3.example.com", "evil-example.com", "api.example.com.evil", ".example.com",
`x.a%_\'?.example.net`, `x.abc\'?.example.net`, ".s3.other.com", "s3.other.com", "a..example.com", ".a.example.com", "a..s3.example.com",
}
rows := make([]analyticsmodel.NodeAccessLog, 0, len(hosts)+3)
for i, host := range hosts {
rows = append(rows, analyticsmodel.NodeAccessLog{
ID: uint64(i + 1), NodeID: "n1", Host: host, LoggedAt: base.Add(time.Duration(i%2)*time.Hour + time.Minute),
RemoteAddr: "1.1.1.1", BytesSent: uint64((i + 1) * 10), StatusCode: 200,
})
}
rows[2].RemoteAddr = "2.2.2.2"
rows[3].RemoteAddr = "" // 空 IP 仍计入请求与字节数,但不计入 UV。
rows = append(rows,
analyticsmodel.NodeAccessLog{ID: 100, NodeID: "other", Host: "api.example.com", LoggedAt: base.Add(time.Minute)},
analyticsmodel.NodeAccessLog{ID: 101, NodeID: "n1", Host: "api.example.com", LoggedAt: base.Add(2 * time.Hour)},
analyticsmodel.NodeAccessLog{ID: 102, NodeID: "n1", Host: "bucket.s3.example.com", LoggedAt: base.Add(-time.Second)},
)
if err := s.BatchInsertNodeAccessLogs(ctx, rows); err != nil {
t.Fatalf("BatchInsertNodeAccessLogs(wildcard fixtures) error = %v, want nil", err)
}
tests := []struct {
name string
hosts []string
host string
wantIDs []uint64
}{
{name: "exact", hosts: []string{" API.EXAMPLE.COM "}, wantIDs: []uint64{1}},
{name: "wildcard_multilevel", hosts: []string{" *.Example.com "}, wantIDs: []uint64{1, 2, 3, 4, 6}},
{name: "nested_wildcard_excludes_suffix_root", hosts: []string{"*.s3.example.com"}, wantIDs: []uint64{3, 4}},
{name: "overlap_does_not_double_count", hosts: []string{"api.example.com", "*.example.com", "*.s3.example.com", "*.EXAMPLE.COM"}, host: "ignored", wantIDs: []uint64{1, 2, 3, 4, 6}},
{name: "exact_root_and_nested_wildcard", hosts: []string{"example.com", "*.s3.example.com"}, wantIDs: []uint64{3, 4, 5}},
{name: "exact_suffix_root", hosts: []string{"example.com"}, wantIDs: []uint64{5}},
{name: "like_metacharacters_are_literal", hosts: []string{`*.a%_\'?.example.net`}, wantIDs: []uint64{10}},
{name: "empty_prefix_is_not_a_subdomain", hosts: []string{"*.s3.other.com"}, wantIDs: []uint64{}},
{name: "single_host_keeps_prefix_search", host: "api.example.com", wantIDs: []uint64{1, 8}},
{name: "single_host_does_not_enable_wildcard", host: "*.example.com", wantIDs: []uint64{}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
q := model.OpenFlareAccessLogQuery{NodeID: "n1", Hosts: tt.hosts, Host: tt.host, Since: base, Until: base.Add(2 * time.Hour)}
type totals struct {
Requests int64
UniqueIP int64
Bytes int64
}
want := totals{Requests: int64(len(tt.wantIDs))}
ips := make(map[string]struct{})
wantBuckets := make(map[int64]totals)
bucketIPs := make(map[int64]map[string]struct{})
for _, id := range tt.wantIDs {
r := rows[id-1]
want.Bytes += int64(r.BytesSent)
if r.RemoteAddr != "" {
ips[r.RemoteAddr] = struct{}{}
}
epoch := r.LoggedAt.Truncate(time.Hour).Unix()
bucket := wantBuckets[epoch]
bucket.Requests++
bucket.Bytes += int64(r.BytesSent)
if bucketIPs[epoch] == nil {
bucketIPs[epoch] = make(map[string]struct{})
}
if r.RemoteAddr != "" {
bucketIPs[epoch][r.RemoteAddr] = struct{}{}
}
bucket.UniqueIP = int64(len(bucketIPs[epoch]))
wantBuckets[epoch] = bucket
}
want.UniqueIP = int64(len(ips))
total, unique, bytes, err := s.Count(ctx, q)
if err != nil {
t.Fatalf("Count(%+v) error = %v, want nil", q, err)
}
if got := (totals{total, unique, bytes}); got != want {
t.Errorf("Count(%+v) = %+v, want %+v", q, got, want)
}
listed, err := s.List(ctx, q)
if err != nil {
t.Fatalf("List(%+v) error = %v, want nil", q, err)
}
gotIDs := make([]uint64, 0, len(listed))
for _, r := range listed {
gotIDs = append(gotIDs, uint64(r.ID))
}
slices.Sort(gotIDs)
if diff := cmp.Diff(tt.wantIDs, gotIDs); diff != "" {
t.Errorf("List(%+v) IDs mismatch (-want +got):\n%s", q, diff)
}
buckets, err := s.BucketAggregates(ctx, q, 3600)
if err != nil {
t.Fatalf("BucketAggregates(%+v, 3600) error = %v, want nil", q, err)
}
gotBuckets := make(map[int64]totals)
for _, b := range buckets {
gotBuckets[b.BucketEpoch] = totals{b.RequestCount, b.UniqueIPCount, b.BytesSent}
}
if diff := cmp.Diff(wantBuckets, gotBuckets); diff != "" {
t.Errorf("BucketAggregates(%+v, 3600) mismatch (-want +got):\n%s", q, diff)
}
bucketCount, err := s.CountBuckets(ctx, q, 3600)
if err != nil {
t.Fatalf("CountBuckets(%+v, 3600) error = %v, want nil", q, err)
}
if bucketCount != int64(len(wantBuckets)) {
t.Errorf("CountBuckets(%+v, 3600) = %d, want %d", q, bucketCount, len(wantBuckets))
}
})
}
}
+22 -4
View File
@@ -814,7 +814,7 @@ func toNodeAccessLogFilter(query model.OpenFlareAccessLogQuery) analyticsmodel.N
}
// buildNodeAccessLogFilterParts 拼装节点访问日志过滤条件(node_id 等值、remote_addr/host/path
// 前缀 LIKE、hosts 走 lower(trim(host)) IN、since/until 时间窗,顺序与 applyNodeAccessLogFilter
// 前缀 LIKE、hosts 精确域名或通配符后缀、since/until 时间窗,顺序与 applyNodeAccessLogFilter
// 完全一致),返回 WHERE 片段与参数;无任何条件时返回空串与 nil。
func buildNodeAccessLogFilterParts(f analyticsmodel.NodeAccessLogFilter) (string, []any) {
var parts []string
@@ -829,8 +829,26 @@ func buildNodeAccessLogFilterParts(f analyticsmodel.NodeAccessLogFilter) (string
}
hosts := normalizeNodeAccessLogHosts(f.Hosts)
if len(hosts) > 0 {
parts = append(parts, "lower(trim(host)) IN ?")
args = append(args, hosts)
var hostParts []string
var exactHosts []string
for _, host := range hosts {
if suffix, wildcard := strings.CutPrefix(host, "*."); wildcard && suffix != "" {
// 与 CH 一致:非空子域名,可跨多层,但不匹配 suffix 根自身。
hostParts = append(hostParts, `(lower(trim(host)) LIKE ? ESCAPE '\' AND trim(host) NOT LIKE '%..%' AND trim(host) NOT LIKE '.%')`)
args = append(args, "_%."+util.EscapeLike(suffix))
continue
}
exactHosts = append(exactHosts, host)
}
if len(exactHosts) > 0 {
hostParts = append(hostParts, "lower(trim(host)) IN ?")
args = append(args, exactHosts)
}
hostClause := strings.Join(hostParts, " OR ")
if len(hostParts) > 1 {
hostClause = "(" + hostClause + ")"
}
parts = append(parts, hostClause)
} else if host := strings.TrimSpace(f.Host); host != "" {
parts = append(parts, `host LIKE ? ESCAPE '\'`)
args = append(args, util.EscapeLike(host)+"%")
@@ -858,7 +876,7 @@ func buildNodeAccessLogFilterParts(f analyticsmodel.NodeAccessLogFilter) (string
}
// applyNodeAccessLogFilter 对齐 CH 过滤语义(node_access_log_filter.go):
// node_id trim 后等值;remote_addr/host/path 前缀 LIKE;hosts 走 lower(trim(host)) IN(参数已归一化);
// node_id trim 后等值;remote_addr/host/path 前缀 LIKE;hosts 精确域名或通配符后缀(参数已归一化);
// since 闭区间 >=;until 开区间 <。
func applyNodeAccessLogFilter(q *gorm.DB, f analyticsmodel.NodeAccessLogFilter) *gorm.DB {
cond, args := buildNodeAccessLogFilterParts(f)