diff --git a/.gitignore b/.gitignore index 315efa15..30fd678c 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ profile.cov /.gomodcache/ *.mmdb !internal/apps/agent/geoipdata/GeoLite2-Country.mmdb +!internal/apps/agent/geoipdata/GeoLite2-City.mmdb /.superpowers/ /.worktrees/ diff --git a/docs/changelog/index.md b/docs/changelog/index.md index 28d1b78e..bf86c0d0 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -29,6 +29,7 @@ sidebar: false ### 变更 - 移除 WAF 规则旧固定黑白名单、地域名单与 PoW 数据库字段;升级后需在发布前重新编排规则。 +- Agent 现同时内嵌 Country 与 City MMDB,首次启动仅从程序内初始化缺失文件,网络下载只用于后续周期更新。 ### 修复 diff --git a/docs/design/waf-design.md b/docs/design/waf-design.md index 973f39ab..b187c0e7 100644 --- a/docs/design/waf-design.md +++ b/docs/design/waf-design.md @@ -8,7 +8,7 @@ Server 保存带坐标和修订号的编辑图,发布时再次校验并编译 IP 组独立于规则拓扑更新。手动、订阅和自动 IP 组由控制面维护,Agent 先原子替换 JSON、最后更新 checksum。协调 Worker 每 5 秒检查 checksum,仅变化时读取完整快照并分发给其它 Worker;失败时保留上一份有效数据。完整运行时快照上限为 20 MiB,Server 发布/同步与 Agent 落盘使用同一序列化校验;OpenResty 使用独立的 64 MiB 共享字典和非淘汰写入,容量不足时拒绝新版本而不破坏已提交快照。 -地域节点使用 Country 与 City MMDB。数据库不可用时地域匹配返回 `false` 并限频告警,不允许因数据损坏意外放行其它执行错误。 +地域节点使用 Country 与 City MMDB。Agent 首次启动时从程序内嵌数据库初始化缺失文件,后续按配置周期下载更新,请求处理始终读取 OpenResty 已加载的数据库。数据库不可用时地域匹配返回 `false` 并限频告警,不允许因数据损坏意外放行其它执行错误。 ## 安全顺序 diff --git a/docs/guide/waf-usage.md b/docs/guide/waf-usage.md index c9c31e66..3e7ea253 100644 --- a/docs/guide/waf-usage.md +++ b/docs/guide/waf-usage.md @@ -8,7 +8,7 @@ OpenFlare WAF 使用可视化有向无环图编排规则。新建规则时只填 - **通过**:结束当前规则;若路由仍有后续规则则继续执行。 - **阻止**:立即按配置的状态码和 HTML 响应终止请求。 - **IP 匹配**:配置 IP、CIDR 或 IP 组,分别连接 `true`、`false`。 -- **地域匹配**:按国家或地区代码分支;City MMDB 不可用时按未匹配处理。 +- **地域匹配**:按国家或地区代码分支;Country 与 City MMDB 缺失时由 Agent 从程序内嵌数据库初始化,并按配置周期更新。City MMDB 不可用时按未匹配处理。 - **PoW**:未完成挑战时接管请求,验证通过后沿 `next` 继续。 服务端会拒绝循环、悬空出口、不可达节点、重复端口连接和无效配置。保存时携带页面加载得到的 `revision`;发生 409 冲突时应重新加载,避免覆盖他人修改。 diff --git a/docs/plan/20260713-waf-orchestration.md b/docs/plan/20260713-waf-orchestration.md index 6ea1976c..04a38e4a 100644 --- a/docs/plan/20260713-waf-orchestration.md +++ b/docs/plan/20260713-waf-orchestration.md @@ -10,7 +10,7 @@ ## 实现状态(2026-07-13) -Tasks 1–11 已实现,包含三段数据库迁移、图模型与编译器、规则 API、发布快照、OpenResty 内存执行器、IP 组协调刷新、React Flow 编辑器、有序绑定、GeoLite2 City/Country 支持以及中文文档与 Swagger 更新。 +Tasks 1–11 已实现,包含三段数据库迁移、图模型与编译器、规则 API、发布快照、OpenResty 内存执行器、IP 组协调刷新、React Flow 编辑器、有序绑定、GeoLite2 City/Country 支持以及中文文档与 Swagger 更新。Country 与 City MMDB 均随 Agent 内嵌,缺失文件在启动时从程序内初始化,网络仅用于后续周期更新。 当前工作区已完成 `go test ./...`、前端全量 Vitest(54 项)、`make swagger`、`make code-check` 与 `git diff --check` 验证。Next.js 生产构建在本机持续停留于 Turbopack 的 `Creating an optimized production build ...`,未返回编译错误或成功状态,故不计为通过。 diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index af66a549..9a743a0f 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -319,8 +319,8 @@ Server 的所有核心基础配置定义在 `config.yaml` 中,且均支持环 | `mmdb_path` | WAF GeoIP mmdb 文件路径 | 否 | `data_dir/etc/openflare/GeoLite2-Country.mmdb` | | `city_mmdb_path` | WAF 地区匹配 City MMDB 文件路径 | 否 | `data_dir/etc/openflare/GeoLite2-City.mmdb` | | `mmdb_update_interval` | WAF GeoIP mmdb 更新间隔 | 否 | `86400000` 毫秒 (24h) | -| `mmdb_download_url` | WAF GeoIP mmdb 下载地址 | 否 | 内置 GeoLite2 Country 下载地址 | -| `city_mmdb_download_url` | WAF City MMDB 下载地址 | 否 | 内置 GeoLite2 City 下载地址 | +| `mmdb_download_url` | WAF GeoIP mmdb 周期更新地址 | 否 | GeoLite2 Country 更新地址;首次缺失时从程序内嵌数据库初始化 | +| `city_mmdb_download_url` | WAF City MMDB 周期更新地址 | 否 | GeoLite2 City 更新地址;首次缺失时从程序内嵌数据库初始化 | | `observability_buffer_path` | 观测补报缓冲文件路径 | 否 | `data_dir/var/lib/openflare/observability-buffer.json` | | `observability_replay_minutes` | 自动补传最近观测窗口分钟数 | 否 | `15` | | `state_path` | Agent 本地状态文件路径 | 否 | `data_dir/var/lib/openflare/agent-state.json` | diff --git a/internal/apps/agent/config/config.go b/internal/apps/agent/config/config.go index 127fe060..dd35e460 100644 --- a/internal/apps/agent/config/config.go +++ b/internal/apps/agent/config/config.go @@ -31,8 +31,8 @@ const ( defaultOpenRestyObservabilityPort = 18081 defaultObservabilityReplayMinutes = 15 defaultMMDBUpdateInterval = 24 * time.Hour - defaultMMDBDownloadURL = "https://raw.githubusercontent.com/Loyalsoldier/geoip/release/GeoLite2-Country.mmdb" - defaultCityMMDBDownloadURL = "https://raw.githubusercontent.com/Loyalsoldier/geoip/release/GeoLite2-City.mmdb" + defaultMMDBDownloadURL = "https://github.com/FyraLabs/geolite2/releases/latest/download/GeoLite2-Country.mmdb" + defaultCityMMDBDownloadURL = "https://github.com/FyraLabs/geolite2/releases/latest/download/GeoLite2-City.mmdb" defaultHeartbeatInterval = 10 * time.Second defaultRequestTimeout = 10 * time.Second configFilePerm = 0o600 diff --git a/internal/apps/agent/geoipdata/GeoLite2-City.mmdb b/internal/apps/agent/geoipdata/GeoLite2-City.mmdb new file mode 100644 index 00000000..e2769ee6 Binary files /dev/null and b/internal/apps/agent/geoipdata/GeoLite2-City.mmdb differ diff --git a/internal/apps/agent/geoipdata/data.go b/internal/apps/agent/geoipdata/data.go index b5572fcb..e0d310f2 100644 --- a/internal/apps/agent/geoipdata/data.go +++ b/internal/apps/agent/geoipdata/data.go @@ -1,12 +1,16 @@ -// Package geoipdata embeds the default MaxMind GeoLite2 country database. +// Package geoipdata embeds the default MaxMind GeoLite2 databases. package geoipdata import "embed" -// FS holds the embedded GeoLite2-Country.mmdb database. +// FS holds the embedded GeoLite2 Country and City databases. // -//go:embed GeoLite2-Country.mmdb +//go:embed GeoLite2-Country.mmdb GeoLite2-City.mmdb var FS embed.FS -// DefaultMMDBName is the filename of the embedded MaxMind country database. -const DefaultMMDBName = "GeoLite2-Country.mmdb" +const ( + // DefaultMMDBName is the filename of the embedded MaxMind Country database. + DefaultMMDBName = "GeoLite2-Country.mmdb" + // DefaultCityMMDBName is the filename of the embedded MaxMind City database. + DefaultCityMMDBName = "GeoLite2-City.mmdb" +) diff --git a/internal/apps/agent/geoipdata/data_test.go b/internal/apps/agent/geoipdata/data_test.go new file mode 100644 index 00000000..79d47c9f --- /dev/null +++ b/internal/apps/agent/geoipdata/data_test.go @@ -0,0 +1,38 @@ +package geoipdata + +import ( + "io/fs" + "strings" + "testing" + + "github.com/oschwald/maxminddb-golang" +) + +func TestEmbeddedDatabasesAreValid(t *testing.T) { + tests := []struct { + name string + filename string + databaseTypePart string + }{ + {name: "Country", filename: DefaultMMDBName, databaseTypePart: "Country"}, + {name: "City", filename: DefaultCityMMDBName, databaseTypePart: "City"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + data, err := fs.ReadFile(FS, test.filename) + if err != nil { + t.Fatalf("read embedded database: %v", err) + } + reader, err := maxminddb.FromBytes(data) + if err != nil { + t.Fatalf("open embedded database: %v", err) + } + defer reader.Close() + + if !strings.Contains(reader.Metadata.DatabaseType, test.databaseTypePart) { + t.Fatalf("unexpected database type %q", reader.Metadata.DatabaseType) + } + }) + } +} diff --git a/internal/apps/agent/geoipupdate/updater.go b/internal/apps/agent/geoipupdate/updater.go index 17eed6e1..e4d684a3 100644 --- a/internal/apps/agent/geoipupdate/updater.go +++ b/internal/apps/agent/geoipupdate/updater.go @@ -31,9 +31,13 @@ type Updater struct { downloadDatabase func(context.Context, string, string) error } -// EnsureInitialDatabase seeds the MMDB file from the embedded database if it does not exist on disk. +// EnsureInitialDatabase seeds the Country MMDB file from the embedded database if it does not exist on disk. func (u *Updater) EnsureInitialDatabase() error { - path := filepath.Clean(u.MMDBPath) + return ensureEmbeddedDatabase(u.MMDBPath, geoipdata.DefaultMMDBName, "Country") +} + +func ensureEmbeddedDatabase(targetPath string, embeddedName string, databaseName string) error { + path := filepath.Clean(targetPath) if path == "" || path == "." { return nil } @@ -42,9 +46,9 @@ func (u *Updater) EnsureInitialDatabase() error { } else if !os.IsNotExist(err) { return fmt.Errorf("stat mmdb file failed: %w", err) } - data, err := fs.ReadFile(geoipdata.FS, geoipdata.DefaultMMDBName) + data, err := fs.ReadFile(geoipdata.FS, embeddedName) if err != nil { - return fmt.Errorf("read embedded mmdb failed: %w", err) + return fmt.Errorf("read embedded %s mmdb failed: %w", databaseName, err) } if err := os.MkdirAll(filepath.Dir(path), mmdbDirPerm); err != nil { return fmt.Errorf("create mmdb directory failed: %w", err) @@ -52,31 +56,19 @@ func (u *Updater) EnsureInitialDatabase() error { if err := os.WriteFile(path, data, mmdbFilePerm); err != nil { return fmt.Errorf("write initial mmdb failed: %w", err) } - slog.Info("initialized GeoIP mmdb from embedded database", "path", path, "size", len(data)) + slog.Info("initialized GeoIP mmdb from embedded database", "database", databaseName, "path", path, "size", len(data)) return nil } -// EnsureInitialDatabases retains the embedded Country seed and immediately -// downloads City when it is absent so subdivision rules work before the first ticker interval. -func (u *Updater) EnsureInitialDatabases(ctx context.Context) error { +// EnsureInitialDatabases seeds both Country and City from embedded databases +// when either managed file is absent. Network downloads are reserved for the periodic updater. +func (u *Updater) EnsureInitialDatabases(_ context.Context) error { var errs []error if err := u.EnsureInitialDatabase(); err != nil { errs = append(errs, err) } - cityPath := filepath.Clean(u.CityMMDBPath) - if cityPath == "" || cityPath == "." || u.CityDownloadURL == "" { - return errors.Join(errs...) - } - if _, err := os.Stat(cityPath); err == nil { - return errors.Join(errs...) - } else if !os.IsNotExist(err) { - errs = append(errs, fmt.Errorf("stat City mmdb file failed: %w", err)) - return errors.Join(errs...) - } - if err := u.download(ctx, cityPath, u.CityDownloadURL); err != nil { - errs = append(errs, fmt.Errorf("download initial City mmdb failed: %w", err)) - } else { - slog.Info("initialized GeoIP City mmdb from provider", "path", cityPath) + if err := ensureEmbeddedDatabase(u.CityMMDBPath, geoipdata.DefaultCityMMDBName, "City"); err != nil { + errs = append(errs, err) } return errors.Join(errs...) } diff --git a/internal/apps/agent/geoipupdate/updater_test.go b/internal/apps/agent/geoipupdate/updater_test.go index 054c312e..cf1555dd 100644 --- a/internal/apps/agent/geoipupdate/updater_test.go +++ b/internal/apps/agent/geoipupdate/updater_test.go @@ -26,7 +26,7 @@ func TestEnsureInitialDatabaseCopiesEmbeddedMMDB(t *testing.T) { } } -func TestEnsureInitialDatabasesDownloadsMissingCity(t *testing.T) { +func TestEnsureInitialDatabasesCopiesEmbeddedCityWithoutDownload(t *testing.T) { tempDir := t.TempDir() countryPath := filepath.Join(tempDir, "GeoLite2-Country.mmdb") cityPath := filepath.Join(tempDir, "GeoLite2-City.mmdb") @@ -35,10 +35,8 @@ func TestEnsureInitialDatabasesDownloadsMissingCity(t *testing.T) { CityMMDBPath: cityPath, CityDownloadURL: "https://geo.example/GeoLite2-City.mmdb", downloadDatabase: func(_ context.Context, path, downloadURL string) error { - if path != cityPath || downloadURL != "https://geo.example/GeoLite2-City.mmdb" { - t.Fatalf("unexpected initial download: %s / %s", path, downloadURL) - } - return os.WriteFile(path, []byte("city-mmdb"), 0o600) + t.Fatalf("initial embedded seed must not download %s from %s", path, downloadURL) + return nil }, } @@ -49,15 +47,18 @@ func TestEnsureInitialDatabasesDownloadsMissingCity(t *testing.T) { t.Fatalf("expected embedded Country database: %v", err) } data, err := os.ReadFile(cityPath) - if err != nil || string(data) != "city-mmdb" { - t.Fatalf("expected downloaded City database, data=%q err=%v", data, err) + if err != nil || len(data) == 0 { + t.Fatalf("expected embedded City database, size=%d err=%v", len(data), err) } } -func TestEnsureInitialDatabasesKeepsCountryFallbackWhenCityDownloadFails(t *testing.T) { +func TestEnsureInitialDatabasesKeepsExistingCityWithoutDownload(t *testing.T) { tempDir := t.TempDir() countryPath := filepath.Join(tempDir, "GeoLite2-Country.mmdb") cityPath := filepath.Join(tempDir, "GeoLite2-City.mmdb") + if err := os.WriteFile(cityPath, []byte("existing-city"), 0o600); err != nil { + t.Fatal(err) + } updater := &Updater{ MMDBPath: countryPath, CityMMDBPath: cityPath, @@ -67,14 +68,15 @@ func TestEnsureInitialDatabasesKeepsCountryFallbackWhenCityDownloadFails(t *test }, } - if err := updater.EnsureInitialDatabases(context.Background()); err == nil { - t.Fatal("expected City download error to be reported") + if err := updater.EnsureInitialDatabases(context.Background()); err != nil { + t.Fatalf("EnsureInitialDatabases failed: %v", err) } if _, err := os.Stat(countryPath); err != nil { t.Fatalf("expected Country fallback to remain available: %v", err) } - if _, err := os.Stat(cityPath); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("expected failed City download not to create a database, err=%v", err) + data, err := os.ReadFile(cityPath) + if err != nil || string(data) != "existing-city" { + t.Fatalf("expected existing City database to remain untouched, data=%q err=%v", data, err) } }