diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 5fb0a806..e794b0a5 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -298,9 +298,7 @@ jobs: with: go-version-file: go.mod - - name: Fetch embedded GeoIP database - run: bash scripts/fetch-agent-geoip-mmdb.sh - + # GeoIP MMDB is not embedded; Docker images COPY mmdb files, bare binaries seed via download on first start. - name: Build Agent env: CGO_ENABLED: 0 diff --git a/.gitignore b/.gitignore index 30fd678c..3be52cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -77,8 +77,8 @@ profile.cov .grok /.gomodcache/ *.mmdb -!internal/apps/agent/geoipdata/GeoLite2-Country.mmdb -!internal/apps/agent/geoipdata/GeoLite2-City.mmdb +# Server control-plane MaxMind Country seed (Country only; Agent does not embed) +!internal/apps/openflare/geoip/data/GeoLite2-Country.mmdb /.superpowers/ /.worktrees/ diff --git a/docker/Dockerfile.agent b/docker/Dockerfile.agent index 9e5519e3..f9c37e8d 100644 --- a/docker/Dockerfile.agent +++ b/docker/Dockerfile.agent @@ -1,4 +1,5 @@ # syntax=docker/dockerfile:1.7 +# Agent image: slim binary + MMDB files on disk (not embedded in the binary). ARG VERSION=dev FROM golang:1.25-alpine AS builder @@ -17,12 +18,14 @@ RUN --mount=type=cache,target=/go/pkg/mod \ go mod download COPY . . -RUN apk add --no-cache bash curl \ - && bash scripts/fetch-agent-geoip-mmdb.sh RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ go build -trimpath -ldflags "-s -w -X 'github.com/Rain-kl/Wavelet/internal/apps/agent/config.Version=$VERSION'" -o /build/bin/openflare-agent ./cmd/agent/main.go +# Fetch MMDB into dist/geoip for COPY into the runtime image (not go:embed). +RUN apk add --no-cache bash curl \ + && bash scripts/fetch-agent-geoip-mmdb.sh + FROM openresty/openresty:alpine RUN apk add --no-cache ca-certificates tzdata perl libmaxminddb su-exec libcap \ @@ -30,7 +33,7 @@ RUN apk add --no-cache ca-certificates tzdata perl libmaxminddb su-exec libcap \ && opm get anjia0532/lua-resty-maxminddb \ && addgroup -S openflare \ && adduser -S -G openflare -H -h /data -s /sbin/nologin openflare \ - && mkdir -p /etc/openflare /data \ + && mkdir -p /etc/openflare /data/etc/openflare \ && chown -R openflare:openflare /etc/openflare /data \ && setcap 'cap_net_bind_service=+ep' /usr/local/openresty/nginx/sbin/nginx @@ -38,9 +41,15 @@ ENV OPENFLARE_OPENRESTY_PATH=openresty \ OPENFLARE_DATA_DIR=/data COPY --from=builder /build/bin/openflare-agent /usr/local/bin/openflare-agent +# Default agent paths: data_dir/etc/openflare/GeoLite2-*.mmdb +COPY --from=builder /build/dist/geoip/GeoLite2-Country.mmdb /data/etc/openflare/GeoLite2-Country.mmdb +COPY --from=builder /build/dist/geoip/GeoLite2-City.mmdb /data/etc/openflare/GeoLite2-City.mmdb +RUN chown openflare:openflare /data/etc/openflare/GeoLite2-Country.mmdb /data/etc/openflare/GeoLite2-City.mmdb \ + && chmod 644 /data/etc/openflare/GeoLite2-Country.mmdb /data/etc/openflare/GeoLite2-City.mmdb + COPY scripts/agent-entrypoint.sh /usr/local/bin/openflare-agent-entrypoint.sh RUN chmod +x /usr/local/bin/openflare-agent-entrypoint.sh EXPOSE 80 443 18081 ENTRYPOINT ["/usr/local/bin/openflare-agent-entrypoint.sh"] -CMD ["-config", "/etc/openflare/agent.json"] \ No newline at end of file +CMD ["-config", "/etc/openflare/agent.json"] diff --git a/docs/changelog/index.md b/docs/changelog/index.md index 4cb79e54..f02e9ca9 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -22,6 +22,10 @@ sidebar: false ## [unreleased] +### 改进 + +- Agent 不再将 GeoLite2 Country/City MMDB 嵌入二进制:Docker 镜像在默认数据目录 COPY 数据库文件,裸二进制首次启动时按需下载,显著减小 Agent 包体积;OpenResty 仍从磁盘路径读取 MMDB。Server 控制面仍仅内嵌 Country MMDB(不含 City),供可选 MaxMind 提供方离线初始化。 + ## [v3.4.4] - 2026-08-06 ### 新增 @@ -31,11 +35,19 @@ sidebar: false ### 修复 +- 修复源站错误页在边缘返回 HTTP 200、页面状态码显示异常(如 0)的问题:错误响应现在正确透传上游状态码,并在页面中展示真实状态码。 - 修复 Agent 在配置已对齐但磁盘校验和不一致时,Pages 等对账成功后仍保留 `LastError` 的问题,避免偶发网络失败被健康事件长期显示为「活动中」且无法自动恢复。 ### 改进 - 删除、撤销与未保存离开等确认操作统一改用页面内 AlertDialog,不再使用浏览器原生 `confirm` 弹窗,交互风格与系统其余对话框保持一致。 +- Cloudflare 分组添加域名成员时支持按顶级域分层展示、搜索筛选与批量勾选,可一次加入多个域名并排队同步。 +- Cloudflare 首页展示域名同步(sync_member)与分组同步(sync_group)任务执行记录,可筛选状态、查看详情与失败重试。 +- Cloudflare 域名/分组同步任务日志补充域名、分组、生效节点 IP、橙云状态及逐域名进度等关键信息,便于排查同步结果。 +- Cloudflare 域名同步与分组同步任务改为可在任务管理中调度的标准任务类型,并提供成员 ID / 分组 ID 参数表单。 +- Cloudflare 首页直接提供指向分组管理,并为分组详情增加自动刷新与手动刷新,减少页面跳转并及时展示同步状态。 +- 统一数据访问分层:业务持久化经 `internal/repository`,`internal/model` 仅保留实体与无 IO 领域规则,避免双轨 CRUD 与职责混淆。 +- 构建检查增加 `internal/model` 禁止直接访问数据库/Redis 的架构守卫,并收敛 model 与 repository 的错误文案定义边界。 ## [v3.4.3] - 2026-07-24 diff --git a/docs/deployment/deployment.md b/docs/deployment/deployment.md index 275e8556..e42e21b8 100644 --- a/docs/deployment/deployment.md +++ b/docs/deployment/deployment.md @@ -65,7 +65,7 @@ Agent: | OpenResty | 本地部署需要可执行 `openresty`,或通过 `--openresty-path` 指定路径 | | Docker | 仅 Docker 部署 Agent 镜像时需要 | | 网络 | Agent 节点必须能访问 Server 地址 | -| GeoIP | WAF 地域规则使用 Agent 本地 MaxMind mmdb;Agent 内置初始库并会定期更新 | +| GeoIP | WAF 地域规则使用 OpenResty 读取本地 MaxMind mmdb;镜像内置文件或首次下载,Agent 负责周期更新 | ### 硬件配置推荐 @@ -206,4 +206,4 @@ export LOG_LEVEL='info' 默认情况下,Agent 在 HTTP 心跳成功后会尝试升级为 WebSocket。升级成功时,Server 发布或激活配置会立即通知 Agent;如果 WebSocket 无法建立或意外断开,Agent 会自动退回 HTTP 心跳同步。 -WAF 地域规则依赖 Agent 本地 `GeoLite2-Country.mmdb`。Agent 启动时会在 `data_dir/etc/openflare/GeoLite2-Country.mmdb` 初始化内置数据库,并按配置周期尝试更新;更新失败只记录警告,不影响配置同步与 OpenResty reload。 +WAF 地域规则依赖 Agent 本地 `GeoLite2-Country.mmdb` / `GeoLite2-City.mmdb`(OpenResty `resty.maxminddb` 读磁盘路径)。Docker 镜像会将 MMDB COPY 到 `data_dir/etc/openflare/`;裸二进制安装时若文件缺失则首次启动按配置 URL 下载。Agent 按配置周期尝试更新;更新失败只记录警告,不影响配置同步与 OpenResty reload。MMDB **不**再嵌入 agent 二进制。Server 控制面可选 MaxMind 提供方仍**仅内嵌 Country**(约 9MB,不含 City)用于离线 seed。 diff --git a/docs/guide/waf-usage.md b/docs/guide/waf-usage.md index 297ffc62..393ace10 100644 --- a/docs/guide/waf-usage.md +++ b/docs/guide/waf-usage.md @@ -8,7 +8,7 @@ OpenFlare WAF 使用可视化有向无环图编排规则。新建规则时只填 - **通过**:结束当前规则;若路由仍有后续规则则继续执行。 - **阻止**:立即按配置的状态码和 HTML 响应终止请求。 - **IP 匹配**:配置 IP、CIDR 或 IP 组,分别连接 `true`、`false`。 -- **地域匹配**:按国家或 ISO 3166-2 一级行政区代码分支;国家列表同时显示中文名称与代码,行政区可按国家名、行政区名或代码搜索。Country 与 City MMDB 缺失时由 Agent 从程序内嵌数据库初始化,并按配置周期更新。City MMDB 不可用时按未匹配处理。 +- **地域匹配**:按国家或 ISO 3166-2 一级行政区代码分支;国家列表同时显示中文名称与代码,行政区可按国家名、行政区名或代码搜索。Country 与 City MMDB 由磁盘文件提供(Docker 镜像会 COPY 到默认路径;裸二进制首次启动时按配置 URL 下载),并按配置周期更新。City MMDB 不可用时按未匹配处理。 - **PoW**:未完成挑战时接管请求,验证通过后沿 `next` 继续。 服务端会拒绝循环、悬空出口、不可达节点、重复端口连接和无效配置。保存时携带页面加载得到的 `revision`;发生 409 冲突时应重新加载,避免覆盖他人修改。 diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 2a32daee..a07e7d3b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -333,8 +333,8 @@ Server 的所有核心基础配置定义在 `config.yaml` 中,且均支持环 | `mmdb_path` | WAF GeoIP mmdb 文件路径 | 否 | `data_dir/etc/openflare/GeoLite2-Country.mmdb` | | `city_mmdb_path` | WAF 地区匹配 City MMDB 文件路径 | 否 | `data_dir/etc/openflare/GeoLite2-City.mmdb` | | `mmdb_update_interval` | WAF GeoIP mmdb 更新间隔 | 否 | `86400000` 毫秒 (24h) | -| `mmdb_download_url` | WAF GeoIP mmdb 周期更新地址 | 否 | GeoLite2 Country 更新地址;首次缺失时从程序内嵌数据库初始化 | -| `city_mmdb_download_url` | WAF City MMDB 周期更新地址 | 否 | GeoLite2 City 更新地址;首次缺失时从程序内嵌数据库初始化 | +| `mmdb_download_url` | WAF GeoIP mmdb 周期更新地址 | 否 | GeoLite2 Country 更新地址;磁盘文件缺失时首次下载(Docker 镜像已 COPY 默认路径文件) | +| `city_mmdb_download_url` | WAF City MMDB 周期更新地址 | 否 | GeoLite2 City 更新地址;磁盘文件缺失时首次下载(Docker 镜像已 COPY 默认路径文件) | | `observability_buffer_path` | 观测补报缓冲文件路径 | 否 | `data_dir/var/lib/openflare/observability-buffer.json` | | `observability_replay_minutes` | 自动补传最近观测窗口分钟数 | 否 | `60` | | `state_path` | Agent 本地状态文件路径 | 否 | `data_dir/var/lib/openflare/agent-state.json` | diff --git a/internal/apps/agent/geoipdata/GeoLite2-City.mmdb b/internal/apps/agent/geoipdata/GeoLite2-City.mmdb deleted file mode 100644 index e2769ee6..00000000 Binary files a/internal/apps/agent/geoipdata/GeoLite2-City.mmdb and /dev/null differ diff --git a/internal/apps/agent/geoipdata/data.go b/internal/apps/agent/geoipdata/data.go index e0d310f2..e661d025 100644 --- a/internal/apps/agent/geoipdata/data.go +++ b/internal/apps/agent/geoipdata/data.go @@ -1,16 +1,13 @@ -// Package geoipdata embeds the default MaxMind GeoLite2 databases. +// Package geoipdata holds shared GeoIP database filename constants. +// +// MaxMind MMDB files are NOT embedded into the agent binary. Docker images +// COPY them onto the default data paths; bare binary installs seed via download +// on first start (see geoipupdate). package geoipdata -import "embed" - -// FS holds the embedded GeoLite2 Country and City databases. -// -//go:embed GeoLite2-Country.mmdb GeoLite2-City.mmdb -var FS embed.FS - const ( - // DefaultMMDBName is the filename of the embedded MaxMind Country database. + // DefaultMMDBName is the default Country database filename. DefaultMMDBName = "GeoLite2-Country.mmdb" - // DefaultCityMMDBName is the filename of the embedded MaxMind City database. + // DefaultCityMMDBName is the default City database filename. DefaultCityMMDBName = "GeoLite2-City.mmdb" ) diff --git a/internal/apps/agent/geoipdata/data_test.go b/internal/apps/agent/geoipdata/data_test.go deleted file mode 100644 index 79d47c9f..00000000 --- a/internal/apps/agent/geoipdata/data_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package geoipdata - -import ( - "io/fs" - "strings" - "testing" - - "github.com/oschwald/maxminddb-golang" -) - -func TestEmbeddedDatabasesAreValid(t *testing.T) { - tests := []struct { - name string - filename string - databaseTypePart string - }{ - {name: "Country", filename: DefaultMMDBName, databaseTypePart: "Country"}, - {name: "City", filename: DefaultCityMMDBName, databaseTypePart: "City"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - data, err := fs.ReadFile(FS, test.filename) - if err != nil { - t.Fatalf("read embedded database: %v", err) - } - reader, err := maxminddb.FromBytes(data) - if err != nil { - t.Fatalf("open embedded database: %v", err) - } - defer reader.Close() - - if !strings.Contains(reader.Metadata.DatabaseType, test.databaseTypePart) { - t.Fatalf("unexpected database type %q", reader.Metadata.DatabaseType) - } - }) - } -} diff --git a/internal/apps/agent/geoipupdate/updater.go b/internal/apps/agent/geoipupdate/updater.go index e4d684a3..f6ad7745 100644 --- a/internal/apps/agent/geoipupdate/updater.go +++ b/internal/apps/agent/geoipupdate/updater.go @@ -5,23 +5,16 @@ import ( "context" "errors" "fmt" - "io/fs" "log/slog" "os" "path/filepath" "time" - "github.com/Rain-kl/Wavelet/internal/apps/agent/geoipdata" "github.com/Rain-kl/Wavelet/pkg/geoip" ) -const ( - mmdbDirPerm = 0o750 - mmdbFilePerm = 0o600 -) - -// Updater periodically downloads a fresh GeoIP MMDB file and seeds the -// initial embedded database when none is present on disk. +// Updater periodically downloads a fresh GeoIP MMDB file and seeds missing +// databases via download (or relies on image-provided files under data_dir). type Updater struct { MMDBPath string DownloadURL string @@ -31,48 +24,58 @@ type Updater struct { downloadDatabase func(context.Context, string, string) error } -// EnsureInitialDatabase seeds the Country MMDB file from the embedded database if it does not exist on disk. -func (u *Updater) EnsureInitialDatabase() error { - return ensureEmbeddedDatabase(u.MMDBPath, geoipdata.DefaultMMDBName, "Country") -} - -func ensureEmbeddedDatabase(targetPath string, embeddedName string, databaseName string) error { - path := filepath.Clean(targetPath) - if path == "" || path == "." { +// EnsureInitialDatabases downloads any missing Country/City MMDB once. +// When files already exist (e.g. Docker image COPY), this is a no-op. +// Network is used only when a managed path is absent — not for binary embeds. +func (u *Updater) EnsureInitialDatabases(ctx context.Context) error { + if u == nil { return nil } - if _, err := os.Stat(path); err == nil { - return nil - } else if !os.IsNotExist(err) { - return fmt.Errorf("stat mmdb file failed: %w", err) - } - data, err := fs.ReadFile(geoipdata.FS, embeddedName) - if err != nil { - return fmt.Errorf("read embedded %s mmdb failed: %w", databaseName, err) - } - if err := os.MkdirAll(filepath.Dir(path), mmdbDirPerm); err != nil { - return fmt.Errorf("create mmdb directory failed: %w", err) - } - if err := os.WriteFile(path, data, mmdbFilePerm); err != nil { - return fmt.Errorf("write initial mmdb failed: %w", err) - } - slog.Info("initialized GeoIP mmdb from embedded database", "database", databaseName, "path", path, "size", len(data)) - return nil + return u.ensureMissingDatabases(ctx) } -// EnsureInitialDatabases seeds both Country and City from embedded databases -// when either managed file is absent. Network downloads are reserved for the periodic updater. -func (u *Updater) EnsureInitialDatabases(_ context.Context) error { +func (u *Updater) ensureMissingDatabases(ctx context.Context) error { + databases := u.managedDatabases() var errs []error - if err := u.EnsureInitialDatabase(); err != nil { - errs = append(errs, err) - } - if err := ensureEmbeddedDatabase(u.CityMMDBPath, geoipdata.DefaultCityMMDBName, "City"); err != nil { - errs = append(errs, err) + for _, database := range databases { + if database.path == "" || database.downloadURL == "" { + continue + } + exists, err := fileExists(database.path) + if err != nil { + errs = append(errs, fmt.Errorf("stat GeoIP %s mmdb failed: %w", database.name, err)) + continue + } + if exists { + continue + } + if err := u.download(ctx, database.path, database.downloadURL); err != nil { + errs = append(errs, fmt.Errorf("seed GeoIP %s mmdb failed: %w", database.name, err)) + continue + } + slog.Info("seeded GeoIP mmdb via download", "database", database.name, "path", database.path) } return errors.Join(errs...) } +func fileExists(path string) (bool, error) { + path = filepath.Clean(path) + if path == "" || path == "." { + return false, nil + } + info, err := os.Stat(path) + if err == nil { + if !info.Mode().IsRegular() { + return false, fmt.Errorf("GeoIP MMDB path is not a regular file: %s", path) + } + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + func (u *Updater) download(ctx context.Context, path string, downloadURL string) error { if u.downloadDatabase != nil { return u.downloadDatabase(ctx, path, downloadURL) @@ -80,8 +83,12 @@ func (u *Updater) download(ctx context.Context, path string, downloadURL string) return geoip.DownloadMaxMindDatabase(ctx, path, downloadURL) } -func (u *Updater) updateDatabases(ctx context.Context) error { - databases := []struct { +func (u *Updater) managedDatabases() []struct { + name string + path string + downloadURL string +} { + return []struct { name string path string downloadURL string @@ -89,9 +96,12 @@ func (u *Updater) updateDatabases(ctx context.Context) error { {name: "Country", path: u.MMDBPath, downloadURL: u.DownloadURL}, {name: "City", path: u.CityMMDBPath, downloadURL: u.CityDownloadURL}, } +} + +func (u *Updater) updateDatabases(ctx context.Context) error { var errs []error - for _, database := range databases { - if database.path == "" || (database.name == "City" && database.downloadURL == "") { + for _, database := range u.managedDatabases() { + if database.path == "" || database.downloadURL == "" { continue } if err := u.download(ctx, database.path, database.downloadURL); err != nil { diff --git a/internal/apps/agent/geoipupdate/updater_test.go b/internal/apps/agent/geoipupdate/updater_test.go index cf1555dd..0fcc442a 100644 --- a/internal/apps/agent/geoipupdate/updater_test.go +++ b/internal/apps/agent/geoipupdate/updater_test.go @@ -6,77 +6,66 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" ) -func TestEnsureInitialDatabaseCopiesEmbeddedMMDB(t *testing.T) { - tempDir := t.TempDir() - path := filepath.Join(tempDir, "GeoLite2-Country.mmdb") - updater := &Updater{MMDBPath: path} - - if err := updater.EnsureInitialDatabase(); err != nil { - t.Fatalf("EnsureInitialDatabase failed: %v", err) - } - info, err := os.Stat(path) - if err != nil { - t.Fatalf("expected mmdb to exist: %v", err) - } - if info.Size() == 0 { - t.Fatal("expected copied mmdb to be non-empty") - } -} - -func TestEnsureInitialDatabasesCopiesEmbeddedCityWithoutDownload(t *testing.T) { - tempDir := t.TempDir() - countryPath := filepath.Join(tempDir, "GeoLite2-Country.mmdb") - cityPath := filepath.Join(tempDir, "GeoLite2-City.mmdb") - updater := &Updater{ - MMDBPath: countryPath, - CityMMDBPath: cityPath, - CityDownloadURL: "https://geo.example/GeoLite2-City.mmdb", - downloadDatabase: func(_ context.Context, path, downloadURL string) error { - t.Fatalf("initial embedded seed must not download %s from %s", path, downloadURL) - return nil - }, - } - - if err := updater.EnsureInitialDatabases(context.Background()); err != nil { - t.Fatalf("EnsureInitialDatabases failed: %v", err) - } - if _, err := os.Stat(countryPath); err != nil { - t.Fatalf("expected embedded Country database: %v", err) - } - data, err := os.ReadFile(cityPath) - if err != nil || len(data) == 0 { - t.Fatalf("expected embedded City database, size=%d err=%v", len(data), err) - } -} - -func TestEnsureInitialDatabasesKeepsExistingCityWithoutDownload(t *testing.T) { +func TestEnsureInitialDatabasesDownloadsMissingOnly(t *testing.T) { tempDir := t.TempDir() countryPath := filepath.Join(tempDir, "GeoLite2-Country.mmdb") cityPath := filepath.Join(tempDir, "GeoLite2-City.mmdb") if err := os.WriteFile(cityPath, []byte("existing-city"), 0o600); err != nil { t.Fatal(err) } + + var downloaded []string updater := &Updater{ MMDBPath: countryPath, + DownloadURL: "https://geo.example/GeoLite2-Country.mmdb", CityMMDBPath: cityPath, CityDownloadURL: "https://geo.example/GeoLite2-City.mmdb", - downloadDatabase: func(_ context.Context, _, _ string) error { - return errors.New("city unavailable") + downloadDatabase: func(_ context.Context, path, _ string) error { + downloaded = append(downloaded, path) + return os.WriteFile(path, []byte("downloaded"), 0o600) }, } if err := updater.EnsureInitialDatabases(context.Background()); err != nil { t.Fatalf("EnsureInitialDatabases failed: %v", err) } - if _, err := os.Stat(countryPath); err != nil { - t.Fatalf("expected Country fallback to remain available: %v", err) + if !slices.Equal(downloaded, []string{countryPath}) { + t.Fatalf("expected only missing Country download, got %#v", downloaded) } - data, err := os.ReadFile(cityPath) - if err != nil || string(data) != "existing-city" { - t.Fatalf("expected existing City database to remain untouched, data=%q err=%v", data, err) + if data, err := os.ReadFile(cityPath); err != nil || string(data) != "existing-city" { + t.Fatalf("existing City must stay untouched, data=%q err=%v", data, err) + } + if data, err := os.ReadFile(countryPath); err != nil || string(data) != "downloaded" { + t.Fatalf("Country should be seeded via download, data=%q err=%v", data, err) + } +} + +func TestEnsureInitialDatabasesNoOpWhenPresent(t *testing.T) { + tempDir := t.TempDir() + countryPath := filepath.Join(tempDir, "GeoLite2-Country.mmdb") + cityPath := filepath.Join(tempDir, "GeoLite2-City.mmdb") + if err := os.WriteFile(countryPath, []byte("c"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cityPath, []byte("city"), 0o600); err != nil { + t.Fatal(err) + } + updater := &Updater{ + MMDBPath: countryPath, + DownloadURL: "https://geo.example/GeoLite2-Country.mmdb", + CityMMDBPath: cityPath, + CityDownloadURL: "https://geo.example/GeoLite2-City.mmdb", + downloadDatabase: func(_ context.Context, path, downloadURL string) error { + t.Fatalf("must not download when files exist: %s %s", path, downloadURL) + return nil + }, + } + if err := updater.EnsureInitialDatabases(context.Background()); err != nil { + t.Fatalf("EnsureInitialDatabases failed: %v", err) } } @@ -101,3 +90,24 @@ func TestUpdateDatabasesAttemptsCityAfterCountryFailure(t *testing.T) { t.Fatalf("expected independent Country then City attempts, paths=%#v err=%v", paths, err) } } + +func TestEnsureInitialDatabasesRejectsDirectoryPath(t *testing.T) { + tempDir := t.TempDir() + // Point Country path at a directory so fileExists must not treat it as seeded. + updater := &Updater{ + MMDBPath: tempDir, + DownloadURL: "https://geo.example/GeoLite2-Country.mmdb", + downloadDatabase: func(_ context.Context, path, downloadURL string) error { + t.Fatalf("must not download when path is a directory: %s %s", path, downloadURL) + return nil + }, + } + + err := updater.EnsureInitialDatabases(context.Background()) + if err == nil { + t.Fatal("expected error when MMDB path is a directory") + } + if !strings.Contains(err.Error(), "not a regular file") { + t.Fatalf("expected regular-file error, got %v", err) + } +} diff --git a/internal/apps/agent/geoipdata/GeoLite2-Country.mmdb b/internal/apps/openflare/geoip/data/GeoLite2-Country.mmdb similarity index 100% rename from internal/apps/agent/geoipdata/GeoLite2-Country.mmdb rename to internal/apps/openflare/geoip/data/GeoLite2-Country.mmdb diff --git a/internal/apps/openflare/geoip/data/data.go b/internal/apps/openflare/geoip/data/data.go new file mode 100644 index 00000000..dfbcae06 --- /dev/null +++ b/internal/apps/openflare/geoip/data/data.go @@ -0,0 +1,15 @@ +// Package data embeds the MaxMind GeoLite2 Country database for the control plane. +// +// Server keeps a Country-only embed so MaxMind provider can seed without network. +// Agent does NOT use this package — Agent MMDB files are image COPY / download only. +package data + +import "embed" + +// FS holds the embedded GeoLite2-Country.mmdb database. +// +//go:embed GeoLite2-Country.mmdb +var FS embed.FS + +// DefaultMMDBName is the filename of the embedded MaxMind Country database. +const DefaultMMDBName = "GeoLite2-Country.mmdb" diff --git a/internal/apps/openflare/geoip/data/data_test.go b/internal/apps/openflare/geoip/data/data_test.go new file mode 100644 index 00000000..8c43a70f --- /dev/null +++ b/internal/apps/openflare/geoip/data/data_test.go @@ -0,0 +1,25 @@ +package data + +import ( + "io/fs" + "strings" + "testing" + + "github.com/oschwald/maxminddb-golang" +) + +func TestEmbeddedCountryDatabaseIsValid(t *testing.T) { + raw, err := fs.ReadFile(FS, DefaultMMDBName) + if err != nil { + t.Fatalf("read embedded Country database: %v", err) + } + reader, err := maxminddb.FromBytes(raw) + if err != nil { + t.Fatalf("open embedded Country database: %v", err) + } + defer reader.Close() + + if !strings.Contains(reader.Metadata.DatabaseType, "Country") { + t.Fatalf("unexpected database type %q", reader.Metadata.DatabaseType) + } +} diff --git a/internal/apps/openflare/geoip/runtime.go b/internal/apps/openflare/geoip/runtime.go index 3207820f..d6bac910 100644 --- a/internal/apps/openflare/geoip/runtime.go +++ b/internal/apps/openflare/geoip/runtime.go @@ -11,7 +11,7 @@ import ( "strings" "sync" - "github.com/Rain-kl/Wavelet/internal/apps/agent/geoipdata" + geodata "github.com/Rain-kl/Wavelet/internal/apps/openflare/geoip/data" "github.com/Rain-kl/Wavelet/internal/model" "github.com/Rain-kl/Wavelet/internal/repository" pkggeoip "github.com/Rain-kl/Wavelet/pkg/geoip" @@ -94,11 +94,12 @@ func ensureServerMMDB() (string, error) { if _, err := os.Stat(path); err == nil { return path, nil } - if !os.IsNotExist(err) { + if err != nil && !os.IsNotExist(err) { return "", err } - data, err := fs.ReadFile(geoipdata.FS, geoipdata.DefaultMMDBName) + // Control plane: seed Country from embedded asset (no City; Agent uses disk/image). + data, err := fs.ReadFile(geodata.FS, geodata.DefaultMMDBName) if err != nil { return "", err } diff --git a/scripts/fetch-agent-geoip-mmdb.sh b/scripts/fetch-agent-geoip-mmdb.sh index 5cadbd23..98a446d8 100755 --- a/scripts/fetch-agent-geoip-mmdb.sh +++ b/scripts/fetch-agent-geoip-mmdb.sh @@ -1,25 +1,32 @@ #!/usr/bin/env bash +# Download MaxMind GeoLite2 Country/City databases for packaging (Docker image COPY), +# not for go:embed into the agent binary. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -TARGET="${ROOT}/internal/apps/agent/geoipdata/GeoLite2-Country.mmdb" -URL="${GEOIP_MMDB_URL:-https://raw.githubusercontent.com/Loyalsoldier/geoip/release/GeoLite2-Country.mmdb}" +OUT_DIR="${GEOIP_OUT_DIR:-${ROOT}/dist/geoip}" +COUNTRY_URL="${GEOIP_MMDB_URL:-https://github.com/FyraLabs/geolite2/releases/latest/download/GeoLite2-Country.mmdb}" +CITY_URL="${GEOIP_CITY_MMDB_URL:-https://github.com/FyraLabs/geolite2/releases/latest/download/GeoLite2-City.mmdb}" -mkdir -p "$(dirname "$TARGET")" +mkdir -p "${OUT_DIR}" -if curl -fsSL -o "${TARGET}.tmp" "$URL"; then - mv "${TARGET}.tmp" "$TARGET" - echo "GeoIP database downloaded: $TARGET" -elif [ -s "$TARGET" ]; then - rm -f "${TARGET}.tmp" - echo "GeoIP download failed, using committed database: $TARGET" -else - rm -f "${TARGET}.tmp" - echo "GeoIP database missing and download failed: $URL" >&2 - exit 1 -fi +download_one() { + local url="$1" + local dest="$2" + local name="$3" + if curl -fsSL -o "${dest}.tmp" "$url" && [ -s "${dest}.tmp" ]; then + mv "${dest}.tmp" "$dest" + echo "GeoIP ${name} downloaded: $dest ($(wc -c <"$dest" | tr -d ' ') bytes)" + return 0 + fi + rm -f "${dest}.tmp" + if [ -s "$dest" ]; then + echo "GeoIP ${name} download failed, keeping existing: $dest" >&2 + return 0 + fi + echo "GeoIP ${name} missing and download failed: $url" >&2 + return 1 +} -if [ ! -s "$TARGET" ]; then - echo "GeoIP database is empty: $TARGET" >&2 - exit 1 -fi \ No newline at end of file +download_one "$COUNTRY_URL" "${OUT_DIR}/GeoLite2-Country.mmdb" "Country" +download_one "$CITY_URL" "${OUT_DIR}/GeoLite2-City.mmdb" "City"