diff --git a/docs/changelog/index.md b/docs/changelog/index.md index 5293b1e9..95a9ac29 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -19,6 +19,8 @@ sidebar: false ### 🛠 修复 +- SQLite 文件数据库启动时自动启用 WAL 模式，改善读写并发；数据库导出改用一致性快照，确保包含 WAL 中已提交的数据。 + - 修复 IP 组按规则自动抓取时会收录内网、回环及链路本地等地址的问题；规则测试同步过滤这些地址，历史抓取结果会在下次同步时清理。 ## [v3.5.8] - 2026-10-08 diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 678c07c6..380a4873 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -87,6 +87,8 @@ Server 的所有核心基础配置定义在 `config.yaml` 中，且均支持环 | `database.max_idle_conn` | `DB_MAX_IDLE_CONN` | 数据库连接池最大空闲连接数 | `16` | | `database.max_open_conn` | `DB_MAX_OPEN_CONN` | 数据库连接池最大打开连接数 | `128` | +SQLite 文件数据库在启动时自动启用 WAL 模式；内存数据库保留 `memory` 模式。请确保数据库所在目录可写，以创建 `-wal` 和 `-shm` 文件。在线备份请使用管理端数据库导出（生成一致性快照），不要仅复制运行中的 `.db` 文件。 + ### 3. Redis 配置 (`redis:`) | 配置文件 YAML 路径 | 对应覆盖环境变量 | 作用说明 | 默认值 | | --- | --- | --- | --- | diff --git a/internal/apps/admin/status/routers.go b/internal/apps/admin/status/routers.go index 37ee1434..a6f47f67 100644 --- a/internal/apps/admin/status/routers.go +++ b/internal/apps/admin/status/routers.go @@ -13,12 +13,15 @@ import ( "net/http" "os" "os/exec" + "path/filepath" "runtime" "strconv" "time" "github.com/Rain-kl/Wavelet/internal/infra/config" db "github.com/Rain-kl/Wavelet/internal/infra/persistence" + "github.com/Rain-kl/Wavelet/internal/repository" + "github.com/Rain-kl/Wavelet/pkg/logger" "github.com/gin-gonic/gin" "github.com/Rain-kl/Wavelet/internal/shared/response" @@ -284,14 +287,27 @@ func ExportDatabase(c *gin.Context) { } } -// exportSQLite 以 HTTP 附件方式下载 SQLite .db 文件 +// exportSQLite 以 HTTP 附件方式下载 SQLite 一致性快照。 func exportSQLite(c *gin.Context) { - path := config.Config.Database.SQLitePath - if path == "" { - path = "./data/openflare.db" + dir, err := os.MkdirTemp("", "openflare-db-export-*") + if err != nil { + logger.ErrorF(c.Request.Context(), "[db-export] create temporary directory failed: %v", err) + response.AbortInternal(c, "无法创建数据库导出文件") + return + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + logger.ErrorF(c.Request.Context(), "[db-export] remove temporary directory failed: %v", err) + } + }() + path := filepath.Join(dir, "openflare.db") + if err := repository.ExportSQLiteSnapshot(c.Request.Context(), path); err != nil { + logger.ErrorF(c.Request.Context(), "[db-export] create SQLite snapshot failed: %v", err) + response.AbortInternal(c, "无法导出数据库") + return } - f, err := os.Open(path) //nolint:gosec // path is loaded from server startup configuration, not user input + f, err := os.Open(path) //nolint:gosec // path is generated in a private temporary directory, not from user input if err != nil { response.AbortInternal(c, "无法打开数据库文件: "+err.Error()) return diff --git a/internal/infra/persistence/postgres.go b/internal/infra/persistence/postgres.go index 78760855..acde888a 100644 --- a/internal/infra/persistence/postgres.go +++ b/internal/infra/persistence/postgres.go @@ -6,6 +6,7 @@ package db import ( "context" + "fmt" "log" "net" "net/url" @@ -58,6 +59,10 @@ func initSQLite() { log.Fatalf("[SQLite] init connection failed: %v\n", err) } + if err = enableSQLiteWAL(db); err != nil { + log.Fatalf("[SQLite] enable WAL failed: %v\n", err) + } + // Trace 注入 if err = db.Use( newGORMTracingPlugin([]attribute.KeyValue{ @@ -71,6 +76,18 @@ func initSQLite() { log.Printf("[SQLite] initialized (path: %s)\n", sqlitePath) } +// enableSQLiteWAL 在文件数据库上启用 WAL；内存数据库仅支持 memory 日志模式。 +func enableSQLiteWAL(database *gorm.DB) error { + var mode string + if err := database.Raw("PRAGMA journal_mode = WAL").Scan(&mode).Error; err != nil { + return err + } + if mode != "wal" && mode != "memory" { + return fmt.Errorf("unexpected SQLite journal mode: %s", mode) + } + return nil +} + // initPostgres 初始化 PostgreSQL 数据库 func initPostgres() { var err error diff --git a/internal/infra/persistence/sqlite_test.go b/internal/infra/persistence/sqlite_test.go new file mode 100644 index 00000000..bb925ad1 --- /dev/null +++ b/internal/infra/persistence/sqlite_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package db + +import ( + "path/filepath" + "testing" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func TestEnableSQLiteWAL(t *testing.T) { + for _, tc := range []struct{ name, path, mode string }{ + {"file", filepath.Join(t.TempDir(), "database.db"), "wal"}, + {"memory", ":memory:", "memory"}, + } { + t.Run(tc.name, func(t *testing.T) { + database, err := gorm.Open(sqlite.Open(tc.path), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, err := database.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := sqlDB.Close(); err != nil { + t.Error(err) + } + }) + if err := enableSQLiteWAL(database); err != nil { + t.Fatal(err) + } + var mode string + if err := database.Raw("PRAGMA journal_mode").Scan(&mode).Error; err != nil { + t.Fatal(err) + } + if mode != tc.mode { + t.Fatalf("journal mode = %q, want %q", mode, tc.mode) + } + if err := enableSQLiteWAL(database); err != nil { + t.Fatalf("enable WAL again: %v", err) + } + }) + } +} diff --git a/internal/repository/database_export.go b/internal/repository/database_export.go new file mode 100644 index 00000000..d914d044 --- /dev/null +++ b/internal/repository/database_export.go @@ -0,0 +1,20 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package repository + +import ( + "context" + "errors" + + db "github.com/Rain-kl/Wavelet/internal/infra/persistence" +) + +// ExportSQLiteSnapshot 生成包含 WAL 中已提交数据的一致性快照，目标文件须不存在或为空。 +func ExportSQLiteSnapshot(ctx context.Context, path string) error { + database := db.DB(ctx) + if database == nil { + return errors.New(errDatabaseNotInitialized) + } + return database.Exec("VACUUM INTO ?", path).Error +} diff --git a/internal/repository/database_export_test.go b/internal/repository/database_export_test.go new file mode 100644 index 00000000..91e60b03 --- /dev/null +++ b/internal/repository/database_export_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 Arctel.net +// SPDX-License-Identifier: Apache-2.0 + +package repository + +import ( + "context" + "path/filepath" + "testing" + + db "github.com/Rain-kl/Wavelet/internal/infra/persistence" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func TestExportSQLiteSnapshotIncludesWAL(t *testing.T) { + dir := t.TempDir() + source, err := gorm.Open(sqlite.Open(filepath.Join(dir, "source.db")), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + sqlDB, err := source.DB() + if err != nil { + t.Fatal(err) + } + previous := db.DB(context.Background()) + db.SetDB(source) + t.Cleanup(func() { + db.SetDB(previous) + if err := sqlDB.Close(); err != nil { + t.Error(err) + } + }) + for _, query := range []string{ + "PRAGMA journal_mode = WAL", + "PRAGMA wal_autocheckpoint = 0", + "CREATE TABLE snapshot_data (value TEXT)", + "INSERT INTO snapshot_data VALUES ('latest committed data')", + } { + if err := source.Exec(query).Error; err != nil { + t.Fatal(err) + } + } + target := filepath.Join(dir, "snapshot.db") + if err := ExportSQLiteSnapshot(context.Background(), target); err != nil { + t.Fatal(err) + } + snapshot, err := gorm.Open(sqlite.Open(target), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + snapshotDB, err := snapshot.DB() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := snapshotDB.Close(); err != nil { + t.Error(err) + } + }) + var value string + if err := snapshot.Raw("SELECT value FROM snapshot_data").Scan(&value).Error; err != nil { + t.Fatal(err) + } + if value != "latest committed data" { + t.Fatalf("unexpected snapshot content: %q", value) + } + var integrity string + if err := snapshot.Raw("PRAGMA integrity_check").Scan(&integrity).Error; err != nil { + t.Fatal(err) + } + if integrity != "ok" { + t.Fatalf("snapshot integrity: %s", integrity) + } +}