fix(sqlite): enable WAL and export consistent snapshots

This commit is contained in:
Noru Wyrms
2026-10-25 02:27:05 +00:00
parent 02a2b79c08
commit ada0550571
7 changed files with 185 additions and 5 deletions
+2
View File
@@ -19,6 +19,8 @@ sidebar: false
### 🛠 修复
- SQLite 文件数据库启动时自动启用 WAL 模式,改善读写并发;数据库导出改用一致性快照,确保包含 WAL 中已提交的数据。
- 修复 IP 组按规则自动抓取时会收录内网、回环及链路本地等地址的问题;规则测试同步过滤这些地址,历史抓取结果会在下次同步时清理。
## [v3.5.8] - 2026-10-08
+2
View File
@@ -87,6 +87,8 @@ Server 的所有核心基础配置定义在 `config.yaml` 中,且均支持环
| `database.max_idle_conn` | `DB_MAX_IDLE_CONN` | 数据库连接池最大空闲连接数 | `16` |
| `database.max_open_conn` | `DB_MAX_OPEN_CONN` | 数据库连接池最大打开连接数 | `128` |
SQLite 文件数据库在启动时自动启用 WAL 模式;内存数据库保留 `memory` 模式。请确保数据库所在目录可写,以创建 `-wal` 和 `-shm` 文件。在线备份请使用管理端数据库导出(生成一致性快照),不要仅复制运行中的 `.db` 文件。
### 3. Redis 配置 (`redis:`)
| 配置文件 YAML 路径 | 对应覆盖环境变量 | 作用说明 | 默认值 |
| --- | --- | --- | --- |
+21 -5
View File
@@ -13,12 +13,15 @@ import (
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"time"
"github.com/Rain-kl/Wavelet/internal/infra/config"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/Rain-kl/Wavelet/internal/repository"
"github.com/Rain-kl/Wavelet/pkg/logger"
"github.com/gin-gonic/gin"
"github.com/Rain-kl/Wavelet/internal/shared/response"
@@ -284,14 +287,27 @@ func ExportDatabase(c *gin.Context) {
}
}
// exportSQLite 以 HTTP 附件方式下载 SQLite .db 文件
// exportSQLite 以 HTTP 附件方式下载 SQLite 一致性快照。
func exportSQLite(c *gin.Context) {
path := config.Config.Database.SQLitePath
if path == "" {
path = "./data/openflare.db"
dir, err := os.MkdirTemp("", "openflare-db-export-*")
if err != nil {
logger.ErrorF(c.Request.Context(), "[db-export] create temporary directory failed: %v", err)
response.AbortInternal(c, "无法创建数据库导出文件")
return
}
defer func() {
if err := os.RemoveAll(dir); err != nil {
logger.ErrorF(c.Request.Context(), "[db-export] remove temporary directory failed: %v", err)
}
}()
path := filepath.Join(dir, "openflare.db")
if err := repository.ExportSQLiteSnapshot(c.Request.Context(), path); err != nil {
logger.ErrorF(c.Request.Context(), "[db-export] create SQLite snapshot failed: %v", err)
response.AbortInternal(c, "无法导出数据库")
return
}
f, err := os.Open(path) //nolint:gosec // path is loaded from server startup configuration, not user input
f, err := os.Open(path) //nolint:gosec // path is generated in a private temporary directory, not from user input
if err != nil {
response.AbortInternal(c, "无法打开数据库文件: "+err.Error())
return
+17
View File
@@ -6,6 +6,7 @@ package db
import (
"context"
"fmt"
"log"
"net"
"net/url"
@@ -58,6 +59,10 @@ func initSQLite() {
log.Fatalf("[SQLite] init connection failed: %v\n", err)
}
if err = enableSQLiteWAL(db); err != nil {
log.Fatalf("[SQLite] enable WAL failed: %v\n", err)
}
// Trace 注入
if err = db.Use(
newGORMTracingPlugin([]attribute.KeyValue{
@@ -71,6 +76,18 @@ func initSQLite() {
log.Printf("[SQLite] initialized (path: %s)\n", sqlitePath)
}
// enableSQLiteWAL 在文件数据库上启用 WAL;内存数据库仅支持 memory 日志模式。
func enableSQLiteWAL(database *gorm.DB) error {
var mode string
if err := database.Raw("PRAGMA journal_mode = WAL").Scan(&mode).Error; err != nil {
return err
}
if mode != "wal" && mode != "memory" {
return fmt.Errorf("unexpected SQLite journal mode: %s", mode)
}
return nil
}
// initPostgres 初始化 PostgreSQL 数据库
func initPostgres() {
var err error
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package db
import (
"path/filepath"
"testing"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func TestEnableSQLiteWAL(t *testing.T) {
for _, tc := range []struct{ name, path, mode string }{
{"file", filepath.Join(t.TempDir(), "database.db"), "wal"},
{"memory", ":memory:", "memory"},
} {
t.Run(tc.name, func(t *testing.T) {
database, err := gorm.Open(sqlite.Open(tc.path), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
sqlDB, err := database.DB()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := sqlDB.Close(); err != nil {
t.Error(err)
}
})
if err := enableSQLiteWAL(database); err != nil {
t.Fatal(err)
}
var mode string
if err := database.Raw("PRAGMA journal_mode").Scan(&mode).Error; err != nil {
t.Fatal(err)
}
if mode != tc.mode {
t.Fatalf("journal mode = %q, want %q", mode, tc.mode)
}
if err := enableSQLiteWAL(database); err != nil {
t.Fatalf("enable WAL again: %v", err)
}
})
}
}
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"errors"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
)
// ExportSQLiteSnapshot 生成包含 WAL 中已提交数据的一致性快照,目标文件须不存在或为空。
func ExportSQLiteSnapshot(ctx context.Context, path string) error {
database := db.DB(ctx)
if database == nil {
return errors.New(errDatabaseNotInitialized)
}
return database.Exec("VACUUM INTO ?", path).Error
}
@@ -0,0 +1,75 @@
// Copyright 2026 Arctel.net
// SPDX-License-Identifier: Apache-2.0
package repository
import (
"context"
"path/filepath"
"testing"
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func TestExportSQLiteSnapshotIncludesWAL(t *testing.T) {
dir := t.TempDir()
source, err := gorm.Open(sqlite.Open(filepath.Join(dir, "source.db")), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
sqlDB, err := source.DB()
if err != nil {
t.Fatal(err)
}
previous := db.DB(context.Background())
db.SetDB(source)
t.Cleanup(func() {
db.SetDB(previous)
if err := sqlDB.Close(); err != nil {
t.Error(err)
}
})
for _, query := range []string{
"PRAGMA journal_mode = WAL",
"PRAGMA wal_autocheckpoint = 0",
"CREATE TABLE snapshot_data (value TEXT)",
"INSERT INTO snapshot_data VALUES ('latest committed data')",
} {
if err := source.Exec(query).Error; err != nil {
t.Fatal(err)
}
}
target := filepath.Join(dir, "snapshot.db")
if err := ExportSQLiteSnapshot(context.Background(), target); err != nil {
t.Fatal(err)
}
snapshot, err := gorm.Open(sqlite.Open(target), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
snapshotDB, err := snapshot.DB()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := snapshotDB.Close(); err != nil {
t.Error(err)
}
})
var value string
if err := snapshot.Raw("SELECT value FROM snapshot_data").Scan(&value).Error; err != nil {
t.Fatal(err)
}
if value != "latest committed data" {
t.Fatalf("unexpected snapshot content: %q", value)
}
var integrity string
if err := snapshot.Raw("PRAGMA integrity_check").Scan(&integrity).Error; err != nil {
t.Fatal(err)
}
if integrity != "ok" {
t.Fatalf("snapshot integrity: %s", integrity)
}
}