mirror of
https://github.com/Rain-kl/OpenFlare.git
synced 2026-11-12 21:44:22 +08:00
fix(sqlite): enable WAL and export consistent snapshots
This commit is contained in:
@@ -19,6 +19,8 @@ sidebar: false
|
||||
|
||||
### 🛠 修复
|
||||
|
||||
- SQLite 文件数据库启动时自动启用 WAL 模式,改善读写并发;数据库导出改用一致性快照,确保包含 WAL 中已提交的数据。
|
||||
|
||||
- 修复 IP 组按规则自动抓取时会收录内网、回环及链路本地等地址的问题;规则测试同步过滤这些地址,历史抓取结果会在下次同步时清理。
|
||||
|
||||
## [v3.5.8] - 2026-10-08
|
||||
|
||||
@@ -87,6 +87,8 @@ Server 的所有核心基础配置定义在 `config.yaml` 中,且均支持环
|
||||
| `database.max_idle_conn` | `DB_MAX_IDLE_CONN` | 数据库连接池最大空闲连接数 | `16` |
|
||||
| `database.max_open_conn` | `DB_MAX_OPEN_CONN` | 数据库连接池最大打开连接数 | `128` |
|
||||
|
||||
SQLite 文件数据库在启动时自动启用 WAL 模式;内存数据库保留 `memory` 模式。请确保数据库所在目录可写,以创建 `-wal` 和 `-shm` 文件。在线备份请使用管理端数据库导出(生成一致性快照),不要仅复制运行中的 `.db` 文件。
|
||||
|
||||
### 3. Redis 配置 (`redis:`)
|
||||
| 配置文件 YAML 路径 | 对应覆盖环境变量 | 作用说明 | 默认值 |
|
||||
| --- | --- | --- | --- |
|
||||
|
||||
@@ -13,12 +13,15 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/infra/config"
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/Rain-kl/Wavelet/internal/repository"
|
||||
"github.com/Rain-kl/Wavelet/pkg/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/Rain-kl/Wavelet/internal/shared/response"
|
||||
@@ -284,14 +287,27 @@ func ExportDatabase(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// exportSQLite 以 HTTP 附件方式下载 SQLite .db 文件
|
||||
// exportSQLite 以 HTTP 附件方式下载 SQLite 一致性快照。
|
||||
func exportSQLite(c *gin.Context) {
|
||||
path := config.Config.Database.SQLitePath
|
||||
if path == "" {
|
||||
path = "./data/openflare.db"
|
||||
dir, err := os.MkdirTemp("", "openflare-db-export-*")
|
||||
if err != nil {
|
||||
logger.ErrorF(c.Request.Context(), "[db-export] create temporary directory failed: %v", err)
|
||||
response.AbortInternal(c, "无法创建数据库导出文件")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
logger.ErrorF(c.Request.Context(), "[db-export] remove temporary directory failed: %v", err)
|
||||
}
|
||||
}()
|
||||
path := filepath.Join(dir, "openflare.db")
|
||||
if err := repository.ExportSQLiteSnapshot(c.Request.Context(), path); err != nil {
|
||||
logger.ErrorF(c.Request.Context(), "[db-export] create SQLite snapshot failed: %v", err)
|
||||
response.AbortInternal(c, "无法导出数据库")
|
||||
return
|
||||
}
|
||||
|
||||
f, err := os.Open(path) //nolint:gosec // path is loaded from server startup configuration, not user input
|
||||
f, err := os.Open(path) //nolint:gosec // path is generated in a private temporary directory, not from user input
|
||||
if err != nil {
|
||||
response.AbortInternal(c, "无法打开数据库文件: "+err.Error())
|
||||
return
|
||||
|
||||
@@ -6,6 +6,7 @@ package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/url"
|
||||
@@ -58,6 +59,10 @@ func initSQLite() {
|
||||
log.Fatalf("[SQLite] init connection failed: %v\n", err)
|
||||
}
|
||||
|
||||
if err = enableSQLiteWAL(db); err != nil {
|
||||
log.Fatalf("[SQLite] enable WAL failed: %v\n", err)
|
||||
}
|
||||
|
||||
// Trace 注入
|
||||
if err = db.Use(
|
||||
newGORMTracingPlugin([]attribute.KeyValue{
|
||||
@@ -71,6 +76,18 @@ func initSQLite() {
|
||||
log.Printf("[SQLite] initialized (path: %s)\n", sqlitePath)
|
||||
}
|
||||
|
||||
// enableSQLiteWAL 在文件数据库上启用 WAL;内存数据库仅支持 memory 日志模式。
|
||||
func enableSQLiteWAL(database *gorm.DB) error {
|
||||
var mode string
|
||||
if err := database.Raw("PRAGMA journal_mode = WAL").Scan(&mode).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if mode != "wal" && mode != "memory" {
|
||||
return fmt.Errorf("unexpected SQLite journal mode: %s", mode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// initPostgres 初始化 PostgreSQL 数据库
|
||||
func initPostgres() {
|
||||
var err error
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestEnableSQLiteWAL(t *testing.T) {
|
||||
for _, tc := range []struct{ name, path, mode string }{
|
||||
{"file", filepath.Join(t.TempDir(), "database.db"), "wal"},
|
||||
{"memory", ":memory:", "memory"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
database, err := gorm.Open(sqlite.Open(tc.path), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, err := database.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
if err := enableSQLiteWAL(database); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mode string
|
||||
if err := database.Raw("PRAGMA journal_mode").Scan(&mode).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mode != tc.mode {
|
||||
t.Fatalf("journal mode = %q, want %q", mode, tc.mode)
|
||||
}
|
||||
if err := enableSQLiteWAL(database); err != nil {
|
||||
t.Fatalf("enable WAL again: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
)
|
||||
|
||||
// ExportSQLiteSnapshot 生成包含 WAL 中已提交数据的一致性快照,目标文件须不存在或为空。
|
||||
func ExportSQLiteSnapshot(ctx context.Context, path string) error {
|
||||
database := db.DB(ctx)
|
||||
if database == nil {
|
||||
return errors.New(errDatabaseNotInitialized)
|
||||
}
|
||||
return database.Exec("VACUUM INTO ?", path).Error
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// Copyright 2026 Arctel.net
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
db "github.com/Rain-kl/Wavelet/internal/infra/persistence"
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestExportSQLiteSnapshotIncludesWAL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
source, err := gorm.Open(sqlite.Open(filepath.Join(dir, "source.db")), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, err := source.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous := db.DB(context.Background())
|
||||
db.SetDB(source)
|
||||
t.Cleanup(func() {
|
||||
db.SetDB(previous)
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
for _, query := range []string{
|
||||
"PRAGMA journal_mode = WAL",
|
||||
"PRAGMA wal_autocheckpoint = 0",
|
||||
"CREATE TABLE snapshot_data (value TEXT)",
|
||||
"INSERT INTO snapshot_data VALUES ('latest committed data')",
|
||||
} {
|
||||
if err := source.Exec(query).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
target := filepath.Join(dir, "snapshot.db")
|
||||
if err := ExportSQLiteSnapshot(context.Background(), target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot, err := gorm.Open(sqlite.Open(target), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshotDB, err := snapshot.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := snapshotDB.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
var value string
|
||||
if err := snapshot.Raw("SELECT value FROM snapshot_data").Scan(&value).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != "latest committed data" {
|
||||
t.Fatalf("unexpected snapshot content: %q", value)
|
||||
}
|
||||
var integrity string
|
||||
if err := snapshot.Raw("PRAGMA integrity_check").Scan(&integrity).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if integrity != "ok" {
|
||||
t.Fatalf("snapshot integrity: %s", integrity)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user