mirror of
https://github.com/truewhile/MeBox.git
synced 2026-10-05 21:06:38 +08:00
perf: 重构 FTS 索引为 rowid 寻址+触发器维护,根治重启 CPU 占满与扫描卡死
- FTS5 普通列(含 UNINDEXED)不支持索引查找,旧版按 media_id 做 NOT EXISTS/DELETE 全是整表扫描:启动回填 O(N^2) 烧 CPU 数小时, 扫描时每个 upsert 一次全表扫描,均隔着全局写锁拖死登录(125s 超时) - v2 布局:FTS rowid 与 media.rowid 对齐,由 INSERT/UPDATE/DELETE 触发器实时维护;回填只剩 rowid 点查兜底,且去掉 ORDER BY - 顺带修复刮削直写 Updates() 后新标题搜不到的问题(触发器覆盖) - 写门改为 context 感知:长写语句不再让登录请求无限期排队 - warmMediaSearchIndex 延迟 30s 错峰启动 - library_scan 周期任务跳过云盘库(由 cloud_sync 夜间窗口负责), 首轮延迟从 15s 改为等满一个周期,重启不再立即全量扫描风暴 - prune 改为只取 id/path 并按 500 条批量删除,缩短写锁占用
This commit is contained in:
@@ -323,7 +323,6 @@ func (r *MediaRepository) upsert(ctx context.Context, m *model.Media) error {
|
||||
m.ScrapeStatus = "pending"
|
||||
}
|
||||
if createErr := r.db.WithContext(ctx).Create(m).Error; createErr == nil {
|
||||
_ = r.refreshSearchIndex(ctx, m.ID)
|
||||
return nil
|
||||
} else if retryErr := r.db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error; retryErr != nil {
|
||||
return createErr
|
||||
@@ -426,7 +425,6 @@ func (r *MediaRepository) upsert(ctx context.Context, m *model.Media) error {
|
||||
Where("id = ?", existing.ID).Updates(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
_ = r.refreshSearchIndex(ctx, existing.ID)
|
||||
// 回写 ID / 不可变字段,让 caller 拿到完整的现有行。
|
||||
*m = existing
|
||||
return nil
|
||||
@@ -518,7 +516,7 @@ func (r *MediaRepository) searchFilteredFTS(ctx context.Context, query string, o
|
||||
var items []model.Media
|
||||
q := r.db.WithContext(ctx).
|
||||
Table("media").
|
||||
Joins("JOIN media_search_fts ON media_search_fts.media_id = media.id").
|
||||
Joins("JOIN media_search_fts ON media_search_fts.rowid = media.rowid").
|
||||
Where("media.deleted_at IS NULL").
|
||||
Where("media_search_fts MATCH ?", ftsQuery)
|
||||
q = applyQualifiedMediaQueryFilter(q, filter)
|
||||
@@ -625,23 +623,6 @@ func escapeLike(value string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
func (r *MediaRepository) refreshSearchIndex(ctx context.Context, mediaID string) error {
|
||||
if strings.TrimSpace(mediaID) == "" {
|
||||
return nil
|
||||
}
|
||||
if !r.searchIndexEnabled(ctx) {
|
||||
return nil
|
||||
}
|
||||
tx := r.db.WithContext(ctx)
|
||||
_ = tx.Exec(`DELETE FROM media_search_fts WHERE media_id = ?`, mediaID).Error
|
||||
return tx.Exec(`
|
||||
INSERT INTO media_search_fts(media_id, title, original_name, path, genres)
|
||||
SELECT id, COALESCE(title, ''), COALESCE(original_name, ''), COALESCE(path, ''), COALESCE(genres, '')
|
||||
FROM media
|
||||
WHERE id = ? AND deleted_at IS NULL
|
||||
`, mediaID).Error
|
||||
}
|
||||
|
||||
func (r *MediaRepository) BackfillSearchIndex(ctx context.Context, batchLimit int) (int64, error) {
|
||||
if batchLimit <= 0 {
|
||||
batchLimit = 1000
|
||||
@@ -649,15 +630,19 @@ func (r *MediaRepository) BackfillSearchIndex(ctx context.Context, batchLimit in
|
||||
if !r.searchIndexEnabled(ctx) {
|
||||
return 0, nil
|
||||
}
|
||||
// 关键性能点:FTS5 普通列(含 UNINDEXED)不支持索引查找,按
|
||||
// media_id 做 NOT EXISTS 是对 FTS 表的整表扫描,再叠加 ORDER BY
|
||||
// 后每个批次都要对全部 media 行探测一遍——大库一次启动回填等于
|
||||
// 上百亿次行访问,曾把 CPU 钉满数小时。v2 布局下 FTS 行 rowid 与
|
||||
// media.rowid 对齐,NOT EXISTS 走 rowid 点查,且无需排序。
|
||||
res := r.db.WithContext(ctx).Exec(`
|
||||
INSERT INTO media_search_fts(media_id, title, original_name, path, genres)
|
||||
SELECT m.id, COALESCE(m.title, ''), COALESCE(m.original_name, ''), COALESCE(m.path, ''), COALESCE(m.genres, '')
|
||||
INSERT INTO media_search_fts(rowid, media_id, title, original_name, path, genres)
|
||||
SELECT m.rowid, m.id, COALESCE(m.title, ''), COALESCE(m.original_name, ''), COALESCE(m.path, ''), COALESCE(m.genres, '')
|
||||
FROM media AS m
|
||||
WHERE m.deleted_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_search_fts AS f WHERE f.media_id = m.id
|
||||
SELECT 1 FROM media_search_fts AS f WHERE f.rowid = m.rowid
|
||||
)
|
||||
ORDER BY m.created_at DESC
|
||||
LIMIT ?
|
||||
`, batchLimit)
|
||||
return res.RowsAffected, res.Error
|
||||
@@ -679,18 +664,13 @@ func (r *MediaRepository) searchIndexEnabled(ctx context.Context) bool {
|
||||
|
||||
// DeleteByLibrary purges all media tied to a library.
|
||||
func (r *MediaRepository) DeleteByLibrary(ctx context.Context, libraryID string) error {
|
||||
if r.searchIndexEnabled(ctx) {
|
||||
_ = r.db.WithContext(ctx).Exec(`DELETE FROM media_search_fts WHERE media_id IN (SELECT id FROM media WHERE library_id = ?)`, libraryID).Error
|
||||
}
|
||||
// FTS 行由 media 表上的触发器同步清理(软删/硬删都覆盖)。
|
||||
return r.db.WithContext(ctx).Where("library_id = ?", libraryID).Delete(&model.Media{}).Error
|
||||
}
|
||||
|
||||
// PurgeByLibrary permanently removes media tied to a library. Used for virtual
|
||||
// cloud mounts where "remove mount" must not populate the recycle bin.
|
||||
func (r *MediaRepository) PurgeByLibrary(ctx context.Context, libraryID string) error {
|
||||
if r.searchIndexEnabled(ctx) {
|
||||
_ = r.db.WithContext(ctx).Exec(`DELETE FROM media_search_fts WHERE media_id IN (SELECT id FROM media WHERE library_id = ?)`, libraryID).Error
|
||||
}
|
||||
return r.db.WithContext(ctx).Unscoped().Where("library_id = ?", libraryID).Delete(&model.Media{}).Error
|
||||
}
|
||||
|
||||
|
||||
@@ -153,12 +153,17 @@ func TestMediaSearchIndexBackfillRunsInBatches(t *testing.T) {
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var before int64
|
||||
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts`).Scan(&before).Error; err != nil {
|
||||
// 插入触发器应当同步维护 FTS 行。
|
||||
var indexed int64
|
||||
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts`).Scan(&indexed).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if before != 0 {
|
||||
t.Fatalf("startup migrate should not synchronously backfill FTS, got %d rows", before)
|
||||
if indexed != 1 {
|
||||
t.Fatalf("insert trigger should index new media, got %d rows", indexed)
|
||||
}
|
||||
// 清空 FTS 模拟旧库升级后索引缺失,回填应按批补齐且 rowid 对齐。
|
||||
if err := repos.DB.Exec(`DELETE FROM media_search_fts`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := repos.Media.BackfillSearchIndex(t.Context(), 1)
|
||||
if err != nil {
|
||||
@@ -167,11 +172,22 @@ func TestMediaSearchIndexBackfillRunsInBatches(t *testing.T) {
|
||||
if n != 1 {
|
||||
t.Fatalf("backfilled rows = %d, want 1", n)
|
||||
}
|
||||
var aligned int64
|
||||
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts f JOIN media m ON f.rowid = m.rowid AND f.media_id = m.id`).Scan(&aligned).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if aligned != 1 {
|
||||
t.Fatalf("fts rows aligned with media rowid = %d, want 1", aligned)
|
||||
}
|
||||
// 软删除后触发器应清理对应 FTS 行,避免搜索命中已删媒体。
|
||||
if err := repos.DB.Delete(&model.Media{}, "id = ?", "m-backfill").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var after int64
|
||||
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts`).Scan(&after).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after != 1 {
|
||||
t.Fatalf("fts rows = %d, want 1", after)
|
||||
if after != 0 {
|
||||
t.Fatalf("soft delete should drop fts row, got %d", after)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user