perf: 重构 FTS 索引为 rowid 寻址+触发器维护,根治重启 CPU 占满与扫描卡死

- FTS5 普通列(含 UNINDEXED)不支持索引查找,旧版按 media_id 做
  NOT EXISTS/DELETE 全是整表扫描:启动回填 O(N^2) 烧 CPU 数小时,
  扫描时每个 upsert 一次全表扫描,均隔着全局写锁拖死登录(125s 超时)
- v2 布局:FTS rowid 与 media.rowid 对齐,由 INSERT/UPDATE/DELETE
  触发器实时维护;回填只剩 rowid 点查兜底,且去掉 ORDER BY
- 顺带修复刮削直写 Updates() 后新标题搜不到的问题(触发器覆盖)
- 写门改为 context 感知:长写语句不再让登录请求无限期排队
- warmMediaSearchIndex 延迟 30s 错峰启动
- library_scan 周期任务跳过云盘库(由 cloud_sync 夜间窗口负责),
  首轮延迟从 15s 改为等满一个周期,重启不再立即全量扫描风暴
- prune 改为只取 id/path 并按 500 条批量删除,缩短写锁占用
This commit is contained in:
ShukeBta
2026-06-12 06:41:14 +00:00
parent 8d1d1f18a8
commit 1568ae127a
6 changed files with 187 additions and 77 deletions
+10 -30
View File
@@ -323,7 +323,6 @@ func (r *MediaRepository) upsert(ctx context.Context, m *model.Media) error {
m.ScrapeStatus = "pending"
}
if createErr := r.db.WithContext(ctx).Create(m).Error; createErr == nil {
_ = r.refreshSearchIndex(ctx, m.ID)
return nil
} else if retryErr := r.db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error; retryErr != nil {
return createErr
@@ -426,7 +425,6 @@ func (r *MediaRepository) upsert(ctx context.Context, m *model.Media) error {
Where("id = ?", existing.ID).Updates(updates).Error; err != nil {
return err
}
_ = r.refreshSearchIndex(ctx, existing.ID)
// 回写 ID / 不可变字段,让 caller 拿到完整的现有行。
*m = existing
return nil
@@ -518,7 +516,7 @@ func (r *MediaRepository) searchFilteredFTS(ctx context.Context, query string, o
var items []model.Media
q := r.db.WithContext(ctx).
Table("media").
Joins("JOIN media_search_fts ON media_search_fts.media_id = media.id").
Joins("JOIN media_search_fts ON media_search_fts.rowid = media.rowid").
Where("media.deleted_at IS NULL").
Where("media_search_fts MATCH ?", ftsQuery)
q = applyQualifiedMediaQueryFilter(q, filter)
@@ -625,23 +623,6 @@ func escapeLike(value string) string {
return value
}
func (r *MediaRepository) refreshSearchIndex(ctx context.Context, mediaID string) error {
if strings.TrimSpace(mediaID) == "" {
return nil
}
if !r.searchIndexEnabled(ctx) {
return nil
}
tx := r.db.WithContext(ctx)
_ = tx.Exec(`DELETE FROM media_search_fts WHERE media_id = ?`, mediaID).Error
return tx.Exec(`
INSERT INTO media_search_fts(media_id, title, original_name, path, genres)
SELECT id, COALESCE(title, ''), COALESCE(original_name, ''), COALESCE(path, ''), COALESCE(genres, '')
FROM media
WHERE id = ? AND deleted_at IS NULL
`, mediaID).Error
}
func (r *MediaRepository) BackfillSearchIndex(ctx context.Context, batchLimit int) (int64, error) {
if batchLimit <= 0 {
batchLimit = 1000
@@ -649,15 +630,19 @@ func (r *MediaRepository) BackfillSearchIndex(ctx context.Context, batchLimit in
if !r.searchIndexEnabled(ctx) {
return 0, nil
}
// 关键性能点:FTS5 普通列(含 UNINDEXED)不支持索引查找,按
// media_id 做 NOT EXISTS 是对 FTS 表的整表扫描,再叠加 ORDER BY
// 后每个批次都要对全部 media 行探测一遍——大库一次启动回填等于
// 上百亿次行访问,曾把 CPU 钉满数小时。v2 布局下 FTS 行 rowid 与
// media.rowid 对齐,NOT EXISTS 走 rowid 点查,且无需排序。
res := r.db.WithContext(ctx).Exec(`
INSERT INTO media_search_fts(media_id, title, original_name, path, genres)
SELECT m.id, COALESCE(m.title, ''), COALESCE(m.original_name, ''), COALESCE(m.path, ''), COALESCE(m.genres, '')
INSERT INTO media_search_fts(rowid, media_id, title, original_name, path, genres)
SELECT m.rowid, m.id, COALESCE(m.title, ''), COALESCE(m.original_name, ''), COALESCE(m.path, ''), COALESCE(m.genres, '')
FROM media AS m
WHERE m.deleted_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM media_search_fts AS f WHERE f.media_id = m.id
SELECT 1 FROM media_search_fts AS f WHERE f.rowid = m.rowid
)
ORDER BY m.created_at DESC
LIMIT ?
`, batchLimit)
return res.RowsAffected, res.Error
@@ -679,18 +664,13 @@ func (r *MediaRepository) searchIndexEnabled(ctx context.Context) bool {
// DeleteByLibrary purges all media tied to a library.
func (r *MediaRepository) DeleteByLibrary(ctx context.Context, libraryID string) error {
if r.searchIndexEnabled(ctx) {
_ = r.db.WithContext(ctx).Exec(`DELETE FROM media_search_fts WHERE media_id IN (SELECT id FROM media WHERE library_id = ?)`, libraryID).Error
}
// FTS 行由 media 表上的触发器同步清理(软删/硬删都覆盖)。
return r.db.WithContext(ctx).Where("library_id = ?", libraryID).Delete(&model.Media{}).Error
}
// PurgeByLibrary permanently removes media tied to a library. Used for virtual
// cloud mounts where "remove mount" must not populate the recycle bin.
func (r *MediaRepository) PurgeByLibrary(ctx context.Context, libraryID string) error {
if r.searchIndexEnabled(ctx) {
_ = r.db.WithContext(ctx).Exec(`DELETE FROM media_search_fts WHERE media_id IN (SELECT id FROM media WHERE library_id = ?)`, libraryID).Error
}
return r.db.WithContext(ctx).Unscoped().Where("library_id = ?", libraryID).Delete(&model.Media{}).Error
}
+22 -6
View File
@@ -153,12 +153,17 @@ func TestMediaSearchIndexBackfillRunsInBatches(t *testing.T) {
}).Error; err != nil {
t.Fatal(err)
}
var before int64
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts`).Scan(&before).Error; err != nil {
// 插入触发器应当同步维护 FTS 行。
var indexed int64
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts`).Scan(&indexed).Error; err != nil {
t.Fatal(err)
}
if before != 0 {
t.Fatalf("startup migrate should not synchronously backfill FTS, got %d rows", before)
if indexed != 1 {
t.Fatalf("insert trigger should index new media, got %d rows", indexed)
}
// 清空 FTS 模拟旧库升级后索引缺失,回填应按批补齐且 rowid 对齐。
if err := repos.DB.Exec(`DELETE FROM media_search_fts`).Error; err != nil {
t.Fatal(err)
}
n, err := repos.Media.BackfillSearchIndex(t.Context(), 1)
if err != nil {
@@ -167,11 +172,22 @@ func TestMediaSearchIndexBackfillRunsInBatches(t *testing.T) {
if n != 1 {
t.Fatalf("backfilled rows = %d, want 1", n)
}
var aligned int64
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts f JOIN media m ON f.rowid = m.rowid AND f.media_id = m.id`).Scan(&aligned).Error; err != nil {
t.Fatal(err)
}
if aligned != 1 {
t.Fatalf("fts rows aligned with media rowid = %d, want 1", aligned)
}
// 软删除后触发器应清理对应 FTS 行,避免搜索命中已删媒体。
if err := repos.DB.Delete(&model.Media{}, "id = ?", "m-backfill").Error; err != nil {
t.Fatal(err)
}
var after int64
if err := repos.DB.Raw(`SELECT COUNT(*) FROM media_search_fts`).Scan(&after).Error; err != nil {
t.Fatal(err)
}
if after != 1 {
t.Fatalf("fts rows = %d, want 1", after)
if after != 0 {
t.Fatalf("soft delete should drop fts row, got %d", after)
}
}