tcjlj/MeBox
1
0
Fork 0
mirror of https://github.com/truewhile/MeBox.git synced 2026-10-03 12:26:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: reduce sqlite write pressure during scans

Browse Source
This commit is contained in:
ShukeBta
2026-06-11 21:45:53 +08:00
parent 0e28d07122
commit 7d0afd0fcb
6 changed files with 233 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/repository/repository.go
+44 -27
View File
@@ -309,6 +309,12 @@ func applyMediaQueryFilter(q *gorm.DB, filter MediaQueryFilter) *gorm.DB {
// 显式写入）。这两个问题都让 EnrichLibrary(WHERE scrape_status='pending')
// 永远捞不到数据。
func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.upsert(ctx, m)
})
}
func (r *MediaRepository) upsert(ctx context.Context, m *model.Media) error {
var existing model.Media
err := r.db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
@@ -328,15 +334,16 @@ func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error {
}
// 已存在：仅刷新文件层面的字段。
updates := map[string]any{
"size_bytes": m.SizeBytes,
"duration_sec": m.DurationSec,
"width": m.Width,
"height": m.Height,
"video_codec": m.VideoCodec,
"audio_codec": m.AudioCodec,
"container": m.Container,
"deleted_at": nil,
updates := map[string]any{}
setIfChanged(updates, "size_bytes", existing.SizeBytes, m.SizeBytes)
setIfChanged(updates, "duration_sec", existing.DurationSec, m.DurationSec)
setIfChanged(updates, "width", existing.Width, m.Width)
setIfChanged(updates, "height", existing.Height, m.Height)
setIfChanged(updates, "video_codec", existing.VideoCodec, m.VideoCodec)
setIfChanged(updates, "audio_codec", existing.AudioCodec, m.AudioCodec)
setIfChanged(updates, "container", existing.Container, m.Container)
if existing.DeletedAt.Valid {
updates["deleted_at"] = nil
}
// 回填硬链接身份标识，便于后续扫描去重（避免重复识别/多倍占用）。
if m.FileID != "" && m.FileID != existing.FileID {
@@ -347,56 +354,56 @@ func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error {
// 真实剧名。仅在 existing 还停留在 'pending'/'' 时回填扫描标题，
// 避免覆盖刮削结果。
if m.ScrapeStatus == "matched" || existing.ScrapeStatus == "pending" || existing.ScrapeStatus == "" || existing.ScrapeStatus == "no_match" {
updates["title"] = m.Title
setIfChanged(updates, "title", existing.Title, m.Title)
if m.Year > 0 {
updates["year"] = m.Year
setIfChanged(updates, "year", existing.Year, m.Year)
}
}
}
if m.ScrapeStatus == "matched" {
updates["scrape_status"] = m.ScrapeStatus
setIfChanged(updates, "scrape_status", existing.ScrapeStatus, m.ScrapeStatus)
if m.OriginalName != "" {
updates["original_name"] = m.OriginalName
setIfChanged(updates, "original_name", existing.OriginalName, m.OriginalName)
}
if m.PosterURL != "" {
updates["poster_url"] = m.PosterURL
setIfChanged(updates, "poster_url", existing.PosterURL, m.PosterURL)
}
if m.BackdropURL != "" {
updates["backdrop_url"] = m.BackdropURL
setIfChanged(updates, "backdrop_url", existing.BackdropURL, m.BackdropURL)
}
if m.Overview != "" {
updates["overview"] = m.Overview
setIfChanged(updates, "overview", existing.Overview, m.Overview)
}
if m.Rating > 0 {
updates["rating"] = m.Rating
setIfChanged(updates, "rating", existing.Rating, m.Rating)
}
if m.Year > 0 {
updates["year"] = m.Year
setIfChanged(updates, "year", existing.Year, m.Year)
}
if m.TMDbID > 0 {
updates["tm_db_id"] = m.TMDbID
setIfChanged(updates, "tm_db_id", existing.TMDbID, m.TMDbID)
}
if m.BangumiID > 0 {
updates["bangumi_id"] = m.BangumiID
setIfChanged(updates, "bangumi_id", existing.BangumiID, m.BangumiID)
}
if m.Languages != "" {
updates["languages"] = m.Languages
setIfChanged(updates, "languages", existing.Languages, m.Languages)
}
if m.Countries != "" {
updates["countries"] = m.Countries
setIfChanged(updates, "countries", existing.Countries, m.Countries)
}
if m.Genres != "" {
updates["genres"] = m.Genres
setIfChanged(updates, "genres", existing.Genres, m.Genres)
}
if m.NSFW {
if m.NSFW && !existing.NSFW {
updates["nsfw"] = true
}
}
if m.PosterURL != "" {
updates["poster_url"] = m.PosterURL
setIfChanged(updates, "poster_url", existing.PosterURL, m.PosterURL)
}
if m.BackdropURL != "" {
updates["backdrop_url"] = m.BackdropURL
setIfChanged(updates, "backdrop_url", existing.BackdropURL, m.BackdropURL)
}
if lib := m.LibraryID; lib != "" && lib != existing.LibraryID {
updates["library_id"] = m.LibraryID
@@ -408,9 +415,13 @@ func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error {
updates["episode_num"] = m.EpisodeNum
}
if m.STRMURL != "" {
updates["strm_url"] = m.STRMURL
setIfChanged(updates, "strm_url", existing.STRMURL, m.STRMURL)
}
if len(updates) == 0 {
*m = existing
return nil
}
if err := r.db.WithContext(ctx).Unscoped().Model(&model.Media{}).
Where("id = ?", existing.ID).Updates(updates).Error; err != nil {
return err
@@ -421,6 +432,12 @@ func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error {
return nil
}
func setIfChanged[T comparable](updates map[string]any, key string, current, next T) {
if current != next {
updates[key] = next
}
}
// FindByID returns the media row or (nil, nil).
func (r *MediaRepository) FindByID(ctx context.Context, id string) (*model.Media, error) {
var m model.Media
internal/repository/repository_test.go
+60
View File
@@ -2,6 +2,7 @@ package repository
import (
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
@@ -10,6 +11,65 @@ import (
"github.com/ShukeBta/MediaStationGo/internal/model"
)
func TestMediaUpsertSkipsUnchangedExistingRow(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := database.AutoMigrate(db); err != nil {
t.Fatalf("migrate: %v", err)
}
repos := New(db)
lib := model.Library{Name: "电影", Path: "/media/movie", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
media := model.Media{
LibraryID: lib.ID,
Title: "已有影片",
Path: "/media/movie/existing.mkv",
SizeBytes: 1024,
DurationSec: 60,
Width: 1920,
Height: 1080,
VideoCodec: "h264",
AudioCodec: "aac",
Container: "matroska,webm",
ScrapeStatus: "pending",
}
if err := repos.Media.Upsert(t.Context(), &media); err != nil {
t.Fatal(err)
}
var before model.Media
if err := repos.DB.Where("path = ?", media.Path).First(&before).Error; err != nil {
t.Fatal(err)
}
time.Sleep(10 * time.Millisecond)
again := model.Media{
LibraryID: lib.ID,
Title: before.Title,
Path: before.Path,
SizeBytes: before.SizeBytes,
DurationSec: before.DurationSec,
Width: before.Width,
Height: before.Height,
VideoCodec: before.VideoCodec,
AudioCodec: before.AudioCodec,
Container: before.Container,
ScrapeStatus: before.ScrapeStatus,
}
if err := repos.Media.Upsert(t.Context(), &again); err != nil {
t.Fatal(err)
}
var after model.Media
if err := repos.DB.Where("path = ?", media.Path).First(&after).Error; err != nil {
t.Fatal(err)
}
if !after.UpdatedAt.Equal(before.UpdatedAt) {
t.Fatalf("unchanged upsert touched updated_at: before=%s after=%s", before.UpdatedAt, after.UpdatedAt)
}
}
func TestMediaSearchFilteredSupportsChineseFuzzyTerms(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {