diff --git a/internal/repository/media_repository_upsert.go b/internal/repository/media_repository_upsert.go index e2c8ab5..6ce2c76 100644 --- a/internal/repository/media_repository_upsert.go +++ b/internal/repository/media_repository_upsert.go @@ -3,6 +3,7 @@ package repository import ( "context" "errors" + "path/filepath" "strings" "gorm.io/gorm" @@ -10,6 +11,18 @@ import ( "github.com/truewhile/MeBox/internal/model" ) +// MediaUpsertItem carries a media row and optional sibling paths from an +// earlier keep_ext naming mode. An alias is only used when the exact new path +// does not exist in the database; in that case the existing row is migrated to +// the new path so scraped metadata survives renames such as: +// +// foo.strm -> foo.mkv.strm +// foo.mkv.strm -> foo.strm +type MediaUpsertItem struct { + Media *model.Media + AliasPaths []string +} + // Upsert inserts or updates a media row keyed by Path (unique index). // // 重要：当一条行已经存在时，scanner 重扫只应该刷新文件级元数据 @@ -22,20 +35,14 @@ import ( // 显式写入）。这两个问题都让 EnrichLibrary(WHERE scrape_status='pending') // 永远捞不到数据。 func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error { - var indexIDs []string - err := withSQLiteBusyRetry(ctx, func() error { - id, uerr := r.upsertWithDB(ctx, r.db, m) - if uerr != nil { - return uerr - } - indexIDs = append(indexIDs[:0], id) - return nil - }) - if err != nil { - return err - } - r.indexByIDBestEffort(ctx, indexIDs) - return nil + return r.UpsertWithAliases(ctx, m, nil) +} + +// UpsertWithAliases is Upsert with explicit, filesystem-verified STRM sibling +// aliases. It preserves the old row's ID, CreatedAt and scraped metadata while +// moving it to the current path. +func (r *MediaRepository) UpsertWithAliases(ctx context.Context, m *model.Media, aliasPaths []string) error { + return r.UpsertBatchWithAliases(ctx, []MediaUpsertItem{{Media: m, AliasPaths: aliasPaths}}) } // UpsertBatch 在单个事务里逐条执行 Upsert：扫描一批只提交（fsync）一次， @@ -46,6 +53,17 @@ func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error { // 事务内会把 SQLite 写锁挂起在网络 IO 上，且批内用非事务连接回读只能 // 拿到提交前的旧版本数据，把陈旧内容写进索引。 func (r *MediaRepository) UpsertBatch(ctx context.Context, items []*model.Media) error { + mapped := make([]MediaUpsertItem, 0, len(items)) + for _, item := range items { + if item != nil { + mapped = append(mapped, MediaUpsertItem{Media: item}) + } + } + return r.UpsertBatchWithAliases(ctx, mapped) +} + +// UpsertBatchWithAliases runs alias-aware upserts in one transaction. +func (r *MediaRepository) UpsertBatchWithAliases(ctx context.Context, items []MediaUpsertItem) error { if len(items) == 0 { return nil } @@ -53,11 +71,11 @@ func (r *MediaRepository) UpsertBatch(ctx context.Context, items []*model.Media) err := withSQLiteBusyRetry(ctx, func() error { indexIDs = indexIDs[:0] return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - for _, m := range items { - if m == nil { + for _, item := range items { + if item.Media == nil { continue } - id, err := r.upsertWithDB(ctx, tx, m) + id, err := r.upsertWithDB(ctx, tx, item.Media, item.AliasPaths) if err != nil { return err } @@ -87,9 +105,9 @@ func (r *MediaRepository) indexByIDBestEffort(ctx context.Context, ids []string) } } -// upsertWithDB 落库（新建或更新），返回需要重建索引的媒体 ID（无则空串）。 -func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *model.Media) (string, error) { - existing, created, err := r.findOrCreateMediaByPath(ctx, db, m) +// upsertWithDB 落库（新建、更新或从旧路径迁移），返回需要重建索引的媒体 ID。 +func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *model.Media, aliasPaths []string) (string, error) { + existing, created, adopted, err := r.findOrCreateMediaByPath(ctx, db, m, aliasPaths) if err != nil { return "", err } @@ -98,6 +116,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode } updates := mediaUpsertUpdates(existing, *m) + if adopted { + updates["path"] = m.Path + if existing.DeletedAt.Valid { + updates["deleted_at"] = nil + } + } if len(updates) == 0 { *m = existing return "", nil @@ -107,31 +131,67 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode return "", err } // 回写 ID / 不可变字段，让 caller 拿到完整的现有行。 + if adopted { + existing.Path = m.Path + existing.DeletedAt = gorm.DeletedAt{} + } *m = existing return existing.ID, nil } -func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.DB, m *model.Media) (model.Media, bool, error) { +func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.DB, m *model.Media, aliasPaths []string) (model.Media, bool, bool, error) { var existing model.Media err := db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error if errors.Is(err, gorm.ErrRecordNotFound) { + if alias, aliasErr := r.findMediaByAlias(ctx, db, m.Path, aliasPaths); aliasErr == nil { + return alias, false, true, nil + } else if !errors.Is(aliasErr, gorm.ErrRecordNotFound) { + return model.Media{}, false, false, aliasErr + } // 新行：保证 scrape_status 走 GORM default:pending（即留空让数据库填）。 if m.ScrapeStatus == "" { m.ScrapeStatus = "pending" } if createErr := db.WithContext(ctx).Create(m).Error; createErr == nil { - return *m, true, nil + return *m, true, false, nil } else if retryErr := db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error; retryErr != nil { - return model.Media{}, false, createErr + return model.Media{}, false, false, createErr } else { // 并发插入竞态：重查已命中既有行，直接走更新分支。 - return existing, false, nil + return existing, false, false, nil } } if err != nil { - return model.Media{}, false, err + return model.Media{}, false, false, err } - return existing, false, nil + return existing, false, false, nil +} + +func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, currentPath string, aliasPaths []string) (model.Media, error) { + aliases := make([]string, 0, len(aliasPaths)) + seen := make(map[string]struct{}, len(aliasPaths)) + for _, alias := range aliasPaths { + alias = filepath.Clean(strings.TrimSpace(alias)) + if alias == "" || alias == "." || alias == filepath.Clean(currentPath) { + continue + } + if _, ok := seen[alias]; ok { + continue + } + seen[alias] = struct{}{} + aliases = append(aliases, alias) + } + if len(aliases) == 0 { + return model.Media{}, gorm.ErrRecordNotFound + } + var existing model.Media + err := db.WithContext(ctx).Unscoped(). + Where("path IN ?", aliases). + Order("CASE WHEN scrape_status = 'matched' THEN 0 ELSE 1 END ASC, " + + "CASE WHEN COALESCE(poster_url, '') <> '' OR COALESCE(overview, '') <> '' THEN 0 ELSE 1 END ASC, " + + "CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END ASC, updated_at DESC, created_at DESC"). + First(&existing).Error + return existing, err } func mediaUpsertUpdates(existing, incoming model.Media) map[string]any { diff --git a/internal/service/local_metadata_artwork.go b/internal/service/local_metadata_artwork.go index 51f692a..0b393bb 100644 --- a/internal/service/local_metadata_artwork.go +++ b/internal/service/local_metadata_artwork.go @@ -158,7 +158,7 @@ func firstExistingImage(dir string, names ...string) string { return "" } for _, name := range names { - for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn"} { + for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn", ".img"} { path := filepath.Join(dir, name+ext) if fileExists(path) { return filepath.Clean(path) @@ -221,7 +221,7 @@ func firstExistingPosterImage(dir string, names ...string) string { if isRejectedPosterName(name) { continue } - for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn"} { + for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn", ".img"} { path := filepath.Join(dir, name+ext) if fileExists(path) && likelyPosterImage(path) { return filepath.Clean(path) @@ -240,7 +240,7 @@ func firstAdultLooseImage(dir, kind string) string { fallback := []string{} for _, path := range matches { ext := strings.ToLower(filepath.Ext(path)) - if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" && ext != ".gif" && ext != ".bmp" && ext != ".tbn" { + if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" && ext != ".gif" && ext != ".bmp" && ext != ".tbn" && ext != ".img" { continue } name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), ext)) diff --git a/internal/service/local_metadata_test.go b/internal/service/local_metadata_test.go index 7790526..87f81df 100644 --- a/internal/service/local_metadata_test.go +++ b/internal/service/local_metadata_test.go @@ -417,3 +417,22 @@ func TestReadLocalMetadataArtworkFallbackOnGarbageShowNFO(t *testing.T) { t.Fatalf("expected episode artwork fallback, got %+v", got) } } + +func TestReadLocalMetadataFindsLegacyImgPoster(t *testing.T) { + root := t.TempDir() + mediaPath := filepath.Join(root, "Jigokuraku.S01E14.mkv.strm") + poster := filepath.Join(root, "Jigokuraku.S01E14-poster.img") + if err := os.WriteFile(mediaPath, []byte("https://example.test/video"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(poster, testJPEG, 0o644); err != nil { + t.Fatal(err) + } + got, err := ReadLocalMetadata(mediaPath, root, true) + if err != nil { + t.Fatal(err) + } + if got == nil || got.PosterURL != poster { + t.Fatalf("PosterURL = %q, want legacy .img poster %q", got.PosterURL, poster) + } +} diff --git a/internal/service/media_path_alias.go b/internal/service/media_path_alias.go new file mode 100644 index 0000000..e0b0e62 --- /dev/null +++ b/internal/service/media_path_alias.go @@ -0,0 +1,101 @@ +package service + +import ( + "os" + "path/filepath" + "sort" + "strings" +) + +// strmPathAliases returns sibling STRM names that can represent the same media +// item when keep_ext is toggled on or off. +// +// Both directions are supported: +// +// foo.strm <-> foo.mkv.strm +// foo.strm <-> foo.mp4.strm +// +// The caller must still verify that the aliases are no longer present on disk. +// keep_ext intentionally allows foo.mkv.strm and foo.mp4.strm to coexist as +// two real versions, so an existing sibling must never be absorbed. +func strmPathAliases(path string) []string { + clean := filepath.Clean(strings.TrimSpace(path)) + if clean == "" || clean == "." || !strings.EqualFold(filepath.Ext(clean), ".strm") { + return nil + } + base := mediaSidecarBase(clean) + if base == "" { + return nil + } + name := filepath.Base(clean) + stem := strings.TrimSuffix(name, filepath.Ext(name)) + lastExt := strings.ToLower(filepath.Ext(stem)) + _, hasVideoExt := videoExtensions[lastExt] + hasVideoExt = hasVideoExt && !strings.EqualFold(lastExt, ".strm") + + dir := filepath.Dir(clean) + out := make([]string, 0, len(videoExtensions)+1) + seen := make(map[string]struct{}, len(videoExtensions)+1) + add := func(candidate string) { + candidate = filepath.Clean(candidate) + if candidate == "" || candidate == "." || samePath(candidate, clean) { + return + } + key := strings.ToLower(candidate) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + out = append(out, candidate) + } + if hasVideoExt { + // Current keep_ext shape: the only legacy shape is the stripped name. + add(filepath.Join(dir, base+".strm")) + return out + } + // Stripped shape: an old keep_ext file may use any configured video ext. + exts := make([]string, 0, len(videoExtensions)) + for ext := range videoExtensions { + if strings.EqualFold(ext, ".strm") { + continue + } + exts = append(exts, strings.ToLower(ext)) + } + sort.Strings(exts) + for _, ext := range exts { + add(filepath.Join(dir, base+ext+".strm")) + if upper := strings.ToUpper(ext); upper != ext { + add(filepath.Join(dir, base+upper+".strm")) + } + } + return out +} + +// missingSTRMPathAliases keeps only aliases which no longer exist on disk. +// This is the key protection for keep_ext=true multi-version libraries: an +// existing foo.mp4.strm is a real second version, not a rename tombstone. +func missingSTRMPathAliases(path string) []string { + aliases := strmPathAliases(path) + if len(aliases) == 0 { + return nil + } + out := make([]string, 0, len(aliases)) + for _, alias := range aliases { + if _, err := os.Lstat(alias); err == nil { + continue + } else if !os.IsNotExist(err) { + continue + } + out = append(out, alias) + } + return out +} + +func liveSTRMPathAlias(path string) bool { + for _, alias := range strmPathAliases(path) { + if info, err := os.Stat(alias); err == nil && !info.IsDir() { + return true + } + } + return false +} diff --git a/internal/service/media_series_resolver.go b/internal/service/media_series_resolver.go index 7037855..feae461 100644 --- a/internal/service/media_series_resolver.go +++ b/internal/service/media_series_resolver.go @@ -67,17 +67,20 @@ func newMediaSeriesKeyResolver(items []model.Media) mediaSeriesKeyResolver { func (r mediaSeriesKeyResolver) key(media model.Media) string { if mediaLooksEpisodicForGrouping(media) { - if key := repeatedSeriesTitleKey(media); key != "" && r.titleCounts[key] > 1 { - return compactSeriesKey(key) - } if pathKey := mediaSeriesRawKey(media); strings.HasPrefix(pathKey, "library-path") { if titleKey := r.pathTitles[pathKey]; titleKey != "" { return compactSeriesKey(titleKey) } + // A series directory is the strongest identity for mixed rows: + // main episodes and specials (CM/NCOP/PV/OVA) may be scraped to + // slightly different titles, but they still belong to one show. if r.pathCounts[pathKey] > 1 { return compactSeriesKey(pathKey) } } + if key := repeatedSeriesTitleKey(media); key != "" && r.titleCounts[key] > 1 { + return compactSeriesKey(key) + } if key := repeatedSeriesExternalKey(media); key != "" && r.externalCounts[key] > 1 { return compactSeriesKey(key) } diff --git a/internal/service/media_series_test.go b/internal/service/media_series_test.go index 601ddc3..1247352 100644 --- a/internal/service/media_series_test.go +++ b/internal/service/media_series_test.go @@ -47,14 +47,14 @@ func TestListRecentSeriesCardsCountsAllEpisodesInSeries(t *testing.T) { if len(cards) != 1 { t.Fatalf("recent cards = %#v, want one series card", cards) } - if cards[0].Count != 40 { - t.Fatalf("recent series count = %d, want full 40 episodes", cards[0].Count) - } - expectedLastAdded := now.Add(40 * time.Minute) - if cards[0].LastAddedAt == nil || !cards[0].LastAddedAt.Equal(expectedLastAdded) { - t.Fatalf("recent series LastAddedAt = %v, want %v", cards[0].LastAddedAt, expectedLastAdded) - } + if cards[0].Count != 40 { + t.Fatalf("recent series count = %d, want full 40 episodes", cards[0].Count) } + expectedLastAdded := now.Add(40 * time.Minute) + if cards[0].LastAddedAt == nil || !cards[0].LastAddedAt.Equal(expectedLastAdded) { + t.Fatalf("recent series LastAddedAt = %v, want %v", cards[0].LastAddedAt, expectedLastAdded) + } +} func TestMediaSeriesKeyCollapsesNestedSpecialFolders(t *testing.T) { main := model.Media{ @@ -377,6 +377,51 @@ func TestGroupMediaSeriesCardsMergesPollutedEpisodeFoldersBySharedShowID(t *test } } +func TestGroupMediaSeriesCardsKeepsMixedTitlesInSameEpisodicDirectoryTogether(t *testing.T) { + items := []model.Media{ + { + Base: model.Base{ID: "main-1"}, + LibraryID: "anime", + Title: "住在拔作岛上的我应该如何是好？", + Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][01][AVC_2×FLAC].mkv.strm`, + SeasonNum: 1, + EpisodeNum: 1, + ScrapeStatus: "matched", + }, + { + Base: model.Base{ID: "main-2"}, + LibraryID: "anime", + Title: "住在拔作岛上的我应该如何是好？", + Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][02][AVC_2×FLAC].mkv.strm`, + SeasonNum: 1, + EpisodeNum: 2, + ScrapeStatus: "matched", + }, + { + Base: model.Base{ID: "special-1"}, + LibraryID: "anime", + Title: "nukitashi", + Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][CM_01][AVC_FLAC].mkv.strm`, + ScrapeStatus: "matched", + }, + { + Base: model.Base{ID: "special-2"}, + LibraryID: "anime", + Title: "nukitashi", + Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][PV_01][AVC_FLAC].mkv.strm`, + ScrapeStatus: "matched", + }, + } + + cards := groupMediaSeriesCards(items) + if len(cards) != 1 { + t.Fatalf("cards=%#v, want main episodes and specials in the same directory folded into one card", cards) + } + if cards[0].Count != 4 { + t.Fatalf("series count=%d, want 4 items", cards[0].Count) + } +} + func TestGroupMediaSeriesCardsKeepsMovieVersionsAsOneMovie(t *testing.T) { items := []model.Media{ { @@ -503,11 +548,11 @@ func TestGroupMediaSeriesCardsKeepsIndependentMoviesSeparateInSharedSubdirectory {LibraryID: "movies", Title: "cd1", Path: `/media/电影/指环王 (2001)/cd1.mkv`}, {LibraryID: "movies", Title: "cd2", Path: `/media/电影/指环王 (2001)/cd2.mkv`}, } - cdCards := groupMediaSeriesCards(cdItems) - if len(cdCards) != 1 { - t.Fatalf("got %d cards for cd1/cd2, want 1 folded movie card", len(cdCards)) - } + cdCards := groupMediaSeriesCards(cdItems) + if len(cdCards) != 1 { + t.Fatalf("got %d cards for cd1/cd2, want 1 folded movie card", len(cdCards)) } +} func TestListMediaEpisodesKeepsIndependentMoviesSeparate(t *testing.T) { db := newServiceTestDB(t, &model.Library{}, &model.Media{}) diff --git a/internal/service/organizer_sidecar.go b/internal/service/organizer_sidecar.go index 99854f3..9573d63 100644 --- a/internal/service/organizer_sidecar.go +++ b/internal/service/organizer_sidecar.go @@ -43,7 +43,7 @@ var artworkSidecarSuffixes = []string{ // artworkSidecarExtensions are the image extensions a sidecar may use. Both // "base-poster.jpg" (bare separator) and "base.poster.jpg" (dotted separator) // rely on suffix matching, so we probe the common image extensions. -var artworkSidecarExtensions = []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn"} +var artworkSidecarExtensions = []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn", ".img"} // transferSidecarArtwork moves/copies/links the scraped poster/backdrop // sidecar files alongside its media using the same transfer mode, mirroring diff --git a/internal/service/scanner_local_ingest.go b/internal/service/scanner_local_ingest.go index b855df4..ca7be6f 100644 --- a/internal/service/scanner_local_ingest.go +++ b/internal/service/scanner_local_ingest.go @@ -200,7 +200,7 @@ func (s *ScannerService) writeLocalScanMedia(in localScanWriteInput) { in.writeBatch.AddWithAfter(in.path, in.media, in.after) return } - if err := s.repo.Media.Upsert(in.ctx, in.media); err != nil { + if err := s.repo.Media.UpsertWithAliases(in.ctx, in.media, missingSTRMPathAliases(in.path)); err != nil { addScanError(in.res, in.path, err) s.log.Warn("upsert media failed", zap.String("path", in.path), zap.Error(err)) return @@ -252,8 +252,10 @@ func (s *ScannerService) duplicateByFileID(ctx context.Context, fileID, path str } func (s *ScannerService) mediaPathExists(ctx context.Context, path string) bool { + paths := []string{path} + paths = append(paths, missingSTRMPathAliases(path)...) var count int64 err := s.repo.DB.WithContext(ctx).Unscoped().Model(&model.Media{}). - Where("path = ?", path).Count(&count).Error + Where("path IN ?", paths).Count(&count).Error return err == nil && count > 0 } diff --git a/internal/service/scanner_local_write_batch.go b/internal/service/scanner_local_write_batch.go index 8f861b2..1f90ba6 100644 --- a/internal/service/scanner_local_write_batch.go +++ b/internal/service/scanner_local_write_batch.go @@ -7,6 +7,7 @@ import ( "go.uber.org/zap" "github.com/truewhile/MeBox/internal/model" + "github.com/truewhile/MeBox/internal/repository" ) type localMediaWriteBatch struct { @@ -18,9 +19,10 @@ type localMediaWriteBatch struct { } type localMediaWriteItem struct { - path string - media *model.Media - after func() + path string + media *model.Media + aliases []string + after func() } func newLocalMediaWriteBatch(scanner *ScannerService, ctx context.Context, res *ScanResult, limit int) *localMediaWriteBatch { @@ -41,7 +43,12 @@ func (b *localMediaWriteBatch) AddWithAfter(path string, media *model.Media, aft if media.ScrapeStatus == "" { media.ScrapeStatus = "pending" } - b.items = append(b.items, localMediaWriteItem{path: path, media: media, after: after}) + b.items = append(b.items, localMediaWriteItem{ + path: path, + media: media, + aliases: missingSTRMPathAliases(path), + after: after, + }) if len(b.items) >= b.limit { b.Flush() } @@ -53,39 +60,35 @@ func (b *localMediaWriteBatch) Flush() { } items := b.items b.items = nil - media := make([]model.Media, 0, len(items)) - for _, item := range items { - if item.media != nil { - media = append(media, *item.media) - } - } - if len(media) == 0 { - return - } - existingPaths := b.existingPaths(items) - upsertItems := make([]*model.Media, 0, len(items)) - upsertAfter := make([]func(), 0, len(items)) + + existingPaths, lookupOK := b.existingPathOrAliasSet(items) + upsertItems := make([]localMediaWriteItem, 0, len(items)) createItems := make([]localMediaWriteItem, 0, len(items)) - createMedia := make([]model.Media, 0, len(items)) for _, item := range items { if item.media == nil { continue } - if existingPaths[filepath.Clean(item.media.Path)] { - // 已存在行：攒起来在一个事务里逐条 upsert（一批一次提交）。 - after := item.after - upsertItems = append(upsertItems, item.media) - upsertAfter = append(upsertAfter, after) + // If the alias lookup failed, route through UpsertWithAliases instead of + // direct-create. That is slower but cannot create a duplicate row. + if !lookupOK || mediaPathOrAliasExists(item.media.Path, item.aliases, existingPaths) { + upsertItems = append(upsertItems, item) continue } createItems = append(createItems, item) - createMedia = append(createMedia, *item.media) } - b.flushUpserts(items, upsertItems, upsertAfter) - if len(createMedia) == 0 { + + b.flushUpserts(upsertItems) + if len(createItems) == 0 { b.publish() return } + + createMedia := make([]model.Media, 0, len(createItems)) + for _, item := range createItems { + if item.media != nil { + createMedia = append(createMedia, *item.media) + } + } if err := b.scanner.repo.DB.WithContext(b.ctx).CreateInBatches(&createMedia, b.limit).Error; err == nil { b.res.Added += len(createMedia) for _, item := range createItems { @@ -96,12 +99,13 @@ func (b *localMediaWriteBatch) Flush() { b.publish() return } + for _, item := range createItems { if item.media == nil { continue } wasExisting := b.mediaPathExists(item.media.Path) - if err := b.scanner.repo.Media.Upsert(b.ctx, item.media); err != nil { + if err := b.scanner.repo.Media.UpsertWithAliases(b.ctx, item.media, item.aliases); err != nil { addScanError(b.res, item.path, err) b.scanner.log.Warn("upsert media failed", zap.String("path", item.path), zap.Error(err)) continue @@ -118,47 +122,90 @@ func (b *localMediaWriteBatch) Flush() { b.publish() } -func (b *localMediaWriteBatch) existingPaths(items []localMediaWriteItem) map[string]bool { +// existingPathOrAliasSet loads exact and alias paths in bounded query chunks. Aliases +// have already been filtered against the filesystem by AddWithAfter, so an +// existing keep_ext sibling is never treated as a rename. +func (b *localMediaWriteBatch) existingPathOrAliasSet(items []localMediaWriteItem) (map[string]bool, bool) { out := map[string]bool{} if b == nil || b.scanner == nil || b.scanner.repo == nil || b.scanner.repo.DB == nil || len(items) == 0 { - return out + return out, false + } + seen := make(map[string]struct{}, len(items)*2) + paths := make([]string, 0, len(items)*2) + add := func(path string) { + path = filepath.Clean(path) + if path == "" || path == "." { + return + } + if _, ok := seen[path]; ok { + return + } + seen[path] = struct{}{} + paths = append(paths, path) } - paths := make([]string, 0, len(items)) for _, item := range items { if item.media == nil || item.media.Path == "" { continue } - paths = append(paths, item.media.Path) + add(item.media.Path) + for _, alias := range item.aliases { + add(alias) + } } if len(paths) == 0 { - return out + return out, true } - var rows []string - if err := b.scanner.repo.DB.WithContext(b.ctx). - Unscoped(). - Model(&model.Media{}). - Where("path IN ?", paths). - Pluck("path", &rows).Error; err != nil { - b.scanner.log.Debug("load existing media paths for scan batch failed", zap.Error(err)) - return out + // Keep SQL variables below the legacy SQLite limit (999). A batch can + // contain 100 STRM rows and every row has many sibling aliases. + const pathLookupChunk = 400 + for start := 0; start < len(paths); start += pathLookupChunk { + end := start + pathLookupChunk + if end > len(paths) { + end = len(paths) + } + var rows []string + if err := b.scanner.repo.DB.WithContext(b.ctx). + Unscoped(). + Model(&model.Media{}). + Where("path IN ?", paths[start:end]). + Pluck("path", &rows).Error; err != nil { + b.scanner.log.Debug("load existing media paths for scan batch failed", zap.Error(err)) + return nil, false + } + for _, path := range rows { + out[filepath.Clean(path)] = true + } } - for _, path := range rows { - out[filepath.Clean(path)] = true - } - return out + return out, true } -// flushUpserts 把已存在行的 upsert 攒成一个事务（一次提交/一组 fsync）。 -// 整批失败（如单条数据触发约束）时退回逐条 Upsert，只丢真正坏的那几条。 -func (b *localMediaWriteBatch) flushUpserts(allItems []localMediaWriteItem, upsertItems []*model.Media, upsertAfter []func()) { +func mediaPathOrAliasExists(path string, aliases []string, existing map[string]bool) bool { + if existing[filepath.Clean(path)] { + return true + } + for _, alias := range aliases { + if existing[filepath.Clean(alias)] { + return true + } + } + return false +} + +// flushUpserts 把已存在或可迁移路径的行攒成一个事务（一次提交/一组 fsync）。 +// 整批失败（如单条数据触发约束）时退回逐条 upsert，只丢真正坏的那几条。 +func (b *localMediaWriteBatch) flushUpserts(upsertItems []localMediaWriteItem) { if len(upsertItems) == 0 { return } - if err := b.scanner.repo.Media.UpsertBatch(b.ctx, upsertItems); err == nil { + batchItems := make([]repository.MediaUpsertItem, 0, len(upsertItems)) + for _, item := range upsertItems { + batchItems = append(batchItems, repository.MediaUpsertItem{Media: item.media, AliasPaths: item.aliases}) + } + if err := b.scanner.repo.Media.UpsertBatchWithAliases(b.ctx, batchItems); err == nil { b.res.Updated += len(upsertItems) - for _, after := range upsertAfter { - if after != nil { - after() + for _, item := range upsertItems { + if item.after != nil { + item.after() } } return @@ -166,13 +213,7 @@ func (b *localMediaWriteBatch) flushUpserts(allItems []localMediaWriteItem, upse b.scanner.log.Warn("batch upsert failed; falling back to per-item upsert", zap.Int("items", len(upsertItems))) } - // 兜底：按原始顺序找回每个条目的 path/after（两个切片同序但可能含 nil）。 - idx := 0 - for _, item := range allItems { - if item.media == nil || idx >= len(upsertItems) || upsertItems[idx] != item.media { - continue - } - idx++ + for _, item := range upsertItems { b.upsertExistingItem(item) } } @@ -181,7 +222,7 @@ func (b *localMediaWriteBatch) upsertExistingItem(item localMediaWriteItem) { if item.media == nil { return } - if err := b.scanner.repo.Media.Upsert(b.ctx, item.media); err != nil { + if err := b.scanner.repo.Media.UpsertWithAliases(b.ctx, item.media, item.aliases); err != nil { addScanError(b.res, item.path, err) b.scanner.log.Warn("upsert media failed", zap.String("path", item.path), zap.Error(err)) return diff --git a/internal/service/scanner_prune.go b/internal/service/scanner_prune.go index 5513bb0..a6d452d 100644 --- a/internal/service/scanner_prune.go +++ b/internal/service/scanner_prune.go @@ -5,13 +5,15 @@ import ( "os" "path/filepath" "strings" + "time" "go.uber.org/zap" "github.com/truewhile/MeBox/internal/model" ) -// RemovePath 物理删除磁盘上已不存在的媒体记录。 +// RemovePath 移除磁盘上已不存在的媒体记录。普通的删除仍做物理删除； +// 可确认是 STRM 扩展名改名的路径则先保留软删除墓碑，供后续 ingest 继承元数据。 func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) { if _, err := os.Stat(path); err == nil { return 0, nil // still exists; nothing to remove @@ -30,24 +32,64 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er Find(&rows).Error; err != nil { return 0, err } - ids := make([]string, 0, len(rows)) + softIDs := make([]string, 0, len(rows)) + hardIDs := make([]string, 0, len(rows)) for _, row := range rows { // LIKE 里的 % _ 是通配符（候选集只会偏大），用 Go 前缀精确过滤， // 避免对含 % / _ 的路径误删。 - if row.Path == path || strings.HasPrefix(filepath.Clean(row.Path), prefix) { - ids = append(ids, row.ID) + if row.Path != path && !strings.HasPrefix(filepath.Clean(row.Path), prefix) { + continue } + // A vanished STRM with a live sibling (foo.strm <-> foo.mkv.strm) + // is a rename. Keep a soft-deleted tombstone so the later ingest can + // adopt its ID and scraped metadata even if fsnotify delivers the + // remove event before the create event. A true deletion stays hard. + if row.Path == path && liveSTRMPathAlias(path) { + softIDs = append(softIDs, row.ID) + continue + } + hardIDs = append(hardIDs, row.ID) } - if len(ids) == 0 { - return 0, nil + var removed int64 + if len(softIDs) > 0 { + res := s.repo.DB.WithContext(ctx). + Where("id IN ?", softIDs). + Delete(&model.Media{}) + if res.Error != nil { + return removed, res.Error + } + removed += res.RowsAffected } - res := s.repo.DB.WithContext(ctx).Unscoped(). - Where("id IN ?", ids). - Delete(&model.Media{}) - if res.Error == nil && res.RowsAffected > 0 { + if len(hardIDs) > 0 { + res := s.repo.DB.WithContext(ctx).Unscoped(). + Where("id IN ?", hardIDs). + Delete(&model.Media{}) + if res.Error != nil { + return removed, res.Error + } + removed += res.RowsAffected + } + if removed > 0 { s.invalidateMediaCache(ctx) } - return res.RowsAffected, res.Error + if len(softIDs) > 0 { + s.purgeExpiredSTRMTombstones(ctx) + } + return removed, nil +} + +const strmRenameTombstoneRetention = 7 * 24 * time.Hour + +// purgeExpiredSTRMTombstones keeps the soft-delete grace period bounded. +// It is deliberately restricted to deleted media rows under a STRM alias +// workflow; normal deletions are still hard-deleted immediately. +func (s *ScannerService) purgeExpiredSTRMTombstones(ctx context.Context) { + cutoff := time.Now().Add(-strmRenameTombstoneRetention) + if err := s.repo.DB.WithContext(ctx).Unscoped(). + Where("deleted_at IS NOT NULL AND deleted_at < ? AND path LIKE ?", cutoff, "%.strm"). + Delete(&model.Media{}).Error; err != nil && s.log != nil { + s.log.Warn("purge expired STRM rename tombstones failed", zap.Error(err)) + } } func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string, seen map[string]struct{}) (int64, error) { diff --git a/internal/service/scanner_scan.go b/internal/service/scanner_scan.go index 7f06697..fcb297a 100644 --- a/internal/service/scanner_scan.go +++ b/internal/service/scanner_scan.go @@ -117,6 +117,10 @@ func (s *ScannerService) scanLibrary(ctx context.Context, libraryID string, auto } continue } + // Flush alias-aware upserts before pruning missing paths. This lets a + // foo.strm <-> foo.mkv.strm rename migrate the existing row (including + // scraped metadata) instead of deleting it as "old path missing". + writeBatch.Flush() scannedRoots++ removed, err := s.pruneMissingMediaForRoot(ctx, lib.ID, root.ID, root.Path, seen) if err != nil { diff --git a/internal/service/scanner_strm_rename_test.go b/internal/service/scanner_strm_rename_test.go new file mode 100644 index 0000000..6327f86 --- /dev/null +++ b/internal/service/scanner_strm_rename_test.go @@ -0,0 +1,212 @@ +package service + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/truewhile/MeBox/internal/model" +) + +func TestSTRMPathAliasesCoverBothKeepExtDirections(t *testing.T) { + toKeepExt := strmPathAliases(filepath.Join(t.TempDir(), "Show.S01E01.strm")) + if !containsPathAlias(toKeepExt, "Show.S01E01.mkv.strm") { + t.Fatalf("missing keep_ext alias: %#v", toKeepExt) + } + fromKeepExt := strmPathAliases(filepath.Join(t.TempDir(), "Show.S01E01.mkv.strm")) + if !containsPathAlias(fromKeepExt, "Show.S01E01.strm") { + t.Fatalf("missing stripped alias: %#v", fromKeepExt) + } +} + +func TestMissingSTRMPathAliasesKeepsRealKeepExtSiblingVersions(t *testing.T) { + dir := t.TempDir() + current := filepath.Join(dir, "Show.S01E01.mkv.strm") + sibling := filepath.Join(dir, "Show.S01E01.mp4.strm") + writeFileContent(t, current, "https://example.test/mkv") + writeFileContent(t, sibling, "https://example.test/mp4") + + aliases := missingSTRMPathAliases(current) + if containsPathAlias(aliases, sibling) { + t.Fatalf("existing keep_ext sibling must remain a separate version: %#v", aliases) + } +} + +func TestScanLibraryMigratesSTRMKeepExtRename(t *testing.T) { + cases := []struct { + name string + oldName string + newName string + }{ + {name: "add video extension", oldName: "Show.S01E01.strm", newName: "Show.S01E01.mkv.strm"}, + {name: "remove video extension", oldName: "Show.S01E01.mkv.strm", newName: "Show.S01E01.strm"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sc, repos := newScannerTestEnv(t) + root := t.TempDir() + lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + oldPath := filepath.Join(root, tc.oldName) + newPath := filepath.Join(root, tc.newName) + writeFileContent(t, oldPath, "https://example.test/video") + if _, err := sc.IngestPath(t.Context(), lib.ID, oldPath); err != nil { + t.Fatal(err) + } + + var before model.Media + if err := repos.DB.First(&before, "path = ?", oldPath).Error; err != nil { + t.Fatal(err) + } + if err := repos.DB.Model(&model.Media{}).Where("id = ?", before.ID).Updates(map[string]any{ + "title": "地狱乐", + "original_name": "Jigokuraku", + "overview": "preserved overview", + "poster_url": "Jigokuraku-poster.jpg", + "backdrop_url": "Jigokuraku-backdrop.jpg", + "year": 2023, + "rating": 8.6, + "tm_db_id": 12345, + "bangumi_id": 67890, + "genres": "Action,Adventure", + "scrape_status": "matched", + }).Error; err != nil { + t.Fatal(err) + } + if err := os.Rename(oldPath, newPath); err != nil { + t.Fatal(err) + } + + res, err := sc.ScanLibrary(t.Context(), lib.ID) + if err != nil { + t.Fatal(err) + } + if res.ErrorCount != 0 { + t.Fatalf("scan errors: %#v", res.Errors) + } + if got := countMedia(t, repos); got != 1 { + t.Fatalf("media count = %d, want 1", got) + } + + var after model.Media + if err := repos.DB.First(&after, "id = ?", before.ID).Error; err != nil { + t.Fatalf("old media row was not preserved: %v", err) + } + if after.Path != newPath { + t.Fatalf("path = %q, want %q", after.Path, newPath) + } + if after.Title != "地狱乐" || after.OriginalName != "Jigokuraku" || after.Overview != "preserved overview" { + t.Fatalf("scraped identity metadata was lost: %#v", after) + } + if after.PosterURL != "Jigokuraku-poster.jpg" || after.BackdropURL != "Jigokuraku-backdrop.jpg" { + t.Fatalf("artwork was lost: poster=%q backdrop=%q", after.PosterURL, after.BackdropURL) + } + if after.TMDbID != 12345 || after.BangumiID != 67890 || after.ScrapeStatus != "matched" { + t.Fatalf("scrape IDs/status changed: tmdb=%d bgm=%d status=%q", after.TMDbID, after.BangumiID, after.ScrapeStatus) + } + }) + } +} + +func TestIngestPathAdoptsSoftDeletedSTRMRenameTombstone(t *testing.T) { + sc, repos := newScannerTestEnv(t) + root := t.TempDir() + lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + oldPath := filepath.Join(root, "Show.S01E01.mkv.strm") + newPath := filepath.Join(root, "Show.S01E01.strm") + writeFileContent(t, oldPath, "https://example.test/video") + if _, err := sc.IngestPath(t.Context(), lib.ID, oldPath); err != nil { + t.Fatal(err) + } + + var before model.Media + if err := repos.DB.First(&before, "path = ?", oldPath).Error; err != nil { + t.Fatal(err) + } + if err := repos.DB.Model(&model.Media{}).Where("id = ?", before.ID).Updates(map[string]any{ + "title": "拔作岛", + "overview": "kept through remove-before-create", + "poster_url": "nukitashi-poster.webp", + "tm_db_id": 222, + "scrape_status": "matched", + }).Error; err != nil { + t.Fatal(err) + } + if err := os.Rename(oldPath, newPath); err != nil { + t.Fatal(err) + } + + // Simulate fsnotify delivering Remove(old) before Create(new). + if removed, err := sc.RemovePath(t.Context(), oldPath); err != nil || removed != 1 { + t.Fatalf("RemovePath() removed=%d err=%v, want 1", removed, err) + } + var tombstone model.Media + if err := repos.DB.Unscoped().First(&tombstone, "id = ?", before.ID).Error; err != nil { + t.Fatal(err) + } + if !tombstone.DeletedAt.Valid { + t.Fatal("rename tombstone should be soft-deleted") + } + if got := countMedia(t, repos); got != 0 { + t.Fatalf("active media count = %d, want 0 before create event", got) + } + + if _, err := sc.IngestPath(t.Context(), lib.ID, newPath); err != nil { + t.Fatal(err) + } + var after model.Media + if err := repos.DB.First(&after, "id = ?", before.ID).Error; err != nil { + t.Fatalf("soft-deleted row was not restored: %v", err) + } + if after.Path != newPath || after.Title != "拔作岛" || after.Overview != "kept through remove-before-create" { + t.Fatalf("rename metadata was not adopted: %#v", after) + } + if after.TMDbID != 222 || after.ScrapeStatus != "matched" { + t.Fatalf("scrape metadata changed: tmdb=%d status=%q", after.TMDbID, after.ScrapeStatus) + } + if got := countMedia(t, repos); got != 1 { + t.Fatalf("media count = %d, want 1", got) + } +} + +func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) { + sc, repos := newScannerTestEnv(t) + root := t.TempDir() + lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "Deleted.S01E01.mkv.strm") + writeFileContent(t, path, "https://example.test/video") + if _, err := sc.IngestPath(t.Context(), lib.ID, path); err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if removed, err := sc.RemovePath(t.Context(), path); err != nil || removed != 1 { + t.Fatalf("RemovePath() removed=%d err=%v, want 1", removed, err) + } + var count int64 + if err := repos.DB.Unscoped().Model(&model.Media{}).Where("path = ?", path).Count(&count).Error; err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("true deletion left %d tombstone row(s)", count) + } +} + +func containsPathAlias(paths []string, name string) bool { + for _, path := range paths { + if strings.EqualFold(filepath.Base(path), filepath.Base(name)) { + return true + } + } + return false +} diff --git a/internal/service/scraper_query_test.go b/internal/service/scraper_query_test.go index 1c59a4e..188052d 100644 --- a/internal/service/scraper_query_test.go +++ b/internal/service/scraper_query_test.go @@ -48,6 +48,17 @@ func TestCleanQuery(t *testing.T) { } } +func TestCleanQueryIgnoresKeepExtSTRMSuffix(t *testing.T) { + plain := filepath.Join("/media/anime", "Jigokuraku 2023 S01E14.strm") + keepExt := filepath.Join("/media/anime", "Jigokuraku 2023 S01E14.mkv.strm") + plainTitle, plainYear := CleanQuery(plain) + keepTitle, keepYear := CleanQuery(keepExt) + if plainTitle != keepTitle || plainYear != keepYear { + t.Fatalf("keep_ext changed scrape identity: plain=(%q,%d) keep_ext=(%q,%d)", + plainTitle, plainYear, keepTitle, keepYear) + } +} + func TestScrapeQueryCandidatesCleanDirtySeriesFolder(t *testing.T) { lib := &model.Library{ Path: `F:\media\电视剧\欧美剧`, diff --git a/internal/service/strm_service.go b/internal/service/strm_service.go index 5bdfddd..1502693 100644 --- a/internal/service/strm_service.go +++ b/internal/service/strm_service.go @@ -49,7 +49,7 @@ const ( const ( StrmDefaultVideoExt = "mkv,mp4,avi,rmvb,rm,mov,ts,wmv,flv,m4v,iso,mpg,mpeg,webm" - StrmDefaultMetaExt = "nfo,jpg,jpeg,png,srt,ass,ssa,sub,txt,bmp,webp" + StrmDefaultMetaExt = "nfo,jpg,jpeg,png,srt,ass,ssa,sub,txt,bmp,webp,img" StrmDefaultExclude = "sample,trailer,预告" ) diff --git a/internal/service/strm_sync.go b/internal/service/strm_sync.go index cc48b0d..9c25975 100644 --- a/internal/service/strm_sync.go +++ b/internal/service/strm_sync.go @@ -612,6 +612,12 @@ func (st *strmSyncState) isVideoExt(ext string, size int64) bool { } func (st *strmSyncState) isMetaExt(ext string) bool { + // Legacy MeBox installs may have persisted strm.meta_ext without img. + // .img is emitted by the artwork writer as a fallback image container, so + // keep accepting existing sidecars without requiring a settings migration. + if strings.EqualFold(ext, ".img") { + return true + } for _, e := range st.cfg.MetaExt { if "."+e == ext { return true diff --git a/internal/service/strm_sync_test.go b/internal/service/strm_sync_test.go index 6bbd5a1..70d549b 100644 --- a/internal/service/strm_sync_test.go +++ b/internal/service/strm_sync_test.go @@ -1680,3 +1680,10 @@ func TestHandleVideoKeepExtWritesAllVersions(t *testing.T) { t.Fatalf("NewStrm=%d want 2", st.rec.NewStrm) } } + +func TestStrmSyncTreatsLegacyImgAsMetadata(t *testing.T) { + st := &strmSyncState{cfg: &strmPathConfig{MetaExt: csvSplit("nfo,jpg")}} + if !st.isMetaExt(".img") { + t.Fatal("legacy .img sidecars must remain metadata even when an old setting omits img") + } +}