Preserve media metadata across STRM path renames

This commit is contained in:
truewhile
2026-09-11 20:45:54 +08:00
parent d6355d5582
commit cfe0c0cf28
16 changed files with 669 additions and 116 deletions
+86 -26
View File
@@ -3,6 +3,7 @@ package repository
import (
"context"
"errors"
"path/filepath"
"strings"
"gorm.io/gorm"
@@ -10,6 +11,18 @@ import (
"github.com/truewhile/MeBox/internal/model"
)
// MediaUpsertItem carries a media row and optional sibling paths from an
// earlier keep_ext naming mode. An alias is only used when the exact new path
// does not exist in the database; in that case the existing row is migrated to
// the new path so scraped metadata survives renames such as:
//
// foo.strm -> foo.mkv.strm
// foo.mkv.strm -> foo.strm
type MediaUpsertItem struct {
Media *model.Media
AliasPaths []string
}
// Upsert inserts or updates a media row keyed by Path (unique index).
//
// 重要:当一条行已经存在时,scanner 重扫只应该刷新文件级元数据
@@ -22,20 +35,14 @@ import (
// 显式写入)。这两个问题都让 EnrichLibrary(WHERE scrape_status='pending')
// 永远捞不到数据。
func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error {
var indexIDs []string
err := withSQLiteBusyRetry(ctx, func() error {
id, uerr := r.upsertWithDB(ctx, r.db, m)
if uerr != nil {
return uerr
}
indexIDs = append(indexIDs[:0], id)
return nil
})
if err != nil {
return err
}
r.indexByIDBestEffort(ctx, indexIDs)
return nil
return r.UpsertWithAliases(ctx, m, nil)
}
// UpsertWithAliases is Upsert with explicit, filesystem-verified STRM sibling
// aliases. It preserves the old row's ID, CreatedAt and scraped metadata while
// moving it to the current path.
func (r *MediaRepository) UpsertWithAliases(ctx context.Context, m *model.Media, aliasPaths []string) error {
return r.UpsertBatchWithAliases(ctx, []MediaUpsertItem{{Media: m, AliasPaths: aliasPaths}})
}
// UpsertBatch 在单个事务里逐条执行 Upsert:扫描一批只提交(fsync)一次,
@@ -46,6 +53,17 @@ func (r *MediaRepository) Upsert(ctx context.Context, m *model.Media) error {
// 事务内会把 SQLite 写锁挂起在网络 IO 上,且批内用非事务连接回读只能
// 拿到提交前的旧版本数据,把陈旧内容写进索引。
func (r *MediaRepository) UpsertBatch(ctx context.Context, items []*model.Media) error {
mapped := make([]MediaUpsertItem, 0, len(items))
for _, item := range items {
if item != nil {
mapped = append(mapped, MediaUpsertItem{Media: item})
}
}
return r.UpsertBatchWithAliases(ctx, mapped)
}
// UpsertBatchWithAliases runs alias-aware upserts in one transaction.
func (r *MediaRepository) UpsertBatchWithAliases(ctx context.Context, items []MediaUpsertItem) error {
if len(items) == 0 {
return nil
}
@@ -53,11 +71,11 @@ func (r *MediaRepository) UpsertBatch(ctx context.Context, items []*model.Media)
err := withSQLiteBusyRetry(ctx, func() error {
indexIDs = indexIDs[:0]
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
for _, m := range items {
if m == nil {
for _, item := range items {
if item.Media == nil {
continue
}
id, err := r.upsertWithDB(ctx, tx, m)
id, err := r.upsertWithDB(ctx, tx, item.Media, item.AliasPaths)
if err != nil {
return err
}
@@ -87,9 +105,9 @@ func (r *MediaRepository) indexByIDBestEffort(ctx context.Context, ids []string)
}
}
// upsertWithDB 落库(新建或更新),返回需要重建索引的媒体 ID(无则空串)。
func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *model.Media) (string, error) {
existing, created, err := r.findOrCreateMediaByPath(ctx, db, m)
// upsertWithDB 落库(新建、更新或从旧路径迁移),返回需要重建索引的媒体 ID。
func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *model.Media, aliasPaths []string) (string, error) {
existing, created, adopted, err := r.findOrCreateMediaByPath(ctx, db, m, aliasPaths)
if err != nil {
return "", err
}
@@ -98,6 +116,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
}
updates := mediaUpsertUpdates(existing, *m)
if adopted {
updates["path"] = m.Path
if existing.DeletedAt.Valid {
updates["deleted_at"] = nil
}
}
if len(updates) == 0 {
*m = existing
return "", nil
@@ -107,31 +131,67 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
return "", err
}
// 回写 ID / 不可变字段,让 caller 拿到完整的现有行。
if adopted {
existing.Path = m.Path
existing.DeletedAt = gorm.DeletedAt{}
}
*m = existing
return existing.ID, nil
}
func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.DB, m *model.Media) (model.Media, bool, error) {
func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.DB, m *model.Media, aliasPaths []string) (model.Media, bool, bool, error) {
var existing model.Media
err := db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
if alias, aliasErr := r.findMediaByAlias(ctx, db, m.Path, aliasPaths); aliasErr == nil {
return alias, false, true, nil
} else if !errors.Is(aliasErr, gorm.ErrRecordNotFound) {
return model.Media{}, false, false, aliasErr
}
// 新行:保证 scrape_status 走 GORM default:pending(即留空让数据库填)。
if m.ScrapeStatus == "" {
m.ScrapeStatus = "pending"
}
if createErr := db.WithContext(ctx).Create(m).Error; createErr == nil {
return *m, true, nil
return *m, true, false, nil
} else if retryErr := db.WithContext(ctx).Unscoped().Where("path = ?", m.Path).First(&existing).Error; retryErr != nil {
return model.Media{}, false, createErr
return model.Media{}, false, false, createErr
} else {
// 并发插入竞态:重查已命中既有行,直接走更新分支。
return existing, false, nil
return existing, false, false, nil
}
}
if err != nil {
return model.Media{}, false, err
return model.Media{}, false, false, err
}
return existing, false, nil
return existing, false, false, nil
}
func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, currentPath string, aliasPaths []string) (model.Media, error) {
aliases := make([]string, 0, len(aliasPaths))
seen := make(map[string]struct{}, len(aliasPaths))
for _, alias := range aliasPaths {
alias = filepath.Clean(strings.TrimSpace(alias))
if alias == "" || alias == "." || alias == filepath.Clean(currentPath) {
continue
}
if _, ok := seen[alias]; ok {
continue
}
seen[alias] = struct{}{}
aliases = append(aliases, alias)
}
if len(aliases) == 0 {
return model.Media{}, gorm.ErrRecordNotFound
}
var existing model.Media
err := db.WithContext(ctx).Unscoped().
Where("path IN ?", aliases).
Order("CASE WHEN scrape_status = 'matched' THEN 0 ELSE 1 END ASC, " +
"CASE WHEN COALESCE(poster_url, '') <> '' OR COALESCE(overview, '') <> '' THEN 0 ELSE 1 END ASC, " +
"CASE WHEN deleted_at IS NULL THEN 0 ELSE 1 END ASC, updated_at DESC, created_at DESC").
First(&existing).Error
return existing, err
}
func mediaUpsertUpdates(existing, incoming model.Media) map[string]any {
+3 -3
View File
@@ -158,7 +158,7 @@ func firstExistingImage(dir string, names ...string) string {
return ""
}
for _, name := range names {
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn"} {
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn", ".img"} {
path := filepath.Join(dir, name+ext)
if fileExists(path) {
return filepath.Clean(path)
@@ -221,7 +221,7 @@ func firstExistingPosterImage(dir string, names ...string) string {
if isRejectedPosterName(name) {
continue
}
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn"} {
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn", ".img"} {
path := filepath.Join(dir, name+ext)
if fileExists(path) && likelyPosterImage(path) {
return filepath.Clean(path)
@@ -240,7 +240,7 @@ func firstAdultLooseImage(dir, kind string) string {
fallback := []string{}
for _, path := range matches {
ext := strings.ToLower(filepath.Ext(path))
if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" && ext != ".gif" && ext != ".bmp" && ext != ".tbn" {
if ext != ".jpg" && ext != ".jpeg" && ext != ".png" && ext != ".webp" && ext != ".gif" && ext != ".bmp" && ext != ".tbn" && ext != ".img" {
continue
}
name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), ext))
+19
View File
@@ -417,3 +417,22 @@ func TestReadLocalMetadataArtworkFallbackOnGarbageShowNFO(t *testing.T) {
t.Fatalf("expected episode artwork fallback, got %+v", got)
}
}
func TestReadLocalMetadataFindsLegacyImgPoster(t *testing.T) {
root := t.TempDir()
mediaPath := filepath.Join(root, "Jigokuraku.S01E14.mkv.strm")
poster := filepath.Join(root, "Jigokuraku.S01E14-poster.img")
if err := os.WriteFile(mediaPath, []byte("https://example.test/video"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(poster, testJPEG, 0o644); err != nil {
t.Fatal(err)
}
got, err := ReadLocalMetadata(mediaPath, root, true)
if err != nil {
t.Fatal(err)
}
if got == nil || got.PosterURL != poster {
t.Fatalf("PosterURL = %q, want legacy .img poster %q", got.PosterURL, poster)
}
}
+101
View File
@@ -0,0 +1,101 @@
package service
import (
"os"
"path/filepath"
"sort"
"strings"
)
// strmPathAliases returns sibling STRM names that can represent the same media
// item when keep_ext is toggled on or off.
//
// Both directions are supported:
//
// foo.strm <-> foo.mkv.strm
// foo.strm <-> foo.mp4.strm
//
// The caller must still verify that the aliases are no longer present on disk.
// keep_ext intentionally allows foo.mkv.strm and foo.mp4.strm to coexist as
// two real versions, so an existing sibling must never be absorbed.
func strmPathAliases(path string) []string {
clean := filepath.Clean(strings.TrimSpace(path))
if clean == "" || clean == "." || !strings.EqualFold(filepath.Ext(clean), ".strm") {
return nil
}
base := mediaSidecarBase(clean)
if base == "" {
return nil
}
name := filepath.Base(clean)
stem := strings.TrimSuffix(name, filepath.Ext(name))
lastExt := strings.ToLower(filepath.Ext(stem))
_, hasVideoExt := videoExtensions[lastExt]
hasVideoExt = hasVideoExt && !strings.EqualFold(lastExt, ".strm")
dir := filepath.Dir(clean)
out := make([]string, 0, len(videoExtensions)+1)
seen := make(map[string]struct{}, len(videoExtensions)+1)
add := func(candidate string) {
candidate = filepath.Clean(candidate)
if candidate == "" || candidate == "." || samePath(candidate, clean) {
return
}
key := strings.ToLower(candidate)
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
out = append(out, candidate)
}
if hasVideoExt {
// Current keep_ext shape: the only legacy shape is the stripped name.
add(filepath.Join(dir, base+".strm"))
return out
}
// Stripped shape: an old keep_ext file may use any configured video ext.
exts := make([]string, 0, len(videoExtensions))
for ext := range videoExtensions {
if strings.EqualFold(ext, ".strm") {
continue
}
exts = append(exts, strings.ToLower(ext))
}
sort.Strings(exts)
for _, ext := range exts {
add(filepath.Join(dir, base+ext+".strm"))
if upper := strings.ToUpper(ext); upper != ext {
add(filepath.Join(dir, base+upper+".strm"))
}
}
return out
}
// missingSTRMPathAliases keeps only aliases which no longer exist on disk.
// This is the key protection for keep_ext=true multi-version libraries: an
// existing foo.mp4.strm is a real second version, not a rename tombstone.
func missingSTRMPathAliases(path string) []string {
aliases := strmPathAliases(path)
if len(aliases) == 0 {
return nil
}
out := make([]string, 0, len(aliases))
for _, alias := range aliases {
if _, err := os.Lstat(alias); err == nil {
continue
} else if !os.IsNotExist(err) {
continue
}
out = append(out, alias)
}
return out
}
func liveSTRMPathAlias(path string) bool {
for _, alias := range strmPathAliases(path) {
if info, err := os.Stat(alias); err == nil && !info.IsDir() {
return true
}
}
return false
}
+6 -3
View File
@@ -67,17 +67,20 @@ func newMediaSeriesKeyResolver(items []model.Media) mediaSeriesKeyResolver {
func (r mediaSeriesKeyResolver) key(media model.Media) string {
if mediaLooksEpisodicForGrouping(media) {
if key := repeatedSeriesTitleKey(media); key != "" && r.titleCounts[key] > 1 {
return compactSeriesKey(key)
}
if pathKey := mediaSeriesRawKey(media); strings.HasPrefix(pathKey, "library-path") {
if titleKey := r.pathTitles[pathKey]; titleKey != "" {
return compactSeriesKey(titleKey)
}
// A series directory is the strongest identity for mixed rows:
// main episodes and specials (CM/NCOP/PV/OVA) may be scraped to
// slightly different titles, but they still belong to one show.
if r.pathCounts[pathKey] > 1 {
return compactSeriesKey(pathKey)
}
}
if key := repeatedSeriesTitleKey(media); key != "" && r.titleCounts[key] > 1 {
return compactSeriesKey(key)
}
if key := repeatedSeriesExternalKey(media); key != "" && r.externalCounts[key] > 1 {
return compactSeriesKey(key)
}
+56 -11
View File
@@ -47,14 +47,14 @@ func TestListRecentSeriesCardsCountsAllEpisodesInSeries(t *testing.T) {
if len(cards) != 1 {
t.Fatalf("recent cards = %#v, want one series card", cards)
}
if cards[0].Count != 40 {
t.Fatalf("recent series count = %d, want full 40 episodes", cards[0].Count)
}
expectedLastAdded := now.Add(40 * time.Minute)
if cards[0].LastAddedAt == nil || !cards[0].LastAddedAt.Equal(expectedLastAdded) {
t.Fatalf("recent series LastAddedAt = %v, want %v", cards[0].LastAddedAt, expectedLastAdded)
}
if cards[0].Count != 40 {
t.Fatalf("recent series count = %d, want full 40 episodes", cards[0].Count)
}
expectedLastAdded := now.Add(40 * time.Minute)
if cards[0].LastAddedAt == nil || !cards[0].LastAddedAt.Equal(expectedLastAdded) {
t.Fatalf("recent series LastAddedAt = %v, want %v", cards[0].LastAddedAt, expectedLastAdded)
}
}
func TestMediaSeriesKeyCollapsesNestedSpecialFolders(t *testing.T) {
main := model.Media{
@@ -377,6 +377,51 @@ func TestGroupMediaSeriesCardsMergesPollutedEpisodeFoldersBySharedShowID(t *test
}
}
func TestGroupMediaSeriesCardsKeepsMixedTitlesInSameEpisodicDirectoryTogether(t *testing.T) {
items := []model.Media{
{
Base: model.Base{ID: "main-1"},
LibraryID: "anime",
Title: "住在拔作岛上的我应该如何是好?",
Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][01][AVC_2×FLAC].mkv.strm`,
SeasonNum: 1,
EpisodeNum: 1,
ScrapeStatus: "matched",
},
{
Base: model.Base{ID: "main-2"},
LibraryID: "anime",
Title: "住在拔作岛上的我应该如何是好?",
Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][02][AVC_2×FLAC].mkv.strm`,
SeasonNum: 1,
EpisodeNum: 2,
ScrapeStatus: "matched",
},
{
Base: model.Base{ID: "special-1"},
LibraryID: "anime",
Title: "nukitashi",
Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][CM_01][AVC_FLAC].mkv.strm`,
ScrapeStatus: "matched",
},
{
Base: model.Base{ID: "special-2"},
LibraryID: "anime",
Title: "nukitashi",
Path: `/media/影视库/动漫/拔作岛/[64bitsub][Nukitashi][PV_01][AVC_FLAC].mkv.strm`,
ScrapeStatus: "matched",
},
}
cards := groupMediaSeriesCards(items)
if len(cards) != 1 {
t.Fatalf("cards=%#v, want main episodes and specials in the same directory folded into one card", cards)
}
if cards[0].Count != 4 {
t.Fatalf("series count=%d, want 4 items", cards[0].Count)
}
}
func TestGroupMediaSeriesCardsKeepsMovieVersionsAsOneMovie(t *testing.T) {
items := []model.Media{
{
@@ -503,11 +548,11 @@ func TestGroupMediaSeriesCardsKeepsIndependentMoviesSeparateInSharedSubdirectory
{LibraryID: "movies", Title: "cd1", Path: `/media/电影/指环王 (2001)/cd1.mkv`},
{LibraryID: "movies", Title: "cd2", Path: `/media/电影/指环王 (2001)/cd2.mkv`},
}
cdCards := groupMediaSeriesCards(cdItems)
if len(cdCards) != 1 {
t.Fatalf("got %d cards for cd1/cd2, want 1 folded movie card", len(cdCards))
}
cdCards := groupMediaSeriesCards(cdItems)
if len(cdCards) != 1 {
t.Fatalf("got %d cards for cd1/cd2, want 1 folded movie card", len(cdCards))
}
}
func TestListMediaEpisodesKeepsIndependentMoviesSeparate(t *testing.T) {
db := newServiceTestDB(t, &model.Library{}, &model.Media{})
+1 -1
View File
@@ -43,7 +43,7 @@ var artworkSidecarSuffixes = []string{
// artworkSidecarExtensions are the image extensions a sidecar may use. Both
// "base-poster.jpg" (bare separator) and "base.poster.jpg" (dotted separator)
// rely on suffix matching, so we probe the common image extensions.
var artworkSidecarExtensions = []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn"}
var artworkSidecarExtensions = []string{".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tbn", ".img"}
// transferSidecarArtwork moves/copies/links the scraped poster/backdrop
// sidecar files alongside its media using the same transfer mode, mirroring
+4 -2
View File
@@ -200,7 +200,7 @@ func (s *ScannerService) writeLocalScanMedia(in localScanWriteInput) {
in.writeBatch.AddWithAfter(in.path, in.media, in.after)
return
}
if err := s.repo.Media.Upsert(in.ctx, in.media); err != nil {
if err := s.repo.Media.UpsertWithAliases(in.ctx, in.media, missingSTRMPathAliases(in.path)); err != nil {
addScanError(in.res, in.path, err)
s.log.Warn("upsert media failed", zap.String("path", in.path), zap.Error(err))
return
@@ -252,8 +252,10 @@ func (s *ScannerService) duplicateByFileID(ctx context.Context, fileID, path str
}
func (s *ScannerService) mediaPathExists(ctx context.Context, path string) bool {
paths := []string{path}
paths = append(paths, missingSTRMPathAliases(path)...)
var count int64
err := s.repo.DB.WithContext(ctx).Unscoped().Model(&model.Media{}).
Where("path = ?", path).Count(&count).Error
Where("path IN ?", paths).Count(&count).Error
return err == nil && count > 0
}
+99 -58
View File
@@ -7,6 +7,7 @@ import (
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
)
type localMediaWriteBatch struct {
@@ -18,9 +19,10 @@ type localMediaWriteBatch struct {
}
type localMediaWriteItem struct {
path string
media *model.Media
after func()
path string
media *model.Media
aliases []string
after func()
}
func newLocalMediaWriteBatch(scanner *ScannerService, ctx context.Context, res *ScanResult, limit int) *localMediaWriteBatch {
@@ -41,7 +43,12 @@ func (b *localMediaWriteBatch) AddWithAfter(path string, media *model.Media, aft
if media.ScrapeStatus == "" {
media.ScrapeStatus = "pending"
}
b.items = append(b.items, localMediaWriteItem{path: path, media: media, after: after})
b.items = append(b.items, localMediaWriteItem{
path: path,
media: media,
aliases: missingSTRMPathAliases(path),
after: after,
})
if len(b.items) >= b.limit {
b.Flush()
}
@@ -53,39 +60,35 @@ func (b *localMediaWriteBatch) Flush() {
}
items := b.items
b.items = nil
media := make([]model.Media, 0, len(items))
for _, item := range items {
if item.media != nil {
media = append(media, *item.media)
}
}
if len(media) == 0 {
return
}
existingPaths := b.existingPaths(items)
upsertItems := make([]*model.Media, 0, len(items))
upsertAfter := make([]func(), 0, len(items))
existingPaths, lookupOK := b.existingPathOrAliasSet(items)
upsertItems := make([]localMediaWriteItem, 0, len(items))
createItems := make([]localMediaWriteItem, 0, len(items))
createMedia := make([]model.Media, 0, len(items))
for _, item := range items {
if item.media == nil {
continue
}
if existingPaths[filepath.Clean(item.media.Path)] {
// 已存在行:攒起来在一个事务里逐条 upsert(一批一次提交)。
after := item.after
upsertItems = append(upsertItems, item.media)
upsertAfter = append(upsertAfter, after)
// If the alias lookup failed, route through UpsertWithAliases instead of
// direct-create. That is slower but cannot create a duplicate row.
if !lookupOK || mediaPathOrAliasExists(item.media.Path, item.aliases, existingPaths) {
upsertItems = append(upsertItems, item)
continue
}
createItems = append(createItems, item)
createMedia = append(createMedia, *item.media)
}
b.flushUpserts(items, upsertItems, upsertAfter)
if len(createMedia) == 0 {
b.flushUpserts(upsertItems)
if len(createItems) == 0 {
b.publish()
return
}
createMedia := make([]model.Media, 0, len(createItems))
for _, item := range createItems {
if item.media != nil {
createMedia = append(createMedia, *item.media)
}
}
if err := b.scanner.repo.DB.WithContext(b.ctx).CreateInBatches(&createMedia, b.limit).Error; err == nil {
b.res.Added += len(createMedia)
for _, item := range createItems {
@@ -96,12 +99,13 @@ func (b *localMediaWriteBatch) Flush() {
b.publish()
return
}
for _, item := range createItems {
if item.media == nil {
continue
}
wasExisting := b.mediaPathExists(item.media.Path)
if err := b.scanner.repo.Media.Upsert(b.ctx, item.media); err != nil {
if err := b.scanner.repo.Media.UpsertWithAliases(b.ctx, item.media, item.aliases); err != nil {
addScanError(b.res, item.path, err)
b.scanner.log.Warn("upsert media failed", zap.String("path", item.path), zap.Error(err))
continue
@@ -118,47 +122,90 @@ func (b *localMediaWriteBatch) Flush() {
b.publish()
}
func (b *localMediaWriteBatch) existingPaths(items []localMediaWriteItem) map[string]bool {
// existingPathOrAliasSet loads exact and alias paths in bounded query chunks. Aliases
// have already been filtered against the filesystem by AddWithAfter, so an
// existing keep_ext sibling is never treated as a rename.
func (b *localMediaWriteBatch) existingPathOrAliasSet(items []localMediaWriteItem) (map[string]bool, bool) {
out := map[string]bool{}
if b == nil || b.scanner == nil || b.scanner.repo == nil || b.scanner.repo.DB == nil || len(items) == 0 {
return out
return out, false
}
seen := make(map[string]struct{}, len(items)*2)
paths := make([]string, 0, len(items)*2)
add := func(path string) {
path = filepath.Clean(path)
if path == "" || path == "." {
return
}
if _, ok := seen[path]; ok {
return
}
seen[path] = struct{}{}
paths = append(paths, path)
}
paths := make([]string, 0, len(items))
for _, item := range items {
if item.media == nil || item.media.Path == "" {
continue
}
paths = append(paths, item.media.Path)
add(item.media.Path)
for _, alias := range item.aliases {
add(alias)
}
}
if len(paths) == 0 {
return out
return out, true
}
var rows []string
if err := b.scanner.repo.DB.WithContext(b.ctx).
Unscoped().
Model(&model.Media{}).
Where("path IN ?", paths).
Pluck("path", &rows).Error; err != nil {
b.scanner.log.Debug("load existing media paths for scan batch failed", zap.Error(err))
return out
// Keep SQL variables below the legacy SQLite limit (999). A batch can
// contain 100 STRM rows and every row has many sibling aliases.
const pathLookupChunk = 400
for start := 0; start < len(paths); start += pathLookupChunk {
end := start + pathLookupChunk
if end > len(paths) {
end = len(paths)
}
var rows []string
if err := b.scanner.repo.DB.WithContext(b.ctx).
Unscoped().
Model(&model.Media{}).
Where("path IN ?", paths[start:end]).
Pluck("path", &rows).Error; err != nil {
b.scanner.log.Debug("load existing media paths for scan batch failed", zap.Error(err))
return nil, false
}
for _, path := range rows {
out[filepath.Clean(path)] = true
}
}
for _, path := range rows {
out[filepath.Clean(path)] = true
}
return out
return out, true
}
// flushUpserts 把已存在行的 upsert 攒成一个事务(一次提交/一组 fsync)。
// 整批失败(如单条数据触发约束)时退回逐条 Upsert,只丢真正坏的那几条。
func (b *localMediaWriteBatch) flushUpserts(allItems []localMediaWriteItem, upsertItems []*model.Media, upsertAfter []func()) {
func mediaPathOrAliasExists(path string, aliases []string, existing map[string]bool) bool {
if existing[filepath.Clean(path)] {
return true
}
for _, alias := range aliases {
if existing[filepath.Clean(alias)] {
return true
}
}
return false
}
// flushUpserts 把已存在或可迁移路径的行攒成一个事务(一次提交/一组 fsync)。
// 整批失败(如单条数据触发约束)时退回逐条 upsert,只丢真正坏的那几条。
func (b *localMediaWriteBatch) flushUpserts(upsertItems []localMediaWriteItem) {
if len(upsertItems) == 0 {
return
}
if err := b.scanner.repo.Media.UpsertBatch(b.ctx, upsertItems); err == nil {
batchItems := make([]repository.MediaUpsertItem, 0, len(upsertItems))
for _, item := range upsertItems {
batchItems = append(batchItems, repository.MediaUpsertItem{Media: item.media, AliasPaths: item.aliases})
}
if err := b.scanner.repo.Media.UpsertBatchWithAliases(b.ctx, batchItems); err == nil {
b.res.Updated += len(upsertItems)
for _, after := range upsertAfter {
if after != nil {
after()
for _, item := range upsertItems {
if item.after != nil {
item.after()
}
}
return
@@ -166,13 +213,7 @@ func (b *localMediaWriteBatch) flushUpserts(allItems []localMediaWriteItem, upse
b.scanner.log.Warn("batch upsert failed; falling back to per-item upsert",
zap.Int("items", len(upsertItems)))
}
// 兜底:按原始顺序找回每个条目的 path/after(两个切片同序但可能含 nil)。
idx := 0
for _, item := range allItems {
if item.media == nil || idx >= len(upsertItems) || upsertItems[idx] != item.media {
continue
}
idx++
for _, item := range upsertItems {
b.upsertExistingItem(item)
}
}
@@ -181,7 +222,7 @@ func (b *localMediaWriteBatch) upsertExistingItem(item localMediaWriteItem) {
if item.media == nil {
return
}
if err := b.scanner.repo.Media.Upsert(b.ctx, item.media); err != nil {
if err := b.scanner.repo.Media.UpsertWithAliases(b.ctx, item.media, item.aliases); err != nil {
addScanError(b.res, item.path, err)
b.scanner.log.Warn("upsert media failed", zap.String("path", item.path), zap.Error(err))
return
+53 -11
View File
@@ -5,13 +5,15 @@ import (
"os"
"path/filepath"
"strings"
"time"
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/model"
)
// RemovePath 物理删除磁盘上已不存在的媒体记录。
// RemovePath 移除磁盘上已不存在的媒体记录。普通的删除仍做物理删除;
// 可确认是 STRM 扩展名改名的路径则先保留软删除墓碑,供后续 ingest 继承元数据。
func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) {
if _, err := os.Stat(path); err == nil {
return 0, nil // still exists; nothing to remove
@@ -30,24 +32,64 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
Find(&rows).Error; err != nil {
return 0, err
}
ids := make([]string, 0, len(rows))
softIDs := make([]string, 0, len(rows))
hardIDs := make([]string, 0, len(rows))
for _, row := range rows {
// LIKE 里的 % _ 是通配符(候选集只会偏大),用 Go 前缀精确过滤,
// 避免对含 % / _ 的路径误删。
if row.Path == path || strings.HasPrefix(filepath.Clean(row.Path), prefix) {
ids = append(ids, row.ID)
if row.Path != path && !strings.HasPrefix(filepath.Clean(row.Path), prefix) {
continue
}
// A vanished STRM with a live sibling (foo.strm <-> foo.mkv.strm)
// is a rename. Keep a soft-deleted tombstone so the later ingest can
// adopt its ID and scraped metadata even if fsnotify delivers the
// remove event before the create event. A true deletion stays hard.
if row.Path == path && liveSTRMPathAlias(path) {
softIDs = append(softIDs, row.ID)
continue
}
hardIDs = append(hardIDs, row.ID)
}
if len(ids) == 0 {
return 0, nil
var removed int64
if len(softIDs) > 0 {
res := s.repo.DB.WithContext(ctx).
Where("id IN ?", softIDs).
Delete(&model.Media{})
if res.Error != nil {
return removed, res.Error
}
removed += res.RowsAffected
}
res := s.repo.DB.WithContext(ctx).Unscoped().
Where("id IN ?", ids).
Delete(&model.Media{})
if res.Error == nil && res.RowsAffected > 0 {
if len(hardIDs) > 0 {
res := s.repo.DB.WithContext(ctx).Unscoped().
Where("id IN ?", hardIDs).
Delete(&model.Media{})
if res.Error != nil {
return removed, res.Error
}
removed += res.RowsAffected
}
if removed > 0 {
s.invalidateMediaCache(ctx)
}
return res.RowsAffected, res.Error
if len(softIDs) > 0 {
s.purgeExpiredSTRMTombstones(ctx)
}
return removed, nil
}
const strmRenameTombstoneRetention = 7 * 24 * time.Hour
// purgeExpiredSTRMTombstones keeps the soft-delete grace period bounded.
// It is deliberately restricted to deleted media rows under a STRM alias
// workflow; normal deletions are still hard-deleted immediately.
func (s *ScannerService) purgeExpiredSTRMTombstones(ctx context.Context) {
cutoff := time.Now().Add(-strmRenameTombstoneRetention)
if err := s.repo.DB.WithContext(ctx).Unscoped().
Where("deleted_at IS NOT NULL AND deleted_at < ? AND path LIKE ?", cutoff, "%.strm").
Delete(&model.Media{}).Error; err != nil && s.log != nil {
s.log.Warn("purge expired STRM rename tombstones failed", zap.Error(err))
}
}
func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string, seen map[string]struct{}) (int64, error) {
+4
View File
@@ -117,6 +117,10 @@ func (s *ScannerService) scanLibrary(ctx context.Context, libraryID string, auto
}
continue
}
// Flush alias-aware upserts before pruning missing paths. This lets a
// foo.strm <-> foo.mkv.strm rename migrate the existing row (including
// scraped metadata) instead of deleting it as "old path missing".
writeBatch.Flush()
scannedRoots++
removed, err := s.pruneMissingMediaForRoot(ctx, lib.ID, root.ID, root.Path, seen)
if err != nil {
@@ -0,0 +1,212 @@
package service
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/truewhile/MeBox/internal/model"
)
func TestSTRMPathAliasesCoverBothKeepExtDirections(t *testing.T) {
toKeepExt := strmPathAliases(filepath.Join(t.TempDir(), "Show.S01E01.strm"))
if !containsPathAlias(toKeepExt, "Show.S01E01.mkv.strm") {
t.Fatalf("missing keep_ext alias: %#v", toKeepExt)
}
fromKeepExt := strmPathAliases(filepath.Join(t.TempDir(), "Show.S01E01.mkv.strm"))
if !containsPathAlias(fromKeepExt, "Show.S01E01.strm") {
t.Fatalf("missing stripped alias: %#v", fromKeepExt)
}
}
func TestMissingSTRMPathAliasesKeepsRealKeepExtSiblingVersions(t *testing.T) {
dir := t.TempDir()
current := filepath.Join(dir, "Show.S01E01.mkv.strm")
sibling := filepath.Join(dir, "Show.S01E01.mp4.strm")
writeFileContent(t, current, "https://example.test/mkv")
writeFileContent(t, sibling, "https://example.test/mp4")
aliases := missingSTRMPathAliases(current)
if containsPathAlias(aliases, sibling) {
t.Fatalf("existing keep_ext sibling must remain a separate version: %#v", aliases)
}
}
func TestScanLibraryMigratesSTRMKeepExtRename(t *testing.T) {
cases := []struct {
name string
oldName string
newName string
}{
{name: "add video extension", oldName: "Show.S01E01.strm", newName: "Show.S01E01.mkv.strm"},
{name: "remove video extension", oldName: "Show.S01E01.mkv.strm", newName: "Show.S01E01.strm"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
sc, repos := newScannerTestEnv(t)
root := t.TempDir()
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
oldPath := filepath.Join(root, tc.oldName)
newPath := filepath.Join(root, tc.newName)
writeFileContent(t, oldPath, "https://example.test/video")
if _, err := sc.IngestPath(t.Context(), lib.ID, oldPath); err != nil {
t.Fatal(err)
}
var before model.Media
if err := repos.DB.First(&before, "path = ?", oldPath).Error; err != nil {
t.Fatal(err)
}
if err := repos.DB.Model(&model.Media{}).Where("id = ?", before.ID).Updates(map[string]any{
"title": "地狱乐",
"original_name": "Jigokuraku",
"overview": "preserved overview",
"poster_url": "Jigokuraku-poster.jpg",
"backdrop_url": "Jigokuraku-backdrop.jpg",
"year": 2023,
"rating": 8.6,
"tm_db_id": 12345,
"bangumi_id": 67890,
"genres": "Action,Adventure",
"scrape_status": "matched",
}).Error; err != nil {
t.Fatal(err)
}
if err := os.Rename(oldPath, newPath); err != nil {
t.Fatal(err)
}
res, err := sc.ScanLibrary(t.Context(), lib.ID)
if err != nil {
t.Fatal(err)
}
if res.ErrorCount != 0 {
t.Fatalf("scan errors: %#v", res.Errors)
}
if got := countMedia(t, repos); got != 1 {
t.Fatalf("media count = %d, want 1", got)
}
var after model.Media
if err := repos.DB.First(&after, "id = ?", before.ID).Error; err != nil {
t.Fatalf("old media row was not preserved: %v", err)
}
if after.Path != newPath {
t.Fatalf("path = %q, want %q", after.Path, newPath)
}
if after.Title != "地狱乐" || after.OriginalName != "Jigokuraku" || after.Overview != "preserved overview" {
t.Fatalf("scraped identity metadata was lost: %#v", after)
}
if after.PosterURL != "Jigokuraku-poster.jpg" || after.BackdropURL != "Jigokuraku-backdrop.jpg" {
t.Fatalf("artwork was lost: poster=%q backdrop=%q", after.PosterURL, after.BackdropURL)
}
if after.TMDbID != 12345 || after.BangumiID != 67890 || after.ScrapeStatus != "matched" {
t.Fatalf("scrape IDs/status changed: tmdb=%d bgm=%d status=%q", after.TMDbID, after.BangumiID, after.ScrapeStatus)
}
})
}
}
func TestIngestPathAdoptsSoftDeletedSTRMRenameTombstone(t *testing.T) {
sc, repos := newScannerTestEnv(t)
root := t.TempDir()
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
oldPath := filepath.Join(root, "Show.S01E01.mkv.strm")
newPath := filepath.Join(root, "Show.S01E01.strm")
writeFileContent(t, oldPath, "https://example.test/video")
if _, err := sc.IngestPath(t.Context(), lib.ID, oldPath); err != nil {
t.Fatal(err)
}
var before model.Media
if err := repos.DB.First(&before, "path = ?", oldPath).Error; err != nil {
t.Fatal(err)
}
if err := repos.DB.Model(&model.Media{}).Where("id = ?", before.ID).Updates(map[string]any{
"title": "拔作岛",
"overview": "kept through remove-before-create",
"poster_url": "nukitashi-poster.webp",
"tm_db_id": 222,
"scrape_status": "matched",
}).Error; err != nil {
t.Fatal(err)
}
if err := os.Rename(oldPath, newPath); err != nil {
t.Fatal(err)
}
// Simulate fsnotify delivering Remove(old) before Create(new).
if removed, err := sc.RemovePath(t.Context(), oldPath); err != nil || removed != 1 {
t.Fatalf("RemovePath() removed=%d err=%v, want 1", removed, err)
}
var tombstone model.Media
if err := repos.DB.Unscoped().First(&tombstone, "id = ?", before.ID).Error; err != nil {
t.Fatal(err)
}
if !tombstone.DeletedAt.Valid {
t.Fatal("rename tombstone should be soft-deleted")
}
if got := countMedia(t, repos); got != 0 {
t.Fatalf("active media count = %d, want 0 before create event", got)
}
if _, err := sc.IngestPath(t.Context(), lib.ID, newPath); err != nil {
t.Fatal(err)
}
var after model.Media
if err := repos.DB.First(&after, "id = ?", before.ID).Error; err != nil {
t.Fatalf("soft-deleted row was not restored: %v", err)
}
if after.Path != newPath || after.Title != "拔作岛" || after.Overview != "kept through remove-before-create" {
t.Fatalf("rename metadata was not adopted: %#v", after)
}
if after.TMDbID != 222 || after.ScrapeStatus != "matched" {
t.Fatalf("scrape metadata changed: tmdb=%d status=%q", after.TMDbID, after.ScrapeStatus)
}
if got := countMedia(t, repos); got != 1 {
t.Fatalf("media count = %d, want 1", got)
}
}
func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
sc, repos := newScannerTestEnv(t)
root := t.TempDir()
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
path := filepath.Join(root, "Deleted.S01E01.mkv.strm")
writeFileContent(t, path, "https://example.test/video")
if _, err := sc.IngestPath(t.Context(), lib.ID, path); err != nil {
t.Fatal(err)
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if removed, err := sc.RemovePath(t.Context(), path); err != nil || removed != 1 {
t.Fatalf("RemovePath() removed=%d err=%v, want 1", removed, err)
}
var count int64
if err := repos.DB.Unscoped().Model(&model.Media{}).Where("path = ?", path).Count(&count).Error; err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatalf("true deletion left %d tombstone row(s)", count)
}
}
func containsPathAlias(paths []string, name string) bool {
for _, path := range paths {
if strings.EqualFold(filepath.Base(path), filepath.Base(name)) {
return true
}
}
return false
}
+11
View File
@@ -48,6 +48,17 @@ func TestCleanQuery(t *testing.T) {
}
}
func TestCleanQueryIgnoresKeepExtSTRMSuffix(t *testing.T) {
plain := filepath.Join("/media/anime", "Jigokuraku 2023 S01E14.strm")
keepExt := filepath.Join("/media/anime", "Jigokuraku 2023 S01E14.mkv.strm")
plainTitle, plainYear := CleanQuery(plain)
keepTitle, keepYear := CleanQuery(keepExt)
if plainTitle != keepTitle || plainYear != keepYear {
t.Fatalf("keep_ext changed scrape identity: plain=(%q,%d) keep_ext=(%q,%d)",
plainTitle, plainYear, keepTitle, keepYear)
}
}
func TestScrapeQueryCandidatesCleanDirtySeriesFolder(t *testing.T) {
lib := &model.Library{
Path: `F:\media\电视剧\欧美剧`,
+1 -1
View File
@@ -49,7 +49,7 @@ const (
const (
StrmDefaultVideoExt = "mkv,mp4,avi,rmvb,rm,mov,ts,wmv,flv,m4v,iso,mpg,mpeg,webm"
StrmDefaultMetaExt = "nfo,jpg,jpeg,png,srt,ass,ssa,sub,txt,bmp,webp"
StrmDefaultMetaExt = "nfo,jpg,jpeg,png,srt,ass,ssa,sub,txt,bmp,webp,img"
StrmDefaultExclude = "sample,trailer,预告"
)
+6
View File
@@ -612,6 +612,12 @@ func (st *strmSyncState) isVideoExt(ext string, size int64) bool {
}
func (st *strmSyncState) isMetaExt(ext string) bool {
// Legacy MeBox installs may have persisted strm.meta_ext without img.
// .img is emitted by the artwork writer as a fallback image container, so
// keep accepting existing sidecars without requiring a settings migration.
if strings.EqualFold(ext, ".img") {
return true
}
for _, e := range st.cfg.MetaExt {
if "."+e == ext {
return true
+7
View File
@@ -1680,3 +1680,10 @@ func TestHandleVideoKeepExtWritesAllVersions(t *testing.T) {
t.Fatalf("NewStrm=%d want 2", st.rec.NewStrm)
}
}
func TestStrmSyncTreatsLegacyImgAsMetadata(t *testing.T) {
st := &strmSyncState{cfg: &strmPathConfig{MetaExt: csvSplit("nfo,jpg")}}
if !st.isMetaExt(".img") {
t.Fatal("legacy .img sidecars must remain metadata even when an old setting omits img")
}
}