mirror of
https://github.com/truewhile/MeBox.git
synced 2026-09-29 11:36:36 +08:00
Deduplicate organize by multi-source metadata
This commit is contained in:
@@ -91,6 +91,8 @@ type Media struct {
|
||||
ScrapeStatus string `gorm:"size:16;default:pending" json:"scrape_status"`
|
||||
TMDbID int `json:"tmdb_id"`
|
||||
BangumiID int `json:"bangumi_id"`
|
||||
DoubanID string `gorm:"column:douban_id;size:32" json:"douban_id,omitempty"`
|
||||
TheTVDBID string `gorm:"column:thetvdb_id;size:64" json:"thetvdb_id,omitempty"`
|
||||
Languages string `gorm:"size:64" json:"languages,omitempty"` // 逗号分隔的 ISO 639-1 代码,如 "zh,en"
|
||||
Countries string `gorm:"size:128" json:"countries,omitempty"` // 逗号分隔的 ISO 3166-1,如 "CN,US"
|
||||
Genres string `gorm:"size:255" json:"genres,omitempty"` // 逗号分隔的类型名,如 "Action,Animation"
|
||||
@@ -150,6 +152,8 @@ type Series struct {
|
||||
Year int `json:"year"`
|
||||
TMDbID int `json:"tmdb_id"`
|
||||
BangumiID int `json:"bangumi_id"`
|
||||
DoubanID string `gorm:"column:douban_id;size:32" json:"douban_id,omitempty"`
|
||||
TheTVDBID string `gorm:"column:thetvdb_id;size:64" json:"thetvdb_id,omitempty"`
|
||||
}
|
||||
|
||||
// PlaybackHistory 记录当前播放位置以支持续播。
|
||||
|
||||
@@ -385,6 +385,12 @@ func (r *MediaRepository) upsert(ctx context.Context, m *model.Media) error {
|
||||
if m.BangumiID > 0 {
|
||||
setIfChanged(updates, "bangumi_id", existing.BangumiID, m.BangumiID)
|
||||
}
|
||||
if m.DoubanID != "" {
|
||||
setIfChanged(updates, "douban_id", existing.DoubanID, m.DoubanID)
|
||||
}
|
||||
if m.TheTVDBID != "" {
|
||||
setIfChanged(updates, "thetvdb_id", existing.TheTVDBID, m.TheTVDBID)
|
||||
}
|
||||
if m.Languages != "" {
|
||||
setIfChanged(updates, "languages", existing.Languages, m.Languages)
|
||||
}
|
||||
|
||||
@@ -206,6 +206,12 @@ func mergeCloudMetadata(dst, src *LocalMetadata) *LocalMetadata {
|
||||
if src.TMDbID > 0 {
|
||||
dst.TMDbID = src.TMDbID
|
||||
}
|
||||
if src.DoubanID != "" {
|
||||
dst.DoubanID = src.DoubanID
|
||||
}
|
||||
if src.TheTVDBID != "" {
|
||||
dst.TheTVDBID = src.TheTVDBID
|
||||
}
|
||||
if src.SeasonNum > 0 {
|
||||
dst.SeasonNum = src.SeasonNum
|
||||
}
|
||||
|
||||
@@ -115,6 +115,23 @@ func (d *DoubanProvider) Search(ctx context.Context, query string) (*DoubanMatch
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *DoubanProvider) SearchMatch(ctx context.Context, query string) (*Match, error) {
|
||||
got, err := d.Search(ctx, query)
|
||||
if err != nil || got == nil {
|
||||
return nil, err
|
||||
}
|
||||
match := &Match{
|
||||
DoubanID: got.DoubanID,
|
||||
Title: got.Title,
|
||||
PosterURL: got.Img,
|
||||
Rating: got.Rating,
|
||||
}
|
||||
if len(got.Year) >= 4 {
|
||||
_, _ = fmt.Sscanf(got.Year[:4], "%d", &match.Year)
|
||||
}
|
||||
return match, nil
|
||||
}
|
||||
|
||||
func (d *DoubanProvider) setHeaders(req *http.Request) {
|
||||
req.Header.Set("User-Agent", userAgents[secureRandomIntn(len(userAgents))])
|
||||
req.Header.Set("Referer", "https://movie.douban.com/")
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
// Package service — duplicate-file finder.
|
||||
//
|
||||
// DuplicateService computes a sparse-sample SHA-256 (head + middle + tail,
|
||||
// 1 MiB each, plus the file size to break collisions) for every media
|
||||
// file and groups identical hashes into "duplicate sets". The first row
|
||||
// (preferring scraped + larger files) is kept as the primary; the rest
|
||||
// get is_duplicate = true and duplicate_of pointing at the primary.
|
||||
// DuplicateService finds duplicate media by two signals:
|
||||
//
|
||||
// - external identity: same TMDb / Bangumi / Douban / TheTVDB id and, for
|
||||
// episodes, same season+episode;
|
||||
// - sparse file hash: same head + middle + tail SHA-256 and same size.
|
||||
//
|
||||
// The first row (preferring scraped + larger files) is kept as the primary;
|
||||
// the rest get is_duplicate = true and duplicate_of pointing at the primary.
|
||||
//
|
||||
// Why sparse: a full hash on a 50 GB Blu-ray remux takes minutes; the
|
||||
// 3-window 3 MiB sample is enough to differentiate real-world copies
|
||||
@@ -121,34 +124,19 @@ func (d *DuplicateService) Detect(ctx context.Context, libraryID string) (*Repor
|
||||
groups[r.FileHash] = append(groups[r.FileHash], r)
|
||||
}
|
||||
|
||||
markedIDs := map[string]struct{}{}
|
||||
for hash, group := range groups {
|
||||
if len(group) < 2 {
|
||||
continue
|
||||
}
|
||||
primary := pickPrimary(group)
|
||||
dupes := make([]model.Media, 0, len(group)-1)
|
||||
for _, m := range group {
|
||||
if m.ID == primary.ID {
|
||||
continue
|
||||
}
|
||||
dupes = append(dupes, m)
|
||||
if err := d.repo.DB.WithContext(ctx).
|
||||
Model(&model.Media{}).
|
||||
Where("id = ?", m.ID).
|
||||
Updates(map[string]any{
|
||||
"is_duplicate": true,
|
||||
"duplicate_of": primary.ID,
|
||||
}).Error; err != nil {
|
||||
d.log.Warn("dup mark failed", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
rep.ItemsMarked++
|
||||
d.markDuplicateGroup(ctx, rep, hash, group, markedIDs)
|
||||
}
|
||||
|
||||
for key, group := range groupByExternalIdentity(rows) {
|
||||
if len(group) < 2 {
|
||||
continue
|
||||
}
|
||||
rep.Groups = append(rep.Groups, Group{
|
||||
Hash: hash,
|
||||
Primary: primary,
|
||||
Duplicates: dupes,
|
||||
})
|
||||
d.markDuplicateGroup(ctx, rep, key, group, markedIDs)
|
||||
}
|
||||
rep.GroupsFound = len(rep.Groups)
|
||||
if d.hub != nil {
|
||||
@@ -161,6 +149,72 @@ func (d *DuplicateService) Detect(ctx context.Context, libraryID string) (*Repor
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
func (d *DuplicateService) markDuplicateGroup(ctx context.Context, rep *Report, key string, group []model.Media, markedIDs map[string]struct{}) {
|
||||
primary := pickPrimary(group)
|
||||
dupes := make([]model.Media, 0, len(group)-1)
|
||||
for _, m := range group {
|
||||
if m.ID == primary.ID || m.DuplicateOf == primary.ID {
|
||||
continue
|
||||
}
|
||||
if _, ok := markedIDs[m.ID]; ok {
|
||||
continue
|
||||
}
|
||||
dupes = append(dupes, m)
|
||||
if err := d.repo.DB.WithContext(ctx).
|
||||
Model(&model.Media{}).
|
||||
Where("id = ?", m.ID).
|
||||
Updates(map[string]any{
|
||||
"is_duplicate": true,
|
||||
"duplicate_of": primary.ID,
|
||||
}).Error; err != nil {
|
||||
d.log.Warn("dup mark failed", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
markedIDs[m.ID] = struct{}{}
|
||||
rep.ItemsMarked++
|
||||
}
|
||||
if len(dupes) == 0 {
|
||||
return
|
||||
}
|
||||
rep.Groups = append(rep.Groups, Group{
|
||||
Hash: key,
|
||||
Primary: primary,
|
||||
Duplicates: dupes,
|
||||
})
|
||||
}
|
||||
|
||||
func groupByExternalIdentity(rows []model.Media) map[string][]model.Media {
|
||||
groups := map[string][]model.Media{}
|
||||
for _, row := range rows {
|
||||
key := mediaExternalIdentityKey(row)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
groups[key] = append(groups[key], row)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func mediaExternalIdentityKey(row model.Media) string {
|
||||
var key string
|
||||
switch {
|
||||
case row.TMDbID > 0:
|
||||
key = fmt.Sprintf("tmdb:%d", row.TMDbID)
|
||||
case row.BangumiID > 0:
|
||||
key = fmt.Sprintf("bangumi:%d", row.BangumiID)
|
||||
case row.DoubanID != "":
|
||||
key = "douban:" + row.DoubanID
|
||||
case row.TheTVDBID != "":
|
||||
key = "thetvdb:" + row.TheTVDBID
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
if row.SeasonNum > 0 || row.EpisodeNum > 0 {
|
||||
key += fmt.Sprintf(":s%d:e%d", row.SeasonNum, row.EpisodeNum)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// Current returns duplicate groups already marked in the database. It keeps
|
||||
// the UI useful after a prior scan and avoids requiring POST on page load.
|
||||
func (d *DuplicateService) Current(ctx context.Context, libraryID string) (*Report, error) {
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
)
|
||||
|
||||
func TestDuplicateDetectMarksExternalIdentityDuplicates(t *testing.T) {
|
||||
repos := newOrganizerTestRepo(t)
|
||||
root := t.TempDir()
|
||||
firstPath := filepath.Join(root, "show-a.mkv")
|
||||
secondPath := filepath.Join(root, "show-b.mkv")
|
||||
writeOrgFile(t, firstPath, "first-release")
|
||||
writeOrgFile(t, secondPath, "second-release")
|
||||
|
||||
lib := model.Library{Name: "剧集", Path: root, Type: "tv", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := model.Media{
|
||||
LibraryID: lib.ID,
|
||||
Title: "间谍过家家",
|
||||
Path: firstPath,
|
||||
SizeBytes: 13,
|
||||
SeasonNum: 1,
|
||||
EpisodeNum: 1,
|
||||
TMDbID: 12345,
|
||||
ScrapeStatus: "matched",
|
||||
}
|
||||
second := model.Media{
|
||||
LibraryID: lib.ID,
|
||||
Title: "Spy Family",
|
||||
Path: secondPath,
|
||||
SizeBytes: 14,
|
||||
SeasonNum: 1,
|
||||
EpisodeNum: 1,
|
||||
TMDbID: 12345,
|
||||
ScrapeStatus: "matched",
|
||||
}
|
||||
if err := repos.DB.Create(&first).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.DB.Create(&second).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
report, err := NewDuplicateService(zap.NewNop(), repos, nil).Detect(t.Context(), lib.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ItemsMarked != 1 || report.GroupsFound != 1 {
|
||||
t.Fatalf("report = %#v, want one external identity duplicate", report)
|
||||
}
|
||||
var rows []model.Media
|
||||
if err := repos.DB.Find(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
marked := 0
|
||||
for _, row := range rows {
|
||||
if row.IsDuplicate && row.DuplicateOf != "" {
|
||||
marked++
|
||||
}
|
||||
}
|
||||
if marked != 1 {
|
||||
t.Fatalf("marked duplicate rows = %d, want 1; rows=%#v", marked, rows)
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,8 @@ type LocalMetadata struct {
|
||||
PosterURL string
|
||||
BackdropURL string
|
||||
TMDbID int
|
||||
DoubanID string
|
||||
TheTVDBID string
|
||||
SeasonNum int
|
||||
EpisodeNum int
|
||||
Genres string
|
||||
@@ -257,6 +259,8 @@ func metadataFromDoc(doc *nfoDocument, baseDir string, seriesLike bool) *LocalMe
|
||||
PosterURL: firstRemoteURL(baseDir, nfoPosterValues(doc)...),
|
||||
BackdropURL: firstRemoteURL(baseDir, nfoBackdropValues(doc)...),
|
||||
TMDbID: doc.TMDbID,
|
||||
DoubanID: externalIDFromUniqueIDs(doc.UniqueIDs, "douban"),
|
||||
TheTVDBID: externalIDFromUniqueIDs(doc.UniqueIDs, "thetvdb", "tvdb"),
|
||||
SeasonNum: doc.Season,
|
||||
EpisodeNum: doc.Episode,
|
||||
Genres: joinNFOValues(adultAwareGenres(doc)),
|
||||
@@ -391,6 +395,12 @@ func mergeEpisodeMetadata(dst, episode *LocalMetadata, doc *nfoDocument) {
|
||||
if episode.TMDbID > 0 {
|
||||
dst.TMDbID = episode.TMDbID
|
||||
}
|
||||
if episode.DoubanID != "" {
|
||||
dst.DoubanID = episode.DoubanID
|
||||
}
|
||||
if episode.TheTVDBID != "" {
|
||||
dst.TheTVDBID = episode.TheTVDBID
|
||||
}
|
||||
if episode.SeasonNum > 0 {
|
||||
dst.SeasonNum = episode.SeasonNum
|
||||
}
|
||||
@@ -409,13 +419,24 @@ func mergeEpisodeMetadata(dst, episode *LocalMetadata, doc *nfoDocument) {
|
||||
}
|
||||
|
||||
func tmdbIDFromUniqueIDs(ids []nfoUniqueID) int {
|
||||
value := externalIDFromUniqueIDs(ids, "tmdb")
|
||||
if value == "" {
|
||||
return 0
|
||||
}
|
||||
v, _ := strconv.Atoi(value)
|
||||
return v
|
||||
}
|
||||
|
||||
func externalIDFromUniqueIDs(ids []nfoUniqueID, types ...string) string {
|
||||
for _, id := range ids {
|
||||
if strings.EqualFold(strings.TrimSpace(id.Type), "tmdb") {
|
||||
v, _ := strconv.Atoi(strings.TrimSpace(id.Value))
|
||||
return v
|
||||
idType := strings.TrimSpace(id.Type)
|
||||
for _, typ := range types {
|
||||
if strings.EqualFold(idType, typ) {
|
||||
return strings.TrimSpace(id.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstRemoteURL(baseDir string, values ...string) string {
|
||||
|
||||
@@ -264,7 +264,9 @@ func (o *OrganizerService) organizeSourceFile(ctx context.Context, src, sourceRo
|
||||
if layout.MediaType == "" {
|
||||
layout.MediaType = o.inferMediaTypeForSourceFile(src, title, season, episode)
|
||||
}
|
||||
var metadataMatch *Match
|
||||
if match := o.lookupOrganizeMetadata(ctx, src, sourceRoot, layout.MediaType, title, year, season, episode, metadataCache); match != nil {
|
||||
metadataMatch = match
|
||||
if matchedTitle := sanitizeFilename(strings.TrimSpace(match.Title)); matchedTitle != "" {
|
||||
title = matchedTitle
|
||||
parsedTitle = strings.TrimSpace(match.Title)
|
||||
@@ -321,9 +323,10 @@ func (o *OrganizerService) organizeSourceFile(ctx context.Context, src, sourceRo
|
||||
|
||||
// 去重候选:合并「目的地媒体库已扫描入库的同一媒体(按标题/年份/季集匹配,
|
||||
// 不受目录大小写或布局影响)」与「目标文件夹内已存在的同名视频文件」。
|
||||
externalExisting := o.existingByExternalIdentity(ctx, destRoot, metadataMatch, season, episode)
|
||||
identityExisting := o.existingByIdentity(ctx, destRoot, parsedTitle, year, season, episode)
|
||||
folderExisting := o.existingByFolder(destDir, episodeTag)
|
||||
existing := mergeExistingVersionPaths(identityExisting, folderExisting)
|
||||
existing := mergeExistingVersionPaths(externalExisting, identityExisting, folderExisting)
|
||||
if len(existing) > 0 {
|
||||
srcArea := o.resolutionArea(ctx, src)
|
||||
bestArea := 0
|
||||
@@ -357,7 +360,7 @@ func (o *OrganizerService) organizeSourceFile(ctx context.Context, src, sourceRo
|
||||
}
|
||||
// 去重:目的地已存在同一媒体且不低于来源分辨率,跳过不再整理过去。
|
||||
reason := organizeSkipTargetExists
|
||||
if len(identityExisting) > 0 || o.allExistingPathsInDB(ctx, existing) {
|
||||
if len(externalExisting) > 0 || len(identityExisting) > 0 || o.allExistingPathsInDB(ctx, existing) {
|
||||
reason = organizeSkipDuplicateLibrary
|
||||
}
|
||||
o.log.Debug("organize skip duplicate",
|
||||
@@ -501,7 +504,9 @@ func (o *OrganizerService) lookupOrganizeMetadata(ctx context.Context, src, sour
|
||||
zap.String("title", match.Title),
|
||||
zap.Int("year", match.Year),
|
||||
zap.Int("tmdb_id", match.TMDbID),
|
||||
zap.Int("bangumi_id", match.BangumiID))
|
||||
zap.Int("bangumi_id", match.BangumiID),
|
||||
zap.String("douban_id", match.DoubanID),
|
||||
zap.String("thetvdb_id", match.TheTVDBID))
|
||||
}
|
||||
return match
|
||||
}
|
||||
@@ -522,6 +527,8 @@ func organizeMatchFromLocalMetadata(local *LocalMetadata) *Match {
|
||||
Year: local.Year,
|
||||
Rating: local.Rating,
|
||||
TMDbID: local.TMDbID,
|
||||
DoubanID: local.DoubanID,
|
||||
TheTVDBID: local.TheTVDBID,
|
||||
NSFW: local.NSFW,
|
||||
}
|
||||
if local.Genres != "" {
|
||||
@@ -980,6 +987,50 @@ func (o *OrganizerService) existingByIdentity(ctx context.Context, destRoot, tit
|
||||
return out
|
||||
}
|
||||
|
||||
func (o *OrganizerService) existingByExternalIdentity(ctx context.Context, destRoot string, match *Match, season, episode int) []string {
|
||||
if o.repo == nil || o.repo.DB == nil || match == nil {
|
||||
return nil
|
||||
}
|
||||
var conds []string
|
||||
var args []any
|
||||
if match.TMDbID > 0 {
|
||||
conds = append(conds, "tm_db_id = ?")
|
||||
args = append(args, match.TMDbID)
|
||||
}
|
||||
if match.BangumiID > 0 {
|
||||
conds = append(conds, "bangumi_id = ?")
|
||||
args = append(args, match.BangumiID)
|
||||
}
|
||||
if strings.TrimSpace(match.DoubanID) != "" {
|
||||
conds = append(conds, "douban_id = ?")
|
||||
args = append(args, strings.TrimSpace(match.DoubanID))
|
||||
}
|
||||
if strings.TrimSpace(match.TheTVDBID) != "" {
|
||||
conds = append(conds, "thetvdb_id = ?")
|
||||
args = append(args, strings.TrimSpace(match.TheTVDBID))
|
||||
}
|
||||
if len(conds) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := o.repo.DB.WithContext(ctx).Model(&model.Media{}).
|
||||
Where("deleted_at IS NULL").
|
||||
Where("("+strings.Join(conds, " OR ")+")", args...)
|
||||
if season > 0 || episode > 0 {
|
||||
q = q.Where("season_num = ? AND episode_num = ?", season, episode)
|
||||
}
|
||||
var rows []model.Media
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, row := range rows {
|
||||
if row.Path != "" && pathWithin(row.Path, destRoot) {
|
||||
out = append(out, row.Path)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// existingByFolder returns video files already present in destDir that
|
||||
// represent the same media. For an episode (episodeTag != "") it matches files
|
||||
// carrying the same SxxExx tag; for a movie it matches every video file in the
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -102,6 +105,107 @@ func TestOrganizeDirectoryUsesScraperMatchBeforeRename(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrganizeDirectoryDedupsByExternalIDBeforeRename(t *testing.T) {
|
||||
scraper, repos, closeServer := newTestScraper(t)
|
||||
defer closeServer()
|
||||
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "downloads")
|
||||
dest := filepath.Join(root, "media")
|
||||
sourceFile := filepath.Join(src, "Spy.x.Family.S01E01.2022.2160p.mkv")
|
||||
writeOrgFile(t, sourceFile, "episode")
|
||||
|
||||
existingPath := filepath.Join(dest, "电视剧", "旧错误名", "Season 01", "旧错误名 - S01E01.mkv")
|
||||
writeOrgFile(t, existingPath, "existing")
|
||||
lib := model.Library{Name: "剧集", Path: filepath.Join(dest, "电视剧"), Type: "tv", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.DB.Create(&model.Media{
|
||||
LibraryID: lib.ID,
|
||||
Title: "旧错误名",
|
||||
Path: existingPath,
|
||||
SeasonNum: 1,
|
||||
EpisodeNum: 1,
|
||||
TMDbID: 12345,
|
||||
ScrapeStatus: "matched",
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
organizer := NewOrganizerService(&config.Config{}, zap.NewNop(), repos)
|
||||
organizer.SetScraper(scraper)
|
||||
res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{
|
||||
SourcePath: src,
|
||||
DestPath: dest,
|
||||
TransferMode: TransferCopy,
|
||||
MediaType: "tv",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("organize directory: %v", err)
|
||||
}
|
||||
if res.Organized != 0 || res.Skipped != 1 {
|
||||
t.Fatalf("organize result = organized %d skipped %d, want 0/1; items=%#v errors=%#v", res.Organized, res.Skipped, res.Items, res.Errors)
|
||||
}
|
||||
if len(res.Items) != 1 || res.Items[0].Reason != organizeSkipDuplicateLibrary {
|
||||
t.Fatalf("source should be skipped as external-id duplicate: %#v", res.Items)
|
||||
}
|
||||
if _, err := os.Stat(sourceFile); err != nil {
|
||||
t.Fatalf("duplicate source should remain untouched: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrganizeDirectoryUsesBangumiForAnimeRename(t *testing.T) {
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/search/subject/frieren" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"results": 1,
|
||||
"list": []map[string]any{{
|
||||
"id": 889,
|
||||
"name": "Frieren",
|
||||
"name_cn": "葬送的芙莉莲",
|
||||
"air_date": "2023-09-29",
|
||||
}},
|
||||
})
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
repos := newOrganizerTestRepo(t)
|
||||
cfg := &config.Config{}
|
||||
bangumi := NewBangumiProvider(cfg, zap.NewNop())
|
||||
bangumi.base = upstream.URL
|
||||
scraper := NewScraperService(cfg, zap.NewNop(), repos, nil, bangumi, nil, nil, NewHub(zap.NewNop()))
|
||||
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "downloads")
|
||||
dest := filepath.Join(root, "media")
|
||||
sourceFile := filepath.Join(src, "Frieren.S01E01.1080p.mkv")
|
||||
writeOrgFile(t, sourceFile, "episode")
|
||||
|
||||
organizer := NewOrganizerService(cfg, zap.NewNop(), repos)
|
||||
organizer.SetScraper(scraper)
|
||||
res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{
|
||||
SourcePath: src,
|
||||
DestPath: dest,
|
||||
TransferMode: TransferCopy,
|
||||
MediaType: "anime",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("organize directory: %v", err)
|
||||
}
|
||||
want := filepath.Join(dest, "电视剧", "葬送的芙莉莲", "Season 01", "葬送的芙莉莲 - S01E01.mkv")
|
||||
if res.Organized != 1 {
|
||||
t.Fatalf("organized = %d, want 1; items=%#v errors=%#v", res.Organized, res.Items, res.Errors)
|
||||
}
|
||||
if _, err := os.Stat(want); err != nil {
|
||||
t.Fatalf("organized file should use Bangumi metadata path %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrganizeScanAndScrapeRetriesNoMatchRows(t *testing.T) {
|
||||
scraper, repos, closeServer := newTestScraper(t)
|
||||
defer closeServer()
|
||||
|
||||
@@ -1734,6 +1734,12 @@ func applyLocalMetadata(m *model.Media, local *LocalMetadata) {
|
||||
if local.TMDbID > 0 {
|
||||
m.TMDbID = local.TMDbID
|
||||
}
|
||||
if local.DoubanID != "" {
|
||||
m.DoubanID = local.DoubanID
|
||||
}
|
||||
if local.TheTVDBID != "" {
|
||||
m.TheTVDBID = local.TheTVDBID
|
||||
}
|
||||
if local.SeasonNum > 0 {
|
||||
m.SeasonNum = local.SeasonNum
|
||||
}
|
||||
@@ -1768,6 +1774,8 @@ func localHasDescriptiveMetadata(local *LocalMetadata) bool {
|
||||
local.Overview != "" ||
|
||||
local.Rating > 0 ||
|
||||
local.TMDbID > 0 ||
|
||||
local.DoubanID != "" ||
|
||||
local.TheTVDBID != "" ||
|
||||
local.Genres != "" ||
|
||||
local.Countries != "" ||
|
||||
local.Languages != ""
|
||||
|
||||
+42
-13
@@ -34,6 +34,7 @@ type ScraperService struct {
|
||||
tmdb *TMDbProvider
|
||||
bangumi *BangumiProvider
|
||||
thetvdb *TheTVDBProvider
|
||||
douban *DoubanProvider
|
||||
fanart *FanartProvider
|
||||
adult *AdultProvider
|
||||
hub *Hub
|
||||
@@ -61,6 +62,10 @@ func NewScraperService(
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ScraperService) SetDouban(douban *DoubanProvider) {
|
||||
s.douban = douban
|
||||
}
|
||||
|
||||
// yearPattern extracts a 4-digit year (1900-2099).
|
||||
var yearPattern = regexp.MustCompile(`(?:^|[^\d])(19\d{2}|20\d{2})(?:[^\d]|$)`)
|
||||
|
||||
@@ -309,6 +314,12 @@ func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media,
|
||||
if match.BangumiID > 0 {
|
||||
updates["bangumi_id"] = match.BangumiID
|
||||
}
|
||||
if match.DoubanID != "" {
|
||||
updates["douban_id"] = match.DoubanID
|
||||
}
|
||||
if match.TheTVDBID != "" {
|
||||
updates["thetvdb_id"] = match.TheTVDBID
|
||||
}
|
||||
if match.NSFW {
|
||||
updates["nsfw"] = true
|
||||
}
|
||||
@@ -368,6 +379,8 @@ func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media,
|
||||
"title": match.Title,
|
||||
"tmdb_id": match.TMDbID,
|
||||
"bangumi_id": match.BangumiID,
|
||||
"douban_id": match.DoubanID,
|
||||
"thetvdb_id": match.TheTVDBID,
|
||||
"source": map[bool]string{true: "adult"}[match.NSFW],
|
||||
})
|
||||
return nil
|
||||
@@ -408,6 +421,12 @@ func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.M
|
||||
if next.BangumiID > 0 {
|
||||
updates["bangumi_id"] = next.BangumiID
|
||||
}
|
||||
if next.DoubanID != "" {
|
||||
updates["douban_id"] = next.DoubanID
|
||||
}
|
||||
if next.TheTVDBID != "" {
|
||||
updates["thetvdb_id"] = next.TheTVDBID
|
||||
}
|
||||
if next.SeasonNum > 0 {
|
||||
updates["season_num"] = next.SeasonNum
|
||||
}
|
||||
@@ -431,10 +450,11 @@ func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.M
|
||||
return err
|
||||
}
|
||||
s.hub.Publish("scrape", map[string]any{
|
||||
"media_id": m.ID,
|
||||
"title": next.Title,
|
||||
"tmdb_id": next.TMDbID,
|
||||
"source": "local_nfo",
|
||||
"media_id": m.ID,
|
||||
"title": next.Title,
|
||||
"tmdb_id": next.TMDbID,
|
||||
"douban_id": next.DoubanID,
|
||||
"source": "local_nfo",
|
||||
})
|
||||
return nil
|
||||
}
|
||||
@@ -558,7 +578,7 @@ func librarySupportsSeasons(lib *model.Library) bool {
|
||||
// 库类型决定首选 provider:
|
||||
//
|
||||
// anime -> Bangumi -> TMDb /search/tv -> TMDb /search/movie
|
||||
// tv -> TheTVDB -> TMDb /search/tv -> TMDb /search/movie
|
||||
// tv -> TMDb /search/tv -> TMDb /search/movie -> TheTVDB
|
||||
// movie -> TMDb /search/movie
|
||||
// (空) -> TMDb /search/movie
|
||||
//
|
||||
@@ -578,14 +598,6 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, query s
|
||||
s.log.Debug("bangumi search failed", zap.String("query", query), zap.Error(err))
|
||||
}
|
||||
}
|
||||
case "tv", "variety", "show", "shows":
|
||||
if s.thetvdb != nil && s.thetvdb.Enabled() {
|
||||
if m, err := s.thetvdb.SearchSeries(ctx, query); err == nil && m != nil {
|
||||
return m
|
||||
} else if err != nil {
|
||||
s.log.Debug("thetvdb search failed", zap.String("query", query), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.tmdb != nil && s.tmdb.Enabled() {
|
||||
// anime / tv 先用 TMDb /search/tv(剧名通常是 TV 类目)。
|
||||
@@ -602,6 +614,20 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, query s
|
||||
s.log.Debug("tmdb movie search failed", zap.String("query", query), zap.Error(err))
|
||||
}
|
||||
}
|
||||
if (kind == "anime" || kind == "tv" || kind == "variety" || kind == "show" || kind == "shows") && s.thetvdb != nil && s.thetvdb.Enabled() {
|
||||
if m, err := s.thetvdb.SearchSeries(ctx, query); err == nil && m != nil {
|
||||
return m
|
||||
} else if err != nil {
|
||||
s.log.Debug("thetvdb search failed", zap.String("query", query), zap.Error(err))
|
||||
}
|
||||
}
|
||||
if s.douban != nil && s.douban.Enabled() {
|
||||
if m, err := s.douban.SearchMatch(ctx, query); err == nil && m != nil {
|
||||
return m
|
||||
} else if err != nil {
|
||||
s.log.Debug("douban search failed", zap.String("query", query), zap.Error(err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -724,6 +750,9 @@ func (s *ScraperService) AnyEnabled() bool {
|
||||
if s.adult != nil && s.adult.Enabled() {
|
||||
return true
|
||||
}
|
||||
if s.douban != nil && s.douban.Enabled() {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -96,9 +96,11 @@ func New(cfg *config.Config, log *zap.Logger, repos *repository.Container) *Cont
|
||||
tmdb := NewTMDbProvider(cfg, log, apiConfig)
|
||||
bangumi := NewBangumiProvider(cfg, log)
|
||||
thetvdb := NewTheTVDBProvider(cfg, log)
|
||||
douban := NewDoubanProvider(cfg, log)
|
||||
fanart := NewFanartProvider(cfg, log)
|
||||
adult := NewAdultProvider(log, apiConfig)
|
||||
scraper := NewScraperService(cfg, log, repos, tmdb, bangumi, thetvdb, fanart, hub, adult)
|
||||
scraper.SetDouban(douban)
|
||||
organizer := NewOrganizerService(cfg, log, repos)
|
||||
organizer.SetProbe(probe)
|
||||
organizer.SetScraper(scraper)
|
||||
@@ -124,7 +126,6 @@ func New(cfg *config.Config, log *zap.Logger, repos *repository.Container) *Cont
|
||||
emby.SetCloudProbe(storageCfg, probe)
|
||||
downloadClients := NewDownloadClientService(log, repos)
|
||||
assistant := NewAssistantService(log, repos, ai)
|
||||
douban := NewDoubanProvider(cfg, log)
|
||||
scheduler := NewSchedulerService(log, repos, scanner, transcoder, organizer, storageCfg, hub, cfg.Cache.CacheDir)
|
||||
scheduler.SetTaskTracker(tasks)
|
||||
|
||||
|
||||
@@ -128,6 +128,7 @@ func (t *TheTVDBProvider) SearchSeries(ctx context.Context, query string) (*Matc
|
||||
}
|
||||
r := p.Data[0]
|
||||
m := &Match{
|
||||
TheTVDBID: r.ID,
|
||||
Title: r.Name,
|
||||
Overview: r.Overview,
|
||||
PosterURL: r.Image,
|
||||
|
||||
@@ -126,6 +126,8 @@ func (t *TMDbProvider) resolveBaseURL(ctx context.Context) string {
|
||||
type Match struct {
|
||||
TMDbID int `json:"tmdb_id"`
|
||||
BangumiID int `json:"bangumi_id"`
|
||||
DoubanID string `json:"douban_id,omitempty"`
|
||||
TheTVDBID string `json:"thetvdb_id,omitempty"`
|
||||
Title string `json:"title"`
|
||||
OriginalName string `json:"original_name,omitempty"`
|
||||
Overview string `json:"overview"`
|
||||
|
||||
Reference in New Issue
Block a user