fix: correct organize media type classification

This commit is contained in:
ShukeBta
2026-06-25 11:50:02 +08:00
parent 16c3ed239a
commit e16fecb3ec
15 changed files with 308 additions and 16 deletions
+1
View File
@@ -228,6 +228,7 @@ func (p *AdultProvider) fetchText(ctx context.Context, targetURL, referer string
func parseAdultDetailHTML(body, code, source, detailURL string) *Match {
match := &Match{
OriginalName: code,
MediaType: "adult",
NSFW: true,
Genres: []string{"Adult", source},
}
+2
View File
@@ -94,6 +94,7 @@ func (b *BangumiProvider) Search(ctx context.Context, query string) (*Match, err
}
m := &Match{
BangumiID: r.ID,
MediaType: "anime",
Title: title,
OriginalName: r.Name,
Overview: r.Summary,
@@ -139,6 +140,7 @@ func (b *BangumiProvider) GetSubject(ctx context.Context, bangumiID int) (*Match
}
m := &Match{
BangumiID: r.ID,
MediaType: "anime",
Title: title,
OriginalName: r.Name,
Overview: r.Summary,
+5
View File
@@ -120,8 +120,13 @@ func (d *DoubanProvider) SearchMatch(ctx context.Context, query string) (*Match,
if err != nil || got == nil {
return nil, err
}
mediaType := ""
if strings.TrimSpace(got.Type) != "" {
mediaType = normalizeMediaType(got.Type, got.Title, "")
}
match := &Match{
DoubanID: got.DoubanID,
MediaType: mediaType,
Title: got.Title,
PosterURL: got.Img,
Rating: got.Rating,
+3
View File
@@ -450,6 +450,9 @@ func mergeManualRequestIntoMatch(match *Match, req ManualScrapeRequest) *Match {
if req.Title != "" {
match.Title = req.Title
}
if mediaType := normalizeOrganizeMediaType(req.MediaType); mediaType != "" {
match.MediaType = mediaType
}
if req.OriginalName != "" {
match.OriginalName = req.OriginalName
}
+9 -6
View File
@@ -15,6 +15,9 @@ var (
classifierEpisodeRE = regexp.MustCompile(`(?i)\bS\d{1,2}E\d{1,3}\b|第\s*\d+\s*[集期]|(?:^|[\s._-])E\d{1,3}(?:[\s._-]|$)`)
classifierSeasonRE = regexp.MustCompile(`(?i)\bS\d{1,2}\b|第\s*\d+\s*季`)
classifierJAVCodeRE = regexp.MustCompile(`(?:^|[\s._\-/\[\]()])[A-Z]{2,6}[-_]?\d{3,5}(?:[\s._\-/\[\]()]|$)`)
classifierMovieRE = regexp.MustCompile(`(?i)(?:^|[^a-z0-9])(?:movies?|films?)(?:[^a-z0-9]|$)`)
classifierTVRE = regexp.MustCompile(`(?i)(?:^|[^a-z0-9])(?:tv|series|shows?|dramas?)(?:[^a-z0-9]|$)`)
classifierAnimeRE = regexp.MustCompile(`(?i)(?:^|[^a-z0-9])(?:anime|bangumi)(?:[^a-z0-9]|$)`)
)
const DownloadSmartClassifySettingKey = "downloads.smart_classify"
@@ -166,11 +169,11 @@ func normalizeMediaType(mediaType, title, category string) string {
return "adult"
case containsAnyText(raw, "综艺", "真人秀"):
return "variety"
case containsAnyText(raw, "国漫", "日漫", "日番", "动漫", "动画", "anime", "bangumi") && !containsAnyText(raw, "动画电影"):
case (containsAnyText(raw, "国漫", "日漫", "日番", "动漫", "动画") || classifierAnimeRE.MatchString(raw)) && !containsAnyText(raw, "动画电影"):
return "anime"
case containsAnyText(raw, "电视剧", "国产剧", "欧美剧", "日韩剧", "日剧", "韩剧", "剧集", "tv", "series"):
case containsAnyText(raw, "电视剧", "国产剧", "欧美剧", "日韩剧", "日剧", "韩剧", "剧集") || classifierTVRE.MatchString(raw):
return "tv"
case containsAnyText(raw, "电影", "movie", "film"):
case containsAnyText(raw, "电影") || classifierMovieRE.MatchString(raw):
return "movie"
}
text := strings.ToLower(title + " " + category)
@@ -179,13 +182,13 @@ func normalizeMediaType(mediaType, title, category string) string {
return "adult"
case containsAnyText(text, "综艺", "真人秀", "脱口秀", "晚会", "春晚", "gala", "festival gala", "reality", "talk show"):
return "variety"
case strings.Contains(text, "movie") || strings.Contains(text, "电影"):
case strings.Contains(text, "电影") || classifierMovieRE.MatchString(text):
return "movie"
case strings.Contains(text, "anime") || strings.Contains(text, "bangumi") || strings.Contains(text, "动漫") || strings.Contains(text, "动画"):
case classifierAnimeRE.MatchString(text) || strings.Contains(text, "动漫") || strings.Contains(text, "动画"):
return "anime"
case strings.Contains(text, "variety") || strings.Contains(text, "综艺") || strings.Contains(text, "真人秀"):
return "variety"
case classifierEpisodeRE.MatchString(text) || classifierSeasonRE.MatchString(text) || strings.Contains(text, "tv") || strings.Contains(text, "剧集") || strings.Contains(text, "电视剧"):
case classifierEpisodeRE.MatchString(text) || classifierSeasonRE.MatchString(text) || classifierTVRE.MatchString(text) || strings.Contains(text, "剧集") || strings.Contains(text, "电视剧"):
return "tv"
default:
return "movie"
+19
View File
@@ -281,6 +281,25 @@ func TestNormalizeMediaTypeAcceptsChineseLibraryTypes(t *testing.T) {
}
}
func TestNormalizeMediaTypeDoesNotTreatReleaseTokensAsTV(t *testing.T) {
tests := []string{
"They Will Kill You 2026 1080p HDTV x264",
"Some Movie 2026 2160p AppleTV WEB-DL",
"Some Movie 2026 2160p ATVP WEB-DL",
}
for _, input := range tests {
t.Run(input, func(t *testing.T) {
if got := normalizeMediaType("", input, ""); got != "movie" {
t.Fatalf("normalizeMediaType(%q) = %q, want movie", input, got)
}
})
}
if got := normalizeMediaType("", "The Last of Us", `F:\media\tv\The Last of Us`); got != "tv" {
t.Fatalf("standalone tv path token = %q, want tv", got)
}
}
func TestSubscriptionResolveClassifiedSavePath(t *testing.T) {
db := newServiceTestDB(t, &model.Setting{})
repos := repository.New(db)
@@ -12,6 +12,7 @@ import (
"go.uber.org/zap"
"github.com/ShukeBta/MediaStationGo/internal/config"
"github.com/ShukeBta/MediaStationGo/internal/model"
)
func TestOrganizeDirectoryClassifiesScraperMatchBeforeRename(t *testing.T) {
@@ -130,6 +131,174 @@ func TestOrganizeDirectoryMetadataCategoryOverridesDownloadFolder(t *testing.T)
}
}
func TestOrganizeDirectoryMovieMetadataOverridesWrongTVFolder(t *testing.T) {
var paths []string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
paths = append(paths, r.URL.Path)
switch {
case r.URL.Path == "/search/movie":
_ = json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{{
"id": 1292695,
"title": "杀的就是你",
"original_title": "They Will Kill You",
"original_language": "en",
"genre_ids": []int{27, 53},
"release_date": "2026-01-16",
"vote_average": 6.3,
}},
})
case r.URL.Path == "/search/tv":
_ = json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{{
"id": 1198994,
"name": "请求救援",
"original_name": "They Will Kill You",
"original_language": "en",
"origin_country": []string{"US"},
"genre_ids": []int{18},
"first_air_date": "2026-01-01",
}},
})
default:
http.NotFound(w, r)
}
}))
defer upstream.Close()
repos := newOrganizerTestRepo(t)
cfg := &config.Config{}
cfg.Organizer.SmartClassify = true
cfg.Secrets.TMDbAPIKey = "test-key"
cfg.Secrets.TMDbAPIProxy = upstream.URL
scraper := NewScraperService(cfg, zap.NewNop(), repos, NewTMDbProvider(cfg, zap.NewNop(), nil), nil, nil, nil, NewHub(zap.NewNop()))
root := t.TempDir()
srcRoot := filepath.Join(root, "downloads")
dest := filepath.Join(root, "media")
sourceFile := filepath.Join(srcRoot, "欧美剧", "They.Will.Kill.You.2026.1080p.HDTV.x264-HiDt.mkv")
writeOrgFile(t, sourceFile, "movie")
organizer := NewOrganizerService(cfg, zap.NewNop(), repos)
organizer.SetScraper(scraper)
res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{
SourcePath: srcRoot,
DestPath: dest,
TransferMode: TransferCopy,
})
if err != nil {
t.Fatalf("organize directory: %v", err)
}
want := filepath.Join(dest, "电影", "外语电影", "杀的就是你 (2026)", "杀的就是你 (2026).mkv")
if res.Organized != 1 || res.Reclassified != 0 {
t.Fatalf("result = %+v, want organized movie only; paths=%v", res, paths)
}
if _, err := os.Stat(want); err != nil {
t.Fatalf("movie in wrong TV folder should organize as movie at %q: %v; items=%#v paths=%v", want, err, res.Items, paths)
}
if len(paths) == 0 || paths[0] != "/search/movie" {
t.Fatalf("first metadata search path = %q, want /search/movie; all=%v", firstQuery(paths), paths)
}
if len(res.Items) != 1 || res.Items[0].MediaType != "movie" || res.Items[0].Category != "外语电影" {
t.Fatalf("organize item = %#v, want movie/外语电影", res.Items)
}
}
func TestOrganizeDirectoryReclassifiesMovieFromDirtyGeneratedEpisodePath(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.URL.Path == "/search/movie":
_ = json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{{
"id": 1198994,
"title": "请求救援",
"original_title": "Request Rescue",
"original_language": "en",
"genre_ids": []int{28, 53},
"release_date": "2026-02-01",
}},
})
case r.URL.Path == "/search/tv":
_ = json.NewEncoder(w).Encode(map[string]any{"results": []any{}})
default:
http.NotFound(w, r)
}
}))
defer upstream.Close()
repos := newOrganizerTestRepo(t)
cfg := &config.Config{}
cfg.Organizer.SmartClassify = true
cfg.Secrets.TMDbAPIKey = "test-key"
cfg.Secrets.TMDbAPIProxy = upstream.URL
scraper := NewScraperService(cfg, zap.NewNop(), repos, NewTMDbProvider(cfg, zap.NewNop(), nil), nil, nil, nil, NewHub(zap.NewNop()))
root := t.TempDir()
dest := filepath.Join(root, "media")
euusLib := model.Library{Name: "欧美剧", Path: filepath.Join(dest, "电视剧", "欧美剧"), Type: "tv", Enabled: true}
foreignMovieLib := model.Library{Name: "外语电影", Path: filepath.Join(dest, "电影", "外语电影"), Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &euusLib); err != nil {
t.Fatal(err)
}
if err := repos.Library.Create(t.Context(), &foreignMovieLib); err != nil {
t.Fatal(err)
}
wrongPath := filepath.Join(euusLib.Path, "请求救援 (2026)", "Season 1", "请求救援 - S01E202-1080p - 第 202 集.mkv")
writeOrgFile(t, wrongPath, "movie")
if err := repos.DB.Create(&model.Media{
LibraryID: euusLib.ID,
Title: "请求救援",
OriginalName: "请求救援",
Path: wrongPath,
SeasonNum: 1,
EpisodeNum: 202,
TMDbID: 1198994,
Year: 2026,
PosterURL: "https://image.tmdb.org/t/p/w500/poster.jpg",
BackdropURL: "https://image.tmdb.org/t/p/w1280/backdrop.jpg",
Overview: "movie overview",
ScrapeStatus: "matched",
Container: "mkv",
SizeBytes: int64(len("movie")),
DurationSec: 7200,
VideoCodec: "h264",
AudioCodec: "aac",
}).Error; err != nil {
t.Fatal(err)
}
organizer := NewOrganizerService(cfg, zap.NewNop(), repos)
organizer.SetScraper(scraper)
res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{
SourcePath: filepath.Dir(filepath.Dir(wrongPath)),
DestPath: euusLib.Path,
TransferMode: TransferCopy,
})
if err != nil {
t.Fatalf("organize dirty generated path: %v", err)
}
want := filepath.Join(foreignMovieLib.Path, "请求救援 (2026)", "请求救援 (2026).mkv")
if res.Reclassified != 1 || res.Organized != 0 {
t.Fatalf("result = %+v, want one reclassified movie", res)
}
if _, err := os.Stat(wrongPath); !os.IsNotExist(err) {
t.Fatalf("wrong generated episode path should be moved away, stat err=%v", err)
}
if _, err := os.Stat(want); err != nil {
t.Fatalf("movie target missing at %q: %v; items=%#v", want, err, res.Items)
}
var got model.Media
if err := repos.DB.First(&got, "path = ?", want).Error; err != nil {
t.Fatal(err)
}
if got.LibraryID != foreignMovieLib.ID || got.SeasonNum != 0 || got.EpisodeNum != 0 {
t.Fatalf("row after reclassify = library %q S%dE%d, want movie lib with cleared episode numbers", got.LibraryID, got.SeasonNum, got.EpisodeNum)
}
}
func TestOrganizeDirectoryRejectedMetadataKeepsExplicitWesternSourceCategory(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@@ -12,7 +12,12 @@ import (
)
func (o *OrganizerService) lookupOrganizeMetadata(ctx context.Context, src, sourceRoot, mediaType, title string, year, season, episode int, cache map[string]*Match) *Match {
seriesLike := isSeriesLibraryType(mediaType) || season > 0 || episode > 0
normalizedType := normalizeOrganizeMediaType(mediaType)
lookupSeason, lookupEpisode := season, episode
if normalizedType == "movie" {
lookupSeason, lookupEpisode = 0, 0
}
seriesLike := isSeriesLibraryType(mediaType) || (normalizedType != "movie" && (season > 0 || episode > 0))
if local, err := ReadLocalMetadata(src, sourceRoot, seriesLike); err == nil && local != nil {
if match := organizeMatchFromLocalMetadata(local); match != nil {
return match
@@ -35,8 +40,8 @@ func (o *OrganizerService) lookupOrganizeMetadata(ctx context.Context, src, sour
Title: title,
Year: year,
Path: src,
SeasonNum: season,
EpisodeNum: episode,
SeasonNum: lookupSeason,
EpisodeNum: lookupEpisode,
}
for _, candidate := range scrapeQueryCandidates(media, lib) {
key := organizeMetadataCacheKey(lib.Type, candidate, year)
@@ -59,6 +64,7 @@ func (o *OrganizerService) lookupOrganizeMetadata(ctx context.Context, src, sour
zap.String("source", src),
zap.String("query", candidate),
zap.String("title", match.Title),
zap.String("media_type", match.MediaType),
zap.Int("source_year", year),
zap.Int("match_year", match.Year),
zap.Int("tmdb_id", match.TMDbID),
@@ -76,6 +82,7 @@ func (o *OrganizerService) lookupOrganizeMetadata(ctx context.Context, src, sour
zap.String("source", src),
zap.String("query", candidate),
zap.String("title", match.Title),
zap.String("media_type", match.MediaType),
zap.Int("year", match.Year),
zap.Int("tmdb_id", match.TMDbID),
zap.Int("bangumi_id", match.BangumiID),
@@ -210,10 +210,15 @@ func (o *OrganizerService) updateReclassifiedMediaRow(ctx context.Context, oldPa
if req.Year > 0 {
updates["year"] = req.Year
}
if req.Season > 0 {
if normalizeOrganizeMediaType(req.MediaType) == "movie" {
updates["season_num"] = 0
updates["episode_num"] = 0
} else if req.Season > 0 {
updates["season_num"] = req.Season
}
if req.Episode > 0 {
if req.Episode > 0 {
updates["episode_num"] = req.Episode
}
} else if req.Episode > 0 {
updates["episode_num"] = req.Episode
}
return o.repo.DB.WithContext(ctx).Model(&model.Media{}).Where("path = ?", oldPath).Updates(updates).Error
+62 -1
View File
@@ -80,7 +80,27 @@ func (o *OrganizerService) buildOrganizeSourceFilePlan(ctx context.Context, req
forcedType := normalizeOrganizeMediaType(req.MediaTypeOverride)
inferredType := o.inferMediaTypeForSourceFile(req.Source, identity.Title, identity.Season, identity.Episode)
layout = applyOrganizeSourceMediaType(layout, forcedType, inferredType)
metadataMatch := o.lookupOrganizeSourceMetadata(ctx, req, layout.MediaType, &identity)
lookupType := organizeSourceMetadataLookupType(req.Source, pathLayout, layout, forcedType, inferredType, identity)
metadataMatch := o.lookupOrganizeSourceMetadata(ctx, req, lookupType, &identity)
if metadataMatch == nil && lookupType != layout.MediaType {
metadataMatch = o.lookupOrganizeSourceMetadata(ctx, req, layout.MediaType, &identity)
}
if forcedType == "" && metadataMatch != nil {
if matchType := normalizeOrganizeMediaType(metadataMatch.MediaType); matchType != "" && matchType != layout.MediaType {
if categoryType, _ := o.mediaTypeForDirectoryCategory(layout.Category); categoryType != "" && !sourceCategoryCompatible(matchType, categoryType) {
layout.Category = ""
}
if o.log != nil {
o.log.Info("organize media type corrected by metadata",
zap.String("source", req.Source),
zap.String("from", layout.MediaType),
zap.String("to", matchType),
zap.String("title", metadataMatch.Title),
zap.Int("tmdb_id", metadataMatch.TMDbID))
}
layout.MediaType = matchType
}
}
layout = o.applyOrganizeSourceCategory(ctx, req, pathLayout, layout, forcedType, identity, metadataMatch)
layoutRoot, targetLibraryID := o.resolveOrganizeSourceLayoutRoot(ctx, req, layout)
target, err := o.buildOrganizeSourceTarget(ctx, req, layout, layoutRoot, identity)
@@ -164,6 +184,47 @@ func applyOrganizeSourceMediaType(layout organizeDirectoryLayout, forcedType, in
return layout
}
func organizeSourceMetadataLookupType(src string, pathLayout, layout organizeDirectoryLayout, forcedType, inferredType string, identity organizeSourceIdentity) string {
if forcedType != "" {
return layout.MediaType
}
if pathLayout.MediaType != "" && pathLayout.MediaType != "movie" && organizeStandaloneMovieSourceHint(src, identity) {
return "movie"
}
return layout.MediaType
}
func organizeStandaloneMovieSourceHint(src string, identity organizeSourceIdentity) bool {
if identity.Season > 0 || identity.Episode > 0 {
return organizeEpisodeLooksSourcedFromMovieYear(src, identity)
}
_, year := CleanQuery(filepath.Base(src))
if year <= 0 {
year = identity.Year
}
if year <= 0 {
return false
}
text := strings.ToLower(strings.Join([]string{filepath.Base(src), identity.Title, identity.ParsedTitle}, " "))
return !classifierEpisodeRE.MatchString(text) && !classifierSeasonRE.MatchString(text)
}
func organizeEpisodeLooksSourcedFromMovieYear(src string, identity organizeSourceIdentity) bool {
year := identity.Year
if identity.SourceMedia != nil && identity.SourceMedia.Year > 0 {
year = identity.SourceMedia.Year
}
if year < 1900 || year > 2099 || identity.Season != 1 || identity.Episode != year/10 {
return false
}
showDir := showDirFromEpisodePath(src)
if showDir == "" {
return false
}
_, folderYear := CleanQuery(filepath.Base(showDir))
return folderYear == year
}
func (o *OrganizerService) lookupOrganizeSourceMetadata(ctx context.Context, req organizeSourceFileRequest, mediaType string, identity *organizeSourceIdentity) *Match {
match := o.lookupOrganizeMetadata(ctx, req.Source, req.SourceRoot, mediaType, identity.Title, identity.Year, identity.Season, identity.Episode, req.MetadataCache)
if match == nil && identity.SourceMedia != nil {
+8 -3
View File
@@ -145,9 +145,14 @@ func (o *OrganizerService) applyOrganizeMedia(ctx context.Context, req organizeM
}
updates := map[string]any{
"path": dst.path,
"season_num": m.SeasonNum,
"episode_num": m.EpisodeNum,
"path": dst.path,
}
if normalizeOrganizeMediaType(dst.mediaType) == "movie" {
updates["season_num"] = 0
updates["episode_num"] = 0
} else {
updates["season_num"] = m.SeasonNum
updates["episode_num"] = m.EpisodeNum
}
if strings.TrimSpace(dst.libraryID) != "" {
updates["library_id"] = strings.TrimSpace(dst.libraryID)
+5
View File
@@ -665,6 +665,11 @@ func (s *ScraperService) determineMediaTypeForMedia(lib *model.Library, media *m
if media != nil && mediaIsEpisodic(media, lib) {
return "tv"
}
if match != nil {
if mediaType := normalizeOrganizeMediaType(match.MediaType); mediaType != "" {
return mediaType
}
}
if lib != nil {
switch lib.Type {
case "tv", "anime", "variety", "show", "shows":
+2
View File
@@ -130,6 +130,7 @@ func (t *TheTVDBProvider) SearchSeries(ctx context.Context, query string) (*Matc
r := p.Data[0]
m := &Match{
TheTVDBID: r.ID,
MediaType: "tv",
Title: r.Name,
Overview: r.Overview,
PosterURL: r.Image,
@@ -240,6 +241,7 @@ func (t *TheTVDBProvider) GetSeriesMatchByID(ctx context.Context, seriesID strin
}
m := &Match{
TheTVDBID: idValue,
MediaType: "tv",
Title: out.Data.Name,
Overview: out.Data.Overview,
PosterURL: firstNonEmpty(out.Data.Image, out.Data.ImageURL),
+3
View File
@@ -128,6 +128,7 @@ type Match struct {
BangumiID int `json:"bangumi_id"`
DoubanID string `json:"douban_id,omitempty"`
TheTVDBID string `json:"thetvdb_id,omitempty"`
MediaType string `json:"media_type,omitempty"`
Title string `json:"title"`
OriginalName string `json:"original_name,omitempty"`
Overview string `json:"overview"`
@@ -195,6 +196,7 @@ func (t *TMDbProvider) GetMovieMatch(ctx context.Context, tmdbID int) (*Match, e
}
m := &Match{
TMDbID: r.ID,
MediaType: "movie",
Title: r.Title,
OriginalName: r.OriginalTitle,
Overview: r.Overview,
@@ -264,6 +266,7 @@ func (t *TMDbProvider) GetTVMatch(ctx context.Context, tmdbID int) (*Match, erro
}
m := &Match{
TMDbID: r.ID,
MediaType: "tv",
Title: r.Name,
OriginalName: r.OriginalName,
Overview: r.Overview,
+2
View File
@@ -89,6 +89,7 @@ func (t *TMDbProvider) SearchMovieCandidates(ctx context.Context, query string,
func (t *TMDbProvider) movieSearchResultToMatch(r tmdbMovieSearchResult) *Match {
m := &Match{
TMDbID: r.ID,
MediaType: "movie",
Title: r.Title,
OriginalName: r.OriginalTitle,
Overview: r.Overview,
@@ -161,6 +162,7 @@ func (t *TMDbProvider) SearchTVCandidates(ctx context.Context, query string, yea
func (t *TMDbProvider) tvSearchResultToMatch(r tmdbTVSearchResult) *Match {
m := &Match{
TMDbID: r.ID,
MediaType: "tv",
Title: r.Name,
OriginalName: r.OriginalName,
Overview: r.Overview,