fix: clean media classification and series grouping

This commit is contained in:
ShukeBta
2026-06-25 05:37:31 +08:00
parent 64a7f78e1b
commit b2c04bb7b6
15 changed files with 305 additions and 9 deletions
+1
View File
@@ -306,6 +306,7 @@ func setDefaults(v *viper.Viper) {
v.SetDefault("organizer.categories.jk_tv", "日韩剧")
v.SetDefault("organizer.categories.jp_anime", "日番")
v.SetDefault("organizer.categories.cn_anime", "国漫")
v.SetDefault("organizer.categories.euus_anime", "欧美动漫")
v.SetDefault("organizer.categories.variety", "综艺")
v.SetDefault("organizer.categories.documentary", "纪录片")
v.SetDefault("organizer.categories.children", "儿童")
+7
View File
@@ -100,6 +100,9 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
if isChineseAnime {
return categoryName(categories, "cn_anime", "国漫")
}
if isWesternByMetadata || (!hasMetadata && containsAnyText(categoryText, "欧美动漫", "欧美动画", "西方动画")) {
return categoryName(categories, "euus_anime", "欧美动漫")
}
return categoryName(categories, "jp_anime", "日番")
case "variety":
return categoryName(categories, "variety", "综艺")
@@ -120,6 +123,9 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
if isChineseAnime {
return categoryName(categories, "cn_anime", "国漫")
}
if isWesternByMetadata || (!hasMetadata && containsAnyText(categoryText, "欧美动漫", "欧美动画", "西方动画")) {
return categoryName(categories, "euus_anime", "欧美动漫")
}
return categoryName(categories, "jp_anime", "日番")
}
if !hasMetadata && sourceHint != "" {
@@ -293,6 +299,7 @@ var sourceCategoryHints = []sourceCategoryHintDef{
{Key: "jk_tv", Fallback: "日韩剧", MediaType: "tv"},
{Key: "cn_anime", Fallback: "国漫", MediaType: "anime"},
{Key: "jp_anime", Fallback: "日番", MediaType: "anime"},
{Key: "euus_anime", Fallback: "欧美动漫", MediaType: "anime"},
{Key: "variety", Fallback: "综艺", MediaType: "variety"},
{Key: "documentary", Fallback: "纪录片", MediaType: "tv"},
{Key: "children", Fallback: "儿童", MediaType: "tv"},
+30
View File
@@ -196,6 +196,36 @@ func TestClassifyMediaCategoryMatchesSmartRules(t *testing.T) {
},
want: "日番",
},
{
name: "western anime metadata uses western anime category",
input: mediaClassifyInput{
MediaType: "anime",
Title: "Family Guy",
Countries: []string{"US"},
Genres: []string{"16"},
Category: "日番",
},
want: "欧美动漫",
},
{
name: "tv animation with western metadata uses western anime category",
input: mediaClassifyInput{
MediaType: "tv",
Title: "The Simpsons",
Countries: []string{"US"},
Genres: []string{"Animation"},
},
want: "欧美动漫",
},
{
name: "western anime source category is preserved without metadata",
input: mediaClassifyInput{
MediaType: "anime",
Title: "The Simpsons S01E01 1080p",
Category: "downloads 欧美动漫",
},
want: "欧美动漫",
},
{
name: "anime with CN country metadata is cn",
input: mediaClassifyInput{
+7 -2
View File
@@ -107,7 +107,7 @@ func groupMediaSeriesCards(items []model.Media) []SeriesCard {
return cards
}
var episodicPathRE = regexp.MustCompile(`(?i)[\\/](?:电视剧|剧集|国产剧|欧美剧|日韩剧|日剧|韩剧|综艺|纪录片|动漫|番剧|国漫|日番|儿童|tv|series|shows?|season[\s._-]*\d|s\d{1,2}(?:[\s._-]|[\\/])|special[\s._-]*episodes?|specials?|sp|ovas?|oads?|extras?|bonus(?:es)?|omake|特别篇|特別篇|番外篇?|特典|外传|外傳|总集篇|總集篇)[\\/]`)
var episodicPathRE = regexp.MustCompile(`(?i)[\\/](?:电视剧|剧集|国产剧|欧美剧|日韩剧|日剧|韩剧|综艺|纪录片|动漫|番剧|国漫|日番|欧美动漫|欧美动画|儿童|tv|series|shows?|season[\s._-]*\d|s\d{1,2}(?:[\s._-]|[\\/])|special[\s._-]*episodes?|specials?|sp|ovas?|oads?|extras?|bonus(?:es)?|omake|特别篇|特別篇|番外篇?|特典|外传|外傳|总集篇|總集篇)[\\/]`)
func mediaSeriesKey(media model.Media) string {
return compactSeriesKey(mediaSeriesRawKey(media))
@@ -188,7 +188,12 @@ func normalizeSeriesTitle(value string) string {
}
func normalizeSeriesPathTitle(value string) string {
title := normalizeSeriesTitle(value)
title, _ := CleanQuery(value)
if title == "" {
title = normalizeSeriesTitle(value)
} else {
title = normalizeSeriesTitle(title)
}
stripped := stripSeriesSpecialSuffix(title)
if stripped != "" {
return stripped
+34
View File
@@ -170,3 +170,37 @@ func TestMediaSeriesKeyCollapsesNumberedSpecialSuffixes(t *testing.T) {
}
}
}
func TestMediaSeriesKeyCleansReleaseNoiseFolders(t *testing.T) {
clean := model.Media{
LibraryID: "lib-variety",
Path: `F:\media\电视剧\综艺\Hntv Spring Festival Gala S01e (2026)\Season 1\Hntv Spring Festival Gala S01e - S01E202.ts`,
SeasonNum: 1,
EpisodeNum: 202,
}
dirty := model.Media{
LibraryID: "lib-variety",
Path: `F:\media\电视剧\综艺\Hntv Spring Festival Gala Fps Hlg Qhstudio S01e (2026)\Season 1\Hntv Spring Festival Gala Fps Hlg Qhstudio S01e - S01E202.ts`,
SeasonNum: 1,
EpisodeNum: 202,
}
if got, want := mediaSeriesKey(dirty), mediaSeriesKey(clean); got != want {
t.Fatalf("dirty folder key=%q, want clean folder key=%q", got, want)
}
noisyRelease := model.Media{
LibraryID: "lib-tv",
Path: `F:\media\电视剧\欧美剧\Motherhood Of Taihang Aac2 Mweb\Season 1\Motherhood Of Taihang Aac2 Mweb - S01E01-Aac2.Mweb.mkv`,
SeasonNum: 1,
EpisodeNum: 1,
}
cleanRelease := model.Media{
LibraryID: "lib-tv",
Path: `F:\media\电视剧\欧美剧\Motherhood Of Taihang\Season 1\Motherhood Of Taihang - S01E01.mkv`,
SeasonNum: 1,
EpisodeNum: 1,
}
if got, want := mediaSeriesKey(noisyRelease), mediaSeriesKey(cleanRelease); got != want {
t.Fatalf("release-noise folder key=%q, want clean key=%q", got, want)
}
}
@@ -116,6 +116,7 @@ func (o *OrganizerService) directoryCategoryTypes() map[string]organizeDirectory
addConfigured("jk_tv", "日韩剧", "tv")
addConfigured("cn_anime", "国漫", "anime")
addConfigured("jp_anime", "日番", "anime")
addConfigured("euus_anime", "欧美动漫", "anime")
addConfigured("variety", "综艺", "variety")
addConfigured("documentary", "纪录片", "tv")
addConfigured("children", "儿童", "tv")
@@ -264,6 +264,8 @@ func (o *OrganizerService) organizeCategoryAliases(mediaType, category string) m
add("日番", "日漫", "日本动漫", "日本动画")
case normalizeOrganizeCategoryKey(categoryName(categories, "cn_anime", "国漫")), "国漫", "国产动漫", "國漫":
add("国漫", "国产动漫")
case normalizeOrganizeCategoryKey(categoryName(categories, "euus_anime", "欧美动漫")), "欧美动漫", "欧美动画", "西方动画":
add("欧美动漫", "欧美动画", "西方动画")
case normalizeOrganizeCategoryKey(categoryName(categories, "domestic_tv", "国产剧")), "国产剧", "国剧", "大陆剧", "国产电视剧":
add("国产剧", "国剧", "大陆剧", "国产电视剧")
case normalizeOrganizeCategoryKey(categoryName(categories, "euus_tv", "欧美剧")), "欧美剧", "欧美电视剧":
+2 -1
View File
@@ -50,8 +50,9 @@ func (o *OrganizerService) categoryPhysicalRootDir(category string) string {
case match(
categoryName(categories, "cn_anime", "国漫"),
categoryName(categories, "jp_anime", "日番"),
categoryName(categories, "euus_anime", "欧美动漫"),
categoryName(categories, "children", "儿童"),
"国漫", "国产动漫", "日番", "番剧", "日漫", "日本动漫", "日本动画", "儿童", "少儿",
"国漫", "国产动漫", "日番", "番剧", "日漫", "日本动漫", "日本动画", "欧美动漫", "欧美动画", "西方动画", "儿童", "少儿",
):
return "动漫"
case match(
@@ -108,6 +108,78 @@ func TestOrganizeDirectoryReclassifiesExistingWrongCategoryMedia(t *testing.T) {
}
}
func TestOrganizeDirectoryCleansReleaseNoiseBeforeMetadataClassify(t *testing.T) {
var queries []string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.URL.Path != "/search/tv" {
http.NotFound(w, r)
return
}
query := r.URL.Query().Get("query")
queries = append(queries, query)
if query != "motherhood of taihang" {
_ = json.NewEncoder(w).Encode(map[string]any{"results": []any{}})
return
}
_ = json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{{
"id": 219630,
"name": "太行之脊",
"original_name": "Motherhood of Taihang",
"original_language": "zh",
"origin_country": []string{"CN"},
"genre_ids": []int{18},
"first_air_date": "2020-08-26",
}},
})
}))
defer upstream.Close()
repos := newOrganizerTestRepo(t)
cfg := &config.Config{}
cfg.Organizer.SmartClassify = true
cfg.Secrets.TMDbAPIKey = "test-key"
cfg.Secrets.TMDbAPIProxy = upstream.URL
scraper := NewScraperService(cfg, zap.NewNop(), repos, NewTMDbProvider(cfg, zap.NewNop(), nil), nil, nil, nil, NewHub(zap.NewNop()))
root := t.TempDir()
srcRoot := filepath.Join(root, "downloads")
dest := filepath.Join(root, "media")
sourceFile := filepath.Join(srcRoot, "Motherhood Of Taihang Aac2 Mweb", "Motherhood Of Taihang Aac2 Mweb - S01E01-Aac2.Mweb - 第 1 集.mkv")
writeOrgFile(t, sourceFile, "episode")
euusLib := model.Library{Name: "欧美剧", Path: filepath.Join(dest, "电视剧", "欧美剧"), Type: "tv", Enabled: true}
domesticLib := model.Library{Name: "国产剧", Path: filepath.Join(dest, "电视剧", "国产剧"), Type: "tv", Enabled: true}
if err := repos.Library.Create(t.Context(), &euusLib); err != nil {
t.Fatal(err)
}
if err := repos.Library.Create(t.Context(), &domesticLib); err != nil {
t.Fatal(err)
}
organizer := NewOrganizerService(cfg, zap.NewNop(), repos)
organizer.SetScraper(scraper)
res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{
SourcePath: srcRoot,
DestPath: euusLib.Path,
TransferMode: TransferCopy,
})
if err != nil {
t.Fatalf("organize directory: %v", err)
}
want := filepath.Join(domesticLib.Path, "太行之脊", "Season 01", "太行之脊 - S01E01.mkv")
if res.Organized != 1 || res.Reclassified != 0 {
t.Fatalf("result = %+v, want organized=1 only; queries=%v", res, queries)
}
if _, err := os.Stat(want); err != nil {
t.Fatalf("organized media missing at %q: %v; items=%#v queries=%v", want, err, res.Items, queries)
}
if len(queries) == 0 || queries[0] != "motherhood of taihang" {
t.Fatalf("first metadata query = %q, want cleaned title; all queries=%v", firstQuery(queries), queries)
}
}
func TestOrganizeDirectoryReclassifiesScannedAnimeUsingDBMetadata(t *testing.T) {
repos := newOrganizerTestRepo(t)
cfg := &config.Config{}
@@ -225,6 +297,60 @@ func TestReclassifyMisclassifiedMediaMovesScannedAnimeToPhysicalAnimeLibrary(t *
}
}
func TestReclassifyMisclassifiedMediaMovesWesternAnimationToWesternAnimeLibrary(t *testing.T) {
repos := newOrganizerTestRepo(t)
cfg := &config.Config{}
cfg.Organizer.SmartClassify = true
root := t.TempDir()
dest := filepath.Join(root, "media")
jpAnimeLib := model.Library{Name: "日番", Path: filepath.Join(dest, "动漫", "日番"), Type: "anime", Enabled: true}
westernAnimeLib := model.Library{Name: "欧美动漫", Path: filepath.Join(dest, "动漫", "欧美动漫"), Type: "anime", Enabled: true}
for _, lib := range []*model.Library{&jpAnimeLib, &westernAnimeLib} {
if err := repos.Library.Create(t.Context(), lib); err != nil {
t.Fatal(err)
}
}
wrongPath := filepath.Join(jpAnimeLib.Path, "Family Guy", "Season 10", "Family Guy - S10E15.mkv")
writeOrgFile(t, wrongPath, "episode")
if err := repos.DB.Create(&model.Media{
LibraryID: jpAnimeLib.ID,
Title: "恶搞之家",
OriginalName: "Family Guy",
Path: wrongPath,
SeasonNum: 10,
EpisodeNum: 15,
TMDbID: 1434,
Languages: "en",
Countries: "US",
Genres: "动画,喜剧",
ScrapeStatus: "matched",
}).Error; err != nil {
t.Fatal(err)
}
organizer := NewOrganizerService(cfg, zap.NewNop(), repos)
res, err := organizer.ReclassifyMisclassifiedMedia(t.Context(), MediaCategoryReclassifyOptions{})
if err != nil {
t.Fatalf("reclassify media: %v", err)
}
want := filepath.Join(westernAnimeLib.Path, "恶搞之家", "Season 10", "恶搞之家 - S10E15.mkv")
if res.Reclassified != 1 {
t.Fatalf("reclassified = %d, want 1; items=%#v errors=%#v", res.Reclassified, res.Items, res.Errors)
}
if _, err := os.Stat(want); err != nil {
t.Fatalf("western animation target missing at %q: %v", want, err)
}
var got model.Media
if err := repos.DB.First(&got, "path = ?", want).Error; err != nil {
t.Fatal(err)
}
if got.LibraryID != westernAnimeLib.ID {
t.Fatalf("library_id = %q, want western anime library %q", got.LibraryID, westernAnimeLib.ID)
}
}
func TestOrganizeDirectoryCleansWrongCategoryDuplicateWhenTargetExists(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
+4 -2
View File
@@ -28,15 +28,17 @@ var noiseTokens = []string{
// 流媒体平台 / 字幕组 / 国家版本(动漫常见)
"netflix", "nf", "amzn", "hulu", "disney", "max", "hbo",
"linetv", "ourtv", "iqiyi", "youku", "bilibili", "qiyi", "krj",
"atvp", "appletv", "apple-tv", "tx", "txweb",
"crunchyroll", "funimation", "anidb", "horriblesubs", "subsplease",
"erai-raws", "judas", "asw", "smcat", "leopard-raws", "ohys-raws", "colortv",
"mweb", "ubweb", "hhweb", "adweb", "chdweb", "kurosawa", "qhstudio",
// 中文字幕标记
"zm", "zw", "ch", "chs", "cht", "cn", "tc", "sc",
"中字", "繁字", "简中", "繁中", "国语", "粤语", "日语",
// 季数前缀残留 — ParseEpisode 已抽取过
"season", "264", "265",
"season", "264", "265", "aac2", "aac5",
}
var noiseTokenSet = func() map[string]struct{} {
@@ -317,7 +319,7 @@ func isGenericMediaCategoryFolder(name string) bool {
"华语电影", "国产电影", "大陆电影",
"外语电影", "欧美电影", "日韩电影",
"动画电影", "动漫电影",
"国漫", "国产动漫", "日番", "日漫", "日本动漫", "日本动画",
"国漫", "国产动漫", "日番", "日漫", "日本动漫", "日本动画", "欧美动漫", "欧美动画", "西方动画",
"综艺", "真人秀",
"纪录片", "纪录",
"儿童", "少儿",
+30
View File
@@ -33,6 +33,9 @@ func TestCleanQuery(t *testing.T) {
{"紫川 (2024) {tmdb-247590}", "紫川", 2024},
{"HNTV.Spring.Festival.Gala.FPS.HLG-QHStudio.S01E202-DD5.QHstudIo.6.4K.ts", "hntv spring festival gala", 0},
{"Hntv Spring Festival Gala S01e (2026)", "hntv spring festival gala", 2026},
{"Motherhood Of Taihang Aac2 Mweb - S01E01-Aac2.Mweb.mkv", "motherhood of taihang", 0},
{"For All Mankind Atvp Hhweb - S05E06-DDP5.HHWEB.4K.mkv", "for all mankind", 0},
{"Hntv Spring Festival Gala Fps Hlg Qhstudio S01e (2026)", "hntv spring festival gala", 2026},
}
for _, tc := range cases {
t.Run(tc.in, func(t *testing.T) {
@@ -45,6 +48,33 @@ func TestCleanQuery(t *testing.T) {
}
}
func TestScrapeQueryCandidatesCleanDirtySeriesFolder(t *testing.T) {
lib := &model.Library{
Path: `F:\media\电视剧\欧美剧`,
Type: "tv",
}
media := &model.Media{
Title: "motherhood of taihang aac2 mweb aac2 mweb",
Path: `F:\media\电视剧\欧美剧\Motherhood Of Taihang Aac2 Mweb\Season 1\Motherhood Of Taihang Aac2 Mweb - S01E01-Aac2.Mweb - 第 1 集.mkv`,
SeasonNum: 1,
EpisodeNum: 1,
}
got := scrapeQueryCandidates(media, lib)
if len(got) == 0 {
t.Fatal("scrapeQueryCandidates returned no candidates")
}
if got[0] != "motherhood of taihang" {
t.Fatalf("first query candidate = %q, want cleaned series title; all candidates=%#v", got[0], got)
}
for _, candidate := range got {
lower := strings.ToLower(candidate)
if strings.Contains(lower, "mweb") || strings.Contains(lower, "aac2") {
t.Fatalf("query candidate kept release noise %q: %#v", candidate, got)
}
}
}
func TestExternalIDHintsFromText(t *testing.T) {
hints := externalIDHintsFromText("国漫/折腰 (2025) {tmdb 296753}/Season 1/折腰.S01E01.mkv")
if hints.TMDbID != 296753 {
+1 -1
View File
@@ -112,7 +112,7 @@ func strmCategoryRoot(part string) string {
return "电影"
case "国产剧", "欧美剧", "日韩剧", "日剧", "韩剧", "综艺", "真人秀", "纪录片", "纪录", "未分类":
return "电视剧"
case "国漫", "国产动漫", "日番", "番剧", "日漫", "日本动漫", "日本动画", "儿童", "少儿":
case "国漫", "国产动漫", "日番", "番剧", "日漫", "日本动漫", "日本动画", "欧美动漫", "欧美动画", "西方动画", "儿童", "少儿":
return "动漫"
case "番号":
return "成人"