mirror of
https://github.com/truewhile/MeBox.git
synced 2026-09-29 11:36:36 +08:00
feat: scrape metadata after organize
This commit is contained in:
@@ -250,6 +250,7 @@ func setDefaults(v *viper.Viper) {
|
||||
v.SetDefault("downloads.smart_classify", true)
|
||||
v.SetDefault("organizer.smart_classify", false)
|
||||
v.SetDefault("organizer.auto_after_download", false)
|
||||
v.SetDefault("organize.scrape_after", false)
|
||||
v.SetDefault("organizer.categories.chinese_movie", "华语电影")
|
||||
v.SetDefault("organizer.categories.animation_movie", "动画电影")
|
||||
v.SetDefault("organizer.categories.foreign_movie", "外语电影")
|
||||
|
||||
@@ -85,6 +85,9 @@ func downloadOrganizeAllHandler(svc *service.Container) gin.HandlerFunc {
|
||||
results = append(results, gin.H{"library": l.Name, "error": err.Error()})
|
||||
continue
|
||||
}
|
||||
if svc.Scan != nil && res != nil && !res.DryRun {
|
||||
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, l.ID, nil)
|
||||
}
|
||||
results = append(results, gin.H{"library": l.Name, "result": res})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"results": results})
|
||||
|
||||
@@ -134,6 +134,9 @@ func organizeBulkHandler(svc *service.Container) gin.HandlerFunc {
|
||||
out = append(out, gin.H{"library": l.Name, "error": err.Error()})
|
||||
continue
|
||||
}
|
||||
if svc.Scan != nil && res != nil && !res.DryRun {
|
||||
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, l.ID, nil)
|
||||
}
|
||||
out = append(out, gin.H{"library": l.Name, "result": res})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"results": out})
|
||||
|
||||
@@ -21,6 +21,7 @@ type organizeReq struct {
|
||||
TransferMode string `json:"transfer_mode"`
|
||||
MediaType string `json:"media_type"`
|
||||
ScanAfter bool `json:"scan_after"`
|
||||
ScrapeAfter *bool `json:"scrape_after"`
|
||||
LibraryID string `json:"library_id"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
@@ -52,24 +53,37 @@ func organizeOptionsFromReq(req organizeReq) service.OrganizeOptions {
|
||||
|
||||
func organizeMediaHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
opts := bindOrganizeOptions(c)
|
||||
var req organizeReq
|
||||
_ = c.ShouldBindJSON(&req)
|
||||
opts := organizeOptionsFromReq(req)
|
||||
dst, err := svc.Organizer.OrganizeMediaWithOptions(c.Request.Context(), c.Param("id"), opts)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"path": dst})
|
||||
payload := gin.H{"path": dst}
|
||||
if req.ScanAfter && !req.DryRun && svc.Scan != nil {
|
||||
scans, scrapes := scanAndScrapeAfterOrganize(c, svc, dst, strings.TrimSpace(req.LibraryID), req.ScrapeAfter)
|
||||
payload["scans"] = scans
|
||||
payload["scrapes"] = scrapes
|
||||
}
|
||||
c.JSON(http.StatusOK, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func organizeLibraryHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
opts := bindOrganizeOptions(c)
|
||||
var req organizeReq
|
||||
_ = c.ShouldBindJSON(&req)
|
||||
opts := organizeOptionsFromReq(req)
|
||||
res, err := svc.Organizer.OrganizeLibraryWithOptions(c.Request.Context(), c.Param("id"), opts)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if req.ScanAfter && !req.DryRun && svc.Scan != nil {
|
||||
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, c.Param("id"), req.ScrapeAfter)
|
||||
}
|
||||
c.JSON(http.StatusOK, res)
|
||||
}
|
||||
}
|
||||
@@ -95,8 +109,16 @@ func organizeDirectoryHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return
|
||||
}
|
||||
if req.ScanAfter && !req.DryRun && svc.Scan != nil {
|
||||
res.Scans = svc.Scan.ScanLibrariesForPath(c.Request.Context(), res.DestPath, strings.TrimSpace(req.LibraryID))
|
||||
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, strings.TrimSpace(req.LibraryID), req.ScrapeAfter)
|
||||
}
|
||||
c.JSON(http.StatusOK, res)
|
||||
}
|
||||
}
|
||||
|
||||
func scanAndScrapeAfterOrganize(c *gin.Context, svc *service.Container, destRoot, preferredLibraryID string, scrapeOverride *bool) ([]service.OrganizeScanSummary, []service.OrganizeScrapeSummary) {
|
||||
scrapeAfter := service.OrganizeScrapeAfterEnabled(c.Request.Context(), svc.Repo)
|
||||
if scrapeOverride != nil {
|
||||
scrapeAfter = *scrapeOverride
|
||||
}
|
||||
return svc.Scan.ScanAndScrapeLibrariesForPath(c.Request.Context(), destRoot, preferredLibraryID, scrapeAfter)
|
||||
}
|
||||
|
||||
@@ -80,6 +80,7 @@ func schemaHandler(_ *service.Container) gin.HandlerFunc {
|
||||
"items": []gin.H{
|
||||
{"key": "organize.auto", "type": "toggle", "label": "整理源目录定时自动整理"},
|
||||
{"key": "organizer.auto_after_download", "type": "toggle"},
|
||||
{"key": "organize.scrape_after", "type": "toggle", "label": "整理后自动刮削"},
|
||||
{"key": "downloads.smart_classify", "type": "toggle"},
|
||||
{"key": "organizer.smart_classify", "type": "toggle"},
|
||||
{"key": "organize.source_dir", "type": "text", "label": "整理源目录"},
|
||||
|
||||
@@ -881,7 +881,7 @@ func (d *DownloadService) onTorrentComplete(ctx context.Context, torrent QBitTor
|
||||
return
|
||||
}
|
||||
if d.scanner != nil && res != nil && strings.TrimSpace(res.DestPath) != "" {
|
||||
res.Scans = d.scanner.ScanLibrariesForPath(ctx, res.DestPath, "")
|
||||
res.Scans, res.Scrapes = d.scanner.ScanAndScrapeLibrariesForPath(ctx, res.DestPath, "", OrganizeScrapeAfterEnabled(ctx, d.repo))
|
||||
}
|
||||
d.log.Info("auto organize completed torrent finished",
|
||||
zap.String("hash", torrent.Hash),
|
||||
@@ -890,6 +890,7 @@ func (d *DownloadService) onTorrentComplete(ctx context.Context, torrent QBitTor
|
||||
zap.Int("organized", res.Organized),
|
||||
zap.Int("replaced", res.Replaced),
|
||||
zap.Int("skipped", res.Skipped),
|
||||
zap.Int("scrapes", len(res.Scrapes)),
|
||||
zap.Int("errors", len(res.Errors)))
|
||||
}
|
||||
|
||||
|
||||
@@ -31,20 +31,30 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
|
||||
genres := normalizeTokens(input.Genres...)
|
||||
countries := normalizeTokens(input.Countries...)
|
||||
languages := normalizeTokens(input.Languages...)
|
||||
text := strings.ToLower(input.Title + " " + input.Category + " " + strings.Join(input.Genres, " "))
|
||||
rawText := input.Title + " " + input.Category + " " + strings.Join(input.Genres, " ")
|
||||
text := strings.ToLower(rawText)
|
||||
|
||||
isChinese := hasAny(languages, "ZH", "ZH-CN", "ZH-TW", "CN") || hasAny(countries, "CN", "TW", "HK", "MO")
|
||||
isJapanese := hasAny(languages, "JA", "JP") || hasAny(countries, "JP") || strings.Contains(text, "日番")
|
||||
isKorean := hasAny(languages, "KO", "KR") || hasAny(countries, "KR", "KP")
|
||||
isChinese := hasAny(languages, "ZH", "ZH-CN", "ZH-TW", "CN") || hasAny(countries, "CN", "TW", "HK", "MO") || containsHan(rawText) || containsAnyText(text, "华语", "国产", "国剧", "国漫")
|
||||
isJapanese := hasAny(languages, "JA", "JP") || hasAny(countries, "JP") || containsJapaneseKana(rawText) || strings.Contains(text, "日番")
|
||||
isKorean := hasAny(languages, "KO", "KR") || hasAny(countries, "KR", "KP") || containsKoreanHangul(rawText)
|
||||
isEastAsian := isJapanese || isKorean || hasAny(countries, "TH", "IN", "SG")
|
||||
isWestern := hasAny(countries,
|
||||
isWesternByMetadata := hasAny(countries,
|
||||
"US", "GB", "UK", "FR", "DE", "CA", "AU", "NZ", "IE", "NL", "SE", "NO", "DK",
|
||||
"FI", "ES", "IT", "PT", "AT", "CH", "BE", "RU",
|
||||
)
|
||||
isLatinFallback := containsLatin(rawText) && !containsHan(rawText) && !containsJapaneseKana(rawText) && !containsKoreanHangul(rawText)
|
||||
isWestern := isWesternByMetadata || (mediaType == "tv" && isLatinFallback)
|
||||
hasAnimeText := containsAnyText(text, "动画", "动漫", "番剧", "年番", "国漫", "日番", "bangumi", "anime")
|
||||
|
||||
hasGenre := func(values ...string) bool {
|
||||
for _, value := range values {
|
||||
if hasAny(genres, strings.ToUpper(value)) || strings.Contains(text, strings.ToLower(value)) {
|
||||
if hasAny(genres, strings.ToUpper(value)) {
|
||||
return true
|
||||
}
|
||||
if isDigits(value) {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(text, strings.ToLower(value)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -62,7 +72,7 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
|
||||
if isEastAsian {
|
||||
return categoryName(categories, "jk_movie", "日韩电影")
|
||||
}
|
||||
if isWestern {
|
||||
if isWesternByMetadata {
|
||||
return categoryName(categories, "euus_movie", "欧美电影")
|
||||
}
|
||||
return categoryName(categories, "foreign_movie", "外语电影")
|
||||
@@ -83,7 +93,7 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
|
||||
if hasGenre("10762", "KIDS", "儿童") {
|
||||
return categoryName(categories, "children", "儿童")
|
||||
}
|
||||
if hasGenre("16", "ANIMATION", "动画", "动漫") {
|
||||
if hasGenre("16", "ANIMATION", "动画", "动漫") || hasAnimeText {
|
||||
if isChinese {
|
||||
return categoryName(categories, "cn_anime", "国漫")
|
||||
}
|
||||
@@ -99,6 +109,8 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
|
||||
return categoryName(categories, "euus_tv", "欧美剧")
|
||||
}
|
||||
return categoryName(categories, "uncategorized_tv", "未分类")
|
||||
case "adult":
|
||||
return categoryName(categories, "adult", "成人")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -158,6 +170,63 @@ func hasAny(values map[string]struct{}, needles ...string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func containsAnyText(text string, needles ...string) bool {
|
||||
for _, needle := range needles {
|
||||
if strings.Contains(text, strings.ToLower(needle)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsHan(text string) bool {
|
||||
for _, r := range text {
|
||||
if r >= '\u4e00' && r <= '\u9fff' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsJapaneseKana(text string) bool {
|
||||
for _, r := range text {
|
||||
if (r >= '\u3040' && r <= '\u30ff') || (r >= '\u31f0' && r <= '\u31ff') {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsKoreanHangul(text string) bool {
|
||||
for _, r := range text {
|
||||
if (r >= '\uac00' && r <= '\ud7af') || (r >= '\u1100' && r <= '\u11ff') || (r >= '\u3130' && r <= '\u318f') {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsLatin(text string) bool {
|
||||
for _, r := range text {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isDigits(text string) bool {
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range text {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func categoryName(categories map[string]string, key, fallback string) string {
|
||||
if categories != nil {
|
||||
if name := strings.TrimSpace(categories[key]); name != "" {
|
||||
|
||||
@@ -57,6 +57,38 @@ func TestClassifyMediaCategoryMatchesMoviePilotStyleRules(t *testing.T) {
|
||||
},
|
||||
want: "纪录片",
|
||||
},
|
||||
{
|
||||
name: "chinese movie title without metadata",
|
||||
input: mediaClassifyInput{
|
||||
MediaType: "movie",
|
||||
Title: "流浪地球2 2023 2160p",
|
||||
},
|
||||
want: "华语电影",
|
||||
},
|
||||
{
|
||||
name: "latin movie title without metadata",
|
||||
input: mediaClassifyInput{
|
||||
MediaType: "movie",
|
||||
Title: "Dune 2021 2160p",
|
||||
},
|
||||
want: "外语电影",
|
||||
},
|
||||
{
|
||||
name: "chinese tv title without metadata",
|
||||
input: mediaClassifyInput{
|
||||
MediaType: "tv",
|
||||
Title: "狂飙 S01E01 1080p",
|
||||
},
|
||||
want: "国产剧",
|
||||
},
|
||||
{
|
||||
name: "latin tv title without metadata",
|
||||
input: mediaClassifyInput{
|
||||
MediaType: "tv",
|
||||
Title: "The Last of Us S01E01 1080p",
|
||||
},
|
||||
want: "欧美剧",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -49,15 +49,16 @@ func (o *OrganizerService) SetProbe(p *FFprobeService) { o.probe = p }
|
||||
|
||||
// OrganizeResult reports what happened.
|
||||
type OrganizeResult struct {
|
||||
Organized int `json:"organized"`
|
||||
Skipped int `json:"skipped"`
|
||||
Replaced int `json:"replaced,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
SourcePath string `json:"source_path,omitempty"`
|
||||
DestPath string `json:"dest_path,omitempty"`
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
Items []OrganizePreviewItem `json:"items,omitempty"`
|
||||
Scans []OrganizeScanSummary `json:"scans,omitempty"`
|
||||
Organized int `json:"organized"`
|
||||
Skipped int `json:"skipped"`
|
||||
Replaced int `json:"replaced,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
SourcePath string `json:"source_path,omitempty"`
|
||||
DestPath string `json:"dest_path,omitempty"`
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
Items []OrganizePreviewItem `json:"items,omitempty"`
|
||||
Scans []OrganizeScanSummary `json:"scans,omitempty"`
|
||||
Scrapes []OrganizeScrapeSummary `json:"scrapes,omitempty"`
|
||||
}
|
||||
|
||||
type OrganizePreviewItem struct {
|
||||
@@ -288,7 +289,7 @@ func (o *OrganizerService) OrganizeLibraryWithOptions(ctx context.Context, libra
|
||||
sourceRoot := o.resolveSourceRoot(ctx, lib, opts.SourcePath)
|
||||
// 目的地目录:已位于该根下的文件视为已整理;受 dest_path 覆盖与设置影响。
|
||||
baseRoot := o.resolveBaseRoot(ctx, lib, opts.DestPath)
|
||||
res := &OrganizeResult{}
|
||||
res := &OrganizeResult{SourcePath: sourceRoot, DestPath: baseRoot, DryRun: opts.DryRun}
|
||||
for i := range rows {
|
||||
// 不在源目录内的文件跳过(不属于本次「从源目录整理」的范围)。
|
||||
if !pathWithin(rows[i].Path, sourceRoot) {
|
||||
|
||||
@@ -201,6 +201,9 @@ func (o *OrganizerService) organizeSourceFile(ctx context.Context, src, sourceRo
|
||||
if layout.MediaType == "" {
|
||||
layout.MediaType = o.inferMediaTypeForSourceFile(src, title, season, episode)
|
||||
}
|
||||
if layout.Category == "" {
|
||||
layout.Category = o.smartClassifySourceFile(ctx, src, sourceRoot, layout.MediaType, title, parsedTitle)
|
||||
}
|
||||
layoutRoot := destRoot
|
||||
if layout.MediaType != "" {
|
||||
layoutRoot = o.organizeRoot(destRoot, layout.MediaType, layout.Category)
|
||||
@@ -338,6 +341,28 @@ func (o *OrganizerService) inferMediaTypeForSourceFile(src, title string, season
|
||||
return normalizeMediaType("", title, src)
|
||||
}
|
||||
|
||||
func (o *OrganizerService) smartClassifySourceFile(ctx context.Context, src, sourceRoot, mediaType, title, parsedTitle string) string {
|
||||
if o == nil || !o.isSmartClassifyEnabled(ctx) {
|
||||
return ""
|
||||
}
|
||||
seriesLike := isSeriesLibraryType(mediaType)
|
||||
input := mediaClassifyInput{
|
||||
MediaType: mediaType,
|
||||
Title: strings.Join([]string{title, parsedTitle, filepath.Base(src)}, " "),
|
||||
Category: strings.Join(organizeDirectoryCategoryCandidates(src, sourceRoot), " "),
|
||||
}
|
||||
if meta, err := ReadLocalMetadata(src, sourceRoot, seriesLike); err == nil && meta != nil && meta.HasNFO {
|
||||
input.Title = strings.Join([]string{meta.Title, meta.OriginalName, title, parsedTitle, filepath.Base(src)}, " ")
|
||||
input.Languages = parseCommaList(meta.Languages)
|
||||
input.Countries = parseCommaList(meta.Countries)
|
||||
input.Genres = parseCommaList(meta.Genres)
|
||||
if meta.NSFW {
|
||||
input.MediaType = "adult"
|
||||
}
|
||||
}
|
||||
return sanitizeFilename(classifyMediaCategory(input, o.categoryMap()))
|
||||
}
|
||||
|
||||
func (o *OrganizerService) inferOrganizeDirectoryLayout(src, sourceRoot string) organizeDirectoryLayout {
|
||||
for _, name := range organizeDirectoryCategoryCandidates(src, sourceRoot) {
|
||||
if mediaType, category := o.mediaTypeForDirectoryCategory(name); mediaType != "" && category != "" {
|
||||
|
||||
@@ -427,6 +427,78 @@ func TestOrganizeDirectoryUsesDownloadCategoryLayout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrganizeDirectorySmartClassifiesUncategorizedSources(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "downloads")
|
||||
dest := filepath.Join(root, "media")
|
||||
writeOrgFile(t, filepath.Join(src, "流浪地球2.2023.2160p.WEB-DL.mkv"), "cn-movie")
|
||||
writeOrgFile(t, filepath.Join(src, "Dune.2021.2160p.WEB-DL.mkv"), "foreign-movie")
|
||||
writeOrgFile(t, filepath.Join(src, "狂飙.S01E01.2023.1080p.WEB-DL.mkv"), "cn-tv")
|
||||
writeOrgFile(t, filepath.Join(src, "The.Last.of.Us.S01E01.2023.1080p.WEB-DL.mkv"), "western-tv")
|
||||
|
||||
repos := newOrganizerTestRepo(t)
|
||||
if err := repos.Setting.Set(t.Context(), "organizer.smart_classify", "true"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
org := NewOrganizerService(&config.Config{}, zap.NewNop(), repos)
|
||||
res, err := org.OrganizeDirectory(t.Context(), OrganizeOptions{
|
||||
SourcePath: src,
|
||||
DestPath: dest,
|
||||
TransferMode: TransferCopy,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("organize directory: %v", err)
|
||||
}
|
||||
if res.Organized != 4 {
|
||||
t.Fatalf("organized = %d, want 4; result=%+v", res.Organized, res)
|
||||
}
|
||||
|
||||
for _, want := range []string{
|
||||
filepath.Join(dest, "电影", "华语电影", "流浪地球2 (2023)", "流浪地球2 (2023).mkv"),
|
||||
filepath.Join(dest, "电影", "外语电影", "Dune (2021)", "Dune (2021).mkv"),
|
||||
filepath.Join(dest, "电视剧", "国产剧", "狂飙", "Season 01", "狂飙 - S01E01.mkv"),
|
||||
filepath.Join(dest, "电视剧", "欧美剧", "The Last Of Us", "Season 01", "The Last Of Us - S01E01.mkv"),
|
||||
} {
|
||||
if _, err := os.Stat(want); err != nil {
|
||||
t.Fatalf("expected smart classified file at %q: %v; items=%+v", want, err, res.Items)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrganizeDirectorySmartClassifiesWithLocalNFO(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "downloads")
|
||||
dest := filepath.Join(root, "media")
|
||||
writeOrgFile(t, filepath.Join(src, "Some.Show.S01E01.2024.1080p.mkv"), "jp-anime")
|
||||
writeOrgFile(t, filepath.Join(src, "tvshow.nfo"), `<tvshow>
|
||||
<title>Some Show</title>
|
||||
<genre>Animation</genre>
|
||||
<country>JP</country>
|
||||
<language>ja</language>
|
||||
</tvshow>`)
|
||||
|
||||
repos := newOrganizerTestRepo(t)
|
||||
if err := repos.Setting.Set(t.Context(), "organizer.smart_classify", "true"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
org := NewOrganizerService(&config.Config{}, zap.NewNop(), repos)
|
||||
res, err := org.OrganizeDirectory(t.Context(), OrganizeOptions{
|
||||
SourcePath: src,
|
||||
DestPath: dest,
|
||||
TransferMode: TransferCopy,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("organize directory: %v", err)
|
||||
}
|
||||
if res.Organized != 1 {
|
||||
t.Fatalf("organized = %d, want 1", res.Organized)
|
||||
}
|
||||
want := filepath.Join(dest, "电视剧", "日番", "Some Show", "Season 01", "Some Show - S01E01.mkv")
|
||||
if _, err := os.Stat(want); err != nil {
|
||||
t.Fatalf("expected NFO classified episode at %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrganizeDirectoryScanAfterRecursesNestedDownloadFolders(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "downloads")
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
"github.com/ShukeBta/MediaStationGo/internal/repository"
|
||||
)
|
||||
|
||||
// OrganizeScanSummary reports a library scan triggered after directory
|
||||
@@ -20,18 +21,53 @@ type OrganizeScanSummary struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// OrganizeScrapeSummary reports metadata enrichment triggered after organize.
|
||||
type OrganizeScrapeSummary struct {
|
||||
LibraryID string `json:"library_id"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Matched int `json:"matched"`
|
||||
Skipped bool `json:"skipped,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// OrganizeScrapeAfterEnabled decides whether organize workflows should run
|
||||
// metadata scraping after scanning organized files into libraries.
|
||||
func OrganizeScrapeAfterEnabled(ctx context.Context, repo *repository.Container) bool {
|
||||
if repo == nil || repo.Setting == nil {
|
||||
return false
|
||||
}
|
||||
if value, err := repo.Setting.Get(ctx, "organize.scrape_after"); err == nil && strings.TrimSpace(value) != "" {
|
||||
return parseBoolSetting(value, false)
|
||||
}
|
||||
if value, err := repo.Setting.Get(ctx, "scrape.auto_on_scan"); err == nil && strings.TrimSpace(value) != "" {
|
||||
return parseBoolSetting(value, false)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ScanLibrariesForPath recursively scans libraries affected by an organize
|
||||
// destination. If preferredLibraryID is set, only that library is scanned.
|
||||
// Otherwise every enabled library whose path intersects destRoot is scanned;
|
||||
// if no path can be matched, we fall back to all enabled libraries to preserve
|
||||
// the old "scan all after ingest" UI behavior.
|
||||
func (s *ScannerService) ScanLibrariesForPath(ctx context.Context, destRoot, preferredLibraryID string) []OrganizeScanSummary {
|
||||
scans, _ := s.ScanAndScrapeLibrariesForPath(ctx, destRoot, preferredLibraryID, false)
|
||||
return scans
|
||||
}
|
||||
|
||||
// ScanAndScrapeLibrariesForPath scans the affected organize target libraries,
|
||||
// then optionally runs the scraper synchronously so manual/automatic organize
|
||||
// can provide deterministic "整理 + 入库 + 刮削" behavior instead of relying on
|
||||
// a background scan hook that may or may not be enabled.
|
||||
func (s *ScannerService) ScanAndScrapeLibrariesForPath(ctx context.Context, destRoot, preferredLibraryID string, scrapeAfter bool) ([]OrganizeScanSummary, []OrganizeScrapeSummary) {
|
||||
if s == nil || s.repo == nil || s.repo.Library == nil {
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
libraries, err := s.repo.Library.List(ctx)
|
||||
if err != nil {
|
||||
return []OrganizeScanSummary{{Error: err.Error()}}
|
||||
return []OrganizeScanSummary{{Error: err.Error()}}, nil
|
||||
}
|
||||
targets := selectOrganizeScanTargets(libraries, destRoot, preferredLibraryID)
|
||||
out := make([]OrganizeScanSummary, 0, len(targets))
|
||||
@@ -41,7 +77,7 @@ func (s *ScannerService) ScanLibrariesForPath(ctx context.Context, destRoot, pre
|
||||
Name: lib.Name,
|
||||
Path: lib.Path,
|
||||
}
|
||||
res, err := s.ScanLibrary(ctx, lib.ID)
|
||||
res, err := s.scanLibrary(ctx, lib.ID, !scrapeAfter)
|
||||
if err != nil {
|
||||
summary.Error = err.Error()
|
||||
out = append(out, summary)
|
||||
@@ -53,6 +89,52 @@ func (s *ScannerService) ScanLibrariesForPath(ctx context.Context, destRoot, pre
|
||||
summary.Removed = res.Removed
|
||||
out = append(out, summary)
|
||||
}
|
||||
return out, s.scrapeOrganizeTargets(ctx, targets, scrapeAfter)
|
||||
}
|
||||
|
||||
func (s *ScannerService) scrapeOrganizeTargets(ctx context.Context, targets []model.Library, scrapeAfter bool) []OrganizeScrapeSummary {
|
||||
if len(targets) == 0 || !scrapeAfter {
|
||||
return nil
|
||||
}
|
||||
out := make([]OrganizeScrapeSummary, 0, len(targets))
|
||||
if s.scraper == nil {
|
||||
for _, lib := range targets {
|
||||
out = append(out, OrganizeScrapeSummary{
|
||||
LibraryID: lib.ID,
|
||||
Name: lib.Name,
|
||||
Path: lib.Path,
|
||||
Skipped: true,
|
||||
Reason: "scraper unavailable",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
if !s.scraper.AnyEnabled() {
|
||||
for _, lib := range targets {
|
||||
out = append(out, OrganizeScrapeSummary{
|
||||
LibraryID: lib.ID,
|
||||
Name: lib.Name,
|
||||
Path: lib.Path,
|
||||
Skipped: true,
|
||||
Reason: "no scraper provider enabled",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
for _, lib := range targets {
|
||||
summary := OrganizeScrapeSummary{
|
||||
LibraryID: lib.ID,
|
||||
Name: lib.Name,
|
||||
Path: lib.Path,
|
||||
}
|
||||
matched, err := s.scraper.EnrichLibrary(ctx, lib.ID)
|
||||
if err != nil {
|
||||
summary.Error = err.Error()
|
||||
} else {
|
||||
summary.Matched = matched
|
||||
}
|
||||
out = append(out, summary)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/config"
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
)
|
||||
|
||||
func TestOrganizeDirectoryScanAndScrapeAfter(t *testing.T) {
|
||||
scraper, repos, closeServer := newTestScraper(t)
|
||||
defer closeServer()
|
||||
if err := repos.DB.AutoMigrate(&model.Setting{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "downloads")
|
||||
dest := filepath.Join(root, "media")
|
||||
sourceFile := filepath.Join(src, "Spy.x.Family.S01E01.2022.1080p.mkv")
|
||||
writeOrgFile(t, sourceFile, "episode")
|
||||
|
||||
lib := model.Library{
|
||||
Name: "剧集",
|
||||
Path: filepath.Join(dest, "电视剧"),
|
||||
Type: "tv",
|
||||
Enabled: true,
|
||||
}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
organizer := NewOrganizerService(&config.Config{}, zap.NewNop(), repos)
|
||||
res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{
|
||||
SourcePath: src,
|
||||
DestPath: dest,
|
||||
TransferMode: TransferCopy,
|
||||
MediaType: "tv",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("organize directory: %v", err)
|
||||
}
|
||||
if res.Organized != 1 {
|
||||
t.Fatalf("organized = %d, want 1", res.Organized)
|
||||
}
|
||||
|
||||
scanner := NewScannerService(&config.Config{}, zap.NewNop(), repos, NewHub(zap.NewNop()), nil, scraper)
|
||||
scans, scrapes := scanner.ScanAndScrapeLibrariesForPath(t.Context(), res.DestPath, "", true)
|
||||
if len(scans) != 1 || scans[0].Added != 1 {
|
||||
t.Fatalf("scans = %#v, want one scan with added=1", scans)
|
||||
}
|
||||
if len(scrapes) != 1 || scrapes[0].Matched != 1 || scrapes[0].Error != "" || scrapes[0].Skipped {
|
||||
t.Fatalf("scrapes = %#v, want one successful matched scrape", scrapes)
|
||||
}
|
||||
|
||||
var media model.Media
|
||||
if err := repos.DB.Where("path LIKE ?", "%Spy Family - S01E01.mkv").First(&media).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.ScrapeStatus != "matched" || media.TMDbID != 12345 {
|
||||
t.Fatalf("media scrape status=%q tmdb=%d, want matched/12345", media.ScrapeStatus, media.TMDbID)
|
||||
}
|
||||
if _, err := os.Stat(media.Path); err != nil {
|
||||
t.Fatalf("organized file missing at %q: %v", media.Path, err)
|
||||
}
|
||||
}
|
||||
@@ -80,6 +80,10 @@ type ScanResult struct {
|
||||
|
||||
// ScanLibrary walks the library root and persists discovered media files.
|
||||
func (s *ScannerService) ScanLibrary(ctx context.Context, libraryID string) (*ScanResult, error) {
|
||||
return s.scanLibrary(ctx, libraryID, true)
|
||||
}
|
||||
|
||||
func (s *ScannerService) scanLibrary(ctx context.Context, libraryID string, autoScrape bool) (*ScanResult, error) {
|
||||
lib, err := s.repo.Library.FindByID(ctx, libraryID)
|
||||
if err != nil || lib == nil {
|
||||
return nil, err
|
||||
@@ -124,7 +128,7 @@ func (s *ScannerService) ScanLibrary(ctx context.Context, libraryID string) (*Sc
|
||||
|
||||
// Online enrichment is opt-in. Local NFO is always consumed first during
|
||||
// the scan, and matched rows are excluded from EnrichLibrary's pending set.
|
||||
if s.scraper != nil && s.scraper.AnyEnabled() && s.autoScrapeEnabled(ctx) {
|
||||
if autoScrape && s.scraper != nil && s.scraper.AnyEnabled() && s.autoScrapeEnabled(ctx) {
|
||||
go func(libID string) {
|
||||
if _, err := s.scraper.EnrichLibrary(context.Background(), libID); err != nil {
|
||||
s.log.Warn("scraper enrich failed", zap.Error(err))
|
||||
|
||||
@@ -255,7 +255,7 @@ func (s *SchedulerService) jobOrganizeSource(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if s.scanner != nil && res != nil && strings.TrimSpace(res.DestPath) != "" {
|
||||
res.Scans = s.scanner.ScanLibrariesForPath(ctx, res.DestPath, "")
|
||||
res.Scans, res.Scrapes = s.scanner.ScanAndScrapeLibrariesForPath(ctx, res.DestPath, "", OrganizeScrapeAfterEnabled(ctx, s.repo))
|
||||
}
|
||||
if s.log != nil && res != nil {
|
||||
s.log.Info("scheduled source organize finished",
|
||||
@@ -264,6 +264,7 @@ func (s *SchedulerService) jobOrganizeSource(ctx context.Context) error {
|
||||
zap.Int("organized", res.Organized),
|
||||
zap.Int("replaced", res.Replaced),
|
||||
zap.Int("skipped", res.Skipped),
|
||||
zap.Int("scrapes", len(res.Scrapes)),
|
||||
zap.Int("errors", len(res.Errors)),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ export interface OrganizeOverrides {
|
||||
transfer_mode?: string
|
||||
media_type?: string
|
||||
scan_after?: boolean
|
||||
scrape_after?: boolean
|
||||
library_id?: string
|
||||
dry_run?: boolean
|
||||
}
|
||||
@@ -76,6 +77,15 @@ export const toolsAPI = {
|
||||
removed: number
|
||||
error?: string
|
||||
}>
|
||||
scrapes?: Array<{
|
||||
library_id: string
|
||||
name: string
|
||||
path: string
|
||||
matched: number
|
||||
skipped?: boolean
|
||||
reason?: string
|
||||
error?: string
|
||||
}>
|
||||
}>(
|
||||
'/admin/organize/source',
|
||||
opts,
|
||||
|
||||
@@ -45,9 +45,19 @@ function formatScanSummary(scans: Array<{ name: string; added: number; updated:
|
||||
return ` · 扫描 ${ok.length}/${scans.length} 个库 · 新入库 ${added} · 更新 ${updated} · 访问 ${visited}`
|
||||
}
|
||||
|
||||
function formatScrapeSummary(scrapes: Array<{ name: string; matched: number; skipped?: boolean; reason?: string; error?: string }>): string {
|
||||
if (scrapes.length === 0) return ''
|
||||
const ok = scrapes.filter((scrape) => !scrape.error && !scrape.skipped)
|
||||
const skipped = scrapes.filter((scrape) => scrape.skipped).length
|
||||
const matched = ok.reduce((sum, scrape) => sum + (scrape.matched ?? 0), 0)
|
||||
if (ok.length === 0 && skipped > 0) return ` · 刮削跳过 ${skipped} 个库`
|
||||
return ` · 刮削 ${ok.length}/${scrapes.length} 个库 · 匹配 ${matched}${skipped ? ` · 跳过 ${skipped}` : ''}`
|
||||
}
|
||||
|
||||
type AutoOrganizeConfig = {
|
||||
enabled: string
|
||||
afterDownload: string
|
||||
scrapeAfter: string
|
||||
sourceDir: string
|
||||
targetDir: string
|
||||
transferMode: string
|
||||
@@ -57,6 +67,7 @@ type AutoOrganizeConfig = {
|
||||
const AUTO_ORGANIZE_DEFAULTS: AutoOrganizeConfig = {
|
||||
enabled: 'false',
|
||||
afterDownload: 'false',
|
||||
scrapeAfter: 'false',
|
||||
sourceDir: '',
|
||||
targetDir: '',
|
||||
transferMode: 'hardlink',
|
||||
@@ -66,6 +77,7 @@ const AUTO_ORGANIZE_DEFAULTS: AutoOrganizeConfig = {
|
||||
const AUTO_ORGANIZE_KEYS: Record<keyof AutoOrganizeConfig, string> = {
|
||||
enabled: 'organize.auto',
|
||||
afterDownload: 'organizer.auto_after_download',
|
||||
scrapeAfter: 'organize.scrape_after',
|
||||
sourceDir: 'organize.source_dir',
|
||||
targetDir: 'organize.target_dir',
|
||||
transferMode: 'organize.transfer_mode',
|
||||
@@ -83,6 +95,7 @@ function mergeAutoOrganizeSettings(rows: Setting[]): AutoOrganizeConfig {
|
||||
return {
|
||||
enabled: idx[AUTO_ORGANIZE_KEYS.enabled] ?? AUTO_ORGANIZE_DEFAULTS.enabled,
|
||||
afterDownload: idx[AUTO_ORGANIZE_KEYS.afterDownload] ?? AUTO_ORGANIZE_DEFAULTS.afterDownload,
|
||||
scrapeAfter: idx[AUTO_ORGANIZE_KEYS.scrapeAfter] ?? AUTO_ORGANIZE_DEFAULTS.scrapeAfter,
|
||||
sourceDir: idx[AUTO_ORGANIZE_KEYS.sourceDir] ?? AUTO_ORGANIZE_DEFAULTS.sourceDir,
|
||||
targetDir: idx[AUTO_ORGANIZE_KEYS.targetDir] ?? AUTO_ORGANIZE_DEFAULTS.targetDir,
|
||||
transferMode: idx[AUTO_ORGANIZE_KEYS.transferMode] ?? AUTO_ORGANIZE_DEFAULTS.transferMode,
|
||||
@@ -114,6 +127,7 @@ export function FileManagerPage() {
|
||||
const [organizeTransferMode, setOrganizeTransferMode] = useState('hardlink')
|
||||
const [organizeMediaType, setOrganizeMediaType] = useState('auto')
|
||||
const [scanAfter, setScanAfter] = useState(true)
|
||||
const [scrapeAfter, setScrapeAfter] = useState(false)
|
||||
const [organizeBusy, setOrganizeBusy] = useState('')
|
||||
const [previewItems, setPreviewItems] = useState<Array<{
|
||||
source: string
|
||||
@@ -160,7 +174,9 @@ export function FileManagerPage() {
|
||||
adminAPI
|
||||
.listSettings()
|
||||
.then((rows) => {
|
||||
setAutoConfig(mergeAutoOrganizeSettings(rows))
|
||||
const nextConfig = mergeAutoOrganizeSettings(rows)
|
||||
setAutoConfig(nextConfig)
|
||||
setScrapeAfter(settingOn(nextConfig.scrapeAfter))
|
||||
setAutoDirty(false)
|
||||
})
|
||||
.catch(() => undefined)
|
||||
@@ -309,7 +325,7 @@ export function FileManagerPage() {
|
||||
if (!dryRun) {
|
||||
const ok = await confirmAction({
|
||||
title: '确认整理入库',
|
||||
message: `来源:${organizeSource}\n目标:${organizeDestPath}\n方式:${organizeTransferMode}${scanAfter ? '\n整理完成后会扫描入库。' : ''}`,
|
||||
message: `来源:${organizeSource}\n目标:${organizeDestPath}\n方式:${organizeTransferMode}${scanAfter ? '\n整理完成后会扫描入库。' : ''}${scanAfter && scrapeAfter ? '\n扫描后会自动刮削。' : ''}`,
|
||||
confirmText: '开始整理',
|
||||
})
|
||||
if (!ok) return
|
||||
@@ -322,6 +338,7 @@ export function FileManagerPage() {
|
||||
transfer_mode: organizeTransferMode,
|
||||
media_type: organizeMediaType === 'auto' ? undefined : organizeMediaType,
|
||||
scan_after: !dryRun && scanAfter,
|
||||
scrape_after: !dryRun && scanAfter && scrapeAfter,
|
||||
library_id: !dryRun && scanAfter && organizeLibraryID ? organizeLibraryID : undefined,
|
||||
dry_run: dryRun,
|
||||
})
|
||||
@@ -340,7 +357,8 @@ export function FileManagerPage() {
|
||||
return
|
||||
}
|
||||
const scanText = scanAfter ? formatScanSummary(result.scans ?? []) : ''
|
||||
toast.success(`整理完成:新增 ${result.organized} · 替换 ${replaced} · 跳过 ${result.skipped}${scanText}`)
|
||||
const scrapeText = scanAfter && scrapeAfter ? formatScrapeSummary(result.scrapes ?? []) : ''
|
||||
toast.success(`整理完成:新增 ${result.organized} · 替换 ${replaced} · 跳过 ${result.skipped}${scanText}${scrapeText}`)
|
||||
refresh()
|
||||
} catch (err: unknown) {
|
||||
toast.error((err as { response?: { data?: { error?: string } } })?.response?.data?.error ?? '整理失败')
|
||||
@@ -498,6 +516,14 @@ export function FileManagerPage() {
|
||||
/>
|
||||
qB 下载完成后自动整理
|
||||
</label>
|
||||
<label className="flex items-center gap-2 rounded-lg border border-gray-200 bg-gray-50 px-2 py-1 text-xs text-ink-100">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={settingOn(autoConfig.scrapeAfter)}
|
||||
onChange={(event) => changeAutoConfig('scrapeAfter', event.target.checked ? 'true' : 'false')}
|
||||
/>
|
||||
整理后自动刮削
|
||||
</label>
|
||||
<span className="text-xs text-sand-500">
|
||||
{autoDirty ? '有未保存设置' : '设置已同步'} · 定时任务名:organize_source
|
||||
</span>
|
||||
@@ -568,6 +594,15 @@ export function FileManagerPage() {
|
||||
<input type="checkbox" checked={scanAfter} onChange={(event) => setScanAfter(event.target.checked)} />
|
||||
整理后扫描入库
|
||||
</label>
|
||||
<label className="flex items-center gap-2 rounded-lg border border-gray-200 bg-gray-50 px-2 py-1 text-xs text-ink-100">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={scanAfter && scrapeAfter}
|
||||
disabled={!scanAfter}
|
||||
onChange={(event) => setScrapeAfter(event.target.checked)}
|
||||
/>
|
||||
整理后自动刮削
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
className="neon-button !border-primary-400/30 !bg-white !text-brand-500"
|
||||
|
||||
@@ -160,6 +160,13 @@ const GROUPS: SettingGroup[] = [
|
||||
type: 'toggle',
|
||||
hint: '开启后 qB 下载完成时,系统会优先使用种子的 content_path 整理该文件/目录,并在整理完成后扫描目标媒体库。',
|
||||
},
|
||||
{
|
||||
key: 'organize.scrape_after',
|
||||
label: '整理后自动刮削',
|
||||
type: 'toggle',
|
||||
hint: '开启后,手动/自动整理完成并扫描入库后,会立即触发 TMDb/豆瓣/Bangumi/JavBus/JavDB 等元数据刮削。需要先配置可用刮削源。',
|
||||
defaultValue: 'false',
|
||||
},
|
||||
{
|
||||
key: 'downloads.smart_classify',
|
||||
label: '下载器智能分类',
|
||||
|
||||
Reference in New Issue
Block a user