feat: scrape metadata after organize

This commit is contained in:
ShukeBta
2026-06-09 17:00:24 +08:00
parent 81f0cabe91
commit 76fe0dbde9
18 changed files with 470 additions and 31 deletions
+1
View File
@@ -250,6 +250,7 @@ func setDefaults(v *viper.Viper) {
v.SetDefault("downloads.smart_classify", true)
v.SetDefault("organizer.smart_classify", false)
v.SetDefault("organizer.auto_after_download", false)
v.SetDefault("organize.scrape_after", false)
v.SetDefault("organizer.categories.chinese_movie", "华语电影")
v.SetDefault("organizer.categories.animation_movie", "动画电影")
v.SetDefault("organizer.categories.foreign_movie", "外语电影")
+3
View File
@@ -85,6 +85,9 @@ func downloadOrganizeAllHandler(svc *service.Container) gin.HandlerFunc {
results = append(results, gin.H{"library": l.Name, "error": err.Error()})
continue
}
if svc.Scan != nil && res != nil && !res.DryRun {
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, l.ID, nil)
}
results = append(results, gin.H{"library": l.Name, "result": res})
}
c.JSON(http.StatusOK, gin.H{"results": results})
+3
View File
@@ -134,6 +134,9 @@ func organizeBulkHandler(svc *service.Container) gin.HandlerFunc {
out = append(out, gin.H{"library": l.Name, "error": err.Error()})
continue
}
if svc.Scan != nil && res != nil && !res.DryRun {
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, l.ID, nil)
}
out = append(out, gin.H{"library": l.Name, "result": res})
}
c.JSON(http.StatusOK, gin.H{"results": out})
+26 -4
View File
@@ -21,6 +21,7 @@ type organizeReq struct {
TransferMode string `json:"transfer_mode"`
MediaType string `json:"media_type"`
ScanAfter bool `json:"scan_after"`
ScrapeAfter *bool `json:"scrape_after"`
LibraryID string `json:"library_id"`
DryRun bool `json:"dry_run"`
}
@@ -52,24 +53,37 @@ func organizeOptionsFromReq(req organizeReq) service.OrganizeOptions {
func organizeMediaHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
opts := bindOrganizeOptions(c)
var req organizeReq
_ = c.ShouldBindJSON(&req)
opts := organizeOptionsFromReq(req)
dst, err := svc.Organizer.OrganizeMediaWithOptions(c.Request.Context(), c.Param("id"), opts)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"path": dst})
payload := gin.H{"path": dst}
if req.ScanAfter && !req.DryRun && svc.Scan != nil {
scans, scrapes := scanAndScrapeAfterOrganize(c, svc, dst, strings.TrimSpace(req.LibraryID), req.ScrapeAfter)
payload["scans"] = scans
payload["scrapes"] = scrapes
}
c.JSON(http.StatusOK, payload)
}
}
func organizeLibraryHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
opts := bindOrganizeOptions(c)
var req organizeReq
_ = c.ShouldBindJSON(&req)
opts := organizeOptionsFromReq(req)
res, err := svc.Organizer.OrganizeLibraryWithOptions(c.Request.Context(), c.Param("id"), opts)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if req.ScanAfter && !req.DryRun && svc.Scan != nil {
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, c.Param("id"), req.ScrapeAfter)
}
c.JSON(http.StatusOK, res)
}
}
@@ -95,8 +109,16 @@ func organizeDirectoryHandler(svc *service.Container) gin.HandlerFunc {
return
}
if req.ScanAfter && !req.DryRun && svc.Scan != nil {
res.Scans = svc.Scan.ScanLibrariesForPath(c.Request.Context(), res.DestPath, strings.TrimSpace(req.LibraryID))
res.Scans, res.Scrapes = scanAndScrapeAfterOrganize(c, svc, res.DestPath, strings.TrimSpace(req.LibraryID), req.ScrapeAfter)
}
c.JSON(http.StatusOK, res)
}
}
func scanAndScrapeAfterOrganize(c *gin.Context, svc *service.Container, destRoot, preferredLibraryID string, scrapeOverride *bool) ([]service.OrganizeScanSummary, []service.OrganizeScrapeSummary) {
scrapeAfter := service.OrganizeScrapeAfterEnabled(c.Request.Context(), svc.Repo)
if scrapeOverride != nil {
scrapeAfter = *scrapeOverride
}
return svc.Scan.ScanAndScrapeLibrariesForPath(c.Request.Context(), destRoot, preferredLibraryID, scrapeAfter)
}
+1
View File
@@ -80,6 +80,7 @@ func schemaHandler(_ *service.Container) gin.HandlerFunc {
"items": []gin.H{
{"key": "organize.auto", "type": "toggle", "label": "整理源目录定时自动整理"},
{"key": "organizer.auto_after_download", "type": "toggle"},
{"key": "organize.scrape_after", "type": "toggle", "label": "整理后自动刮削"},
{"key": "downloads.smart_classify", "type": "toggle"},
{"key": "organizer.smart_classify", "type": "toggle"},
{"key": "organize.source_dir", "type": "text", "label": "整理源目录"},
+2 -1
View File
@@ -881,7 +881,7 @@ func (d *DownloadService) onTorrentComplete(ctx context.Context, torrent QBitTor
return
}
if d.scanner != nil && res != nil && strings.TrimSpace(res.DestPath) != "" {
res.Scans = d.scanner.ScanLibrariesForPath(ctx, res.DestPath, "")
res.Scans, res.Scrapes = d.scanner.ScanAndScrapeLibrariesForPath(ctx, res.DestPath, "", OrganizeScrapeAfterEnabled(ctx, d.repo))
}
d.log.Info("auto organize completed torrent finished",
zap.String("hash", torrent.Hash),
@@ -890,6 +890,7 @@ func (d *DownloadService) onTorrentComplete(ctx context.Context, torrent QBitTor
zap.Int("organized", res.Organized),
zap.Int("replaced", res.Replaced),
zap.Int("skipped", res.Skipped),
zap.Int("scrapes", len(res.Scrapes)),
zap.Int("errors", len(res.Errors)))
}
+77 -8
View File
@@ -31,20 +31,30 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
genres := normalizeTokens(input.Genres...)
countries := normalizeTokens(input.Countries...)
languages := normalizeTokens(input.Languages...)
text := strings.ToLower(input.Title + " " + input.Category + " " + strings.Join(input.Genres, " "))
rawText := input.Title + " " + input.Category + " " + strings.Join(input.Genres, " ")
text := strings.ToLower(rawText)
isChinese := hasAny(languages, "ZH", "ZH-CN", "ZH-TW", "CN") || hasAny(countries, "CN", "TW", "HK", "MO")
isJapanese := hasAny(languages, "JA", "JP") || hasAny(countries, "JP") || strings.Contains(text, "日番")
isKorean := hasAny(languages, "KO", "KR") || hasAny(countries, "KR", "KP")
isChinese := hasAny(languages, "ZH", "ZH-CN", "ZH-TW", "CN") || hasAny(countries, "CN", "TW", "HK", "MO") || containsHan(rawText) || containsAnyText(text, "华语", "国产", "国剧", "国漫")
isJapanese := hasAny(languages, "JA", "JP") || hasAny(countries, "JP") || containsJapaneseKana(rawText) || strings.Contains(text, "日番")
isKorean := hasAny(languages, "KO", "KR") || hasAny(countries, "KR", "KP") || containsKoreanHangul(rawText)
isEastAsian := isJapanese || isKorean || hasAny(countries, "TH", "IN", "SG")
isWestern := hasAny(countries,
isWesternByMetadata := hasAny(countries,
"US", "GB", "UK", "FR", "DE", "CA", "AU", "NZ", "IE", "NL", "SE", "NO", "DK",
"FI", "ES", "IT", "PT", "AT", "CH", "BE", "RU",
)
isLatinFallback := containsLatin(rawText) && !containsHan(rawText) && !containsJapaneseKana(rawText) && !containsKoreanHangul(rawText)
isWestern := isWesternByMetadata || (mediaType == "tv" && isLatinFallback)
hasAnimeText := containsAnyText(text, "动画", "动漫", "番剧", "年番", "国漫", "日番", "bangumi", "anime")
hasGenre := func(values ...string) bool {
for _, value := range values {
if hasAny(genres, strings.ToUpper(value)) || strings.Contains(text, strings.ToLower(value)) {
if hasAny(genres, strings.ToUpper(value)) {
return true
}
if isDigits(value) {
continue
}
if strings.Contains(text, strings.ToLower(value)) {
return true
}
}
@@ -62,7 +72,7 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
if isEastAsian {
return categoryName(categories, "jk_movie", "日韩电影")
}
if isWestern {
if isWesternByMetadata {
return categoryName(categories, "euus_movie", "欧美电影")
}
return categoryName(categories, "foreign_movie", "外语电影")
@@ -83,7 +93,7 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
if hasGenre("10762", "KIDS", "儿童") {
return categoryName(categories, "children", "儿童")
}
if hasGenre("16", "ANIMATION", "动画", "动漫") {
if hasGenre("16", "ANIMATION", "动画", "动漫") || hasAnimeText {
if isChinese {
return categoryName(categories, "cn_anime", "国漫")
}
@@ -99,6 +109,8 @@ func classifyMediaCategory(input mediaClassifyInput, categories map[string]strin
return categoryName(categories, "euus_tv", "欧美剧")
}
return categoryName(categories, "uncategorized_tv", "未分类")
case "adult":
return categoryName(categories, "adult", "成人")
}
return ""
}
@@ -158,6 +170,63 @@ func hasAny(values map[string]struct{}, needles ...string) bool {
return false
}
func containsAnyText(text string, needles ...string) bool {
for _, needle := range needles {
if strings.Contains(text, strings.ToLower(needle)) {
return true
}
}
return false
}
func containsHan(text string) bool {
for _, r := range text {
if r >= '\u4e00' && r <= '\u9fff' {
return true
}
}
return false
}
func containsJapaneseKana(text string) bool {
for _, r := range text {
if (r >= '\u3040' && r <= '\u30ff') || (r >= '\u31f0' && r <= '\u31ff') {
return true
}
}
return false
}
func containsKoreanHangul(text string) bool {
for _, r := range text {
if (r >= '\uac00' && r <= '\ud7af') || (r >= '\u1100' && r <= '\u11ff') || (r >= '\u3130' && r <= '\u318f') {
return true
}
}
return false
}
func containsLatin(text string) bool {
for _, r := range text {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') {
return true
}
}
return false
}
func isDigits(text string) bool {
if text == "" {
return false
}
for _, r := range text {
if r < '0' || r > '9' {
return false
}
}
return true
}
func categoryName(categories map[string]string, key, fallback string) string {
if categories != nil {
if name := strings.TrimSpace(categories[key]); name != "" {
+32
View File
@@ -57,6 +57,38 @@ func TestClassifyMediaCategoryMatchesMoviePilotStyleRules(t *testing.T) {
},
want: "纪录片",
},
{
name: "chinese movie title without metadata",
input: mediaClassifyInput{
MediaType: "movie",
Title: "流浪地球2 2023 2160p",
},
want: "华语电影",
},
{
name: "latin movie title without metadata",
input: mediaClassifyInput{
MediaType: "movie",
Title: "Dune 2021 2160p",
},
want: "外语电影",
},
{
name: "chinese tv title without metadata",
input: mediaClassifyInput{
MediaType: "tv",
Title: "狂飙 S01E01 1080p",
},
want: "国产剧",
},
{
name: "latin tv title without metadata",
input: mediaClassifyInput{
MediaType: "tv",
Title: "The Last of Us S01E01 1080p",
},
want: "欧美剧",
},
}
for _, tt := range tests {
+11 -10
View File
@@ -49,15 +49,16 @@ func (o *OrganizerService) SetProbe(p *FFprobeService) { o.probe = p }
// OrganizeResult reports what happened.
type OrganizeResult struct {
Organized int `json:"organized"`
Skipped int `json:"skipped"`
Replaced int `json:"replaced,omitempty"`
Errors []string `json:"errors,omitempty"`
SourcePath string `json:"source_path,omitempty"`
DestPath string `json:"dest_path,omitempty"`
DryRun bool `json:"dry_run,omitempty"`
Items []OrganizePreviewItem `json:"items,omitempty"`
Scans []OrganizeScanSummary `json:"scans,omitempty"`
Organized int `json:"organized"`
Skipped int `json:"skipped"`
Replaced int `json:"replaced,omitempty"`
Errors []string `json:"errors,omitempty"`
SourcePath string `json:"source_path,omitempty"`
DestPath string `json:"dest_path,omitempty"`
DryRun bool `json:"dry_run,omitempty"`
Items []OrganizePreviewItem `json:"items,omitempty"`
Scans []OrganizeScanSummary `json:"scans,omitempty"`
Scrapes []OrganizeScrapeSummary `json:"scrapes,omitempty"`
}
type OrganizePreviewItem struct {
@@ -288,7 +289,7 @@ func (o *OrganizerService) OrganizeLibraryWithOptions(ctx context.Context, libra
sourceRoot := o.resolveSourceRoot(ctx, lib, opts.SourcePath)
// 目的地目录:已位于该根下的文件视为已整理;受 dest_path 覆盖与设置影响。
baseRoot := o.resolveBaseRoot(ctx, lib, opts.DestPath)
res := &OrganizeResult{}
res := &OrganizeResult{SourcePath: sourceRoot, DestPath: baseRoot, DryRun: opts.DryRun}
for i := range rows {
// 不在源目录内的文件跳过(不属于本次「从源目录整理」的范围)。
if !pathWithin(rows[i].Path, sourceRoot) {
+25
View File
@@ -201,6 +201,9 @@ func (o *OrganizerService) organizeSourceFile(ctx context.Context, src, sourceRo
if layout.MediaType == "" {
layout.MediaType = o.inferMediaTypeForSourceFile(src, title, season, episode)
}
if layout.Category == "" {
layout.Category = o.smartClassifySourceFile(ctx, src, sourceRoot, layout.MediaType, title, parsedTitle)
}
layoutRoot := destRoot
if layout.MediaType != "" {
layoutRoot = o.organizeRoot(destRoot, layout.MediaType, layout.Category)
@@ -338,6 +341,28 @@ func (o *OrganizerService) inferMediaTypeForSourceFile(src, title string, season
return normalizeMediaType("", title, src)
}
func (o *OrganizerService) smartClassifySourceFile(ctx context.Context, src, sourceRoot, mediaType, title, parsedTitle string) string {
if o == nil || !o.isSmartClassifyEnabled(ctx) {
return ""
}
seriesLike := isSeriesLibraryType(mediaType)
input := mediaClassifyInput{
MediaType: mediaType,
Title: strings.Join([]string{title, parsedTitle, filepath.Base(src)}, " "),
Category: strings.Join(organizeDirectoryCategoryCandidates(src, sourceRoot), " "),
}
if meta, err := ReadLocalMetadata(src, sourceRoot, seriesLike); err == nil && meta != nil && meta.HasNFO {
input.Title = strings.Join([]string{meta.Title, meta.OriginalName, title, parsedTitle, filepath.Base(src)}, " ")
input.Languages = parseCommaList(meta.Languages)
input.Countries = parseCommaList(meta.Countries)
input.Genres = parseCommaList(meta.Genres)
if meta.NSFW {
input.MediaType = "adult"
}
}
return sanitizeFilename(classifyMediaCategory(input, o.categoryMap()))
}
func (o *OrganizerService) inferOrganizeDirectoryLayout(src, sourceRoot string) organizeDirectoryLayout {
for _, name := range organizeDirectoryCategoryCandidates(src, sourceRoot) {
if mediaType, category := o.mediaTypeForDirectoryCategory(name); mediaType != "" && category != "" {
@@ -427,6 +427,78 @@ func TestOrganizeDirectoryUsesDownloadCategoryLayout(t *testing.T) {
}
}
func TestOrganizeDirectorySmartClassifiesUncategorizedSources(t *testing.T) {
root := t.TempDir()
src := filepath.Join(root, "downloads")
dest := filepath.Join(root, "media")
writeOrgFile(t, filepath.Join(src, "流浪地球2.2023.2160p.WEB-DL.mkv"), "cn-movie")
writeOrgFile(t, filepath.Join(src, "Dune.2021.2160p.WEB-DL.mkv"), "foreign-movie")
writeOrgFile(t, filepath.Join(src, "狂飙.S01E01.2023.1080p.WEB-DL.mkv"), "cn-tv")
writeOrgFile(t, filepath.Join(src, "The.Last.of.Us.S01E01.2023.1080p.WEB-DL.mkv"), "western-tv")
repos := newOrganizerTestRepo(t)
if err := repos.Setting.Set(t.Context(), "organizer.smart_classify", "true"); err != nil {
t.Fatal(err)
}
org := NewOrganizerService(&config.Config{}, zap.NewNop(), repos)
res, err := org.OrganizeDirectory(t.Context(), OrganizeOptions{
SourcePath: src,
DestPath: dest,
TransferMode: TransferCopy,
})
if err != nil {
t.Fatalf("organize directory: %v", err)
}
if res.Organized != 4 {
t.Fatalf("organized = %d, want 4; result=%+v", res.Organized, res)
}
for _, want := range []string{
filepath.Join(dest, "电影", "华语电影", "流浪地球2 (2023)", "流浪地球2 (2023).mkv"),
filepath.Join(dest, "电影", "外语电影", "Dune (2021)", "Dune (2021).mkv"),
filepath.Join(dest, "电视剧", "国产剧", "狂飙", "Season 01", "狂飙 - S01E01.mkv"),
filepath.Join(dest, "电视剧", "欧美剧", "The Last Of Us", "Season 01", "The Last Of Us - S01E01.mkv"),
} {
if _, err := os.Stat(want); err != nil {
t.Fatalf("expected smart classified file at %q: %v; items=%+v", want, err, res.Items)
}
}
}
func TestOrganizeDirectorySmartClassifiesWithLocalNFO(t *testing.T) {
root := t.TempDir()
src := filepath.Join(root, "downloads")
dest := filepath.Join(root, "media")
writeOrgFile(t, filepath.Join(src, "Some.Show.S01E01.2024.1080p.mkv"), "jp-anime")
writeOrgFile(t, filepath.Join(src, "tvshow.nfo"), `<tvshow>
<title>Some Show</title>
<genre>Animation</genre>
<country>JP</country>
<language>ja</language>
</tvshow>`)
repos := newOrganizerTestRepo(t)
if err := repos.Setting.Set(t.Context(), "organizer.smart_classify", "true"); err != nil {
t.Fatal(err)
}
org := NewOrganizerService(&config.Config{}, zap.NewNop(), repos)
res, err := org.OrganizeDirectory(t.Context(), OrganizeOptions{
SourcePath: src,
DestPath: dest,
TransferMode: TransferCopy,
})
if err != nil {
t.Fatalf("organize directory: %v", err)
}
if res.Organized != 1 {
t.Fatalf("organized = %d, want 1", res.Organized)
}
want := filepath.Join(dest, "电视剧", "日番", "Some Show", "Season 01", "Some Show - S01E01.mkv")
if _, err := os.Stat(want); err != nil {
t.Fatalf("expected NFO classified episode at %q: %v", want, err)
}
}
func TestOrganizeDirectoryScanAfterRecursesNestedDownloadFolders(t *testing.T) {
root := t.TempDir()
src := filepath.Join(root, "downloads")
+85 -3
View File
@@ -5,6 +5,7 @@ import (
"strings"
"github.com/ShukeBta/MediaStationGo/internal/model"
"github.com/ShukeBta/MediaStationGo/internal/repository"
)
// OrganizeScanSummary reports a library scan triggered after directory
@@ -20,18 +21,53 @@ type OrganizeScanSummary struct {
Error string `json:"error,omitempty"`
}
// OrganizeScrapeSummary reports metadata enrichment triggered after organize.
type OrganizeScrapeSummary struct {
LibraryID string `json:"library_id"`
Name string `json:"name"`
Path string `json:"path"`
Matched int `json:"matched"`
Skipped bool `json:"skipped,omitempty"`
Reason string `json:"reason,omitempty"`
Error string `json:"error,omitempty"`
}
// OrganizeScrapeAfterEnabled decides whether organize workflows should run
// metadata scraping after scanning organized files into libraries.
func OrganizeScrapeAfterEnabled(ctx context.Context, repo *repository.Container) bool {
if repo == nil || repo.Setting == nil {
return false
}
if value, err := repo.Setting.Get(ctx, "organize.scrape_after"); err == nil && strings.TrimSpace(value) != "" {
return parseBoolSetting(value, false)
}
if value, err := repo.Setting.Get(ctx, "scrape.auto_on_scan"); err == nil && strings.TrimSpace(value) != "" {
return parseBoolSetting(value, false)
}
return false
}
// ScanLibrariesForPath recursively scans libraries affected by an organize
// destination. If preferredLibraryID is set, only that library is scanned.
// Otherwise every enabled library whose path intersects destRoot is scanned;
// if no path can be matched, we fall back to all enabled libraries to preserve
// the old "scan all after ingest" UI behavior.
func (s *ScannerService) ScanLibrariesForPath(ctx context.Context, destRoot, preferredLibraryID string) []OrganizeScanSummary {
scans, _ := s.ScanAndScrapeLibrariesForPath(ctx, destRoot, preferredLibraryID, false)
return scans
}
// ScanAndScrapeLibrariesForPath scans the affected organize target libraries,
// then optionally runs the scraper synchronously so manual/automatic organize
// can provide deterministic "整理 + 入库 + 刮削" behavior instead of relying on
// a background scan hook that may or may not be enabled.
func (s *ScannerService) ScanAndScrapeLibrariesForPath(ctx context.Context, destRoot, preferredLibraryID string, scrapeAfter bool) ([]OrganizeScanSummary, []OrganizeScrapeSummary) {
if s == nil || s.repo == nil || s.repo.Library == nil {
return nil
return nil, nil
}
libraries, err := s.repo.Library.List(ctx)
if err != nil {
return []OrganizeScanSummary{{Error: err.Error()}}
return []OrganizeScanSummary{{Error: err.Error()}}, nil
}
targets := selectOrganizeScanTargets(libraries, destRoot, preferredLibraryID)
out := make([]OrganizeScanSummary, 0, len(targets))
@@ -41,7 +77,7 @@ func (s *ScannerService) ScanLibrariesForPath(ctx context.Context, destRoot, pre
Name: lib.Name,
Path: lib.Path,
}
res, err := s.ScanLibrary(ctx, lib.ID)
res, err := s.scanLibrary(ctx, lib.ID, !scrapeAfter)
if err != nil {
summary.Error = err.Error()
out = append(out, summary)
@@ -53,6 +89,52 @@ func (s *ScannerService) ScanLibrariesForPath(ctx context.Context, destRoot, pre
summary.Removed = res.Removed
out = append(out, summary)
}
return out, s.scrapeOrganizeTargets(ctx, targets, scrapeAfter)
}
func (s *ScannerService) scrapeOrganizeTargets(ctx context.Context, targets []model.Library, scrapeAfter bool) []OrganizeScrapeSummary {
if len(targets) == 0 || !scrapeAfter {
return nil
}
out := make([]OrganizeScrapeSummary, 0, len(targets))
if s.scraper == nil {
for _, lib := range targets {
out = append(out, OrganizeScrapeSummary{
LibraryID: lib.ID,
Name: lib.Name,
Path: lib.Path,
Skipped: true,
Reason: "scraper unavailable",
})
}
return out
}
if !s.scraper.AnyEnabled() {
for _, lib := range targets {
out = append(out, OrganizeScrapeSummary{
LibraryID: lib.ID,
Name: lib.Name,
Path: lib.Path,
Skipped: true,
Reason: "no scraper provider enabled",
})
}
return out
}
for _, lib := range targets {
summary := OrganizeScrapeSummary{
LibraryID: lib.ID,
Name: lib.Name,
Path: lib.Path,
}
matched, err := s.scraper.EnrichLibrary(ctx, lib.ID)
if err != nil {
summary.Error = err.Error()
} else {
summary.Matched = matched
}
out = append(out, summary)
}
return out
}
+70
View File
@@ -0,0 +1,70 @@
package service
import (
"os"
"path/filepath"
"testing"
"go.uber.org/zap"
"github.com/ShukeBta/MediaStationGo/internal/config"
"github.com/ShukeBta/MediaStationGo/internal/model"
)
func TestOrganizeDirectoryScanAndScrapeAfter(t *testing.T) {
scraper, repos, closeServer := newTestScraper(t)
defer closeServer()
if err := repos.DB.AutoMigrate(&model.Setting{}); err != nil {
t.Fatal(err)
}
root := t.TempDir()
src := filepath.Join(root, "downloads")
dest := filepath.Join(root, "media")
sourceFile := filepath.Join(src, "Spy.x.Family.S01E01.2022.1080p.mkv")
writeOrgFile(t, sourceFile, "episode")
lib := model.Library{
Name: "剧集",
Path: filepath.Join(dest, "电视剧"),
Type: "tv",
Enabled: true,
}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
organizer := NewOrganizerService(&config.Config{}, zap.NewNop(), repos)
res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{
SourcePath: src,
DestPath: dest,
TransferMode: TransferCopy,
MediaType: "tv",
})
if err != nil {
t.Fatalf("organize directory: %v", err)
}
if res.Organized != 1 {
t.Fatalf("organized = %d, want 1", res.Organized)
}
scanner := NewScannerService(&config.Config{}, zap.NewNop(), repos, NewHub(zap.NewNop()), nil, scraper)
scans, scrapes := scanner.ScanAndScrapeLibrariesForPath(t.Context(), res.DestPath, "", true)
if len(scans) != 1 || scans[0].Added != 1 {
t.Fatalf("scans = %#v, want one scan with added=1", scans)
}
if len(scrapes) != 1 || scrapes[0].Matched != 1 || scrapes[0].Error != "" || scrapes[0].Skipped {
t.Fatalf("scrapes = %#v, want one successful matched scrape", scrapes)
}
var media model.Media
if err := repos.DB.Where("path LIKE ?", "%Spy Family - S01E01.mkv").First(&media).Error; err != nil {
t.Fatal(err)
}
if media.ScrapeStatus != "matched" || media.TMDbID != 12345 {
t.Fatalf("media scrape status=%q tmdb=%d, want matched/12345", media.ScrapeStatus, media.TMDbID)
}
if _, err := os.Stat(media.Path); err != nil {
t.Fatalf("organized file missing at %q: %v", media.Path, err)
}
}
+5 -1
View File
@@ -80,6 +80,10 @@ type ScanResult struct {
// ScanLibrary walks the library root and persists discovered media files.
func (s *ScannerService) ScanLibrary(ctx context.Context, libraryID string) (*ScanResult, error) {
return s.scanLibrary(ctx, libraryID, true)
}
func (s *ScannerService) scanLibrary(ctx context.Context, libraryID string, autoScrape bool) (*ScanResult, error) {
lib, err := s.repo.Library.FindByID(ctx, libraryID)
if err != nil || lib == nil {
return nil, err
@@ -124,7 +128,7 @@ func (s *ScannerService) ScanLibrary(ctx context.Context, libraryID string) (*Sc
// Online enrichment is opt-in. Local NFO is always consumed first during
// the scan, and matched rows are excluded from EnrichLibrary's pending set.
if s.scraper != nil && s.scraper.AnyEnabled() && s.autoScrapeEnabled(ctx) {
if autoScrape && s.scraper != nil && s.scraper.AnyEnabled() && s.autoScrapeEnabled(ctx) {
go func(libID string) {
if _, err := s.scraper.EnrichLibrary(context.Background(), libID); err != nil {
s.log.Warn("scraper enrich failed", zap.Error(err))
+2 -1
View File
@@ -255,7 +255,7 @@ func (s *SchedulerService) jobOrganizeSource(ctx context.Context) error {
return err
}
if s.scanner != nil && res != nil && strings.TrimSpace(res.DestPath) != "" {
res.Scans = s.scanner.ScanLibrariesForPath(ctx, res.DestPath, "")
res.Scans, res.Scrapes = s.scanner.ScanAndScrapeLibrariesForPath(ctx, res.DestPath, "", OrganizeScrapeAfterEnabled(ctx, s.repo))
}
if s.log != nil && res != nil {
s.log.Info("scheduled source organize finished",
@@ -264,6 +264,7 @@ func (s *SchedulerService) jobOrganizeSource(ctx context.Context) error {
zap.Int("organized", res.Organized),
zap.Int("replaced", res.Replaced),
zap.Int("skipped", res.Skipped),
zap.Int("scrapes", len(res.Scrapes)),
zap.Int("errors", len(res.Errors)),
)
}
+10
View File
@@ -12,6 +12,7 @@ export interface OrganizeOverrides {
transfer_mode?: string
media_type?: string
scan_after?: boolean
scrape_after?: boolean
library_id?: string
dry_run?: boolean
}
@@ -76,6 +77,15 @@ export const toolsAPI = {
removed: number
error?: string
}>
scrapes?: Array<{
library_id: string
name: string
path: string
matched: number
skipped?: boolean
reason?: string
error?: string
}>
}>(
'/admin/organize/source',
opts,
+38 -3
View File
@@ -45,9 +45,19 @@ function formatScanSummary(scans: Array<{ name: string; added: number; updated:
return ` · 扫描 ${ok.length}/${scans.length} 个库 · 新入库 ${added} · 更新 ${updated} · 访问 ${visited}`
}
function formatScrapeSummary(scrapes: Array<{ name: string; matched: number; skipped?: boolean; reason?: string; error?: string }>): string {
if (scrapes.length === 0) return ''
const ok = scrapes.filter((scrape) => !scrape.error && !scrape.skipped)
const skipped = scrapes.filter((scrape) => scrape.skipped).length
const matched = ok.reduce((sum, scrape) => sum + (scrape.matched ?? 0), 0)
if (ok.length === 0 && skipped > 0) return ` · 刮削跳过 ${skipped} 个库`
return ` · 刮削 ${ok.length}/${scrapes.length} 个库 · 匹配 ${matched}${skipped ? ` · 跳过 ${skipped}` : ''}`
}
type AutoOrganizeConfig = {
enabled: string
afterDownload: string
scrapeAfter: string
sourceDir: string
targetDir: string
transferMode: string
@@ -57,6 +67,7 @@ type AutoOrganizeConfig = {
const AUTO_ORGANIZE_DEFAULTS: AutoOrganizeConfig = {
enabled: 'false',
afterDownload: 'false',
scrapeAfter: 'false',
sourceDir: '',
targetDir: '',
transferMode: 'hardlink',
@@ -66,6 +77,7 @@ const AUTO_ORGANIZE_DEFAULTS: AutoOrganizeConfig = {
const AUTO_ORGANIZE_KEYS: Record<keyof AutoOrganizeConfig, string> = {
enabled: 'organize.auto',
afterDownload: 'organizer.auto_after_download',
scrapeAfter: 'organize.scrape_after',
sourceDir: 'organize.source_dir',
targetDir: 'organize.target_dir',
transferMode: 'organize.transfer_mode',
@@ -83,6 +95,7 @@ function mergeAutoOrganizeSettings(rows: Setting[]): AutoOrganizeConfig {
return {
enabled: idx[AUTO_ORGANIZE_KEYS.enabled] ?? AUTO_ORGANIZE_DEFAULTS.enabled,
afterDownload: idx[AUTO_ORGANIZE_KEYS.afterDownload] ?? AUTO_ORGANIZE_DEFAULTS.afterDownload,
scrapeAfter: idx[AUTO_ORGANIZE_KEYS.scrapeAfter] ?? AUTO_ORGANIZE_DEFAULTS.scrapeAfter,
sourceDir: idx[AUTO_ORGANIZE_KEYS.sourceDir] ?? AUTO_ORGANIZE_DEFAULTS.sourceDir,
targetDir: idx[AUTO_ORGANIZE_KEYS.targetDir] ?? AUTO_ORGANIZE_DEFAULTS.targetDir,
transferMode: idx[AUTO_ORGANIZE_KEYS.transferMode] ?? AUTO_ORGANIZE_DEFAULTS.transferMode,
@@ -114,6 +127,7 @@ export function FileManagerPage() {
const [organizeTransferMode, setOrganizeTransferMode] = useState('hardlink')
const [organizeMediaType, setOrganizeMediaType] = useState('auto')
const [scanAfter, setScanAfter] = useState(true)
const [scrapeAfter, setScrapeAfter] = useState(false)
const [organizeBusy, setOrganizeBusy] = useState('')
const [previewItems, setPreviewItems] = useState<Array<{
source: string
@@ -160,7 +174,9 @@ export function FileManagerPage() {
adminAPI
.listSettings()
.then((rows) => {
setAutoConfig(mergeAutoOrganizeSettings(rows))
const nextConfig = mergeAutoOrganizeSettings(rows)
setAutoConfig(nextConfig)
setScrapeAfter(settingOn(nextConfig.scrapeAfter))
setAutoDirty(false)
})
.catch(() => undefined)
@@ -309,7 +325,7 @@ export function FileManagerPage() {
if (!dryRun) {
const ok = await confirmAction({
title: '确认整理入库',
message: `来源:${organizeSource}\n目标:${organizeDestPath}\n方式:${organizeTransferMode}${scanAfter ? '\n整理完成后会扫描入库。' : ''}`,
message: `来源:${organizeSource}\n目标:${organizeDestPath}\n方式:${organizeTransferMode}${scanAfter ? '\n整理完成后会扫描入库。' : ''}${scanAfter && scrapeAfter ? '\n扫描后会自动刮削。' : ''}`,
confirmText: '开始整理',
})
if (!ok) return
@@ -322,6 +338,7 @@ export function FileManagerPage() {
transfer_mode: organizeTransferMode,
media_type: organizeMediaType === 'auto' ? undefined : organizeMediaType,
scan_after: !dryRun && scanAfter,
scrape_after: !dryRun && scanAfter && scrapeAfter,
library_id: !dryRun && scanAfter && organizeLibraryID ? organizeLibraryID : undefined,
dry_run: dryRun,
})
@@ -340,7 +357,8 @@ export function FileManagerPage() {
return
}
const scanText = scanAfter ? formatScanSummary(result.scans ?? []) : ''
toast.success(`整理完成:新增 ${result.organized} · 替换 ${replaced} · 跳过 ${result.skipped}${scanText}`)
const scrapeText = scanAfter && scrapeAfter ? formatScrapeSummary(result.scrapes ?? []) : ''
toast.success(`整理完成:新增 ${result.organized} · 替换 ${replaced} · 跳过 ${result.skipped}${scanText}${scrapeText}`)
refresh()
} catch (err: unknown) {
toast.error((err as { response?: { data?: { error?: string } } })?.response?.data?.error ?? '整理失败')
@@ -498,6 +516,14 @@ export function FileManagerPage() {
/>
qB 下载完成后自动整理
</label>
<label className="flex items-center gap-2 rounded-lg border border-gray-200 bg-gray-50 px-2 py-1 text-xs text-ink-100">
<input
type="checkbox"
checked={settingOn(autoConfig.scrapeAfter)}
onChange={(event) => changeAutoConfig('scrapeAfter', event.target.checked ? 'true' : 'false')}
/>
整理后自动刮削
</label>
<span className="text-xs text-sand-500">
{autoDirty ? '有未保存设置' : '设置已同步'} · 定时任务名:organize_source
</span>
@@ -568,6 +594,15 @@ export function FileManagerPage() {
<input type="checkbox" checked={scanAfter} onChange={(event) => setScanAfter(event.target.checked)} />
整理后扫描入库
</label>
<label className="flex items-center gap-2 rounded-lg border border-gray-200 bg-gray-50 px-2 py-1 text-xs text-ink-100">
<input
type="checkbox"
checked={scanAfter && scrapeAfter}
disabled={!scanAfter}
onChange={(event) => setScrapeAfter(event.target.checked)}
/>
整理后自动刮削
</label>
<button
type="button"
className="neon-button !border-primary-400/30 !bg-white !text-brand-500"
+7
View File
@@ -160,6 +160,13 @@ const GROUPS: SettingGroup[] = [
type: 'toggle',
hint: '开启后 qB 下载完成时,系统会优先使用种子的 content_path 整理该文件/目录,并在整理完成后扫描目标媒体库。',
},
{
key: 'organize.scrape_after',
label: '整理后自动刮削',
type: 'toggle',
hint: '开启后,手动/自动整理完成并扫描入库后,会立即触发 TMDb/豆瓣/Bangumi/JavBus/JavDB 等元数据刮削。需要先配置可用刮削源。',
defaultValue: 'false',
},
{
key: 'downloads.smart_classify',
label: '下载器智能分类',