refactor: split modules and harden scraping workflows

This commit is contained in:
ShukeBta
2026-06-24 11:59:18 +08:00
parent efca3cbe69
commit 192f35d9fa
470 changed files with 47957 additions and 33839 deletions
+131 -540
View File
@@ -8,7 +8,6 @@ package service
import (
"context"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@@ -33,6 +32,24 @@ type ScraperService struct {
adult *AdultProvider
hub *Hub
notify *NotifyChannelService
cache *RuntimeCacheService
images *ImageProxy
}
type ScrapeOptions struct {
RetryNoMatch bool
IncludeMatched bool
EpisodeArtwork *bool
DeferEpisodeDetails bool
}
func (o ScrapeOptions) episodeArtworkEnabled() bool {
return o.EpisodeArtwork == nil || *o.EpisodeArtwork
}
func skipEpisodeArtworkOptions(retryNoMatch bool) ScrapeOptions {
episodeArtwork := false
return ScrapeOptions{RetryNoMatch: retryNoMatch, EpisodeArtwork: &episodeArtwork}
}
// NewScraperService is the constructor.
@@ -67,148 +84,34 @@ func (s *ScraperService) SetNotifyChannels(notify *NotifyChannelService) {
}
}
// yearPattern extracts a 4-digit year (1900-2099).
var yearPattern = regexp.MustCompile(`(?:^|[^\d])(19\d{2}|20\d{2})(?:[^\d]|$)`)
func (s *ScraperService) SetRuntimeCache(cache *RuntimeCacheService) *ScraperService {
if s != nil {
s.cache = cache
}
return s
}
func (s *ScraperService) SetImageProxy(images *ImageProxy) *ScraperService {
if s != nil {
s.images = images
}
return s
}
var tmdbDetailsTimeout = 8 * time.Second
// noiseTokens are stripped before search.
var noiseTokens = []string{
// 视频规格
"1080p", "2160p", "4k", "720p", "480p", "uhd", "ds4k", "fhd",
"bd", "bdrip", "brrip", "dvd", "dvdrip", "hdtv", "pdtv", "webdl",
"hdrip", "bluray", "blu-ray", "webrip", "web-dl", "web",
"x264", "x265", "h264", "h265", "hevc", "avc", "10bit", "8bit", "hi10p", "hi10",
"hdr", "hdr10", "sdr", "dts", "ddp", "ddp5", "dd5", "dd2", "eac3", "truehd",
"dovi", "atmos", "aac", "ac3", "flac",
"remux", "extended", "uncut", "remastered", "repack", "proper", "internal",
"limited", "imax", "directors-cut", "directors_cut",
"hkfree", "yify", "rarbg", "ettv", "fgt", "tgx", "ctrlhd", "ntb", "flux",
// 流媒体平台 / 字幕组 / 国家版本(动漫常见)
"netflix", "nf", "amzn", "hulu", "disney", "max", "hbo",
"linetv", "ourtv", "iqiyi", "youku", "bilibili", "qiyi", "krj",
"crunchyroll", "funimation", "anidb", "horriblesubs", "subsplease",
"erai-raws", "judas", "asw", "smcat", "leopard-raws", "ohys-raws", "colortv",
// 中文字幕标记
"zm", "zw", "ch", "chs", "cht", "cn", "tc", "sc",
"中字", "繁字", "简中", "繁中", "国语", "粤语", "日语",
// 季数前缀残留 — ParseEpisode 已抽取过
"season", "264", "265",
}
var noiseTokenSet = func() map[string]struct{} {
set := make(map[string]struct{}, len(noiseTokens)+1)
for _, token := range noiseTokens {
set[token] = struct{}{}
}
set["dl"] = struct{}{}
return set
}()
var releaseBoundaryTokenSet = map[string]struct{}{
"1080p": {}, "2160p": {}, "4k": {}, "720p": {}, "480p": {}, "uhd": {}, "fhd": {},
"bd": {}, "bdrip": {}, "brrip": {}, "dvd": {}, "dvdrip": {}, "hdtv": {}, "pdtv": {},
"webdl": {}, "hdrip": {}, "bluray": {}, "webrip": {}, "web": {}, "remux": {},
"x264": {}, "x265": {}, "h264": {}, "h265": {}, "hevc": {}, "avc": {},
}
var strictSeasonFolderPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)^(?:s|season)\.?\s*(\d{1,2})$`),
regexp.MustCompile(`^第\s*([0-9一二三四五六七八九十百零两]+)\s*季$`),
}
// bracketedTag matches "[anything]", "(anything)" or "{anything}" segments.
var bracketedTag = regexp.MustCompile(`[\[\(\{][^\]\)\}]*[\]\)\}]`)
var multiWordNoise = []*regexp.Regexp{
regexp.MustCompile(`(?i)\bweb[\s._-]*dl\b`),
regexp.MustCompile(`(?i)\bblu[\s._-]*ray\b`),
regexp.MustCompile(`(?i)\bdirectors[\s._-]*cut\b`),
regexp.MustCompile(`(?i)\berai[\s._-]*raws\b`),
regexp.MustCompile(`(?i)\bohys[\s._-]*raws\b`),
}
const (
defaultScrapeDelayMinMS = 250
defaultScrapeDelayMaxMS = 500
maxScrapeDelayMS = 5 * 60 * 1000
)
// CleanQuery converts a filename like "Inception.2010.1080p.BluRay.x264.mkv"
// into a TMDb-friendly title plus an optional year hint.
func CleanQuery(raw string) (title string, year int) {
name := strings.TrimSuffix(filepath.Base(raw), filepath.Ext(raw))
lower := strings.ToLower(name)
if m := yearPattern.FindStringSubmatch(lower); len(m) >= 2 {
if v, err := strconv.Atoi(m[1]); err == nil {
year = v
lower = strings.ReplaceAll(lower, m[1], " ")
}
}
lower = bracketedTag.ReplaceAllString(lower, " ")
lower = patSEnE.ReplaceAllString(lower, " ")
lower = patNxE.ReplaceAllString(lower, " ")
lower = patEP.ReplaceAllString(lower, " ")
lower = patCN.ReplaceAllString(lower, " ")
// 去掉中文季/部标记(如「第二季」「第2部」),避免残留在标题里既污染
// 搜索查询又导致整理后的目录名重复季信息。
lower = patSeasonOnly.ReplaceAllString(lower, " ")
lower = patCNSeason.ReplaceAllString(lower, " ")
for _, pat := range multiWordNoise {
lower = pat.ReplaceAllString(lower, " ")
}
for _, sep := range []string{".", "_", "-", "[", "]", "(", ")", "×"} {
lower = strings.ReplaceAll(lower, sep, " ")
}
// 拆分后丢掉过短(≤1)且全为 ASCII 数字 / 字母的"碎片",避免
// 「2」「0」「v」之类残留干扰 TMDb 搜索。中文字符不算碎片。
out := make([]string, 0, 8)
seenReleaseBoundary := false
for _, w := range strings.Fields(lower) {
if _, ok := noiseTokenSet[w]; ok {
if _, boundary := releaseBoundaryTokenSet[w]; boundary {
seenReleaseBoundary = true
}
continue
}
if seenReleaseBoundary && isASCIIWord(w) {
continue
}
if len(w) <= 1 {
r := []rune(w)
if len(r) == 1 && r[0] < 128 {
continue
}
}
out = append(out, w)
}
title = strings.TrimSpace(strings.Join(out, " "))
return title, year
}
func isASCIIWord(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if r >= 128 {
return false
}
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
continue
}
return false
}
return true
}
// EnrichOne runs the provider chain for a single media row.
func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
return s.EnrichOneWithOptions(ctx, m, ScrapeOptions{})
}
func (s *ScraperService) EnrichOneWithOptions(ctx context.Context, m *model.Media, options ScrapeOptions) error {
lib, err := s.repo.Library.FindByID(ctx, m.LibraryID)
if err != nil {
return err
@@ -220,12 +123,14 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
if !cloudMedia {
if found, err := ReadLocalMetadata(m.Path, lib.Path, seriesLike); err == nil && found != nil {
local = found
applyLocalMetadata(m, local)
} else if err != nil {
s.log.Warn("read local metadata before scrape failed", zap.String("media_id", m.ID), zap.Error(err))
}
} else if hinted, _ := pathHintMetadata(m.Path, seriesLike); hinted != nil {
local = mergeCloudMetadata(local, hinted)
}
if hinted, _ := pathHintMetadata(m.Path, seriesLike); hinted != nil {
local = mergeScrapePathHintMetadata(local, hinted)
}
if local != nil {
applyLocalMetadata(m, local)
}
@@ -235,7 +140,7 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
if code := firstText(localAdultCode(local), AdultCodeFromMediaPath(m.Path), normalizeAdultCode(m.OriginalName), normalizeAdultCode(m.Title)); code != "" {
if adultMatch, err := s.adult.Search(ctx, code); err == nil && adultMatch != nil {
mergeLocalMetadataIntoMatch(adultMatch, local)
return s.applyProviderMatch(ctx, m, lib, adultMatch)
return s.applyProviderMatchWithOptions(ctx, m, lib, adultMatch, options)
} else if err != nil {
s.log.Debug("adult metadata search failed", zap.String("media_id", m.ID), zap.String("code", code), zap.Error(err))
}
@@ -245,7 +150,7 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
if match := s.matchFromMediaExternalIDs(ctx, m, lib); match != nil {
s.applyFanartArtwork(ctx, match)
mergeLocalMetadataIntoMatch(match, local)
return s.applyProviderMatch(ctx, m, lib, match)
return s.applyProviderMatchWithOptions(ctx, m, lib, match, options)
}
candidates := scrapeQueryCandidates(m, lib)
@@ -253,7 +158,7 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
match := (*Match)(nil)
for _, candidate := range candidates {
query = candidate
candidateMatch := s.lookup(ctx, lib, candidate, year)
candidateMatch := s.lookup(ctx, lib, m, candidate, year)
if candidateMatch == nil {
continue
}
@@ -276,11 +181,12 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
}
}
if match == nil {
if local != nil {
if local != nil && !local.PathHint {
return s.applyLocalMetadataMatch(ctx, m, local)
}
_ = s.repo.DB.Model(&model.Media{}).Where("id = ?", m.ID).
Update("scrape_status", "no_match").Error
s.invalidateMediaCache(ctx)
s.log.Info("metadata scrape no match",
zap.String("media_id", m.ID),
zap.String("query", query),
@@ -290,7 +196,7 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
s.applyFanartArtwork(ctx, match)
mergeLocalMetadataIntoMatch(match, local)
return s.applyProviderMatch(ctx, m, lib, match)
return s.applyProviderMatchWithOptions(ctx, m, lib, match, options)
}
func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model.Media, lib *model.Library) *Match {
@@ -400,6 +306,10 @@ func mergeLocalMetadataIntoMatch(match *Match, local *LocalMetadata) {
if match == nil || local == nil {
return
}
if local.PathHint {
mergePathHintIDsIntoMatch(match, local)
return
}
if local.Title != "" {
match.Title = local.Title
}
@@ -451,12 +361,41 @@ func mergeLocalMetadataIntoMatch(match *Match, local *LocalMetadata) {
}
}
func mergePathHintIDsIntoMatch(match *Match, local *LocalMetadata) {
if match == nil || local == nil {
return
}
if local.TMDbID > 0 {
match.TMDbID = local.TMDbID
}
if local.BangumiID > 0 {
match.BangumiID = local.BangumiID
}
if local.DoubanID != "" {
match.DoubanID = local.DoubanID
}
if local.TheTVDBID != "" {
match.TheTVDBID = local.TheTVDBID
}
if match.Year <= 0 && local.Year > 0 {
match.Year = local.Year
}
}
func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media, lib *model.Library, match *Match) error {
return s.applyProviderMatchWithOptions(ctx, m, lib, match, ScrapeOptions{})
}
func (s *ScraperService) applyProviderMatchWithOptions(ctx context.Context, m *model.Media, lib *model.Library, match *Match, options ScrapeOptions) error {
posterCandidate := match.PosterURL
backdropCandidate := match.BackdropURL
posterURL, removePoster := s.prepareScrapedArtworkURL(ctx, m.ID, "poster_url", m.PosterURL, posterCandidate)
backdropURL, removeBackdrop := s.prepareScrapedArtworkURL(ctx, m.ID, "backdrop_url", m.BackdropURL, backdropCandidate)
updates := map[string]any{
"title": match.Title,
"overview": match.Overview,
"poster_url": match.PosterURL,
"backdrop_url": match.BackdropURL,
"poster_url": posterURL,
"backdrop_url": backdropURL,
"rating": match.Rating,
"year": match.Year,
"scrape_status": "matched",
@@ -464,6 +403,9 @@ func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media,
if match.OriginalName != "" {
updates["original_name"] = match.OriginalName
}
if strings.TrimSpace(m.EpisodeTitle) != "" {
updates["episode_title"] = strings.TrimSpace(m.EpisodeTitle)
}
if match.TMDbID > 0 {
updates["tm_db_id"] = match.TMDbID
}
@@ -493,104 +435,22 @@ func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media,
Updates(updates).Error; err != nil {
return err
}
s.removeCachedScrapedArtwork(removePoster, removeBackdrop)
// Fetch extended metadata after the selected match is already saved.
// Manual cloud/batch applies must not fail just because an optional provider
// details request is slow or unavailable.
if match.TMDbID > 0 && s.tmdb != nil && s.tmdb.Enabled() {
mediaType := s.determineMediaTypeForMedia(lib, m, match)
detailCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), tmdbDetailsTimeout)
details, err := s.tmdb.GetDetails(detailCtx, match.TMDbID, mediaType)
cancel()
if err != nil {
s.log.Warn("failed to get details from tmdb",
zap.Int("tmdb_id", match.TMDbID),
zap.String("type", mediaType),
zap.Error(err))
} else if details != nil {
detailUpdates := map[string]any{}
if len(details.Languages) > 0 {
detailUpdates["languages"] = strings.Join(details.Languages, ",")
}
if len(details.Countries) > 0 {
detailUpdates["countries"] = strings.Join(details.Countries, ",")
}
if len(details.Genres) > 0 {
detailUpdates["genres"] = strings.Join(details.Genres, ",")
}
if len(detailUpdates) > 0 {
if err := s.repo.DB.Model(&model.Media{}).Where("id = ?", m.ID).
Updates(detailUpdates).Error; err != nil {
s.log.Warn("failed to save tmdb extended metadata",
zap.String("media_id", m.ID),
zap.Int("tmdb_id", match.TMDbID),
zap.Error(err))
}
}
s.log.Debug("enrich: saved extended metadata",
zap.String("media_id", m.ID),
zap.Strings("languages", details.Languages),
zap.Strings("countries", details.Countries),
zap.Strings("genres", details.Genres))
}
if mediaType == "tv" && m != nil && m.EpisodeNum > 0 {
episodeCtx, episodeCancel := context.WithTimeout(context.WithoutCancel(ctx), tmdbDetailsTimeout)
episode, err := s.tmdb.GetTVEpisodeDetails(episodeCtx, match.TMDbID, m.SeasonNum, m.EpisodeNum)
episodeCancel()
if err != nil {
s.log.Debug("failed to get tmdb episode details",
zap.String("media_id", m.ID),
zap.Int("tmdb_id", match.TMDbID),
zap.Int("season", m.SeasonNum),
zap.Int("episode", m.EpisodeNum),
zap.Error(err))
} else if episode != nil {
episodeUpdates := map[string]any{}
// 注意: 不要把 episode.Name(单集名,如"觉醒"/"Pilot"/"第1集")写入
// original_name —— 该字段是「整剧原名」,是合集分组键的回退依据。
// 若每集都写成各自的单集名,同一部剧的各集 original_name 互不相同,
// 会被前端 getSeriesKey / 后端 mediaVersionGroupKey 拆成多个独立卡片,
// 导致同剧无法合并成合集。单集名属于单集信息,这里只回填不影响合集
// 分组的单集字段(简介/剧照/评分/时长)。
if strings.TrimSpace(episode.Overview) != "" {
episodeUpdates["overview"] = strings.TrimSpace(episode.Overview)
}
if strings.TrimSpace(episode.StillURL) != "" {
episodeUpdates["backdrop_url"] = strings.TrimSpace(episode.StillURL)
}
if episode.Rating > 0 {
episodeUpdates["rating"] = episode.Rating
}
if episode.AirYear > 0 && match.Year <= 0 {
episodeUpdates["year"] = episode.AirYear
}
if episode.Runtime > 0 && m.DurationSec <= 0 {
episodeUpdates["duration_sec"] = episode.Runtime * 60
}
if len(episodeUpdates) > 0 {
if err := s.repo.DB.Model(&model.Media{}).Where("id = ?", m.ID).
Updates(episodeUpdates).Error; err != nil {
s.log.Warn("failed to save tmdb episode metadata",
zap.String("media_id", m.ID),
zap.Int("tmdb_id", match.TMDbID),
zap.Int("season", m.SeasonNum),
zap.Int("episode", m.EpisodeNum),
zap.Error(err))
}
}
}
s.fetchAndSaveTMDbExtendedMetadata(ctx, m.ID, match.TMDbID, mediaType)
if mediaType == "tv" && !options.DeferEpisodeDetails {
s.fetchAndSaveTMDbEpisodeDetails(ctx, m, match.TMDbID, match.Year, options)
}
}
cloudMedia := isCloudMediaPath(m.Path) || (lib != nil && isCloudMediaPath(lib.Path))
if !cloudMedia {
if refreshed, err := s.repo.Media.FindByID(ctx, m.ID); err == nil && refreshed != nil {
if path, err := WriteMediaNFO(refreshed); err != nil {
s.log.Warn("write nfo after scrape failed", zap.String("media_id", m.ID), zap.Error(err))
} else {
s.log.Debug("write nfo after scrape", zap.String("media_id", m.ID), zap.String("path", path))
}
}
if !(options.DeferEpisodeDetails && m != nil && m.EpisodeNum > 0) {
s.writeMediaNFOAfterScrape(ctx, m, lib)
}
s.invalidateMediaCache(ctx)
s.hub.Publish("scrape", map[string]any{
"media_id": m.ID,
"title": match.Title,
@@ -603,6 +463,38 @@ func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media,
return nil
}
func mergeScrapePathHintMetadata(dst, src *LocalMetadata) *LocalMetadata {
if src == nil {
return dst
}
if dst == nil {
return cloneLocalMetadata(src)
}
hasLocalMetadata := localMetadataMarksMatched(dst)
if dst.Title == "" && src.Title != "" {
dst.Title = src.Title
}
if dst.Year <= 0 && src.Year > 0 {
dst.Year = src.Year
}
if src.TMDbID > 0 {
dst.TMDbID = src.TMDbID
}
if src.BangumiID > 0 {
dst.BangumiID = src.BangumiID
}
if strings.TrimSpace(src.DoubanID) != "" {
dst.DoubanID = strings.TrimSpace(src.DoubanID)
}
if strings.TrimSpace(src.TheTVDBID) != "" {
dst.TheTVDBID = strings.TrimSpace(src.TheTVDBID)
}
if !hasLocalMetadata {
dst.PathHint = dst.PathHint || src.PathHint
}
return dst
}
func isCloudMediaPath(value string) bool {
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(value)), "cloud://")
}
@@ -621,6 +513,9 @@ func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.M
if next.OriginalName != "" {
updates["original_name"] = next.OriginalName
}
if next.EpisodeTitle != "" {
updates["episode_title"] = next.EpisodeTitle
}
if next.Overview != "" {
updates["overview"] = next.Overview
}
@@ -670,6 +565,7 @@ func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.M
Where("id = ?", m.ID).Updates(updates).Error; err != nil {
return err
}
s.invalidateMediaCache(ctx)
s.hub.Publish("scrape", map[string]any{
"media_id": m.ID,
"title": next.Title,
@@ -680,316 +576,11 @@ func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.M
return nil
}
func scrapeQueryCandidates(m *model.Media, lib *model.Library) []string {
seen := map[string]struct{}{}
var out []string
add := func(raw string) {
cleaned, _ := CleanQuery(raw)
if cleaned == "" {
cleaned = strings.TrimSpace(raw)
}
for _, candidate := range titleCandidates(cleaned) {
key := strings.ToLower(candidate)
if _, ok := seen[key]; ok || candidate == "" {
continue
}
seen[key] = struct{}{}
out = append(out, candidate)
}
func (s *ScraperService) invalidateMediaCache(ctx context.Context) {
if s != nil && s.cache != nil {
s.cache.DeletePrefix(ctx, "media:")
s.cache.DeletePrefix(ctx, "stats:")
}
if lib != nil && mediaIsEpisodic(m, lib) {
add(seriesFolderTitle(m.Path, lib.Path))
}
add(m.Title)
add(m.Path)
if len(out) == 0 {
out = append(out, strings.TrimSuffix(filepath.Base(m.Path), filepath.Ext(m.Path)))
}
return out
}
func seriesFolderTitle(mediaPath, libraryRoot string) string {
dir := filepath.Dir(mediaPath)
if strictSeasonFolderMatched(filepath.Base(dir)) {
dir = filepath.Dir(dir)
}
if libraryRoot != "" && samePath(dir, filepath.Clean(libraryRoot)) {
return ""
}
base := filepath.Base(dir)
if base == "." || base == string(filepath.Separator) {
return ""
}
if isGenericMediaCategoryFolder(base) {
return ""
}
return base
}
func isGenericMediaCategoryFolder(name string) bool {
key := strings.ToLower(strings.TrimSpace(name))
key = strings.Trim(key, `\/`)
switch key {
case "",
"电影", "movies", "movie",
"电视剧", "剧集", "tv", "shows", "series",
"动漫", "动画", "anime", "bangumi",
"国产剧", "国剧", "大陆剧", "国产电视剧",
"欧美剧", "欧美电视剧",
"日韩剧", "日剧", "韩剧",
"华语电影", "国产电影", "大陆电影",
"外语电影", "欧美电影", "日韩电影",
"动画电影", "动漫电影",
"国漫", "国产动漫", "日番", "日漫", "日本动漫", "日本动画",
"综艺", "真人秀",
"纪录片", "纪录",
"儿童", "少儿",
"成人", "番号", "9kg",
"未分类", "uncategorized":
return true
default:
return false
}
}
func strictSeasonFolder(name string) int {
if season, ok := seasonFromDir(name); ok {
return season
}
return 0
}
func strictSeasonFolderMatched(name string) bool {
_, ok := seasonFromDir(name)
return ok
}
func titleCandidates(title string) []string {
title = strings.Join(strings.Fields(strings.TrimSpace(title)), " ")
if title == "" {
return nil
}
out := make([]string, 0, 2)
if cjk := cjkTitleOnly(title); cjk != "" {
out = append(out, cjk)
if cjk != title {
return out
}
}
out = append(out, title)
return out
}
func cjkTitleOnly(title string) string {
parts := make([]string, 0, 4)
for _, field := range strings.Fields(title) {
if containsCJK(field) {
parts = append(parts, field)
}
}
return strings.Join(parts, " ")
}
func containsCJK(s string) bool {
for _, r := range s {
switch {
case r >= '\u3400' && r <= '\u4dbf':
return true
case r >= '\u4e00' && r <= '\u9fff':
return true
case r >= '\uf900' && r <= '\ufaff':
return true
}
}
return false
}
func mediaIsEpisodic(m *model.Media, lib *model.Library) bool {
if m != nil && (m.SeasonNum > 0 || m.EpisodeNum > 0) {
return true
}
return librarySupportsSeasons(lib)
}
func librarySupportsSeasons(lib *model.Library) bool {
if lib == nil {
return false
}
switch strings.ToLower(strings.TrimSpace(lib.Type)) {
case "tv", "anime", "variety", "show", "shows":
return true
default:
return false
}
}
// lookup runs the provider chain after local NFO has been considered:
// TMDb -> Douban -> Bangumi -> TheTVDB. Douban and Bangumi do not require API
// keys; providers that are unavailable or return an error are skipped.
func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, query string, year int) *Match {
kind := ""
if lib != nil {
kind = lib.Type
}
if s.tmdb != nil && s.tmdb.Enabled() {
// anime / tv 先用 TMDb /search/tv(剧名通常是 TV 类目)。
if kind == "anime" || kind == "tv" || kind == "variety" || kind == "show" || kind == "shows" {
if m, err := s.tmdb.SearchTV(ctx, query, year); err == nil && m != nil {
return m
} else if err != nil {
s.log.Debug("tmdb tv search failed", zap.String("query", query), zap.Error(err))
}
}
if m, err := s.tmdb.SearchMovie(ctx, query, year); err == nil && m != nil {
return m
} else if err != nil {
s.log.Debug("tmdb movie search failed", zap.String("query", query), zap.Error(err))
}
}
if s.douban != nil && s.douban.Enabled() {
if m, err := s.douban.SearchMatch(ctx, query); err == nil && m != nil {
return m
} else if err != nil {
s.log.Debug("douban search failed", zap.String("query", query), zap.Error(err))
}
}
if s.bangumi != nil && s.bangumi.Enabled() {
if m, err := s.bangumi.Search(ctx, query); err == nil && m != nil {
return m
} else if err != nil {
s.log.Debug("bangumi search failed", zap.String("query", query), zap.Error(err))
}
}
if (kind == "anime" || kind == "tv" || kind == "variety" || kind == "show" || kind == "shows") && s.thetvdb != nil && s.thetvdb.Enabled() {
if m, err := s.thetvdb.SearchSeries(ctx, query); err == nil && m != nil {
return m
} else if err != nil {
s.log.Debug("thetvdb search failed", zap.String("query", query), zap.Error(err))
}
}
return nil
}
// EnrichLibrary runs the provider chain for every pending media in a library.
// When retryNoMatch is true it also retries rows previously marked no_match,
// which is the expected behaviour for a manual "重新刮削" action. Scanner-driven
// automatic enrichment keeps the default false path to avoid repeated scraping.
//
// Pending status includes both the canonical "pending" string and the
// empty / NULL values, because MediaRepository.Upsert can wipe the GORM
// default when re-running a scan over an already-existing row.
func (s *ScraperService) EnrichLibrary(ctx context.Context, libraryID string, retryNoMatch ...bool) (int, error) {
var rows []model.Media
statusFilter := "scrape_status IS NULL OR scrape_status = '' OR scrape_status = ?"
statusArgs := []any{"pending"}
if len(retryNoMatch) > 0 && retryNoMatch[0] {
statusFilter += " OR scrape_status = ?"
statusArgs = append(statusArgs, "no_match")
}
q := s.repo.DB.Where(statusFilter, statusArgs...)
if libraryID != "" {
q = q.Where("library_id = ?", libraryID)
}
if err := q.Find(&rows).Error; err != nil {
return 0, err
}
matched := 0
processed := 0
for i := range rows {
select {
case <-ctx.Done():
return matched, ctx.Err()
default:
}
if err := s.EnrichOne(ctx, &rows[i]); err != nil {
s.log.Warn("enrich failed", zap.String("media", rows[i].ID), zap.Error(err))
s.notifyScrapeFailed(rows[i], err)
continue
}
processed++
if s.mediaIsMatched(ctx, rows[i].ID) {
matched++
}
if i < len(rows)-1 {
if delay := s.scrapeDelay(ctx); delay > 0 {
select {
case <-ctx.Done():
return matched, ctx.Err()
case <-time.After(delay):
}
}
}
}
s.hub.Publish("scrape", map[string]any{
"library_id": libraryID,
"finished": true,
"matched": matched,
"processed": processed,
})
return matched, nil
}
func (s *ScraperService) notifyScrapeFailed(m model.Media, err error) {
if s == nil || s.notify == nil || err == nil {
return
}
body := strings.TrimSpace(m.Title)
if body == "" {
body = m.Path
}
body = "媒体:" + body + "\n错误:" + err.Error()
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
s.notify.Broadcast(ctx, "MediaStationGo 刮削失败", body, EventScrapeFailed)
}()
}
func (s *ScraperService) scrapeDelay(ctx context.Context) time.Duration {
minMS := s.scrapeDelaySetting(ctx, "scrape.delay_min_ms", defaultScrapeDelayMinMS)
maxMS := s.scrapeDelaySetting(ctx, "scrape.delay_max_ms", defaultScrapeDelayMaxMS)
if minMS < 0 {
minMS = 0
}
if maxMS < 0 {
maxMS = 0
}
if minMS > maxScrapeDelayMS {
minMS = maxScrapeDelayMS
}
if maxMS > maxScrapeDelayMS {
maxMS = maxScrapeDelayMS
}
if maxMS < minMS {
maxMS = minMS
}
if maxMS == 0 {
return 0
}
if maxMS == minMS {
return time.Duration(minMS) * time.Millisecond
}
return time.Duration(minMS+secureRandomIntn(maxMS-minMS+1)) * time.Millisecond
}
func (s *ScraperService) scrapeDelaySetting(ctx context.Context, key string, fallback int) int {
if s == nil || s.repo == nil || s.repo.Setting == nil {
return fallback
}
value, err := s.repo.Setting.Get(ctx, key)
if err != nil || strings.TrimSpace(value) == "" {
return fallback
}
return parseIntSettingDefault(strings.TrimSpace(value), fallback)
}
func (s *ScraperService) mediaIsMatched(ctx context.Context, mediaID string) bool {
var status string
err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
Select("scrape_status").
Where("id = ?", mediaID).
Scan(&status).Error
return err == nil && status == "matched"
}
// AnyEnabled reports whether at least one provider can run.