tcjlj/MeBox
1
0
Fork 0
mirror of https://github.com/truewhile/MeBox.git synced 2026-10-05 21:06:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor scraper service helpers

Browse Source
This commit is contained in:
ShukeBta
2026-06-26 14:08:37 +08:00
parent c08460baa7
commit 09c4e9252c
4 changed files with 392 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/service/scraper.go
-373
View File
@@ -7,8 +7,6 @@ package service
import ( import (
"context" "context"
"path/filepath"
"strconv"
"strings" "strings"
"time" "time"
@@ -36,22 +34,6 @@ type ScraperService struct {
images *ImageProxy images *ImageProxy
} }
type ScrapeOptions struct {
RetryNoMatch bool
IncludeMatched bool
EpisodeArtwork *bool
DeferEpisodeDetails bool
}
func (o ScrapeOptions) episodeArtworkEnabled() bool {
return o.EpisodeArtwork == nil || *o.EpisodeArtwork
}
func skipEpisodeArtworkOptions(retryNoMatch bool) ScrapeOptions {
episodeArtwork := false
return ScrapeOptions{RetryNoMatch: retryNoMatch, EpisodeArtwork: &episodeArtwork}
}
// NewScraperService is the constructor. // NewScraperService is the constructor.
func NewScraperService( func NewScraperService(
cfg *config.Config, cfg *config.Config,
@@ -200,241 +182,6 @@ func (s *ScraperService) EnrichOneWithOptions(ctx context.Context, m *model.Medi
return s.applyProviderMatchWithOptions(ctx, m, lib, match, options) return s.applyProviderMatchWithOptions(ctx, m, lib, match, options)
} }
func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model.Media, lib *model.Library) *Match {
if s == nil || m == nil {
return nil
}
mediaType := ""
if lib != nil {
mediaType = lib.Type
}
if m.TMDbID > 0 {
if mediaIsEpisodic(m, lib) {
mediaType = "tv"
}
if match := s.manualTMDbMatchByID(ctx, m.TMDbID, normalizeMediaType(mediaType, m.Title, "")); match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "tmdb") {
return match
}
}
}
if strings.TrimSpace(m.DoubanID) != "" && s.douban != nil && s.douban.Enabled() {
if match, err := s.douban.GetMatchByID(ctx, strings.TrimSpace(m.DoubanID)); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "douban") {
return match
}
} else if err != nil {
s.log.Debug("douban id lookup failed", zap.String("media_id", m.ID), zap.String("douban_id", m.DoubanID), zap.Error(err))
}
}
if m.BangumiID > 0 && s.bangumi != nil && s.bangumi.Enabled() {
if match, err := s.bangumi.GetSubject(ctx, m.BangumiID); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "bangumi") {
return match
}
} else if err != nil {
s.log.Debug("bangumi id lookup failed", zap.String("media_id", m.ID), zap.Int("bangumi_id", m.BangumiID), zap.Error(err))
}
}
if strings.TrimSpace(m.TheTVDBID) != "" && s.thetvdb != nil && s.thetvdb.Enabled() {
if match, err := s.thetvdb.GetSeriesMatchByID(ctx, strings.TrimSpace(m.TheTVDBID)); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "thetvdb") {
return match
}
} else if err != nil {
s.log.Debug("thetvdb id lookup failed", zap.String("media_id", m.ID), zap.String("thetvdb_id", m.TheTVDBID), zap.Error(err))
}
}
return nil
}
func (s *ScraperService) mediaExternalIDMatchTrusted(m *model.Media, lib *model.Library, match *Match, source string) bool {
if match == nil || strings.TrimSpace(match.Title) == "" {
return false
}
if !mediaIsEpisodic(m, lib) {
return true
}
for _, candidate := range scrapeQueryCandidates(m, lib) {
if unsafeAutomaticEpisodeQuery(candidate) {
continue
}
if automaticMetadataTitleTrusted(candidate, match) {
return true
}
}
if s != nil && s.log != nil && m != nil {
s.log.Warn("episode external id match rejected",
zap.String("media_id", m.ID),
zap.String("source", source),
zap.String("title", match.Title),
zap.String("original_name", match.OriginalName),
zap.Int("tmdb_id", match.TMDbID),
zap.Int("bangumi_id", match.BangumiID),
zap.String("douban_id", match.DoubanID),
zap.String("thetvdb_id", match.TheTVDBID))
}
return false
}
func (s *ScraperService) applyFanartArtwork(ctx context.Context, match *Match) {
if s == nil || s.fanart == nil || !s.fanart.Enabled() || match == nil {
return
}
apply := func(a *Artwork) {
if a == nil {
return
}
if a.Poster != "" {
match.PosterURL = a.Poster
}
if a.Backdrop != "" {
match.BackdropURL = a.Backdrop
}
}
if match.TMDbID > 0 {
if a, err := s.fanart.MovieArtwork(ctx, match.TMDbID); err == nil {
apply(a)
} else {
s.log.Debug("fanart movie artwork failed", zap.Int("tmdb_id", match.TMDbID), zap.Error(err))
}
}
if strings.TrimSpace(match.TheTVDBID) != "" {
if a, err := s.fanart.TVArtwork(ctx, strings.TrimSpace(match.TheTVDBID)); err == nil {
apply(a)
} else {
s.log.Debug("fanart tv artwork failed", zap.String("thetvdb_id", match.TheTVDBID), zap.Error(err))
}
}
}
func mediaYearHint(m *model.Media) int {
if m == nil {
return 0
}
if m.SeasonNum > 0 || m.EpisodeNum > 0 {
return seriesPathYearHint(m.Path)
}
if season, episode := ParseEpisode(m.Path); season > 0 || episode > 0 {
return seriesPathYearHint(m.Path)
}
if m.Year > 0 {
return m.Year
}
if _, year := CleanQuery(filepath.Base(m.Path)); year > 0 {
return year
}
return yearFromText(m.Path)
}
func seriesPathYearHint(path string) int {
if showDir := showDirFromEpisodePath(path); showDir != "" {
if _, year := CleanQuery(showDir); year > 0 {
return year
}
}
return 0
}
func yearFromText(raw string) int {
if raw == "" {
return 0
}
matches := yearPattern.FindStringSubmatch(strings.ToLower(raw))
if len(matches) < 2 {
return 0
}
year, _ := strconv.Atoi(matches[1])
return year
}
func localAdultCode(local *LocalMetadata) string {
if local == nil {
return ""
}
return local.AdultCode
}
func mergeLocalMetadataIntoMatch(match *Match, local *LocalMetadata) {
if match == nil || local == nil {
return
}
if local.PathHint {
mergePathHintIDsIntoMatch(match, local)
return
}
if local.Title != "" {
match.Title = local.Title
}
if local.OriginalName != "" {
match.OriginalName = local.OriginalName
}
if local.AdultCode != "" {
match.OriginalName = local.AdultCode
match.NSFW = true
}
if local.Overview != "" {
match.Overview = local.Overview
}
if local.PosterURL != "" {
match.PosterURL = local.PosterURL
}
if local.BackdropURL != "" {
match.BackdropURL = local.BackdropURL
}
if local.Rating > 0 {
match.Rating = local.Rating
}
if local.Year > 0 {
match.Year = local.Year
}
if local.TMDbID > 0 {
match.TMDbID = local.TMDbID
}
if local.BangumiID > 0 {
match.BangumiID = local.BangumiID
}
if local.DoubanID != "" {
match.DoubanID = local.DoubanID
}
if local.TheTVDBID != "" {
match.TheTVDBID = local.TheTVDBID
}
if local.Genres != "" {
match.Genres = splitNFOList(local.Genres)
}
if local.Countries != "" {
match.Countries = splitNFOList(local.Countries)
}
if local.Languages != "" {
match.Languages = splitNFOList(local.Languages)
}
if local.NSFW {
match.NSFW = true
}
}
func mergePathHintIDsIntoMatch(match *Match, local *LocalMetadata) {
if match == nil || local == nil {
return
}
if local.TMDbID > 0 {
match.TMDbID = local.TMDbID
}
if local.BangumiID > 0 {
match.BangumiID = local.BangumiID
}
if local.DoubanID != "" {
match.DoubanID = local.DoubanID
}
if local.TheTVDBID != "" {
match.TheTVDBID = local.TheTVDBID
}
if match.Year <= 0 && local.Year > 0 {
match.Year = local.Year
}
}
func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media, lib *model.Library, match *Match) error { func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media, lib *model.Library, match *Match) error {
return s.applyProviderMatchWithOptions(ctx, m, lib, match, ScrapeOptions{}) return s.applyProviderMatchWithOptions(ctx, m, lib, match, ScrapeOptions{})
} }
@@ -516,126 +263,6 @@ func (s *ScraperService) applyProviderMatchWithOptions(ctx context.Context, m *m
return nil return nil
} }
func mergeScrapePathHintMetadata(dst, src *LocalMetadata) *LocalMetadata {
if src == nil {
return dst
}
if dst == nil {
return cloneLocalMetadata(src)
}
hasLocalMetadata := localMetadataMarksMatched(dst)
if dst.Title == "" && src.Title != "" {
dst.Title = src.Title
}
if dst.Year <= 0 && src.Year > 0 {
dst.Year = src.Year
}
if src.TMDbID > 0 {
dst.TMDbID = src.TMDbID
}
if src.BangumiID > 0 {
dst.BangumiID = src.BangumiID
}
if strings.TrimSpace(src.DoubanID) != "" {
dst.DoubanID = strings.TrimSpace(src.DoubanID)
}
if strings.TrimSpace(src.TheTVDBID) != "" {
dst.TheTVDBID = strings.TrimSpace(src.TheTVDBID)
}
if !hasLocalMetadata {
dst.PathHint = dst.PathHint || src.PathHint
}
return dst
}
func isCloudMediaPath(value string) bool {
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(value)), "cloud://")
}
func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.Media, local *LocalMetadata) error {
next := *m
applyLocalMetadata(&next, local)
status := "matched"
if !localMetadataMarksMatched(local) {
status = "pending"
}
updates := map[string]any{
"title": next.Title,
"scrape_status": status,
}
if next.OriginalName != "" {
updates["original_name"] = next.OriginalName
}
if next.EpisodeTitle != "" {
updates["episode_title"] = next.EpisodeTitle
}
if next.Overview != "" {
updates["overview"] = next.Overview
}
if next.PosterURL != "" {
updates["poster_url"] = next.PosterURL
}
if next.BackdropURL != "" {
updates["backdrop_url"] = next.BackdropURL
}
if next.Rating > 0 {
updates["rating"] = next.Rating
}
if next.Year > 0 {
updates["year"] = next.Year
}
if next.TMDbID > 0 {
updates["tm_db_id"] = next.TMDbID
}
if next.BangumiID > 0 {
updates["bangumi_id"] = next.BangumiID
}
if next.DoubanID != "" {
updates["douban_id"] = next.DoubanID
}
if next.TheTVDBID != "" {
updates["thetvdb_id"] = next.TheTVDBID
}
if next.SeasonNum > 0 || next.EpisodeNum > 0 {
updates["season_num"] = next.SeasonNum
}
if next.EpisodeNum > 0 {
updates["episode_num"] = next.EpisodeNum
}
if next.Genres != "" {
updates["genres"] = next.Genres
}
if next.Countries != "" {
updates["countries"] = next.Countries
}
if next.Languages != "" {
updates["languages"] = next.Languages
}
if next.NSFW {
updates["nsfw"] = true
}
if err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
Where("id = ?", m.ID).Updates(updates).Error; err != nil {
return err
}
s.invalidateMediaCache(ctx)
s.hub.Publish("scrape", map[string]any{
"media_id": m.ID,
"title": next.Title,
"tmdb_id": next.TMDbID,
"douban_id": next.DoubanID,
"source": "local_nfo",
})
return nil
}
func (s *ScraperService) invalidateMediaCache(ctx context.Context) {
if s != nil && s.cache != nil {
s.cache.DeletePrefix(ctx, "media:")
s.cache.DeletePrefix(ctx, "stats:")
}
}
// AnyEnabled reports whether at least one provider can run. // AnyEnabled reports whether at least one provider can run.
func (s *ScraperService) AnyEnabled() bool { func (s *ScraperService) AnyEnabled() bool {
if s.tmdb != nil && s.tmdb.Enabled() { if s.tmdb != nil && s.tmdb.Enabled() {
internal/service/scraper_local_metadata.go
+257
View File
@@ -0,0 +1,257 @@
package service
import (
"context"
"path/filepath"
"strconv"
"strings"
"github.com/ShukeBta/MediaStationGo/internal/model"
)
func mediaYearHint(m *model.Media) int {
if m == nil {
return 0
}
if m.SeasonNum > 0 || m.EpisodeNum > 0 {
return seriesPathYearHint(m.Path)
}
if season, episode := ParseEpisode(m.Path); season > 0 || episode > 0 {
return seriesPathYearHint(m.Path)
}
if m.Year > 0 {
return m.Year
}
if _, year := CleanQuery(filepath.Base(m.Path)); year > 0 {
return year
}
return yearFromText(m.Path)
}
func seriesPathYearHint(path string) int {
if showDir := showDirFromEpisodePath(path); showDir != "" {
if _, year := CleanQuery(showDir); year > 0 {
return year
}
}
return 0
}
func yearFromText(raw string) int {
if raw == "" {
return 0
}
matches := yearPattern.FindStringSubmatch(strings.ToLower(raw))
if len(matches) < 2 {
return 0
}
year, _ := strconv.Atoi(matches[1])
return year
}
func localAdultCode(local *LocalMetadata) string {
if local == nil {
return ""
}
return local.AdultCode
}
func mergeLocalMetadataIntoMatch(match *Match, local *LocalMetadata) {
if match == nil || local == nil {
return
}
if local.PathHint {
mergePathHintIDsIntoMatch(match, local)
return
}
if local.Title != "" {
match.Title = local.Title
}
if local.OriginalName != "" {
match.OriginalName = local.OriginalName
}
if local.AdultCode != "" {
match.OriginalName = local.AdultCode
match.NSFW = true
}
if local.Overview != "" {
match.Overview = local.Overview
}
if local.PosterURL != "" {
match.PosterURL = local.PosterURL
}
if local.BackdropURL != "" {
match.BackdropURL = local.BackdropURL
}
if local.Rating > 0 {
match.Rating = local.Rating
}
if local.Year > 0 {
match.Year = local.Year
}
if local.TMDbID > 0 {
match.TMDbID = local.TMDbID
}
if local.BangumiID > 0 {
match.BangumiID = local.BangumiID
}
if local.DoubanID != "" {
match.DoubanID = local.DoubanID
}
if local.TheTVDBID != "" {
match.TheTVDBID = local.TheTVDBID
}
if local.Genres != "" {
match.Genres = splitNFOList(local.Genres)
}
if local.Countries != "" {
match.Countries = splitNFOList(local.Countries)
}
if local.Languages != "" {
match.Languages = splitNFOList(local.Languages)
}
if local.NSFW {
match.NSFW = true
}
}
func mergePathHintIDsIntoMatch(match *Match, local *LocalMetadata) {
if match == nil || local == nil {
return
}
if local.TMDbID > 0 {
match.TMDbID = local.TMDbID
}
if local.BangumiID > 0 {
match.BangumiID = local.BangumiID
}
if local.DoubanID != "" {
match.DoubanID = local.DoubanID
}
if local.TheTVDBID != "" {
match.TheTVDBID = local.TheTVDBID
}
if match.Year <= 0 && local.Year > 0 {
match.Year = local.Year
}
}
func mergeScrapePathHintMetadata(dst, src *LocalMetadata) *LocalMetadata {
if src == nil {
return dst
}
if dst == nil {
return cloneLocalMetadata(src)
}
hasLocalMetadata := localMetadataMarksMatched(dst)
if dst.Title == "" && src.Title != "" {
dst.Title = src.Title
}
if dst.Year <= 0 && src.Year > 0 {
dst.Year = src.Year
}
if src.TMDbID > 0 {
dst.TMDbID = src.TMDbID
}
if src.BangumiID > 0 {
dst.BangumiID = src.BangumiID
}
if strings.TrimSpace(src.DoubanID) != "" {
dst.DoubanID = strings.TrimSpace(src.DoubanID)
}
if strings.TrimSpace(src.TheTVDBID) != "" {
dst.TheTVDBID = strings.TrimSpace(src.TheTVDBID)
}
if !hasLocalMetadata {
dst.PathHint = dst.PathHint || src.PathHint
}
return dst
}
func isCloudMediaPath(value string) bool {
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(value)), "cloud://")
}
func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.Media, local *LocalMetadata) error {
next := *m
applyLocalMetadata(&next, local)
status := "matched"
if !localMetadataMarksMatched(local) {
status = "pending"
}
updates := map[string]any{
"title": next.Title,
"scrape_status": status,
}
if next.OriginalName != "" {
updates["original_name"] = next.OriginalName
}
if next.EpisodeTitle != "" {
updates["episode_title"] = next.EpisodeTitle
}
if next.Overview != "" {
updates["overview"] = next.Overview
}
if next.PosterURL != "" {
updates["poster_url"] = next.PosterURL
}
if next.BackdropURL != "" {
updates["backdrop_url"] = next.BackdropURL
}
if next.Rating > 0 {
updates["rating"] = next.Rating
}
if next.Year > 0 {
updates["year"] = next.Year
}
if next.TMDbID > 0 {
updates["tm_db_id"] = next.TMDbID
}
if next.BangumiID > 0 {
updates["bangumi_id"] = next.BangumiID
}
if next.DoubanID != "" {
updates["douban_id"] = next.DoubanID
}
if next.TheTVDBID != "" {
updates["thetvdb_id"] = next.TheTVDBID
}
if next.SeasonNum > 0 || next.EpisodeNum > 0 {
updates["season_num"] = next.SeasonNum
}
if next.EpisodeNum > 0 {
updates["episode_num"] = next.EpisodeNum
}
if next.Genres != "" {
updates["genres"] = next.Genres
}
if next.Countries != "" {
updates["countries"] = next.Countries
}
if next.Languages != "" {
updates["languages"] = next.Languages
}
if next.NSFW {
updates["nsfw"] = true
}
if err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
Where("id = ?", m.ID).Updates(updates).Error; err != nil {
return err
}
s.invalidateMediaCache(ctx)
s.hub.Publish("scrape", map[string]any{
"media_id": m.ID,
"title": next.Title,
"tmdb_id": next.TMDbID,
"douban_id": next.DoubanID,
"source": "local_nfo",
})
return nil
}
func (s *ScraperService) invalidateMediaCache(ctx context.Context) {
if s != nil && s.cache != nil {
s.cache.DeletePrefix(ctx, "media:")
s.cache.DeletePrefix(ctx, "stats:")
}
}
internal/service/scraper_lookup.go
+118
View File
@@ -0,0 +1,118 @@
package service
import (
"context"
"strings"
"go.uber.org/zap"
"github.com/ShukeBta/MediaStationGo/internal/model"
)
func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model.Media, lib *model.Library) *Match {
if s == nil || m == nil {
return nil
}
mediaType := ""
if lib != nil {
mediaType = lib.Type
}
if m.TMDbID > 0 {
if mediaIsEpisodic(m, lib) {
mediaType = "tv"
}
if match := s.manualTMDbMatchByID(ctx, m.TMDbID, normalizeMediaType(mediaType, m.Title, "")); match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "tmdb") {
return match
}
}
}
if strings.TrimSpace(m.DoubanID) != "" && s.douban != nil && s.douban.Enabled() {
if match, err := s.douban.GetMatchByID(ctx, strings.TrimSpace(m.DoubanID)); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "douban") {
return match
}
} else if err != nil {
s.log.Debug("douban id lookup failed", zap.String("media_id", m.ID), zap.String("douban_id", m.DoubanID), zap.Error(err))
}
}
if m.BangumiID > 0 && s.bangumi != nil && s.bangumi.Enabled() {
if match, err := s.bangumi.GetSubject(ctx, m.BangumiID); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "bangumi") {
return match
}
} else if err != nil {
s.log.Debug("bangumi id lookup failed", zap.String("media_id", m.ID), zap.Int("bangumi_id", m.BangumiID), zap.Error(err))
}
}
if strings.TrimSpace(m.TheTVDBID) != "" && s.thetvdb != nil && s.thetvdb.Enabled() {
if match, err := s.thetvdb.GetSeriesMatchByID(ctx, strings.TrimSpace(m.TheTVDBID)); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "thetvdb") {
return match
}
} else if err != nil {
s.log.Debug("thetvdb id lookup failed", zap.String("media_id", m.ID), zap.String("thetvdb_id", m.TheTVDBID), zap.Error(err))
}
}
return nil
}
func (s *ScraperService) mediaExternalIDMatchTrusted(m *model.Media, lib *model.Library, match *Match, source string) bool {
if match == nil || strings.TrimSpace(match.Title) == "" {
return false
}
if !mediaIsEpisodic(m, lib) {
return true
}
for _, candidate := range scrapeQueryCandidates(m, lib) {
if unsafeAutomaticEpisodeQuery(candidate) {
continue
}
if automaticMetadataTitleTrusted(candidate, match) {
return true
}
}
if s != nil && s.log != nil && m != nil {
s.log.Warn("episode external id match rejected",
zap.String("media_id", m.ID),
zap.String("source", source),
zap.String("title", match.Title),
zap.String("original_name", match.OriginalName),
zap.Int("tmdb_id", match.TMDbID),
zap.Int("bangumi_id", match.BangumiID),
zap.String("douban_id", match.DoubanID),
zap.String("thetvdb_id", match.TheTVDBID))
}
return false
}
func (s *ScraperService) applyFanartArtwork(ctx context.Context, match *Match) {
if s == nil || s.fanart == nil || !s.fanart.Enabled() || match == nil {
return
}
apply := func(a *Artwork) {
if a == nil {
return
}
if a.Poster != "" {
match.PosterURL = a.Poster
}
if a.Backdrop != "" {
match.BackdropURL = a.Backdrop
}
}
if match.TMDbID > 0 {
if a, err := s.fanart.MovieArtwork(ctx, match.TMDbID); err == nil {
apply(a)
} else {
s.log.Debug("fanart movie artwork failed", zap.Int("tmdb_id", match.TMDbID), zap.Error(err))
}
}
if strings.TrimSpace(match.TheTVDBID) != "" {
if a, err := s.fanart.TVArtwork(ctx, strings.TrimSpace(match.TheTVDBID)); err == nil {
apply(a)
} else {
s.log.Debug("fanart tv artwork failed", zap.String("thetvdb_id", match.TheTVDBID), zap.Error(err))
}
}
}
internal/service/scraper_options.go
+17
View File
@@ -0,0 +1,17 @@
package service
type ScrapeOptions struct {
RetryNoMatch bool
IncludeMatched bool
EpisodeArtwork *bool
DeferEpisodeDetails bool
}
func (o ScrapeOptions) episodeArtworkEnabled() bool {
return o.EpisodeArtwork == nil || *o.EpisodeArtwork
}
func skipEpisodeArtworkOptions(retryNoMatch bool) ScrapeOptions {
episodeArtwork := false
return ScrapeOptions{RetryNoMatch: retryNoMatch, EpisodeArtwork: &episodeArtwork}
}