diff --git a/internal/service/adult_scraper_parse.go b/internal/service/adult_scraper_parse.go index 1b38d49..9827e34 100644 --- a/internal/service/adult_scraper_parse.go +++ b/internal/service/adult_scraper_parse.go @@ -23,6 +23,7 @@ func parseAdultDetailHTML(body, code, source, detailURL string) *Match { } match := &Match{ + Provider: strings.ToLower(strings.TrimSpace(source)), OriginalName: code, MediaType: "adult", NSFW: true, diff --git a/internal/service/metatube.go b/internal/service/metatube.go index b27375e..4cae2c9 100644 --- a/internal/service/metatube.go +++ b/internal/service/metatube.go @@ -310,6 +310,7 @@ func (p *MetaTubeProvider) convertSearchResultToMatch(cfg MetaTubeConfig, query } return &Match{ + Provider: "metatube", MediaType: "adult", Title: formattedTitle, OriginalName: code, @@ -383,6 +384,7 @@ func (p *MetaTubeProvider) convertMovieInfoToMatch(cfg MetaTubeConfig, movie *Me } return &Match{ + Provider: "metatube", MediaType: "adult", Title: formattedTitle, OriginalName: code, diff --git a/internal/service/metatube_test.go b/internal/service/metatube_test.go index 4eb32f7..040bdc2 100644 --- a/internal/service/metatube_test.go +++ b/internal/service/metatube_test.go @@ -62,6 +62,9 @@ func TestMetaTubeProviderSearch(t *testing.T) { t.Fatalf("expected 1 match, got %d", len(matches)) } m := matches[0] + if m.Provider != "metatube" { + t.Errorf("expected provider metatube, got %q", m.Provider) + } if m.OriginalName != "IPX-235" { t.Errorf("expected original name IPX-235, got %s", m.OriginalName) } diff --git a/internal/service/scraper.go b/internal/service/scraper.go index d0b1058..d608c91 100644 --- a/internal/service/scraper.go +++ b/internal/service/scraper.go @@ -93,7 +93,11 @@ func (s *ScraperService) EnrichOneWithOptions(ctx context.Context, m *model.Medi } if match == nil { if local != nil && !local.PathHint { - return s.applyLocalMetadataMatch(ctx, m, local) + err := s.applyLocalMetadataMatch(ctx, m, local) + if err == nil { + options.recordProvider("local") + } + return err } if err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).Where("id = ?", m.ID). Update("scrape_status", "no_match").Error; err != nil { @@ -227,6 +231,11 @@ func (s *ScraperService) applyProviderMatchWithOptions(ctx context.Context, m *m "thetvdb_id": match.TheTVDBID, "source": map[bool]string{true: "adult"}[match.NSFW], }) + provider := strings.ToLower(strings.TrimSpace(match.Provider)) + if provider == "" { + provider = "unknown" + } + options.recordProvider(provider) return nil } diff --git a/internal/service/scraper_artwork.go b/internal/service/scraper_artwork.go index eab2a72..3934e29 100644 --- a/internal/service/scraper_artwork.go +++ b/internal/service/scraper_artwork.go @@ -2,6 +2,7 @@ package service import ( "context" + "net/url" "strings" "time" @@ -20,10 +21,34 @@ func (s *ScraperService) prepareScrapedArtworkURL(ctx context.Context, mediaID, if s == nil || s.images == nil || !isHTTPish(candidate) { return candidate, "" } - fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + originalSource := metaTubeOriginalArtworkURL(candidate) + timeout := 15 * time.Second + if originalSource != "" { + // MetaTube face processing can become very slow under a batch scrape. + // Fail over quickly so the same source can be processed locally. + timeout = 8 * time.Second + } + fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) err := s.images.PrefetchRemote(fetchCtx, candidate) cancel() if err != nil { + if originalSource != "" { + fallbackCtx, fallbackCancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + fallbackErr := s.images.PrefetchRemote(fallbackCtx, originalSource) + fallbackCancel() + if fallbackErr == nil { + s.log.Warn("MetaTube artwork processing failed; using local face-aware crop", + zap.String("media_id", mediaID), + zap.String("field", field), + zap.String("candidate", candidate), + zap.String("source", originalSource), + zap.Error(err)) + if current != "" && isHTTPish(current) { + return originalSource, current + } + return originalSource, "" + } + } if current != "" { s.log.Warn("scrape artwork prefetch failed; keeping existing artwork", zap.String("media_id", mediaID), @@ -46,6 +71,18 @@ func (s *ScraperService) prepareScrapedArtworkURL(ctx context.Context, mediaID, return candidate, "" } +func metaTubeOriginalArtworkURL(raw string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || !strings.Contains(parsed.Path, "/v1/images/primary/") { + return "" + } + source := strings.TrimSpace(parsed.Query().Get("url")) + if !isHTTPish(source) { + return "" + } + return source +} + func (s *ScraperService) removeCachedScrapedArtwork(urls ...string) { if s == nil || s.images == nil { return diff --git a/internal/service/scraper_artwork_test.go b/internal/service/scraper_artwork_test.go index 001b987..a6d12ca 100644 --- a/internal/service/scraper_artwork_test.go +++ b/internal/service/scraper_artwork_test.go @@ -3,8 +3,11 @@ package service import ( "bytes" "errors" + "image" + "image/color" "io" "net/http" + "net/url" "os" "path/filepath" "strings" @@ -67,10 +70,14 @@ func TestApplyProviderMatchInvalidatesMediaCache(t *testing.T) { if err := repos.DB.Create(&media).Error; err != nil { t.Fatal(err) } - match := &Match{Title: "Matched", PosterURL: "https://image.tmdb.org/t/p/w500/poster.jpg", BackdropURL: "https://image.tmdb.org/t/p/w1280/backdrop.jpg"} - if err := scraper.applyProviderMatch(t.Context(), &media, &lib, match); err != nil { + match := &Match{Provider: "metatube", Title: "Matched", PosterURL: "https://image.tmdb.org/t/p/w500/poster.jpg", BackdropURL: "https://image.tmdb.org/t/p/w1280/backdrop.jpg"} + usedProvider := "" + if err := scraper.applyProviderMatchWithOptions(t.Context(), &media, &lib, match, ScrapeOptions{resultProvider: &usedProvider}); err != nil { t.Fatal(err) } + if usedProvider != "metatube" { + t.Fatalf("recorded provider = %q, want metatube", usedProvider) + } var stale map[string]string if cache.GetJSON(t.Context(), "media:list:stale", &stale) { @@ -145,6 +152,92 @@ func TestApplyProviderMatchKeepsExistingArtworkWhenNewPrefetchFails(t *testing.T } } +func TestApplyProviderMatchFallsBackToLocalFaceCropWhenMetaTubeImageFails(t *testing.T) { + scraper, repos, closeServer := newTestScraper(t) + defer closeServer() + + wideCover := createTestImage( + 900, + 600, + color.RGBA{R: 255, A: 255}, + color.RGBA{B: 255, A: 255}, + ) + sourceURL := "https://images.example.com/wide-cover.jpg" + images := NewImageProxy(&config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}, zap.NewNop()) + images.client = &http.Client{Transport: imageRoundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host == "metatube.example.com" { + return &http.Response{ + StatusCode: http.StatusGatewayTimeout, + Status: "504 Gateway Timeout", + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("timeout")), + Request: req, + }, nil + } + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Header: http.Header{"Content-Type": []string{"image/jpeg"}}, + Body: io.NopCloser(bytes.NewReader(wideCover)), + Request: req, + }, nil + })} + scraper.SetImageProxy(images) + + libPath := t.TempDir() + lib := model.Library{Name: "Adult", Path: libPath, Type: "adult", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: lib.ID, + Title: "Raw", + Path: filepath.Join(libPath, "IPX-235.mp4"), + PosterURL: filepath.Join(libPath, "old-poster.jpg"), + NSFW: true, + ScrapeStatus: "matched", + } + if err := repos.DB.Create(&media).Error; err != nil { + t.Fatal(err) + } + query := url.Values{ + "auto": {"true"}, + "pos": {"1"}, + "quality": {"90"}, + "ratio": {"-1"}, + "url": {sourceURL}, + } + match := &Match{ + Title: "IPX-235 Matched", + MediaType: "adult", + NSFW: true, + PosterURL: "https://metatube.example.com/v1/images/primary/AVE/123?" + query.Encode(), + } + if err := scraper.applyProviderMatch(t.Context(), &media, &lib, match); err != nil { + t.Fatal(err) + } + + var stored model.Media + if err := repos.DB.First(&stored, "id = ?", media.ID).Error; err != nil { + t.Fatal(err) + } + if filepath.Base(stored.PosterURL) != "IPX-235-poster.jpg" { + t.Fatalf("poster = %q, want local face-cropped sidecar", stored.PosterURL) + } + data, err := os.ReadFile(stored.PosterURL) + if err != nil { + t.Fatal(err) + } + cropped, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + ratio := float64(cropped.Bounds().Dx()) / float64(cropped.Bounds().Dy()) + if ratio < 0.65 || ratio > 0.68 { + t.Fatalf("local fallback poster ratio = %.3f", ratio) + } +} + func TestApplyProviderMatchReplacesArtworkAndRemovesOldCache(t *testing.T) { scraper, repos, closeServer := newTestScraper(t) defer closeServer() diff --git a/internal/service/scraper_library.go b/internal/service/scraper_library.go index f9d6277..172b195 100644 --- a/internal/service/scraper_library.go +++ b/internal/service/scraper_library.go @@ -27,11 +27,13 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, media * } if s.tmdb != nil && s.tmdb.Enabled() { if match := s.lookupAutomaticTMDb(ctx, kind, query, year); match != nil { + match.Provider = "tmdb" return match } } if s.douban != nil && s.douban.Enabled() { if m, err := s.douban.SearchMatch(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) { + m.Provider = "douban" return m } else if err != nil { s.log.Debug("douban search failed", zap.String("query", query), zap.Error(err)) @@ -39,6 +41,7 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, media * } if s.bangumi != nil && s.bangumi.Enabled() { if m, err := s.bangumi.Search(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) { + m.Provider = "bangumi" return m } else if err != nil { s.log.Debug("bangumi search failed", zap.String("query", query), zap.Error(err)) @@ -46,6 +49,7 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, media * } if (kind == "anime" || kind == "tv" || kind == "variety" || kind == "show" || kind == "shows") && s.thetvdb != nil && s.thetvdb.Enabled() { if m, err := s.thetvdb.SearchSeries(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) { + m.Provider = "thetvdb" return m } else if err != nil { s.log.Debug("thetvdb search failed", zap.String("query", query), zap.Error(err)) diff --git a/internal/service/scraper_lookup.go b/internal/service/scraper_lookup.go index d74b875..7fd6727 100644 --- a/internal/service/scraper_lookup.go +++ b/internal/service/scraper_lookup.go @@ -23,6 +23,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model } if match := s.manualTMDbMatchByID(ctx, m.TMDbID, normalizeMediaType(mediaType, m.Title, "")); match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "tmdb") { + match.Provider = "tmdb" preferExistingLocalizedEpisodeTitle(m, lib, match) return match } @@ -31,6 +32,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model if strings.TrimSpace(m.DoubanID) != "" && s.douban != nil && s.douban.Enabled() { if match, err := s.douban.GetMatchByID(ctx, strings.TrimSpace(m.DoubanID)); err == nil && match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "douban") { + match.Provider = "douban" preferExistingLocalizedEpisodeTitle(m, lib, match) return match } @@ -41,6 +43,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model if m.BangumiID > 0 && s.bangumi != nil && s.bangumi.Enabled() { if match, err := s.bangumi.GetSubject(ctx, m.BangumiID); err == nil && match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "bangumi") { + match.Provider = "bangumi" preferExistingLocalizedEpisodeTitle(m, lib, match) return match } @@ -51,6 +54,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model if strings.TrimSpace(m.TheTVDBID) != "" && s.thetvdb != nil && s.thetvdb.Enabled() { if match, err := s.thetvdb.GetSeriesMatchByID(ctx, strings.TrimSpace(m.TheTVDBID)); err == nil && match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "thetvdb") { + match.Provider = "thetvdb" preferExistingLocalizedEpisodeTitle(m, lib, match) return match } diff --git a/internal/service/scraper_options.go b/internal/service/scraper_options.go index c597d55..da45f18 100644 --- a/internal/service/scraper_options.go +++ b/internal/service/scraper_options.go @@ -8,6 +8,7 @@ type ScrapeOptions struct { DeferEpisodeDetails bool ForceRematch bool RebuildIdentity bool + resultProvider *string } func (o ScrapeOptions) episodeArtworkEnabled() bool { @@ -18,3 +19,9 @@ func skipEpisodeArtworkOptions(retryNoMatch bool) ScrapeOptions { episodeArtwork := false return ScrapeOptions{RetryNoMatch: retryNoMatch, EpisodeArtwork: &episodeArtwork} } + +func (o ScrapeOptions) recordProvider(provider string) { + if o.resultProvider != nil { + *o.resultProvider = provider + } +} diff --git a/internal/service/scraper_queue.go b/internal/service/scraper_queue.go index a782e90..2d16b3d 100644 --- a/internal/service/scraper_queue.go +++ b/internal/service/scraper_queue.go @@ -115,10 +115,12 @@ func (s *ScraperService) processScrapeTask(ctx context.Context, task *model.Scra } epArtwork := task.EpisodeImages + usedProvider := "" options := ScrapeOptions{ EpisodeArtwork: &epArtwork, IncludeMatched: task.RefreshMatched, RetryNoMatch: true, + resultProvider: &usedProvider, } enrichErr := s.EnrichOneWithOptions(ctx, media, options) @@ -133,16 +135,9 @@ func (s *ScraperService) processScrapeTask(ctx context.Context, task *model.Scra task.MatchedYear = refreshed.Year task.PosterURL = refreshed.PosterURL task.BackdropURL = refreshed.BackdropURL - if refreshed.TMDbID > 0 { - task.Provider = "tmdb" - } else if strings.TrimSpace(refreshed.DoubanID) != "" { - task.Provider = "douban" - } else if refreshed.BangumiID > 0 { - task.Provider = "bangumi" - } else if strings.TrimSpace(refreshed.TheTVDBID) != "" { - task.Provider = "thetvdb" - } else { - task.Provider = "metatube" + task.Provider = strings.ToLower(strings.TrimSpace(usedProvider)) + if task.Provider == "" { + task.Provider = "unknown" } } else { task.Status = model.ScrapeTaskFailed diff --git a/internal/service/tmdb_types.go b/internal/service/tmdb_types.go index bbd8822..64ba330 100644 --- a/internal/service/tmdb_types.go +++ b/internal/service/tmdb_types.go @@ -4,6 +4,7 @@ package service // across providers; provider-specific IDs sit side-by-side so the scraper // orchestrator can write them all into a single update. type Match struct { + Provider string `json:"provider,omitempty"` TMDbID int `json:"tmdb_id"` BangumiID int `json:"bangumi_id"` DoubanID string `json:"douban_id,omitempty"`