This commit is contained in:
truewhile
2026-09-09 00:03:14 +08:00
parent fb4e762026
commit 8eb017fedc
11 changed files with 170 additions and 14 deletions
+1
View File
@@ -23,6 +23,7 @@ func parseAdultDetailHTML(body, code, source, detailURL string) *Match {
} }
match := &Match{ match := &Match{
Provider: strings.ToLower(strings.TrimSpace(source)),
OriginalName: code, OriginalName: code,
MediaType: "adult", MediaType: "adult",
NSFW: true, NSFW: true,
+2
View File
@@ -310,6 +310,7 @@ func (p *MetaTubeProvider) convertSearchResultToMatch(cfg MetaTubeConfig, query
} }
return &Match{ return &Match{
Provider: "metatube",
MediaType: "adult", MediaType: "adult",
Title: formattedTitle, Title: formattedTitle,
OriginalName: code, OriginalName: code,
@@ -383,6 +384,7 @@ func (p *MetaTubeProvider) convertMovieInfoToMatch(cfg MetaTubeConfig, movie *Me
} }
return &Match{ return &Match{
Provider: "metatube",
MediaType: "adult", MediaType: "adult",
Title: formattedTitle, Title: formattedTitle,
OriginalName: code, OriginalName: code,
+3
View File
@@ -62,6 +62,9 @@ func TestMetaTubeProviderSearch(t *testing.T) {
t.Fatalf("expected 1 match, got %d", len(matches)) t.Fatalf("expected 1 match, got %d", len(matches))
} }
m := matches[0] m := matches[0]
if m.Provider != "metatube" {
t.Errorf("expected provider metatube, got %q", m.Provider)
}
if m.OriginalName != "IPX-235" { if m.OriginalName != "IPX-235" {
t.Errorf("expected original name IPX-235, got %s", m.OriginalName) t.Errorf("expected original name IPX-235, got %s", m.OriginalName)
} }
+10 -1
View File
@@ -93,7 +93,11 @@ func (s *ScraperService) EnrichOneWithOptions(ctx context.Context, m *model.Medi
} }
if match == nil { if match == nil {
if local != nil && !local.PathHint { if local != nil && !local.PathHint {
return s.applyLocalMetadataMatch(ctx, m, local) err := s.applyLocalMetadataMatch(ctx, m, local)
if err == nil {
options.recordProvider("local")
}
return err
} }
if err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).Where("id = ?", m.ID). if err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).Where("id = ?", m.ID).
Update("scrape_status", "no_match").Error; err != nil { Update("scrape_status", "no_match").Error; err != nil {
@@ -227,6 +231,11 @@ func (s *ScraperService) applyProviderMatchWithOptions(ctx context.Context, m *m
"thetvdb_id": match.TheTVDBID, "thetvdb_id": match.TheTVDBID,
"source": map[bool]string{true: "adult"}[match.NSFW], "source": map[bool]string{true: "adult"}[match.NSFW],
}) })
provider := strings.ToLower(strings.TrimSpace(match.Provider))
if provider == "" {
provider = "unknown"
}
options.recordProvider(provider)
return nil return nil
} }
+38 -1
View File
@@ -2,6 +2,7 @@ package service
import ( import (
"context" "context"
"net/url"
"strings" "strings"
"time" "time"
@@ -20,10 +21,34 @@ func (s *ScraperService) prepareScrapedArtworkURL(ctx context.Context, mediaID,
if s == nil || s.images == nil || !isHTTPish(candidate) { if s == nil || s.images == nil || !isHTTPish(candidate) {
return candidate, "" return candidate, ""
} }
fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) originalSource := metaTubeOriginalArtworkURL(candidate)
timeout := 15 * time.Second
if originalSource != "" {
// MetaTube face processing can become very slow under a batch scrape.
// Fail over quickly so the same source can be processed locally.
timeout = 8 * time.Second
}
fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout)
err := s.images.PrefetchRemote(fetchCtx, candidate) err := s.images.PrefetchRemote(fetchCtx, candidate)
cancel() cancel()
if err != nil { if err != nil {
if originalSource != "" {
fallbackCtx, fallbackCancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
fallbackErr := s.images.PrefetchRemote(fallbackCtx, originalSource)
fallbackCancel()
if fallbackErr == nil {
s.log.Warn("MetaTube artwork processing failed; using local face-aware crop",
zap.String("media_id", mediaID),
zap.String("field", field),
zap.String("candidate", candidate),
zap.String("source", originalSource),
zap.Error(err))
if current != "" && isHTTPish(current) {
return originalSource, current
}
return originalSource, ""
}
}
if current != "" { if current != "" {
s.log.Warn("scrape artwork prefetch failed; keeping existing artwork", s.log.Warn("scrape artwork prefetch failed; keeping existing artwork",
zap.String("media_id", mediaID), zap.String("media_id", mediaID),
@@ -46,6 +71,18 @@ func (s *ScraperService) prepareScrapedArtworkURL(ctx context.Context, mediaID,
return candidate, "" return candidate, ""
} }
func metaTubeOriginalArtworkURL(raw string) string {
parsed, err := url.Parse(strings.TrimSpace(raw))
if err != nil || !strings.Contains(parsed.Path, "/v1/images/primary/") {
return ""
}
source := strings.TrimSpace(parsed.Query().Get("url"))
if !isHTTPish(source) {
return ""
}
return source
}
func (s *ScraperService) removeCachedScrapedArtwork(urls ...string) { func (s *ScraperService) removeCachedScrapedArtwork(urls ...string) {
if s == nil || s.images == nil { if s == nil || s.images == nil {
return return
+95 -2
View File
@@ -3,8 +3,11 @@ package service
import ( import (
"bytes" "bytes"
"errors" "errors"
"image"
"image/color"
"io" "io"
"net/http" "net/http"
"net/url"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -67,10 +70,14 @@ func TestApplyProviderMatchInvalidatesMediaCache(t *testing.T) {
if err := repos.DB.Create(&media).Error; err != nil { if err := repos.DB.Create(&media).Error; err != nil {
t.Fatal(err) t.Fatal(err)
} }
match := &Match{Title: "Matched", PosterURL: "https://image.tmdb.org/t/p/w500/poster.jpg", BackdropURL: "https://image.tmdb.org/t/p/w1280/backdrop.jpg"} match := &Match{Provider: "metatube", Title: "Matched", PosterURL: "https://image.tmdb.org/t/p/w500/poster.jpg", BackdropURL: "https://image.tmdb.org/t/p/w1280/backdrop.jpg"}
if err := scraper.applyProviderMatch(t.Context(), &media, &lib, match); err != nil { usedProvider := ""
if err := scraper.applyProviderMatchWithOptions(t.Context(), &media, &lib, match, ScrapeOptions{resultProvider: &usedProvider}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if usedProvider != "metatube" {
t.Fatalf("recorded provider = %q, want metatube", usedProvider)
}
var stale map[string]string var stale map[string]string
if cache.GetJSON(t.Context(), "media:list:stale", &stale) { if cache.GetJSON(t.Context(), "media:list:stale", &stale) {
@@ -145,6 +152,92 @@ func TestApplyProviderMatchKeepsExistingArtworkWhenNewPrefetchFails(t *testing.T
} }
} }
func TestApplyProviderMatchFallsBackToLocalFaceCropWhenMetaTubeImageFails(t *testing.T) {
scraper, repos, closeServer := newTestScraper(t)
defer closeServer()
wideCover := createTestImage(
900,
600,
color.RGBA{R: 255, A: 255},
color.RGBA{B: 255, A: 255},
)
sourceURL := "https://images.example.com/wide-cover.jpg"
images := NewImageProxy(&config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}, zap.NewNop())
images.client = &http.Client{Transport: imageRoundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host == "metatube.example.com" {
return &http.Response{
StatusCode: http.StatusGatewayTimeout,
Status: "504 Gateway Timeout",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader("timeout")),
Request: req,
}, nil
}
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: http.Header{"Content-Type": []string{"image/jpeg"}},
Body: io.NopCloser(bytes.NewReader(wideCover)),
Request: req,
}, nil
})}
scraper.SetImageProxy(images)
libPath := t.TempDir()
lib := model.Library{Name: "Adult", Path: libPath, Type: "adult", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
media := model.Media{
LibraryID: lib.ID,
Title: "Raw",
Path: filepath.Join(libPath, "IPX-235.mp4"),
PosterURL: filepath.Join(libPath, "old-poster.jpg"),
NSFW: true,
ScrapeStatus: "matched",
}
if err := repos.DB.Create(&media).Error; err != nil {
t.Fatal(err)
}
query := url.Values{
"auto": {"true"},
"pos": {"1"},
"quality": {"90"},
"ratio": {"-1"},
"url": {sourceURL},
}
match := &Match{
Title: "IPX-235 Matched",
MediaType: "adult",
NSFW: true,
PosterURL: "https://metatube.example.com/v1/images/primary/AVE/123?" + query.Encode(),
}
if err := scraper.applyProviderMatch(t.Context(), &media, &lib, match); err != nil {
t.Fatal(err)
}
var stored model.Media
if err := repos.DB.First(&stored, "id = ?", media.ID).Error; err != nil {
t.Fatal(err)
}
if filepath.Base(stored.PosterURL) != "IPX-235-poster.jpg" {
t.Fatalf("poster = %q, want local face-cropped sidecar", stored.PosterURL)
}
data, err := os.ReadFile(stored.PosterURL)
if err != nil {
t.Fatal(err)
}
cropped, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
t.Fatal(err)
}
ratio := float64(cropped.Bounds().Dx()) / float64(cropped.Bounds().Dy())
if ratio < 0.65 || ratio > 0.68 {
t.Fatalf("local fallback poster ratio = %.3f", ratio)
}
}
func TestApplyProviderMatchReplacesArtworkAndRemovesOldCache(t *testing.T) { func TestApplyProviderMatchReplacesArtworkAndRemovesOldCache(t *testing.T) {
scraper, repos, closeServer := newTestScraper(t) scraper, repos, closeServer := newTestScraper(t)
defer closeServer() defer closeServer()
+4
View File
@@ -27,11 +27,13 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, media *
} }
if s.tmdb != nil && s.tmdb.Enabled() { if s.tmdb != nil && s.tmdb.Enabled() {
if match := s.lookupAutomaticTMDb(ctx, kind, query, year); match != nil { if match := s.lookupAutomaticTMDb(ctx, kind, query, year); match != nil {
match.Provider = "tmdb"
return match return match
} }
} }
if s.douban != nil && s.douban.Enabled() { if s.douban != nil && s.douban.Enabled() {
if m, err := s.douban.SearchMatch(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) { if m, err := s.douban.SearchMatch(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) {
m.Provider = "douban"
return m return m
} else if err != nil { } else if err != nil {
s.log.Debug("douban search failed", zap.String("query", query), zap.Error(err)) s.log.Debug("douban search failed", zap.String("query", query), zap.Error(err))
@@ -39,6 +41,7 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, media *
} }
if s.bangumi != nil && s.bangumi.Enabled() { if s.bangumi != nil && s.bangumi.Enabled() {
if m, err := s.bangumi.Search(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) { if m, err := s.bangumi.Search(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) {
m.Provider = "bangumi"
return m return m
} else if err != nil { } else if err != nil {
s.log.Debug("bangumi search failed", zap.String("query", query), zap.Error(err)) s.log.Debug("bangumi search failed", zap.String("query", query), zap.Error(err))
@@ -46,6 +49,7 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, media *
} }
if (kind == "anime" || kind == "tv" || kind == "variety" || kind == "show" || kind == "shows") && s.thetvdb != nil && s.thetvdb.Enabled() { if (kind == "anime" || kind == "tv" || kind == "variety" || kind == "show" || kind == "shows") && s.thetvdb != nil && s.thetvdb.Enabled() {
if m, err := s.thetvdb.SearchSeries(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) { if m, err := s.thetvdb.SearchSeries(ctx, query); err == nil && m != nil && metadataMatchCompatibleWithType(kind, m) {
m.Provider = "thetvdb"
return m return m
} else if err != nil { } else if err != nil {
s.log.Debug("thetvdb search failed", zap.String("query", query), zap.Error(err)) s.log.Debug("thetvdb search failed", zap.String("query", query), zap.Error(err))
+4
View File
@@ -23,6 +23,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model
} }
if match := s.manualTMDbMatchByID(ctx, m.TMDbID, normalizeMediaType(mediaType, m.Title, "")); match != nil { if match := s.manualTMDbMatchByID(ctx, m.TMDbID, normalizeMediaType(mediaType, m.Title, "")); match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "tmdb") { if s.mediaExternalIDMatchTrusted(m, lib, match, "tmdb") {
match.Provider = "tmdb"
preferExistingLocalizedEpisodeTitle(m, lib, match) preferExistingLocalizedEpisodeTitle(m, lib, match)
return match return match
} }
@@ -31,6 +32,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model
if strings.TrimSpace(m.DoubanID) != "" && s.douban != nil && s.douban.Enabled() { if strings.TrimSpace(m.DoubanID) != "" && s.douban != nil && s.douban.Enabled() {
if match, err := s.douban.GetMatchByID(ctx, strings.TrimSpace(m.DoubanID)); err == nil && match != nil { if match, err := s.douban.GetMatchByID(ctx, strings.TrimSpace(m.DoubanID)); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "douban") { if s.mediaExternalIDMatchTrusted(m, lib, match, "douban") {
match.Provider = "douban"
preferExistingLocalizedEpisodeTitle(m, lib, match) preferExistingLocalizedEpisodeTitle(m, lib, match)
return match return match
} }
@@ -41,6 +43,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model
if m.BangumiID > 0 && s.bangumi != nil && s.bangumi.Enabled() { if m.BangumiID > 0 && s.bangumi != nil && s.bangumi.Enabled() {
if match, err := s.bangumi.GetSubject(ctx, m.BangumiID); err == nil && match != nil { if match, err := s.bangumi.GetSubject(ctx, m.BangumiID); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "bangumi") { if s.mediaExternalIDMatchTrusted(m, lib, match, "bangumi") {
match.Provider = "bangumi"
preferExistingLocalizedEpisodeTitle(m, lib, match) preferExistingLocalizedEpisodeTitle(m, lib, match)
return match return match
} }
@@ -51,6 +54,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model
if strings.TrimSpace(m.TheTVDBID) != "" && s.thetvdb != nil && s.thetvdb.Enabled() { if strings.TrimSpace(m.TheTVDBID) != "" && s.thetvdb != nil && s.thetvdb.Enabled() {
if match, err := s.thetvdb.GetSeriesMatchByID(ctx, strings.TrimSpace(m.TheTVDBID)); err == nil && match != nil { if match, err := s.thetvdb.GetSeriesMatchByID(ctx, strings.TrimSpace(m.TheTVDBID)); err == nil && match != nil {
if s.mediaExternalIDMatchTrusted(m, lib, match, "thetvdb") { if s.mediaExternalIDMatchTrusted(m, lib, match, "thetvdb") {
match.Provider = "thetvdb"
preferExistingLocalizedEpisodeTitle(m, lib, match) preferExistingLocalizedEpisodeTitle(m, lib, match)
return match return match
} }
+7
View File
@@ -8,6 +8,7 @@ type ScrapeOptions struct {
DeferEpisodeDetails bool DeferEpisodeDetails bool
ForceRematch bool ForceRematch bool
RebuildIdentity bool RebuildIdentity bool
resultProvider *string
} }
func (o ScrapeOptions) episodeArtworkEnabled() bool { func (o ScrapeOptions) episodeArtworkEnabled() bool {
@@ -18,3 +19,9 @@ func skipEpisodeArtworkOptions(retryNoMatch bool) ScrapeOptions {
episodeArtwork := false episodeArtwork := false
return ScrapeOptions{RetryNoMatch: retryNoMatch, EpisodeArtwork: &episodeArtwork} return ScrapeOptions{RetryNoMatch: retryNoMatch, EpisodeArtwork: &episodeArtwork}
} }
func (o ScrapeOptions) recordProvider(provider string) {
if o.resultProvider != nil {
*o.resultProvider = provider
}
}
+5 -10
View File
@@ -115,10 +115,12 @@ func (s *ScraperService) processScrapeTask(ctx context.Context, task *model.Scra
} }
epArtwork := task.EpisodeImages epArtwork := task.EpisodeImages
usedProvider := ""
options := ScrapeOptions{ options := ScrapeOptions{
EpisodeArtwork: &epArtwork, EpisodeArtwork: &epArtwork,
IncludeMatched: task.RefreshMatched, IncludeMatched: task.RefreshMatched,
RetryNoMatch: true, RetryNoMatch: true,
resultProvider: &usedProvider,
} }
enrichErr := s.EnrichOneWithOptions(ctx, media, options) enrichErr := s.EnrichOneWithOptions(ctx, media, options)
@@ -133,16 +135,9 @@ func (s *ScraperService) processScrapeTask(ctx context.Context, task *model.Scra
task.MatchedYear = refreshed.Year task.MatchedYear = refreshed.Year
task.PosterURL = refreshed.PosterURL task.PosterURL = refreshed.PosterURL
task.BackdropURL = refreshed.BackdropURL task.BackdropURL = refreshed.BackdropURL
if refreshed.TMDbID > 0 { task.Provider = strings.ToLower(strings.TrimSpace(usedProvider))
task.Provider = "tmdb" if task.Provider == "" {
} else if strings.TrimSpace(refreshed.DoubanID) != "" { task.Provider = "unknown"
task.Provider = "douban"
} else if refreshed.BangumiID > 0 {
task.Provider = "bangumi"
} else if strings.TrimSpace(refreshed.TheTVDBID) != "" {
task.Provider = "thetvdb"
} else {
task.Provider = "metatube"
} }
} else { } else {
task.Status = model.ScrapeTaskFailed task.Status = model.ScrapeTaskFailed
+1
View File
@@ -4,6 +4,7 @@ package service
// across providers; provider-specific IDs sit side-by-side so the scraper // across providers; provider-specific IDs sit side-by-side so the scraper
// orchestrator can write them all into a single update. // orchestrator can write them all into a single update.
type Match struct { type Match struct {
Provider string `json:"provider,omitempty"`
TMDbID int `json:"tmdb_id"` TMDbID int `json:"tmdb_id"`
BangumiID int `json:"bangumi_id"` BangumiID int `json:"bangumi_id"`
DoubanID string `json:"douban_id,omitempty"` DoubanID string `json:"douban_id,omitempty"`