Fix local adult metadata and watch history removal

internal/service/scraper.go

@@ -3,9 +3,9 @@
 // ScraperService takes a Media row and tries to enrich it with metadata
 // from one or more providers. Selection is driven by the library type:
 //
-//   library.type == "anime"   -> Bangumi  (fallback: TMDb)
-//   library.type == "tv"      -> TheTVDB  (fallback: TMDb)
-//   default                   -> TMDb
+//	library.type == "anime"   -> Bangumi  (fallback: TMDb)
+//	library.type == "tv"      -> TheTVDB  (fallback: TMDb)
+//	default                   -> TMDb
 //
 // After the primary match we optionally upgrade poster / backdrop with
 // Fanart.tv when an API key is configured.
@@ -35,6 +35,7 @@ type ScraperService struct {
 	bangumi *BangumiProvider
 	thetvdb *TheTVDBProvider
 	fanart  *FanartProvider
+	adult   *AdultProvider
 	hub     *Hub
 }
 
@@ -48,10 +49,15 @@ func NewScraperService(
 	thetvdb *TheTVDBProvider,
 	fanart *FanartProvider,
 	hub *Hub,
+	adult ...*AdultProvider,
 ) *ScraperService {
+	var adultProvider *AdultProvider
+	if len(adult) > 0 {
+		adultProvider = adult[0]
+	}
 	return &ScraperService{
 		cfg: cfg, log: log, repo: repo,
-		tmdb: tmdb, bangumi: bangumi, thetvdb: thetvdb, fanart: fanart, hub: hub,
+		tmdb: tmdb, bangumi: bangumi, thetvdb: thetvdb, fanart: fanart, adult: adultProvider, hub: hub,
 	}
 }
 
@@ -82,8 +88,24 @@ var noiseTokens = []string{
 	"season",
 }
 
+var noiseTokenSet = func() map[string]struct{} {
+	set := make(map[string]struct{}, len(noiseTokens)+1)
+	for _, token := range noiseTokens {
+		set[token] = struct{}{}
+	}
+	set["dl"] = struct{}{}
+	return set
+}()
+
 // bracketedTag matches "[anything]" or "(anything)" segments.
 var bracketedTag = regexp.MustCompile(`[\[\(][^\]\)]*[\]\)]`)
+var multiWordNoise = []*regexp.Regexp{
+	regexp.MustCompile(`(?i)\bweb[\s._-]*dl\b`),
+	regexp.MustCompile(`(?i)\bblu[\s._-]*ray\b`),
+	regexp.MustCompile(`(?i)\bdirectors[\s._-]*cut\b`),
+	regexp.MustCompile(`(?i)\berai[\s._-]*raws\b`),
+	regexp.MustCompile(`(?i)\bohys[\s._-]*raws\b`),
+}
 
 // CleanQuery converts a filename like "Inception.2010.1080p.BluRay.x264.mkv"
 // into a TMDb-friendly title plus an optional year hint.
@@ -105,8 +127,8 @@ func CleanQuery(raw string) (title string, year int) {
 	lower = patEP.ReplaceAllString(lower, " ")
 	lower = patCN.ReplaceAllString(lower, " ")
 
-	for _, t := range noiseTokens {
-		lower = strings.ReplaceAll(lower, t, " ")
+	for _, pat := range multiWordNoise {
+		lower = pat.ReplaceAllString(lower, " ")
 	}
 	for _, sep := range []string{".", "_", "-", "[", "]", "(", ")", "×"} {
 		lower = strings.ReplaceAll(lower, sep, " ")
@@ -115,6 +137,9 @@ func CleanQuery(raw string) (title string, year int) {
 	// 「2」「0」「v」之类残留干扰 TMDb 搜索。中文字符不算碎片。
 	out := make([]string, 0, 8)
 	for _, w := range strings.Fields(lower) {
+		if _, ok := noiseTokenSet[w]; ok {
+			continue
+		}
 		if len(w) <= 1 {
 			r := []rune(w)
 			if len(r) == 1 && r[0] < 128 {
@@ -134,21 +159,51 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
 		return err
 	}
 
-	query := m.Title
-	if query == "" {
-		query, _ = CleanQuery(m.Path)
-	} else {
-		query, _ = CleanQuery(query)
+	seriesLike := mediaIsEpisodic(m, lib)
+	var local *LocalMetadata
+	if found, err := ReadLocalMetadata(m.Path, lib.Path, seriesLike); err == nil && found != nil {
+		local = found
+		applyLocalMetadata(m, local)
+	} else if err != nil {
+		s.log.Warn("read local metadata before scrape failed", zap.String("media_id", m.ID), zap.Error(err))
 	}
+
 	year := m.Year
 	if year == 0 {
 		_, year = CleanQuery(filepath.Base(m.Path))
 	}
 
-	match := s.lookup(ctx, lib, query, year)
+	if s.adult != nil && s.adult.Enabled() {
+		if code := firstText(localAdultCode(local), AdultCodeFromMediaPath(m.Path), normalizeAdultCode(m.OriginalName), normalizeAdultCode(m.Title)); code != "" {
+			if adultMatch, err := s.adult.Search(ctx, code); err == nil && adultMatch != nil {
+				mergeLocalMetadataIntoMatch(adultMatch, local)
+				return s.applyProviderMatch(ctx, m, lib, adultMatch)
+			} else if err != nil {
+				s.log.Debug("adult metadata search failed", zap.String("media_id", m.ID), zap.String("code", code), zap.Error(err))
+			}
+		}
+	}
+
+	candidates := scrapeQueryCandidates(m, lib)
+	var query string
+	match := (*Match)(nil)
+	for _, candidate := range candidates {
+		match = s.lookup(ctx, lib, candidate, year)
+		query = candidate
+		if match != nil {
+			break
+		}
+	}
 	if match == nil {
+		if local != nil {
+			return s.applyLocalMetadataMatch(ctx, m, local)
+		}
 		_ = s.repo.DB.Model(&model.Media{}).Where("id = ?", m.ID).
 			Update("scrape_status", "no_match").Error
+		s.log.Info("metadata scrape no match",
+			zap.String("media_id", m.ID),
+			zap.String("query", query),
+			zap.String("library_type", lib.Type))
 		return nil
 	}
 	// Optional Fanart upgrade.
@@ -162,7 +217,68 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
 			}
 		}
 	}
+	mergeLocalMetadataIntoMatch(match, local)
 
+	return s.applyProviderMatch(ctx, m, lib, match)
+}
+
+func localAdultCode(local *LocalMetadata) string {
+	if local == nil {
+		return ""
+	}
+	return local.AdultCode
+}
+
+func mergeLocalMetadataIntoMatch(match *Match, local *LocalMetadata) {
+	if match == nil || local == nil {
+		return
+	}
+	if local.Title != "" {
+		match.Title = local.Title
+	}
+	if local.OriginalName != "" {
+		match.OriginalName = local.OriginalName
+	}
+	if local.AdultCode != "" {
+		match.OriginalName = local.AdultCode
+		match.NSFW = true
+	}
+	if local.Overview != "" {
+		match.Overview = local.Overview
+	}
+	if local.PosterURL != "" {
+		match.PosterURL = local.PosterURL
+	}
+	if local.BackdropURL != "" {
+		match.BackdropURL = local.BackdropURL
+	}
+	if local.Rating > 0 {
+		match.Rating = local.Rating
+	}
+	if local.Year > 0 {
+		match.Year = local.Year
+	}
+	if local.TMDbID > 0 {
+		match.TMDbID = local.TMDbID
+	}
+	if local.BangumiID > 0 {
+		match.BangumiID = local.BangumiID
+	}
+	if local.Genres != "" {
+		match.Genres = splitNFOList(local.Genres)
+	}
+	if local.Countries != "" {
+		match.Countries = splitNFOList(local.Countries)
+	}
+	if local.Languages != "" {
+		match.Languages = splitNFOList(local.Languages)
+	}
+	if local.NSFW {
+		match.NSFW = true
+	}
+}
+
+func (s *ScraperService) applyProviderMatch(ctx context.Context, m *model.Media, lib *model.Library, match *Match) error {
 	updates := map[string]any{
 		"title":         match.Title,
 		"overview":      match.Overview,
@@ -172,12 +288,27 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
 		"year":          match.Year,
 		"scrape_status": "matched",
 	}
+	if match.OriginalName != "" {
+		updates["original_name"] = match.OriginalName
+	}
 	if match.TMDbID > 0 {
-		updates["tmdb_id"] = match.TMDbID
+		updates["tm_db_id"] = match.TMDbID
 	}
 	if match.BangumiID > 0 {
 		updates["bangumi_id"] = match.BangumiID
 	}
+	if match.NSFW {
+		updates["nsfw"] = true
+	}
+	if len(match.Genres) > 0 {
+		updates["genres"] = strings.Join(match.Genres, ",")
+	}
+	if len(match.Countries) > 0 {
+		updates["countries"] = strings.Join(match.Countries, ",")
+	}
+	if len(match.Languages) > 0 {
+		updates["languages"] = strings.Join(match.Languages, ",")
+	}
 
 	// Fetch extended metadata (languages, countries, genres) from TMDb
 	if match.TMDbID > 0 && s.tmdb != nil && s.tmdb.Enabled() {
@@ -210,15 +341,198 @@ func (s *ScraperService) EnrichOne(ctx context.Context, m *model.Media) error {
 		Updates(updates).Error; err != nil {
 		return err
 	}
+	if refreshed, err := s.repo.Media.FindByID(ctx, m.ID); err == nil && refreshed != nil {
+		if path, err := WriteMediaNFO(refreshed); err != nil {
+			s.log.Warn("write nfo after scrape failed", zap.String("media_id", m.ID), zap.Error(err))
+		} else {
+			s.log.Debug("write nfo after scrape", zap.String("media_id", m.ID), zap.String("path", path))
+		}
+	}
 	s.hub.Publish("scrape", map[string]any{
 		"media_id":   m.ID,
 		"title":      match.Title,
 		"tmdb_id":    match.TMDbID,
 		"bangumi_id": match.BangumiID,
+		"source":     map[bool]string{true: "adult"}[match.NSFW],
 	})
 	return nil
 }
 
+func (s *ScraperService) applyLocalMetadataMatch(ctx context.Context, m *model.Media, local *LocalMetadata) error {
+	next := *m
+	applyLocalMetadata(&next, local)
+	updates := map[string]any{
+		"title":         next.Title,
+		"scrape_status": "matched",
+	}
+	if next.OriginalName != "" {
+		updates["original_name"] = next.OriginalName
+	}
+	if next.Overview != "" {
+		updates["overview"] = next.Overview
+	}
+	if next.PosterURL != "" {
+		updates["poster_url"] = next.PosterURL
+	}
+	if next.BackdropURL != "" {
+		updates["backdrop_url"] = next.BackdropURL
+	}
+	if next.Rating > 0 {
+		updates["rating"] = next.Rating
+	}
+	if next.Year > 0 {
+		updates["year"] = next.Year
+	}
+	if next.TMDbID > 0 {
+		updates["tm_db_id"] = next.TMDbID
+	}
+	if next.BangumiID > 0 {
+		updates["bangumi_id"] = next.BangumiID
+	}
+	if next.SeasonNum > 0 {
+		updates["season_num"] = next.SeasonNum
+	}
+	if next.EpisodeNum > 0 {
+		updates["episode_num"] = next.EpisodeNum
+	}
+	if next.Genres != "" {
+		updates["genres"] = next.Genres
+	}
+	if next.Countries != "" {
+		updates["countries"] = next.Countries
+	}
+	if next.Languages != "" {
+		updates["languages"] = next.Languages
+	}
+	if next.NSFW {
+		updates["nsfw"] = true
+	}
+	if err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
+		Where("id = ?", m.ID).Updates(updates).Error; err != nil {
+		return err
+	}
+	s.hub.Publish("scrape", map[string]any{
+		"media_id": m.ID,
+		"title":    next.Title,
+		"tmdb_id":  next.TMDbID,
+		"source":   "local_nfo",
+	})
+	return nil
+}
+
+func scrapeQueryCandidates(m *model.Media, lib *model.Library) []string {
+	seen := map[string]struct{}{}
+	var out []string
+	add := func(raw string) {
+		cleaned, _ := CleanQuery(raw)
+		if cleaned == "" {
+			cleaned = strings.TrimSpace(raw)
+		}
+		for _, candidate := range titleCandidates(cleaned) {
+			key := strings.ToLower(candidate)
+			if _, ok := seen[key]; ok || candidate == "" {
+				continue
+			}
+			seen[key] = struct{}{}
+			out = append(out, candidate)
+		}
+	}
+	if lib != nil && mediaIsEpisodic(m, lib) {
+		add(seriesFolderTitle(m.Path, lib.Path))
+	}
+	add(m.Title)
+	add(m.Path)
+	if len(out) == 0 {
+		out = append(out, strings.TrimSuffix(filepath.Base(m.Path), filepath.Ext(m.Path)))
+	}
+	return out
+}
+
+func seriesFolderTitle(mediaPath, libraryRoot string) string {
+	dir := filepath.Dir(mediaPath)
+	if seasonFromDir(filepath.Base(dir)) > 0 {
+		dir = filepath.Dir(dir)
+	}
+	if libraryRoot != "" && samePath(dir, filepath.Clean(libraryRoot)) {
+		return ""
+	}
+	base := filepath.Base(dir)
+	if base == "." || base == string(filepath.Separator) {
+		return ""
+	}
+	return base
+}
+
+func seasonFromDir(name string) int {
+	if m := patSeasonFolder.FindStringSubmatch(name); len(m) >= 3 {
+		for _, group := range m[1:] {
+			if group != "" {
+				return mustAtoi(group)
+			}
+		}
+	}
+	return 0
+}
+
+func titleCandidates(title string) []string {
+	title = strings.Join(strings.Fields(strings.TrimSpace(title)), " ")
+	if title == "" {
+		return nil
+	}
+	out := make([]string, 0, 2)
+	if cjk := cjkTitleOnly(title); cjk != "" {
+		out = append(out, cjk)
+		if cjk != title {
+			return out
+		}
+	}
+	out = append(out, title)
+	return out
+}
+
+func cjkTitleOnly(title string) string {
+	parts := make([]string, 0, 4)
+	for _, field := range strings.Fields(title) {
+		if containsCJK(field) {
+			parts = append(parts, field)
+		}
+	}
+	return strings.Join(parts, " ")
+}
+
+func containsCJK(s string) bool {
+	for _, r := range s {
+		switch {
+		case r >= '\u3400' && r <= '\u4dbf':
+			return true
+		case r >= '\u4e00' && r <= '\u9fff':
+			return true
+		case r >= '\uf900' && r <= '\ufaff':
+			return true
+		}
+	}
+	return false
+}
+
+func mediaIsEpisodic(m *model.Media, lib *model.Library) bool {
+	if m != nil && (m.SeasonNum > 0 || m.EpisodeNum > 0) {
+		return true
+	}
+	return librarySupportsSeasons(lib)
+}
+
+func librarySupportsSeasons(lib *model.Library) bool {
+	if lib == nil {
+		return false
+	}
+	switch strings.ToLower(strings.TrimSpace(lib.Type)) {
+	case "tv", "anime", "variety", "show", "shows":
+		return true
+	default:
+		return false
+	}
+}
+
 // lookup runs the provider chain. When the library is missing we fall
 // back to TMDb only.
 //
@@ -245,7 +559,7 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, query s
 				s.log.Debug("bangumi search failed", zap.String("query", query), zap.Error(err))
 			}
 		}
-	case "tv":
+	case "tv", "variety", "show", "shows":
 		if s.thetvdb != nil && s.thetvdb.Enabled() {
 			if m, err := s.thetvdb.SearchSeries(ctx, query); err == nil && m != nil {
 				return m
@@ -256,7 +570,7 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, query s
 	}
 	if s.tmdb != nil && s.tmdb.Enabled() {
 		// anime / tv 先用 TMDb /search/tv（剧名通常是 TV 类目）。
-		if kind == "anime" || kind == "tv" {
+		if kind == "anime" || kind == "tv" || kind == "variety" || kind == "show" || kind == "shows" {
 			if m, err := s.tmdb.SearchTV(ctx, query, year); err == nil && m != nil {
 				return m
 			} else if err != nil {
@@ -272,15 +586,23 @@ func (s *ScraperService) lookup(ctx context.Context, lib *model.Library, query s
 	return nil
 }
 
-// EnrichLibrary runs the provider chain for every "pending" media in a
-// library. It throttles to 4 RPS and publishes a summary event when done.
+// EnrichLibrary runs the provider chain for every pending media in a library.
+// When retryNoMatch is true it also retries rows previously marked no_match,
+// which is the expected behaviour for a manual "重新刮削" action. Scanner-driven
+// automatic enrichment keeps the default false path to avoid repeated scraping.
 //
 // Pending status includes both the canonical "pending" string and the
 // empty / NULL values, because MediaRepository.Upsert can wipe the GORM
 // default when re-running a scan over an already-existing row.
-func (s *ScraperService) EnrichLibrary(ctx context.Context, libraryID string) (int, error) {
+func (s *ScraperService) EnrichLibrary(ctx context.Context, libraryID string, retryNoMatch ...bool) (int, error) {
 	var rows []model.Media
-	q := s.repo.DB.Where("scrape_status IS NULL OR scrape_status = '' OR scrape_status = ?", "pending")
+	statusFilter := "scrape_status IS NULL OR scrape_status = '' OR scrape_status = ?"
+	statusArgs := []any{"pending"}
+	if len(retryNoMatch) > 0 && retryNoMatch[0] {
+		statusFilter += " OR scrape_status = ?"
+		statusArgs = append(statusArgs, "no_match")
+	}
+	q := s.repo.DB.Where(statusFilter, statusArgs...)
 	if libraryID != "" {
 		q = q.Where("library_id = ?", libraryID)
 	}
@@ -288,6 +610,7 @@ func (s *ScraperService) EnrichLibrary(ctx context.Context, libraryID string) (i
 		return 0, err
 	}
 	matched := 0
+	processed := 0
 	for i := range rows {
 		select {
 		case <-ctx.Done():
@@ -298,17 +621,30 @@ func (s *ScraperService) EnrichLibrary(ctx context.Context, libraryID string) (i
 			s.log.Warn("enrich failed", zap.String("media", rows[i].ID), zap.Error(err))
 			continue
 		}
-		matched++
+		processed++
+		if s.mediaIsMatched(ctx, rows[i].ID) {
+			matched++
+		}
 		time.Sleep(250 * time.Millisecond) // ~4 RPS
 	}
 	s.hub.Publish("scrape", map[string]any{
 		"library_id": libraryID,
 		"finished":   true,
 		"matched":    matched,
+		"processed":  processed,
 	})
 	return matched, nil
 }
 
+func (s *ScraperService) mediaIsMatched(ctx context.Context, mediaID string) bool {
+	var status string
+	err := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
+		Select("scrape_status").
+		Where("id = ?", mediaID).
+		Scan(&status).Error
+	return err == nil && status == "matched"
+}
+
 // AnyEnabled reports whether at least one provider can run.
 func (s *ScraperService) AnyEnabled() bool {
 	if s.tmdb != nil && s.tmdb.Enabled() {
@@ -320,6 +656,9 @@ func (s *ScraperService) AnyEnabled() bool {
 	if s.thetvdb != nil && s.thetvdb.Enabled() {
 		return true
 	}
+	if s.adult != nil && s.adult.Enabled() {
+		return true
+	}
 	return false
 }
 
@@ -328,7 +667,7 @@ func (s *ScraperService) AnyEnabled() bool {
 func (s *ScraperService) determineMediaType(lib *model.Library, match *Match) string {
 	if lib != nil {
 		switch lib.Type {
-		case "tv", "anime":
+		case "tv", "anime", "variety", "show", "shows":
 			return "tv"
 		}
 	}