fix: dedupe RSS subscription variants

2026-09-28 11:16:37 +08:00 · 2026-06-08 21:49:51 +08:00
parent 691b1cee17
commit 76d6e0c8ff
3 changed files with 162 additions and 42 deletions
@@ -61,18 +61,20 @@ func (s *SubscriptionService) Stop() { close(s.stop) }
 type rssFeed struct {
 	XMLName xml.Name `xml:"rss"`
 	Channel struct {
-		Items []struct {
-			Title       string `xml:"title"`
-			Link        string `xml:"link"`
-			GUID        string `xml:"guid"`
-			Description string `xml:"description"`
-			Enclosure   struct {
-				URL string `xml:"url,attr"`
-			} `xml:"enclosure"`
-		} `xml:"item"`
+		Items []rssItem `xml:"item"`
 	} `xml:"channel"`
 }

+type rssItem struct {
+	Title       string `xml:"title"`
+	Link        string `xml:"link"`
+	GUID        string `xml:"guid"`
+	Description string `xml:"description"`
+	Enclosure   struct {
+		URL string `xml:"url,attr"`
+	} `xml:"enclosure"`
+}
+
 // Create persists a new subscription.
 func (s *SubscriptionService) Create(ctx context.Context, sub *model.Subscription) error {
 	if sub.Name == "" || sub.FeedURL == "" {
@@ -186,42 +188,21 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio
 		seenSet[g] = struct{}{}
 	}

-	// 非洗版订阅：成功下载一次即满足，预先算一次媒体库与下载中任务的
-	// 可用性，用于跳过已入库/已在下载队列中的电影或剧集（对齐 MoviePilot）。
 	washOff := !sub.WashEnabled
-	var avail LocalAvailability
-	availQuery := ""
-	if washOff {
-		availQuery = availabilityQuery(subscriptionName(sub), subscriptionFilter(sub))
-		avail = mergeLocalAvailability(
-			SubscriptionLocalAvailability(ctx, s.repo, sub),
-			s.pendingDownloadAvailability(ctx, sub),
-		)
-	}
+	availQuery := availabilityQuery(subscriptionName(sub), subscriptionFilter(sub))
+	// RSS 和站点搜索统一使用候选规划：先按订阅规则过滤，再按洗版优先级/集数去重择优。
+	// 非洗版订阅成功下载一次即满足，媒体库与下载中任务会作为可用性输入避免重复下载。
+	avail := mergeLocalAvailability(
+		SubscriptionLocalAvailability(ctx, s.repo, sub),
+		s.pendingDownloadAvailability(ctx, sub),
+	)
+	candidates := selectRSSSubscriptionCandidates(feed.Channel.Items, sub, filter, seenSet, avail)

 	queued := 0
-	for _, item := range feed.Channel.Items {
-		guid := stableRSSItemGUID(item.Title, item.GUID, item.Link, item.Enclosure.URL)
-		if _, ok := seenSet[guid]; ok {
-			continue
-		}
-		if filter != nil && !filter.MatchString(item.Title) {
-			continue
-		}
-		// 应用订阅高级规则（排除词/分辨率/质量/特效/发布组）—— 此前 RSS 路径完全跳过，导致排除不生效。
-		if !matchesSubscriptionRules(sub, item.Title) {
-			continue
-		}
-		if washOff && subscriptionItemAlreadyAvailable(sub, avail, item.Title) {
-			continue
-		}
-		download := item.Enclosure.URL
-		if download == "" {
-			download = item.Link
-		}
-		if download == "" {
-			continue
-		}
+	for _, candidate := range candidates {
+		item := candidate.Item
+		guid := candidate.GUID
+		download := candidate.Download
 		mediaType, mediaCategory := s.classifySubscriptionItem(ctx, sub, item.Title, "")
 		savePath := s.resolveSubscriptionSavePath(ctx, sub, mediaType, mediaCategory)
 		if s.downloadPathHasCandidate(ctx, sub, item.Title, savePath) {
@@ -78,6 +78,52 @@ func selectSiteSearchCandidatesWithAvailability(results []SearchResult, sub *mod
 			Score:    score,
 		})
 	}
+	return selectPreparedSubscriptionCandidates(candidates, sub, local)
+}
+
+func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, filter *regexp.Regexp, seenSet map[string]struct{}, local LocalAvailability) []siteSearchCandidate {
+	if seenSet == nil {
+		seenSet = map[string]struct{}{}
+	}
+	candidates := make([]siteSearchCandidate, 0, len(items))
+	for _, item := range items {
+		title := strings.TrimSpace(item.Title)
+		if title == "" {
+			continue
+		}
+		if filter != nil && !filter.MatchString(title) {
+			continue
+		}
+		if !matchesSubscriptionRules(sub, title) {
+			continue
+		}
+		download := strings.TrimSpace(item.Enclosure.URL)
+		if download == "" {
+			download = strings.TrimSpace(item.Link)
+		}
+		if download == "" {
+			continue
+		}
+		guid := stableRSSItemGUID(title, item.GUID, item.Link, item.Enclosure.URL)
+		if _, ok := seenSet[guid]; ok {
+			continue
+		}
+		searchItem := SearchResult{Title: title, DownloadURL: download}
+		season, episode := ParseEpisode(title)
+		candidates = append(candidates, siteSearchCandidate{
+			Item:     searchItem,
+			Download: download,
+			GUID:     guid,
+			Season:   season,
+			Episode:  episode,
+			Pack:     isSeriesPackTitle(title),
+			Score:    subscriptionCandidateScore(sub, searchItem),
+		})
+	}
+	return selectPreparedSubscriptionCandidates(candidates, sub, local)
+}
+
+func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub *model.Subscription, local LocalAvailability) []siteSearchCandidate {
 	if len(candidates) > 1 {
 		sort.SliceStable(candidates, func(i, j int) bool {
 			if candidates[i].Score != candidates[j].Score {
@@ -1,6 +1,7 @@
 package service

 import (
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -593,6 +594,98 @@ func TestSubscriptionRunOneSkipsSameEpisodeAddedEarlierInFeed(t *testing.T) {
 	}
 }

+func TestSubscriptionRunOneRSSWashQueuesOnlyBestMovieVariant(t *testing.T) {
+	rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/rss+xml")
+		_, _ = w.Write([]byte(`<?xml version="1.0"?>
+<rss><channel>
+  <item>
+    <title>Dune 2021 1080p WEB-DL</title>
+    <guid>dune-1080-web</guid>
+    <link>magnet:?xt=urn:btih:dddddddddddddddddddddddddddddddddddddddd&amp;dn=Dune+2021+1080p+WEB-DL</link>
+  </item>
+  <item>
+    <title>Dune 2021 2160p UHD BluRay REMUX HDR</title>
+    <guid>dune-2160-remux</guid>
+    <link>magnet:?xt=urn:btih:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee&amp;dn=Dune+2021+2160p+REMUX</link>
+  </item>
+  <item>
+    <title>Dune 2021 720p HDTV</title>
+    <guid>dune-720-hdtv</guid>
+    <link>magnet:?xt=urn:btih:ffffffffffffffffffffffffffffffffffffffff&amp;dn=Dune+2021+720p+HDTV</link>
+  </item>
+</channel></rss>`))
+	}))
+	defer rss.Close()
+
+	var addCalls int32
+	var addedTitles []string
+	addedHashes := make([]string, 0, 3)
+	qb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v2/auth/login":
+			_, _ = w.Write([]byte("Ok."))
+		case "/api/v2/torrents/info":
+			if len(addedHashes) == 0 {
+				_, _ = w.Write([]byte(`[]`))
+				return
+			}
+			var items []string
+			for _, hash := range addedHashes {
+				items = append(items, `{"hash":"`+hash+`","name":"Dune 2021","state":"downloading","progress":0.1}`)
+			}
+			_, _ = w.Write([]byte(`[` + strings.Join(items, ",") + `]`))
+		case "/api/v2/torrents/add":
+			call := atomic.AddInt32(&addCalls, 1)
+			_ = r.ParseMultipartForm(10 << 20)
+			addedTitles = append(addedTitles, r.FormValue("urls"))
+			addedHashes = append(addedHashes, strings.Repeat(fmt.Sprintf("%x", call), 40))
+			_, _ = w.Write([]byte("Ok."))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer qb.Close()
+
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.AutoMigrate(&model.Subscription{}, &model.Setting{}, &model.DownloadTask{}, &model.Media{}, &model.DownloadClient{}); err != nil {
+		t.Fatal(err)
+	}
+	repos := repository.New(db)
+	configureTestDefaultQB(t, repos, qb.URL)
+	downloads := NewDownloadService(zap.NewNop(), repos, NewHub(zap.NewNop()), nil)
+	svc := NewSubscriptionService(nil, zap.NewNop(), repos, downloads, nil, NewHub(zap.NewNop()))
+
+	sub := &model.Subscription{
+		Name:         "Dune 自动订阅",
+		FeedURL:      rss.URL,
+		Filter:       "Dune 2021",
+		MediaType:    "movie",
+		WashEnabled:  true,
+		WashPriority: "resolution",
+		SavePath:     "/downloads/movies",
+	}
+	if err := repos.Subscription.Create(t.Context(), sub); err != nil {
+		t.Fatal(err)
+	}
+	queued, err := svc.runOne(t.Context(), sub)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if queued != 1 {
+		t.Fatalf("queued = %d, want 1 best movie variant", queued)
+	}
+	if got := atomic.LoadInt32(&addCalls); got != 1 {
+		t.Fatalf("qb add calls = %d, want 1", got)
+	}
+	if len(addedTitles) != 1 || !strings.Contains(addedTitles[0], "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") {
+		t.Fatalf("added %#v, want 2160p REMUX variant only", addedTitles)
+	}
+}
+
 func TestSubscriptionRunOneDoesNotUseDeletedDownloader(t *testing.T) {
 	rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.Header().Set("Content-Type", "application/rss+xml")