From 76d6e0c8ff3bd8181bfa77e95cdd2a5bfc12a8e8 Mon Sep 17 00:00:00 2001 From: ShukeBta Date: Mon, 8 Jun 2026 21:49:51 +0800 Subject: [PATCH] fix: dedupe RSS subscription variants --- internal/service/subscription.go | 65 ++++++----------- internal/service/subscription_planner.go | 46 ++++++++++++ internal/service/subscription_test.go | 93 ++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 42 deletions(-) diff --git a/internal/service/subscription.go b/internal/service/subscription.go index 38c1db3..27579bd 100644 --- a/internal/service/subscription.go +++ b/internal/service/subscription.go @@ -61,18 +61,20 @@ func (s *SubscriptionService) Stop() { close(s.stop) } type rssFeed struct { XMLName xml.Name `xml:"rss"` Channel struct { - Items []struct { - Title string `xml:"title"` - Link string `xml:"link"` - GUID string `xml:"guid"` - Description string `xml:"description"` - Enclosure struct { - URL string `xml:"url,attr"` - } `xml:"enclosure"` - } `xml:"item"` + Items []rssItem `xml:"item"` } `xml:"channel"` } +type rssItem struct { + Title string `xml:"title"` + Link string `xml:"link"` + GUID string `xml:"guid"` + Description string `xml:"description"` + Enclosure struct { + URL string `xml:"url,attr"` + } `xml:"enclosure"` +} + // Create persists a new subscription. func (s *SubscriptionService) Create(ctx context.Context, sub *model.Subscription) error { if sub.Name == "" || sub.FeedURL == "" { @@ -186,42 +188,21 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio seenSet[g] = struct{}{} } - // 非洗版订阅：成功下载一次即满足，预先算一次媒体库与下载中任务的 - // 可用性，用于跳过已入库/已在下载队列中的电影或剧集（对齐 MoviePilot）。 washOff := !sub.WashEnabled - var avail LocalAvailability - availQuery := "" - if washOff { - availQuery = availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)) - avail = mergeLocalAvailability( - SubscriptionLocalAvailability(ctx, s.repo, sub), - s.pendingDownloadAvailability(ctx, sub), - ) - } + availQuery := availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)) + // RSS 和站点搜索统一使用候选规划：先按订阅规则过滤，再按洗版优先级/集数去重择优。 + // 非洗版订阅成功下载一次即满足，媒体库与下载中任务会作为可用性输入避免重复下载。 + avail := mergeLocalAvailability( + SubscriptionLocalAvailability(ctx, s.repo, sub), + s.pendingDownloadAvailability(ctx, sub), + ) + candidates := selectRSSSubscriptionCandidates(feed.Channel.Items, sub, filter, seenSet, avail) queued := 0 - for _, item := range feed.Channel.Items { - guid := stableRSSItemGUID(item.Title, item.GUID, item.Link, item.Enclosure.URL) - if _, ok := seenSet[guid]; ok { - continue - } - if filter != nil && !filter.MatchString(item.Title) { - continue - } - // 应用订阅高级规则（排除词/分辨率/质量/特效/发布组）—— 此前 RSS 路径完全跳过，导致排除不生效。 - if !matchesSubscriptionRules(sub, item.Title) { - continue - } - if washOff && subscriptionItemAlreadyAvailable(sub, avail, item.Title) { - continue - } - download := item.Enclosure.URL - if download == "" { - download = item.Link - } - if download == "" { - continue - } + for _, candidate := range candidates { + item := candidate.Item + guid := candidate.GUID + download := candidate.Download mediaType, mediaCategory := s.classifySubscriptionItem(ctx, sub, item.Title, "") savePath := s.resolveSubscriptionSavePath(ctx, sub, mediaType, mediaCategory) if s.downloadPathHasCandidate(ctx, sub, item.Title, savePath) { diff --git a/internal/service/subscription_planner.go b/internal/service/subscription_planner.go index 8562695..4a49d8d 100644 --- a/internal/service/subscription_planner.go +++ b/internal/service/subscription_planner.go @@ -78,6 +78,52 @@ func selectSiteSearchCandidatesWithAvailability(results []SearchResult, sub *mod Score: score, }) } + return selectPreparedSubscriptionCandidates(candidates, sub, local) +} + +func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, filter *regexp.Regexp, seenSet map[string]struct{}, local LocalAvailability) []siteSearchCandidate { + if seenSet == nil { + seenSet = map[string]struct{}{} + } + candidates := make([]siteSearchCandidate, 0, len(items)) + for _, item := range items { + title := strings.TrimSpace(item.Title) + if title == "" { + continue + } + if filter != nil && !filter.MatchString(title) { + continue + } + if !matchesSubscriptionRules(sub, title) { + continue + } + download := strings.TrimSpace(item.Enclosure.URL) + if download == "" { + download = strings.TrimSpace(item.Link) + } + if download == "" { + continue + } + guid := stableRSSItemGUID(title, item.GUID, item.Link, item.Enclosure.URL) + if _, ok := seenSet[guid]; ok { + continue + } + searchItem := SearchResult{Title: title, DownloadURL: download} + season, episode := ParseEpisode(title) + candidates = append(candidates, siteSearchCandidate{ + Item: searchItem, + Download: download, + GUID: guid, + Season: season, + Episode: episode, + Pack: isSeriesPackTitle(title), + Score: subscriptionCandidateScore(sub, searchItem), + }) + } + return selectPreparedSubscriptionCandidates(candidates, sub, local) +} + +func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub *model.Subscription, local LocalAvailability) []siteSearchCandidate { if len(candidates) > 1 { sort.SliceStable(candidates, func(i, j int) bool { if candidates[i].Score != candidates[j].Score { diff --git a/internal/service/subscription_test.go b/internal/service/subscription_test.go index a5a29e7..48d1b6d 100644 --- a/internal/service/subscription_test.go +++ b/internal/service/subscription_test.go @@ -1,6 +1,7 @@ package service import ( + "fmt" "net/http" "net/http/httptest" "os" @@ -593,6 +594,98 @@ func TestSubscriptionRunOneSkipsSameEpisodeAddedEarlierInFeed(t *testing.T) { } } +func TestSubscriptionRunOneRSSWashQueuesOnlyBestMovieVariant(t *testing.T) { + rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/rss+xml") + _, _ = w.Write([]byte(` + + + Dune 2021 1080p WEB-DL + dune-1080-web + magnet:?xt=urn:btih:dddddddddddddddddddddddddddddddddddddddd&dn=Dune+2021+1080p+WEB-DL + + + Dune 2021 2160p UHD BluRay REMUX HDR + dune-2160-remux + magnet:?xt=urn:btih:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee&dn=Dune+2021+2160p+REMUX + + + Dune 2021 720p HDTV + dune-720-hdtv + magnet:?xt=urn:btih:ffffffffffffffffffffffffffffffffffffffff&dn=Dune+2021+720p+HDTV + +`)) + })) + defer rss.Close() + + var addCalls int32 + var addedTitles []string + addedHashes := make([]string, 0, 3) + qb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v2/auth/login": + _, _ = w.Write([]byte("Ok.")) + case "/api/v2/torrents/info": + if len(addedHashes) == 0 { + _, _ = w.Write([]byte(`[]`)) + return + } + var items []string + for _, hash := range addedHashes { + items = append(items, `{"hash":"`+hash+`","name":"Dune 2021","state":"downloading","progress":0.1}`) + } + _, _ = w.Write([]byte(`[` + strings.Join(items, ",") + `]`)) + case "/api/v2/torrents/add": + call := atomic.AddInt32(&addCalls, 1) + _ = r.ParseMultipartForm(10 << 20) + addedTitles = append(addedTitles, r.FormValue("urls")) + addedHashes = append(addedHashes, strings.Repeat(fmt.Sprintf("%x", call), 40)) + _, _ = w.Write([]byte("Ok.")) + default: + http.NotFound(w, r) + } + })) + defer qb.Close() + + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Subscription{}, &model.Setting{}, &model.DownloadTask{}, &model.Media{}, &model.DownloadClient{}); err != nil { + t.Fatal(err) + } + repos := repository.New(db) + configureTestDefaultQB(t, repos, qb.URL) + downloads := NewDownloadService(zap.NewNop(), repos, NewHub(zap.NewNop()), nil) + svc := NewSubscriptionService(nil, zap.NewNop(), repos, downloads, nil, NewHub(zap.NewNop())) + + sub := &model.Subscription{ + Name: "Dune 自动订阅", + FeedURL: rss.URL, + Filter: "Dune 2021", + MediaType: "movie", + WashEnabled: true, + WashPriority: "resolution", + SavePath: "/downloads/movies", + } + if err := repos.Subscription.Create(t.Context(), sub); err != nil { + t.Fatal(err) + } + queued, err := svc.runOne(t.Context(), sub) + if err != nil { + t.Fatal(err) + } + if queued != 1 { + t.Fatalf("queued = %d, want 1 best movie variant", queued) + } + if got := atomic.LoadInt32(&addCalls); got != 1 { + t.Fatalf("qb add calls = %d, want 1", got) + } + if len(addedTitles) != 1 || !strings.Contains(addedTitles[0], "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") { + t.Fatalf("added %#v, want 2160p REMUX variant only", addedTitles) + } +} + func TestSubscriptionRunOneDoesNotUseDeletedDownloader(t *testing.T) { rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/rss+xml")