mirror of
https://github.com/truewhile/MeBox.git
synced 2026-10-03 04:26:36 +08:00
fix: dedupe RSS subscription variants
This commit is contained in:
@@ -61,18 +61,20 @@ func (s *SubscriptionService) Stop() { close(s.stop) }
|
|||||||
type rssFeed struct {
|
type rssFeed struct {
|
||||||
XMLName xml.Name `xml:"rss"`
|
XMLName xml.Name `xml:"rss"`
|
||||||
Channel struct {
|
Channel struct {
|
||||||
Items []struct {
|
Items []rssItem `xml:"item"`
|
||||||
Title string `xml:"title"`
|
|
||||||
Link string `xml:"link"`
|
|
||||||
GUID string `xml:"guid"`
|
|
||||||
Description string `xml:"description"`
|
|
||||||
Enclosure struct {
|
|
||||||
URL string `xml:"url,attr"`
|
|
||||||
} `xml:"enclosure"`
|
|
||||||
} `xml:"item"`
|
|
||||||
} `xml:"channel"`
|
} `xml:"channel"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type rssItem struct {
|
||||||
|
Title string `xml:"title"`
|
||||||
|
Link string `xml:"link"`
|
||||||
|
GUID string `xml:"guid"`
|
||||||
|
Description string `xml:"description"`
|
||||||
|
Enclosure struct {
|
||||||
|
URL string `xml:"url,attr"`
|
||||||
|
} `xml:"enclosure"`
|
||||||
|
}
|
||||||
|
|
||||||
// Create persists a new subscription.
|
// Create persists a new subscription.
|
||||||
func (s *SubscriptionService) Create(ctx context.Context, sub *model.Subscription) error {
|
func (s *SubscriptionService) Create(ctx context.Context, sub *model.Subscription) error {
|
||||||
if sub.Name == "" || sub.FeedURL == "" {
|
if sub.Name == "" || sub.FeedURL == "" {
|
||||||
@@ -186,42 +188,21 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio
|
|||||||
seenSet[g] = struct{}{}
|
seenSet[g] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 非洗版订阅:成功下载一次即满足,预先算一次媒体库与下载中任务的
|
|
||||||
// 可用性,用于跳过已入库/已在下载队列中的电影或剧集(对齐 MoviePilot)。
|
|
||||||
washOff := !sub.WashEnabled
|
washOff := !sub.WashEnabled
|
||||||
var avail LocalAvailability
|
availQuery := availabilityQuery(subscriptionName(sub), subscriptionFilter(sub))
|
||||||
availQuery := ""
|
// RSS 和站点搜索统一使用候选规划:先按订阅规则过滤,再按洗版优先级/集数去重择优。
|
||||||
if washOff {
|
// 非洗版订阅成功下载一次即满足,媒体库与下载中任务会作为可用性输入避免重复下载。
|
||||||
availQuery = availabilityQuery(subscriptionName(sub), subscriptionFilter(sub))
|
avail := mergeLocalAvailability(
|
||||||
avail = mergeLocalAvailability(
|
SubscriptionLocalAvailability(ctx, s.repo, sub),
|
||||||
SubscriptionLocalAvailability(ctx, s.repo, sub),
|
s.pendingDownloadAvailability(ctx, sub),
|
||||||
s.pendingDownloadAvailability(ctx, sub),
|
)
|
||||||
)
|
candidates := selectRSSSubscriptionCandidates(feed.Channel.Items, sub, filter, seenSet, avail)
|
||||||
}
|
|
||||||
|
|
||||||
queued := 0
|
queued := 0
|
||||||
for _, item := range feed.Channel.Items {
|
for _, candidate := range candidates {
|
||||||
guid := stableRSSItemGUID(item.Title, item.GUID, item.Link, item.Enclosure.URL)
|
item := candidate.Item
|
||||||
if _, ok := seenSet[guid]; ok {
|
guid := candidate.GUID
|
||||||
continue
|
download := candidate.Download
|
||||||
}
|
|
||||||
if filter != nil && !filter.MatchString(item.Title) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// 应用订阅高级规则(排除词/分辨率/质量/特效/发布组)—— 此前 RSS 路径完全跳过,导致排除不生效。
|
|
||||||
if !matchesSubscriptionRules(sub, item.Title) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if washOff && subscriptionItemAlreadyAvailable(sub, avail, item.Title) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
download := item.Enclosure.URL
|
|
||||||
if download == "" {
|
|
||||||
download = item.Link
|
|
||||||
}
|
|
||||||
if download == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
mediaType, mediaCategory := s.classifySubscriptionItem(ctx, sub, item.Title, "")
|
mediaType, mediaCategory := s.classifySubscriptionItem(ctx, sub, item.Title, "")
|
||||||
savePath := s.resolveSubscriptionSavePath(ctx, sub, mediaType, mediaCategory)
|
savePath := s.resolveSubscriptionSavePath(ctx, sub, mediaType, mediaCategory)
|
||||||
if s.downloadPathHasCandidate(ctx, sub, item.Title, savePath) {
|
if s.downloadPathHasCandidate(ctx, sub, item.Title, savePath) {
|
||||||
|
|||||||
@@ -78,6 +78,52 @@ func selectSiteSearchCandidatesWithAvailability(results []SearchResult, sub *mod
|
|||||||
Score: score,
|
Score: score,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
return selectPreparedSubscriptionCandidates(candidates, sub, local)
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, filter *regexp.Regexp, seenSet map[string]struct{}, local LocalAvailability) []siteSearchCandidate {
|
||||||
|
if seenSet == nil {
|
||||||
|
seenSet = map[string]struct{}{}
|
||||||
|
}
|
||||||
|
candidates := make([]siteSearchCandidate, 0, len(items))
|
||||||
|
for _, item := range items {
|
||||||
|
title := strings.TrimSpace(item.Title)
|
||||||
|
if title == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if filter != nil && !filter.MatchString(title) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !matchesSubscriptionRules(sub, title) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
download := strings.TrimSpace(item.Enclosure.URL)
|
||||||
|
if download == "" {
|
||||||
|
download = strings.TrimSpace(item.Link)
|
||||||
|
}
|
||||||
|
if download == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
guid := stableRSSItemGUID(title, item.GUID, item.Link, item.Enclosure.URL)
|
||||||
|
if _, ok := seenSet[guid]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
searchItem := SearchResult{Title: title, DownloadURL: download}
|
||||||
|
season, episode := ParseEpisode(title)
|
||||||
|
candidates = append(candidates, siteSearchCandidate{
|
||||||
|
Item: searchItem,
|
||||||
|
Download: download,
|
||||||
|
GUID: guid,
|
||||||
|
Season: season,
|
||||||
|
Episode: episode,
|
||||||
|
Pack: isSeriesPackTitle(title),
|
||||||
|
Score: subscriptionCandidateScore(sub, searchItem),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return selectPreparedSubscriptionCandidates(candidates, sub, local)
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub *model.Subscription, local LocalAvailability) []siteSearchCandidate {
|
||||||
if len(candidates) > 1 {
|
if len(candidates) > 1 {
|
||||||
sort.SliceStable(candidates, func(i, j int) bool {
|
sort.SliceStable(candidates, func(i, j int) bool {
|
||||||
if candidates[i].Score != candidates[j].Score {
|
if candidates[i].Score != candidates[j].Score {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package service
|
package service
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
@@ -593,6 +594,98 @@ func TestSubscriptionRunOneSkipsSameEpisodeAddedEarlierInFeed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubscriptionRunOneRSSWashQueuesOnlyBestMovieVariant(t *testing.T) {
|
||||||
|
rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/rss+xml")
|
||||||
|
_, _ = w.Write([]byte(`<?xml version="1.0"?>
|
||||||
|
<rss><channel>
|
||||||
|
<item>
|
||||||
|
<title>Dune 2021 1080p WEB-DL</title>
|
||||||
|
<guid>dune-1080-web</guid>
|
||||||
|
<link>magnet:?xt=urn:btih:dddddddddddddddddddddddddddddddddddddddd&dn=Dune+2021+1080p+WEB-DL</link>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>Dune 2021 2160p UHD BluRay REMUX HDR</title>
|
||||||
|
<guid>dune-2160-remux</guid>
|
||||||
|
<link>magnet:?xt=urn:btih:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee&dn=Dune+2021+2160p+REMUX</link>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>Dune 2021 720p HDTV</title>
|
||||||
|
<guid>dune-720-hdtv</guid>
|
||||||
|
<link>magnet:?xt=urn:btih:ffffffffffffffffffffffffffffffffffffffff&dn=Dune+2021+720p+HDTV</link>
|
||||||
|
</item>
|
||||||
|
</channel></rss>`))
|
||||||
|
}))
|
||||||
|
defer rss.Close()
|
||||||
|
|
||||||
|
var addCalls int32
|
||||||
|
var addedTitles []string
|
||||||
|
addedHashes := make([]string, 0, 3)
|
||||||
|
qb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/api/v2/auth/login":
|
||||||
|
_, _ = w.Write([]byte("Ok."))
|
||||||
|
case "/api/v2/torrents/info":
|
||||||
|
if len(addedHashes) == 0 {
|
||||||
|
_, _ = w.Write([]byte(`[]`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var items []string
|
||||||
|
for _, hash := range addedHashes {
|
||||||
|
items = append(items, `{"hash":"`+hash+`","name":"Dune 2021","state":"downloading","progress":0.1}`)
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`[` + strings.Join(items, ",") + `]`))
|
||||||
|
case "/api/v2/torrents/add":
|
||||||
|
call := atomic.AddInt32(&addCalls, 1)
|
||||||
|
_ = r.ParseMultipartForm(10 << 20)
|
||||||
|
addedTitles = append(addedTitles, r.FormValue("urls"))
|
||||||
|
addedHashes = append(addedHashes, strings.Repeat(fmt.Sprintf("%x", call), 40))
|
||||||
|
_, _ = w.Write([]byte("Ok."))
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer qb.Close()
|
||||||
|
|
||||||
|
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := db.AutoMigrate(&model.Subscription{}, &model.Setting{}, &model.DownloadTask{}, &model.Media{}, &model.DownloadClient{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
repos := repository.New(db)
|
||||||
|
configureTestDefaultQB(t, repos, qb.URL)
|
||||||
|
downloads := NewDownloadService(zap.NewNop(), repos, NewHub(zap.NewNop()), nil)
|
||||||
|
svc := NewSubscriptionService(nil, zap.NewNop(), repos, downloads, nil, NewHub(zap.NewNop()))
|
||||||
|
|
||||||
|
sub := &model.Subscription{
|
||||||
|
Name: "Dune 自动订阅",
|
||||||
|
FeedURL: rss.URL,
|
||||||
|
Filter: "Dune 2021",
|
||||||
|
MediaType: "movie",
|
||||||
|
WashEnabled: true,
|
||||||
|
WashPriority: "resolution",
|
||||||
|
SavePath: "/downloads/movies",
|
||||||
|
}
|
||||||
|
if err := repos.Subscription.Create(t.Context(), sub); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
queued, err := svc.runOne(t.Context(), sub)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if queued != 1 {
|
||||||
|
t.Fatalf("queued = %d, want 1 best movie variant", queued)
|
||||||
|
}
|
||||||
|
if got := atomic.LoadInt32(&addCalls); got != 1 {
|
||||||
|
t.Fatalf("qb add calls = %d, want 1", got)
|
||||||
|
}
|
||||||
|
if len(addedTitles) != 1 || !strings.Contains(addedTitles[0], "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") {
|
||||||
|
t.Fatalf("added %#v, want 2160p REMUX variant only", addedTitles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubscriptionRunOneDoesNotUseDeletedDownloader(t *testing.T) {
|
func TestSubscriptionRunOneDoesNotUseDeletedDownloader(t *testing.T) {
|
||||||
rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
w.Header().Set("Content-Type", "application/rss+xml")
|
w.Header().Set("Content-Type", "application/rss+xml")
|
||||||
|
|||||||
Reference in New Issue
Block a user