fix(subscription): prevent duplicate qb downloads

This commit is contained in:
ShukeBta
2026-06-07 10:05:49 +08:00
parent 22b64d3d47
commit d94330b30f
22 changed files with 990 additions and 75 deletions
+74 -5
View File
@@ -214,10 +214,7 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio
queued := 0
for _, item := range feed.Channel.Items {
guid := item.GUID
if guid == "" {
guid = item.Link
}
guid := stableRSSItemGUID(item.Title, item.GUID, item.Link, item.Enclosure.URL)
if _, ok := seenSet[guid]; ok {
continue
}
@@ -253,6 +250,11 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio
BackdropURL: sub.BackdropURL,
Overview: sub.Overview,
}); err != nil {
if errors.Is(err, ErrDownloadAlreadyExists) {
seen = append(seen, guid)
seenSet[guid] = struct{}{}
continue
}
s.log.Warn("subscription enqueue failed",
zap.String("title", item.Title),
zap.String("media_type", mediaType),
@@ -338,6 +340,11 @@ func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subs
BackdropURL: sub.BackdropURL,
Overview: sub.Overview,
}); err != nil {
if errors.Is(err, ErrDownloadAlreadyExists) {
seen = append(seen, candidate.GUID)
seenSet[candidate.GUID] = struct{}{}
continue
}
lastEnqueueErr = err
s.log.Warn("site-search subscription enqueue failed",
zap.String("subscription", sub.Name),
@@ -389,7 +396,7 @@ func selectSiteSearchCandidates(results []SearchResult, sub *model.Subscription,
if download == "" {
continue
}
guid := download
guid := stableSiteSearchGUID(item, download)
if _, ok := seenSet[guid]; ok {
continue
}
@@ -472,6 +479,68 @@ func selectSiteSearchCandidates(results []SearchResult, sub *model.Subscription,
return selected
}
func stableRSSItemGUID(title, guid, link, enclosureURL string) string {
parts := []string{"rss", strings.ToLower(strings.TrimSpace(title))}
for _, raw := range []string{guid, enclosureURL, link} {
if key := stableDownloadURLKey(raw); key != "" {
parts = append(parts, key)
return strings.Join(parts, "|")
}
if raw = strings.TrimSpace(raw); raw != "" {
parts = append(parts, strings.ToLower(raw))
return strings.Join(parts, "|")
}
}
return strings.Join(parts, "|")
}
func stableSiteSearchGUID(item SearchResult, download string) string {
parts := []string{
"site",
strings.ToLower(strings.TrimSpace(firstNonEmpty(item.SiteID, item.SiteName))),
strings.ToLower(strings.TrimSpace(item.Category)),
strings.ToLower(strings.TrimSpace(item.Title)),
fmt.Sprintf("%d", item.Size),
}
if key := stableDownloadURLKey(download); key != "" {
parts = append(parts, key)
}
return strings.Join(parts, "|")
}
func stableDownloadURLKey(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
u, err := url.Parse(raw)
if err != nil {
return strings.ToLower(raw)
}
if strings.EqualFold(u.Scheme, "magnet") {
xt := strings.ToLower(strings.TrimSpace(u.Query().Get("xt")))
if xt != "" {
return "magnet:" + xt
}
return strings.ToLower(raw)
}
if u.Host == "" {
return strings.ToLower(raw)
}
q := u.Query()
kept := make([]string, 0, 4)
for _, key := range []string{"id", "tid", "torrent", "torrent_id", "torrentid", "hash", "info_hash"} {
if value := strings.TrimSpace(q.Get(key)); value != "" {
kept = append(kept, key+"="+strings.ToLower(value))
}
}
base := strings.ToLower(strings.TrimRight(u.Host, "/") + "/" + strings.TrimLeft(u.Path, "/"))
if len(kept) > 0 {
return base + "?" + strings.Join(kept, "&")
}
return base
}
// defaultExcludeWords 是参考 MoviePilot 默认过滤的「垃圾版本」排除清单,对所有订阅生效,
// 与用户自定义排除词合并。拉丁词在 containsAnyExcludeToken 里按词边界匹配以避免子串误伤。
const defaultExcludeWords = "cam,ts,tc,telesync,telecine,hdcam,hdts,枪版,抢先,抢鲜,预告,trailer,sample"