From db65e54c450729057970c963a38718baeca6db07 Mon Sep 17 00:00:00 2001 From: ShukeBta Date: Fri, 29 May 2026 17:39:57 +0800 Subject: [PATCH] fix: avoid duplicate subscription downloads --- README.md | 24 +-- README_EN.md | 22 +-- docker-compose.yml | 2 +- internal/service/subscription.go | 213 +++++++++++++++++++++++++- internal/service/subscription_test.go | 54 +++++++ 5 files changed, 290 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 2b0a708..6e53e64 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ mkdir -p data cache media downloads ```bash cat > .env <<'EOF' # 固定版本；需要升级时改成新的 MediaStationGo-vX.Y.Z 后执行 docker compose pull && docker compose up -d -MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.19 +MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.20 MEDIASTATION_HTTP_PORT=18080 # 程序数据和缓存建议放在 MediaStationGo 部署目录下，便于备份和迁移。 @@ -307,7 +307,7 @@ vim docker-compose.yml # # 镜像版本： # 默认拉取 latest；如需固定版本，创建 .env 并写入： -# MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.19 +# MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.20 # # 路径映射总览： # /data 程序数据目录。保存 SQLite 数据库、JWT secret、系统配置等，必须持久化。 @@ -516,7 +516,7 @@ docker compose up -d ```bash cat > .env <<'EOF' -MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.19 +MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.20 MEDIASTATION_HTTP_PORT=18080 MEDIASTATION_DATA_DIR=./data MEDIASTATION_CACHE_DIR=./cache @@ -779,26 +779,26 @@ cd MediaStationGo | 平台 | 包名示例 | | --- | --- | -| Linux x86_64 | `MediaStationGo-v0.0.19-linux-amd64.tar.gz` | -| Linux ARM64 | `MediaStationGo-v0.0.19-linux-arm64.tar.gz` | -| Windows x86_64 | `MediaStationGo-v0.0.19-windows-amd64.zip` | -| macOS Intel | `MediaStationGo-v0.0.19-darwin-amd64.tar.gz` | -| macOS Apple Silicon | `MediaStationGo-v0.0.19-darwin-arm64.tar.gz` | +| Linux x86_64 | `MediaStationGo-v0.0.20-linux-amd64.tar.gz` | +| Linux ARM64 | `MediaStationGo-v0.0.20-linux-arm64.tar.gz` | +| Windows x86_64 | `MediaStationGo-v0.0.20-windows-amd64.zip` | +| macOS Intel | `MediaStationGo-v0.0.20-darwin-amd64.tar.gz` | +| macOS Apple Silicon | `MediaStationGo-v0.0.20-darwin-arm64.tar.gz` | 部署步骤： ```bash # Linux 示例 -tar -xzf MediaStationGo-v0.0.19-linux-amd64.tar.gz -cd MediaStationGo-v0.0.19-linux-amd64 +tar -xzf MediaStationGo-v0.0.20-linux-amd64.tar.gz +cd MediaStationGo-v0.0.20-linux-amd64 MEDIASTATION_APP_PORT=18080 ./mediastation-go ``` Windows： ```powershell -Expand-Archive .\MediaStationGo-v0.0.19-windows-amd64.zip -cd .\MediaStationGo-v0.0.19-windows-amd64 +Expand-Archive .\MediaStationGo-v0.0.20-windows-amd64.zip +cd .\MediaStationGo-v0.0.20-windows-amd64 $env:MEDIASTATION_APP_PORT = "18080" .\mediastation-go.exe ``` diff --git a/README_EN.md b/README_EN.md index 5b485a1..c47966f 100644 --- a/README_EN.md +++ b/README_EN.md @@ -235,7 +235,7 @@ mkdir -p data cache media downloads ```bash cat > .env <<'EOF' -MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.19 +MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.20 MEDIASTATION_HTTP_PORT=18080 MEDIASTATION_DATA_DIR=./data MEDIASTATION_CACHE_DIR=./cache @@ -344,7 +344,7 @@ For production, pin a specific release tag instead of using `latest`. Recommende ```bash cat > .env <<'EOF' -MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.19 +MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.20 MEDIASTATION_HTTP_PORT=18080 MEDIASTATION_DATA_DIR=./data MEDIASTATION_CACHE_DIR=./cache @@ -593,25 +593,25 @@ Each release provides multi-platform archives: | Platform | Package example | | --- | --- | -| Linux x86_64 | `MediaStationGo-v0.0.19-linux-amd64.tar.gz` | -| Linux ARM64 | `MediaStationGo-v0.0.19-linux-arm64.tar.gz` | -| Windows x86_64 | `MediaStationGo-v0.0.19-windows-amd64.zip` | -| macOS Intel | `MediaStationGo-v0.0.19-darwin-amd64.tar.gz` | -| macOS Apple Silicon | `MediaStationGo-v0.0.19-darwin-arm64.tar.gz` | +| Linux x86_64 | `MediaStationGo-v0.0.20-linux-amd64.tar.gz` | +| Linux ARM64 | `MediaStationGo-v0.0.20-linux-arm64.tar.gz` | +| Windows x86_64 | `MediaStationGo-v0.0.20-windows-amd64.zip` | +| macOS Intel | `MediaStationGo-v0.0.20-darwin-amd64.tar.gz` | +| macOS Apple Silicon | `MediaStationGo-v0.0.20-darwin-arm64.tar.gz` | Linux example: ```bash -tar -xzf MediaStationGo-v0.0.19-linux-amd64.tar.gz -cd MediaStationGo-v0.0.19-linux-amd64 +tar -xzf MediaStationGo-v0.0.20-linux-amd64.tar.gz +cd MediaStationGo-v0.0.20-linux-amd64 MEDIASTATION_APP_PORT=18080 ./mediastation-go ``` Windows example: ```powershell -Expand-Archive .\MediaStationGo-v0.0.19-windows-amd64.zip -cd .\MediaStationGo-v0.0.19-windows-amd64 +Expand-Archive .\MediaStationGo-v0.0.20-windows-amd64.zip +cd .\MediaStationGo-v0.0.20-windows-amd64 $env:MEDIASTATION_APP_PORT = "18080" .\mediastation-go.exe ``` diff --git a/docker-compose.yml b/docker-compose.yml index f352f83..4a4a022 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ # # 镜像版本： # 默认拉取 latest；如需固定版本，创建 .env 并写入： -# MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.19 +# MEDIASTATION_IMAGE_TAG=MediaStationGo-v0.0.20 # # 路径映射总览： # /data 程序数据目录。保存 SQLite 数据库、JWT secret、系统配置等，必须持久化。 diff --git a/internal/service/subscription.go b/internal/service/subscription.go index d89f25c..cc1a1be 100644 --- a/internal/service/subscription.go +++ b/internal/service/subscription.go @@ -14,10 +14,13 @@ import ( "io" "net/http" "net/url" + "os" + "path/filepath" "regexp" "sort" "strings" "time" + "unicode" "go.uber.org/zap" @@ -223,6 +226,11 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio } mediaType, mediaCategory := s.classifySubscriptionItem(ctx, sub, item.Title, "") savePath := s.resolveSubscriptionSavePath(ctx, sub, mediaType, mediaCategory) + if s.downloadPathHasCandidate(ctx, sub, item.Title, savePath) { + seen = append(seen, guid) + seenSet[guid] = struct{}{} + continue + } if _, err := s.downloads.AddDownloadWithMeta(ctx, sub.UserID, download, savePath, DownloadTaskMeta{ Title: firstNonEmpty(item.Title, sub.Name), PosterURL: sub.PosterURL, @@ -285,7 +293,10 @@ func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subs seenSet[g] = struct{}{} } - availability := SubscriptionLocalAvailability(ctx, s.repo, sub) + availability := mergeLocalAvailability( + SubscriptionLocalAvailability(ctx, s.repo, sub), + s.pendingDownloadAvailability(ctx, sub), + ) candidates := selectSiteSearchCandidates(results, sub, seenSet, availability) var lastEnqueueErr error queued := 0 @@ -300,6 +311,11 @@ func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subs } realURL := s.site.ResolveDownloadURL(ctx, candidate.Download) savePath := s.resolveSubscriptionSavePath(ctx, sub, mediaType, mediaCategory) + if s.downloadPathHasCandidate(ctx, sub, candidate.Item.Title, savePath) { + seen = append(seen, candidate.GUID) + seenSet[candidate.GUID] = struct{}{} + continue + } if _, err := s.downloads.AddDownloadWithMeta(ctx, sub.UserID, realURL, savePath, DownloadTaskMeta{ Title: firstNonEmpty(item.Title, sub.Name), PosterURL: sub.PosterURL, @@ -630,6 +646,201 @@ func (s *SubscriptionService) shouldSkipExistingTorrent(ctx context.Context, med return s.downloads.TorrentExistsByName(ctx, candidate.Item.Title) } +func (s *SubscriptionService) pendingDownloadAvailability(ctx context.Context, sub *model.Subscription) LocalAvailability { + out := LocalAvailability{ + ExistingEpisodeKeys: map[string]struct{}{}, + MissingEpisodeKeys: map[string]struct{}{}, + } + if sub != nil { + out.TotalEpisodes = sub.TotalEpisodes + } + root := s.subscriptionBaseSavePath(ctx, sub) + query := availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)) + if root == "" || query == "" { + return out + } + _ = scanDownloadPath(ctx, root, query, func(_ string, season, episode int) bool { + out.LocalMediaCount++ + out.InLibrary = true + if episode > 0 { + out.ExistingEpisodeKeys[episodeKey(season, episode)] = struct{}{} + } + return true + }) + mediaType := "" + if sub != nil { + mediaType = sub.MediaType + } + if isSubscriptionSeriesType(mediaType) || len(out.ExistingEpisodeKeys) > 0 { + out.DownloadedEpisodes = len(out.ExistingEpisodeKeys) + out.MissingEpisodes = missingEpisodes(out.ExistingEpisodeKeys, out.TotalEpisodes) + for _, episode := range out.MissingEpisodes { + out.MissingEpisodeKeys[episodeKey(1, episode)] = struct{}{} + } + } else if out.LocalMediaCount > 0 { + out.DownloadedEpisodes = 1 + if out.TotalEpisodes == 0 { + out.TotalEpisodes = 1 + } + } + return out +} + +func (s *SubscriptionService) subscriptionBaseSavePath(ctx context.Context, sub *model.Subscription) string { + if sub == nil { + return "" + } + base := strings.TrimSpace(sub.SavePath) + if base == "" && s != nil && s.repo != nil && s.repo.Setting != nil { + base, _ = s.repo.Setting.Get(ctx, "qbittorrent.savepath") + } + return base +} + +func subscriptionName(sub *model.Subscription) string { + if sub == nil { + return "" + } + return sub.Name +} + +func subscriptionFilter(sub *model.Subscription) string { + if sub == nil { + return "" + } + return sub.Filter +} + +func mergeLocalAvailability(values ...LocalAvailability) LocalAvailability { + out := LocalAvailability{ + ExistingEpisodeKeys: map[string]struct{}{}, + MissingEpisodeKeys: map[string]struct{}{}, + } + for _, value := range values { + if out.TotalEpisodes == 0 { + out.TotalEpisodes = value.TotalEpisodes + } + out.LocalMediaCount += value.LocalMediaCount + out.InLibrary = out.InLibrary || value.InLibrary + for key := range value.ExistingEpisodeKeys { + out.ExistingEpisodeKeys[key] = struct{}{} + } + } + out.DownloadedEpisodes = len(out.ExistingEpisodeKeys) + if out.TotalEpisodes > 0 { + out.MissingEpisodes = missingEpisodes(out.ExistingEpisodeKeys, out.TotalEpisodes) + for _, episode := range out.MissingEpisodes { + out.MissingEpisodeKeys[episodeKey(1, episode)] = struct{}{} + } + } + if out.DownloadedEpisodes == 0 && out.LocalMediaCount > 0 { + out.DownloadedEpisodes = out.LocalMediaCount + if out.TotalEpisodes == 0 { + out.TotalEpisodes = 1 + } + } + return out +} + +func (s *SubscriptionService) downloadPathHasCandidate(ctx context.Context, sub *model.Subscription, title, savePath string) bool { + savePath = strings.TrimSpace(savePath) + if savePath == "" { + savePath = s.subscriptionBaseSavePath(ctx, sub) + } + query := availabilityQuery(title, subscriptionFilter(sub)) + if savePath == "" || query == "" { + return false + } + wantSeason, wantEpisode := ParseEpisode(title) + if wantSeason <= 0 { + wantSeason = 1 + } + found := false + _ = scanDownloadPath(ctx, savePath, query, func(path string, season, episode int) bool { + if wantEpisode <= 0 { + found = true + return false + } + if episode <= 0 { + return true + } + if season <= 0 { + season = 1 + } + if episodeKey(season, episode) == episodeKey(wantSeason, wantEpisode) { + found = true + return false + } + return true + }) + return found +} + +func scanDownloadPath(ctx context.Context, root, query string, visit func(path string, season, episode int) bool) error { + if strings.TrimSpace(root) == "" { + return nil + } + info, err := os.Stat(root) + if err != nil || !info.IsDir() { + return nil + } + normalizedQuery := normalizeAvailabilityComparable(query) + if normalizedQuery == "" { + return nil + } + visited := 0 + return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + if d.IsDir() { + if path != root && strings.HasPrefix(filepath.Base(path), ".") { + return filepath.SkipDir + } + return nil + } + if !isDownloadMediaPath(path) { + return nil + } + visited++ + if visited > 10000 { + return filepath.SkipAll + } + if !strings.Contains(normalizeAvailabilityComparable(path), normalizedQuery) { + return nil + } + season, episode := ParseEpisode(path) + if !visit(path, season, episode) { + return filepath.SkipAll + } + return nil + }) +} + +func isDownloadMediaPath(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".!qb", ".part", ".aria2", ".crdownload": + path = strings.TrimSuffix(path, filepath.Ext(path)) + ext = strings.ToLower(filepath.Ext(path)) + } + _, ok := videoExtensions[ext] + return ok +} + +func normalizeAvailabilityComparable(value string) string { + var b strings.Builder + for _, r := range strings.ToLower(value) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } + } + return b.String() +} + func siteSearchKeyword(sub *model.Subscription) string { if sub == nil { return "" diff --git a/internal/service/subscription_test.go b/internal/service/subscription_test.go index 257288b..231638f 100644 --- a/internal/service/subscription_test.go +++ b/internal/service/subscription_test.go @@ -1,6 +1,8 @@ package service import ( + "os" + "path/filepath" "testing" "github.com/ShukeBta/MediaStationGo/internal/model" @@ -138,3 +140,55 @@ func TestSelectSiteSearchCandidatesWithUnknownTotalSkipsExistingEpisodes(t *test t.Fatalf("selected %#v, want only not-yet-local episode 3", got) } } + +func TestSubscriptionPendingDownloadAvailabilitySkipsUnorganizedEpisodes(t *testing.T) { + root := t.TempDir() + seasonDir := filepath.Join(root, "间谍过家家", "Season 01") + if err := os.MkdirAll(seasonDir, 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{ + "间谍过家家 - S01E01.mkv", + "间谍过家家 - S01E02.mkv.!qB", + } { + if err := os.WriteFile(filepath.Join(seasonDir, name), []byte("video"), 0o644); err != nil { + t.Fatal(err) + } + } + + sub := &model.Subscription{ + Name: "间谍过家家 自动订阅", + Filter: "间谍过家家", + MediaType: "tv", + SavePath: root, + TotalEpisodes: 3, + } + svc := NewSubscriptionService(nil, nil, nil, nil, nil, nil) + availability := svc.pendingDownloadAvailability(t.Context(), sub) + if availability.DownloadedEpisodes != 2 { + t.Fatalf("downloaded episodes = %d, want 2", availability.DownloadedEpisodes) + } + if _, ok := availability.ExistingEpisodeKeys[episodeKey(1, 1)]; !ok { + t.Fatalf("missing pending E01 key: %#v", availability.ExistingEpisodeKeys) + } + if _, ok := availability.ExistingEpisodeKeys[episodeKey(1, 2)]; !ok { + t.Fatalf("missing pending E02 key: %#v", availability.ExistingEpisodeKeys) + } + + results := []SearchResult{ + {Title: "间谍过家家 S01 Complete 1080p", DownloadURL: "https://pt/download/pack", Seeders: 100}, + {Title: "间谍过家家 S01E01 1080p", DownloadURL: "https://pt/download/1", Seeders: 90}, + {Title: "间谍过家家 S01E02 1080p", DownloadURL: "https://pt/download/2", Seeders: 80}, + {Title: "间谍过家家 S01E03 1080p", DownloadURL: "https://pt/download/3", Seeders: 70}, + } + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}, availability) + if len(got) != 1 || got[0].Episode != 3 { + t.Fatalf("selected %#v, want only not-yet-downloaded episode 3", got) + } + if !svc.downloadPathHasCandidate(t.Context(), sub, "间谍过家家 S01E02 1080p", root) { + t.Fatal("expected existing pending E02 file to be detected") + } + if svc.downloadPathHasCandidate(t.Context(), sub, "间谍过家家 S01E03 1080p", root) { + t.Fatal("did not expect missing E03 to be detected") + } +}