From 941bb273b93144f9f9007d3df55a2e8bec69963f Mon Sep 17 00:00:00 2001 From: ShukeBta <272197458+ShukeBta@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:18:02 +0800 Subject: [PATCH] fix media organize and subscription workflows --- internal/handler/discover_extra.go | 149 ++++++++- internal/handler/discover_extra_test.go | 93 ++++++ .../handler/routes_authenticated_features.go | 1 + internal/handler/strm.go | 56 +++- internal/handler/subscription_extra.go | 20 ++ internal/handler/subscriptions.go | 58 ++-- internal/model/download_subscription.go | 5 + .../repository/media_repository_upsert.go | 1 + internal/repository/repository_test.go | 73 +++++ internal/service/cloud_mount_display.go | 6 +- internal/service/cloud_mount_merge.go | 6 +- internal/service/discover.go | 16 +- internal/service/discover_section_cache.go | 104 ++++++ .../service/discover_section_cache_test.go | 44 +++ internal/service/emby_movie_items.go | 4 +- internal/service/local_availability.go | 74 +++-- internal/service/manual_scrape_helpers.go | 77 ++++- internal/service/manual_scrape_providers.go | 25 +- internal/service/manual_scrape_search.go | 9 +- internal/service/manual_scrape_test.go | 167 +++++++++- internal/service/media_classifier.go | 4 +- internal/service/media_paths.go | 2 +- internal/service/media_series_key.go | 37 ++- internal/service/media_series_test.go | 55 ++++ internal/service/metadata_match_trust.go | 30 -- internal/service/organize_pipeline.go | 26 +- internal/service/organize_pipeline_test.go | 43 +++ internal/service/organizer.go | 7 +- .../service/organizer_classification_test.go | 91 +++++- internal/service/organizer_directory.go | 12 +- .../service/organizer_directory_dedup_test.go | 67 ++++ .../service/organizer_directory_layout.go | 16 + .../service/organizer_directory_metadata.go | 52 +++ .../organizer_directory_metadata_persist.go | 76 +++++ .../service/organizer_directory_reclassify.go | 17 + .../organizer_directory_source_existing.go | 1 + .../organizer_library_classification.go | 12 +- internal/service/organizer_media.go | 10 +- internal/service/organizer_media_metadata.go | 117 +++++++ internal/service/organizer_paths.go | 8 +- .../service/organizer_reclassify_scanned.go | 50 ++- internal/service/organizer_reclassify_test.go | 2 +- internal/service/organizer_scan.go | 4 +- internal/service/organizer_scrape_test.go | 251 +++++++++++++++ internal/service/organizer_settings.go | 36 +++ internal/service/scanner.go | 5 + internal/service/scanner_strm.go | 9 + internal/service/scraper_library.go | 22 +- internal/service/scraper_lookup.go | 73 +++++ internal/service/scraper_lookup_test.go | 54 ++++ internal/service/scraper_options.go | 1 + internal/service/scraper_query_trust_test.go | 20 +- internal/service/site_adapter.go | 1 + .../service/site_adapter_nexusphp_list.go | 28 ++ internal/service/site_adapter_test.go | 25 ++ internal/service/site_search.go | 2 + internal/service/strm_generate.go | 8 + internal/service/strm_output_dir.go | 2 - internal/service/strm_svc_test.go | 84 +++++ internal/service/strm_tree_cleanup_test.go | 150 +++++++++ internal/service/strm_tree_import.go | 141 +++++++++ internal/service/strm_tree_import_test.go | 83 +++++ internal/service/strm_tree_normalize.go | 284 +++++++++++++++++ internal/service/strm_tree_parse.go | 165 ++++++++++ internal/service/strm_tree_parse_test.go | 234 ++++++++++++++ internal/service/strm_tree_paths.go | 76 +++++ internal/service/strm_tree_url_test.go | 251 +++++++++++++++ internal/service/strm_url.go | 46 +++ internal/service/subscription_archive.go | 4 +- internal/service/subscription_archive_test.go | 1 + .../service/subscription_candidate_collect.go | 94 +++++- .../subscription_candidate_defaults_test.go | 299 ++++++++++++++++++ .../service/subscription_candidate_select.go | 2 +- .../subscription_candidate_selection_test.go | 288 +---------------- .../service/subscription_downloader_test.go | 88 ++++++ .../service/subscription_exclude_rules.go | 4 +- .../service/subscription_rss_defaults_test.go | 35 ++ internal/service/subscription_rss_run.go | 4 +- internal/service/subscription_rules.go | 62 +++- internal/service/subscription_rules_test.go | 48 ++- .../service/subscription_run_wash_test.go | 147 +++++++-- internal/service/subscription_score.go | 42 ++- .../subscription_site_search_enqueue.go | 2 +- .../subscription_site_search_keywords_test.go | 110 +++++++ .../subscription_site_search_limit_test.go | 129 ++++++++ .../service/subscription_torrent_rules.go | 56 ++++ .../subscription_torrent_rules_test.go | 59 ++++ web/src/api/discover.ts | 5 + web/src/api/strm.ts | 17 + web/src/api/subscriptions.ts | 5 + web/src/pages/DiscoverPage.tsx | 56 +++- web/src/pages/DiscoverPageSections.tsx | 7 +- web/src/pages/StrmGenerateSection.tsx | 4 + web/src/pages/StrmGenerateSectionParts.tsx | 42 ++- web/src/pages/StrmPage.tsx | 7 + web/src/pages/StrmTreeGenerateSection.tsx | 182 +++++++++++ web/src/pages/SubscriptionForm.tsx | 36 +++ web/src/pages/SubscriptionsPage.tsx | 19 ++ web/src/pages/discoverPageModel.ts | 94 ++++++ web/src/pages/strmPageModel.ts | 1 - web/src/pages/subscriptionFormModel.ts | 12 +- web/src/pages/subscriptionPageModel.ts | 40 ++- web/src/pages/useStrmGenerateForm.ts | 10 +- web/src/pages/useStrmPage.ts | 3 + web/src/pages/useStrmTreeGenerateForm.ts | 115 +++++++ web/src/types/subscriptions.ts | 5 + web/src/utils/groupSeries.ts | 34 +- 107 files changed, 5420 insertions(+), 526 deletions(-) create mode 100644 internal/service/discover_section_cache.go create mode 100644 internal/service/discover_section_cache_test.go create mode 100644 internal/service/organizer_directory_metadata_persist.go create mode 100644 internal/service/organizer_media_metadata.go create mode 100644 internal/service/scraper_lookup_test.go create mode 100644 internal/service/strm_tree_cleanup_test.go create mode 100644 internal/service/strm_tree_import.go create mode 100644 internal/service/strm_tree_import_test.go create mode 100644 internal/service/strm_tree_normalize.go create mode 100644 internal/service/strm_tree_parse.go create mode 100644 internal/service/strm_tree_parse_test.go create mode 100644 internal/service/strm_tree_paths.go create mode 100644 internal/service/strm_tree_url_test.go create mode 100644 internal/service/subscription_candidate_defaults_test.go create mode 100644 internal/service/subscription_downloader_test.go create mode 100644 internal/service/subscription_rss_defaults_test.go create mode 100644 internal/service/subscription_site_search_keywords_test.go create mode 100644 internal/service/subscription_site_search_limit_test.go create mode 100644 internal/service/subscription_torrent_rules.go create mode 100644 internal/service/subscription_torrent_rules_test.go create mode 100644 web/src/pages/StrmTreeGenerateSection.tsx create mode 100644 web/src/pages/useStrmTreeGenerateForm.ts diff --git a/internal/handler/discover_extra.go b/internal/handler/discover_extra.go index 6c65a24..d1a4965 100644 --- a/internal/handler/discover_extra.go +++ b/internal/handler/discover_extra.go @@ -8,11 +8,14 @@ package handler import ( "context" + "errors" "net/http" "strconv" "strings" + "time" "github.com/gin-gonic/gin" + "go.uber.org/zap" "github.com/ShukeBta/MediaStationGo/internal/service" ) @@ -38,6 +41,9 @@ var discoverSectionCatalog = []discoverSectionDef{ {Key: "bangumi_calendar", Label: "Bangumi 每日放送", Provider: "bangumi"}, } +const discoverFeedSectionTimeout = 15 * time.Second +const discoverFeedSlowSectionThreshold = 2 * time.Second + // discoverSectionsHandler returns the catalog of sections the UI can // pick from. The names match the upstream Vue UI so existing settings // keep working. @@ -76,24 +82,155 @@ func discoverFeedHandler(svc *service.Container) gin.HandlerFunc { } if provider := discoverSectionProvider(k); provider != "" && !discoverProviderEnabled(c.Request.Context(), svc, provider) { out[k] = []service.ExternalMediaResult{} - meta[k] = gin.H{"page": page, "has_next": false} + meta[k] = gin.H{"page": page, "has_next": false, "disabled": true} continue } - items, err := discoverSectionItems(c.Request.Context(), svc, k, page) + sectionCtx, cancel := context.WithTimeout(c.Request.Context(), discoverFeedSectionTimeout) + started := time.Now() + items, err := discoverSectionItems(sectionCtx, svc, k, page) + elapsed := time.Since(started) + cancel() + metaEntry := gin.H{"page": page, "has_next": false, "duration_ms": elapsed.Milliseconds()} if err != nil { - svc.Log.Debug("discover fetch failed") - items = nil + logDiscoverFetchFailed(svc, k, page, elapsed, err) + if cached, ok := cachedDiscoverSection(svc, k, page); ok { + items = cached + metaEntry["stale"] = true + metaEntry["warning"] = discoverFeedStaleMessage(err) + } else if fallbackItems, fallbackKey, ok := fallbackDiscoverSectionItems(c.Request.Context(), svc, k, page); ok { + items = fallbackItems + metaEntry["fallback"] = fallbackKey + metaEntry["warning"] = discoverFeedFallbackMessage(fallbackKey, err) + rememberDiscoverSection(svc, k, page, items) + } else { + metaEntry["error"] = discoverFeedErrorMessage(err) + items = nil + } + } else { + logDiscoverFetchSlow(svc, k, page, elapsed, len(items)) + rememberDiscoverSection(svc, k, page, items) } artworkItems = append(artworkItems, items...) out[k] = items - meta[k] = gin.H{"page": page, "has_next": discoverSectionHasNext(k, len(items))} + metaEntry["has_next"] = discoverSectionHasNext(k, len(items)) + meta[k] = metaEntry } out["_meta"] = meta - svc.Discover.WarmExternalArtwork(artworkItems) + if svc != nil && svc.Discover != nil { + svc.Discover.WarmExternalArtwork(artworkItems) + } c.JSON(http.StatusOK, out) } } +func cachedDiscoverSection(svc *service.Container, key string, page int) ([]service.ExternalMediaResult, bool) { + if svc == nil || svc.Discover == nil { + return nil, false + } + return svc.Discover.CachedSection(key, page) +} + +func rememberDiscoverSection(svc *service.Container, key string, page int, items []service.ExternalMediaResult) { + if svc == nil || svc.Discover == nil { + return + } + svc.Discover.RememberSection(key, page, items) +} + +func fallbackDiscoverSectionItems(parent context.Context, svc *service.Container, key string, page int) ([]service.ExternalMediaResult, string, bool) { + fallbackKey := fallbackDiscoverSectionKey(key) + if fallbackKey == "" || svc == nil || svc.Discover == nil { + return nil, "", false + } + ctx, cancel := context.WithTimeout(parent, discoverFeedSectionTimeout) + defer cancel() + items, err := discoverSectionItems(ctx, svc, fallbackKey, page) + if err != nil || len(items) == 0 { + return nil, fallbackKey, false + } + if svc.Log != nil { + svc.Log.Info("discover section fallback used", + zap.String("section", key), + zap.String("fallback_section", fallbackKey), + zap.Int("page", page), + zap.Int("items", len(items))) + } + return items, fallbackKey, true +} + +func fallbackDiscoverSectionKey(key string) string { + switch key { + case "douban_hot_movie": + return "tmdb_popular_movie" + case "douban_hot_tv": + return "tmdb_popular_tv" + case "douban_top_movie": + return "tmdb_top_rated_movie" + default: + return "" + } +} + +func logDiscoverFetchFailed(svc *service.Container, key string, page int, elapsed time.Duration, err error) { + if svc == nil || svc.Log == nil || err == nil { + return + } + svc.Log.Warn("discover section fetch failed", + zap.String("section", key), + zap.String("provider", discoverSectionProvider(key)), + zap.Int("page", page), + zap.Duration("duration", elapsed), + zap.Int64("duration_ms", elapsed.Milliseconds()), + zap.Duration("timeout", discoverFeedSectionTimeout), + zap.Error(err)) +} + +func logDiscoverFetchSlow(svc *service.Container, key string, page int, elapsed time.Duration, itemCount int) { + if svc == nil || svc.Log == nil || elapsed < discoverFeedSlowSectionThreshold { + return + } + svc.Log.Info("discover section fetch slow", + zap.String("section", key), + zap.String("provider", discoverSectionProvider(key)), + zap.Int("page", page), + zap.Int("items", itemCount), + zap.Duration("duration", elapsed), + zap.Int64("duration_ms", elapsed.Milliseconds()), + zap.Duration("slow_threshold", discoverFeedSlowSectionThreshold)) +} + +func discoverFeedErrorMessage(err error) string { + if err == nil { + return "" + } + if errors.Is(err, context.DeadlineExceeded) { + return "推荐源响应超时,已跳过本次加载" + } + var timeout interface{ Timeout() bool } + if errors.As(err, &timeout) && timeout.Timeout() { + return "推荐源响应超时,已跳过本次加载" + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context deadline exceeded") { + return "推荐源响应超时,已跳过本次加载" + } + return "推荐源暂时不可用,已跳过本次加载" +} + +func discoverFeedStaleMessage(err error) string { + if discoverFeedErrorMessage(err) == "推荐源响应超时,已跳过本次加载" { + return "推荐源响应超时,已显示上次成功结果" + } + return "推荐源暂时不可用,已显示上次成功结果" +} + +func discoverFeedFallbackMessage(fallbackKey string, err error) string { + if strings.TrimSpace(fallbackKey) == "" { + return discoverFeedErrorMessage(err) + } + return "推荐源暂时不可用,已显示同类备用榜单" +} + func enabledDiscoverSections(ctx context.Context, svc *service.Container) []discoverSectionDef { sections := make([]discoverSectionDef, 0, len(discoverSectionCatalog)) for _, section := range discoverSectionCatalog { diff --git a/internal/handler/discover_extra_test.go b/internal/handler/discover_extra_test.go index c1b8745..fe92044 100644 --- a/internal/handler/discover_extra_test.go +++ b/internal/handler/discover_extra_test.go @@ -1,10 +1,14 @@ package handler import ( + "context" + "errors" "testing" + "time" "github.com/glebarez/sqlite" "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" "gorm.io/gorm" "github.com/ShukeBta/MediaStationGo/internal/model" @@ -36,6 +40,81 @@ func TestDiscoverProviderEnabledHonorsAPIConfigToggle(t *testing.T) { } } +func TestDiscoverFetchFailureLogIncludesDiagnostics(t *testing.T) { + core, observed := observer.New(zap.WarnLevel) + logger := zap.New(core) + + logDiscoverFetchFailed( + &service.Container{Log: logger}, + "tmdb_latest_movie", + 2, + 1500*time.Millisecond, + context.DeadlineExceeded, + ) + + entries := observed.FilterMessage("discover section fetch failed").All() + if len(entries) != 1 { + t.Fatalf("expected one failure log entry, got %d", len(entries)) + } + fields := entries[0].ContextMap() + if fields["section"] != "tmdb_latest_movie" || fields["provider"] != "tmdb" { + t.Fatalf("unexpected section/provider fields: %#v", fields) + } + if fields["page"] != int64(2) && fields["page"] != 2 { + t.Fatalf("page field missing or wrong: %#v", fields["page"]) + } + if fields["duration_ms"] != int64(1500) && fields["duration_ms"] != 1500 { + t.Fatalf("duration_ms field missing or wrong: %#v", fields["duration_ms"]) + } + if _, ok := fields["timeout"]; !ok { + t.Fatalf("timeout field missing: %#v", fields) + } +} + +func TestDiscoverSlowFetchLogIncludesSectionTiming(t *testing.T) { + core, observed := observer.New(zap.InfoLevel) + logger := zap.New(core) + + logDiscoverFetchSlow(&service.Container{Log: logger}, "douban_hot_movie", 1, discoverFeedSlowSectionThreshold-time.Millisecond, 24) + if got := observed.FilterMessage("discover section fetch slow").Len(); got != 0 { + t.Fatalf("fast section should not log, got %d entries", got) + } + + logDiscoverFetchSlow(&service.Container{Log: logger}, "douban_hot_movie", 1, discoverFeedSlowSectionThreshold, 24) + entries := observed.FilterMessage("discover section fetch slow").All() + if len(entries) != 1 { + t.Fatalf("expected one slow log entry, got %d", len(entries)) + } + fields := entries[0].ContextMap() + if fields["section"] != "douban_hot_movie" || fields["provider"] != "douban" { + t.Fatalf("unexpected section/provider fields: %#v", fields) + } + if fields["items"] != int64(24) && fields["items"] != 24 { + t.Fatalf("items field missing or wrong: %#v", fields["items"]) + } + if _, ok := fields["duration_ms"]; !ok { + t.Fatalf("duration_ms field missing: %#v", fields) + } + if _, ok := fields["slow_threshold"]; !ok { + t.Fatalf("slow_threshold field missing: %#v", fields) + } +} + +func TestDiscoverFeedErrorMessageHidesTechnicalTimeout(t *testing.T) { + for _, err := range []error{ + context.DeadlineExceeded, + errors.New("timeout of 30000ms exceeded"), + } { + got := discoverFeedErrorMessage(err) + if got != "推荐源响应超时,已跳过本次加载" { + t.Fatalf("message for %q = %q", err, got) + } + } + if got := discoverFeedErrorMessage(errors.New("upstream 503")); got != "推荐源暂时不可用,已跳过本次加载" { + t.Fatalf("generic message = %q", got) + } +} + func TestDefaultDiscoverSectionKeysSkipDisabledProviders(t *testing.T) { db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) if err != nil { @@ -78,3 +157,17 @@ func TestDefaultDiscoverSectionKeysIncludeLatestTMDbRails(t *testing.T) { } } } + +func TestFallbackDiscoverSectionKeyUsesTMDbForDoubanRails(t *testing.T) { + cases := map[string]string{ + "douban_hot_movie": "tmdb_popular_movie", + "douban_hot_tv": "tmdb_popular_tv", + "douban_top_movie": "tmdb_top_rated_movie", + "tmdb_latest_tv": "", + } + for key, want := range cases { + if got := fallbackDiscoverSectionKey(key); got != want { + t.Fatalf("fallbackDiscoverSectionKey(%q) = %q, want %q", key, got, want) + } + } +} diff --git a/internal/handler/routes_authenticated_features.go b/internal/handler/routes_authenticated_features.go index 72fe5c3..c5baf0c 100644 --- a/internal/handler/routes_authenticated_features.go +++ b/internal/handler/routes_authenticated_features.go @@ -54,6 +54,7 @@ func registerAuthedSTRMRoutes(authed *gin.RouterGroup, svc *service.Container) { authed.DELETE("/media/:id/strm", middleware.AdminRequired(), clearSTRMHandler(svc)) authed.POST("/strm/import", middleware.AdminRequired(), importSTRMHandler(svc)) authed.POST("/strm/generate", middleware.AdminRequired(), generateSTRMHandler(svc)) + authed.POST("/strm/generate-from-tree", middleware.AdminRequired(), generateSTRMFromTreeHandler(svc)) } func registerAuthedDuplicateRoutes(authed *gin.RouterGroup, svc *service.Container) { diff --git a/internal/handler/strm.go b/internal/handler/strm.go index 52dc22a..4bc9a00 100644 --- a/internal/handler/strm.go +++ b/internal/handler/strm.go @@ -104,7 +104,20 @@ type generateSTRMReq struct { BaseURL string `json:"base_url"` Enabled bool `json:"enabled"` Overwrite bool `json:"overwrite"` - IncludeLocal bool `json:"include_local"` + IncludeLocal *bool `json:"include_local"` + PreserveTree bool `json:"preserve_tree"` +} + +type generateSTRMTreeReq struct { + Provider string `json:"provider"` + TreeText string `json:"tree_text"` + Paths []string `json:"paths"` + SourceRoot string `json:"source_root"` + OutputPrefix string `json:"output_prefix"` + OutputDir string `json:"output_dir"` + BaseURL string `json:"base_url"` + Overwrite bool `json:"overwrite"` + Cleanup bool `json:"cleanup"` } func generateSTRMHandler(svc *service.Container) gin.HandlerFunc { @@ -122,13 +135,18 @@ func generateSTRMHandler(svc *service.Container) gin.HandlerFunc { if baseURL == "" { baseURL = strings.TrimRight(absoluteRequestURL(c, "/"), "/") } + includeLocal := true + if req.IncludeLocal != nil { + includeLocal = *req.IncludeLocal + } options := service.GenerateSTRMOptions{ LibraryID: req.LibraryID, OutputDir: req.OutputDir, BaseURL: baseURL, Enabled: req.Enabled, Overwrite: req.Overwrite, - IncludeLocal: true, + IncludeLocal: includeLocal, + PreserveTree: req.PreserveTree, PlaybackToken: strmPlaybackTokenForRequest(c, svc), } var res *service.GenerateSTRMResult @@ -146,6 +164,40 @@ func generateSTRMHandler(svc *service.Container) gin.HandlerFunc { } } +func generateSTRMFromTreeHandler(svc *service.Container) gin.HandlerFunc { + return func(c *gin.Context) { + var req generateSTRMTreeReq + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + strmSvc := svc.STRM + if strmSvc == nil { + strmSvc = service.NewSTRMService(svc.Log, svc.Repo, svc.Cfg) + } + baseURL := strings.TrimRight(strings.TrimSpace(req.BaseURL), "/") + if baseURL == "" { + baseURL = strings.TrimRight(absoluteRequestURL(c, "/"), "/") + } + res, err := strmSvc.GenerateFromTree(c.Request.Context(), service.GenerateSTRMTreeOptions{ + Provider: req.Provider, + TreeText: req.TreeText, + Paths: req.Paths, + SourceRoot: req.SourceRoot, + OutputPrefix: req.OutputPrefix, + OutputDir: req.OutputDir, + BaseURL: baseURL, + Overwrite: req.Overwrite, + Cleanup: req.Cleanup, + }) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, res) + } +} + func strmPlaybackTokenForRequest(c *gin.Context, svc *service.Container) string { if svc == nil || svc.Auth == nil || svc.Repo == nil || svc.Repo.User == nil { return "" diff --git a/internal/handler/subscription_extra.go b/internal/handler/subscription_extra.go index 8598bf2..e0bcb87 100644 --- a/internal/handler/subscription_extra.go +++ b/internal/handler/subscription_extra.go @@ -33,6 +33,11 @@ type subscriptionPatchReq struct { Effects *string `json:"effects"` ReleaseGroups *string `json:"release_groups"` ExcludeWords *string `json:"exclude_words"` + MinSeeders *int `json:"min_seeders"` + MaxSeeders *int `json:"max_seeders"` + MinSizeGB *float64 `json:"min_size_gb"` + MaxSizeGB *float64 `json:"max_size_gb"` + FreeOnly *bool `json:"free_only"` WashEnabled *bool `json:"wash_enabled"` WashPriority *string `json:"wash_priority"` TotalEpisodes *int `json:"total_episodes"` @@ -137,6 +142,21 @@ func subscriptionPatchUpdates(patch subscriptionPatchReq) map[string]any { if patch.ExcludeWords != nil { updates["exclude_words"] = *patch.ExcludeWords } + if patch.MinSeeders != nil { + updates["min_seeders"] = *patch.MinSeeders + } + if patch.MaxSeeders != nil { + updates["max_seeders"] = *patch.MaxSeeders + } + if patch.MinSizeGB != nil { + updates["min_size_gb"] = *patch.MinSizeGB + } + if patch.MaxSizeGB != nil { + updates["max_size_gb"] = *patch.MaxSizeGB + } + if patch.FreeOnly != nil { + updates["free_only"] = *patch.FreeOnly + } if patch.WashEnabled != nil { updates["wash_enabled"] = *patch.WashEnabled } diff --git a/internal/handler/subscriptions.go b/internal/handler/subscriptions.go index e1f5eaf..423dd1b 100644 --- a/internal/handler/subscriptions.go +++ b/internal/handler/subscriptions.go @@ -14,30 +14,35 @@ import ( ) type subscriptionReq struct { - Name string `json:"name" binding:"required"` - FeedURL string `json:"feed_url" binding:"required"` - Filter string `json:"filter"` - MediaType string `json:"media_type"` - MediaCategory string `json:"media_category"` - SavePath string `json:"save_path"` - SearchMode string `json:"search_mode"` - IMDBID string `json:"imdb_id"` - Source string `json:"source"` - PosterURL string `json:"poster_url"` - BackdropURL string `json:"backdrop_url"` - Overview string `json:"overview"` - OriginalName string `json:"original_name"` - Year int `json:"year"` - Resolution string `json:"resolution"` - Quality string `json:"quality"` - Effects string `json:"effects"` - ReleaseGroups string `json:"release_groups"` - ExcludeWords string `json:"exclude_words"` - WashEnabled bool `json:"wash_enabled"` - WashPriority string `json:"wash_priority"` - TotalEpisodes int `json:"total_episodes"` - Priority int `json:"priority"` - Enabled *bool `json:"enabled"` + Name string `json:"name" binding:"required"` + FeedURL string `json:"feed_url" binding:"required"` + Filter string `json:"filter"` + MediaType string `json:"media_type"` + MediaCategory string `json:"media_category"` + SavePath string `json:"save_path"` + SearchMode string `json:"search_mode"` + IMDBID string `json:"imdb_id"` + Source string `json:"source"` + PosterURL string `json:"poster_url"` + BackdropURL string `json:"backdrop_url"` + Overview string `json:"overview"` + OriginalName string `json:"original_name"` + Year int `json:"year"` + Resolution string `json:"resolution"` + Quality string `json:"quality"` + Effects string `json:"effects"` + ReleaseGroups string `json:"release_groups"` + ExcludeWords string `json:"exclude_words"` + MinSeeders int `json:"min_seeders"` + MaxSeeders int `json:"max_seeders"` + MinSizeGB float64 `json:"min_size_gb"` + MaxSizeGB float64 `json:"max_size_gb"` + FreeOnly bool `json:"free_only"` + WashEnabled bool `json:"wash_enabled"` + WashPriority string `json:"wash_priority"` + TotalEpisodes int `json:"total_episodes"` + Priority int `json:"priority"` + Enabled *bool `json:"enabled"` } func createSubscriptionHandler(svc *service.Container) gin.HandlerFunc { @@ -73,6 +78,11 @@ func createSubscriptionHandler(svc *service.Container) gin.HandlerFunc { Effects: req.Effects, ReleaseGroups: req.ReleaseGroups, ExcludeWords: req.ExcludeWords, + MinSeeders: req.MinSeeders, + MaxSeeders: req.MaxSeeders, + MinSizeGB: req.MinSizeGB, + MaxSizeGB: req.MaxSizeGB, + FreeOnly: req.FreeOnly, WashEnabled: req.WashEnabled, WashPriority: req.WashPriority, TotalEpisodes: req.TotalEpisodes, diff --git a/internal/model/download_subscription.go b/internal/model/download_subscription.go index 73f8ed1..f1d45f9 100644 --- a/internal/model/download_subscription.go +++ b/internal/model/download_subscription.go @@ -57,6 +57,11 @@ type Subscription struct { Effects string `gorm:"size:128" json:"effects,omitempty"` // hdr,dolby-vision,atmos ReleaseGroups string `gorm:"size:255" json:"release_groups,omitempty"` // comma separated ExcludeWords string `gorm:"size:255" json:"exclude_words,omitempty"` // comma separated + MinSeeders int `gorm:"default:0" json:"min_seeders,omitempty"` + MaxSeeders int `gorm:"default:0" json:"max_seeders,omitempty"` + MinSizeGB float64 `gorm:"default:0" json:"min_size_gb,omitempty"` + MaxSizeGB float64 `gorm:"default:0" json:"max_size_gb,omitempty"` + FreeOnly bool `gorm:"default:false" json:"free_only,omitempty"` WashEnabled bool `gorm:"default:false" json:"wash_enabled"` WashPriority string `gorm:"size:32" json:"wash_priority,omitempty"` // balanced / resolution / quality / effects / seeders TotalEpisodes int `gorm:"default:0" json:"total_episodes,omitempty"` diff --git a/internal/repository/media_repository_upsert.go b/internal/repository/media_repository_upsert.go index 2974716..f17ed08 100644 --- a/internal/repository/media_repository_upsert.go +++ b/internal/repository/media_repository_upsert.go @@ -240,6 +240,7 @@ func (r *MediaRepository) applyMediaUpsertUpdates(ctx context.Context, m *model. // 回写 ID / 不可变字段,让 caller 拿到完整的现有行。 *m = existing if fresh, err := r.FindByID(ctx, existing.ID); err == nil && fresh != nil { + *m = *fresh r.indexMediaBestEffort(ctx, *fresh) } return nil diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index b97b512..3693345 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -186,6 +186,79 @@ func TestMediaUpsertMatchedIncomingRefreshesScrapedMetadata(t *testing.T) { } } +func TestMediaUpsertScanDoesNotClearMatchedMetadata(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := database.AutoMigrate(db); err != nil { + t.Fatalf("migrate: %v", err) + } + repos := New(db) + lib := model.Library{Name: "剧集", Path: "/media/tv", Type: "tv", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + path := "/media/tv/间谍过家家/Season 01/间谍过家家 - S01E01.mkv" + existing := model.Media{ + LibraryID: lib.ID, + Title: "间谍过家家", + OriginalName: "SPY×FAMILY", + Path: path, + PosterURL: "/poster.jpg", + BackdropURL: "/backdrop.jpg", + Overview: "剧情简介", + Year: 2022, + SeasonNum: 1, + EpisodeNum: 1, + ScrapeStatus: "matched", + TMDbID: 12345, + BangumiID: 67890, + DoubanID: "db-spy", + TheTVDBID: "tvdb-spy", + } + if err := repos.Media.Upsert(t.Context(), &existing); err != nil { + t.Fatal(err) + } + scan := model.Media{ + LibraryID: lib.ID, + Title: "Spy.x.Family.S01E01.2022.1080p.WEB-DL", + Path: path, + SizeBytes: 2048, + DurationSec: 1500, + Width: 1920, + Height: 1080, + VideoCodec: "h264", + AudioCodec: "aac", + Container: "mkv", + SeasonNum: 1, + EpisodeNum: 1, + } + if err := repos.Media.Upsert(t.Context(), &scan); err != nil { + t.Fatal(err) + } + + var got model.Media + if err := repos.DB.Where("path = ?", path).First(&got).Error; err != nil { + t.Fatal(err) + } + if got.Title != "间谍过家家" || got.OriginalName != "SPY×FAMILY" || got.ScrapeStatus != "matched" { + t.Fatalf("matched names/status were overwritten by scan: %#v", got) + } + if got.TMDbID != 12345 || got.BangumiID != 67890 || got.DoubanID != "db-spy" || got.TheTVDBID != "tvdb-spy" { + t.Fatalf("matched provider ids were cleared by scan: %#v", got) + } + if got.PosterURL != "/poster.jpg" || got.BackdropURL != "/backdrop.jpg" || got.Overview != "剧情简介" { + t.Fatalf("matched artwork/overview were overwritten by scan: %#v", got) + } + if got.SizeBytes != 2048 || got.DurationSec != 1500 || got.Width != 1920 || got.Height != 1080 || got.Container != "mkv" { + t.Fatalf("file scan fields were not refreshed: %#v", got) + } + if scan.ID != got.ID || scan.Title != got.Title || scan.TMDbID != got.TMDbID || scan.ScrapeStatus != "matched" { + t.Fatalf("upsert caller did not receive fresh matched row: %#v want %#v", scan, got) + } +} + // TestMediaUpsertMigratesCloudLibraryIDOnRescan 复现"一键挂载子目录后媒体消失"的 // 回归:同一 cloud:// 文件先被父目录库扫描入库,之后用户按二级分类重新挂载到更 // 精确的分类库并扫描,library_id 必须迁移到新分类库,否则媒体被钉死在旧库、新库 diff --git a/internal/service/cloud_mount_display.go b/internal/service/cloud_mount_display.go index 3cde0d8..e7fbd6f 100644 --- a/internal/service/cloud_mount_display.go +++ b/internal/service/cloud_mount_display.go @@ -124,11 +124,11 @@ func canonicalLibraryCategoryName(libraryType, name string) string { } case "tvshows": switch name { - case "国剧", "大陆剧", "国产电视剧", "华语剧": + case "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "港剧", "台剧", "港台剧": return "国产剧" case "欧美电视剧", "美剧", "英剧", "未分类", "uncategorized": return "欧美剧" - case "日剧", "韩剧": + case "日韩电视剧", "日剧", "韩剧", "泰剧": return "日韩剧" case "真人秀": return "综艺" @@ -222,7 +222,7 @@ func canonicalLibraryCategoryDisplayType(name string) string { switch normalizeLibraryMergeName(name) { case "演唱会", "音乐会", "动画电影", "动漫电影", "华语电影", "国产电影", "大陆电影", "欧美电影", "外语电影", "外国电影", "日韩电影", "日本电影", "韩国电影": return "movie" - case "国产剧", "国剧", "大陆剧", "国产电视剧", "华语剧", "欧美剧", "欧美电视剧", "美剧", "英剧", "未分类", "uncategorized", "日韩剧", "日剧", "韩剧", "综艺", "真人秀", "儿童", "少儿": + case "国产剧", "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "港剧", "台剧", "港台剧", "欧美剧", "欧美电视剧", "美剧", "英剧", "未分类", "uncategorized", "日韩剧", "日韩电视剧", "日剧", "韩剧", "泰剧", "综艺", "真人秀", "儿童", "少儿": return "tv" case "国漫", "国产动漫", "国产动画", "日番", "日漫", "番剧", "日本动漫", "日本动画", "韩漫", "韩国动漫", "韩国动画", "美漫", "欧美动漫", "欧美动画", "西方动画", "其他", "其他动漫", "其它动漫", "other": return "anime" diff --git a/internal/service/cloud_mount_merge.go b/internal/service/cloud_mount_merge.go index dfc6419..f7e8b65 100644 --- a/internal/service/cloud_mount_merge.go +++ b/internal/service/cloud_mount_merge.go @@ -66,11 +66,11 @@ func cloudLibraryMergeNameKey(typeKey, name string) string { } case "tvshows": switch name { - case "国产剧", "大陆剧", "华语剧", "国剧": + case "国产剧", "大陆剧", "华语剧", "国剧", "国产电视剧", "大陆电视剧", "华语电视剧", "港剧", "台剧", "港台剧": return "国产剧" - case "欧美剧", "美剧", "英剧": + case "欧美剧", "欧美电视剧", "美剧", "英剧": return "欧美剧" - case "日韩剧", "日剧", "韩剧": + case "日韩剧", "日韩电视剧", "日剧", "韩剧", "泰剧": return "日韩剧" case "国漫", "国产动漫", "国产动画": return "国漫" diff --git a/internal/service/discover.go b/internal/service/discover.go index 957276d..1729478 100644 --- a/internal/service/discover.go +++ b/internal/service/discover.go @@ -21,18 +21,20 @@ import ( // DiscoverService talks to TMDb's /trending and /movie/popular endpoints. type DiscoverService struct { - log *zap.Logger - tmdb *TMDbProvider - client *http.Client - images *ImageProxy + log *zap.Logger + tmdb *TMDbProvider + client *http.Client + images *ImageProxy + sectionCache *DiscoverSectionCache } // NewDiscoverService is the constructor. func NewDiscoverService(log *zap.Logger, tmdb *TMDbProvider) *DiscoverService { return &DiscoverService{ - log: log, - tmdb: tmdb, - client: NewExternalHTTPClient(15 * time.Second), + log: log, + tmdb: tmdb, + client: NewExternalHTTPClient(15 * time.Second), + sectionCache: NewDiscoverSectionCache(6 * time.Hour), } } diff --git a/internal/service/discover_section_cache.go b/internal/service/discover_section_cache.go new file mode 100644 index 0000000..63a949c --- /dev/null +++ b/internal/service/discover_section_cache.go @@ -0,0 +1,104 @@ +package service + +import ( + "fmt" + "sync" + "time" +) + +// DiscoverSectionCache keeps the last good discover rail in memory so a slow +// upstream provider does not turn a populated page into empty rows. +type DiscoverSectionCache struct { + ttl time.Duration + mu sync.RWMutex + entries map[string]discoverSectionCacheEntry +} + +type discoverSectionCacheEntry struct { + items []ExternalMediaResult + storedAt time.Time +} + +func NewDiscoverSectionCache(ttl time.Duration) *DiscoverSectionCache { + if ttl <= 0 { + ttl = 6 * time.Hour + } + return &DiscoverSectionCache{ + ttl: ttl, + entries: map[string]discoverSectionCacheEntry{}, + } +} + +func (d *DiscoverService) RememberSection(key string, page int, items []ExternalMediaResult) { + if d == nil || d.sectionCache == nil || len(items) == 0 { + return + } + d.sectionCache.Set(key, page, items) +} + +func (d *DiscoverService) CachedSection(key string, page int) ([]ExternalMediaResult, bool) { + if d == nil || d.sectionCache == nil { + return nil, false + } + return d.sectionCache.Get(key, page) +} + +func (c *DiscoverSectionCache) Set(key string, page int, items []ExternalMediaResult) { + if c == nil || key == "" || page < 1 || len(items) == 0 { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.entries[discoverSectionCacheKey(key, page)] = discoverSectionCacheEntry{ + items: cloneExternalMediaResults(items), + storedAt: time.Now(), + } +} + +func (c *DiscoverSectionCache) Get(key string, page int) ([]ExternalMediaResult, bool) { + if c == nil || key == "" || page < 1 { + return nil, false + } + c.mu.RLock() + entry, ok := c.entries[discoverSectionCacheKey(key, page)] + c.mu.RUnlock() + if !ok || time.Since(entry.storedAt) > c.ttl || len(entry.items) == 0 { + return nil, false + } + return cloneExternalMediaResults(entry.items), true +} + +func discoverSectionCacheKey(key string, page int) string { + return fmt.Sprintf("%s:%d", key, page) +} + +func cloneExternalMediaResults(items []ExternalMediaResult) []ExternalMediaResult { + out := make([]ExternalMediaResult, len(items)) + for i, item := range items { + out[i] = item + out[i].SubscribeAliases = cloneStrings(item.SubscribeAliases) + out[i].MissingEpisodes = cloneInts(item.MissingEpisodes) + out[i].Languages = cloneStrings(item.Languages) + out[i].Countries = cloneStrings(item.Countries) + out[i].Genres = cloneStrings(item.Genres) + } + return out +} + +func cloneStrings(items []string) []string { + if len(items) == 0 { + return nil + } + out := make([]string, len(items)) + copy(out, items) + return out +} + +func cloneInts(items []int) []int { + if len(items) == 0 { + return nil + } + out := make([]int, len(items)) + copy(out, items) + return out +} diff --git a/internal/service/discover_section_cache_test.go b/internal/service/discover_section_cache_test.go new file mode 100644 index 0000000..97d511f --- /dev/null +++ b/internal/service/discover_section_cache_test.go @@ -0,0 +1,44 @@ +package service + +import ( + "testing" + "time" +) + +func TestDiscoverSectionCacheReturnsClone(t *testing.T) { + cache := NewDiscoverSectionCache(time.Hour) + cache.Set("douban_hot_movie", 1, []ExternalMediaResult{{ + Title: "第一部", + SubscribeAliases: []string{"别名"}, + MissingEpisodes: []int{1}, + Languages: []string{"zh"}, + }}) + + got, ok := cache.Get("douban_hot_movie", 1) + if !ok || len(got) != 1 || got[0].Title != "第一部" { + t.Fatalf("cached section = %#v, %v", got, ok) + } + + got[0].Title = "被修改" + got[0].SubscribeAliases[0] = "别名被改" + got[0].MissingEpisodes[0] = 9 + got[0].Languages[0] = "en" + again, ok := cache.Get("douban_hot_movie", 1) + if !ok || + again[0].Title != "第一部" || + again[0].SubscribeAliases[0] != "别名" || + again[0].MissingEpisodes[0] != 1 || + again[0].Languages[0] != "zh" { + t.Fatalf("cache should return a clone, got %#v", again) + } +} + +func TestDiscoverSectionCacheExpires(t *testing.T) { + cache := NewDiscoverSectionCache(time.Nanosecond) + cache.Set("tmdb_latest_movie", 1, []ExternalMediaResult{{Title: "旧数据"}}) + time.Sleep(time.Millisecond) + + if got, ok := cache.Get("tmdb_latest_movie", 1); ok || len(got) != 0 { + t.Fatalf("expired cache should miss, got %#v", got) + } +} diff --git a/internal/service/emby_movie_items.go b/internal/service/emby_movie_items.go index 31cc461..1021cc3 100644 --- a/internal/service/emby_movie_items.go +++ b/internal/service/emby_movie_items.go @@ -207,7 +207,7 @@ func embyLikelyEpisodicPathSQL() (string, []any) { "%/season %/%", "%/season.%/%", "%/season-%/%", "%/season_%/%", "%/s0%/%", "%/s1%/%", "%/s2%/%", "%/s3%/%", "%/s4%/%", "%/s5%/%", "%/s6%/%", "%/s7%/%", "%/s8%/%", "%/s9%/%", "%/special/%", "%/specials/%", "%/sp/%", "%/ova/%", "%/oad/%", "%/extra/%", "%/extras/%", - "%/电视剧/%", "%/剧集/%", "%/国产剧/%", "%/欧美剧/%", "%/日韩剧/%", "%/日剧/%", "%/韩剧/%", + "%/电视剧/%", "%/剧集/%", "%/连续剧/%", "%/短剧/%", "%/国产剧/%", "%/国剧/%", "%/大陆剧/%", "%/华语剧/%", "%/国产电视剧/%", "%/大陆电视剧/%", "%/华语电视剧/%", "%/欧美剧/%", "%/欧美电视剧/%", "%/美剧/%", "%/英剧/%", "%/日韩剧/%", "%/日韩电视剧/%", "%/日剧/%", "%/韩剧/%", "%/港剧/%", "%/台剧/%", "%/港台剧/%", "%/泰剧/%", "%/日番/%", "%/国漫/%", "%/番剧/%", "%/动漫/%", "%/特别篇/%", "%/特別篇/%", "%/番外/%", "%/特典/%", } clauses := make([]string, 0, len(patterns)*2) @@ -230,7 +230,7 @@ func embyMediaPathLooksEpisodic(path string) bool { } for _, marker := range []string{ "/season ", "/season.", "/season-", "/season_", "/special/", "/specials/", "/sp/", "/ova/", "/oad/", "/extra/", "/extras/", - "/电视剧/", "/剧集/", "/国产剧/", "/欧美剧/", "/日韩剧/", "/日剧/", "/韩剧/", + "/电视剧/", "/剧集/", "/连续剧/", "/短剧/", "/国产剧/", "/国剧/", "/大陆剧/", "/华语剧/", "/国产电视剧/", "/大陆电视剧/", "/华语电视剧/", "/欧美剧/", "/欧美电视剧/", "/美剧/", "/英剧/", "/日韩剧/", "/日韩电视剧/", "/日剧/", "/韩剧/", "/港剧/", "/台剧/", "/港台剧/", "/泰剧/", "/日番/", "/国漫/", "/番剧/", "/动漫/", "/特别篇/", "/特別篇/", "/番外/", "/特典/", } { if strings.Contains(normalized, marker) { diff --git a/internal/service/local_availability.go b/internal/service/local_availability.go index 87ef52b..edb59d3 100644 --- a/internal/service/local_availability.go +++ b/internal/service/local_availability.go @@ -164,30 +164,66 @@ func missingEpisodeSet(availability LocalAvailability) map[int]struct{} { } func sortedEpisodeCandidates(candidates []siteSearchCandidate) []siteSearchCandidate { - byEpisode := make(map[string]siteSearchCandidate) - order := make([]string, 0, len(candidates)) + selected := make([]siteSearchCandidate, 0, len(candidates)) + covered := make(map[string]struct{}, len(candidates)) for _, candidate := range candidates { - if candidate.Episode <= 0 { + keys := candidateEpisodeKeys(candidate) + if len(keys) == 0 { continue } - season := candidate.Season - if season <= 0 { - season = 1 - } - key := episodeKey(season, candidate.Episode) - if current, ok := byEpisode[key]; ok { - if current.Score < candidate.Score { - byEpisode[key] = candidate - } + if episodeKeysOverlap(covered, keys) { continue } - byEpisode[key] = candidate - order = append(order, key) - } - sort.Strings(order) - selected := make([]siteSearchCandidate, 0, len(order)) - for _, key := range order { - selected = append(selected, byEpisode[key]) + selected = append(selected, candidate) + for _, key := range keys { + covered[key] = struct{}{} + } } + sort.SliceStable(selected, func(i, j int) bool { + return candidateFirstEpisodeKey(selected[i]) < candidateFirstEpisodeKey(selected[j]) + }) return selected } + +func candidateEpisodeKeys(candidate siteSearchCandidate) []string { + episodes := candidateEpisodeNumbers(candidate) + if len(episodes) == 0 { + return nil + } + season := candidate.Season + if season <= 0 { + season = 1 + } + keys := make([]string, 0, len(episodes)) + seen := make(map[string]struct{}, len(episodes)) + for _, episode := range episodes { + if episode <= 0 { + continue + } + key := episodeKey(season, episode) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func candidateFirstEpisodeKey(candidate siteSearchCandidate) string { + keys := candidateEpisodeKeys(candidate) + if len(keys) == 0 { + return "" + } + return keys[0] +} + +func episodeKeysOverlap(covered map[string]struct{}, keys []string) bool { + for _, key := range keys { + if _, ok := covered[key]; ok { + return true + } + } + return false +} diff --git a/internal/service/manual_scrape_helpers.go b/internal/service/manual_scrape_helpers.go index 91b6762..c6149bb 100644 --- a/internal/service/manual_scrape_helpers.go +++ b/internal/service/manual_scrape_helpers.go @@ -66,17 +66,84 @@ func isTVLikeTMDbMatch(match *Match, mediaType string) bool { } func parsePositiveInt(value string) (int, bool) { - value = strings.TrimSpace(value) - if strings.Contains(value, ":") { - value = value[strings.LastIndex(value, ":")+1:] + idString, ok := parsePositiveIDString(value) + if !ok { + return 0, false } - id, err := strconv.Atoi(strings.TrimSpace(value)) + id, err := strconv.Atoi(idString) return id, err == nil && id > 0 } func parsePositiveIDString(value string) (string, bool) { - id, ok := parsePositiveInt(value) + for _, provider := range []string{"tmdb", "bangumi", "douban", "thetvdb"} { + if id, ok := parseProviderIDString(value, provider); ok { + return id, true + } + } + return parseBarePositiveIDString(value) +} + +func parseProviderIDInt(value, provider string) (int, bool) { + idString, ok := parseProviderIDString(value, provider) if !ok { + return 0, false + } + id, err := strconv.Atoi(idString) + return id, err == nil && id > 0 +} + +func parseProviderIDString(value, provider string) (string, bool) { + hints := externalIDHintsFromText(value) + switch strings.ToLower(strings.TrimSpace(provider)) { + case "tmdb": + if hints.TMDbID > 0 { + return strconv.Itoa(hints.TMDbID), true + } + case "bangumi", "bgm": + if hints.BangumiID > 0 { + return strconv.Itoa(hints.BangumiID), true + } + case "douban", "db": + if strings.TrimSpace(hints.DoubanID) != "" { + return strings.TrimSpace(hints.DoubanID), true + } + case "thetvdb", "tvdb": + if strings.TrimSpace(hints.TheTVDBID) != "" { + return strings.TrimSpace(hints.TheTVDBID), true + } + } + if hints.useful() { + return "", false + } + return parseBarePositiveIDString(value) +} + +func providerIDHintMismatched(value, provider string) bool { + hints := externalIDHintsFromText(value) + if !hints.useful() { + return false + } + switch strings.ToLower(strings.TrimSpace(provider)) { + case "tmdb": + return hints.TMDbID <= 0 + case "bangumi", "bgm": + return hints.BangumiID <= 0 + case "douban", "db": + return strings.TrimSpace(hints.DoubanID) == "" + case "thetvdb", "tvdb": + return strings.TrimSpace(hints.TheTVDBID) == "" + default: + return false + } +} + +func parseBarePositiveIDString(value string) (string, bool) { + value = strings.Trim(strings.TrimSpace(value), "[]{}()") + if strings.Contains(value, ":") { + value = value[strings.LastIndex(value, ":")+1:] + } + id, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil || id <= 0 { return "", false } return strconv.Itoa(id), true diff --git a/internal/service/manual_scrape_providers.go b/internal/service/manual_scrape_providers.go index 51f3d7c..4e9a406 100644 --- a/internal/service/manual_scrape_providers.go +++ b/internal/service/manual_scrape_providers.go @@ -16,7 +16,7 @@ func (s *ScraperService) manualTMDbCandidates(ctx context.Context, query string, if s.tmdb == nil || !s.tmdb.Enabled() { return nil } - if id, ok := parsePositiveInt(query); ok { + if id, ok := parseProviderIDInt(query, "tmdb"); ok { out := make([]manualTMDbCandidate, 0, 2) for _, typ := range manualTMDbIDSearchTypes(mediaType) { if match := s.manualTMDbMatchByIDForType(ctx, id, typ); match != nil { @@ -25,6 +25,9 @@ func (s *ScraperService) manualTMDbCandidates(ctx context.Context, query string, } return out } + if providerIDHintMismatched(query, "tmdb") { + return nil + } out := make([]manualTMDbCandidate, 0, 4) for _, typ := range manualTMDbSearchTypes(mediaType) { switch typ { @@ -152,11 +155,14 @@ func (s *ScraperService) manualDoubanMatch(ctx context.Context, query string) *M if s.douban == nil || !s.douban.Enabled() { return nil } - if id, ok := parsePositiveIDString(query); ok { + if id, ok := parseProviderIDString(query, "douban"); ok { if match, err := s.douban.GetMatchByID(ctx, id); err == nil && match != nil { return match } } + if providerIDHintMismatched(query, "douban") { + return nil + } match, err := s.douban.SearchMatch(ctx, query) if err != nil { return nil @@ -168,11 +174,14 @@ func (s *ScraperService) manualBangumiMatch(ctx context.Context, query string) * if s.bangumi == nil || !s.bangumi.Enabled() { return nil } - if id, ok := parsePositiveInt(query); ok { + if id, ok := parseProviderIDInt(query, "bangumi"); ok { if match, err := s.bangumi.GetSubject(ctx, id); err == nil && match != nil { return match } } + if providerIDHintMismatched(query, "bangumi") { + return nil + } match, err := s.bangumi.Search(ctx, query) if err != nil { return nil @@ -184,7 +193,15 @@ func (s *ScraperService) manualTheTVDBMatch(ctx context.Context, query string) * if s.thetvdb == nil || !s.thetvdb.Enabled() { return nil } - if id, ok := parsePositiveIDString(normalizeTheTVDBSeriesID(query)); ok { + if id, ok := parseProviderIDString(query, "thetvdb"); ok { + if match, err := s.thetvdb.GetSeriesMatchByID(ctx, id); err == nil && match != nil { + return match + } + } + if providerIDHintMismatched(query, "thetvdb") { + return nil + } + if id, ok := parseBarePositiveIDString(normalizeTheTVDBSeriesID(query)); ok { if match, err := s.thetvdb.GetSeriesMatchByID(ctx, id); err == nil && match != nil { return match } diff --git a/internal/service/manual_scrape_search.go b/internal/service/manual_scrape_search.go index 8a06fde..3e6fc99 100644 --- a/internal/service/manual_scrape_search.go +++ b/internal/service/manual_scrape_search.go @@ -61,6 +61,9 @@ func (s *ScraperService) ManualSearch(ctx context.Context, media *model.Media, q if providers.want("adult") { for _, candidateQuery := range queries { + if externalIDHintsFromText(candidateQuery).useful() { + continue + } for _, match := range s.manualAdultMatches(ctx, media, candidateQuery) { add("adult", "adult", match) } @@ -140,7 +143,11 @@ func (s *ScraperService) manualSearchQueries(ctx context.Context, media *model.M out = append(out, value) } - add(ApplyRecognitionWords(ctx, s.repo, query)) + recognizedQuery := ApplyRecognitionWords(ctx, s.repo, query) + add(recognizedQuery) + if externalIDHintsFromText(recognizedQuery).useful() { + return out + } if strings.TrimSpace(query) == "" && media != nil { add(firstText(media.Title, media.OriginalName)) } diff --git a/internal/service/manual_scrape_test.go b/internal/service/manual_scrape_test.go index fe83689..2497123 100644 --- a/internal/service/manual_scrape_test.go +++ b/internal/service/manual_scrape_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" + "sync/atomic" "testing" "time" @@ -34,10 +36,12 @@ func TestManualRequestMatchFallsBackToCandidatePayload(t *testing.T) { func TestParsePositiveIDStringAcceptsProviderPrefixes(t *testing.T) { cases := map[string]string{ - "12345": "12345", - "tmdb:12345": "12345", - "douban:67890": "67890", - "thetvdb:24680": "24680", + "12345": "12345", + "tmdb:12345": "12345", + "[tmdbid-12345]": "12345", + "douban:67890": "67890", + "{douban=67890}": "67890", + "thetvdb:24680": "24680", } for input, want := range cases { got, ok := parsePositiveIDString(input) @@ -48,6 +52,9 @@ func TestParsePositiveIDStringAcceptsProviderPrefixes(t *testing.T) { if got, ok := parsePositiveIDString("tmdb:not-a-number"); ok || got != "" { t.Fatalf("parsePositiveIDString invalid = %q,%v; want empty,false", got, ok) } + if got, ok := parseProviderIDString("[tmdbid-12345]", "douban"); ok || got != "" { + t.Fatalf("parseProviderIDString provider mismatch = %q,%v; want empty,false", got, ok) + } } func TestManualSearchReturnsTMDbCandidatePage(t *testing.T) { @@ -279,7 +286,7 @@ func TestManualSearchAllProvidersTMDbNumericIDTriesMovieAndTVNamespaces(t *testi if err := repos.DB.Create(&lib).Error; err != nil { t.Fatal(err) } - media := model.Media{LibraryID: lib.ID, Title: "待匹配", Path: `/media/movie/raw.mkv`} + media := model.Media{LibraryID: lib.ID, Title: "待匹配", Path: `/media/movie/raw-provider-id.mkv`} if err := repos.DB.Create(&media).Error; err != nil { t.Fatal(err) } @@ -296,6 +303,156 @@ func TestManualSearchAllProvidersTMDbNumericIDTriesMovieAndTVNamespaces(t *testi } } +func TestManualSearchTMDbProviderIDUsesIDLookupOnly(t *testing.T) { + var paths []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/movie/1208850": + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": 1208850, + "title": "多拉特行动", + "original_title": "Malbatt: Misi Bakara", + "overview": "ID matched movie.", + "poster_path": "/malbatt.jpg", + "release_date": "2024-01-11", + "vote_average": 6.9, + }) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Library{}, &model.Series{}, &model.Media{}); err != nil { + t.Fatal(err) + } + repos := repository.New(db) + cfg := &config.Config{} + cfg.Secrets.TMDbAPIKey = "test-key" + cfg.Secrets.TMDbAPIProxy = upstream.URL + log := zap.NewNop() + scraper := NewScraperService(cfg, log, repos, NewTMDbProvider(cfg, log, nil), nil, nil, nil, NewHub(log)) + + lib := model.Library{Name: "电影", Path: `/media/movie`, Type: "movie", Enabled: true} + if err := repos.DB.Create(&lib).Error; err != nil { + t.Fatal(err) + } + media := model.Media{LibraryID: lib.ID, Title: "待匹配", Path: `/media/movie/raw-provider-id-skip-adult.mkv`} + if err := repos.DB.Create(&media).Error; err != nil { + t.Fatal(err) + } + + results, err := scraper.ManualSearch(t.Context(), &media, "[tmdbid-1208850]", "all", "movie") + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].TMDbID != 1208850 || results[0].Title != "多拉特行动" { + t.Fatalf("manual TMDb provider-id results=%#v, paths=%v", results, paths) + } + for _, path := range paths { + if strings.HasPrefix(path, "/search/") { + t.Fatalf("provider-id search should not use fuzzy search; paths=%v", paths) + } + } + if len(paths) == 0 || paths[0] != "/movie/1208850" { + t.Fatalf("tmdb provider-id paths=%v, want /movie/1208850 first", paths) + } +} + +func TestManualSearchAllProvidersProviderIDSkipsAdultSource(t *testing.T) { + var tmdbPaths []string + tmdbUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tmdbPaths = append(tmdbPaths, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/movie/1208850": + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": 1208850, + "title": "多拉特行动", + "original_title": "Malbatt: Misi Bakara", + "release_date": "2024-01-11", + }) + default: + http.NotFound(w, r) + } + })) + defer tmdbUpstream.Close() + + var adultCalls atomic.Int32 + adultUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + adultCalls.Add(1) + http.NotFound(w, r) + })) + defer adultUpstream.Close() + + db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Library{}, &model.Series{}, &model.Media{}, &model.APIConfig{}); err != nil { + t.Fatal(err) + } + repos := repository.New(db) + cfg := &config.Config{} + cfg.Secrets.TMDbAPIKey = "test-key" + cfg.Secrets.TMDbAPIProxy = tmdbUpstream.URL + log := zap.NewNop() + apiConfig := NewAPIConfigService(log, repos, NewCryptoService("", log)) + adultBaseURL := adultUpstream.URL + if _, err := apiConfig.Update(t.Context(), "adult", APIConfigPatch{BaseURL: &adultBaseURL}); err != nil { + t.Fatal(err) + } + scraper := NewScraperService(cfg, log, repos, NewTMDbProvider(cfg, log, nil), nil, nil, nil, NewHub(log), NewAdultProvider(log, apiConfig)) + + lib := model.Library{Name: "电影", Path: `/media/movie`, Type: "movie", Enabled: true} + if err := repos.DB.Create(&lib).Error; err != nil { + t.Fatal(err) + } + media := model.Media{LibraryID: lib.ID, Title: "待匹配", Path: `/media/movie/raw.mkv`} + if err := repos.DB.Create(&media).Error; err != nil { + t.Fatal(err) + } + + results, err := scraper.ManualSearch(t.Context(), &media, "[tmdbid-1208850]", "all", "movie") + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].TMDbID != 1208850 || results[0].Source != "tmdb" { + t.Fatalf("manual provider-id results=%#v, tmdb paths=%v", results, tmdbPaths) + } + if calls := adultCalls.Load(); calls != 0 { + t.Fatalf("adult provider was called %d times for explicit tmdb id", calls) + } +} + +func TestManualTMDbCandidatesSkipOtherProviderIDs(t *testing.T) { + var paths []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + http.NotFound(w, r) + })) + defer upstream.Close() + + cfg := &config.Config{} + cfg.Secrets.TMDbAPIKey = "test-key" + cfg.Secrets.TMDbAPIProxy = upstream.URL + log := zap.NewNop() + scraper := NewScraperService(cfg, log, nil, NewTMDbProvider(cfg, log, nil), nil, nil, nil, NewHub(log)) + + if got := scraper.manualTMDbCandidates(t.Context(), "{douban=36941123}", 0, "movie"); len(got) != 0 { + t.Fatalf("tmdb candidates for douban id = %#v, want none", got) + } + if len(paths) != 0 { + t.Fatalf("tmdb should not be called for douban id hint, paths=%v", paths) + } +} + func TestManualSearchIncludesAdultProvider(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { diff --git a/internal/service/media_classifier.go b/internal/service/media_classifier.go index 229c9b1..6f9e6f1 100644 --- a/internal/service/media_classifier.go +++ b/internal/service/media_classifier.go @@ -182,7 +182,7 @@ func normalizeMediaType(mediaType, title, category string) string { return "variety" case (containsAnyText(raw, "国漫", "日漫", "日番", "韩漫", "美漫", "欧美动漫", "其他动漫", "动漫", "动画") || classifierAnimeRE.MatchString(raw)) && !containsAnyText(raw, "动画电影"): return "anime" - case containsAnyText(raw, "电视剧", "国产剧", "欧美剧", "日韩剧", "日剧", "韩剧", "剧集") || classifierTVRE.MatchString(raw): + case containsAnyText(raw, "电视剧", "剧集", "连续剧", "短剧", "国产剧", "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "欧美剧", "欧美电视剧", "美剧", "英剧", "日韩剧", "日韩电视剧", "日剧", "韩剧", "港剧", "台剧", "港台剧", "泰剧") || classifierTVRE.MatchString(raw): return "tv" case containsAnyText(raw, "电影", "演唱会") || classifierMovieRE.MatchString(raw): return "movie" @@ -199,7 +199,7 @@ func normalizeMediaType(mediaType, title, category string) string { return "anime" case strings.Contains(text, "variety") || strings.Contains(text, "综艺") || strings.Contains(text, "真人秀"): return "variety" - case classifierEpisodeRE.MatchString(text) || classifierSeasonRE.MatchString(text) || classifierTVRE.MatchString(text) || strings.Contains(text, "剧集") || strings.Contains(text, "电视剧"): + case classifierEpisodeRE.MatchString(text) || classifierSeasonRE.MatchString(text) || classifierTVRE.MatchString(text) || containsAnyText(text, "剧集", "电视剧", "连续剧", "短剧", "国产电视剧", "大陆电视剧", "华语电视剧", "欧美电视剧", "日韩电视剧", "美剧", "英剧", "港剧", "台剧", "港台剧", "泰剧"): return "tv" default: return "movie" diff --git a/internal/service/media_paths.go b/internal/service/media_paths.go index 950b943..5ab1dd1 100644 --- a/internal/service/media_paths.go +++ b/internal/service/media_paths.go @@ -18,7 +18,7 @@ func inferLibraryKind(name, path, requested string) string { return "variety" case containsAnyText(text, "国漫", "日漫", "日番", "动漫", "动画", "anime", "bangumi") && !containsAnyText(text, "动画电影"): return "anime" - case containsAnyText(text, "电视剧", "国产剧", "欧美剧", "日韩剧", "日剧", "韩剧", "剧集", "tv", "series"): + case containsAnyText(text, "电视剧", "剧集", "连续剧", "短剧", "国产剧", "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "欧美剧", "欧美电视剧", "美剧", "英剧", "日韩剧", "日韩电视剧", "日剧", "韩剧", "港剧", "台剧", "港台剧", "泰剧", "tv", "series"): return "tv" case containsAnyText(text, "电影", "movie", "film"): return "movie" diff --git a/internal/service/media_series_key.go b/internal/service/media_series_key.go index 136483c..6822fbb 100644 --- a/internal/service/media_series_key.go +++ b/internal/service/media_series_key.go @@ -9,7 +9,7 @@ import ( "github.com/ShukeBta/MediaStationGo/internal/model" ) -var episodicPathRE = regexp.MustCompile(`(?i)[\\/](?:电视剧|剧集|国产剧|欧美剧|日韩剧|日剧|韩剧|综艺|纪录片|儿童|动漫|番剧|国漫|日番|韩漫|美漫|欧美动漫|欧美动画|其他动漫|tv|series|shows?|season[\s._-]*\d|s\d{1,2}(?:[\s._-]|[\\/])|special[\s._-]*episodes?|specials?|sp|ovas?|oads?|extras?|bonus(?:es)?|omake|特别篇|特別篇|番外篇?|特典|外传|外傳|总集篇|總集篇)[\\/]`) +var episodicPathRE = regexp.MustCompile(`(?i)[\\/](?:电视剧|剧集|连续剧|短剧|国产剧|国剧|大陆剧|华语剧|国产电视剧|大陆电视剧|华语电视剧|欧美剧|欧美电视剧|美剧|英剧|日韩剧|日韩电视剧|日剧|韩剧|港剧|台剧|港台剧|泰剧|综艺|纪录片|儿童|动漫|番剧|国漫|日番|韩漫|美漫|欧美动漫|欧美动画|其他动漫|tv|series|shows?|season[\s._-]*\d|s\d{1,2}(?:[\s._-]|[\\/])|special[\s._-]*episodes?|specials?|sp|ovas?|oads?|extras?|bonus(?:es)?|omake|特别篇|特別篇|番外篇?|特典|外传|外傳|总集篇|總集篇)[\\/]`) func mediaSeriesKey(media model.Media) string { return compactSeriesKey(mediaSeriesRawKey(media)) @@ -21,6 +21,9 @@ func mediaSeriesRawKey(media model.Media) string { if fromPath != "" { return seriesFingerprint("library-path", mediaTargetLibraryID(media), fromPath) } + if idKey := seriesExternalIDKeyFromPath(media.Path); idKey != "" { + return seriesFingerprint("library-path-id", mediaTargetLibraryID(media), idKey) + } if media.TMDbID > 0 { return fmt.Sprintf("tmdb:%d", media.TMDbID) } @@ -122,6 +125,9 @@ func seriesTitleFromMediaPath(path string) string { return "" } dirIndex := len(parts) - 2 + if last := parts[len(parts)-1]; !seriesPathPartLooksLikeFile(last) && !seriesSeasonDirRE.MatchString(filepath.Base(last)) { + dirIndex = len(parts) - 1 + } for dirIndex >= 0 && seriesSeasonDirRE.MatchString(filepath.Base(parts[dirIndex])) { dirIndex-- } @@ -135,6 +141,19 @@ func seriesTitleFromMediaPath(path string) string { return title } +func seriesPathPartLooksLikeFile(part string) bool { + ext := strings.ToLower(filepath.Ext(part)) + if _, ok := videoExtensions[ext]; ok { + return true + } + switch ext { + case ".iso", ".nfo", ".srt", ".ass", ".ssa", ".vtt", ".sub", ".idx", ".jpg", ".jpeg", ".png", ".webp": + return true + default: + return false + } +} + func seriesDisplayTitle(media model.Media) string { if fromPath := seriesTitleFromMediaPath(media.Path); fromPath != "" { return fromPath @@ -148,6 +167,22 @@ func seriesDisplayTitle(media model.Media) string { return "未命名节目" } +func seriesExternalIDKeyFromPath(path string) string { + _, hints := pathHintMetadata(path, true) + switch { + case hints.TMDbID > 0: + return fmt.Sprintf("tmdb:%d", hints.TMDbID) + case hints.BangumiID > 0: + return fmt.Sprintf("bgm:%d", hints.BangumiID) + case strings.TrimSpace(hints.DoubanID) != "": + return "douban:" + strings.TrimSpace(hints.DoubanID) + case strings.TrimSpace(hints.TheTVDBID) != "": + return "thetvdb:" + strings.TrimSpace(hints.TheTVDBID) + default: + return "" + } +} + func mediaTargetLibraryID(media model.Media) string { if strings.TrimSpace(media.DisplayLibraryID) != "" { return media.DisplayLibraryID diff --git a/internal/service/media_series_test.go b/internal/service/media_series_test.go index 308f4f2..2f7b0ba 100644 --- a/internal/service/media_series_test.go +++ b/internal/service/media_series_test.go @@ -204,3 +204,58 @@ func TestMediaSeriesKeyCleansReleaseNoiseFolders(t *testing.T) { t.Fatalf("release-noise folder key=%q, want clean key=%q", got, want) } } + +func TestMediaSeriesKeyTreatsDomesticTelevisionFolderAsSeries(t *testing.T) { + main := model.Media{ + LibraryID: "lib-domestic-tv", + Path: `/media/国产电视剧/人世间 (2022) [TMDBID-156568]/人世间.S01E01.mkv`, + SeasonNum: 1, + EpisodeNum: 1, + TMDbID: 156568, + } + weakEpisode := model.Media{ + LibraryID: "lib-domestic-tv", + Path: `/media/国产电视剧/人世间 (2022) [TMDBID-156568]/人世间.S01E02.mkv`, + // Some local/cloud scans may miss S/E at first while local NFO or + // scraper metadata already carries an episode-level TMDb id. + TMDbID: 4375419, + } + folderRecord := model.Media{ + LibraryID: "lib-domestic-tv", + Path: `/media/国产电视剧/人世间 (2022) [TMDBID-156568]`, + Title: "人世间", + TMDbID: 156568, + } + + if got, want := mediaSeriesKey(weakEpisode), mediaSeriesKey(main); got != want { + t.Fatalf("domestic television folder key=%q, want main key=%q", got, want) + } + if got, want := mediaSeriesKey(folderRecord), mediaSeriesKey(main); got != want { + t.Fatalf("domestic television folder record key=%q, want main key=%q", got, want) + } + + cards := groupMediaSeriesCards([]model.Media{main, weakEpisode, folderRecord}) + if len(cards) != 1 || cards[0].Count != 3 { + t.Fatalf("cards=%#v, want one merged series card with three items", cards) + } +} + +func TestMediaSeriesKeyUsesSeriesDirectoryExternalID(t *testing.T) { + episodeIDOnly := model.Media{ + LibraryID: "lib-domestic-tv", + Path: `/media/电视剧/国产剧/人世间 (2022)/Season 01/人世间.S01E03.{tmdb-7129826}.mkv`, + SeasonNum: 1, + EpisodeNum: 3, + TMDbID: 7129826, + } + cleanFolder := model.Media{ + LibraryID: "lib-domestic-tv", + Path: `/media/电视剧/国产剧/人世间 (2022)/Season 01/人世间.S01E04.mkv`, + SeasonNum: 1, + EpisodeNum: 4, + TMDbID: 156568, + } + if got, want := mediaSeriesKey(episodeIDOnly), mediaSeriesKey(cleanFolder); got != want { + t.Fatalf("episode filename tmdb id should not split clean folder key=%q, want %q", got, want) + } +} diff --git a/internal/service/metadata_match_trust.go b/internal/service/metadata_match_trust.go index 9b3cbe3..ec8827b 100644 --- a/internal/service/metadata_match_trust.go +++ b/internal/service/metadata_match_trust.go @@ -24,9 +24,6 @@ func automaticMetadataTitleTrusted(query string, match *Match) bool { return true } } - if metadataTrustChineseReleaseAlias(queryKey, match) { - return true - } if metadataTrustLocalizedSearchKeyword(queryKey, match) { return true } @@ -89,23 +86,6 @@ func metadataTrustTokenOverlap(queryKey, titleKey string) bool { return queryCoverage >= 0.80 && titleCoverage >= 0.50 } -func metadataTrustChineseReleaseAlias(queryKey string, match *Match) bool { - if match == nil || !metadataMatchHasChineseOrigin(match) { - return false - } - tokens := metadataTrustSignificantTokens(queryKey) - if len(tokens) < 2 { - return false - } - latinTokens := 0 - for _, token := range tokens { - if containsLatin(token) { - latinTokens++ - } - } - return latinTokens >= 2 -} - func metadataTrustLocalizedSearchKeyword(queryKey string, match *Match) bool { if match == nil || !metadataMatchHasExternalID(match) { return false @@ -157,16 +137,6 @@ func metadataMatchHasExternalID(match *Match) bool { strings.TrimSpace(match.TheTVDBID) != "") } -func metadataMatchHasChineseOrigin(match *Match) bool { - if match == nil { - return false - } - languages := normalizeTokens(match.Languages...) - countries := normalizeTokens(match.Countries...) - return hasAny(languages, "ZH", "ZH-CN", "ZH-TW", "CN") || - hasAny(countries, "CN", "TW", "HK", "MO") -} - func metadataTrustSignificantTokens(key string) []string { fields := strings.Fields(key) out := make([]string, 0, len(fields)) diff --git a/internal/service/organize_pipeline.go b/internal/service/organize_pipeline.go index ef85884..b1ebb7d 100644 --- a/internal/service/organize_pipeline.go +++ b/internal/service/organize_pipeline.go @@ -134,10 +134,7 @@ func (p *OrganizePipelineService) Run(ctx context.Context, req OrganizePipelineR if scanRoot == "" && strings.TrimSpace(path) != "" { scanRoot = filepath.Dir(path) } - preferredLibraryID := strings.TrimSpace(req.PreferredLibraryID) - if preferredLibraryID == "" && req.Scope == OrganizeScopeLibrary { - preferredLibraryID = strings.TrimSpace(req.LibraryID) - } + preferredLibraryID := p.scanPreferredLibraryID(ctx, req) res.Scans, res.Scrapes = p.scanner.ScanAndScrapeLibrariesForPath(ctx, scanRoot, preferredLibraryID, p.scrapeAfter(ctx, req)) } else if p.log != nil && res != nil && !req.DryRun { p.log.Info("organize pipeline skipped scan; no destination changes", @@ -211,3 +208,24 @@ func (p *OrganizePipelineService) scrapeAfter(ctx context.Context, req OrganizeP } return OrganizeScrapeAfterEnabled(ctx, p.repo) } + +func (p *OrganizePipelineService) scanPreferredLibraryID(ctx context.Context, req OrganizePipelineRequest) string { + if preferred := strings.TrimSpace(req.PreferredLibraryID); preferred != "" { + return preferred + } + switch req.Scope { + case OrganizeScopeMedia: + if p == nil || p.repo == nil || p.repo.Media == nil { + return "" + } + media, err := p.repo.Media.FindByID(ctx, strings.TrimSpace(req.MediaID)) + if err != nil || media == nil { + return "" + } + return strings.TrimSpace(media.LibraryID) + case OrganizeScopeLibrary: + return strings.TrimSpace(req.LibraryID) + default: + return "" + } +} diff --git a/internal/service/organize_pipeline_test.go b/internal/service/organize_pipeline_test.go index c217927..ca933b8 100644 --- a/internal/service/organize_pipeline_test.go +++ b/internal/service/organize_pipeline_test.go @@ -7,6 +7,8 @@ import ( "testing" "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/model" ) func TestOrganizeScanRootUsesActualOrganizedTarget(t *testing.T) { @@ -85,3 +87,44 @@ func TestOrganizePipelineFailsWhenEveryOrganizeItemErrors(t *testing.T) { t.Fatalf("task details = %#v, want transfer error detail", task.Details) } } + +func TestOrganizePipelinePrefersCurrentMediaLibraryAfterMediaOrganize(t *testing.T) { + repos := newOrganizerTestRepo(t) + sourceLib := model.Library{Name: "待整理", Path: filepath.Join(t.TempDir(), "incoming"), Type: "tv", Enabled: true} + targetLib := model.Library{Name: "国产剧", Path: filepath.Join(t.TempDir(), "media", "电视剧", "国产剧"), Type: "tv", Enabled: true} + if err := repos.Library.Create(t.Context(), &sourceLib); err != nil { + t.Fatal(err) + } + if err := repos.Library.Create(t.Context(), &targetLib); err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: targetLib.ID, + Title: "南部档案", + Path: filepath.Join(targetLib.Path, "南部档案", "Season 01", "南部档案 - S01E01.mkv"), + SeasonNum: 1, + EpisodeNum: 1, + ScrapeStatus: "matched", + } + if err := repos.Media.Upsert(t.Context(), &media); err != nil { + t.Fatal(err) + } + + pipeline := NewOrganizePipelineService(zap.NewNop(), repos, nil, nil, nil) + got := pipeline.scanPreferredLibraryID(t.Context(), OrganizePipelineRequest{ + Scope: OrganizeScopeMedia, + MediaID: media.ID, + }) + if got != targetLib.ID { + t.Fatalf("preferred library = %q, want current media library %q", got, targetLib.ID) + } + + got = pipeline.scanPreferredLibraryID(t.Context(), OrganizePipelineRequest{ + Scope: OrganizeScopeMedia, + MediaID: media.ID, + PreferredLibraryID: sourceLib.ID, + }) + if got != sourceLib.ID { + t.Fatalf("explicit preferred library = %q, want %q", got, sourceLib.ID) + } +} diff --git a/internal/service/organizer.go b/internal/service/organizer.go index b162ac7..d3c6798 100644 --- a/internal/service/organizer.go +++ b/internal/service/organizer.go @@ -88,6 +88,11 @@ func (o *OrganizerService) OrganizeMediaWithOptions(ctx context.Context, mediaID // Skip if already in place. if req.media.Path == dst.path { + if !req.dryRun { + if err := o.persistOrganizedMediaMetadata(ctx, req.media); err != nil { + return "", err + } + } return dst.path, nil } if req.dryRun { @@ -136,7 +141,7 @@ func (o *OrganizerService) OrganizeLibraryWithOptions(ctx context.Context, libra } res := &OrganizeResult{SourcePath: sourceRoot, DestPath: baseRoot, DryRun: opts.DryRun} for i := range rows { - if changed, err := o.reclassifyScannedMedia(ctx, rows[i], *lib, "", opts.DryRun, res); err != nil { + if changed, err := o.reclassifyScannedMedia(ctx, rows[i], *lib, "", opts, opts.DryRun, res); err != nil { res.Errors = append(res.Errors, fmt.Sprintf("%s: %s", rows[i].Title, err.Error())) continue } else if changed { diff --git a/internal/service/organizer_classification_test.go b/internal/service/organizer_classification_test.go index 522a212..7d6a412 100644 --- a/internal/service/organizer_classification_test.go +++ b/internal/service/organizer_classification_test.go @@ -299,7 +299,7 @@ func TestOrganizeDirectoryReclassifiesMovieFromDirtyGeneratedEpisodePath(t *test } } -func TestOrganizeDirectoryChineseAliasMetadataOverridesWrongWesternSourceCategory(t *testing.T) { +func TestOrganizeDirectoryRejectsLooseChineseAliasMetadataForWesternSourceCategory(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if r.URL.Path != "/search/tv" { @@ -344,19 +344,19 @@ func TestOrganizeDirectoryChineseAliasMetadataOverridesWrongWesternSourceCategor if err != nil { t.Fatalf("organize directory: %v", err) } - want := filepath.Join(dest, "动漫", "国漫", "镖人", "Season 02", "镖人 - S02E01.mkv") + want := filepath.Join(dest, "电视剧", "欧美剧", "Blades Of The Guardians", "Season 02", "Blades Of The Guardians - S02E01.mkv") if res.Organized != 1 { t.Fatalf("organized = %d, want 1; items=%#v errors=%#v", res.Organized, res.Items, res.Errors) } if _, err := os.Stat(want); err != nil { - t.Fatalf("Chinese alias metadata should override wrong western source category at %q: %v; items=%#v", want, err, res.Items) + t.Fatalf("loose Chinese alias metadata should be rejected and keep source category at %q: %v; items=%#v", want, err, res.Items) } - wrong := filepath.Join(dest, "电视剧", "欧美剧", "Blades Of The Guardians") + wrong := filepath.Join(dest, "动漫", "国漫", "镖人") if _, err := os.Stat(wrong); !os.IsNotExist(err) { - t.Fatalf("wrong western category should not remain at %q, err=%v", wrong, err) + t.Fatalf("rejected loose alias should not create Chinese anime category at %q, err=%v", wrong, err) } - if len(res.Items) != 1 || res.Items[0].Category != "国漫" || res.Items[0].MediaType != "anime" || res.Items[0].Title != "镖人" { - t.Fatalf("organize item = %#v, want 镖人 in 国漫/anime", res.Items) + if len(res.Items) != 1 || res.Items[0].Category != "欧美剧" || res.Items[0].MediaType != "tv" || res.Items[0].Title != "Blades Of The Guardians" { + t.Fatalf("organize item = %#v, want Blades Of The Guardians in 欧美剧/tv", res.Items) } } @@ -437,6 +437,83 @@ func TestOrganizeDirectoryDoesNotScrapeByDownloadCategoryFolder(t *testing.T) { } } +func TestOrganizeDirectoryUsesPathTMDbIDBeforeTitleSearch(t *testing.T) { + var paths []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + paths = append(paths, r.URL.Path) + switch r.URL.Path { + case "/tv/156568": + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": 156568, + "name": "人世间", + "original_name": "A Lifelong Journey", + "overview": "正确的剧集条目", + "poster_path": "/lifelong.jpg", + "first_air_date": "2022-01-28", + "origin_country": []string{"CN"}, + "genre_ids": []int{18}, + "vote_average": 8.1, + "original_language": "zh", + }) + case "/search/tv", "/search/movie": + _ = json.NewEncoder(w).Encode(map[string]any{ + "results": []map[string]any{{ + "id": 999999, + "name": "错误候选", + "first_air_date": "2022-01-01", + }}, + }) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + repos := newOrganizerTestRepo(t) + cfg := &config.Config{} + cfg.Organizer.SmartClassify = true + cfg.Secrets.TMDbAPIKey = "test-key" + cfg.Secrets.TMDbAPIProxy = upstream.URL + scraper := NewScraperService(cfg, zap.NewNop(), repos, NewTMDbProvider(cfg, zap.NewNop(), nil), nil, nil, nil, NewHub(zap.NewNop())) + + root := t.TempDir() + srcRoot := filepath.Join(root, "downloads") + dest := filepath.Join(root, "media") + sourceFile := filepath.Join(srcRoot, "东南亚电影", "人世间 (2022) [tmdbid-156568]", "人世间.S01E01.1080p.mkv") + writeOrgFile(t, sourceFile, "episode") + + organizer := NewOrganizerService(cfg, zap.NewNop(), repos) + organizer.SetScraper(scraper) + res, err := organizer.OrganizeDirectory(t.Context(), OrganizeOptions{ + SourcePath: srcRoot, + DestPath: filepath.Join(dest, "电视剧", "国产电视剧"), + MediaType: "tv", + TransferMode: TransferCopy, + }) + if err != nil { + t.Fatalf("organize directory: %v", err) + } + want := filepath.Join(dest, "电视剧", "国产剧", "人世间", "Season 01", "人世间 - S01E01.mkv") + if res.Organized != 1 { + t.Fatalf("organized = %d, want 1; items=%#v errors=%#v paths=%v", res.Organized, res.Items, res.Errors, paths) + } + if _, err := os.Stat(want); err != nil { + t.Fatalf("tmdb-id organize target missing at %q: %v; items=%#v paths=%v", want, err, res.Items, paths) + } + if len(paths) == 0 || paths[0] != "/tv/156568" { + t.Fatalf("first metadata lookup path=%q, want /tv/156568; all=%v", firstQuery(paths), paths) + } + for _, path := range paths { + if strings.HasPrefix(path, "/search/") { + t.Fatalf("path tmdb id should avoid fuzzy search; paths=%v", paths) + } + } + if len(res.Items) != 1 || res.Items[0].Category != "国产剧" || res.Items[0].MediaType != "tv" { + t.Fatalf("organize item = %#v, want 国产剧/tv", res.Items) + } +} + func TestOrganizeDirectoryEpisodeMarkerOverridesMovieSourceFolder(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/service/organizer_directory.go b/internal/service/organizer_directory.go index 44c808d..eb8cc63 100644 --- a/internal/service/organizer_directory.go +++ b/internal/service/organizer_directory.go @@ -49,7 +49,9 @@ func (o *OrganizerService) OrganizeDirectory(ctx context.Context, opts OrganizeO if _, ok := ParseCloudLibraryMount(requestedDest); ok { return nil, errors.New("organize destination must be a local writable media directory; enable cloud transfer in external storage when writing to cloud") } - dest := normalizeOrganizeDestinationRoot(resolveMappedDestinationPath(requestedDest)) + resolvedDest := resolveMappedDestinationPath(requestedDest) + mediaTypeOverride, mediaCategoryOverride := o.effectiveOrganizeOverrides(opts, resolvedDest) + dest := normalizeOrganizeDestinationRoot(resolvedDest) if dest == "" || dest == "." { return nil, errors.New("destination path required") } @@ -84,8 +86,8 @@ func (o *OrganizerService) OrganizeDirectory(ctx context.Context, opts OrganizeO SourceRoot: filepath.Dir(source), DestRoot: dest, Mode: mode, - MediaTypeOverride: opts.MediaType, - MediaCategoryOverride: opts.MediaCategory, + MediaTypeOverride: mediaTypeOverride, + MediaCategoryOverride: mediaCategoryOverride, DryRun: opts.DryRun, AllowReplaceExisting: opts.AllowReplaceExisting, MetadataCache: metadataCache, @@ -120,8 +122,8 @@ func (o *OrganizerService) OrganizeDirectory(ctx context.Context, opts OrganizeO SourceRoot: source, DestRoot: dest, Mode: mode, - MediaTypeOverride: opts.MediaType, - MediaCategoryOverride: opts.MediaCategory, + MediaTypeOverride: mediaTypeOverride, + MediaCategoryOverride: mediaCategoryOverride, DryRun: opts.DryRun, AllowReplaceExisting: opts.AllowReplaceExisting, MetadataCache: metadataCache, diff --git a/internal/service/organizer_directory_dedup_test.go b/internal/service/organizer_directory_dedup_test.go index e01319c..c7ff86f 100644 --- a/internal/service/organizer_directory_dedup_test.go +++ b/internal/service/organizer_directory_dedup_test.go @@ -245,6 +245,73 @@ func TestOrganizeDirectoryKeepsHigherResolutionExisting(t *testing.T) { } } +func TestOrganizeDirectoryDoesNotReclassifySameEpisodeVersionInTargetDir(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "downloads") + dest := filepath.Join(root, "media") + + source := filepath.Join(src, "南部档案.S01E16.1080p.IQ.WEB-DL.H264.AAC-UBWEB.mkv") + writeOrgFile(t, source, "download-1080p") + + repos := newOrganizerTestRepo(t) + if err := repos.Setting.Set(t.Context(), "organize.tv_format", "{title}/Season {season:02}/{title} - {episode_tag}{% if video_format %}-{{video_format}}{% endif %}{fileExt}"); err != nil { + t.Fatal(err) + } + + lib := model.Library{Name: "国产剧", Path: filepath.Join(dest, "电视剧", "国产剧"), Type: "tv", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + + existing := filepath.Join(lib.Path, "南部档案", "Season 01", "南部档案 - S01E16-2160p.IQ.WEB-DL.H265.DDP5.1-UBWEB.mkv") + writeOrgFile(t, existing, "library-2160p") + row := model.Media{ + LibraryID: lib.ID, + Title: "南部档案", + Path: existing, + Container: "mkv", + SeasonNum: 1, + EpisodeNum: 16, + Width: 3840, + Height: 2160, + TMDbID: 123456, + ScrapeStatus: "matched", + } + if err := repos.Media.Upsert(t.Context(), &row); err != nil { + t.Fatal(err) + } + + org := NewOrganizerService(&config.Config{}, zap.NewNop(), repos) + res, err := org.OrganizeDirectory(t.Context(), OrganizeOptions{ + SourcePath: src, + DestPath: dest, + MediaType: "tv", + MediaCategory: "国产剧", + TransferMode: TransferCopy, + AllowReplaceExisting: false, + }) + if err != nil { + t.Fatalf("organize directory: %v", err) + } + if res.Reclassified != 0 || res.Replaced != 0 || res.Organized != 0 || res.Skipped != 1 { + t.Fatalf("result = %+v, want skipped duplicate without reclassify", res) + } + if _, err := os.Stat(existing); err != nil { + t.Fatalf("existing higher version should stay at original path: %v", err) + } + wrongRename := filepath.Join(lib.Path, "南部档案", "Season 01", "南部档案 - S01E16-1080p.IQ.WEB-DL.H264.AAC-UBWEB.mkv") + if _, err := os.Stat(wrongRename); !os.IsNotExist(err) { + t.Fatalf("existing version must not be renamed to incoming release path, stat err=%v", err) + } + var got model.Media + if err := repos.DB.First(&got, "path = ?", existing).Error; err != nil { + t.Fatal(err) + } + if got.Title != "南部档案" || got.TMDbID != 123456 || got.ScrapeStatus != "matched" { + t.Fatalf("metadata changed after duplicate organize: title=%q tmdb=%d status=%q", got.Title, got.TMDbID, got.ScrapeStatus) + } +} + // TestOrganizeDirectoryTVEpisodeDedup verifies per-episode dedup for TV media. func TestOrganizeDirectoryTVEpisodeDedup(t *testing.T) { root := t.TempDir() diff --git a/internal/service/organizer_directory_layout.go b/internal/service/organizer_directory_layout.go index e2ec183..d6594d2 100644 --- a/internal/service/organizer_directory_layout.go +++ b/internal/service/organizer_directory_layout.go @@ -125,8 +125,24 @@ func (o *OrganizerService) directoryCategoryTypes() map[string]organizeDirectory addAlias("外语电影", "euus_movie", "欧美电影", "movie") addAlias("外国电影", "euus_movie", "欧美电影", "movie") addConfigured("domestic_tv", "国产剧", "tv") + addAlias("国剧", "domestic_tv", "国产剧", "tv") + addAlias("大陆剧", "domestic_tv", "国产剧", "tv") + addAlias("华语剧", "domestic_tv", "国产剧", "tv") + addAlias("国产电视剧", "domestic_tv", "国产剧", "tv") + addAlias("大陆电视剧", "domestic_tv", "国产剧", "tv") + addAlias("华语电视剧", "domestic_tv", "国产剧", "tv") + addAlias("港剧", "domestic_tv", "国产剧", "tv") + addAlias("台剧", "domestic_tv", "国产剧", "tv") + addAlias("港台剧", "domestic_tv", "国产剧", "tv") addConfigured("euus_tv", "欧美剧", "tv") + addAlias("欧美电视剧", "euus_tv", "欧美剧", "tv") + addAlias("美剧", "euus_tv", "欧美剧", "tv") + addAlias("英剧", "euus_tv", "欧美剧", "tv") addConfigured("jk_tv", "日韩剧", "tv") + addAlias("日韩电视剧", "jk_tv", "日韩剧", "tv") + addAlias("日剧", "jk_tv", "日韩剧", "tv") + addAlias("韩剧", "jk_tv", "日韩剧", "tv") + addAlias("泰剧", "jk_tv", "日韩剧", "tv") addConfigured("cn_anime", "国漫", "anime") addConfigured("jp_anime", "日番", "anime") addConfigured("kr_anime", "韩漫", "anime") diff --git a/internal/service/organizer_directory_metadata.go b/internal/service/organizer_directory_metadata.go index 12673a5..06de588 100644 --- a/internal/service/organizer_directory_metadata.go +++ b/internal/service/organizer_directory_metadata.go @@ -31,6 +31,9 @@ func (o *OrganizerService) lookupOrganizeMetadata(ctx context.Context, src, sour if o == nil || o.scraper == nil || !o.scraper.AnyEnabled() { return nil } + if match := o.lookupOrganizeMetadataByPathHints(ctx, src, sourceRoot, normalizedType, title, year, lookupSeason, lookupEpisode, seriesLike); match != nil { + return match + } libType := normalizeOrganizeMediaType(mediaType) if libType == "" { libType = organizeLibraryModelType(mediaType) @@ -99,6 +102,55 @@ func (o *OrganizerService) lookupOrganizeMetadata(ctx context.Context, src, sour return nil } +func (o *OrganizerService) lookupOrganizeMetadataByPathHints(ctx context.Context, src, sourceRoot, mediaType, title string, year, season, episode int, seriesLike bool) *Match { + if o == nil || o.scraper == nil { + return nil + } + meta, hints := pathHintMetadata(src, seriesLike) + if !hints.useful() { + return nil + } + if meta != nil { + if strings.TrimSpace(title) == "" { + title = strings.TrimSpace(meta.Title) + } + if year <= 0 { + year = meta.Year + } + } + libType := normalizeOrganizeMediaType(mediaType) + if libType == "" { + libType = organizeLibraryModelType(mediaType) + } + lib := &model.Library{Path: sourceRoot, Type: libType, Enabled: true} + media := &model.Media{ + Title: title, + Year: year, + Path: src, + SeasonNum: season, + EpisodeNum: episode, + TMDbID: hints.TMDbID, + BangumiID: hints.BangumiID, + DoubanID: strings.TrimSpace(hints.DoubanID), + TheTVDBID: strings.TrimSpace(hints.TheTVDBID), + } + match := o.scraper.matchFromMediaExternalIDs(ctx, media, lib) + if match == nil || strings.TrimSpace(match.Title) == "" { + return nil + } + if o.log != nil { + o.log.Info("organize metadata matched by path id before rename", + zap.String("source", src), + zap.String("title", match.Title), + zap.String("media_type", match.MediaType), + zap.Int("tmdb_id", match.TMDbID), + zap.Int("bangumi_id", match.BangumiID), + zap.String("douban_id", match.DoubanID), + zap.String("thetvdb_id", match.TheTVDBID)) + } + return match +} + func (o *OrganizerService) lookupOrganizeAdultMetadata(ctx context.Context, src, mediaType, title string) *Match { if o == nil || o.scraper == nil || o.scraper.adult == nil || !o.scraper.adult.Enabled() { return nil diff --git a/internal/service/organizer_directory_metadata_persist.go b/internal/service/organizer_directory_metadata_persist.go new file mode 100644 index 0000000..f784e8b --- /dev/null +++ b/internal/service/organizer_directory_metadata_persist.go @@ -0,0 +1,76 @@ +package service + +import ( + "context" + "os" + "path/filepath" + "strings" + + "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func (o *OrganizerService) persistOrganizedSourceMetadata(ctx context.Context, plan organizeSourceFilePlan) { + if o == nil || o.repo == nil || o.repo.Media == nil || plan.MetadataMatch == nil { + return + } + libraryID := strings.TrimSpace(plan.TargetLibraryID) + if libraryID == "" { + return + } + media := organizedSourceMediaFromPlan(libraryID, plan) + if info, err := os.Stat(plan.Target.Path); err == nil && !info.IsDir() { + media.SizeBytes = info.Size() + } + if fileID, ok := fileIdentity(plan.Target.Path); ok { + media.FileID = fileID + } + if err := o.repo.Media.Upsert(ctx, media); err != nil && o.log != nil { + o.log.Warn("persist organized metadata failed", + zap.String("path", plan.Target.Path), + zap.String("library_id", libraryID), + zap.Error(err)) + } +} + +func organizedSourceMediaFromPlan(libraryID string, plan organizeSourceFilePlan) *model.Media { + match := plan.MetadataMatch + media := &model.Media{ + LibraryID: libraryID, + Title: strings.TrimSpace(firstNonEmpty(match.Title, plan.Identity.ParsedTitle, plan.Identity.Title)), + OriginalName: strings.TrimSpace(match.OriginalName), + Overview: match.Overview, + PosterURL: match.PosterURL, + BackdropURL: match.BackdropURL, + Year: firstPositiveInt(match.Year, plan.Identity.Year), + Rating: match.Rating, + Path: plan.Target.Path, + Container: strings.TrimPrefix(strings.ToLower(filepath.Ext(plan.Target.Path)), "."), + SeasonNum: plan.Identity.Season, + EpisodeNum: plan.Identity.Episode, + TMDbID: match.TMDbID, + BangumiID: match.BangumiID, + DoubanID: strings.TrimSpace(match.DoubanID), + TheTVDBID: strings.TrimSpace(match.TheTVDBID), + Genres: strings.Join(match.Genres, ","), + Countries: strings.Join(match.Countries, ","), + Languages: strings.Join(match.Languages, ","), + NSFW: match.NSFW, + ScrapeStatus: "matched", + } + if normalizeOrganizeMediaType(plan.Layout.MediaType) == "movie" { + media.SeasonNum = 0 + media.EpisodeNum = 0 + } + return media +} + +func firstPositiveInt(values ...int) int { + for _, value := range values { + if value > 0 { + return value + } + } + return 0 +} diff --git a/internal/service/organizer_directory_reclassify.go b/internal/service/organizer_directory_reclassify.go index c279839..942e78d 100644 --- a/internal/service/organizer_directory_reclassify.go +++ b/internal/service/organizer_directory_reclassify.go @@ -39,6 +39,7 @@ func (o *OrganizerService) reclassifyExistingMedia(ctx context.Context, req orga return false, nil } candidates := reclassifyExistingCandidates(req.Existing, target, req.DestRoot) + candidates = reclassifyMoveCandidates(candidates, target) if len(candidates) == 0 { return false, nil } @@ -112,6 +113,22 @@ func reclassifyExistingCandidates(existing []string, target, destRoot string) [] return out } +func reclassifyMoveCandidates(existing []string, target string) []string { + targetDir := filepath.Clean(filepath.Dir(target)) + out := make([]string, 0, len(existing)) + for _, path := range existing { + cleaned := filepath.Clean(strings.TrimSpace(path)) + if cleaned == "" || cleaned == "." { + continue + } + if strings.EqualFold(filepath.Clean(filepath.Dir(cleaned)), targetDir) { + continue + } + out = append(out, cleaned) + } + return out +} + func (o *OrganizerService) cleanupReclassifiedDuplicates(ctx context.Context, req organizeExistingReclassifyRequest, target string, candidates []string) (int, error) { cleaned := 0 for _, oldPath := range candidates { diff --git a/internal/service/organizer_directory_source_existing.go b/internal/service/organizer_directory_source_existing.go index 29a01a2..5880d7f 100644 --- a/internal/service/organizer_directory_source_existing.go +++ b/internal/service/organizer_directory_source_existing.go @@ -127,6 +127,7 @@ func (o *OrganizerService) writeOrganizedSourceFile(ctx context.Context, req org o.log.Warn("organize sidecar nfo failed", zap.String("from", req.Source), zap.String("to", plan.Target.Path), zap.Error(err)) } + o.persistOrganizedSourceMetadata(ctx, plan) req.Result.Organized++ return nil } diff --git a/internal/service/organizer_library_classification.go b/internal/service/organizer_library_classification.go index dad421d..d23a6ee 100644 --- a/internal/service/organizer_library_classification.go +++ b/internal/service/organizer_library_classification.go @@ -56,12 +56,12 @@ func (o *OrganizerService) organizeCategoryAliases(mediaType, category string) m add("美漫", categoryName(categories, "us_anime", "美漫")) case normalizeOrganizeCategoryKey(categoryName(categories, "other_anime", "其他")), "其他", "其他动漫", "其它动漫": add("其他", categoryName(categories, "other_anime", "其他")) - case normalizeOrganizeCategoryKey(categoryName(categories, "domestic_tv", "国产剧")), "国产剧", "国剧", "大陆剧", "国产电视剧": - add("国产剧", categoryName(categories, "domestic_tv", "国产剧")) - case normalizeOrganizeCategoryKey(categoryName(categories, "euus_tv", "欧美剧")), "欧美剧", "欧美电视剧": - add("欧美剧", categoryName(categories, "euus_tv", "欧美剧")) - case normalizeOrganizeCategoryKey(categoryName(categories, "jk_tv", "日韩剧")), "日韩剧", "日剧", "韩剧": - add("日韩剧", categoryName(categories, "jk_tv", "日韩剧")) + case normalizeOrganizeCategoryKey(categoryName(categories, "domestic_tv", "国产剧")), "国产剧", "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "港剧", "台剧", "港台剧": + add("国产剧", "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "港剧", "台剧", "港台剧", categoryName(categories, "domestic_tv", "国产剧")) + case normalizeOrganizeCategoryKey(categoryName(categories, "euus_tv", "欧美剧")), "欧美剧", "欧美电视剧", "美剧", "英剧": + add("欧美剧", "欧美电视剧", "美剧", "英剧", categoryName(categories, "euus_tv", "欧美剧")) + case normalizeOrganizeCategoryKey(categoryName(categories, "jk_tv", "日韩剧")), "日韩剧", "日韩电视剧", "日剧", "韩剧", "泰剧": + add("日韩剧", "日韩电视剧", "日剧", "韩剧", "泰剧", categoryName(categories, "jk_tv", "日韩剧")) case normalizeOrganizeCategoryKey(categoryName(categories, "variety", "综艺")), "综艺", "真人秀": add("综艺", categoryName(categories, "variety", "综艺")) case normalizeOrganizeCategoryKey(categoryName(categories, "documentary", "纪录片")), "纪录片", "纪录": diff --git a/internal/service/organizer_media.go b/internal/service/organizer_media.go index ad5bb1a..9c9fc4d 100644 --- a/internal/service/organizer_media.go +++ b/internal/service/organizer_media.go @@ -41,7 +41,9 @@ func (o *OrganizerService) resolveOrganizeMediaRequest(ctx context.Context, medi if _, ok := ParseCloudLibraryMount(lib.Path); ok { return organizeMediaRequest{}, errors.New("local organize cannot use cloud libraries directly; use external storage scan/mount for cloud media or enable cloud transfer to write to cloud") } - baseRoot := normalizeOrganizeDestinationRoot(o.resolveBaseRoot(ctx, lib, opts.DestPath)) + requestedBaseRoot := o.resolveBaseRoot(ctx, lib, opts.DestPath) + mediaType, mediaCategory := o.effectiveOrganizeOverrides(opts, requestedBaseRoot) + baseRoot := normalizeOrganizeDestinationRoot(requestedBaseRoot) if _, ok := ParseCloudLibraryMount(baseRoot); ok { return organizeMediaRequest{}, errors.New("organize destination must be a local writable media directory; enable cloud transfer in external storage when writing to cloud") } @@ -55,12 +57,13 @@ func (o *OrganizerService) resolveOrganizeMediaRequest(ctx context.Context, medi return organizeMediaRequest{}, err } } + o.refreshOrganizeMediaMetadata(ctx, m, lib, opts.MediaType) return organizeMediaRequest{ media: m, library: lib, baseRoot: baseRoot, - mediaType: strings.TrimSpace(opts.MediaType), - mediaCategory: strings.TrimSpace(opts.MediaCategory), + mediaType: mediaType, + mediaCategory: mediaCategory, dryRun: opts.DryRun, transferMode: o.resolveTransferMode(ctx, opts.TransferMode), }, nil @@ -149,6 +152,7 @@ func (o *OrganizerService) applyOrganizeMedia(ctx context.Context, req organizeM updates := map[string]any{ "path": dst.path, } + addOrganizedMediaMetadataUpdates(updates, *m) if normalizeOrganizeMediaType(dst.mediaType) == "movie" { updates["season_num"] = 0 updates["episode_num"] = 0 diff --git a/internal/service/organizer_media_metadata.go b/internal/service/organizer_media_metadata.go new file mode 100644 index 0000000..bb77b9c --- /dev/null +++ b/internal/service/organizer_media_metadata.go @@ -0,0 +1,117 @@ +package service + +import ( + "context" + "strings" + + "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func (o *OrganizerService) refreshOrganizeMediaMetadata(ctx context.Context, media *model.Media, lib *model.Library, requestedType string) { + if o == nil || media == nil || lib == nil || !organizeMediaNeedsMetadataRefresh(*media) { + return + } + mediaType := normalizeOrganizeMediaType(requestedType) + if mediaType == "" { + mediaType = normalizeOrganizeMediaType(lib.Type) + } + match := o.lookupReclassifyMetadata(ctx, *media, *lib, mediaType) + if match == nil { + return + } + refreshed := mediaWithReclassifyMatch(*media, match) + *media = refreshed + if o.log != nil { + o.log.Info("organize media metadata refreshed before rename", + zap.String("media", media.ID), + zap.String("path", media.Path), + zap.String("title", media.Title), + zap.Int("tmdb_id", media.TMDbID), + zap.Int("bangumi_id", media.BangumiID), + zap.String("douban_id", media.DoubanID), + zap.String("thetvdb_id", media.TheTVDBID)) + } +} + +func organizeMediaNeedsMetadataRefresh(media model.Media) bool { + if strings.TrimSpace(media.ScrapeStatus) != "matched" { + return true + } + if organizeMediaTitleLooksLikeRelease(media.Title) { + return true + } + return media.TMDbID <= 0 && + media.BangumiID <= 0 && + strings.TrimSpace(media.DoubanID) == "" && + strings.TrimSpace(media.TheTVDBID) == "" +} + +func organizeMediaTitleLooksLikeRelease(title string) bool { + title = strings.TrimSpace(title) + if title == "" || organizeWeakFileTitle(title) { + return true + } + if season, episode := ParseEpisode(title); season > 0 || episode > 0 { + return true + } + normalized := strings.ToLower(strings.NewReplacer(".", " ", "_", " ", "-", " ").Replace(title)) + for _, field := range strings.Fields(normalized) { + if _, ok := releaseBoundaryTokenSet[field]; ok { + return true + } + } + return false +} + +func addOrganizedMediaMetadataUpdates(updates map[string]any, media model.Media) { + if updates == nil || strings.TrimSpace(media.ScrapeStatus) != "matched" { + return + } + setNonEmptyUpdate(updates, "title", media.Title) + setNonEmptyUpdate(updates, "original_name", media.OriginalName) + setNonEmptyUpdate(updates, "episode_title", media.EpisodeTitle) + setNonEmptyUpdate(updates, "poster_url", media.PosterURL) + setNonEmptyUpdate(updates, "backdrop_url", media.BackdropURL) + setNonEmptyUpdate(updates, "overview", media.Overview) + setNonEmptyUpdate(updates, "languages", media.Languages) + setNonEmptyUpdate(updates, "countries", media.Countries) + setNonEmptyUpdate(updates, "genres", media.Genres) + if media.Year > 0 { + updates["year"] = media.Year + } + if media.Rating > 0 { + updates["rating"] = media.Rating + } + if media.TMDbID > 0 { + updates["tm_db_id"] = media.TMDbID + } + if media.BangumiID > 0 { + updates["bangumi_id"] = media.BangumiID + } + setNonEmptyUpdate(updates, "douban_id", media.DoubanID) + setNonEmptyUpdate(updates, "thetvdb_id", media.TheTVDBID) + if media.NSFW { + updates["nsfw"] = true + } + updates["scrape_status"] = "matched" +} + +func (o *OrganizerService) persistOrganizedMediaMetadata(ctx context.Context, media *model.Media) error { + if o == nil || o.repo == nil || o.repo.DB == nil || media == nil { + return nil + } + updates := map[string]any{} + addOrganizedMediaMetadataUpdates(updates, *media) + if len(updates) == 0 { + return nil + } + return o.repo.DB.WithContext(ctx).Model(&model.Media{}).Where("id = ?", media.ID).Updates(updates).Error +} + +func setNonEmptyUpdate(updates map[string]any, key, value string) { + if strings.TrimSpace(value) != "" { + updates[key] = value + } +} diff --git a/internal/service/organizer_paths.go b/internal/service/organizer_paths.go index a8ffacd..8d09bd6 100644 --- a/internal/service/organizer_paths.go +++ b/internal/service/organizer_paths.go @@ -70,7 +70,9 @@ func (o *OrganizerService) categoryPhysicalRootDirForType(mediaType, category st categoryName(categories, "variety", "综艺"), categoryName(categories, "documentary", "纪录片"), categoryName(categories, "children", "儿童"), - "国产剧", "欧美剧", "日韩剧", "日剧", "韩剧", "综艺", "真人秀", "纪录片", "纪录", "儿童", "少儿", "未分类", + "国产剧", "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "港剧", "台剧", "港台剧", + "欧美剧", "欧美电视剧", "美剧", "英剧", "日韩剧", "日韩电视剧", "日剧", "韩剧", "泰剧", + "综艺", "真人秀", "纪录片", "纪录", "儿童", "少儿", "未分类", ) { return "电视剧" } @@ -124,7 +126,9 @@ func (o *OrganizerService) categoryPhysicalRootDir(category string) string { categoryName(categories, "variety", "综艺"), categoryName(categories, "documentary", "纪录片"), categoryName(categories, "children", "儿童"), - "国产剧", "欧美剧", "日韩剧", "日剧", "韩剧", "综艺", "真人秀", "纪录片", "纪录", "儿童", "少儿", "未分类", + "国产剧", "国剧", "大陆剧", "华语剧", "国产电视剧", "大陆电视剧", "华语电视剧", "港剧", "台剧", "港台剧", + "欧美剧", "欧美电视剧", "美剧", "英剧", "日韩剧", "日韩电视剧", "日剧", "韩剧", "泰剧", + "综艺", "真人秀", "纪录片", "纪录", "儿童", "少儿", "未分类", ): return "电视剧" case match( diff --git a/internal/service/organizer_reclassify_scanned.go b/internal/service/organizer_reclassify_scanned.go index c6bce03..a4f1fbd 100644 --- a/internal/service/organizer_reclassify_scanned.go +++ b/internal/service/organizer_reclassify_scanned.go @@ -69,7 +69,7 @@ func (o *OrganizerService) ReclassifyMisclassifiedMedia(ctx context.Context, opt if !ok { continue } - changed, err := o.reclassifyScannedMedia(ctx, rows[i], lib, typeHints[rows[i].ID], opts.DryRun, res) + changed, err := o.reclassifyScannedMedia(ctx, rows[i], lib, typeHints[rows[i].ID], OrganizeOptions{}, opts.DryRun, res) if err != nil { res.Errors = append(res.Errors, fmt.Sprintf("%s: %s", rows[i].Title, err.Error())) if o.log != nil { @@ -103,7 +103,7 @@ func normalizeReclassifyMediaTypeHints(values map[string]string) map[string]stri return out } -func (o *OrganizerService) reclassifyScannedMedia(ctx context.Context, media model.Media, lib model.Library, mediaTypeHint string, dryRun bool, res *OrganizeResult) (bool, error) { +func (o *OrganizerService) reclassifyScannedMedia(ctx context.Context, media model.Media, lib model.Library, mediaTypeHint string, opts OrganizeOptions, dryRun bool, res *OrganizeResult) (bool, error) { if res == nil || !lib.Enabled || strings.TrimSpace(media.Path) == "" { return false, nil } @@ -114,11 +114,20 @@ func (o *OrganizerService) reclassifyScannedMedia(ctx context.Context, media mod return false, nil } + requestedBaseRoot := o.resolveBaseRoot(ctx, &lib, opts.DestPath) + overrideType, overrideCategory := o.effectiveOrganizeOverrides(opts, requestedBaseRoot) + explicitType := overrideType != "" + explicitCategory := overrideCategory != "" + mediaType := normalizeOrganizeMediaType(lib.Type) if mediaTypeHint != "" { mediaType = mediaTypeHint } + if explicitType { + mediaType = overrideType + } metadataMatch := organizeMatchFromMedia(&media) + metadataRefreshed := false if metadataMatch != nil && mediaTypeHint != "" { metadataMatch.MediaType = mediaTypeHint } @@ -128,17 +137,28 @@ func (o *OrganizerService) reclassifyScannedMedia(ctx context.Context, media mod return false, nil } media = mediaWithReclassifyMatch(media, metadataMatch) + metadataRefreshed = true } - if matchType := normalizeOrganizeMediaType(metadataMatchMediaType(metadataMatch)); matchType != "" { - mediaType = matchType + if !explicitType { + if matchType := normalizeOrganizeMediaType(metadataMatchMediaType(metadataMatch)); matchType != "" { + mediaType = matchType + } + } + category := overrideCategory + if category == "" { + category = o.classifyMedia(ctx, &media, mediaType) } - category := o.classifyMedia(ctx, &media, mediaType) if category == "" { return false, nil } if impliedType, normalizedCategory := o.mediaTypeForDirectoryCategory(category); impliedType != "" { - mediaType = impliedType category = normalizedCategory + if !explicitType { + mediaType = impliedType + } + } + if explicitCategory && overrideCategory != "" { + category = overrideCategory } if mediaType == "" { mediaType = normalizeOrganizeMediaType(lib.Type) @@ -147,7 +167,7 @@ func (o *OrganizerService) reclassifyScannedMedia(ctx context.Context, media mod return false, nil } - baseRoot := normalizeOrganizeDestinationRoot(o.resolveBaseRoot(ctx, &lib, "")) + baseRoot := normalizeOrganizeDestinationRoot(requestedBaseRoot) targetLibrary, matched := o.organizeLibraryForLayout(ctx, baseRoot, mediaType, category) if !matched || strings.TrimSpace(targetLibrary.ID) == "" || strings.TrimSpace(targetLibrary.Path) == "" { targetRoot := categoryRoot(o.organizeRoot(baseRoot, mediaType, category), category) @@ -160,6 +180,22 @@ func (o *OrganizerService) reclassifyScannedMedia(ctx context.Context, media mod } } if strings.EqualFold(targetLibrary.ID, lib.ID) && pathWithin(media.Path, targetLibrary.Path) { + if metadataRefreshed && !dryRun { + if err := o.persistOrganizedMediaMetadata(ctx, &media); err != nil { + return false, err + } + if o.log != nil { + o.log.Info("media metadata refreshed without reclassify", + zap.String("media", media.ID), + zap.String("path", media.Path), + zap.String("title", media.Title), + zap.Int("tmdb_id", media.TMDbID), + zap.Int("bangumi_id", media.BangumiID), + zap.String("douban_id", media.DoubanID), + zap.String("thetvdb_id", media.TheTVDBID)) + } + return true, nil + } return false, nil } if pathWithin(media.Path, targetLibrary.Path) { diff --git a/internal/service/organizer_reclassify_test.go b/internal/service/organizer_reclassify_test.go index afff23c..3876b70 100644 --- a/internal/service/organizer_reclassify_test.go +++ b/internal/service/organizer_reclassify_test.go @@ -25,7 +25,7 @@ func TestOrganizeDirectoryReclassifiesExistingWrongCategoryMedia(t *testing.T) { "results": []map[string]any{{ "id": 292696, "name": "莫离", - "original_name": "莫离", + "original_name": "The First Jasmine", "original_language": "zh", "origin_country": []string{"CN"}, "genre_ids": []int{18}, diff --git a/internal/service/organizer_scan.go b/internal/service/organizer_scan.go index a4aa722..3f99128 100644 --- a/internal/service/organizer_scan.go +++ b/internal/service/organizer_scan.go @@ -164,7 +164,9 @@ func (s *ScannerService) scrapeOrganizeTargets(ctx context.Context, targets []mo // Organize is an explicit ingest workflow: after rename/classification, // previously failed no_match rows should be retried so the operator does // not need to run a separate manual scrape. - result, err := s.scraper.EnrichLibraryDetailedWithOptions(ctx, lib.ID, skipEpisodeArtworkOptions(true)) + options := skipEpisodeArtworkOptions(true) + options.RefreshWeakMatched = true + result, err := s.scraper.EnrichLibraryDetailedWithOptions(ctx, lib.ID, options) if err != nil { summary.Error = err.Error() } else { diff --git a/internal/service/organizer_scrape_test.go b/internal/service/organizer_scrape_test.go index 6c516ae..52cd530 100644 --- a/internal/service/organizer_scrape_test.go +++ b/internal/service/organizer_scrape_test.go @@ -100,6 +100,212 @@ func TestOrganizeDirectoryUsesScraperMatchBeforeRename(t *testing.T) { if len(res.Items) != 1 || res.Items[0].Target != want || res.Items[0].Title != "间谍过家家" { t.Fatalf("organize preview did not use scraper metadata: %#v", res.Items) } + var media model.Media + if err := repos.DB.First(&media, "path = ?", want).Error; err != nil { + t.Fatalf("organized metadata should be persisted before scan: %v", err) + } + if media.Title != "间谍过家家" || media.TMDbID != 12345 || media.ScrapeStatus != "matched" { + t.Fatalf("persisted media = title=%q tmdb=%d status=%q, want localized matched metadata", media.Title, media.TMDbID, media.ScrapeStatus) + } +} + +func TestOrganizeMediaRefreshesMetadataBeforeRename(t *testing.T) { + scraper, repos, closeServer := newTestScraper(t) + defer closeServer() + + root := t.TempDir() + src := filepath.Join(root, "downloads") + dest := filepath.Join(root, "media", "电视剧") + sourceFile := filepath.Join(src, "Spy.x.Family.S01E01.2022.1080p.mkv") + writeOrgFile(t, sourceFile, "episode") + + lib := model.Library{ + Name: "剧集", + Path: dest, + Type: "tv", + Enabled: true, + } + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: lib.ID, + Title: "Spy x Family S01E01 2022 1080p", + Path: sourceFile, + Container: "mkv", + SeasonNum: 1, + EpisodeNum: 1, + ScrapeStatus: "pending", + } + if err := repos.Media.Upsert(t.Context(), &media); err != nil { + t.Fatal(err) + } + + organizer := NewOrganizerService(&config.Config{}, zap.NewNop(), repos) + organizer.SetScraper(scraper) + dst, err := organizer.OrganizeMediaWithOptions(t.Context(), media.ID, OrganizeOptions{ + TransferMode: TransferCopy, + }) + if err != nil { + t.Fatalf("organize media: %v", err) + } + want := filepath.Join(dest, "间谍过家家", "Season 01", "间谍过家家 - S01E01.mkv") + if dst != want { + t.Fatalf("dst = %q, want %q", dst, want) + } + + var got model.Media + if err := repos.DB.First(&got, "id = ?", media.ID).Error; err != nil { + t.Fatal(err) + } + if got.Title != "间谍过家家" || got.TMDbID != 12345 || got.ScrapeStatus != "matched" { + t.Fatalf("media = title=%q tmdb=%d status=%q, want localized matched metadata", got.Title, got.TMDbID, got.ScrapeStatus) + } + if got.Path != want { + t.Fatalf("media path = %q, want %q", got.Path, want) + } +} + +func TestOrganizeMediaRefreshesMatchedReleaseTitleBeforeRename(t *testing.T) { + scraper, repos, closeServer := newTestScraper(t) + defer closeServer() + + root := t.TempDir() + src := filepath.Join(root, "downloads") + dest := filepath.Join(root, "media", "电视剧") + sourceFile := filepath.Join(src, "Spy.x.Family.S01E01.2022.1080p.WEB-DL.mkv") + writeOrgFile(t, sourceFile, "episode") + + lib := model.Library{Name: "剧集", Path: dest, Type: "tv", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: lib.ID, + Title: "Spy.x.Family.S01E01.2022.1080p.WEB-DL", + Path: sourceFile, + Container: "mkv", + SeasonNum: 1, + EpisodeNum: 1, + TMDbID: 12345, + ScrapeStatus: "matched", + } + if err := repos.Media.Upsert(t.Context(), &media); err != nil { + t.Fatal(err) + } + + organizer := NewOrganizerService(&config.Config{}, zap.NewNop(), repos) + organizer.SetScraper(scraper) + dst, err := organizer.OrganizeMediaWithOptions(t.Context(), media.ID, OrganizeOptions{ + TransferMode: TransferCopy, + }) + if err != nil { + t.Fatalf("organize media: %v", err) + } + want := filepath.Join(dest, "间谍过家家", "Season 01", "间谍过家家 - S01E01.mkv") + if dst != want { + t.Fatalf("dst = %q, want %q", dst, want) + } + + var got model.Media + if err := repos.DB.First(&got, "id = ?", media.ID).Error; err != nil { + t.Fatal(err) + } + if got.Title != "间谍过家家" || got.TMDbID != 12345 || got.OriginalName != "SPY×FAMILY" { + t.Fatalf("media = title=%q original=%q tmdb=%d, want refreshed localized metadata", got.Title, got.OriginalName, got.TMDbID) + } +} + +func TestOrganizeMediaPersistsMetadataWhenAlreadyInPlace(t *testing.T) { + scraper, repos, closeServer := newTestScraper(t) + defer closeServer() + + root := t.TempDir() + libRoot := filepath.Join(root, "media", "电视剧") + mediaPath := filepath.Join(libRoot, "间谍过家家", "Season 01", "间谍过家家 - S01E01.mkv") + writeOrgFile(t, mediaPath, "episode") + + lib := model.Library{Name: "剧集", Path: libRoot, Type: "tv", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: lib.ID, + Title: "Spy.x.Family.S01E01.2022.1080p.WEB-DL", + Path: mediaPath, + Container: "mkv", + SeasonNum: 1, + EpisodeNum: 1, + ScrapeStatus: "matched", + } + if err := repos.Media.Upsert(t.Context(), &media); err != nil { + t.Fatal(err) + } + + organizer := NewOrganizerService(&config.Config{}, zap.NewNop(), repos) + organizer.SetScraper(scraper) + dst, err := organizer.OrganizeMediaWithOptions(t.Context(), media.ID, OrganizeOptions{ + TransferMode: TransferCopy, + }) + if err != nil { + t.Fatalf("organize media already in place: %v", err) + } + if dst != mediaPath { + t.Fatalf("dst = %q, want existing path %q", dst, mediaPath) + } + + var got model.Media + if err := repos.DB.First(&got, "id = ?", media.ID).Error; err != nil { + t.Fatal(err) + } + if got.Title != "间谍过家家" || got.TMDbID != 12345 || got.OriginalName != "SPY×FAMILY" || got.ScrapeStatus != "matched" { + t.Fatalf("metadata not persisted for already-in-place media: title=%q original=%q tmdb=%d status=%q", got.Title, got.OriginalName, got.TMDbID, got.ScrapeStatus) + } +} + +func TestOrganizeLibraryPersistsMetadataForInPlaceWeakRows(t *testing.T) { + scraper, repos, closeServer := newTestScraper(t) + defer closeServer() + + root := t.TempDir() + libRoot := filepath.Join(root, "media", "电视剧", "欧美剧") + mediaPath := filepath.Join(libRoot, "间谍过家家", "Season 01", "间谍过家家 - S01E01.mkv") + writeOrgFile(t, mediaPath, "episode") + + lib := model.Library{Name: "欧美剧", Path: libRoot, Type: "tv", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: lib.ID, + Title: "Spy.x.Family.S01E01.2022.1080p.WEB-DL", + Path: mediaPath, + Container: "mkv", + SeasonNum: 1, + EpisodeNum: 1, + ScrapeStatus: "matched", + } + if err := repos.Media.Upsert(t.Context(), &media); err != nil { + t.Fatal(err) + } + + organizer := NewOrganizerService(&config.Config{}, zap.NewNop(), repos) + organizer.SetScraper(scraper) + res, err := organizer.OrganizeLibraryWithOptions(t.Context(), lib.ID, OrganizeOptions{}) + if err != nil { + t.Fatalf("organize library: %v", err) + } + if res.Organized != 0 || res.Reclassified != 0 { + t.Fatalf("result = %+v, want metadata-only refresh without move", res) + } + + var got model.Media + if err := repos.DB.First(&got, "id = ?", media.ID).Error; err != nil { + t.Fatal(err) + } + if got.Title != "间谍过家家" || got.TMDbID != 12345 || got.OriginalName != "SPY×FAMILY" || got.ScrapeStatus != "matched" { + t.Fatalf("metadata not persisted for in-place library row: title=%q original=%q tmdb=%d status=%q", got.Title, got.OriginalName, got.TMDbID, got.ScrapeStatus) + } } func TestOrganizeScanAndScrapeRetriesNoMatchRows(t *testing.T) { @@ -147,6 +353,51 @@ func TestOrganizeScanAndScrapeRetriesNoMatchRows(t *testing.T) { } } +func TestOrganizeScanAndScrapeRepairsWeakMatchedReleaseTitle(t *testing.T) { + scraper, repos, closeServer := newTestScraper(t) + defer closeServer() + + root := t.TempDir() + libRoot := filepath.Join(root, "media", "电视剧") + mediaPath := filepath.Join(libRoot, "Spy.x.Family", "Season 01", "Spy.x.Family.S01E01.2022.1080p.WEB-DL.mkv") + writeOrgFile(t, mediaPath, "episode") + + lib := model.Library{ + Name: "剧集", + Path: libRoot, + Type: "tv", + Enabled: true, + } + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: lib.ID, + Title: "Spy.x.Family.S01E01.2022.1080p.WEB-DL", + Path: mediaPath, + SeasonNum: 1, + EpisodeNum: 1, + ScrapeStatus: "matched", + } + if err := repos.Media.Upsert(t.Context(), &media); err != nil { + t.Fatal(err) + } + + scanner := NewScannerService(&config.Config{}, zap.NewNop(), repos, NewHub(zap.NewNop()), nil, scraper) + _, scrapes := scanner.ScanAndScrapeLibrariesForPath(t.Context(), libRoot, "", true) + if len(scrapes) != 1 || scrapes[0].Matched != 1 || scrapes[0].Processed != 1 || scrapes[0].Error != "" || scrapes[0].Skipped { + t.Fatalf("scrapes = %#v, want one repaired matched release row", scrapes) + } + + var got model.Media + if err := repos.DB.First(&got, "path = ?", mediaPath).Error; err != nil { + t.Fatal(err) + } + if got.Title != "间谍过家家" || got.TMDbID != 12345 || got.ScrapeStatus != "matched" { + t.Fatalf("media = title=%q tmdb=%d status=%q, want localized matched metadata", got.Title, got.TMDbID, got.ScrapeStatus) + } +} + func TestOrganizeResultNeedsVisibilitySyncIgnoresScannedDuplicates(t *testing.T) { if OrganizeResultNeedsVisibilitySync(&OrganizeResult{ Skipped: 1, diff --git a/internal/service/organizer_settings.go b/internal/service/organizer_settings.go index 2fade57..244386e 100644 --- a/internal/service/organizer_settings.go +++ b/internal/service/organizer_settings.go @@ -2,6 +2,7 @@ package service import ( "context" + "path/filepath" "strings" "github.com/ShukeBta/MediaStationGo/internal/model" @@ -84,3 +85,38 @@ func (o *OrganizerService) autoAddLibraryEnabled(ctx context.Context) bool { } return parseBoolSetting(v, true) } + +func (o *OrganizerService) effectiveOrganizeOverrides(opts OrganizeOptions, explicitDest string) (string, string) { + mediaType := normalizeOrganizeMediaType(opts.MediaType) + category := sanitizeFilename(strings.TrimSpace(opts.MediaCategory)) + hasExplicitLayout := mediaType != "" || category != "" + if category != "" { + if impliedType, normalizedCategory := o.mediaTypeForDirectoryCategory(category); impliedType != "" { + category = normalizedCategory + if mediaType == "" { + mediaType = impliedType + } + } + return mediaType, category + } + if hasExplicitLayout { + if layout := o.organizeLayoutFromDestPath(explicitDest); layout.Category != "" { + category = sanitizeFilename(layout.Category) + if mediaType == "" { + mediaType = layout.MediaType + } + } + } + return mediaType, category +} + +func (o *OrganizerService) organizeLayoutFromDestPath(dest string) organizeDirectoryLayout { + dest = filepath.Clean(strings.TrimSpace(dest)) + if dest == "" || dest == "." { + return organizeDirectoryLayout{} + } + if mediaType, category := o.mediaTypeForDirectoryCategory(filepath.Base(dest)); mediaType != "" && category != "" { + return organizeDirectoryLayout{MediaType: mediaType, Category: category} + } + return organizeDirectoryLayout{} +} diff --git a/internal/service/scanner.go b/internal/service/scanner.go index fb877e1..e529d12 100644 --- a/internal/service/scanner.go +++ b/internal/service/scanner.go @@ -32,7 +32,12 @@ var videoExtensions = map[string]struct{}{ ".avi": {}, ".mov": {}, ".webm": {}, + ".flv": {}, + ".wmv": {}, ".ts": {}, + ".m2ts": {}, + ".mts": {}, + ".vob": {}, ".rmvb": {}, ".rm": {}, ".3gp": {}, diff --git a/internal/service/scanner_strm.go b/internal/service/scanner_strm.go index 87dfd7f..186b980 100644 --- a/internal/service/scanner_strm.go +++ b/internal/service/scanner_strm.go @@ -76,6 +76,7 @@ func (s *ScannerService) maybeGenerateSTRMAfterScan(libraryID string) { Enabled: true, IncludeLocal: true, Overwrite: true, + PreserveTree: s.autoSTRMPreserveTree(ctx), SkipSettingsSave: true, } if outDir, scope := s.autoSTRMOutputDir(ctx); outDir != "" { @@ -103,3 +104,11 @@ func (s *ScannerService) autoSTRMOutputDir(ctx context.Context) (string, string) scope, _ := s.repo.Setting.Get(ctx, "strm.output_scope") return resolveMappedDestinationPath(strings.TrimSpace(outDir)), strings.ToLower(strings.TrimSpace(scope)) } + +func (s *ScannerService) autoSTRMPreserveTree(ctx context.Context) bool { + if s == nil || s.repo == nil || s.repo.Setting == nil { + return false + } + value, err := s.repo.Setting.Get(ctx, "strm.preserve_tree") + return err == nil && parseBoolSetting(value, false) +} diff --git a/internal/service/scraper_library.go b/internal/service/scraper_library.go index 662c28a..37d6e92 100644 --- a/internal/service/scraper_library.go +++ b/internal/service/scraper_library.go @@ -154,7 +154,7 @@ func (s *ScraperService) scrapeCandidateRows(ctx context.Context, libraryID stri statusFilter += " OR scrape_status = ?" statusArgs = append(statusArgs, "no_match") } - if options.IncludeMatched { + if options.IncludeMatched || options.RefreshWeakMatched { statusFilter += " OR scrape_status = ?" statusArgs = append(statusArgs, "matched") } @@ -168,9 +168,29 @@ func (s *ScraperService) scrapeCandidateRows(ctx context.Context, libraryID stri Find(&rows).Error; err != nil { return nil, err } + if options.RefreshWeakMatched && !options.IncludeMatched { + rows = filterWeakMatchedScrapeRows(rows) + } return rows, nil } +func filterWeakMatchedScrapeRows(rows []model.Media) []model.Media { + out := rows[:0] + for _, row := range rows { + if shouldScrapeCandidateRow(row) { + out = append(out, row) + } + } + return out +} + +func shouldScrapeCandidateRow(media model.Media) bool { + if strings.TrimSpace(media.ScrapeStatus) != "matched" { + return true + } + return organizeMediaTitleLooksLikeRelease(media.Title) +} + func (s *ScraperService) notifyScrapeFailed(m model.Media, err error) { if s == nil || s.notify == nil || err == nil { return diff --git a/internal/service/scraper_lookup.go b/internal/service/scraper_lookup.go index 9f39892..d8ada68 100644 --- a/internal/service/scraper_lookup.go +++ b/internal/service/scraper_lookup.go @@ -23,6 +23,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model } if match := s.manualTMDbMatchByID(ctx, m.TMDbID, normalizeMediaType(mediaType, m.Title, "")); match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "tmdb") { + preferExistingLocalizedEpisodeTitle(m, lib, match) return match } } @@ -30,6 +31,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model if strings.TrimSpace(m.DoubanID) != "" && s.douban != nil && s.douban.Enabled() { if match, err := s.douban.GetMatchByID(ctx, strings.TrimSpace(m.DoubanID)); err == nil && match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "douban") { + preferExistingLocalizedEpisodeTitle(m, lib, match) return match } } else if err != nil { @@ -39,6 +41,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model if m.BangumiID > 0 && s.bangumi != nil && s.bangumi.Enabled() { if match, err := s.bangumi.GetSubject(ctx, m.BangumiID); err == nil && match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "bangumi") { + preferExistingLocalizedEpisodeTitle(m, lib, match) return match } } else if err != nil { @@ -48,6 +51,7 @@ func (s *ScraperService) matchFromMediaExternalIDs(ctx context.Context, m *model if strings.TrimSpace(m.TheTVDBID) != "" && s.thetvdb != nil && s.thetvdb.Enabled() { if match, err := s.thetvdb.GetSeriesMatchByID(ctx, strings.TrimSpace(m.TheTVDBID)); err == nil && match != nil { if s.mediaExternalIDMatchTrusted(m, lib, match, "thetvdb") { + preferExistingLocalizedEpisodeTitle(m, lib, match) return match } } else if err != nil { @@ -61,9 +65,15 @@ func (s *ScraperService) mediaExternalIDMatchTrusted(m *model.Media, lib *model. if match == nil || strings.TrimSpace(match.Title) == "" { return false } + if mediaPathHintMatchesExternalID(m, lib, match, source) { + return true + } if !mediaIsEpisodic(m, lib) { return true } + if mediaExternalIDSourceMatches(m, match, source) && mediaExternalIDLanguageFallbackTrusted(m, match) { + return true + } for _, candidate := range scrapeQueryCandidates(m, lib) { if unsafeAutomaticEpisodeQuery(candidate) { continue @@ -86,6 +96,69 @@ func (s *ScraperService) mediaExternalIDMatchTrusted(m *model.Media, lib *model. return false } +func mediaExternalIDSourceMatches(m *model.Media, match *Match, source string) bool { + if m == nil || match == nil { + return false + } + switch strings.ToLower(strings.TrimSpace(source)) { + case "tmdb": + return m.TMDbID > 0 && match.TMDbID == m.TMDbID + case "bangumi": + return m.BangumiID > 0 && match.BangumiID == m.BangumiID + case "douban": + return strings.TrimSpace(m.DoubanID) != "" && + strings.TrimSpace(match.DoubanID) == strings.TrimSpace(m.DoubanID) + case "thetvdb": + return strings.TrimSpace(m.TheTVDBID) != "" && + strings.TrimSpace(match.TheTVDBID) == strings.TrimSpace(m.TheTVDBID) + default: + return false + } +} + +func mediaPathHintMatchesExternalID(m *model.Media, lib *model.Library, match *Match, source string) bool { + if m == nil || match == nil { + return false + } + _, hints := pathHintMetadata(m.Path, mediaIsEpisodic(m, lib)) + switch strings.ToLower(strings.TrimSpace(source)) { + case "tmdb": + return hints.TMDbID > 0 && hints.TMDbID == match.TMDbID + case "bangumi": + return hints.BangumiID > 0 && hints.BangumiID == match.BangumiID + case "douban": + return strings.TrimSpace(hints.DoubanID) != "" && + strings.TrimSpace(hints.DoubanID) == strings.TrimSpace(match.DoubanID) + case "thetvdb": + return strings.TrimSpace(hints.TheTVDBID) != "" && + strings.TrimSpace(hints.TheTVDBID) == strings.TrimSpace(match.TheTVDBID) + default: + return false + } +} + +func mediaExternalIDLanguageFallbackTrusted(m *model.Media, match *Match) bool { + if m == nil || match == nil || containsCJK(match.Title) { + return false + } + title := strings.TrimSpace(m.Title) + return title != "" && + containsCJK(title) && + !unsafeAutomaticEpisodeQuery(title) && + !organizeMediaTitleLooksLikeRelease(title) +} + +func preferExistingLocalizedEpisodeTitle(m *model.Media, lib *model.Library, match *Match) { + if !mediaIsEpisodic(m, lib) || !mediaExternalIDLanguageFallbackTrusted(m, match) { + return + } + title := strings.TrimSpace(m.Title) + if strings.TrimSpace(match.OriginalName) == "" { + match.OriginalName = strings.TrimSpace(match.Title) + } + match.Title = title +} + func (s *ScraperService) applyFanartArtwork(ctx context.Context, match *Match) { if s == nil || s.fanart == nil || !s.fanart.Enabled() || match == nil { return diff --git a/internal/service/scraper_lookup_test.go b/internal/service/scraper_lookup_test.go new file mode 100644 index 0000000..cac7e2f --- /dev/null +++ b/internal/service/scraper_lookup_test.go @@ -0,0 +1,54 @@ +package service + +import ( + "testing" + + "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func TestMediaExternalIDMatchTrustsExactEpisodeID(t *testing.T) { + scraper := &ScraperService{log: zap.NewNop()} + media := &model.Media{ + Title: "电锯人", + Path: "/media/anime/电锯人/Season 01/电锯人 - S01E01.mkv", + TMDbID: 114410, + SeasonNum: 1, + EpisodeNum: 1, + } + lib := &model.Library{Type: "anime", Path: "/media/anime"} + match := &Match{ + Title: "チェンソーマン", + OriginalName: "チェンソーマン", + TMDbID: 114410, + } + + if !scraper.mediaExternalIDMatchTrusted(media, lib, match, "tmdb") { + t.Fatal("exact external id match should be trusted for episodic media") + } +} + +func TestPreferExistingLocalizedMediaTitleKeepsCleanChineseTitle(t *testing.T) { + media := &model.Media{Title: "电锯人", SeasonNum: 1, EpisodeNum: 1} + lib := &model.Library{Type: "anime"} + match := &Match{Title: "チェンソーマン"} + + preferExistingLocalizedEpisodeTitle(media, lib, match) + + if match.Title != "电锯人" || match.OriginalName != "チェンソーマン" { + t.Fatalf("match title=%q original=%q, want localized title with original preserved", match.Title, match.OriginalName) + } +} + +func TestPreferExistingLocalizedMediaTitleIgnoresReleaseTitle(t *testing.T) { + media := &model.Media{Title: "电锯人.S01E01.1080p.WEB-DL", SeasonNum: 1, EpisodeNum: 1} + lib := &model.Library{Type: "anime"} + match := &Match{Title: "チェンソーマン"} + + preferExistingLocalizedEpisodeTitle(media, lib, match) + + if match.Title != "チェンソーマン" { + t.Fatalf("release-like media title should not replace provider title, got %q", match.Title) + } +} diff --git a/internal/service/scraper_options.go b/internal/service/scraper_options.go index 536cf56..4615417 100644 --- a/internal/service/scraper_options.go +++ b/internal/service/scraper_options.go @@ -3,6 +3,7 @@ package service type ScrapeOptions struct { RetryNoMatch bool IncludeMatched bool + RefreshWeakMatched bool EpisodeArtwork *bool DeferEpisodeDetails bool } diff --git a/internal/service/scraper_query_trust_test.go b/internal/service/scraper_query_trust_test.go index a2670a5..af53a76 100644 --- a/internal/service/scraper_query_trust_test.go +++ b/internal/service/scraper_query_trust_test.go @@ -26,7 +26,7 @@ func TestOrganizeMetadataTrustsCleanedBroadcastReleaseQuery(t *testing.T) { } } -func TestOrganizeMetadataTrustsChineseReleaseAliasWithoutOriginalName(t *testing.T) { +func TestOrganizeMetadataRejectsChineseReleaseAliasWithoutOriginalName(t *testing.T) { match := &Match{ Title: "莫离", Languages: []string{"zh"}, @@ -34,8 +34,22 @@ func TestOrganizeMetadataTrustsChineseReleaseAliasWithoutOriginalName(t *testing Year: 2026, TMDbID: 292696, } - if !organizeMetadataMatchTrusted("the first jasmine", 2026, match) { - t.Fatal("multi-word English release alias should be trusted for Chinese-origin metadata") + if organizeMetadataMatchTrusted("the first jasmine", 2026, match) { + t.Fatal("multi-word English release alias must not be trusted without title or original-name evidence") + } +} + +func TestOrganizeMetadataRejectsLooseChineseOriginEnglishAlias(t *testing.T) { + match := &Match{ + Title: "镖人", + OriginalName: "Biao Ren", + Languages: []string{"zh"}, + Countries: []string{"CN"}, + Year: 2023, + TMDbID: 107463, + } + if organizeMetadataMatchTrusted("blades of the guardians", 2023, match) { + t.Fatal("unrelated English query must not trust Chinese-origin metadata just because it has multiple Latin tokens") } } diff --git a/internal/service/site_adapter.go b/internal/service/site_adapter.go index 4d3ec62..a435fcc 100644 --- a/internal/service/site_adapter.go +++ b/internal/service/site_adapter.go @@ -45,6 +45,7 @@ type TorrentItem struct { ID string `json:"id"` Title string `json:"title"` Subtitle string `json:"subtitle"` + Labels string `json:"labels,omitempty"` Category string `json:"category"` Size int64 `json:"size"` Seeders int `json:"seeders"` diff --git a/internal/service/site_adapter_nexusphp_list.go b/internal/service/site_adapter_nexusphp_list.go index 483c018..1d2d553 100644 --- a/internal/service/site_adapter_nexusphp_list.go +++ b/internal/service/site_adapter_nexusphp_list.go @@ -9,6 +9,11 @@ import ( "time" ) +var ( + nexusPHPFreeLabelRE = regexp.MustCompile(`(?i)(class="[^"]*(?:free|free2|twoupfree|free_download)[^"]*"|促销|免费)`) + nexusPHPRiskLabelRE = regexp.MustCompile(`(?i)(?:class|title|alt)=["'][^"']*\bhr\b[^"']*["']`) +) + // parseNexusPHPHTML 解析 NexusPHP 种子列表 HTML。 func parseNexusPHPHTML(html, siteName, baseURL string) (*SiteSearchResult, error) { result := &SiteSearchResult{ @@ -56,6 +61,7 @@ func parseNexusPHPRow(row, baseURL string) TorrentItem { item.ID = link.query.Get("id") item.Title = nexusPHPTitleFromLink(*link) item.Subtitle = nexusPHPSubtitle(row) + item.Labels = nexusPHPRowLabels(row) item.DetailURL = resolveSiteURL(baseURL, link.href) } if link := firstNexusPHPLink(row, "download.php"); link != nil { @@ -166,6 +172,28 @@ func nexusPHPSubtitle(row string) string { return "" } +func nexusPHPRowLabels(row string) string { + labels := make([]string, 0, 4) + lower := strings.ToLower(row) + add := func(label string) { + for _, existing := range labels { + if existing == label { + return + } + } + labels = append(labels, label) + } + if nexusPHPFreeLabelRE.MatchString(row) { + add("free") + } + if strings.Contains(lower, "hit and run") || strings.Contains(lower, "hit&run") || strings.Contains(lower, "h&r") || + nexusPHPRiskLabelRE.MatchString(row) || + strings.Contains(row, "禁转") || strings.Contains(row, "禁止转载") || strings.Contains(row, "禁下") || strings.Contains(row, "禁止下载") { + add("HR") + } + return strings.Join(labels, " ") +} + func nexusPHPIntByClass(row, className string) (int, bool) { pattern := regexp.MustCompile(`(?is)]*(?:class|id)\s*=\s*["'][^"']*` + regexp.QuoteMeta(className) + `[^"']*["'][^>]*>(.*?)`) if match := pattern.FindStringSubmatch(row); len(match) >= 2 { diff --git a/internal/service/site_adapter_test.go b/internal/service/site_adapter_test.go index 7983ad4..ab21326 100644 --- a/internal/service/site_adapter_test.go +++ b/internal/service/site_adapter_test.go @@ -266,6 +266,31 @@ func TestParseNexusPHPHTMLModernRows(t *testing.T) { } } +func TestParseNexusPHPHTMLCapturesRiskAndPromotionLabels(t *testing.T) { + page := ` + + + + + + +
Some Show免费H&R下载
` + result, err := parseNexusPHPHTML(page, "Nexus", "https://pt.example") + if err != nil { + t.Fatal(err) + } + if len(result.Items) != 1 { + t.Fatalf("items = %#v", result.Items) + } + item := result.Items[0] + if !item.Free { + t.Fatalf("item.Free = false, want free promotion detected: %#v", item) + } + if !strings.Contains(item.Labels, "HR") || !strings.Contains(item.Labels, "free") { + t.Fatalf("labels = %q, want HR and free", item.Labels) + } +} + func TestParseNexusPHPHTMLIgnoresUserDetailsLinks(t *testing.T) { page := ` diff --git a/internal/service/site_search.go b/internal/service/site_search.go index 70a0de6..f6b7008 100644 --- a/internal/service/site_search.go +++ b/internal/service/site_search.go @@ -20,6 +20,7 @@ type SearchResult struct { SiteID string `json:"site_id"` Title string `json:"title"` Subtitle string `json:"subtitle,omitempty"` + Labels string `json:"labels,omitempty"` TorrentURL string `json:"torrent_url"` DownloadURL string `json:"download_url"` Category string `json:"category,omitempty"` @@ -202,6 +203,7 @@ func siteSearchResultsFromItems(site model.Site, result *SiteSearchResult, keywo SiteID: site.ID, Title: item.Title, Subtitle: item.Subtitle, + Labels: item.Labels, TorrentURL: item.DetailURL, DownloadURL: item.DownloadURL, Category: item.Category, diff --git a/internal/service/strm_generate.go b/internal/service/strm_generate.go index d63f2c9..0709086 100644 --- a/internal/service/strm_generate.go +++ b/internal/service/strm_generate.go @@ -19,6 +19,7 @@ type GenerateSTRMOptions struct { Enabled bool `json:"enabled"` Overwrite bool `json:"overwrite"` IncludeLocal bool `json:"include_local"` + PreserveTree bool `json:"preserve_tree"` PlaybackToken string `json:"-"` SkipSettingsSave bool `json:"-"` } @@ -127,6 +128,7 @@ func (s *STRMService) GenerateForAllLibraries(ctx context.Context, opts Generate if baseOutputDir != "" && baseOutputDir != "." && s.repo.Setting != nil { _ = s.repo.Setting.Set(ctx, "strm.output_dir", baseOutputDir) _ = s.repo.Setting.Set(ctx, "strm.output_scope", "all") + _ = s.repo.Setting.Set(ctx, "strm.preserve_tree", strconv.FormatBool(opts.PreserveTree)) result.OutputDir = baseOutputDir } return result, nil @@ -160,6 +162,7 @@ func (s *STRMService) saveSTRMGenerationSettings(ctx context.Context, outputDir _ = s.repo.Setting.Set(ctx, "strm.auto_generate_enabled", strconv.FormatBool(opts.Enabled)) _ = s.repo.Setting.Set(ctx, "strm.output_dir", outputDir) _ = s.repo.Setting.Set(ctx, "strm.output_scope", "library") + _ = s.repo.Setting.Set(ctx, "strm.preserve_tree", strconv.FormatBool(opts.PreserveTree)) } func (s *STRMService) librarySTRMMedia(ctx context.Context, libraryID string) ([]model.Media, error) { @@ -193,6 +196,11 @@ func (s *STRMService) generateOne(ctx context.Context, lib model.Library, media return item } rel := s.strmRelativePath(lib, media) + if opts.PreserveTree { + if treeRel := s.strmTreeRelativePath(media); treeRel != "" { + rel = treeRel + } + } if rel == "" { item.Action = "skipped" item.Reason = "cannot build file name" diff --git a/internal/service/strm_output_dir.go b/internal/service/strm_output_dir.go index 60e25c4..81ec8ac 100644 --- a/internal/service/strm_output_dir.go +++ b/internal/service/strm_output_dir.go @@ -177,8 +177,6 @@ func strmCanonicalCategory(part string) string { return "综艺" case "少儿": return "儿童" - case "未分类": - return "欧美剧" case "国产动漫": return "国漫" case "番剧", "日漫", "日本动漫", "日本动画": diff --git a/internal/service/strm_svc_test.go b/internal/service/strm_svc_test.go index 4019944..641e37d 100644 --- a/internal/service/strm_svc_test.go +++ b/internal/service/strm_svc_test.go @@ -217,6 +217,11 @@ func TestSTRMLibraryOutputSubdirUsesLibraryCategoryPath(t *testing.T) { lib: model.Library{Name: "欧美剧", Path: `F:\media\电视剧\欧美剧`, Type: "tv"}, want: filepath.Join("电视剧", "欧美剧"), }, + { + name: "uncategorized tv category stays uncategorized", + lib: model.Library{Name: "OpenList · 未分类", Path: BuildCloudLibraryPath("openlist", "/电视剧/未分类", "/电视剧/未分类"), Type: "tv"}, + want: filepath.Join("电视剧", "未分类"), + }, { name: "fallback to type root", lib: model.Library{Name: "Archive", Path: `F:\archive`, Type: "movie"}, @@ -314,6 +319,85 @@ func TestGenerateSTRMForLibraryUsesPathEpisodeFallback(t *testing.T) { assertFileContains(t, filepath.Join(libraryOutDir, "南部档案", "Season 01", "南部档案 - S01E02.strm"), "http://nas.example:18080/api/stream/ep-2?token=strm-token") } +func TestGenerateSTRMForLibraryPreservesSourceTree(t *testing.T) { + db := newServiceTestDB(t, &model.Library{}, &model.Media{}, &model.STRMRecord{}, &model.Setting{}) + repos := repository.New(db) + lib := model.Library{Name: "OpenList · 国产剧", Path: BuildCloudLibraryPath("openlist", "/电视剧/国产剧", "/电视剧/国产剧"), Type: "tv", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + media := model.Media{ + Base: model.Base{ID: "ep-1"}, + LibraryID: lib.ID, + Title: "南部档案", + Path: "cloud://openlist/电视剧/国产剧/南部档案/Season 01/Archives.The.Nanyang.Mystery.S01E01.mkv", + STRMURL: "/api/cloud/play/openlist?ref=ep1", + } + if err := repos.DB.Create(&media).Error; err != nil { + t.Fatal(err) + } + + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), repos, &config.Config{}) + res, err := svc.GenerateForLibrary(t.Context(), GenerateSTRMOptions{ + LibraryID: lib.ID, + OutputDir: outDir, + BaseURL: "http://nas.example:18080", + IncludeLocal: true, + PreserveTree: true, + PlaybackToken: "strm-token", + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || res.Skipped != 0 { + t.Fatalf("result = %#v, want generated=1 skipped=0", res) + } + wantPath := filepath.Join(outDir, "电视剧", "国产剧", "南部档案", "Season 01", "Archives.The.Nanyang.Mystery.S01E01.strm") + assertFileContains(t, wantPath, "http://nas.example:18080/api/stream/ep-1?token=strm-token") + if got, err := repos.Setting.Get(t.Context(), "strm.preserve_tree"); err != nil || got != "true" { + t.Fatalf("strm.preserve_tree = %q, %v; want true", got, err) + } +} + +func TestGenerateSTRMForLibraryCanSkipLocalMedia(t *testing.T) { + db := newServiceTestDB(t, &model.Library{}, &model.Media{}, &model.STRMRecord{}, &model.Setting{}) + repos := repository.New(db) + lib := model.Library{Name: "电影", Path: "cloud://openlist/电影", Type: "movie", Enabled: true} + if err := repos.Library.Create(t.Context(), &lib); err != nil { + t.Fatal(err) + } + rows := []model.Media{ + {Base: model.Base{ID: "cloud-media"}, LibraryID: lib.ID, Title: "云盘电影", Year: 2026, Path: "cloud://openlist/电影/云盘电影.mkv", STRMURL: "/api/cloud/play/openlist?ref=movie"}, + {Base: model.Base{ID: "local-media"}, LibraryID: lib.ID, Title: "本地电影", Year: 2025, Path: filepath.Join(t.TempDir(), "本地电影.mkv")}, + } + for i := range rows { + if err := repos.DB.Create(&rows[i]).Error; err != nil { + t.Fatal(err) + } + } + + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), repos, &config.Config{}) + res, err := svc.GenerateForLibrary(t.Context(), GenerateSTRMOptions{ + LibraryID: lib.ID, + OutputDir: outDir, + BaseURL: "http://nas.example:18080", + IncludeLocal: false, + PlaybackToken: "strm-token", + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || res.Skipped != 1 { + t.Fatalf("result = %#v, want generated=1 skipped=1", res) + } + assertFileContains(t, filepath.Join(outDir, "电影", "云盘电影 (2026)", "云盘电影 (2026).strm"), "http://nas.example:18080/api/stream/cloud-media?token=strm-token") + if _, err := os.Stat(filepath.Join(outDir, "电影", "本地电影 (2025)", "本地电影 (2025).strm")); !os.IsNotExist(err) { + t.Fatalf("local media strm should not exist, stat err=%v", err) + } +} + func TestGenerateSTRMForAllLibrariesWritesPerLibraryFolders(t *testing.T) { db := newServiceTestDB(t, &model.Library{}, &model.Media{}, &model.STRMRecord{}, &model.Setting{}) repos := repository.New(db) diff --git a/internal/service/strm_tree_cleanup_test.go b/internal/service/strm_tree_cleanup_test.go new file mode 100644 index 0000000..da23821 --- /dev/null +++ b/internal/service/strm_tree_cleanup_test.go @@ -0,0 +1,150 @@ +package service + +import ( + "os" + "path/filepath" + "testing" + + "go.uber.org/zap" +) + +func TestGenerateSTRMFromTreeOverwriteAndTraversal(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + existing := filepath.Join(outDir, "Movies", "Movie.strm") + if err := os.MkdirAll(filepath.Dir(existing), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(existing, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{"Movies/Movie.mkv", "../escape.mkv"}, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 0 || res.Skipped != 1 || len(res.Errors) != 1 { + t.Fatalf("result = %#v, want existing skipped and traversal rejected", res) + } + if got := readSTRM(t, existing); got != "old" { + t.Fatalf("existing strm = %q, want unchanged", got) + } + + res, err = svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{"Movies/Movie.mkv"}, + OutputDir: outDir, + Overwrite: true, + }) + if err != nil { + t.Fatal(err) + } + if res.Updated != 1 { + t.Fatalf("updated = %d, want 1", res.Updated) + } + if got := readSTRM(t, existing); got == "old" { + t.Fatalf("existing strm should be overwritten, got %q", got) + } +} + +func TestGenerateSTRMFromTreeCleanupStaleFiles(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + current := filepath.Join(outDir, "Shows", "Show.S01E01.strm") + stale := filepath.Join(outDir, "Shows", "Show.S01E02.strm") + for _, file := range []string{current, stale} { + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + } + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{"Shows/Show.S01E01.mkv"}, + OutputDir: outDir, + Cleanup: true, + }) + if err != nil { + t.Fatal(err) + } + if res.Skipped != 1 || res.Cleaned != 1 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want current skipped and one stale file cleaned", res) + } + if _, err := os.Stat(current); err != nil { + t.Fatalf("current strm should remain: %v", err) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("stale strm should be removed, stat err=%v", err) + } +} + +func TestGenerateSTRMFromTreeCleanupWithOutputPrefixStaysInPrefix(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + current := filepath.Join(outDir, "电影", "欧美电影", "Movie.strm") + staleInPrefix := filepath.Join(outDir, "电影", "欧美电影", "Old.strm") + otherCategory := filepath.Join(outDir, "电视剧", "国产剧", "Show.strm") + for _, file := range []string{current, staleInPrefix, otherCategory} { + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + } + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{"Movie.mkv"}, + OutputPrefix: "电影/欧美电影", + OutputDir: outDir, + Cleanup: true, + }) + if err != nil { + t.Fatal(err) + } + if res.Skipped != 1 || res.Cleaned != 1 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want one stale file cleaned inside prefix only", res) + } + if _, err := os.Stat(staleInPrefix); !os.IsNotExist(err) { + t.Fatalf("stale prefixed strm should be removed, stat err=%v", err) + } + if _, err := os.Stat(otherCategory); err != nil { + t.Fatalf("other category strm should remain: %v", err) + } +} + +func TestGenerateSTRMFromTreeCleanupSkipsWhenNoValidSources(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + existing := filepath.Join(outDir, "Movies", "Movie.strm") + if err := os.MkdirAll(filepath.Dir(existing), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(existing, []byte("keep\n"), 0o644); err != nil { + t.Fatal(err) + } + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{"Movies/poster.jpg"}, + OutputDir: outDir, + Cleanup: true, + }) + if err != nil { + t.Fatal(err) + } + if res.Cleaned != 0 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want cleanup skipped without valid media sources", res) + } + if got := readSTRM(t, existing); got != "keep" { + t.Fatalf("existing strm = %q, want kept", got) + } +} diff --git a/internal/service/strm_tree_import.go b/internal/service/strm_tree_import.go new file mode 100644 index 0000000..62ce36d --- /dev/null +++ b/internal/service/strm_tree_import.go @@ -0,0 +1,141 @@ +package service + +import ( + "context" + "errors" + "os" + "path" + "path/filepath" + "strings" +) + +type GenerateSTRMTreeOptions struct { + Provider string `json:"provider"` + TreeText string `json:"tree_text,omitempty"` + Paths []string `json:"paths,omitempty"` + SourceRoot string `json:"source_root,omitempty"` + OutputPrefix string `json:"output_prefix,omitempty"` + OutputDir string `json:"output_dir"` + BaseURL string `json:"base_url,omitempty"` + Overwrite bool `json:"overwrite"` + Cleanup bool `json:"cleanup"` +} + +type strmTreeSource struct { + Provider string + Path string + RefPath string +} + +func (s *STRMService) GenerateFromTree(ctx context.Context, opts GenerateSTRMTreeOptions) (*GenerateSTRMResult, error) { + provider := normalizeSTRMTreeProvider(opts.Provider) + if provider == "" { + return nil, errors.New("provider required") + } + outputDir := resolveMappedDestinationPath(strings.TrimSpace(opts.OutputDir)) + if outputDir == "" || outputDir == "." { + return nil, errors.New("output_dir required") + } + if err := os.MkdirAll(outputDir, 0o755); err != nil { // #nosec G301 -- STRM output directories must be readable by media players. + return nil, err + } + result := &GenerateSTRMResult{LibraryID: provider, OutputDir: outputDir} + expectedFiles := make(map[string]struct{}) + for _, source := range collectSTRMTreeSources(opts) { + select { + case <-ctx.Done(): + return result, ctx.Err() + default: + } + item := generateTreeSTRMItem(outputDir, source, opts) + if item.FilePath != "" && item.Action != "error" { + expectedFiles[filepath.Clean(item.FilePath)] = struct{}{} + } + result.addItem(item) + } + if opts.Cleanup && len(expectedFiles) > 0 { + cleanupDir := outputDir + if prefix, err := strmTreeOutputPrefixPath(opts.OutputPrefix); err == nil && prefix != "" { + cleanupDir = filepath.Join(outputDir, prefix) + } + cleaned, err := removeStaleSTRMFiles(cleanupDir, expectedFiles) + result.Cleaned += cleaned + if err != nil { + result.Errors = append(result.Errors, err.Error()) + } + } + return result, nil +} + +func generateTreeSTRMItem(outputDir string, source strmTreeSource, opts GenerateSTRMTreeOptions) GenerateSTRMItem { + relSource := strmTreeRelativeSource(source.Path, opts.SourceRoot) + relPath, err := strmTreeOutputRelativePath(relSource) + item := GenerateSTRMItem{Title: strings.TrimSuffix(path.Base(source.Path), path.Ext(source.Path))} + if err != nil { + item.Action = "error" + item.Reason = err.Error() + return item + } + prefix, err := strmTreeOutputPrefixPath(opts.OutputPrefix) + if err != nil { + item.Action = "error" + item.Reason = err.Error() + return item + } + filePath := filepath.Join(outputDir, prefix, relPath) + item.FilePath = filePath + item.URL = absolutizeSTRMURL(BuildRelativeCloudPlayURL(source.Provider, strmTreeCloudRef(source.cloudRefPath(), opts.SourceRoot)), opts.BaseURL) + if _, err := os.Stat(filePath); err == nil && !opts.Overwrite { + item.Action = "skipped" + item.Reason = "target exists" + return item + } + action := "generated" + if _, err := os.Stat(filePath); err == nil { + action = "updated" + } + if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { // #nosec G301 -- STRM output directories must be readable by media players. + item.Action = "error" + item.Reason = err.Error() + return item + } + if err := os.WriteFile(filePath, []byte(item.URL+"\n"), 0o644); err != nil { // #nosec G306 -- STRM files are media sidecars intended to be readable by players. + item.Action = "error" + item.Reason = err.Error() + return item + } + item.Action = action + return item +} + +func collectSTRMTreeSources(opts GenerateSTRMTreeOptions) []strmTreeSource { + fallbackProvider := normalizeSTRMTreeProvider(opts.Provider) + out := make([]strmTreeSource, 0, len(opts.Paths)) + seen := map[string]struct{}{} + add := func(value string) { + source := normalizeSTRMTreeSourceWithProvider(value, fallbackProvider) + if source.Provider == "" || source.Path == "" || !strmTreeSourceIsVideo(source.Path) { + return + } + key := strings.ToLower(source.Provider) + "\x00" + strings.ToLower(source.Path) + "\x00" + strings.ToLower(source.cloudRefPath()) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + out = append(out, source) + } + for _, value := range opts.Paths { + add(value) + } + for _, value := range parseSTRMTreeText(opts.TreeText) { + add(value) + } + return out +} + +func (s strmTreeSource) cloudRefPath() string { + if strings.TrimSpace(s.RefPath) != "" { + return s.RefPath + } + return s.Path +} diff --git a/internal/service/strm_tree_import_test.go b/internal/service/strm_tree_import_test.go new file mode 100644 index 0000000..5e63436 --- /dev/null +++ b/internal/service/strm_tree_import_test.go @@ -0,0 +1,83 @@ +package service + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "go.uber.org/zap" +) + +func TestGenerateSTRMFromTreePaths(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "115", + Paths: []string{"/电视剧/国产剧/南部档案/Season 01/Archives.S01E01.mkv", "/电视剧/国产剧/南部档案/poster.jpg", "/电视剧/国产剧/南部档案/Existing.strm"}, + TreeText: "电视剧\n└── 国产剧\n └── 南部档案\n └── Existing.Tree.strm", + SourceRoot: "/电视剧", + OutputDir: outDir, + BaseURL: "https://media.example.com", + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || res.Skipped != 0 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want one generated video and ignored sidecar", res) + } + path := filepath.Join(outDir, "国产剧", "南部档案", "Season 01", "Archives.S01E01.strm") + got := readSTRM(t, path) + if !strings.HasPrefix(got, "https://media.example.com/api/cloud/play/cloud115?") { + t.Fatalf("strm url = %q, want cloud115 play url", got) + } + if !strings.Contains(got, "ref=%2F%E7%94%B5%E8%A7%86%E5%89%A7%2F%E5%9B%BD%E4%BA%A7%E5%89%A7%2F%E5%8D%97%E9%83%A8%E6%A1%A3%E6%A1%88%2FSeason+01%2FArchives.S01E01.mkv") { + t.Fatalf("strm url = %q, missing encoded source ref", got) + } + if _, err := os.Stat(filepath.Join(outDir, "国产剧", "南部档案", "Existing.strm")); !os.IsNotExist(err) { + t.Fatalf("existing .strm source should be ignored by tree generator, stat err=%v", err) + } + if _, err := os.Stat(filepath.Join(outDir, "电视剧", "国产剧", "南部档案", "Existing.Tree.strm")); !os.IsNotExist(err) { + t.Fatalf("tree .strm source should be ignored by tree generator, stat err=%v", err) + } +} + +func TestGenerateSTRMFromTreeSupportsCommonVideoExtensions(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{ + "/Movies/BluRay.Stream.2026.m2ts", + "/Movies/Camera.Source.2026.MTS", + "/Movies/DVD.Feature.2026.vob", + "/Movies/Legacy.Video.2026.wmv", + "/Movies/Web.Legacy.2026.flv", + "/Movies/Disc.Image.2026.iso", + }, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 5 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want five common video sources generated and iso ignored", res) + } + for _, name := range []string{ + "BluRay.Stream.2026", + "Camera.Source.2026", + "DVD.Feature.2026", + "Legacy.Video.2026", + "Web.Legacy.2026", + } { + got := readSTRM(t, filepath.Join(outDir, "Movies", name+".strm")) + if !strings.Contains(got, "/api/cloud/play/openlist?") { + t.Fatalf("%s strm url = %q, want cloud play url", name, got) + } + } + if _, err := os.Stat(filepath.Join(outDir, "Movies", "Disc.Image.2026.strm")); !os.IsNotExist(err) { + t.Fatalf("iso source should stay ignored by tree generator, stat err=%v", err) + } +} diff --git a/internal/service/strm_tree_normalize.go b/internal/service/strm_tree_normalize.go new file mode 100644 index 0000000..a72c20b --- /dev/null +++ b/internal/service/strm_tree_normalize.go @@ -0,0 +1,284 @@ +package service + +import ( + "net/url" + "path" + "regexp" + "strings" +) + +var strmTreeListPrefixRE = regexp.MustCompile(`^\s*(?:[-*+•·]\s+|\d{1,4}[\.)、]\s+)`) + +var strmTreeURLPathQueryKeys = []string{"path", "ref", "file", "filepath", "filename", "name"} + +func normalizeSTRMTreeProvider(raw string) string { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "115", "115pan", "pan115", "cloud115": + return "cloud115" + case "alist", "openlist", "webdav", "clouddrive2", "s3": + return strings.ToLower(strings.TrimSpace(raw)) + default: + return strings.ToLower(strings.TrimSpace(raw)) + } +} + +func normalizeSTRMTreeSource(raw string) string { + value := strings.TrimSpace(strings.Trim(raw, `"'`)) + if value == "" { + return "" + } + if strings.HasPrefix(strings.ToLower(value), "cloud://") { + parts := strmLibraryPathParts(value) + if len(parts) == 0 { + return "" + } + return "/" + path.Join(parts...) + } + if parsed, err := url.Parse(value); err == nil && parsed.Scheme != "" && parsed.Scheme != "file" { + if ref := strmTreeSourceFromURL(parsed); ref != "" { + value = ref + } + } + value = trimSTRMTreeMediaMetadata(value) + value = strings.ReplaceAll(value, "\\", "/") + value = pathAfterWindowsDrivePrefix(value) + value = cleanSTRMTreePathDecorations(value) + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if strmTreeHasUnsafeSegment(value) { + return strings.Trim(value, "/") + } + if strings.HasPrefix(value, "/") { + return path.Clean(value) + } + return strings.TrimPrefix(path.Clean("/"+value), "/") +} + +func normalizeSTRMTreeSourceWithProvider(raw, fallbackProvider string) strmTreeSource { + provider := normalizeSTRMTreeProvider(fallbackProvider) + value := strings.TrimSpace(strings.Trim(raw, `"'`)) + if value == "" { + return strmTreeSource{} + } + if info, ok := ParseCloudLibraryMount(value); ok { + source := normalizeSTRMTreeSource(info.DisplayDir) + if source == "" { + return strmTreeSource{} + } + ref := normalizeSTRMTreeSource(info.ScanDir) + if ref == "" { + ref = source + } + return strmTreeSource{Provider: normalizeSTRMTreeProvider(info.Provider), Path: source, RefPath: ref} + } + if parsed, err := url.Parse(value); err == nil { + if cloudProvider, ref := strmTreeCloudPlaySourceFromURL(parsed); cloudProvider != "" && ref != "" { + source := normalizeSTRMTreeSource(ref) + if source == "" { + return strmTreeSource{} + } + return strmTreeSource{Provider: normalizeSTRMTreeProvider(cloudProvider), Path: source} + } + } + source := normalizeSTRMTreeSource(value) + if source == "" { + return strmTreeSource{} + } + return strmTreeSource{Provider: provider, Path: source} +} + +func strmTreeCloudPlaySourceFromURL(parsed *url.URL) (string, string) { + if parsed == nil { + return "", "" + } + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(parts) < 4 || !strings.EqualFold(parts[0], "api") || !strings.EqualFold(parts[1], "cloud") || !strings.EqualFold(parts[2], "play") { + return "", "" + } + provider := strings.TrimSpace(parts[3]) + if decoded, err := url.PathUnescape(provider); err == nil { + provider = decoded + } + // /api/cloud/play is generated by this app with url.Values, where '+' + // represents a space. Decode the standard query view first so re-importing + // existing STRM links does not turn "My Movie.mkv" into "My+Movie.mkv". + for _, value := range parsed.Query()["ref"] { + if ref := strmTreeURLQueryMediaSource(value); ref != "" { + return provider, ref + } + } + for _, value := range strmTreeRawQueryValues(parsed.RawQuery, "ref") { + if ref := strmTreeURLQueryMediaSource(value); ref != "" { + return provider, ref + } + } + return "", "" +} + +func strmTreeSourceFromURL(parsed *url.URL) string { + if parsed == nil { + return "" + } + for _, key := range strmTreeURLPathQueryKeys { + for _, value := range strmTreeRawQueryValues(parsed.RawQuery, key) { + if ref := strmTreeURLQueryMediaSource(value); ref != "" { + return ref + } + } + for _, value := range parsed.Query()[key] { + if ref := strmTreeURLQueryMediaSource(value); ref != "" { + return ref + } + } + } + if parsed.Path != "" && strmTreeURLQueryMediaSource(parsed.Path) != "" { + return parsed.Path + } + return "" +} + +func strmTreeRawQueryValues(rawQuery, key string) []string { + key = strings.ToLower(strings.TrimSpace(key)) + if rawQuery == "" || key == "" { + return nil + } + values := make([]string, 0, 1) + for _, pair := range strings.Split(rawQuery, "&") { + if pair == "" { + continue + } + rawKey, rawValue, _ := strings.Cut(pair, "=") + decodedKey, err := url.QueryUnescape(rawKey) + if err != nil { + decodedKey = rawKey + } + if strings.ToLower(strings.TrimSpace(decodedKey)) != key { + continue + } + decodedValue, err := url.PathUnescape(rawValue) + if err != nil { + decodedValue = rawValue + } + values = append(values, decodedValue) + } + return values +} + +func strmTreeURLQueryMediaSource(raw string) string { + value := strings.TrimSpace(strings.Trim(raw, `"'`)) + if value == "" { + return "" + } + if parsed, err := url.Parse(value); err == nil && parsed.Scheme != "" && parsed.Scheme != "file" { + value = parsed.Path + } + value = trimSTRMTreeMediaMetadata(value) + if strmTreeSourceIsVideo(value) { + return value + } + return "" +} + +func strmTreeHasUnsafeSegment(value string) bool { + for _, part := range strings.Split(strings.ReplaceAll(value, "\\", "/"), "/") { + if strings.TrimSpace(part) == ".." { + return true + } + } + return false +} + +func strmTreeSourceIsVideo(source string) bool { + ext := strings.ToLower(path.Ext(trimSTRMTreeMediaMetadata(cleanSTRMTreeItemName(source)))) + if ext == ".strm" { + return false + } + _, ok := videoExtensions[ext] + return ok +} + +func cleanSTRMTreePathDecorations(value string) string { + if value == "" { + return "" + } + leadingSlash := strings.HasPrefix(value, "/") + parts := strings.Split(value, "/") + for i, part := range parts { + if part == "" { + continue + } + parts[i] = cleanSTRMTreeItemName(part) + } + cleaned := strings.Join(parts, "/") + if leadingSlash && !strings.HasPrefix(cleaned, "/") { + cleaned = "/" + cleaned + } + return cleaned +} + +func cleanSTRMTreeItemName(value string) string { + value = strings.TrimSpace(strings.Trim(value, `"'`)) + for { + before := value + value = strings.TrimSpace(strmTreeListPrefixRE.ReplaceAllString(value, "")) + value = strings.TrimSpace(strings.TrimLeft(value, "\ufeff📁📂📄📃🎬🎞🎥📺🗂️")) + lower := strings.ToLower(value) + for _, marker := range []string{ + "[file]", "[f]", "[文件]", "[檔案]", "[档案]", + "[dir]", "[d]", "[folder]", "[目录]", "[目錄]", "[文件夹]", "[資料夾]", + "(file)", "(f)", "(文件)", "(檔案)", "(档案)", + "(dir)", "(d)", "(folder)", "(目录)", "(目錄)", "(文件夹)", "(資料夾)", + } { + if strings.HasPrefix(lower, marker) { + value = strings.TrimLeft(strings.TrimSpace(value[len(marker):]), "-_:: ") + break + } + } + if value == before { + return value + } + } +} + +func trimSTRMTreeMediaMetadata(value string) string { + value = strings.TrimSpace(value) + lower := strings.ToLower(value) + bestEnd := -1 + for ext := range videoExtensions { + from := 0 + for { + idx := strings.Index(lower[from:], ext) + if idx < 0 { + break + } + idx += from + end := idx + len(ext) + if strmTreeMediaExtBoundary(value, end) && end > bestEnd { + bestEnd = end + } + from = idx + 1 + if from >= len(lower) { + break + } + } + } + if bestEnd > 0 && bestEnd < len(value) { + return strings.TrimSpace(value[:bestEnd]) + } + return value +} + +func strmTreeMediaExtBoundary(value string, end int) bool { + if end >= len(value) { + return true + } + for _, r := range value[end:] { + if r == '(' || r == '[' || r == '{' || r == '<' || r == '(' || r == '【' || r == ',' || r == ',' { + return true + } + return r == ' ' || r == '\t' || r == '\r' || r == '\n' + } + return true +} diff --git a/internal/service/strm_tree_parse.go b/internal/service/strm_tree_parse.go new file mode 100644 index 0000000..d80e2ba --- /dev/null +++ b/internal/service/strm_tree_parse.go @@ -0,0 +1,165 @@ +package service + +import ( + "path" + "strings" +) + +func parseSTRMTreeText(raw string) []string { + var out []string + stack := make([]string, 0, 8) + plainIndents := make([]int, 0, 8) + rootOffset := 0 + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimRight(line, " \t\r") + if strings.TrimSpace(line) == "" { + continue + } + if item, level, ok := parseSTRMTreeLine(line); ok { + plainIndents = plainIndents[:0] + level += rootOffset + stack = stack[:min(level, len(stack))] + if strmTreeItemIsVideoSource(item) { + out = append(out, strmTreeJoinedSource(stack, item)) + continue + } + stack = append(stack, item) + continue + } + if item, level, ok := parseSTRMTreeContinuationLine(line, len(plainIndents) > 0); ok { + plainIndents = plainIndents[:0] + level += rootOffset + stack = stack[:min(level, len(stack))] + out = append(out, path.Join(append(append([]string{}, stack...), item)...)) + continue + } + item := cleanSTRMTreeItemName(line) + if indent := strmTreePlainIndent(line); indent > 0 && !strings.ContainsAny(item, `/\`) { + for len(plainIndents) > 0 && indent <= plainIndents[len(plainIndents)-1] { + plainIndents = plainIndents[:len(plainIndents)-1] + } + level := rootOffset + len(plainIndents) + stack = stack[:min(level, len(stack))] + if strmTreeItemIsVideoSource(item) { + out = append(out, strmTreeJoinedSource(stack, item)) + continue + } + stack = append(stack, item) + plainIndents = append(plainIndents, indent) + continue + } + if strmTreeItemIsVideoSource(item) || strings.ContainsAny(item, `/\`) { + plainIndents = plainIndents[:0] + out = append(out, item) + continue + } + stack = []string{item} + plainIndents = plainIndents[:0] + rootOffset = 1 + } + return out +} + +func parseSTRMTreeLine(line string) (string, int, bool) { + if idx := strings.Index(line, "──"); idx >= 0 { + prefix := line[:idx] + level := strmTreeIndentLevelWithWidth(prefix, 4) + item := cleanSTRMTreeItemName(strings.Trim(strings.TrimSpace(line[idx+len("──"):]), "─- ")) + return item, level, item != "" + } + if idx := strings.Index(line, "─"); idx >= 0 { + prefix := line[:idx] + level := strmTreeIndentLevelWithWidth(prefix, 3) + item := cleanSTRMTreeItemName(strings.Trim(strings.TrimSpace(line[idx+len("─"):]), "─- ")) + return item, level, item != "" + } + for _, marker := range []string{"|--", "+--", "`--"} { + if idx := strings.Index(line, marker); idx >= 0 { + item := cleanSTRMTreeItemName(line[idx+len(marker):]) + return item, strmTreeIndentLevel(line[:idx]), item != "" + } + } + return "", 0, false +} + +func parseSTRMTreeContinuationLine(line string, plainTreeActive bool) (string, int, bool) { + prefixLen := 0 + hasGuide := false + for _, r := range line { + switch r { + case ' ', '\t', '│', '|': + if r == '│' || r == '|' { + hasGuide = true + } + prefixLen += len(string(r)) + default: + if plainTreeActive && !hasGuide { + return "", 0, false + } + item := cleanSTRMTreeItemName(line[prefixLen:]) + if item == "" || !strmTreeItemIsVideoSource(item) { + return "", 0, false + } + return item, strmTreeIndentLevel(line[:prefixLen]), true + } + } + return "", 0, false +} + +func strmTreeIndentLevel(prefix string) int { + return strmTreeIndentLevelWithWidth(prefix, 4) +} + +func strmTreeIndentLevelWithWidth(prefix string, width int) int { + if prefix == "" { + return 0 + } + if width <= 0 { + width = 4 + } + verticals := strings.Count(prefix, "│") + strings.Count(prefix, "|") + runeLen := len([]rune(strings.ReplaceAll(strings.ReplaceAll(prefix, "│", " "), "|", " "))) + byWidth := 0 + if runeLen > 0 { + byWidth = (runeLen - 1) / width + } + if verticals > byWidth { + return verticals + } + return byWidth +} + +func strmTreePlainIndent(line string) int { + indent := 0 + for _, r := range line { + switch r { + case ' ': + indent++ + case '\t': + indent += 4 + default: + return indent + } + } + return indent +} + +func strmTreeItemIsVideoSource(item string) bool { + if strmTreeSourceIsVideo(item) { + return true + } + source := normalizeSTRMTreeSourceWithProvider(item, "openlist") + return source.Path != "" && strmTreeSourceIsVideo(source.Path) +} + +func strmTreeJoinedSource(stack []string, item string) string { + if strmTreeItemIsAbsoluteSource(item) { + return item + } + return path.Join(append(append([]string{}, stack...), item)...) +} + +func strmTreeItemIsAbsoluteSource(item string) bool { + value := strings.ToLower(strings.TrimSpace(item)) + return strings.Contains(value, "://") || strings.HasPrefix(value, "/api/") +} diff --git a/internal/service/strm_tree_parse_test.go b/internal/service/strm_tree_parse_test.go new file mode 100644 index 0000000..36235f8 --- /dev/null +++ b/internal/service/strm_tree_parse_test.go @@ -0,0 +1,234 @@ +package service + +import ( + "path/filepath" + "strings" + "testing" + + "go.uber.org/zap" +) + +func TestGenerateSTRMFromTreeTextPreservesRootTree(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "电视剧", + "├── 国产剧", + "│ └── 南部档案", + "│ ├── Archives.S01E01.mkv", + "│ └── Archives.S01E01.nfo", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + TreeText: tree, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want one generated video", res) + } + got := readSTRM(t, filepath.Join(outDir, "电视剧", "国产剧", "南部档案", "Archives.S01E01.strm")) + if got != "/api/cloud/play/openlist?ref=%2F%E7%94%B5%E8%A7%86%E5%89%A7%2F%E5%9B%BD%E4%BA%A7%E5%89%A7%2F%E5%8D%97%E9%83%A8%E6%A1%A3%E6%A1%88%2FArchives.S01E01.mkv" { + t.Fatalf("strm url = %q", got) + } +} + +func TestGenerateSTRMFromTreeTextSupportsSingleLineTreeMarkers(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "动漫", + "├─ 国漫", + "│ └─ 凡人修仙传", + "│ └─ Season 01", + "│ └─ Mortal.Journey.S01E01.mp4", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "115", + TreeText: tree, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want one generated video", res) + } + got := readSTRM(t, filepath.Join(outDir, "动漫", "国漫", "凡人修仙传", "Season 01", "Mortal.Journey.S01E01.strm")) + if !strings.Contains(got, "/api/cloud/play/cloud115?") { + t.Fatalf("strm url = %q, want cloud115 play url", got) + } +} + +func TestGenerateSTRMFromTreeTextSupportsConnectorURLSources(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "电影", + "├── https://media.example.com/api/cloud/play/openlist?ref=%2FMovies%2FLinked.Movie.2026.mkv", + "└── cloud://openlist/%E7%94%B5%E5%BD%B1/%E5%88%AB%E5%90%8D/Cloud.Query.2026.mkv?dir=%2Factual%2Fcloud%2FCloud.Query.2026.mkv", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "115", + TreeText: tree, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 2 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want two generated videos from connector URL rows", res) + } + linked := readSTRM(t, filepath.Join(outDir, "Movies", "Linked.Movie.2026.strm")) + if !strings.Contains(linked, "/api/cloud/play/openlist?") || !strings.Contains(linked, "ref=%2FMovies%2FLinked.Movie.2026.mkv") { + t.Fatalf("cloud play connector url = %q, want preserved provider/ref", linked) + } + cloud := readSTRM(t, filepath.Join(outDir, "电影", "别名", "Cloud.Query.2026.strm")) + if !strings.Contains(cloud, "/api/cloud/play/openlist?") || !strings.Contains(cloud, "ref=%2Factual%2Fcloud%2FCloud.Query.2026.mkv") { + t.Fatalf("cloud mount connector url = %q, want display path output and scan dir ref", cloud) + } +} + +func TestGenerateSTRMFromTreeTextSupportsPlainIndentedTree(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "电视剧", + " 欧美剧", + " House of the Dragon", + " Season 03", + " House.of.the.Dragon.S03E01.mkv", + " House.of.the.Dragon.S03E02.mkv", + " The Last of Us", + " Season 02", + " The.Last.of.Us.S02E01.mkv", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + TreeText: tree, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 3 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want three generated videos from plain indented tree", res) + } + got := readSTRM(t, filepath.Join(outDir, "电视剧", "欧美剧", "House of the Dragon", "Season 03", "House.of.the.Dragon.S03E01.strm")) + if !strings.Contains(got, "House.of.the.Dragon.S03E01.mkv") { + t.Fatalf("strm url = %q, want first plain-indented source ref", got) + } + got = readSTRM(t, filepath.Join(outDir, "电视剧", "欧美剧", "The Last of Us", "Season 02", "The.Last.of.Us.S02E01.strm")) + if !strings.Contains(got, "The.Last.of.Us.S02E01.mkv") { + t.Fatalf("strm url = %q, want sibling folder source ref", got) + } +} + +func TestGenerateSTRMFromTreeTextSupportsWindowsTreeFileRows(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "电视剧", + "├─欧美剧", + "│ ├─House of the Dragon", + "│ │ House.of.the.Dragon.S03E01.mkv", + "│ │ House.of.the.Dragon.S03E02.mkv", + "│ └─The Last of Us", + "│ The.Last.of.Us.S02E01.mkv", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + TreeText: tree, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 3 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want three generated videos from Windows tree rows", res) + } + got := readSTRM(t, filepath.Join(outDir, "电视剧", "欧美剧", "House of the Dragon", "House.of.the.Dragon.S03E02.strm")) + if !strings.Contains(got, "House.of.the.Dragon.S03E02.mkv") { + t.Fatalf("strm url = %q, want vertical-prefix file row ref", got) + } + got = readSTRM(t, filepath.Join(outDir, "电视剧", "欧美剧", "The Last of Us", "The.Last.of.Us.S02E01.strm")) + if !strings.Contains(got, "The.Last.of.Us.S02E01.mkv") { + t.Fatalf("strm url = %q, want blank-prefix sibling file row ref", got) + } +} + +func TestGenerateSTRMFromTreeStripsDecoratedTreeNames(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "📁 电视剧", + "├── [目录] 欧美剧", + "│ └── (folder) House of the Dragon", + "│ │ 📄 House.of.the.Dragon.S03E01.mkv", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + TreeText: tree, + Paths: []string{"/[目录] 动漫/[folder] 日番/[文件] Frieren.S01E01.mp4"}, + SourceRoot: "/动漫", + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 2 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want two generated videos with decorations stripped", res) + } + showPath := filepath.Join(outDir, "电视剧", "欧美剧", "House of the Dragon", "House.of.the.Dragon.S03E01.strm") + show := readSTRM(t, showPath) + if strings.Contains(showPath, "目录") || strings.Contains(showPath, "folder") || strings.Contains(showPath, "📄") { + t.Fatalf("decorated local path was not cleaned: %q", showPath) + } + if !strings.Contains(show, "House.of.the.Dragon.S03E01.mkv") || strings.Contains(show, "%5B") || strings.Contains(show, "%F0%9F") { + t.Fatalf("decorated tree ref was not cleaned: %q", show) + } + episode := readSTRM(t, filepath.Join(outDir, "日番", "Frieren.S01E01.strm")) + if !strings.Contains(episode, "ref=%2F%E5%8A%A8%E6%BC%AB%2F%E6%97%A5%E7%95%AA%2FFrieren.S01E01.mp4") { + t.Fatalf("decorated direct path ref was not cleaned: %q", episode) + } +} + +func TestGenerateSTRMFromTreeStripsExportedFileMetadata(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "电影", + "└── 欧美电影", + " └── Dune.Part.Two.2024.2160p.WEB-DL.mkv 18.6 GB 2024-04-01 12:30", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + TreeText: tree, + Paths: []string{"/电视剧/欧美剧/Show/Season 01/Show.S01E01.mp4 (2.1 GB)"}, + SourceRoot: "/电视剧", + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 2 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want two generated videos with metadata suffix stripped", res) + } + movie := readSTRM(t, filepath.Join(outDir, "电影", "欧美电影", "Dune.Part.Two.2024.2160p.WEB-DL.strm")) + if !strings.Contains(movie, "Dune.Part.Two.2024.2160p.WEB-DL.mkv") || strings.Contains(movie, "18.6") { + t.Fatalf("movie strm url = %q, want clean media ref without size metadata", movie) + } + episode := readSTRM(t, filepath.Join(outDir, "欧美剧", "Show", "Season 01", "Show.S01E01.strm")) + if !strings.Contains(episode, "Show.S01E01.mp4") || strings.Contains(episode, "2.1") { + t.Fatalf("episode strm url = %q, want clean media ref without size metadata", episode) + } +} diff --git a/internal/service/strm_tree_paths.go b/internal/service/strm_tree_paths.go new file mode 100644 index 0000000..ced42eb --- /dev/null +++ b/internal/service/strm_tree_paths.go @@ -0,0 +1,76 @@ +package service + +import ( + "errors" + "fmt" + "path" + "path/filepath" + "strings" +) + +func strmTreeRelativeSource(source, sourceRoot string) string { + source = normalizeSTRMTreeSource(source) + root := normalizeRemotePath(sourceRoot) + if root != "/" && strings.HasPrefix(source, root+"/") { + return strings.TrimPrefix(strings.TrimPrefix(source, root), "/") + } + return strings.TrimPrefix(source, "/") +} + +func strmTreeCloudRef(source, sourceRoot string) string { + source = normalizeSTRMTreeSource(source) + if strings.HasPrefix(source, "/") { + return source + } + if strings.TrimSpace(sourceRoot) != "" { + return joinRemotePath(sourceRoot, source) + } + return normalizeRemotePath(source) +} + +func strmTreeOutputRelativePath(source string) (string, error) { + parts := strings.Split(strings.Trim(strings.ReplaceAll(source, "\\", "/"), "/"), "/") + if len(parts) == 0 { + return "", errors.New("empty source path") + } + out := make([]string, 0, len(parts)) + for i, part := range parts { + part = strings.TrimSpace(part) + if part == "" || part == "." || part == ".." { + return "", fmt.Errorf("unsafe path segment %q", part) + } + if i == len(parts)-1 { + ext := strings.ToLower(path.Ext(part)) + if _, ok := videoExtensions[ext]; !ok { + return "", fmt.Errorf("unsupported media extension %q", ext) + } + part = strings.TrimSuffix(part, path.Ext(part)) + ".strm" + } + safe := sanitizeFilename(part) + if safe == "" { + return "", errors.New("empty sanitized path segment") + } + out = append(out, safe) + } + return filepath.Join(out...), nil +} + +func strmTreeOutputPrefixPath(prefix string) (string, error) { + parts := strings.Split(strings.Trim(strings.ReplaceAll(prefix, "\\", "/"), "/"), "/") + out := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if part == "." || part == ".." { + return "", fmt.Errorf("unsafe output prefix segment %q", part) + } + safe := sanitizeFilename(part) + if safe == "" { + return "", errors.New("empty sanitized output prefix segment") + } + out = append(out, safe) + } + return filepath.Join(out...), nil +} diff --git a/internal/service/strm_tree_url_test.go b/internal/service/strm_tree_url_test.go new file mode 100644 index 0000000..e4d01e4 --- /dev/null +++ b/internal/service/strm_tree_url_test.go @@ -0,0 +1,251 @@ +package service + +import ( + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "go.uber.org/zap" +) + +func TestGenerateSTRMFromTreeUsesMediaPathFromURLQuery(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{ + "https://openlist.example.com/api/fs/get?path=%2FMovies%2FDune.Part.Two.2024.mkv", + "https://openlist.example.com/api/fs/get?path=%2FMovies%2FA%2BB.2026.mkv", + "https://openlist.example.com/api/fs/get?path=/Movies/A+B.Raw.2026.mkv", + "https://openlist.example.com/api/raw?ref=/Shows/Some.Show/S01E01.mp4", + "https://cdn.example.com/media/Movies/Direct.Movie.2026.mkv?token=secret", + "https://cdn.example.com/media/Movies/A+B.Direct.2026.mkv?token=secret", + "https://openlist.example.com/api/fs/get?id=12345", + }, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 6 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want six generated videos and non-media API URL ignored", res) + } + movie := readSTRM(t, filepath.Join(outDir, "Movies", "Dune.Part.Two.2024.strm")) + if !strings.Contains(movie, "ref=%2FMovies%2FDune.Part.Two.2024.mkv") || strings.Contains(movie, "api%2Ffs%2Fget") { + t.Fatalf("movie strm url = %q, want query media path as cloud ref", movie) + } + plusMovie := readSTRM(t, filepath.Join(outDir, "Movies", "A+B.2026.strm")) + if !strings.Contains(plusMovie, "ref=%2FMovies%2FA%2BB.2026.mkv") || strings.Contains(plusMovie, "A+B.2026.mkv") { + t.Fatalf("plus movie strm url = %q, want literal plus preserved and encoded in ref", plusMovie) + } + rawPlusMovie := readSTRM(t, filepath.Join(outDir, "Movies", "A+B.Raw.2026.strm")) + if !strings.Contains(rawPlusMovie, "ref=%2FMovies%2FA%2BB.Raw.2026.mkv") || strings.Contains(rawPlusMovie, "A+B.Raw.2026.mkv") { + t.Fatalf("raw plus movie strm url = %q, want raw query plus preserved and encoded in ref", rawPlusMovie) + } + show := readSTRM(t, filepath.Join(outDir, "Shows", "Some.Show", "S01E01.strm")) + if !strings.Contains(show, "ref=%2FShows%2FSome.Show%2FS01E01.mp4") { + t.Fatalf("show strm url = %q, want ref query media path", show) + } + direct := readSTRM(t, filepath.Join(outDir, "media", "Movies", "Direct.Movie.2026.strm")) + if !strings.Contains(direct, "ref=%2Fmedia%2FMovies%2FDirect.Movie.2026.mkv") { + t.Fatalf("direct url strm = %q, want normal URL path media ref", direct) + } + plusDirect := readSTRM(t, filepath.Join(outDir, "media", "Movies", "A+B.Direct.2026.strm")) + if !strings.Contains(plusDirect, "ref=%2Fmedia%2FMovies%2FA%2BB.Direct.2026.mkv") { + t.Fatalf("plus direct url strm = %q, want URL path plus preserved", plusDirect) + } + if _, err := os.Stat(filepath.Join(outDir, "api", "fs", "get.strm")); !os.IsNotExist(err) { + t.Fatalf("non-media API URL should not generate STRM, stat err=%v", err) + } +} + +func TestGenerateSTRMFromTreePreservesSourceProvider(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "115", + Paths: []string{ + "cloud://openlist/Movies/OpenList.Movie.2026.mkv", + "/api/cloud/play/cloud115?ref=%2FShows%2FCloud115.Show.S01E01.mkv", + "/api/cloud/play/openlist?ref=%2FMovies%2FMy+Space.Movie.2026.mkv", + "https://media.example.com/api/cloud/play/openlist?ref=%2FMovies%2FRemote.OpenList.Movie.2026.mkv", + "/Movies/Fallback.115.Movie.2026.mkv", + }, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 5 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want five generated videos", res) + } + openlist := readSTRM(t, filepath.Join(outDir, "Movies", "OpenList.Movie.2026.strm")) + if !strings.Contains(openlist, "/api/cloud/play/openlist?") || !strings.Contains(openlist, "ref=%2FMovies%2FOpenList.Movie.2026.mkv") { + t.Fatalf("cloud:// source url = %q, want openlist provider and original ref", openlist) + } + cloud115 := readSTRM(t, filepath.Join(outDir, "Shows", "Cloud115.Show.S01E01.strm")) + if !strings.Contains(cloud115, "/api/cloud/play/cloud115?") || !strings.Contains(cloud115, "ref=%2FShows%2FCloud115.Show.S01E01.mkv") { + t.Fatalf("cloud play source url = %q, want cloud115 provider preserved", cloud115) + } + spaceMoviePath := filepath.Join(outDir, "Movies", "My Space.Movie.2026.strm") + spaceMovie := readSTRM(t, spaceMoviePath) + if _, err := os.Stat(filepath.Join(outDir, "Movies", "My+Space.Movie.2026.strm")); !os.IsNotExist(err) { + t.Fatalf("cloud play source should not create literal-plus local path, stat err=%v", err) + } + spaceURL, err := url.Parse(spaceMovie) + if err != nil { + t.Fatalf("parse cloud play source url %q: %v", spaceMovie, err) + } + if got := spaceURL.Query().Get("ref"); got != "/Movies/My Space.Movie.2026.mkv" { + t.Fatalf("cloud play source ref = %q, want decoded space path", got) + } + remoteOpenlist := readSTRM(t, filepath.Join(outDir, "Movies", "Remote.OpenList.Movie.2026.strm")) + if !strings.Contains(remoteOpenlist, "/api/cloud/play/openlist?") || !strings.Contains(remoteOpenlist, "ref=%2FMovies%2FRemote.OpenList.Movie.2026.mkv") { + t.Fatalf("absolute cloud play source url = %q, want openlist provider preserved", remoteOpenlist) + } + fallback := readSTRM(t, filepath.Join(outDir, "Movies", "Fallback.115.Movie.2026.strm")) + if !strings.Contains(fallback, "/api/cloud/play/cloud115?") || !strings.Contains(fallback, "ref=%2FMovies%2FFallback.115.Movie.2026.mkv") { + t.Fatalf("plain source url = %q, want fallback cloud115 provider", fallback) + } +} + +func TestGenerateSTRMFromTreeCloudMountUsesScanDirForRef(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "115", + Paths: []string{ + "cloud://openlist/%E7%94%B5%E5%BD%B1/%E5%88%AB%E5%90%8D%E7%9B%AE%E5%BD%95/Alias.Movie.2026.mkv?dir=%2Factual%2Fcloud%2Fmovies%2FAlias.Movie.2026.mkv", + }, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want one generated video", res) + } + got := readSTRM(t, filepath.Join(outDir, "电影", "别名目录", "Alias.Movie.2026.strm")) + if !strings.Contains(got, "/api/cloud/play/openlist?") { + t.Fatalf("strm url = %q, want provider from cloud mount", got) + } + if !strings.Contains(got, "ref=%2Factual%2Fcloud%2Fmovies%2FAlias.Movie.2026.mkv") { + t.Fatalf("strm url = %q, want dir scan path as playable ref", got) + } + if strings.Contains(got, "%E5%88%AB%E5%90%8D%E7%9B%AE%E5%BD%95") { + t.Fatalf("strm url = %q, display path leaked into playable ref", got) + } +} + +func TestGenerateSTRMFromTreeDoesNotDedupeDifferentProviders(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "115", + Paths: []string{"cloud://openlist/Movies/Same.Movie.mkv", "cloud://cloud115/Movies/Same.Movie.mkv"}, + OutputDir: outDir, + Overwrite: true, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || res.Updated != 1 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want both provider-specific sources processed", res) + } + got := readSTRM(t, filepath.Join(outDir, "Movies", "Same.Movie.strm")) + if !strings.Contains(got, "/api/cloud/play/cloud115?") || !strings.Contains(got, "ref=%2FMovies%2FSame.Movie.mkv") { + t.Fatalf("final strm url = %q, want second provider write to prove it was not deduped", got) + } +} + +func TestGenerateSTRMFromTreeStripsPathListPrefixes(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + tree := strings.Join([]string{ + "- /Movies/Dune.Part.Two.2024.mkv", + "* https://openlist.example.com/api/fs/get?path=%2FShows%2FSome.Show%2FS01E01.mp4", + "1. /Anime/Frieren/Frieren.S01E01.mp4", + "2) /Anime/Frieren/Frieren.S01E02.mp4", + "• /Documentaries/Earth.2026.mkv", + }, "\n") + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + TreeText: tree, + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 5 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want five generated videos with list prefixes stripped", res) + } + moviePath := filepath.Join(outDir, "Movies", "Dune.Part.Two.2024.strm") + movie := readSTRM(t, moviePath) + if strings.Contains(moviePath, "- ") || !strings.Contains(movie, "ref=%2FMovies%2FDune.Part.Two.2024.mkv") { + t.Fatalf("markdown bullet prefix leaked into movie path/ref: path=%q url=%q", moviePath, movie) + } + show := readSTRM(t, filepath.Join(outDir, "Shows", "Some.Show", "S01E01.strm")) + if strings.Contains(show, "api%2Ffs%2Fget") || !strings.Contains(show, "ref=%2FShows%2FSome.Show%2FS01E01.mp4") { + t.Fatalf("bullet URL query source was not cleaned: %q", show) + } + if _, err := os.Stat(filepath.Join(outDir, "1. ", "Anime", "Frieren", "Frieren.S01E01.strm")); !os.IsNotExist(err) { + t.Fatalf("numbered prefix should not create a literal prefix directory, stat err=%v", err) + } + if got := readSTRM(t, filepath.Join(outDir, "Documentaries", "Earth.2026.strm")); !strings.Contains(got, "ref=%2FDocumentaries%2FEarth.2026.mkv") { + t.Fatalf("round bullet source ref = %q", got) + } +} + +func TestGenerateSTRMFromTreeOutputPrefixOnlyAffectsLocalPath(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{"/cloud/Dune.Part.Two.2024.mkv"}, + SourceRoot: "/cloud", + OutputPrefix: "电影/欧美电影", + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 1 || len(res.Errors) != 0 { + t.Fatalf("result = %#v, want one generated video", res) + } + got := readSTRM(t, filepath.Join(outDir, "电影", "欧美电影", "Dune.Part.Two.2024.strm")) + if strings.Contains(got, "%E7%94%B5%E5%BD%B1") || strings.Contains(got, "%E6%AC%A7%E7%BE%8E%E7%94%B5%E5%BD%B1") { + t.Fatalf("strm url = %q, output prefix should not be injected into cloud ref", got) + } + if !strings.Contains(got, "ref=%2Fcloud%2FDune.Part.Two.2024.mkv") { + t.Fatalf("strm url = %q, want original cloud ref", got) + } +} + +func TestGenerateSTRMFromTreeRejectsUnsafeOutputPrefix(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "strm") + svc := NewSTRMService(zap.NewNop(), nil, nil) + + res, err := svc.GenerateFromTree(t.Context(), GenerateSTRMTreeOptions{ + Provider: "openlist", + Paths: []string{"Movies/Movie.mkv"}, + OutputPrefix: "../escape", + OutputDir: outDir, + }) + if err != nil { + t.Fatal(err) + } + if res.Generated != 0 || len(res.Errors) != 1 { + t.Fatalf("result = %#v, want unsafe prefix rejected", res) + } + if _, err := os.Stat(filepath.Join(outDir, "..", "escape", "Movies", "Movie.strm")); !os.IsNotExist(err) { + t.Fatalf("unsafe prefixed strm should not be written, stat err=%v", err) + } +} diff --git a/internal/service/strm_url.go b/internal/service/strm_url.go index cba70c8..6481d0f 100644 --- a/internal/service/strm_url.go +++ b/internal/service/strm_url.go @@ -114,6 +114,52 @@ func (s *STRMService) strmRelativePath(lib model.Library, media model.Media) str return filepath.Join(safe, safe+".strm") } +func (s *STRMService) strmTreeRelativePath(media model.Media) string { + parts := strmLibraryPathParts(media.Path) + if len(parts) == 0 { + return "" + } + parts = strmDropCategoryPrefix(parts) + if len(parts) == 0 { + return "" + } + last := parts[len(parts)-1] + ext := filepath.Ext(last) + if ext == "" { + return "" + } + parts[len(parts)-1] = strings.TrimSuffix(last, ext) + ".strm" + clean := make([]string, 0, len(parts)) + for _, part := range parts { + if safe := sanitizeFilename(part); safe != "" { + clean = append(clean, safe) + } + } + if len(clean) == 0 { + return "" + } + return filepath.Join(clean...) +} + +func strmDropCategoryPrefix(parts []string) []string { + if len(parts) == 0 { + return nil + } + for i, part := range parts { + if strmCanonicalRoot(part) == "" && strmCategoryRoot(part) == "" { + continue + } + next := i + 1 + if strmCanonicalRoot(part) != "" && next < len(parts) && strmCategoryRoot(parts[next]) != "" { + next++ + } + if next < len(parts) { + return append([]string(nil), parts[next:]...) + } + } + return append([]string(nil), parts...) +} + func absolutizeSTRMURL(raw, baseURL string) string { raw = strings.TrimSpace(raw) if raw == "" || strings.HasPrefix(raw, "//") { diff --git a/internal/service/subscription_archive.go b/internal/service/subscription_archive.go index 702c5c6..172e61b 100644 --- a/internal/service/subscription_archive.go +++ b/internal/service/subscription_archive.go @@ -81,7 +81,7 @@ func (s *SubscriptionService) archiveCompletedSubscription(ctx context.Context, } func subscriptionShouldArchive(sub *model.Subscription, availability LocalAvailability) bool { - if sub == nil || sub.WashEnabled || sub.ArchivedAt != nil { + if sub == nil || subscriptionAllowsWash(sub) || sub.ArchivedAt != nil { return false } mediaType := normalizeMediaType(sub.MediaType, sub.Name+" "+sub.Filter, "") @@ -128,7 +128,7 @@ func maxAvailabilityEpisode(keys map[string]struct{}) int { } func subscriptionArchiveReason(sub *model.Subscription, availability LocalAvailability) string { - if sub != nil && sub.WashEnabled { + if subscriptionAllowsWash(sub) { return "" } if availability.HasSeriesPack { diff --git a/internal/service/subscription_archive_test.go b/internal/service/subscription_archive_test.go index a0f0999..b9cef12 100644 --- a/internal/service/subscription_archive_test.go +++ b/internal/service/subscription_archive_test.go @@ -185,6 +185,7 @@ func TestSubscriptionArchiveKeepsWashSubscriptionActive(t *testing.T) { FeedURL: "site-search://search?keyword=Dune", Filter: "Dune 2021", MediaType: "movie", + Resolution: "2160p", WashEnabled: true, Enabled: true, } diff --git a/internal/service/subscription_candidate_collect.go b/internal/service/subscription_candidate_collect.go index bf9f9f9..b7b4184 100644 --- a/internal/service/subscription_candidate_collect.go +++ b/internal/service/subscription_candidate_collect.go @@ -2,11 +2,14 @@ package service import ( "regexp" + "strconv" "strings" "github.com/ShukeBta/MediaStationGo/internal/model" ) +var looseSubscriptionEpisodeRE = regexp.MustCompile(`(?i)(?:^|[\s._\-\[\(])0?(\d{1,3})(?:v\d+)?(?:$|[\s._\-\]\)])`) + func collectSiteSearchCandidates(results []SearchResult, sub *model.Subscription, seenSet map[string]struct{}, allowQueryMismatch bool, stats *siteSearchSelectionStats) []siteSearchCandidate { candidates := make([]siteSearchCandidate, 0, len(results)) for _, item := range results { @@ -20,7 +23,7 @@ func collectSiteSearchCandidates(results []SearchResult, sub *model.Subscription continue } } - if !matchesSubscriptionRules(sub, matchText) { + if !matchesSubscriptionRules(sub, matchText) || !matchesSubscriptionTorrentRules(sub, item) { stats.RuleMismatch++ continue } @@ -37,8 +40,13 @@ func collectSiteSearchCandidates(results []SearchResult, sub *model.Subscription stats.Seen++ continue } + refs := subscriptionCandidateEpisodeRefs(sub, matchText) season, episode := ParseEpisode(matchText) - episodes := episodeNumbersFromRefs(episodeRefsFromTitle(matchText), season) + if episode <= 0 && len(refs) > 0 { + season = refs[0].Season + episode = refs[0].Episode + } + episodes := episodeNumbersFromRefs(refs, season) score := subscriptionCandidateScore(sub, item) stats.Prepared++ candidates = append(candidates, siteSearchCandidate{ @@ -78,7 +86,7 @@ func shouldRelaxSiteSearchQueryMatch(sub *model.Subscription, local LocalAvailab } func subscriptionSearchResultText(item SearchResult) string { - return strings.TrimSpace(strings.Join([]string{item.Title, item.Subtitle}, " ")) + return strings.TrimSpace(strings.Join([]string{item.Title, item.Subtitle, item.Labels}, " ")) } func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, filter *regexp.Regexp, seenSet map[string]struct{}, local LocalAvailability) []siteSearchCandidate { @@ -94,7 +102,8 @@ func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, f if filter != nil && !filter.MatchString(title) { continue } - if !matchesSubscriptionRules(sub, title) { + searchItem := SearchResult{Title: title} + if !matchesSubscriptionRules(sub, title) || !matchesSubscriptionTorrentRules(sub, searchItem) { continue } download := strings.TrimSpace(item.Enclosure.URL) @@ -108,9 +117,14 @@ func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, f if _, ok := seenSet[guid]; ok { continue } - searchItem := SearchResult{Title: title, DownloadURL: download} + searchItem.DownloadURL = download + refs := subscriptionCandidateEpisodeRefs(sub, title) season, episode := ParseEpisode(title) - episodes := episodeNumbersFromRefs(episodeRefsFromTitle(title), season) + if episode <= 0 && len(refs) > 0 { + season = refs[0].Season + episode = refs[0].Episode + } + episodes := episodeNumbersFromRefs(refs, season) candidates = append(candidates, siteSearchCandidate{ Item: searchItem, Download: download, @@ -125,6 +139,74 @@ func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, f return selectPreparedSubscriptionCandidates(candidates, sub, local) } +func subscriptionCandidateEpisodeRefs(sub *model.Subscription, text string) []episodeRef { + if refs := episodeRefsFromTitle(text); len(refs) > 0 { + return refs + } + if sub == nil || !isSubscriptionSeriesType(sub.MediaType) || isSeriesPackTitle(text) || patSeasonOnly.MatchString(text) { + return nil + } + episode := inferLooseSubscriptionEpisode(maskSubscriptionTitleQueries(sub, text)) + if episode <= 0 { + return nil + } + return []episodeRef{{Season: 1, Episode: episode}} +} + +func maskSubscriptionTitleQueries(sub *model.Subscription, text string) string { + if sub == nil || strings.TrimSpace(text) == "" { + return text + } + out := text + outFold := strings.ToLower(out) + for _, query := range subscriptionTitleMatchQueries(sub) { + query = strings.TrimSpace(query) + if query == "" { + continue + } + queryFold := strings.ToLower(query) + for { + idx := strings.Index(outFold, queryFold) + if idx < 0 { + break + } + out = out[:idx] + strings.Repeat(" ", len(query)) + out[idx+len(query):] + outFold = strings.ToLower(out) + } + } + return out +} + +func inferLooseSubscriptionEpisode(text string) int { + for _, match := range looseSubscriptionEpisodeRE.FindAllStringSubmatchIndex(text, -1) { + if len(match) < 4 || match[2] < 0 || match[3] < 0 { + continue + } + if isDecimalFractionMatch(text, match[2]) { + continue + } + value, err := strconv.Atoi(text[match[2]:match[3]]) + if err != nil || !looksLikeLooseEpisodeNumber(value) { + continue + } + return value + } + return 0 +} + +func isDecimalFractionMatch(text string, digitStart int) bool { + return digitStart >= 2 && text[digitStart-1] == '.' && text[digitStart-2] >= '0' && text[digitStart-2] <= '9' +} + +func looksLikeLooseEpisodeNumber(value int) bool { + switch { + case value <= 0, value > 200: + return false + default: + return true + } +} + func episodeNumbersFromRefs(refs []episodeRef, fallbackSeason int) []int { if len(refs) == 0 { return nil diff --git a/internal/service/subscription_candidate_defaults_test.go b/internal/service/subscription_candidate_defaults_test.go new file mode 100644 index 0000000..122b619 --- /dev/null +++ b/internal/service/subscription_candidate_defaults_test.go @@ -0,0 +1,299 @@ +package service + +import ( + "testing" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func TestSelectSiteSearchCandidatesRejectsKeywordOriginWithConflictingYear(t *testing.T) { + sub := &model.Subscription{ + Name: "玩具总动员 5 自动订阅", + Filter: "玩具总动员 5 2026", + MediaType: "movie", + Year: 2026, + } + results := []SearchResult{{ + Title: "Toy Story 4 2019 2160p DSNP WEB-DL", + DownloadURL: "https://pt/download/toy-story-4", + SearchKeyword: "玩具总动员 5", + Seeders: 90, + }} + + got, stats := selectSiteSearchCandidatesWithStats(results, sub, map[string]struct{}{}, LocalAvailability{}) + if len(got) != 0 { + t.Fatalf("selected %#v, want conflicting-year keyword-origin result rejected", got) + } + if stats.QueryMismatch != 1 || stats.Prepared != 0 { + t.Fatalf("stats = %#v, want query mismatch for conflicting year", stats) + } +} + +func TestSelectSiteSearchCandidatesDoesNotWashByDefault(t *testing.T) { + sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", WashPriority: "resolution"} + results := []SearchResult{ + {Title: "Inception 2010 1080p", DownloadURL: "https://pt/download/1080", Seeders: 90}, + {Title: "Inception 2010 2160p", DownloadURL: "https://pt/download/2160", Seeders: 80}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/2160" { + t.Fatalf("selected %#v, want default best single result when wash disabled", got) + } +} + +func TestSelectSiteSearchCandidatesWashNeedsExplicitUpgradeCriteria(t *testing.T) { + sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", WashEnabled: true, WashPriority: "resolution"} + local := LocalAvailability{LocalMediaCount: 1, InLibrary: true} + results := []SearchResult{ + {Title: "Inception 2010 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/1080", Seeders: 90}, + {Title: "Inception 2010 2160p WEB-DL H264 AAC", DownloadURL: "https://pt/download/2160", Seeders: 80}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}, local) + if len(got) != 0 { + t.Fatalf("selected %#v, want no wash download without explicit upgrade criteria", got) + } + + sub.Resolution = "2160p" + got = selectSiteSearchCandidates(results, sub, map[string]struct{}{}, local) + if len(got) != 1 || got[0].Download != "https://pt/download/2160" { + t.Fatalf("selected %#v, want explicit 2160p wash candidate", got) + } +} + +func TestSelectSiteSearchCandidatesWashWithoutCriteriaUsesDefaultQuality(t *testing.T) { + sub := &model.Subscription{Name: "Dune 自动订阅", Filter: "Dune 2021", MediaType: "movie", WashEnabled: true, WashPriority: "quality"} + results := []SearchResult{ + {Title: "Dune 2021 2160p REMUX H264 AAC", DownloadURL: "https://pt/download/remux", Seeders: 80}, + {Title: "Dune 2021 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/webdl", Seeders: 60}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/webdl" { + t.Fatalf("selected %#v, want default compatible WEB-DL when wash has no explicit criteria", got) + } +} + +func TestSelectSiteSearchCandidatesDefaultsToOnePreferredVersionPerEpisode(t *testing.T) { + sub := &model.Subscription{Name: "House of the Dragon 自动订阅", Filter: "House of the Dragon", MediaType: "tv"} + results := []SearchResult{ + {Title: "House of the Dragon S03E01 1080p HDTV", DownloadURL: "https://pt/download/e01-hdtv", Seeders: 50000}, + {Title: "House of the Dragon S03E01 1080p WEB-DL", DownloadURL: "https://pt/download/e01-webdl-1080", Seeders: 100}, + {Title: "House of the Dragon S03E01 2160p WEB-DL", DownloadURL: "https://pt/download/e01-webdl-2160", Seeders: 80}, + {Title: "House of the Dragon S03E02 720p HDTV", DownloadURL: "https://pt/download/e02-hdtv", Seeders: 500}, + {Title: "House of the Dragon S03E02 1080p WEBRip", DownloadURL: "https://pt/download/e02-webrip", Seeders: 60}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 2 { + t.Fatalf("selected %d candidates, want one per episode", len(got)) + } + if got[0].Download != "https://pt/download/e01-webdl-2160" { + t.Fatalf("episode 1 selected %q, want best WEB-DL version", got[0].Download) + } + if got[1].Download != "https://pt/download/e02-webrip" { + t.Fatalf("episode 2 selected %q, want WEBRip over high-seeder HDTV", got[1].Download) + } +} + +func TestSelectSiteSearchCandidatesDefaultQualityRecognizesWebDLVariants(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv"} + results := []SearchResult{ + {Title: "Some Show S01E01 1080p BluRay H264 AAC", DownloadURL: "https://pt/download/e01-bluray", Seeders: 900}, + {Title: "Some Show S01E01 2160p WEB.DL H264 AAC", DownloadURL: "https://pt/download/e01-webdotdl", Seeders: 40}, + {Title: "Some Show S01E01 1080p WEB DL H264 AAC", DownloadURL: "https://pt/download/e01-webdl", Seeders: 50}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-webdotdl" { + t.Fatalf("selected %#v, want one best WEB-DL variant", got) + } +} + +func TestSelectSiteSearchCandidatesDefaultPrefersWebDLBeforeResolution(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv"} + results := []SearchResult{ + {Title: "Some Show S01E01 2160p BluRay H264 AAC", DownloadURL: "https://pt/download/e01-bluray-2160", Seeders: 900}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-webdl-1080", Seeders: 50}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-webdl-1080" { + t.Fatalf("selected %#v, want one compatible WEB-DL version before higher-resolution BluRay", got) + } +} + +func TestSelectSiteSearchCandidatesDefaultPrefersFreeWithinSameQualityBand(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv"} + results := []SearchResult{ + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-nonfree", Seeders: 5000}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-free", Seeders: 80, Free: true}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-free" { + t.Fatalf("selected %#v, want free candidate within same quality/resolution band", got) + } +} + +func TestSelectSiteSearchCandidatesDefaultDoesNotLetFreeOverrideBetterQuality(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv"} + results := []SearchResult{ + {Title: "Some Show S01E01 1080p HDTV H264 AAC", DownloadURL: "https://pt/download/e01-free-hdtv", Seeders: 80, Free: true}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-webdl", Seeders: 50}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-webdl" { + t.Fatalf("selected %#v, want WEB-DL quality to stay ahead of free HDTV", got) + } +} + +func TestSelectSiteSearchCandidatesDefaultDoesNotLetFreeOverrideBetterResolution(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv"} + results := []SearchResult{ + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-free-1080", Seeders: 80, Free: true}, + {Title: "Some Show S01E01 2160p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-2160", Seeders: 50}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-2160" { + t.Fatalf("selected %#v, want better resolution to stay ahead of free lower-resolution release", got) + } +} + +func TestSelectSiteSearchCandidatesDefaultsToOneLooseNumberedEpisode(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv"} + results := []SearchResult{ + {Title: "Some Show 01 1080p HDTV H264 AAC", DownloadURL: "https://pt/download/e01-hdtv", Seeders: 5000}, + {Title: "Some Show 01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-webdl-1080", Seeders: 100}, + {Title: "Some Show 01 2160p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-webdl-2160", Seeders: 80}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 { + t.Fatalf("selected %d candidates, want one preferred loose-numbered episode", len(got)) + } + if got[0].Download != "https://pt/download/e01-webdl-2160" || got[0].Episode != 1 { + t.Fatalf("selected %#v, want episode 1 best WEB-DL version", got) + } +} + +func TestSelectSiteSearchCandidatesDoesNotTreatTitleNumberAsLooseEpisode(t *testing.T) { + sub := &model.Subscription{Name: "问心2 自动订阅", Filter: "问心2 2023", MediaType: "tv"} + results := []SearchResult{ + {Title: "问心2 2023 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/season", Seeders: 100}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 { + t.Fatalf("selected %#v, want one fallback candidate", got) + } + if got[0].Episode != 0 { + t.Fatalf("episode = %d, want title number not treated as episode", got[0].Episode) + } +} + +func TestSelectSiteSearchCandidatesRejectsRiskyLabelsFromSiteResult(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv"} + results := []SearchResult{ + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", Labels: "HR", DownloadURL: "https://pt/download/e01-hr", Seeders: 900}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-safe", Seeders: 80}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-safe" { + t.Fatalf("selected %#v, want non-HR candidate only", got) + } +} + +func TestSelectSiteSearchCandidatesWashPriorityDoesNotLetFreeOverrideResolution(t *testing.T) { + sub := &model.Subscription{Name: "Some Show 自动订阅", Filter: "Some Show", MediaType: "tv", WashEnabled: true, WashPriority: "resolution"} + results := []SearchResult{ + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-free-1080", Seeders: 80, Free: true}, + {Title: "Some Show S01E01 2160p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-2160", Seeders: 50}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-2160" { + t.Fatalf("selected %#v, want wash resolution priority to stay ahead of free lower-resolution release", got) + } +} + +func TestSelectSiteSearchCandidatesRejectsDefaultCompatibilityVersions(t *testing.T) { + sub := &model.Subscription{Name: "House of the Dragon 自动订阅", Filter: "House of the Dragon", MediaType: "tv"} + results := []SearchResult{ + {Title: "House of the Dragon S03E01 2160p WEB-DL HEVC 10bit DoVi Atmos", DownloadURL: "https://pt/download/e01-dovi", Seeders: 900}, + {Title: "House of the Dragon S03E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-webdl", Seeders: 80}, + {Title: "House of the Dragon S03E01 1080p HDTV H264 AAC", DownloadURL: "https://pt/download/e01-hdtv", Seeders: 5000}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-webdl" { + t.Fatalf("selected %#v, want compatible WEB-DL only", got) + } +} + +func TestSelectSiteSearchCandidatesKeepsCompatibilityExcludesWithCustomExcludeWords(t *testing.T) { + sub := &model.Subscription{ + Name: "House of the Dragon 自动订阅", + Filter: "House of the Dragon", + MediaType: "tv", + ExcludeWords: "官中,无字幕", + } + results := []SearchResult{ + {Title: "House of the Dragon S03E01 2160p WEB-DL HEVC 10bit DoVi Atmos", DownloadURL: "https://pt/download/e01-dovi", Seeders: 900}, + {Title: "House of the Dragon S03E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-webdl", Seeders: 80}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/e01-webdl" { + t.Fatalf("selected %#v, want custom exclude words to keep default compatible WEB-DL guard", got) + } +} + +func TestSelectSiteSearchCandidatesAvoidsOverlappingEpisodeRanges(t *testing.T) { + sub := &model.Subscription{Name: "House of the Dragon 自动订阅", Filter: "House of the Dragon", MediaType: "tv", WashEnabled: true, WashPriority: "quality"} + availability := LocalAvailability{ + LocalMediaCount: 1, + TotalEpisodes: 3, + MissingEpisodes: []int{1, 2, 3}, + ExistingEpisodeKeys: map[string]struct{}{}, + } + results := []SearchResult{ + {Title: "House of the Dragon S03E01-E02 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e01-e02-pack", Seeders: 90}, + {Title: "House of the Dragon S03E02 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e02-duplicate", Seeders: 80}, + {Title: "House of the Dragon S03E03 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/e03", Seeders: 70}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}, availability) + if len(got) != 2 { + t.Fatalf("selected %d candidates, want pack plus non-overlapping episode", len(got)) + } + if got[0].Download != "https://pt/download/e01-e02-pack" || got[1].Download != "https://pt/download/e03" { + t.Fatalf("selected %#v, want overlapping E02 duplicate skipped", got) + } +} + +func TestSelectSiteSearchCandidatesAppliesQualityRules(t *testing.T) { + sub := &model.Subscription{ + Name: "Dune 自动订阅", + Filter: "Dune 2021", + MediaType: "movie", + Resolution: "2160p", + Quality: "remux", + Effects: "hdr", + ExcludeWords: "cam,ts", + } + results := []SearchResult{ + {Title: "Dune 2021 2160p WEB-DL HDR", DownloadURL: "https://pt/download/web", Seeders: 100}, + {Title: "Dune 2021 2160p UHD BluRay REMUX HDR", DownloadURL: "https://pt/download/remux", Seeders: 30}, + {Title: "Dune 2021 2160p REMUX HDR CAM", DownloadURL: "https://pt/download/cam", Seeders: 200}, + } + + got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) + if len(got) != 1 || got[0].Download != "https://pt/download/remux" { + t.Fatalf("selected %#v, want filtered remux", got) + } +} diff --git a/internal/service/subscription_candidate_select.go b/internal/service/subscription_candidate_select.go index d14fa62..187fa69 100644 --- a/internal/service/subscription_candidate_select.go +++ b/internal/service/subscription_candidate_select.go @@ -29,7 +29,7 @@ func selectPreparedSubscriptionCandidatesWithStats(candidates []siteSearchCandid mediaType := normalizeMediaType(sub.MediaType, sub.Name+" "+sub.Filter, "") if !isSubscriptionSeriesType(mediaType) { // 非洗版订阅成功下载一次即满足,媒体库/下载中已存在则不再重复下载。 - if (sub == nil || !sub.WashEnabled) && local.LocalMediaCount > 0 { + if !subscriptionAllowsWash(sub) && local.LocalMediaCount > 0 { if stats != nil { stats.LocalAlreadySatisfied = true } diff --git a/internal/service/subscription_candidate_selection_test.go b/internal/service/subscription_candidate_selection_test.go index 6377ab9..6fee3bc 100644 --- a/internal/service/subscription_candidate_selection_test.go +++ b/internal/service/subscription_candidate_selection_test.go @@ -1,18 +1,10 @@ package service import ( - "errors" - "net/http" - "net/http/httptest" "strings" - "sync/atomic" "testing" - "time" - - "go.uber.org/zap" "github.com/ShukeBta/MediaStationGo/internal/model" - "github.com/ShukeBta/MediaStationGo/internal/repository" ) func TestSelectSiteSearchCandidatesPrefersSeriesPack(t *testing.T) { @@ -56,12 +48,12 @@ func TestSelectSiteSearchCandidatesQueuesDistinctEpisodesWhenNoPack(t *testing.T func TestSelectSiteSearchCandidatesKeepsMovieSingleBest(t *testing.T) { sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", WashPriority: "seeders"} results := []SearchResult{ - {Title: "Inception 2010 1080p", DownloadURL: "https://pt/download/1080", Seeders: 90}, - {Title: "Inception 2010 2160p", DownloadURL: "https://pt/download/2160", Seeders: 80}, + {Title: "Inception 2010 1080p HDTV", DownloadURL: "https://pt/download/1080-hdtv", Seeders: 900}, + {Title: "Inception 2010 2160p WEB-DL", DownloadURL: "https://pt/download/2160-webdl", Seeders: 80}, } got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) - if len(got) != 1 || got[0].Download != "https://pt/download/1080" { + if len(got) != 1 || got[0].Download != "https://pt/download/2160-webdl" { t.Fatalf("selected %#v, want movie best only", got) } } @@ -197,277 +189,3 @@ func TestDedupeSiteSearchResultsKeepsMatchedSearchKeyword(t *testing.T) { t.Fatalf("stats = %#v, want merged keyword to avoid query mismatch", stats) } } - -func TestSearchSubscriptionSitesStopsAfterRateLimit(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requests.Add(1) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"code":"0","message":"SUCCESS","data":{"total":"0","data":[]}}`)) - })) - defer upstream.Close() - - db := newServiceTestDB(t, &model.Site{}, &model.Setting{}) - repos := repository.New(db) - siteSvc := NewSiteService(zap.NewNop(), repos, "") - limiter := &staticSiteAPIRateLimiter{err: &siteAPIRateLimitError{ - Bucket: "torrent_search_24h", - Limit: 1500, - Window: 24 * time.Hour, - RetryAfter: time.Hour, - }} - siteSvc.apiRateLimiter = limiter - if err := siteSvc.Create(t.Context(), &model.Site{ - Name: "馒头", - Type: "mteam", - URL: upstream.URL, - AuthType: "api_key", - APIKey: "token-123", - Enabled: true, - Timeout: 5, - }); err != nil { - t.Fatal(err) - } - svc := NewSubscriptionService(nil, zap.NewNop(), repos, nil, siteSvc, NewHub(zap.NewNop())) - sub := &model.Subscription{Name: "问心2 自动订阅", Filter: "问心2", MediaType: "tv"} - - _, err := svc.searchSubscriptionSites(t.Context(), sub, []string{"问心2", "问心", "问心2 2023"}) - var limited *siteAPIRateLimitError - if !errors.As(err, &limited) { - t.Fatalf("searchSubscriptionSites error = %v, want siteAPIRateLimitError", err) - } - if limiter.calls != 1 { - t.Fatalf("rate limiter calls = %d, want 1 keyword attempt", limiter.calls) - } - if got := requests.Load(); got != 0 { - t.Fatalf("HTTP requests = %d, want 0 after local rate limit", got) - } -} - -func TestSubscriptionRunAllStopsSweepAfterRateLimit(t *testing.T) { - var requests atomic.Int32 - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requests.Add(1) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"code":"0","message":"SUCCESS","data":{"total":"0","data":[]}}`)) - })) - defer upstream.Close() - - db := newServiceTestDB(t, &model.Site{}, &model.Setting{}, &model.Subscription{}) - repos := repository.New(db) - siteSvc := NewSiteService(zap.NewNop(), repos, "") - limiter := &staticSiteAPIRateLimiter{err: &siteAPIRateLimitError{ - Bucket: "torrent_search_24h", - Limit: 1500, - Window: 24 * time.Hour, - RetryAfter: time.Hour, - }} - siteSvc.apiRateLimiter = limiter - if err := siteSvc.Create(t.Context(), &model.Site{ - Name: "馒头", - Type: "mteam", - URL: upstream.URL, - AuthType: "api_key", - APIKey: "token-123", - Enabled: true, - Timeout: 5, - }); err != nil { - t.Fatal(err) - } - for _, name := range []string{"问心2 自动订阅", "南部档案 自动订阅"} { - sub := &model.Subscription{ - Name: name, - FeedURL: "site-search://search?keyword=" + name, - Filter: name, - Enabled: true, - } - if err := repos.Subscription.Create(t.Context(), sub); err != nil { - t.Fatal(err) - } - } - svc := NewSubscriptionService(nil, zap.NewNop(), repos, nil, siteSvc, NewHub(zap.NewNop())) - - svc.runAll(t.Context()) - if limiter.calls != 1 { - t.Fatalf("rate limiter calls = %d, want sweep to stop after first quota failure", limiter.calls) - } - if got := requests.Load(); got != 0 { - t.Fatalf("HTTP requests = %d, want 0 after local rate limit", got) - } -} - -func TestSubscriptionSiteSearchStopsAfterTransientSiteErrors(t *testing.T) { - for _, errText := range []string{ - `search: Post "https://api.m-team.cc/api/torrent/search": context deadline exceeded`, - `search: Post "https://api.m-team.cc/api/torrent/search": net/http: TLS handshake timeout`, - `search: Post "https://api.m-team.cc/api/torrent/search": unexpected EOF`, - `search: Post "https://api.m-team.cc/api/torrent/search": read tcp 127.0.0.1: connection reset by peer`, - } { - if !subscriptionSiteSearchShouldStopOnError(errors.New(errText)) { - t.Fatalf("subscriptionSiteSearchShouldStopOnError(%q) = false, want true", errText) - } - } - if subscriptionSiteSearchShouldStopOnError(errors.New("temporary parser warning: no matching torrent rows")) { - t.Fatal("non-upstream-failure errors should not stop alias search") - } -} - -func TestSelectSiteSearchCandidatesRejectsKeywordOriginWithConflictingYear(t *testing.T) { - sub := &model.Subscription{ - Name: "玩具总动员 5 自动订阅", - Filter: "玩具总动员 5 2026", - MediaType: "movie", - Year: 2026, - } - results := []SearchResult{{ - Title: "Toy Story 4 2019 2160p DSNP WEB-DL", - DownloadURL: "https://pt/download/toy-story-4", - SearchKeyword: "玩具总动员 5", - Seeders: 90, - }} - - got, stats := selectSiteSearchCandidatesWithStats(results, sub, map[string]struct{}{}, LocalAvailability{}) - if len(got) != 0 { - t.Fatalf("selected %#v, want conflicting-year keyword-origin result rejected", got) - } - if stats.QueryMismatch != 1 || stats.Prepared != 0 { - t.Fatalf("stats = %#v, want query mismatch for conflicting year", stats) - } -} - -func TestSelectSiteSearchCandidatesDoesNotWashByDefault(t *testing.T) { - sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", WashPriority: "resolution"} - results := []SearchResult{ - {Title: "Inception 2010 1080p", DownloadURL: "https://pt/download/1080", Seeders: 90}, - {Title: "Inception 2010 2160p", DownloadURL: "https://pt/download/2160", Seeders: 80}, - } - - got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) - if len(got) != 1 || got[0].Download != "https://pt/download/1080" { - t.Fatalf("selected %#v, want seeders best when wash disabled", got) - } -} - -func TestSelectSiteSearchCandidatesAppliesQualityRules(t *testing.T) { - sub := &model.Subscription{ - Name: "Dune 自动订阅", - Filter: "Dune 2021", - MediaType: "movie", - Resolution: "2160p", - Quality: "remux", - Effects: "hdr", - ExcludeWords: "cam,ts", - } - results := []SearchResult{ - {Title: "Dune 2021 2160p WEB-DL HDR", DownloadURL: "https://pt/download/web", Seeders: 100}, - {Title: "Dune 2021 2160p UHD BluRay REMUX HDR", DownloadURL: "https://pt/download/remux", Seeders: 30}, - {Title: "Dune 2021 2160p REMUX HDR CAM", DownloadURL: "https://pt/download/cam", Seeders: 200}, - } - - got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}) - if len(got) != 1 || got[0].Download != "https://pt/download/remux" { - t.Fatalf("selected %#v, want filtered remux", got) - } -} - -func TestSiteSearchKeywordCanUseIMDB(t *testing.T) { - sub := &model.Subscription{Name: "沙丘 自动订阅", Filter: "Dune 2021", SearchMode: "imdb", IMDBID: "tt1160419"} - if got := siteSearchKeyword(sub); got != "tt1160419" { - t.Fatalf("keyword = %q, want imdb id", got) - } -} - -func TestSiteSearchKeywordsIncludeAliasesAndCleanedKeywords(t *testing.T) { - sub := &model.Subscription{ - Name: "真人快打2 自动订阅", - FeedURL: "site-search://search?keyword=%E7%9C%9F%E4%BA%BA%E5%BF%AB%E6%89%932%202026&alias=Mortal%20Kombat%20II%202026", - Filter: "真人快打2 2026", - } - - got := siteSearchKeywords(sub) - for _, want := range []string{"真人快打2 2026", "Mortal Kombat II 2026", "真人快打2", "Mortal Kombat II"} { - if !containsString(got, want) { - t.Fatalf("keywords = %#v, missing %q", got, want) - } - } - if got[0] != "真人快打2 2026" { - t.Fatalf("primary keyword = %q, want feed keyword first", got[0]) - } -} - -func TestSiteSearchKeywordsUseCleanMetadataAliases(t *testing.T) { - sub := &model.Subscription{ - Name: "玩具总动员 4 自动订阅", - Filter: "玩具总动员 4 2019", - OriginalName: "Toy Story 4", - Year: 2019, - } - - got := siteSearchKeywords(sub) - for _, want := range []string{"玩具总动员 4 2019", "Toy Story 4", "Toy Story 4 2019", "玩具总动员 4"} { - if !containsString(got, want) { - t.Fatalf("keywords = %#v, missing %q", got, want) - } - } - for _, unwanted := range []string{"玩具总动员 4 自动订阅", "玩具总动员 4 自动订阅 2019", "玩具总动员 4 2019 2019"} { - if containsString(got, unwanted) { - t.Fatalf("keywords = %#v, should not contain %q", got, unwanted) - } - } -} - -func containsString(values []string, want string) bool { - for _, value := range values { - if value == want { - return true - } - } - return false -} - -func TestStableSiteSearchGUIDIgnoresPrivateTokenChanges(t *testing.T) { - item := SearchResult{ - SiteID: "mteam", - Title: "Some Show S01E01 1080p", - Category: "TV", - Size: 1024, - } - first := stableSiteSearchGUID(item, "https://pt.example/download?id=123&passkey=old") - second := stableSiteSearchGUID(item, "https://pt.example/download?id=123&passkey=new") - if first != second { - t.Fatalf("stableSiteSearchGUID changed with token: %q != %q", first, second) - } - if strings.Contains(first, "passkey") || strings.Contains(first, "old") || strings.Contains(first, "new") { - t.Fatalf("stableSiteSearchGUID leaked private token: %q", first) - } -} - -func TestSelectSiteSearchCandidatesWithStatsExplainsFiltering(t *testing.T) { - sub := &model.Subscription{Name: "Stats Show 自动订阅", Filter: "Stats Show", MediaType: "tv"} - seenItem := SearchResult{Title: "Stats Show S01E02 1080p", DownloadURL: "https://pt/download/seen", Seeders: 50} - seenGUID := stableSiteSearchGUID(seenItem, seenItem.DownloadURL) - results := []SearchResult{ - {Title: "Different Show S01E01 1080p", DownloadURL: "https://pt/download/wrong", Seeders: 90}, - {Title: "Stats Show S01E01 CAM", DownloadURL: "https://pt/download/cam", Seeders: 80}, - {Title: "Stats Show S01E02 1080p", Seeders: 70}, - seenItem, - {Title: "Stats Show S01E03 1080p", DownloadURL: "https://pt/download/right", Seeders: 60}, - } - - got, stats := selectSiteSearchCandidatesWithStats(results, sub, map[string]struct{}{seenGUID: {}}, LocalAvailability{}) - if len(got) != 1 || got[0].Download != "https://pt/download/right" { - t.Fatalf("selected %#v, want only unfiltered candidate", got) - } - if stats.Total != 5 || - stats.QueryMismatch != 1 || - stats.RuleMismatch != 1 || - stats.MissingDownload != 1 || - stats.Seen != 1 || - stats.Prepared != 1 || - stats.Selected != 1 { - t.Fatalf("unexpected stats: %#v", stats) - } - if len(stats.QueryMismatchExamples) != 1 || stats.QueryMismatchExamples[0] != "Different Show S01E01 1080p" { - t.Fatalf("query mismatch examples = %#v", stats.QueryMismatchExamples) - } -} diff --git a/internal/service/subscription_downloader_test.go b/internal/service/subscription_downloader_test.go new file mode 100644 index 0000000..f273cfa --- /dev/null +++ b/internal/service/subscription_downloader_test.go @@ -0,0 +1,88 @@ +package service + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/model" + "github.com/ShukeBta/MediaStationGo/internal/repository" +) + +func TestSubscriptionRunOneDoesNotUseDeletedDownloader(t *testing.T) { + rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/rss+xml") + _, _ = w.Write([]byte(` + + + Deleted Downloader Show S01E01 1080p + deleted-downloader-episode-1 + magnet:?xt=urn:btih:cccccccccccccccccccccccccccccccccccccccc&dn=Deleted+Downloader+Show+S01E01 + +`)) + })) + defer rss.Close() + + var qbCalls int32 + qb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&qbCalls, 1) + switch r.URL.Path { + case "/api/v2/auth/login": + _, _ = w.Write([]byte("Ok.")) + case "/api/v2/torrents/info": + _, _ = w.Write([]byte(`[]`)) + case "/api/v2/torrents/add": + _, _ = w.Write([]byte("Ok.")) + default: + http.NotFound(w, r) + } + })) + defer qb.Close() + + db := newServiceTestDB(t, &model.Subscription{}, &model.Setting{}, &model.DownloadTask{}, &model.Media{}, &model.DownloadClient{}) + repos := repository.New(db) + client := &model.DownloadClient{Name: "qB deleted", Type: "qbittorrent", Host: qb.URL, Username: "admin", Password: "admin", IsDefault: true, Enabled: true} + if err := repos.DownloadClient.Create(t.Context(), client); err != nil { + t.Fatal(err) + } + if err := repos.Setting.Set(t.Context(), settingDownloadClientsManaged, "true"); err != nil { + t.Fatal(err) + } + if err := repos.DownloadClient.Delete(t.Context(), client.ID); err != nil { + t.Fatal(err) + } + + downloads := NewDownloadService(zap.NewNop(), repos, NewHub(zap.NewNop()), nil) + svc := NewSubscriptionService(nil, zap.NewNop(), repos, downloads, nil, NewHub(zap.NewNop())) + sub := &model.Subscription{ + Name: "Deleted Downloader Show 自动订阅", + FeedURL: rss.URL, + Filter: "Deleted Downloader Show", + MediaType: "tv", + SavePath: "/downloads/tv", + } + if err := repos.Subscription.Create(t.Context(), sub); err != nil { + t.Fatal(err) + } + + queued, err := svc.runOne(t.Context(), sub) + if err != nil { + t.Fatal(err) + } + if queued != 0 { + t.Fatalf("queued = %d, want 0 when default downloader was deleted", queued) + } + if got := atomic.LoadInt32(&qbCalls); got != 0 { + t.Fatalf("qB calls = %d, want 0 after downloader deletion", got) + } + rows, err := repos.Download.List(t.Context()) + if err != nil { + t.Fatal(err) + } + if len(rows) != 0 { + t.Fatalf("download rows = %d, want 0", len(rows)) + } +} diff --git a/internal/service/subscription_exclude_rules.go b/internal/service/subscription_exclude_rules.go index b5a9b80..90cd4fe 100644 --- a/internal/service/subscription_exclude_rules.go +++ b/internal/service/subscription_exclude_rules.go @@ -7,14 +7,12 @@ import ( // defaultExcludeWords 是默认过滤的「垃圾版本」排除清单,对所有订阅生效。 // 拉丁词在 containsAnyExcludeToken 里按词边界匹配以避免子串误伤。 -const defaultExcludeWords = "cam,ts,tc,telesync,telecine,hdcam,hdts,枪版,抢先,抢鲜,预告,trailer,sample" +const defaultExcludeWords = "cam,ts,tc,telesync,telecine,hdcam,hdts,枪版,抢先,抢鲜,预告,trailer,sample,hr,h&r,hit and run,hit&run,hit-and-run,禁转,禁止转载,禁下,禁止下载" // defaultCompatibilityExcludeWords 是面向自动订阅的兼容性默认排除清单。 // 仅在用户未真正自定义排除词时启用,避免默认命中 DoVi/H.265/10bit/杜比音轨等版本。 const defaultCompatibilityExcludeWords = "dovi,dv,dolby vision,dolby,杜比视界,杜比,h265,h.265,h-265,h_265,h 265,hevc,x265,10bit,10-bit,10 bit,hi10p,atmos,truehd,ddp,dd+,eac3" -const legacyFrontendExcludeWords = "cam,ts,tc,枪版" - func containsAnyToken(titleFold, csv string) bool { for _, token := range strings.FieldsFunc(strings.ToLower(csv), func(r rune) bool { return r == ',' || r == '/' || r == '|' || r == ';' || r == ',' diff --git a/internal/service/subscription_rss_defaults_test.go b/internal/service/subscription_rss_defaults_test.go new file mode 100644 index 0000000..c4e15d8 --- /dev/null +++ b/internal/service/subscription_rss_defaults_test.go @@ -0,0 +1,35 @@ +package service + +import ( + "testing" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func TestSelectRSSSubscriptionCandidatesDefaultKeepsOnlyBestWebDLPerEpisode(t *testing.T) { + sub := &model.Subscription{Name: "House of the Dragon 自动订阅", Filter: "House of the Dragon", MediaType: "tv"} + items := []rssItem{ + rssSubscriptionItem("House of the Dragon S03E01 1080p WEB-DL H264 AAC", "https://pt/download/e01-1080"), + rssSubscriptionItem("House of the Dragon S03E01 2160p WEB-DL H264 AAC", "https://pt/download/e01-2160"), + rssSubscriptionItem("House of the Dragon S03E01 720p WEBRip H264 AAC", "https://pt/download/e01-720"), + rssSubscriptionItem("House of the Dragon S03E02 1080p HDTV H264 AAC", "https://pt/download/e02-hdtv"), + rssSubscriptionItem("House of the Dragon S03E02 1080p WEB-DL H264 AAC", "https://pt/download/e02-webdl"), + } + + got := selectRSSSubscriptionCandidates(items, sub, compileFilter(sub.Filter), nil, LocalAvailability{}) + if len(got) != 2 { + t.Fatalf("selected %d candidates, want one best release per episode", len(got)) + } + if got[0].Download != "https://pt/download/e01-2160" { + t.Fatalf("episode 1 selected %q, want 2160p WEB-DL", got[0].Download) + } + if got[1].Download != "https://pt/download/e02-webdl" { + t.Fatalf("episode 2 selected %q, want WEB-DL over HDTV", got[1].Download) + } +} + +func rssSubscriptionItem(title, download string) rssItem { + item := rssItem{Title: title, Link: download, GUID: download} + item.Enclosure.URL = download + return item +} diff --git a/internal/service/subscription_rss_run.go b/internal/service/subscription_rss_run.go index 7580d4b..e9158cb 100644 --- a/internal/service/subscription_rss_run.go +++ b/internal/service/subscription_rss_run.go @@ -60,7 +60,7 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio seenSet: seenSet, availability: mergeLocalAvailability(SubscriptionLocalAvailability(ctx, s.repo, sub), s.pendingDownloadAvailability(ctx, sub)), availabilityQuery: availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)), - washOff: !sub.WashEnabled, + washOff: !subscriptionAllowsWash(sub), } candidates := selectRSSSubscriptionCandidates(feed.Channel.Items, sub, filter, runState.seenSet, runState.availability) queued = s.enqueueRSSSubscriptionCandidates(ctx, sub, candidates, runState) @@ -94,7 +94,7 @@ func (s *SubscriptionService) enqueueRSSSubscriptionCandidate(ctx context.Contex Overview: sub.Overview, MediaType: mediaType, MediaCategory: mediaCategory, - AllowExistingLibrary: sub.WashEnabled, + AllowExistingLibrary: subscriptionAllowsWash(sub), }); err != nil { if IsDownloadDedupError(err) { if s.subscriptionCandidateConfirmedAvailable(ctx, sub, candidate) { diff --git a/internal/service/subscription_rules.go b/internal/service/subscription_rules.go index 5d57f21..2fe52af 100644 --- a/internal/service/subscription_rules.go +++ b/internal/service/subscription_rules.go @@ -14,7 +14,7 @@ func matchesSubscriptionRules(sub *model.Subscription, title string) bool { if sub == nil { return true } - if shouldApplyDefaultCompatibilityExcludes(sub.ExcludeWords) && containsAnyExcludeToken(titleFold, defaultCompatibilityExcludeWords) { + if compatibilityExcludes := defaultCompatibilityExcludesForSubscription(sub); compatibilityExcludes != "" && containsAnyExcludeToken(titleFold, compatibilityExcludes) { return false } if sub.ExcludeWords != "" && containsAnyExcludeToken(titleFold, sub.ExcludeWords) { @@ -35,13 +35,42 @@ func matchesSubscriptionRules(sub *model.Subscription, title string) bool { return true } -func shouldApplyDefaultCompatibilityExcludes(excludeWords string) bool { - normalized := normalizeExcludeWords(excludeWords) - return normalized == "" || normalized == normalizeExcludeWords(legacyFrontendExcludeWords) +func defaultCompatibilityExcludesForSubscription(sub *model.Subscription) string { + if sub == nil { + return defaultCompatibilityExcludeWords + } + requested := strings.ToLower(strings.Join([]string{sub.Effects, sub.Quality}, ",")) + if strings.TrimSpace(requested) == "" { + return defaultCompatibilityExcludeWords + } + tokens := excludeWordTokens(defaultCompatibilityExcludeWords) + out := make([]string, 0, len(tokens)) + for _, token := range tokens { + if token == "" || compatibilityTokenRequested(requested, token) { + continue + } + out = append(out, token) + } + return strings.Join(out, ",") } -func normalizeExcludeWords(csv string) string { - return strings.Join(excludeWordTokens(csv), ",") +func compatibilityTokenRequested(requested, token string) bool { + switch token { + case "dovi", "dv", "dolby vision", "杜比视界": + return containsAnyEffect(requested, "dolby-vision") || containsAnyToken(requested, "dovi,dv,dolby vision,杜比视界") + case "dolby", "杜比": + return containsAnyEffect(requested, "dolby-vision") || containsAnyToken(requested, "dolby,dolby vision,杜比,杜比视界,atmos,dolby atmos,杜比全景声") + case "atmos": + return containsAnyToken(requested, "atmos,dolby atmos,杜比全景声") + case "h265", "h.265", "h-265", "h_265", "h 265", "hevc", "x265": + return containsAnyToken(requested, "h265,h.265,h-265,h_265,h 265,hevc,x265") + case "10bit", "10-bit", "10 bit", "hi10p": + return containsAnyToken(requested, "10bit,10-bit,10 bit,hi10p") + case "truehd", "ddp", "dd+", "eac3": + return containsAnyToken(requested, "truehd,ddp,dd+,eac3") + default: + return containsAnyToken(requested, token) + } } func isSubscriptionSeriesType(mediaType string) bool { @@ -52,3 +81,24 @@ func isSubscriptionSeriesType(mediaType string) bool { return false } } + +func subscriptionAllowsWash(sub *model.Subscription) bool { + if sub == nil || !sub.WashEnabled { + return false + } + return subscriptionHasExplicitUpgradeCriteria(sub) +} + +func subscriptionHasExplicitUpgradeCriteria(sub *model.Subscription) bool { + if sub == nil { + return false + } + if value := strings.TrimSpace(strings.ToLower(sub.Resolution)); value != "" && value != "best" { + return true + } + if value := strings.TrimSpace(strings.ToLower(sub.Quality)); value != "" && value != "best" { + return true + } + return strings.TrimSpace(sub.Effects) != "" || + strings.TrimSpace(sub.ReleaseGroups) != "" +} diff --git a/internal/service/subscription_rules_test.go b/internal/service/subscription_rules_test.go index fc3051c..ebf60aa 100644 --- a/internal/service/subscription_rules_test.go +++ b/internal/service/subscription_rules_test.go @@ -75,6 +75,21 @@ func TestMatchesSubscriptionRulesDefaultExcludesJunkReleases(t *testing.T) { } } +func TestMatchesSubscriptionRulesDefaultExcludesRiskyTorrentLabels(t *testing.T) { + sub := &model.Subscription{} + for _, title := range []string{ + "Some Show S01E01 1080p WEB-DL HR", + "Some Show S01E01 1080p WEB-DL H&R", + "Some Show S01E01 1080p WEB-DL Hit and Run", + "Some Show S01E01 1080p WEB-DL 禁转", + "Some Show S01E01 1080p WEB-DL 禁止下载", + } { + if matchesSubscriptionRules(sub, title) { + t.Errorf("expected default rules to exclude risky torrent label %q", title) + } + } +} + func TestMatchesSubscriptionRulesDefaultExcludesCompatibilityReleases(t *testing.T) { cases := []struct { name string @@ -82,6 +97,7 @@ func TestMatchesSubscriptionRulesDefaultExcludesCompatibilityReleases(t *testing }{ {name: "empty exclude words", sub: &model.Subscription{}}, {name: "legacy frontend defaults", sub: &model.Subscription{ExcludeWords: "cam,ts,tc,枪版"}}, + {name: "custom exclude words", sub: &model.Subscription{ExcludeWords: "官中,无字幕"}}, } for _, c := range cases { @@ -89,6 +105,7 @@ func TestMatchesSubscriptionRulesDefaultExcludesCompatibilityReleases(t *testing for _, title := range []string{ "Some Movie 2024 2160p DoVi H.265 10bit", "Some Movie 2024 2160p H-265", + "Some Movie 2024 2160p H 265 10 bit", "Some Movie 2024 1080p HEVC", "Some Movie 2024 1080p x265", "Some Movie 2024 2160p Dolby Vision Atmos", @@ -103,17 +120,38 @@ func TestMatchesSubscriptionRulesDefaultExcludesCompatibilityReleases(t *testing } } -func TestMatchesSubscriptionRulesCustomExcludeWordsCanOptOutOfCompatibilityDefaults(t *testing.T) { +func TestMatchesSubscriptionRulesCustomExcludeWordsKeepCompatibilityDefaults(t *testing.T) { sub := &model.Subscription{ExcludeWords: "sample"} title := "Some Movie 2024 2160p DoVi HEVC 10bit" - if !matchesSubscriptionRules(sub, title) { - t.Fatalf("custom exclude words should not force default compatibility excludes for %q", title) + if matchesSubscriptionRules(sub, title) { + t.Fatalf("custom exclude words should keep default compatibility excludes for %q", title) } if matchesSubscriptionRules(sub, "Some Movie 2024 1080p SAMPLE") { t.Fatal("custom exclude words should still apply") } } +func TestMatchesSubscriptionRulesExplicitEffectsCanRequestCompatibilityFormats(t *testing.T) { + sub := &model.Subscription{Effects: "dolby vision"} + title := "Some Movie 2024 2160p DoVi WEB-DL" + if !matchesSubscriptionRules(sub, title) { + t.Fatalf("explicit requested effects should allow compatibility format for %q", title) + } +} + +func TestMatchesSubscriptionRulesExplicitAtmosDoesNotAllowOtherCompatibilityFormats(t *testing.T) { + sub := &model.Subscription{Effects: "atmos"} + if !matchesSubscriptionRules(sub, "Some Movie 2024 1080p WEB-DL Atmos") { + t.Fatal("explicit atmos should allow an Atmos-only release") + } + if !matchesSubscriptionRules(sub, "Some Movie 2024 1080p WEB-DL Dolby Atmos") { + t.Fatal("explicit atmos should allow Dolby Atmos wording") + } + if matchesSubscriptionRules(sub, "Some Movie 2024 2160p WEB-DL HEVC 10bit DoVi Atmos") { + t.Fatal("explicit atmos should not also allow DoVi/HEVC/10bit") + } +} + func TestMatchesSubscriptionRulesWordBoundaryAvoidsFalsePositives(t *testing.T) { sub := &model.Subscription{} // "ts" / "cam" / "tc" 作为子串出现在合法标题里时不应被默认排除误伤。 @@ -142,8 +180,8 @@ func TestSelectSiteSearchCandidatesSkipsExistingMovieWhenNotWashing(t *testing.T } } -func TestSelectSiteSearchCandidatesAllowsMovieWashUpgrade(t *testing.T) { - sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", WashEnabled: true, WashPriority: "resolution"} +func TestSelectSiteSearchCandidatesAllowsMovieWashUpgradeWithExplicitCriteria(t *testing.T) { + sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", Resolution: "2160p", WashEnabled: true, WashPriority: "resolution"} results := []SearchResult{ {Title: "Inception 2010 2160p REMUX", DownloadURL: "https://pt/download/2160", Seeders: 80}, {Title: "Inception 2010 1080p WEB-DL", DownloadURL: "https://pt/download/1080", Seeders: 200}, diff --git a/internal/service/subscription_run_wash_test.go b/internal/service/subscription_run_wash_test.go index 0e9642a..321c1f4 100644 --- a/internal/service/subscription_run_wash_test.go +++ b/internal/service/subscription_run_wash_test.go @@ -78,6 +78,8 @@ func TestSubscriptionRunOneRSSWashQueuesOnlyBestMovieVariant(t *testing.T) { FeedURL: rss.URL, Filter: "Dune 2021", MediaType: "movie", + Resolution: "2160p", + Quality: "remux", WashEnabled: true, WashPriority: "resolution", SavePath: "/downloads/movies", @@ -100,29 +102,52 @@ func TestSubscriptionRunOneRSSWashQueuesOnlyBestMovieVariant(t *testing.T) { } } -func TestSubscriptionRunOneDoesNotUseDeletedDownloader(t *testing.T) { +func TestSubscriptionRunOneRSSDefaultQueuesOnlyBestEpisodeVariant(t *testing.T) { rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/rss+xml") _, _ = w.Write([]byte(` - Deleted Downloader Show S01E01 1080p - deleted-downloader-episode-1 - magnet:?xt=urn:btih:cccccccccccccccccccccccccccccccccccccccc&dn=Deleted+Downloader+Show+S01E01 + House of the Dragon S03E01 2160p BluRay H264 AAC + hotd-e01-bluray + magnet:?xt=urn:btih:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&dn=House+of+the+Dragon+S03E01+2160p+BluRay + + + House of the Dragon S03E01 1080p WEB-DL H264 AAC + hotd-e01-webdl + magnet:?xt=urn:btih:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb&dn=House+of+the+Dragon+S03E01+1080p+WEB-DL + + + House of the Dragon S03E01 720p HDTV H264 AAC + hotd-e01-hdtv + magnet:?xt=urn:btih:cccccccccccccccccccccccccccccccccccccccc&dn=House+of+the+Dragon+S03E01+720p+HDTV `)) })) defer rss.Close() - var qbCalls int32 + var addCalls int32 + var addedURLs []string + addedHashes := make([]string, 0, 3) qb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atomic.AddInt32(&qbCalls, 1) switch r.URL.Path { case "/api/v2/auth/login": _, _ = w.Write([]byte("Ok.")) case "/api/v2/torrents/info": - _, _ = w.Write([]byte(`[]`)) + if len(addedHashes) == 0 { + _, _ = w.Write([]byte(`[]`)) + return + } + var items []string + for _, hash := range addedHashes { + items = append(items, `{"hash":"`+hash+`","name":"House of the Dragon S03E01","state":"downloading","progress":0.1}`) + } + _, _ = w.Write([]byte(`[` + strings.Join(items, ",") + `]`)) case "/api/v2/torrents/add": + call := atomic.AddInt32(&addCalls, 1) + _ = r.ParseMultipartForm(10 << 20) + addedURLs = append(addedURLs, r.FormValue("urls")) + addedHashes = append(addedHashes, strings.Repeat(fmt.Sprintf("%x", call), 40)) _, _ = w.Write([]byte("Ok.")) default: http.NotFound(w, r) @@ -132,45 +157,111 @@ func TestSubscriptionRunOneDoesNotUseDeletedDownloader(t *testing.T) { db := newServiceTestDB(t, &model.Subscription{}, &model.Setting{}, &model.DownloadTask{}, &model.Media{}, &model.DownloadClient{}) repos := repository.New(db) - client := &model.DownloadClient{Name: "qB deleted", Type: "qbittorrent", Host: qb.URL, Username: "admin", Password: "admin", IsDefault: true, Enabled: true} - if err := repos.DownloadClient.Create(t.Context(), client); err != nil { - t.Fatal(err) - } - if err := repos.Setting.Set(t.Context(), settingDownloadClientsManaged, "true"); err != nil { - t.Fatal(err) - } - if err := repos.DownloadClient.Delete(t.Context(), client.ID); err != nil { - t.Fatal(err) - } - + configureTestDefaultQB(t, repos, qb.URL) downloads := NewDownloadService(zap.NewNop(), repos, NewHub(zap.NewNop()), nil) svc := NewSubscriptionService(nil, zap.NewNop(), repos, downloads, nil, NewHub(zap.NewNop())) + sub := &model.Subscription{ - Name: "Deleted Downloader Show 自动订阅", + Name: "House of the Dragon 自动订阅", FeedURL: rss.URL, - Filter: "Deleted Downloader Show", + Filter: "House of the Dragon", MediaType: "tv", SavePath: "/downloads/tv", } if err := repos.Subscription.Create(t.Context(), sub); err != nil { t.Fatal(err) } - queued, err := svc.runOne(t.Context(), sub) if err != nil { t.Fatal(err) } - if queued != 0 { - t.Fatalf("queued = %d, want 0 when default downloader was deleted", queued) + if queued != 1 { + t.Fatalf("queued = %d, want one best episode variant", queued) } - if got := atomic.LoadInt32(&qbCalls); got != 0 { - t.Fatalf("qB calls = %d, want 0 after downloader deletion", got) + if got := atomic.LoadInt32(&addCalls); got != 1 { + t.Fatalf("qb add calls = %d, want 1", got) } - rows, err := repos.Download.List(t.Context()) + if len(addedURLs) != 1 || !strings.Contains(addedURLs[0], "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") { + t.Fatalf("added %#v, want 1080p WEB-DL variant only", addedURLs) + } +} + +func TestSubscriptionRunOneRSSCustomExcludeStillSkipsIncompatibleVariants(t *testing.T) { + rss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/rss+xml") + _, _ = w.Write([]byte(` + + + House of the Dragon S03E01 2160p WEB-DL HEVC 10bit DoVi Atmos + hotd-e01-dovi + magnet:?xt=urn:btih:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&dn=House+of+the+Dragon+S03E01+2160p+DoVi + + + House of the Dragon S03E01 1080p WEB-DL H264 AAC + hotd-e01-webdl + magnet:?xt=urn:btih:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb&dn=House+of+the+Dragon+S03E01+1080p+WEB-DL + +`)) + })) + defer rss.Close() + + var addCalls int32 + var addedURLs []string + addedHashes := make([]string, 0, 2) + qb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v2/auth/login": + _, _ = w.Write([]byte("Ok.")) + case "/api/v2/torrents/info": + if len(addedHashes) == 0 { + _, _ = w.Write([]byte(`[]`)) + return + } + var items []string + for _, hash := range addedHashes { + items = append(items, `{"hash":"`+hash+`","name":"House of the Dragon S03E01","state":"downloading","progress":0.1}`) + } + _, _ = w.Write([]byte(`[` + strings.Join(items, ",") + `]`)) + case "/api/v2/torrents/add": + call := atomic.AddInt32(&addCalls, 1) + _ = r.ParseMultipartForm(10 << 20) + addedURLs = append(addedURLs, r.FormValue("urls")) + addedHashes = append(addedHashes, strings.Repeat(fmt.Sprintf("%x", call), 40)) + _, _ = w.Write([]byte("Ok.")) + default: + http.NotFound(w, r) + } + })) + defer qb.Close() + + db := newServiceTestDB(t, &model.Subscription{}, &model.Setting{}, &model.DownloadTask{}, &model.Media{}, &model.DownloadClient{}) + repos := repository.New(db) + configureTestDefaultQB(t, repos, qb.URL) + downloads := NewDownloadService(zap.NewNop(), repos, NewHub(zap.NewNop()), nil) + svc := NewSubscriptionService(nil, zap.NewNop(), repos, downloads, nil, NewHub(zap.NewNop())) + + sub := &model.Subscription{ + Name: "House of the Dragon 自动订阅", + FeedURL: rss.URL, + Filter: "House of the Dragon", + MediaType: "tv", + SavePath: "/downloads/tv", + ExcludeWords: "官中,无字幕", + } + if err := repos.Subscription.Create(t.Context(), sub); err != nil { + t.Fatal(err) + } + queued, err := svc.runOne(t.Context(), sub) if err != nil { t.Fatal(err) } - if len(rows) != 0 { - t.Fatalf("download rows = %d, want 0", len(rows)) + if queued != 1 { + t.Fatalf("queued = %d, want one compatible WEB-DL variant", queued) + } + if got := atomic.LoadInt32(&addCalls); got != 1 { + t.Fatalf("qb add calls = %d, want 1", got) + } + if len(addedURLs) != 1 || !strings.Contains(addedURLs[0], "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") { + t.Fatalf("added %#v, want compatible 1080p WEB-DL only", addedURLs) } } diff --git a/internal/service/subscription_score.go b/internal/service/subscription_score.go index 3a62be8..87090a6 100644 --- a/internal/service/subscription_score.go +++ b/internal/service/subscription_score.go @@ -7,12 +7,25 @@ import ( "github.com/ShukeBta/MediaStationGo/internal/model" ) +var ( + dolbyVisionTokenRE = regexp.MustCompile(`\bdv\b`) + webDLTokenRE = regexp.MustCompile(`\bweb[\s._-]?dl\b`) + webRipTokenRE = regexp.MustCompile(`\bweb[\s._-]?rip\b`) + bluRayTokenRE = regexp.MustCompile(`\b(?:blu[\s._-]?ray|bdrip|bdremux|uhd[\s._-]?blu[\s._-]?ray)\b`) +) + +const ( + defaultSubscriptionFreePromotionScore = 50_000 + washSubscriptionFreePromotionScore = 25 +) + func subscriptionCandidateScore(sub *model.Subscription, item SearchResult) int { title := strings.ToLower(subscriptionSearchResultText(item)) score := item.Seeders - if sub == nil || !sub.WashEnabled { + if !subscriptionAllowsWash(sub) { + score += detectDefaultSubscriptionQualityScore(title)*1_000_000 + detectResolutionScore(title)*100_000 if item.Free { - score += 25 + score += defaultSubscriptionFreePromotionScore } return score } @@ -37,7 +50,7 @@ func subscriptionCandidateScore(sub *model.Subscription, item SearchResult) int score += resolutionScore*500 + qualityScore*300 + effectScore*150 } if item.Free { - score += 25 + score += washSubscriptionFreePromotionScore } return score } @@ -52,7 +65,7 @@ func containsAnyEffect(titleFold, csv string) bool { } switch token { case "dolby-vision", "dolby vision", "dv": - if strings.Contains(titleFold, "dolby vision") || strings.Contains(titleFold, "dovi") || regexp.MustCompile(`\bdv\b`).MatchString(titleFold) { + if strings.Contains(titleFold, "dolby vision") || strings.Contains(titleFold, "dovi") || dolbyVisionTokenRE.MatchString(titleFold) { return true } default: @@ -80,9 +93,11 @@ func titleMatchesResolution(titleFold, resolution string) bool { func titleMatchesQuality(titleFold, quality string) bool { switch strings.ToLower(strings.TrimSpace(quality)) { case "webdl", "web-dl": - return strings.Contains(titleFold, "web-dl") || strings.Contains(titleFold, "webdl") + return webDLTokenRE.MatchString(titleFold) + case "webrip", "web-rip": + return webRipTokenRE.MatchString(titleFold) case "bluray", "blu-ray": - return strings.Contains(titleFold, "bluray") || strings.Contains(titleFold, "blu-ray") || strings.Contains(titleFold, "bdrip") + return bluRayTokenRE.MatchString(titleFold) case "remux": return strings.Contains(titleFold, "remux") case "hdtv": @@ -92,6 +107,21 @@ func titleMatchesQuality(titleFold, quality string) bool { } } +func detectDefaultSubscriptionQualityScore(titleFold string) int { + switch { + case titleMatchesQuality(titleFold, "web-dl"): + return 5 + case titleMatchesQuality(titleFold, "web-rip"): + return 4 + case titleMatchesQuality(titleFold, "bluray"), titleMatchesQuality(titleFold, "remux"): + return 3 + case titleMatchesQuality(titleFold, "hdtv"): + return 2 + default: + return 1 + } +} + func detectResolutionScore(titleFold string) int { switch { case titleMatchesResolution(titleFold, "2160p"): diff --git a/internal/service/subscription_site_search_enqueue.go b/internal/service/subscription_site_search_enqueue.go index e002a10..8ff6a78 100644 --- a/internal/service/subscription_site_search_enqueue.go +++ b/internal/service/subscription_site_search_enqueue.go @@ -68,7 +68,7 @@ func (s *SubscriptionService) enqueueSiteSearchCandidate(ctx context.Context, su MediaType: mediaType, MediaCategory: mediaCategory, SourceCategory: item.Category, - AllowExistingLibrary: sub.WashEnabled, + AllowExistingLibrary: subscriptionAllowsWash(sub), }); err != nil { if IsDownloadDedupError(err) { if s.subscriptionCandidateConfirmedAvailable(ctx, sub, candidate) { diff --git a/internal/service/subscription_site_search_keywords_test.go b/internal/service/subscription_site_search_keywords_test.go new file mode 100644 index 0000000..3b02d8e --- /dev/null +++ b/internal/service/subscription_site_search_keywords_test.go @@ -0,0 +1,110 @@ +package service + +import ( + "strings" + "testing" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func TestSiteSearchKeywordCanUseIMDB(t *testing.T) { + sub := &model.Subscription{Name: "沙丘 自动订阅", Filter: "Dune 2021", SearchMode: "imdb", IMDBID: "tt1160419"} + if got := siteSearchKeyword(sub); got != "tt1160419" { + t.Fatalf("keyword = %q, want imdb id", got) + } +} + +func TestSiteSearchKeywordsIncludeAliasesAndCleanedKeywords(t *testing.T) { + sub := &model.Subscription{ + Name: "真人快打2 自动订阅", + FeedURL: "site-search://search?keyword=%E7%9C%9F%E4%BA%BA%E5%BF%AB%E6%89%932%202026&alias=Mortal%20Kombat%20II%202026", + Filter: "真人快打2 2026", + } + + got := siteSearchKeywords(sub) + for _, want := range []string{"真人快打2 2026", "Mortal Kombat II 2026", "真人快打2", "Mortal Kombat II"} { + if !containsString(got, want) { + t.Fatalf("keywords = %#v, missing %q", got, want) + } + } + if got[0] != "真人快打2 2026" { + t.Fatalf("primary keyword = %q, want feed keyword first", got[0]) + } +} + +func TestSiteSearchKeywordsUseCleanMetadataAliases(t *testing.T) { + sub := &model.Subscription{ + Name: "玩具总动员 4 自动订阅", + Filter: "玩具总动员 4 2019", + OriginalName: "Toy Story 4", + Year: 2019, + } + + got := siteSearchKeywords(sub) + for _, want := range []string{"玩具总动员 4 2019", "Toy Story 4", "Toy Story 4 2019", "玩具总动员 4"} { + if !containsString(got, want) { + t.Fatalf("keywords = %#v, missing %q", got, want) + } + } + for _, unwanted := range []string{"玩具总动员 4 自动订阅", "玩具总动员 4 自动订阅 2019", "玩具总动员 4 2019 2019"} { + if containsString(got, unwanted) { + t.Fatalf("keywords = %#v, should not contain %q", got, unwanted) + } + } +} + +func TestStableSiteSearchGUIDIgnoresPrivateTokenChanges(t *testing.T) { + item := SearchResult{ + SiteID: "mteam", + Title: "Some Show S01E01 1080p", + Category: "TV", + Size: 1024, + } + first := stableSiteSearchGUID(item, "https://pt.example/download?id=123&passkey=old") + second := stableSiteSearchGUID(item, "https://pt.example/download?id=123&passkey=new") + if first != second { + t.Fatalf("stableSiteSearchGUID changed with token: %q != %q", first, second) + } + if strings.Contains(first, "passkey") || strings.Contains(first, "old") || strings.Contains(first, "new") { + t.Fatalf("stableSiteSearchGUID leaked private token: %q", first) + } +} + +func TestSelectSiteSearchCandidatesWithStatsExplainsFiltering(t *testing.T) { + sub := &model.Subscription{Name: "Stats Show 自动订阅", Filter: "Stats Show", MediaType: "tv"} + seenItem := SearchResult{Title: "Stats Show S01E02 1080p", DownloadURL: "https://pt/download/seen", Seeders: 50} + seenGUID := stableSiteSearchGUID(seenItem, seenItem.DownloadURL) + results := []SearchResult{ + {Title: "Different Show S01E01 1080p", DownloadURL: "https://pt/download/wrong", Seeders: 90}, + {Title: "Stats Show S01E01 CAM", DownloadURL: "https://pt/download/cam", Seeders: 80}, + {Title: "Stats Show S01E02 1080p", Seeders: 70}, + seenItem, + {Title: "Stats Show S01E03 1080p", DownloadURL: "https://pt/download/right", Seeders: 60}, + } + + got, stats := selectSiteSearchCandidatesWithStats(results, sub, map[string]struct{}{seenGUID: {}}, LocalAvailability{}) + if len(got) != 1 || got[0].Download != "https://pt/download/right" { + t.Fatalf("selected %#v, want only unfiltered candidate", got) + } + if stats.Total != 5 || + stats.QueryMismatch != 1 || + stats.RuleMismatch != 1 || + stats.MissingDownload != 1 || + stats.Seen != 1 || + stats.Prepared != 1 || + stats.Selected != 1 { + t.Fatalf("unexpected stats: %#v", stats) + } + if len(stats.QueryMismatchExamples) != 1 || stats.QueryMismatchExamples[0] != "Different Show S01E01 1080p" { + t.Fatalf("query mismatch examples = %#v", stats.QueryMismatchExamples) + } +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/internal/service/subscription_site_search_limit_test.go b/internal/service/subscription_site_search_limit_test.go new file mode 100644 index 0000000..38519e4 --- /dev/null +++ b/internal/service/subscription_site_search_limit_test.go @@ -0,0 +1,129 @@ +package service + +import ( + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/model" + "github.com/ShukeBta/MediaStationGo/internal/repository" +) + +func TestSearchSubscriptionSitesStopsAfterRateLimit(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"code":"0","message":"SUCCESS","data":{"total":"0","data":[]}}`)) + })) + defer upstream.Close() + + db := newServiceTestDB(t, &model.Site{}, &model.Setting{}) + repos := repository.New(db) + siteSvc := NewSiteService(zap.NewNop(), repos, "") + limiter := &staticSiteAPIRateLimiter{err: &siteAPIRateLimitError{ + Bucket: "torrent_search_24h", + Limit: 1500, + Window: 24 * time.Hour, + RetryAfter: time.Hour, + }} + siteSvc.apiRateLimiter = limiter + if err := siteSvc.Create(t.Context(), &model.Site{ + Name: "馒头", + Type: "mteam", + URL: upstream.URL, + AuthType: "api_key", + APIKey: "token-123", + Enabled: true, + Timeout: 5, + }); err != nil { + t.Fatal(err) + } + svc := NewSubscriptionService(nil, zap.NewNop(), repos, nil, siteSvc, NewHub(zap.NewNop())) + sub := &model.Subscription{Name: "问心2 自动订阅", Filter: "问心2", MediaType: "tv"} + + _, err := svc.searchSubscriptionSites(t.Context(), sub, []string{"问心2", "问心", "问心2 2023"}) + var limited *siteAPIRateLimitError + if !errors.As(err, &limited) { + t.Fatalf("searchSubscriptionSites error = %v, want siteAPIRateLimitError", err) + } + if limiter.calls != 1 { + t.Fatalf("rate limiter calls = %d, want 1 keyword attempt", limiter.calls) + } + if got := requests.Load(); got != 0 { + t.Fatalf("HTTP requests = %d, want 0 after local rate limit", got) + } +} + +func TestSubscriptionRunAllStopsSweepAfterRateLimit(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"code":"0","message":"SUCCESS","data":{"total":"0","data":[]}}`)) + })) + defer upstream.Close() + + db := newServiceTestDB(t, &model.Site{}, &model.Setting{}, &model.Subscription{}) + repos := repository.New(db) + siteSvc := NewSiteService(zap.NewNop(), repos, "") + limiter := &staticSiteAPIRateLimiter{err: &siteAPIRateLimitError{ + Bucket: "torrent_search_24h", + Limit: 1500, + Window: 24 * time.Hour, + RetryAfter: time.Hour, + }} + siteSvc.apiRateLimiter = limiter + if err := siteSvc.Create(t.Context(), &model.Site{ + Name: "馒头", + Type: "mteam", + URL: upstream.URL, + AuthType: "api_key", + APIKey: "token-123", + Enabled: true, + Timeout: 5, + }); err != nil { + t.Fatal(err) + } + for _, name := range []string{"问心2 自动订阅", "南部档案 自动订阅"} { + sub := &model.Subscription{ + Name: name, + FeedURL: "site-search://search?keyword=" + name, + Filter: name, + Enabled: true, + } + if err := repos.Subscription.Create(t.Context(), sub); err != nil { + t.Fatal(err) + } + } + svc := NewSubscriptionService(nil, zap.NewNop(), repos, nil, siteSvc, NewHub(zap.NewNop())) + + svc.runAll(t.Context()) + if limiter.calls != 1 { + t.Fatalf("rate limiter calls = %d, want sweep to stop after first quota failure", limiter.calls) + } + if got := requests.Load(); got != 0 { + t.Fatalf("HTTP requests = %d, want 0 after local rate limit", got) + } +} + +func TestSubscriptionSiteSearchStopsAfterTransientSiteErrors(t *testing.T) { + for _, errText := range []string{ + `search: Post "https://api.m-team.cc/api/torrent/search": context deadline exceeded`, + `search: Post "https://api.m-team.cc/api/torrent/search": net/http: TLS handshake timeout`, + `search: Post "https://api.m-team.cc/api/torrent/search": unexpected EOF`, + `search: Post "https://api.m-team.cc/api/torrent/search": read tcp 127.0.0.1: connection reset by peer`, + } { + if !subscriptionSiteSearchShouldStopOnError(errors.New(errText)) { + t.Fatalf("subscriptionSiteSearchShouldStopOnError(%q) = false, want true", errText) + } + } + if subscriptionSiteSearchShouldStopOnError(errors.New("temporary parser warning: no matching torrent rows")) { + t.Fatal("non-upstream-failure errors should not stop alias search") + } +} diff --git a/internal/service/subscription_torrent_rules.go b/internal/service/subscription_torrent_rules.go new file mode 100644 index 0000000..3d790ef --- /dev/null +++ b/internal/service/subscription_torrent_rules.go @@ -0,0 +1,56 @@ +package service + +import ( + "strings" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +const bytesPerGiB = 1024 * 1024 * 1024 + +func matchesSubscriptionTorrentRules(sub *model.Subscription, item SearchResult) bool { + if sub == nil { + return true + } + if sub.MinSeeders > 0 && item.Seeders < sub.MinSeeders { + return false + } + if sub.MaxSeeders > 0 && item.Seeders > sub.MaxSeeders { + return false + } + if !subscriptionSizeInRange(item.Size, sub.MinSizeGB, sub.MaxSizeGB) { + return false + } + if sub.FreeOnly && !subscriptionResultIsFree(item) { + return false + } + return true +} + +func subscriptionSizeInRange(sizeBytes int64, minGB, maxGB float64) bool { + if minGB <= 0 && maxGB <= 0 { + return true + } + if sizeBytes <= 0 { + return false + } + sizeGB := float64(sizeBytes) / bytesPerGiB + if minGB > 0 && sizeGB < minGB { + return false + } + if maxGB > 0 && sizeGB > maxGB { + return false + } + return true +} + +func subscriptionResultIsFree(item SearchResult) bool { + if item.Free { + return true + } + text := strings.ToLower(subscriptionSearchResultText(item)) + return strings.Contains(text, "freeleech") || + strings.Contains(text, "2xfree") || + strings.Contains(text, "免费") || + matchesWordBoundary(text, "free") +} diff --git a/internal/service/subscription_torrent_rules_test.go b/internal/service/subscription_torrent_rules_test.go new file mode 100644 index 0000000..55e9372 --- /dev/null +++ b/internal/service/subscription_torrent_rules_test.go @@ -0,0 +1,59 @@ +package service + +import ( + "testing" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +func TestSelectSiteSearchCandidatesAppliesSeederSizeAndFreeRules(t *testing.T) { + sub := &model.Subscription{ + Name: "Some Show 自动订阅", + Filter: "Some Show", + MediaType: "tv", + MinSeeders: 10, + MaxSeeders: 100, + MinSizeGB: 1, + MaxSizeGB: 8, + FreeOnly: true, + } + results := []SearchResult{ + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/low-seed", Seeders: 3, Size: 2 * bytesPerGiB, Free: true}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/hot", Seeders: 500, Size: 2 * bytesPerGiB, Free: true}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/small", Seeders: 50, Size: bytesPerGiB / 2, Free: true}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/nonfree", Seeders: 50, Size: 2 * bytesPerGiB}, + {Title: "Some Show S01E01 1080p WEB-DL H264 AAC", DownloadURL: "https://pt/download/right", Seeders: 50, Size: 2 * bytesPerGiB, Free: true}, + } + + got, stats := selectSiteSearchCandidatesWithStats(results, sub, nil, LocalAvailability{}) + if len(got) != 1 || got[0].Download != "https://pt/download/right" { + t.Fatalf("selected %#v, want only torrent matching seed/size/free rules", got) + } + if stats.RuleMismatch != 4 || stats.Prepared != 1 || stats.Selected != 1 { + t.Fatalf("stats = %#v, want four rule mismatches and one selected", stats) + } +} + +func TestSubscriptionTorrentRulesRecognizeFreeLabels(t *testing.T) { + sub := &model.Subscription{FreeOnly: true} + for _, item := range []SearchResult{ + {Title: "Some Movie 2026 1080p WEB-DL FREE"}, + {Title: "Some Movie 2026 1080p WEB-DL FreeLeech"}, + {Title: "Some Movie 2026 1080p WEB-DL 免费"}, + {Title: "Some Movie 2026 1080p WEB-DL", Free: true}, + } { + if !matchesSubscriptionTorrentRules(sub, item) { + t.Fatalf("expected free rule to accept %#v", item) + } + } + if matchesSubscriptionTorrentRules(sub, SearchResult{Title: "Some Movie 2026 1080p WEB-DL"}) { + t.Fatal("free-only rule accepted non-free result") + } +} + +func TestSubscriptionTorrentRulesRejectUnknownSizeWhenSizeRangeConfigured(t *testing.T) { + sub := &model.Subscription{MinSizeGB: 1} + if matchesSubscriptionTorrentRules(sub, SearchResult{Title: "Some Movie 2026 1080p WEB-DL"}) { + t.Fatal("size range accepted result without size metadata") + } +} diff --git a/web/src/api/discover.ts b/web/src/api/discover.ts index d5e2dd5..8370cf2 100644 --- a/web/src/api/discover.ts +++ b/web/src/api/discover.ts @@ -33,6 +33,11 @@ export interface DiscoverSection { export interface DiscoverFeedMeta { page: number has_next: boolean + duration_ms?: number + error?: string + warning?: string + stale?: boolean + disabled?: boolean } export interface DiscoverFeedResult { diff --git a/web/src/api/strm.ts b/web/src/api/strm.ts index 8b9ed70..901f955 100644 --- a/web/src/api/strm.ts +++ b/web/src/api/strm.ts @@ -7,6 +7,7 @@ export type GenerateSTRMInput = { enabled?: boolean overwrite?: boolean include_local?: boolean + preserve_tree?: boolean } export type GenerateSTRMResult = { @@ -27,6 +28,18 @@ export type GenerateSTRMResult = { }> } +export type GenerateSTRMTreeInput = { + provider: string + tree_text?: string + paths?: string[] + source_root?: string + output_prefix?: string + output_dir: string + base_url?: string + overwrite?: boolean + cleanup?: boolean +} + export const strmAPI = { set: (mediaID: string, url: string) => api.put(`/media/${mediaID}/strm`, { url }).then((r) => r.data), @@ -37,4 +50,8 @@ export const strmAPI = { api .post('/strm/generate', input, { timeout: BATCH_REQUEST_TIMEOUT }) .then((r) => r.data), + generateFromTree: (input: GenerateSTRMTreeInput) => + api + .post('/strm/generate-from-tree', input, { timeout: BATCH_REQUEST_TIMEOUT }) + .then((r) => r.data), } diff --git a/web/src/api/subscriptions.ts b/web/src/api/subscriptions.ts index 2d1d783..3c9b39d 100644 --- a/web/src/api/subscriptions.ts +++ b/web/src/api/subscriptions.ts @@ -71,6 +71,11 @@ export const subscriptionsAPI = { effects?: string release_groups?: string exclude_words?: string + min_seeders?: number + max_seeders?: number + min_size_gb?: number + max_size_gb?: number + free_only?: boolean wash_enabled?: boolean wash_priority?: string total_episodes?: number diff --git a/web/src/pages/DiscoverPage.tsx b/web/src/pages/DiscoverPage.tsx index fa6e0ea..60d9821 100644 --- a/web/src/pages/DiscoverPage.tsx +++ b/web/src/pages/DiscoverPage.tsx @@ -7,8 +7,10 @@ import { DiscoverEmptySelection, DiscoverHeader, DiscoverResults } from './Disco import { defaultSections, discoverStorageKey, + readCachedDiscoverRows, readSavedSections, serializeSavedSections, + writeCachedDiscoverRow, } from './discoverPageModel' export function DiscoverPage() { @@ -38,8 +40,11 @@ export function DiscoverPage() { const available = new Set(items.map((item) => item.key)) const fallback = defaultSections.filter((key) => available.has(key)) const nextSelected = saved.length > 0 ? saved : fallback + const cached = readCachedDiscoverRows(nextSelected) setSelected(nextSelected) setRowPages(Object.fromEntries(nextSelected.map((key) => [key, 1]))) + setRows(cached.rows) + setRowCanNext(cached.rowCanNext) setSectionsReady(true) }) .catch(() => { @@ -93,14 +98,31 @@ export function DiscoverPage() { .feed([key], page) .then((feed) => { if (cancelled) return - setRows((current) => ({ ...current, [key]: feed.items[key] ?? [] })) - setRowCanNext((current) => ({ ...current, [key]: Boolean(feed.meta[key]?.has_next) })) + const error = feed.meta[key]?.error + const nextItems = feed.items[key] ?? [] + const nextCanNext = Boolean(feed.meta[key]?.has_next) + setRows((current) => { + if (error && nextItems.length === 0 && (current[key]?.length ?? 0) > 0) { + return current + } + return { ...current, [key]: nextItems } + }) + setRowCanNext((current) => { + if (error && nextItems.length === 0 && key in current) { + return current + } + return { ...current, [key]: nextCanNext } + }) + if (!error) { + writeCachedDiscoverRow(key, page, nextItems, nextCanNext) + } + setRowErrors((current) => updateDiscoverRowError(current, key, error)) }) .catch((err) => { if (cancelled) return - const message = err instanceof Error ? err.message : String(err) - setRows((current) => ({ ...current, [key]: [] })) - setRowCanNext((current) => ({ ...current, [key]: false })) + const message = discoverRequestErrorMessage(err) + setRows((current) => ((current[key]?.length ?? 0) > 0 ? current : { ...current, [key]: [] })) + setRowCanNext((current) => (key in current ? current : { ...current, [key]: false })) setRowErrors((current) => ({ ...current, [key]: message })) }) .finally(() => { @@ -191,3 +213,27 @@ export function DiscoverPage() { ) } + +function updateDiscoverRowError( + current: Record, + key: string, + error?: string, +): Record { + if (error) return { ...current, [key]: error } + if (!(key in current)) return current + const next = { ...current } + delete next[key] + return next +} + +function discoverRequestErrorMessage(err: unknown): string { + const raw = err instanceof Error ? err.message : String(err) + const lower = raw.toLowerCase() + if (lower.includes('timeout') || lower.includes('deadline')) { + return '推荐源请求超时,已跳过本次加载' + } + if (lower.includes('network')) { + return '推荐源网络不可用,已跳过本次加载' + } + return '推荐源暂时不可用,已跳过本次加载' +} diff --git a/web/src/pages/DiscoverPageSections.tsx b/web/src/pages/DiscoverPageSections.tsx index 62a4c39..6ffafc4 100644 --- a/web/src/pages/DiscoverPageSections.tsx +++ b/web/src/pages/DiscoverPageSections.tsx @@ -152,9 +152,10 @@ function DiscoverRowErrors({ sectionLabel: SectionLabel }) { return ( -
- -
+
+ +
+

部分推荐源暂不可用,其他已加载内容不受影响。

{Object.entries(rowErrors).map(([key, message]) => (

{sectionLabel(key)}:{message}

))} diff --git a/web/src/pages/StrmGenerateSection.tsx b/web/src/pages/StrmGenerateSection.tsx index 600c37d..66357a5 100644 --- a/web/src/pages/StrmGenerateSection.tsx +++ b/web/src/pages/StrmGenerateSection.tsx @@ -23,6 +23,8 @@ export type StrmGenerateSectionProps = { autoGenerate: boolean savingSettings: boolean overwrite: boolean + includeLocal: boolean + preserveTree: boolean generating: boolean generateResult: GenerateSTRMResult | null playbackStatus: string @@ -36,6 +38,8 @@ export type StrmGenerateSectionProps = { setRedirectProxyEnabled: (value: boolean) => void setAutoGenerate: (value: boolean) => void setOverwrite: (value: boolean) => void + setIncludeLocal: (value: boolean) => void + setPreserveTree: (value: boolean) => void } export function StrmGenerateSection(props: StrmGenerateSectionProps) { diff --git a/web/src/pages/StrmGenerateSectionParts.tsx b/web/src/pages/StrmGenerateSectionParts.tsx index ebe6470..bc45360 100644 --- a/web/src/pages/StrmGenerateSectionParts.tsx +++ b/web/src/pages/StrmGenerateSectionParts.tsx @@ -142,12 +142,16 @@ type StrmGenerateFormProps = Pick< | 'baseURL' | 'outputDir' | 'overwrite' + | 'includeLocal' + | 'preserveTree' | 'generating' | 'onGenerate' | 'setGenerateLibraryID' | 'setBaseURL' | 'setOutputDir' | 'setOverwrite' + | 'setIncludeLocal' + | 'setPreserveTree' > export function StrmGenerateForm({ @@ -156,12 +160,16 @@ export function StrmGenerateForm({ baseURL, outputDir, overwrite, + includeLocal, + preserveTree, generating, onGenerate, setGenerateLibraryID, setBaseURL, setOutputDir, setOverwrite, + setIncludeLocal, + setPreserveTree, }: StrmGenerateFormProps) { return (
@@ -180,16 +188,13 @@ export function StrmGenerateForm({ > 使用当前访问地址 - +
+ + + +
setOutputDir(e.target.value)} @@ -202,6 +207,25 @@ export function StrmGenerateForm({ ) } +type CompactOptionProps = { + checked: boolean + label: string + onChange: (value: boolean) => void +} + +function CompactOption({ checked, label, onChange }: CompactOptionProps) { + return ( + + ) +} + type LibrarySelectProps = { libraries: Library[] value: string diff --git a/web/src/pages/StrmPage.tsx b/web/src/pages/StrmPage.tsx index 1a52df1..6b00fd8 100644 --- a/web/src/pages/StrmPage.tsx +++ b/web/src/pages/StrmPage.tsx @@ -2,6 +2,7 @@ import { StrmAttachSection } from './StrmAttachSection' import { StrmGenerateSection } from './StrmGenerateSection' import { StrmImportSection } from './StrmImportSection' import { StrmPageHeader } from './StrmPageHeader' +import { StrmTreeGenerateSection } from './StrmTreeGenerateSection' import { useStrmPage } from './useStrmPage' // StrmPage exposes the URL-as-file admin tooling backed by the Go server: @@ -28,6 +29,8 @@ export function StrmPage() { autoGenerate={strm.generate.autoGenerate} savingSettings={strm.generate.savingSettings} overwrite={strm.generate.overwrite} + includeLocal={strm.generate.includeLocal} + preserveTree={strm.generate.preserveTree} generating={strm.generate.generating} generateResult={strm.generate.generateResult} playbackStatus={strm.generate.playbackStatus} @@ -41,8 +44,12 @@ export function StrmPage() { setRedirectProxyEnabled={strm.generate.setRedirectProxyEnabled} setAutoGenerate={strm.generate.setAutoGenerate} setOverwrite={strm.generate.setOverwrite} + setIncludeLocal={strm.generate.setIncludeLocal} + setPreserveTree={strm.generate.setPreserveTree} /> + + + +const outputPrefixPresets = [ + '电影/演唱会', + '电影/纪录片', + '电影/动画电影', + '电影/华语电影', + '电影/日韩电影', + '电影/欧美电影', + '电视剧/纪录片', + '电视剧/儿童', + '电视剧/综艺', + '电视剧/国产剧', + '电视剧/日韩剧', + '电视剧/欧美剧', + '动漫/日番', + '动漫/国漫', + '动漫/韩漫', + '动漫/美漫', + '动漫/其他', + '成人', +] + +export function StrmTreeGenerateSection({ + baseURL, + cleanup, + generating, + onGenerate, + onImportTreeFile, + outputDir, + outputPrefix, + overwrite, + pathsText, + provider, + result, + setBaseURL, + setCleanup, + setOutputDir, + setOutputPrefix, + setOverwrite, + setPathsText, + setProvider, + setSourceRoot, + setTreeText, + sourceRoot, + treeText, +}: StrmTreeGenerateSectionProps) { + return ( +
+
+

目录树生成 STRM

+

从网盘目录树或路径列表直接生成 .strm 文件。

+
+ + + setSourceRoot(e.target.value)} + /> + setOutputDir(e.target.value)} + /> + + setOutputPrefix(e.target.value)} + /> + setBaseURL(e.target.value)} + /> + +