From 65d1ad5ae53e96a66902e6b066ac0eb4517fc1ff Mon Sep 17 00:00:00 2001 From: truewhile <62226914+truewhile@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:06:49 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=95=AA=E5=8F=B7=E5=A4=84?= =?UTF-8?q?=E7=90=86=E9=80=BB=E8=BE=91=EF=BC=8C=E6=B7=BB=E5=8A=A0=E5=AF=B9?= =?UTF-8?q?MetaTube=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 优化番号处理逻辑，添加对MetaTube的支持 --- internal/handler/admin_settings.go | 143 ++++ internal/handler/api_config.go | 12 + internal/handler/routes_admin.go | 2 + internal/model/api_config.go | 5 +- internal/service/adult_scraper.go | 240 +++++- internal/service/adult_scraper_config.go | 19 + internal/service/adult_scraper_parse.go | 26 +- .../service/adult_scraper_routing_test.go | 91 +++ internal/service/adult_scraper_test.go | 58 +- internal/service/api_config.go | 5 +- internal/service/api_config_connection.go | 93 ++- internal/service/image_proxy_remote_fetch.go | 26 +- internal/service/manual_scrape_providers.go | 16 +- internal/service/metatube.go | 404 ++++++++++ internal/service/metatube_test.go | 176 +++++ internal/service/metatube_types.go | 92 +++ internal/service/scraper_test.go | 52 +- internal/service/service_builder.go | 2 +- web/src/api/admin.ts | 18 + web/src/api/api_configs.ts | 2 + web/src/components/APIConfigsPanel.tsx | 32 +- web/src/pages/APIConfigsPage.tsx | 54 +- web/src/pages/AdultSettingsPanel.tsx | 703 ++++++++++++++++++ web/src/pages/SettingsPage.tsx | 10 +- web/src/pages/settingsGroupAccess.ts | 85 ++- 25 files changed, 2283 insertions(+), 83 deletions(-) create mode 100644 internal/service/adult_scraper_routing_test.go create mode 100644 internal/service/metatube.go create mode 100644 internal/service/metatube_test.go create mode 100644 internal/service/metatube_types.go create mode 100644 web/src/pages/AdultSettingsPanel.tsx diff --git a/internal/handler/admin_settings.go b/internal/handler/admin_settings.go index 9d2f229..af5f605 100644 --- a/internal/handler/admin_settings.go +++ b/internal/handler/admin_settings.go @@ -1,7 +1,11 @@ package handler import ( + "fmt" + "io" "net/http" + "strings" + "time" "github.com/gin-gonic/gin" @@ -62,6 +66,145 @@ func updateSettingHandler(svc *service.Container) gin.HandlerFunc { } } +type testAdultScraperReq struct { + Engine string `json:"engine"` + ServerURL string `json:"server_url"` + Token string `json:"token"` + JavDBURL string `json:"javdb_url"` + JavBusURL string `json:"javbus_url"` + Cookie string `json:"cookie"` +} + +func testAdultScraperHandler(svc *service.Container) gin.HandlerFunc { + return func(c *gin.Context) { + var req testAdultScraperReq + _ = c.ShouldBindJSON(&req) + + engine := strings.ToLower(strings.TrimSpace(req.Engine)) + if engine == "" { + engine = "metatube" + } + + if engine == "metatube" { + serverURL := strings.TrimSpace(req.ServerURL) + if serverURL == "" { + serverURL, _ = svc.Repo.Setting.Get(c.Request.Context(), "adult.scraper.metatube_server") + } + if serverURL == "" { + serverURL = "http://127.0.0.1:7700" + } + token := strings.TrimSpace(req.Token) + if token == "" { + token, _ = svc.Repo.Setting.Get(c.Request.Context(), "adult.scraper.metatube_token") + } + + client := service.NewMetaTubeProvider(svc.Log) + res, err := client.TestConnection(c.Request.Context(), serverURL, token) + if err != nil { + c.JSON(http.StatusOK, gin.H{ + "success": false, + "latency_ms": res.LatencyMs, + "error": err.Error(), + }) + return + } + c.JSON(http.StatusOK, gin.H{ + "success": res.Success, + "latency_ms": res.LatencyMs, + "providers": res.Providers, + "error": res.Error, + }) + return + } + + // Builtin scraper test + bases := []string{} + if req.JavDBURL != "" { + bases = append(bases, strings.TrimSpace(req.JavDBURL)) + } + if req.JavBusURL != "" { + bases = append(bases, strings.Split(req.JavBusURL, ",")...) + } + if len(bases) == 0 { + if s, _ := svc.Repo.Setting.Get(c.Request.Context(), "adult.scraper.builtin_javdb_url"); s != "" { + bases = append(bases, s) + } + if s, _ := svc.Repo.Setting.Get(c.Request.Context(), "adult.scraper.builtin_javbus_url"); s != "" { + bases = append(bases, strings.Split(s, ",")...) + } + } + if len(bases) == 0 { + bases = []string{"https://javdb.com", "https://javbus.sbs", "https://www.javbus.com"} + } + + cookie := strings.TrimSpace(req.Cookie) + if cookie == "" { + cookie, _ = svc.Repo.Setting.Get(c.Request.Context(), "adult.scraper.builtin_cookie") + } + if !strings.Contains(cookie, "age=") { + cookie = cookie + "; age=verified" + } + + httpClient := service.NewExternalHTTPClient(8 * time.Second) + start := time.Now() + var lastErr error + success := false + for _, b := range bases { + b = strings.TrimRight(strings.TrimSpace(b), "/") + if b == "" { + continue + } + if !strings.HasPrefix(b, "http://") && !strings.HasPrefix(b, "https://") { + b = "https://" + b + } + httpReq, err := http.NewRequestWithContext(c.Request.Context(), http.MethodGet, b, nil) + if err != nil { + lastErr = err + continue + } + httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + httpReq.Header.Set("Cookie", cookie) + resp, err := httpClient.Do(httpReq) + if err != nil { + lastErr = err + continue + } + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 256<<10)) + _ = resp.Body.Close() + if resp.StatusCode >= 400 { + lastErr = fmt.Errorf("%s returned HTTP %d", b, resp.StatusCode) + continue + } + text := string(bodyBytes) + if strings.Contains(text, "driver-verify") || strings.Contains(text, "Age Verification") { + lastErr = fmt.Errorf("%s intercepted by age verification", b) + continue + } + success = true + break + } + + latency := time.Since(start).Milliseconds() + if success { + c.JSON(http.StatusOK, gin.H{ + "success": true, + "latency_ms": latency, + "message": "内置刮削源连接正常", + }) + return + } + errMsg := "内置源连接失败" + if lastErr != nil { + errMsg = lastErr.Error() + } + c.JSON(http.StatusOK, gin.H{ + "success": false, + "latency_ms": latency, + "error": errMsg, + }) + } +} + func recentLogsHandler(svc *service.Container) gin.HandlerFunc { return func(c *gin.Context) { rows, err := svc.Repo.Log.Recent(c.Request.Context(), 200) diff --git a/internal/handler/api_config.go b/internal/handler/api_config.go index 9ebdb46..2521ce9 100644 --- a/internal/handler/api_config.go +++ b/internal/handler/api_config.go @@ -63,3 +63,15 @@ func deleteAPIConfigHandler(svc *service.Container) gin.HandlerFunc { c.Status(http.StatusNoContent) } } + +func testAPIConfigHandler(svc *service.Container) gin.HandlerFunc { + return func(c *gin.Context) { + provider := c.Param("provider") + result, err := svc.ApiConfig.TestConnection(c.Request.Context(), provider) + if err != nil { + c.JSON(http.StatusOK, gin.H{"result": result, "error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"result": result}) + } +} diff --git a/internal/handler/routes_admin.go b/internal/handler/routes_admin.go index d25ea2f..3651963 100644 --- a/internal/handler/routes_admin.go +++ b/internal/handler/routes_admin.go @@ -38,6 +38,7 @@ func registerAdminUserRoutes(admin *gin.RouterGroup, svc *service.Container) { admin.DELETE("/users/:id", deleteUserHandler(svc)) admin.GET("/settings", listSettingsHandler(svc)) admin.PUT("/settings", updateSettingHandler(svc)) + admin.POST("/adult/test-scraper", testAdultScraperHandler(svc)) admin.GET("/logs", recentLogsHandler(svc)) } @@ -125,6 +126,7 @@ func registerAdminAPIConfigRoutes(admin *gin.RouterGroup, svc *service.Container admin.GET("/api-configs/:provider", getAPIConfigHandler(svc)) admin.PUT("/api-configs/:provider", updateAPIConfigHandler(svc)) admin.DELETE("/api-configs/:provider", deleteAPIConfigHandler(svc)) + admin.POST("/api-configs/:provider/test", testAPIConfigHandler(svc)) } func registerAdminSchedulerRoutes(admin *gin.RouterGroup, svc *service.Container) { diff --git a/internal/model/api_config.go b/internal/model/api_config.go index 2af6f99..fd487ee 100644 --- a/internal/model/api_config.go +++ b/internal/model/api_config.go @@ -39,6 +39,7 @@ func PredefinedProviders() []ApiProvider { {ID: "openai", Name: "OpenAI", Description: "GPT 系列模型", HasAPIKey: true, HasBaseURL: true}, {ID: "deepseek", Name: "DeepSeek", Description: "DeepSeek 大模型", HasAPIKey: true, HasBaseURL: true}, {ID: "siliconflow", Name: "SiliconFlow", Description: "AI 模型聚合 API", HasAPIKey: true, HasBaseURL: true}, - {ID: "adult", Name: "Adult / 番号", Description: "JavDB/JavBus 成人内容元数据", HasAPIKey: false, HasBaseURL: true}, - } + {ID: "adult", Name: "Adult / 番号", Description: "JavDB/JavBus 成人内容元数据与 Cookie 凭据", HasAPIKey: true, HasBaseURL: true}, + {ID: "metatube", Name: "MetaTube", Description: "MetaTube Server 番号元数据后端服务", HasAPIKey: true, HasBaseURL: true}, + } } diff --git a/internal/service/adult_scraper.go b/internal/service/adult_scraper.go index 416f871..a3befec 100644 --- a/internal/service/adult_scraper.go +++ b/internal/service/adult_scraper.go @@ -12,6 +12,8 @@ import ( "time" "go.uber.org/zap" + + "github.com/ShukeBta/MediaStationGo/internal/repository" ) var ( @@ -50,12 +52,20 @@ type AdultProvider struct { log *zap.Logger client *http.Client apiConfig *APIConfigService + repo *repository.Container + metatube *MetaTubeProvider } -func NewAdultProvider(log *zap.Logger, apiConfig *APIConfigService) *AdultProvider { +func NewAdultProvider(log *zap.Logger, apiConfig *APIConfigService, repos ...*repository.Container) *AdultProvider { + var repo *repository.Container + if len(repos) > 0 { + repo = repos[0] + } return &AdultProvider{ log: log, apiConfig: apiConfig, + repo: repo, + metatube: NewMetaTubeProvider(log), client: NewExternalHTTPClient(12 * time.Second), } } @@ -64,11 +74,150 @@ func (p *AdultProvider) Enabled() bool { return p != nil } +// MetaTubeClient 返回 MetaTube 客户端实例。 +func (p *AdultProvider) MetaTubeClient() *MetaTubeProvider { + if p == nil { + return nil + } + return p.metatube +} + +// ResolveMetaTubeConfig 解析当前生效的 MetaTube 配置。 +func (p *AdultProvider) ResolveMetaTubeConfig(ctx context.Context) MetaTubeConfig { + serverURL := p.getSetting(ctx, "adult.scraper.metatube_server", "") + token := p.getSetting(ctx, "adult.scraper.metatube_token", "") + provider := p.getSetting(ctx, "adult.scraper.metatube_provider", "") + enableActor := p.getSettingBool(ctx, "adult.scraper.enable_actor", true) + enableTrailer := p.getSettingBool(ctx, "adult.scraper.enable_trailer", false) + cropCover := p.getSettingBool(ctx, "adult.scraper.crop_cover", true) + badge := p.getSettingBool(ctx, "adult.scraper.badge", false) + + if serverURL == "" && p.apiConfig != nil { + if resolved, err := p.apiConfig.Resolve(ctx, "metatube"); err == nil && resolved.BaseURL != "" { + serverURL = resolved.BaseURL + if token == "" { + token = resolved.APIKey + } + } + } + + return MetaTubeConfig{ + ServerURL: serverURL, + Token: token, + DefaultProvider: provider, + EnableActor: enableActor, + EnableTrailer: enableTrailer, + CropCover: cropCover, + Badge: badge, + } +} + +// Search 执行单个番号刮削（根据配置自动在 MetaTube、内置源或智能混合之间路由）。 func (p *AdultProvider) Search(ctx context.Context, code string) (*Match, error) { code = normalizeAdultCode(code) if code == "" { return nil, errors.New("empty adult code") } + + engine := strings.ToLower(p.getSetting(ctx, "adult.scraper.engine", "builtin")) + switch engine { + case "metatube": + mtCfg := p.ResolveMetaTubeConfig(ctx) + if mtCfg.ServerURL != "" { + match, err := p.metatube.SearchAndGetBestMatch(ctx, mtCfg, code) + if err != nil { + if p.log != nil { + p.log.Warn("metatube scrape failed", zap.String("code", code), zap.Error(err)) + } + return nil, err + } + if match != nil { + match.OriginalName = code + match.Title = FormatAdultTitle(code, match.Title) + match.NSFW = true + return match, nil + } + } + return nil, nil + + case "auto": + mtCfg := p.ResolveMetaTubeConfig(ctx) + if mtCfg.ServerURL != "" { + match, err := p.metatube.SearchAndGetBestMatch(ctx, mtCfg, code) + if err == nil && match != nil { + match.OriginalName = code + match.Title = FormatAdultTitle(code, match.Title) + match.NSFW = true + return match, nil + } + if p.log != nil && err != nil { + p.log.Debug("metatube search in auto mode failed, falling back to builtin", zap.String("code", code), zap.Error(err)) + } + } + return p.searchBuiltin(ctx, code) + + default: // "builtin" + return p.searchBuiltin(ctx, code) + } +} + +// SearchCandidates 执行搜索并返回所有候选 Match 列表（用于手动刮削弹窗）。 +func (p *AdultProvider) SearchCandidates(ctx context.Context, query string) ([]*Match, error) { + code := normalizeAdultCode(query) + if code == "" { + code = strings.TrimSpace(query) + } + if code == "" { + return nil, errors.New("empty adult query") + } + + engine := strings.ToLower(p.getSetting(ctx, "adult.scraper.engine", "builtin")) + switch engine { + case "metatube": + mtCfg := p.ResolveMetaTubeConfig(ctx) + if mtCfg.ServerURL != "" { + matches, err := p.metatube.Search(ctx, mtCfg, code) + if err != nil { + return nil, err + } + for _, m := range matches { + m.OriginalName = code + m.Title = FormatAdultTitle(code, m.Title) + m.NSFW = true + } + return matches, nil + } + return nil, nil + + case "auto": + mtCfg := p.ResolveMetaTubeConfig(ctx) + if mtCfg.ServerURL != "" { + matches, err := p.metatube.Search(ctx, mtCfg, code) + if err == nil && len(matches) > 0 { + for _, m := range matches { + m.OriginalName = code + m.Title = FormatAdultTitle(code, m.Title) + m.NSFW = true + } + return matches, nil + } + } + m, err := p.searchBuiltin(ctx, code) + if err != nil || m == nil { + return nil, err + } + return []*Match{m}, nil + + default: + m, err := p.searchBuiltin(ctx, code) + if err != nil || m == nil { + return nil, err + } + return []*Match{m}, nil + } +} + +func (p *AdultProvider) searchBuiltin(ctx context.Context, code string) (*Match, error) { bases := p.resolveBases(ctx) if len(bases) == 0 { return nil, nil @@ -90,37 +239,74 @@ func (p *AdultProvider) Search(ctx context.Context, code string) (*Match, error) } continue } - if match != nil { - match.OriginalName = code - match.Title = FormatAdultTitle(code, match.Title) - match.NSFW = true - return match, nil - } + if match != nil { + match.OriginalName = code + match.Title = FormatAdultTitle(code, match.Title) + match.NSFW = true + return match, nil + } } return nil, lastErr } func (p *AdultProvider) resolveBases(ctx context.Context) []string { - out := append([]string{}, defaultAdultBases...) - if p.apiConfig == nil { - return out - } - resolved, err := p.apiConfig.Resolve(ctx, "adult") - if err != nil { - return out - } - if !resolved.Enabled && (resolved.BaseURL != "" || resolved.Extra != "" || resolved.APIKey != "") { - return nil - } + customJavDB := p.getSetting(ctx, "adult.scraper.builtin_javdb_url", "") + customJavBus := p.getSetting(ctx, "adult.scraper.builtin_javbus_url", "") + configured := []string{} - configured = append(configured, adultConfiguredBases(resolved.BaseURL)...) - configured = append(configured, adultConfiguredBases(resolved.Extra)...) + if customJavDB != "" { + configured = append(configured, adultConfiguredBases(customJavDB)...) + } + if customJavBus != "" { + configured = append(configured, adultConfiguredBases(customJavBus)...) + } + + out := append([]string{}, defaultAdultBases...) + if p.apiConfig != nil { + if resolved, err := p.apiConfig.Resolve(ctx, "adult"); err == nil { + if !resolved.Enabled && (resolved.BaseURL != "" || resolved.Extra != "" || resolved.APIKey != "") { + return nil + } + configured = append(configured, adultConfiguredBases(resolved.BaseURL)...) + configured = append(configured, adultConfiguredBases(resolved.Extra)...) + } + } if len(configured) > 0 { out = append(configured, out...) } return dedupeStrings(out) } +const defaultAdultCookies = "age=verified; existmag=all" + +func (p *AdultProvider) resolveCookie(ctx context.Context) string { + customCookie := p.getSetting(ctx, "adult.scraper.builtin_cookie", "") + if customCookie != "" { + if !strings.Contains(customCookie, "age=") { + customCookie = customCookie + "; age=verified" + } + if !strings.Contains(customCookie, "existmag=") { + customCookie = customCookie + "; existmag=all" + } + return customCookie + } + if p == nil || p.apiConfig == nil { + return defaultAdultCookies + } + resolved, err := p.apiConfig.Resolve(ctx, "adult") + if err != nil || strings.TrimSpace(resolved.APIKey) == "" { + return defaultAdultCookies + } + userCookie := strings.TrimSpace(resolved.APIKey) + if !strings.Contains(userCookie, "age=") { + userCookie = userCookie + "; age=verified" + } + if !strings.Contains(userCookie, "existmag=") { + userCookie = userCookie + "; existmag=all" + } + return userCookie +} + func (p *AdultProvider) scrapeJavDB(ctx context.Context, base, code string) (*Match, error) { searchURL := base + "/search?q=" + url.QueryEscape(code) + "&f=all" body, err := p.fetchText(ctx, searchURL, base) @@ -164,7 +350,8 @@ func (p *AdultProvider) fetchText(ctx context.Context, targetURL, referer string if err != nil { return "", err } - applyAdultHeaders(req, referer) + cookie := p.resolveCookie(ctx) + applyAdultHeaders(req, referer, cookie) resp, err := p.client.Do(req) if err != nil { return "", err @@ -180,13 +367,20 @@ func (p *AdultProvider) fetchText(ctx context.Context, targetURL, referer string if err != nil { return "", err } - return string(body), nil + text := string(body) + if strings.Contains(text, "driver-verify") || strings.Contains(text, "Age Verification") || strings.Contains(text, "你是否已經成年") { + return "", fmt.Errorf("adult source %s intercepted by age verification", targetURL) + } + return text, nil } -func applyAdultHeaders(req *http.Request, referer string) { +func applyAdultHeaders(req *http.Request, referer, cookie string) { req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,ja;q=0.8,en;q=0.7") + if cookie != "" { + req.Header.Set("Cookie", cookie) + } if referer != "" { req.Header.Set("Referer", referer) } diff --git a/internal/service/adult_scraper_config.go b/internal/service/adult_scraper_config.go index f6499c4..3cfb365 100644 --- a/internal/service/adult_scraper_config.go +++ b/internal/service/adult_scraper_config.go @@ -1,6 +1,7 @@ package service import ( + "context" "net/url" "strings" ) @@ -65,3 +66,21 @@ func dedupeStrings(values []string) []string { } return out } + +func (p *AdultProvider) getSetting(ctx context.Context, key, fallback string) string { + if p != nil && p.repo != nil && p.repo.Setting != nil { + if val, err := p.repo.Setting.Get(ctx, key); err == nil && strings.TrimSpace(val) != "" { + return strings.TrimSpace(val) + } + } + return fallback +} + +func (p *AdultProvider) getSettingBool(ctx context.Context, key string, fallback bool) bool { + val := p.getSetting(ctx, key, "") + if val == "" { + return fallback + } + val = strings.ToLower(val) + return val == "true" || val == "1" || val == "yes" || val == "on" +} diff --git a/internal/service/adult_scraper_parse.go b/internal/service/adult_scraper_parse.go index 25f1ff2..1b38d49 100644 --- a/internal/service/adult_scraper_parse.go +++ b/internal/service/adult_scraper_parse.go @@ -9,21 +9,26 @@ import ( ) func parseAdultDetailHTML(body, code, source, detailURL string) *Match { + if strings.Contains(body, "driver-verify") || strings.Contains(body, "Age Verification") || strings.Contains(body, "你是否已經成年") { + return nil + } code = normalizeAdultCode(code) + rawTitle := firstAdultTitle(body, code) + title := FormatAdultTitle(code, rawTitle) + if title == "" { + title = code + } + if title == "" { + return nil + } + match := &Match{ OriginalName: code, MediaType: "adult", NSFW: true, + Title: title, Genres: []string{"Adult", source}, } - rawTitle := firstAdultTitle(body, code) - match.Title = FormatAdultTitle(code, rawTitle) - if match.Title == "" { - match.Title = code - } - if match.Title == "" { - return nil - } if source == "javbus" { if m := adultJavBusCoverPattern.FindStringSubmatch(body); len(m) > 1 { match.PosterURL = absolutizeURL(detailURL, m[1]) @@ -39,6 +44,11 @@ func parseAdultDetailHTML(body, code, source, detailURL string) *Match { } match.Year = firstYearInText(body) match.Rating = firstRatingInText(body) + + // 如果没有提取到任何封面且标题仅是番号本身，判定为无效或被拦截页面 + if match.PosterURL == "" && rawTitle == "" { + return nil + } return match } diff --git a/internal/service/adult_scraper_routing_test.go b/internal/service/adult_scraper_routing_test.go new file mode 100644 index 0000000..0cafa64 --- /dev/null +++ b/internal/service/adult_scraper_routing_test.go @@ -0,0 +1,91 @@ +package service + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/glebarez/sqlite" + "go.uber.org/zap" + "gorm.io/gorm" + + "github.com/ShukeBta/MediaStationGo/internal/model" + "github.com/ShukeBta/MediaStationGo/internal/repository" +) + +func TestAdultProviderRouting(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatalf("failed to open sqlite: %v", err) + } + _ = db.AutoMigrate(&model.Setting{}, &model.ApiConfig{}) + + repos := repository.New(db) + + mtServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/movies/search" { + q := r.URL.Query().Get("q") + if q == "SSIS-001" { + results := struct { + Data []MetaTubeSearchResult `json:"data"` + }{ + Data: []MetaTubeSearchResult{ + { + ID: "999", + Number: "SSIS-001", + Title: "河北彩花 専属デビュー", + Provider: "javdb", + Actors: []string{"河北彩花"}, + ReleaseDate: "2021-06-19", + Score: 4.9, + }, + }, + } + _ = json.NewEncoder(w).Encode(results) + return + } + } + http.NotFound(w, r) + })) + defer mtServer.Close() + + // 1. Configure metatube settings + _ = repos.Setting.Set(context.Background(), "adult.scraper.engine", "metatube") + _ = repos.Setting.Set(context.Background(), "adult.scraper.metatube_server", mtServer.URL) + + adultProvider := NewAdultProvider(zap.NewNop(), nil, repos) + + // Test Search + match, err := adultProvider.Search(context.Background(), "SSIS-001") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if match == nil { + t.Fatal("expected match, got nil") + } + if match.OriginalName != "SSIS-001" { + t.Errorf("expected SSIS-001, got %s", match.OriginalName) + } + if match.Year != 2021 { + t.Errorf("expected 2021, got %d", match.Year) + } + + // Test SearchCandidates + candidates, err := adultProvider.SearchCandidates(context.Background(), "SSIS-001") + if err != nil { + t.Fatalf("unexpected error on candidates: %v", err) + } + if len(candidates) != 1 { + t.Fatalf("expected 1 candidate, got %d", len(candidates)) + } + + // 2. Test auto mode with failing metatube query + _ = repos.Setting.Set(context.Background(), "adult.scraper.engine", "auto") + // Search for something not on mock metatube server; will attempt fallback + mUnknown, _ := adultProvider.Search(context.Background(), "NONEXISTENT-999") + if mUnknown != nil { + t.Errorf("expected nil for nonexistent in auto mode when sources unavailable") + } +} diff --git a/internal/service/adult_scraper_test.go b/internal/service/adult_scraper_test.go index bde64d7..4d1a119 100644 --- a/internal/service/adult_scraper_test.go +++ b/internal/service/adult_scraper_test.go @@ -135,9 +135,61 @@ func TestAdultProviderDefaultBases(t *testing.T) { if len(got) != len(want) { t.Fatalf("resolveBases len = %d, want %d: %v", len(got), len(want), got) } - for i := range want { - if got[i] != want[i] { - t.Fatalf("resolveBases[%d] = %q, want %q", i, got[i], want[i]) + for i := range want { + if got[i] != want[i] { + t.Fatalf("resolveBases[%d] = %q, want %q", i, got[i], want[i]) + } } } + +func TestParseAdultDetailHTMLRejectsAgeVerification(t *testing.T) { + verifyHTML := ` +Age Verification JavBus - JavBus +

你是否已經成年?

driver-verify +` + got := parseAdultDetailHTML(verifyHTML, "IPX-235", "javbus", "https://www.javbus.com/IPX-235") + if got != nil { + t.Fatalf("expected nil for age verification page, got: %+v", got) + } +} + +func TestAdultProviderPassesDefaultCookie(t *testing.T) { + receivedCookie := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedCookie = r.Header.Get("Cookie") + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(`

IPX-235 测试番号

`)) + })) + defer server.Close() + + baseURL := server.URL + provider := &AdultProvider{ + client: server.Client(), + } + match, err := provider.scrapeJavBus(context.Background(), baseURL, "IPX-235") + if err != nil { + t.Fatalf("scrapeJavBus failed: %v", err) + } + if match == nil || match.PosterURL != baseURL+"/pics/cover/6v0a_b.jpg" { + t.Fatalf("unexpected match: %+v", match) + } + if receivedCookie != "age=verified; existmag=all" { + t.Fatalf("receivedCookie = %q, want 'age=verified; existmag=all'", receivedCookie) + } +} + +func TestLiveAdultProviderJavBus(t *testing.T) { + provider := NewAdultProvider(zap.NewNop(), nil) + match, err := provider.scrapeJavBus(context.Background(), "https://www.javbus.com", "IPX-235") + if err != nil { + t.Logf("Live test skipped or failed due to network: %v", err) + return + } + if match == nil { + t.Fatal("expected match from live JavBus") + } + t.Logf("Live scrape success: Title=%q, PosterURL=%q, BackdropURL=%q", match.Title, match.PosterURL, match.BackdropURL) + if match.PosterURL == "" { + t.Fatal("expected non-empty PosterURL") + } } diff --git a/internal/service/api_config.go b/internal/service/api_config.go index a2ccaab..6197855 100644 --- a/internal/service/api_config.go +++ b/internal/service/api_config.go @@ -41,8 +41,9 @@ func (s *APIConfigService) SeedDefaults(ctx context.Context) error { {Provider: "thetvdb", BaseURL: "https://api4.thetvdb.com/v4", Description: "TheTVDB (tv)", Enabled: true}, {Provider: "fanart", BaseURL: "https://webservice.fanart.tv/v3", Description: "Fanart.tv (artwork)", Enabled: true}, {Provider: "douban", Description: "Douban cookie (zh metadata)", Enabled: true}, - {Provider: "adult", BaseURL: "https://javdb.com", Extra: "https://javbus.sbs,https://www.javbus.com,https://www.cdnbus.cyou,https://www.javsee.cyou,https://www.busjav.cyou", Description: "Adult / 番号元数据（JavDB/JavBus）", Enabled: true}, - {Provider: "openai", BaseURL: "https://api.openai.com/v1", Description: "OpenAI-compatible (smart search)", Enabled: true}, + {Provider: "adult", BaseURL: "https://javdb.com", Extra: "https://javbus.sbs,https://www.javbus.com,https://www.cdnbus.cyou,https://www.javsee.cyou,https://www.busjav.cyou", Description: "Adult / 番号元数据（JavDB/JavBus）", Enabled: true}, + {Provider: "metatube", BaseURL: "http://127.0.0.1:7700", Description: "MetaTube Server 番号刮削后端", Enabled: true}, + {Provider: "openai", BaseURL: "https://api.openai.com/v1", Description: "OpenAI-compatible (smart search)", Enabled: true}, } for i := range defaults { var existing model.APIConfig diff --git a/internal/service/api_config_connection.go b/internal/service/api_config_connection.go index 8b75613..117f1a2 100644 --- a/internal/service/api_config_connection.go +++ b/internal/service/api_config_connection.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "net/http" "net/url" "strings" @@ -29,7 +30,11 @@ func (s *ApiConfigService) TestConnection(ctx context.Context, provider string) return s.testDeepSeek(cfg) case "siliconflow": return s.testSiliconFlow(cfg) - default: + case "adult": + return s.testAdult(ctx, cfg) + case "metatube": + return s.testMetaTube(ctx, cfg) + default: return "unknown", fmt.Errorf("no test implemented for provider: %s", provider) } } @@ -169,3 +174,89 @@ func (s *ApiConfigService) testSiliconFlow(cfg *model.ApiConfig) (string, error) } return "error", fmt.Errorf("SiliconFlow API returned status %d", resp.StatusCode) } + +// testAdult 测试 Adult (JavDB/JavBus) 刮削数据源连接与年龄验证。 +func (s *ApiConfigService) testAdult(ctx context.Context, cfg *model.ApiConfig) (string, error) { + bases := []string{} + if cfg.BaseURL != "" { + bases = append(bases, adultConfiguredBases(cfg.BaseURL)...) + } + if cfg.Extra != "" { + bases = append(bases, adultConfiguredBases(cfg.Extra)...) + } + if len(bases) == 0 { + bases = defaultAdultBases + } + cookie := defaultAdultCookies + if strings.TrimSpace(cfg.APIKey) != "" { + cookie = strings.TrimSpace(cfg.APIKey) + if !strings.Contains(cookie, "age=") { + cookie = cookie + "; age=verified" + } + if !strings.Contains(cookie, "existmag=") { + cookie = cookie + "; existmag=all" + } + } + + client := NewExternalHTTPClient(10 * time.Second) + var lastErr error + successCount := 0 + + for _, base := range bases { + base = strings.TrimRight(base, "/") + probeURL := base + if adultSourceKind(base) == "javbus" { + probeURL = base + "/IPX-235" + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil) + if err != nil { + lastErr = err + continue + } + applyAdultHeaders(req, base, cookie) + resp, err := client.Do(req) + if err != nil { + lastErr = fmt.Errorf("%s: %w", base, err) + continue + } + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 512<<10)) + _ = resp.Body.Close() + + if resp.StatusCode >= 400 { + lastErr = fmt.Errorf("%s returned %d", base, resp.StatusCode) + continue + } + text := string(respBody) + if strings.Contains(text, "driver-verify") || strings.Contains(text, "Age Verification") || strings.Contains(text, "你是否已經成年") { + lastErr = fmt.Errorf("%s: age verification required / intercepted", base) + continue + } + successCount++ + break + } + + if successCount > 0 { + return "success", nil + } + if lastErr != nil { + return "error", fmt.Errorf("adult sources test failed: %w", lastErr) + } + return "error", errors.New("no adult source available") +} + +// testMetaTube 测试 MetaTube Server 连接与 Token。 +func (s *ApiConfigService) testMetaTube(ctx context.Context, cfg *model.ApiConfig) (string, error) { + serverURL := strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/") + if serverURL == "" { + serverURL = "http://127.0.0.1:7700" + } + provider := NewMetaTubeProvider(s.log) + res, err := provider.TestConnection(ctx, serverURL, cfg.APIKey) + if err != nil { + return "error", err + } + if !res.Success { + return "error", errors.New(res.Error) + } + return "success", nil +} diff --git a/internal/service/image_proxy_remote_fetch.go b/internal/service/image_proxy_remote_fetch.go index 27c16ca..a7c2088 100644 --- a/internal/service/image_proxy_remote_fetch.go +++ b/internal/service/image_proxy_remote_fetch.go @@ -85,11 +85,26 @@ func applyRemoteImageHeaders(req *http.Request, host string) { req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,ja;q=0.8,en;q=0.7") req.Header.Set("Cache-Control", "no-cache") req.Header.Set("Pragma", "no-cache") + if cookie := remoteImageCookie(host); cookie != "" { + req.Header.Set("Cookie", cookie) + } if referer := remoteImageReferer(host); referer != "" { req.Header.Set("Referer", referer) } } +func remoteImageCookie(host string) string { + h := strings.ToLower(strings.TrimSpace(host)) + switch { + case strings.Contains(h, "javbus") || strings.Contains(h, "busjav") || strings.Contains(h, "dmmbus") || strings.Contains(h, "javsee") || strings.Contains(h, "cdnbus"): + return "age=verified; existmag=all" + case strings.Contains(h, "javdb") || strings.Contains(h, "jdbstatic"): + return "over18=1" + default: + return "" + } +} + func remoteImageReferer(host string) string { h := strings.ToLower(strings.TrimSpace(host)) switch { @@ -143,10 +158,13 @@ func fetchRemoteImageWithCurl(ctx context.Context, raw, host string) ([]byte, st "--header", "Cache-Control: no-cache", "--header", "Pragma: no-cache", } - if referer := remoteImageReferer(host); referer != "" { - args = append(args, "--referer", referer) - } - args = append(args, "--", raw) + if referer := remoteImageReferer(host); referer != "" { + args = append(args, "--referer", referer) + } + if cookie := remoteImageCookie(host); cookie != "" { + args = append(args, "--cookie", cookie) + } + args = append(args, "--", raw) cmd := exec.CommandContext(curlCtx, bin, args...) // #nosec G204 -- bin is resolved by LookPath and args are not shell-expanded. stderr := bytes.Buffer{} diff --git a/internal/service/manual_scrape_providers.go b/internal/service/manual_scrape_providers.go index 4e9a406..cd5603e 100644 --- a/internal/service/manual_scrape_providers.go +++ b/internal/service/manual_scrape_providers.go @@ -83,14 +83,20 @@ func manualTMDbSearchTypes(mediaType string) []string { } func (s *ScraperService) manualAdultMatches(ctx context.Context, media *model.Media, query string) []*Match { + if s.adult == nil || !s.adult.Enabled() { + return nil + } candidates := []string{query} if media != nil { candidates = append(candidates, media.Path, media.OriginalName, media.Title) } - out := make([]*Match, 0, 1) + out := make([]*Match, 0, 4) seen := map[string]struct{}{} for _, candidate := range candidates { code := normalizeAdultCode(candidate) + if code == "" { + code = strings.TrimSpace(candidate) + } if code == "" { continue } @@ -98,8 +104,12 @@ func (s *ScraperService) manualAdultMatches(ctx context.Context, media *model.Me continue } seen[code] = struct{}{} - if match := s.manualAdultMatch(ctx, code); match != nil { - out = append(out, match) + if matches, err := s.adult.SearchCandidates(ctx, code); err == nil && len(matches) > 0 { + for _, m := range matches { + if m != nil && m.Title != "" { + out = append(out, m) + } + } } } return out diff --git a/internal/service/metatube.go b/internal/service/metatube.go new file mode 100644 index 0000000..6b7df6c --- /dev/null +++ b/internal/service/metatube.go @@ -0,0 +1,404 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "go.uber.org/zap" +) + +// MetaTubeProvider 是对接 MetaTube Server REST API 的刮削客户端。 +type MetaTubeProvider struct { + log *zap.Logger + client *http.Client +} + +// NewMetaTubeProvider 创建 MetaTube 刮削服务实例。 +func NewMetaTubeProvider(log *zap.Logger) *MetaTubeProvider { + return &MetaTubeProvider{ + log: log, + client: NewExternalHTTPClient(15 * time.Second), + } +} + +// Search 执行番号/关键词搜索，返回标准化 Match 列表。 +func (p *MetaTubeProvider) Search(ctx context.Context, cfg MetaTubeConfig, query string) ([]*Match, error) { + serverURL := strings.TrimRight(strings.TrimSpace(cfg.ServerURL), "/") + if serverURL == "" { + return nil, errors.New("metatube server url not configured") + } + + searchURL := fmt.Sprintf("%s/v1/movies/search?q=%s", serverURL, url.QueryEscape(query)) + if strings.TrimSpace(cfg.DefaultProvider) != "" { + searchURL += "&provider=" + url.QueryEscape(strings.TrimSpace(cfg.DefaultProvider)) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil) + if err != nil { + return nil, err + } + p.applyAuthHeader(req, cfg.Token) + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("metatube search request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return nil, fmt.Errorf("metatube search returned status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return nil, err + } + + var results []MetaTubeSearchResult + var wrapped struct { + Data []MetaTubeSearchResult `json:"data"` + } + if wrapErr := json.Unmarshal(bodyBytes, &wrapped); wrapErr == nil && len(wrapped.Data) > 0 { + results = wrapped.Data + } else if directErr := json.Unmarshal(bodyBytes, &results); directErr != nil { + return nil, fmt.Errorf("decode metatube search response failed: %w", directErr) + } + + if len(results) == 0 { + return nil, nil + } + + matches := make([]*Match, 0, len(results)) + for _, res := range results { + match := p.convertSearchResultToMatch(query, &res) + if match != nil { + matches = append(matches, match) + } + } + return matches, nil +} + +// GetMovie 根据 Provider 与影片 ID 获取完整元数据。 +func (p *MetaTubeProvider) GetMovie(ctx context.Context, cfg MetaTubeConfig, provider, id string) (*Match, error) { + serverURL := strings.TrimRight(strings.TrimSpace(cfg.ServerURL), "/") + if serverURL == "" { + return nil, errors.New("metatube server url not configured") + } + + movieURL := fmt.Sprintf("%s/v1/movies/%s/%s", serverURL, url.PathEscape(provider), url.PathEscape(id)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, movieURL, nil) + if err != nil { + return nil, err + } + p.applyAuthHeader(req, cfg.Token) + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("metatube movie request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return nil, fmt.Errorf("metatube movie returned status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return nil, err + } + + var movie MetaTubeMovieInfo + var wrapped struct { + Data MetaTubeMovieInfo `json:"data"` + } + if wrapErr := json.Unmarshal(bodyBytes, &wrapped); wrapErr == nil && wrapped.Data.ID != "" { + movie = wrapped.Data + } else if directErr := json.Unmarshal(bodyBytes, &movie); directErr != nil { + return nil, fmt.Errorf("decode metatube movie response failed: %w", directErr) + } + + return p.convertMovieInfoToMatch(&movie), nil +} + +// SearchAndGetBestMatch 执行搜索并拉取首个最佳结果的完整电影详情。 +func (p *MetaTubeProvider) SearchAndGetBestMatch(ctx context.Context, cfg MetaTubeConfig, code string) (*Match, error) { + results, err := p.Search(ctx, cfg, code) + if err != nil { + return nil, err + } + if len(results) == 0 { + return nil, nil + } + + // 优先寻找带有 ID 和 Provider 的结果拉取详情 + for _, candidate := range results { + if candidate.DoubanID != "" && candidate.TheTVDBID != "" { + if detailed, err := p.GetMovie(ctx, cfg, candidate.TheTVDBID, candidate.DoubanID); err == nil && detailed != nil { + return detailed, nil + } + } + } + + return results[0], nil +} + +// GetProviders 获取 MetaTube 服务端支持的 Provider 列表。 +func (p *MetaTubeProvider) GetProviders(ctx context.Context, cfg MetaTubeConfig) ([]string, error) { + serverURL := strings.TrimRight(strings.TrimSpace(cfg.ServerURL), "/") + if serverURL == "" { + return nil, errors.New("metatube server url not configured") + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL+"/v1/providers", nil) + if err != nil { + return nil, err + } + p.applyAuthHeader(req, cfg.Token) + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("metatube providers request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("metatube providers returned status %d", resp.StatusCode) + } + + bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, err + } + + var respObj MetaTubeProvidersResponse + if err := json.Unmarshal(bodyBytes, &respObj); err == nil && len(respObj.Data.MovieProviders) > 0 { + providers := make([]string, 0, len(respObj.Data.MovieProviders)) + for prov := range respObj.Data.MovieProviders { + providers = append(providers, prov) + } + sort.Strings(providers) + return providers, nil + } + + // 兼容纯数组返回 + var arrayProviders []string + if err := json.Unmarshal(bodyBytes, &arrayProviders); err == nil { + return arrayProviders, nil + } + + return nil, fmt.Errorf("decode metatube providers failed") +} + +// TestConnection 测试与 MetaTube 服务端的连通性。 +func (p *MetaTubeProvider) TestConnection(ctx context.Context, serverURL, token string) (*MetaTubeTestResult, error) { + serverURL = strings.TrimRight(strings.TrimSpace(serverURL), "/") + if serverURL == "" { + return &MetaTubeTestResult{Success: false, Error: "server URL is required"}, errors.New("server URL is required") + } + + start := time.Now() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL+"/v1/providers", nil) + if err != nil { + return &MetaTubeTestResult{Success: false, Error: err.Error()}, err + } + p.applyAuthHeader(req, token) + + resp, err := p.client.Do(req) + latency := time.Since(start).Milliseconds() + if err != nil { + return &MetaTubeTestResult{ + Success: false, + LatencyMs: latency, + Error: fmt.Sprintf("connection failed: %v", err), + }, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return &MetaTubeTestResult{ + Success: false, + LatencyMs: latency, + Error: "unauthorized: invalid token", + }, errors.New("unauthorized: invalid token") + } + + if resp.StatusCode != http.StatusOK { + return &MetaTubeTestResult{ + Success: false, + LatencyMs: latency, + Error: fmt.Sprintf("server returned HTTP %d", resp.StatusCode), + }, fmt.Errorf("server returned HTTP %d", resp.StatusCode) + } + + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + var respObj MetaTubeProvidersResponse + var providers []string + if err := json.Unmarshal(bodyBytes, &respObj); err == nil && len(respObj.Data.MovieProviders) > 0 { + for prov := range respObj.Data.MovieProviders { + providers = append(providers, prov) + } + sort.Strings(providers) + } else { + _ = json.Unmarshal(bodyBytes, &providers) + } + + return &MetaTubeTestResult{ + Success: true, + LatencyMs: latency, + Providers: providers, + }, nil +} + +func (p *MetaTubeProvider) applyAuthHeader(req *http.Request, token string) { + token = strings.TrimSpace(token) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("X-Token", token) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "MediaStationGo/1.0 (MetaTube Client)") +} + +func (p *MetaTubeProvider) convertSearchResultToMatch(query string, res *MetaTubeSearchResult) *Match { + if res == nil { + return nil + } + code := strings.TrimSpace(res.Number) + if code == "" { + code = normalizeAdultCode(query) + } + + title := strings.TrimSpace(res.Title) + if title == "" { + title = code + } + formattedTitle := FormatAdultTitle(code, title) + + year := parseYearFromDate(res.ReleaseDate) + + genres := make([]string, 0, len(res.Actors)) + for _, a := range res.Actors { + if strings.TrimSpace(a) != "" { + genres = append(genres, strings.TrimSpace(a)) + } + } + + posterURL := firstNonEmpty(res.BigCoverURL, res.CoverURL, res.BigThumbURL, res.ThumbURL) + + return &Match{ + MediaType: "adult", + Title: formattedTitle, + OriginalName: code, + PosterURL: posterURL, + BackdropURL: posterURL, + Year: year, + ReleaseDate: cleanDateString(res.ReleaseDate), + Rating: res.Score, + Genres: genres, + NSFW: true, + DoubanID: res.ID, // 借用字段存储原始 ID 便于详情反查 + TheTVDBID: res.Provider, // 借用字段存储 Provider + } +} + +func (p *MetaTubeProvider) convertMovieInfoToMatch(movie *MetaTubeMovieInfo) *Match { + if movie == nil { + return nil + } + + code := strings.TrimSpace(movie.Number) + if code == "" { + code = movie.ID + } + + title := strings.TrimSpace(movie.Title) + if title == "" { + title = code + } + formattedTitle := FormatAdultTitle(code, title) + + year := parseYearFromDate(movie.ReleaseDate) + + posterURL := firstNonEmpty(movie.BigCoverURL, movie.CoverURL, movie.BigThumbURL, movie.ThumbURL) + backdropURL := posterURL + if len(movie.PreviewImages) > 0 && movie.PreviewImages[0] != "" { + backdropURL = movie.PreviewImages[0] + } + + genres := make([]string, 0, len(movie.Genres)+len(movie.Actors)+4) + for _, g := range movie.Genres { + if strings.TrimSpace(g) != "" { + genres = append(genres, strings.TrimSpace(g)) + } + } + if strings.TrimSpace(movie.Maker) != "" { + genres = append(genres, strings.TrimSpace(movie.Maker)) + } + if strings.TrimSpace(movie.Label) != "" && movie.Label != movie.Maker { + genres = append(genres, strings.TrimSpace(movie.Label)) + } + for _, a := range movie.Actors { + if strings.TrimSpace(a) != "" { + genres = append(genres, strings.TrimSpace(a)) + } + } + if strings.TrimSpace(movie.Director) != "" { + genres = append(genres, strings.TrimSpace(movie.Director)) + } + for _, d := range movie.Directors { + if strings.TrimSpace(d) != "" { + genres = append(genres, strings.TrimSpace(d)) + } + } + + return &Match{ + MediaType: "adult", + Title: formattedTitle, + OriginalName: code, + Overview: strings.TrimSpace(movie.Summary), + PosterURL: posterURL, + BackdropURL: backdropURL, + Year: year, + ReleaseDate: cleanDateString(movie.ReleaseDate), + Rating: movie.Score, + Genres: dedupeStrings(genres), + NSFW: true, + DoubanID: movie.ID, + TheTVDBID: movie.Provider, + } +} + +func parseYearFromDate(dateStr string) int { + dateStr = strings.TrimSpace(dateStr) + if len(dateStr) >= 4 { + if y, err := strconv.Atoi(dateStr[:4]); err == nil && y > 1900 && y < 2100 { + return y + } + } + return 0 +} + +func cleanDateString(dateStr string) string { + dateStr = strings.TrimSpace(dateStr) + if len(dateStr) >= 10 { + return dateStr[:10] + } + return dateStr +} diff --git a/internal/service/metatube_test.go b/internal/service/metatube_test.go new file mode 100644 index 0000000..5dfd569 --- /dev/null +++ b/internal/service/metatube_test.go @@ -0,0 +1,176 @@ +package service + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "go.uber.org/zap" +) + +func TestMetaTubeProviderSearch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/movies/search" { + http.NotFound(w, r) + return + } + q := r.URL.Query().Get("q") + if q != "IPX-235" { + http.Error(w, "unexpected query", http.StatusBadRequest) + return + } + auth := r.Header.Get("Authorization") + if auth != "Bearer secret-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + results := struct { + Data []MetaTubeSearchResult `json:"data"` + }{ + Data: []MetaTubeSearchResult{ + { + ID: "123456", + Number: "IPX-235", + Title: "相沢みなみ 専属デビュー", + Provider: "javdb", + Actors: []string{"相沢みなみ"}, + CoverURL: "https://example.com/cover.jpg", + ReleaseDate: "2018-11-13", + Score: 4.6, + }, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(results) + })) + defer server.Close() + + provider := NewMetaTubeProvider(zap.NewNop()) + cfg := MetaTubeConfig{ + ServerURL: server.URL, + Token: "secret-token", + } + + matches, err := provider.Search(context.Background(), cfg, "IPX-235") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(matches) != 1 { + t.Fatalf("expected 1 match, got %d", len(matches)) + } + m := matches[0] + if m.OriginalName != "IPX-235" { + t.Errorf("expected original name IPX-235, got %s", m.OriginalName) + } + if m.Year != 2018 { + t.Errorf("expected year 2018, got %d", m.Year) + } + if !m.NSFW { + t.Errorf("expected NSFW true") + } + if len(m.Genres) != 1 || m.Genres[0] != "相沢みなみ" { + t.Errorf("unexpected genres: %v", m.Genres) + } +} + +func TestMetaTubeProviderGetMovie(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/movies/javdb/123456" { + http.NotFound(w, r) + return + } + + info := struct { + Data MetaTubeMovieInfo `json:"data"` + }{ + Data: MetaTubeMovieInfo{ + ID: "123456", + Number: "IPX-235", + Title: "相沢みなみ 専属デビュー", + Summary: "超绝美少女相沢みなみ出道作品！", + Provider: "javdb", + Actors: []string{"相沢みなみ"}, + Directors: []string{"导演A"}, + Maker: "IDEA POCKET", + Label: "Tissue", + Genres: []string{"单体作品", "美少女"}, + Score: 4.8, + ReleaseDate: "2018-11-13", + CoverURL: "https://example.com/cover.jpg", + BigCoverURL: "https://example.com/big_cover.jpg", + PreviewImages: []string{"https://example.com/preview1.jpg"}, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(info) + })) + defer server.Close() + + provider := NewMetaTubeProvider(zap.NewNop()) + cfg := MetaTubeConfig{ + ServerURL: server.URL, + } + + match, err := provider.GetMovie(context.Background(), cfg, "javdb", "123456") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if match == nil { + t.Fatal("expected non-nil match") + } + if match.Overview != "超绝美少女相沢みなみ出道作品！" { + t.Errorf("unexpected overview: %s", match.Overview) + } + if match.PosterURL != "https://example.com/big_cover.jpg" { + t.Errorf("unexpected poster URL: %s", match.PosterURL) + } + if match.BackdropURL != "https://example.com/preview1.jpg" { + t.Errorf("unexpected backdrop URL: %s", match.BackdropURL) + } + if match.Year != 2018 { + t.Errorf("expected year 2018, got %d", match.Year) + } +} + +func TestMetaTubeProviderTestConnection(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/providers" { + http.NotFound(w, r) + return + } + auth := r.Header.Get("Authorization") + if auth != "Bearer valid-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + resp := MetaTubeProvidersResponse{} + resp.Data.MovieProviders = map[string]string{ + "JavBus": "https://www.javbus.com/", + "FANZA": "https://www.dmm.co.jp/", + "FC2": "https://adult.contents.fc2.com/", + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + provider := NewMetaTubeProvider(zap.NewNop()) + + // Test valid token + res, err := provider.TestConnection(context.Background(), server.URL, "valid-token") + if err != nil || !res.Success { + t.Fatalf("expected success, got %v, err=%v", res, err) + } + if len(res.Providers) != 3 { + t.Errorf("expected 3 providers, got %d (%v)", len(res.Providers), res.Providers) + } + + // Test invalid token + resFail, errFail := provider.TestConnection(context.Background(), server.URL, "wrong-token") + if errFail == nil && resFail.Success { + t.Fatalf("expected failure for wrong token, got %v", resFail) + } +} diff --git a/internal/service/metatube_types.go b/internal/service/metatube_types.go new file mode 100644 index 0000000..75aac1d --- /dev/null +++ b/internal/service/metatube_types.go @@ -0,0 +1,92 @@ +package service + +import "time" + +// MetaTubeSearchResult 表示 MetaTube 搜索结果项。 +type MetaTubeSearchResult struct { + ID string `json:"id"` + Number string `json:"number"` + Title string `json:"title"` + Provider string `json:"provider"` + Homepage string `json:"homepage,omitempty"` + Actors []string `json:"actors,omitempty"` + CoverURL string `json:"cover_url,omitempty"` + BigCoverURL string `json:"big_cover_url,omitempty"` + ThumbURL string `json:"thumb_url,omitempty"` + BigThumbURL string `json:"big_thumb_url,omitempty"` + Score float32 `json:"score,omitempty"` + ReleaseDate string `json:"release_date,omitempty"` +} + +// MetaTubeMovieInfo 表示 MetaTube 影片完整详情。 +type MetaTubeMovieInfo struct { + ID string `json:"id"` + Number string `json:"number"` + Title string `json:"title"` + Summary string `json:"summary,omitempty"` + Provider string `json:"provider"` + Homepage string `json:"homepage,omitempty"` + Director string `json:"director,omitempty"` + Directors []string `json:"directors,omitempty"` + Actors []string `json:"actors,omitempty"` + Maker string `json:"maker,omitempty"` + Label string `json:"label,omitempty"` + Series string `json:"series,omitempty"` + Genres []string `json:"genres,omitempty"` + Score float32 `json:"score,omitempty"` + Runtime int `json:"runtime,omitempty"` + ReleaseDate string `json:"release_date,omitempty"` + ThumbURL string `json:"thumb_url,omitempty"` + BigThumbURL string `json:"big_thumb_url,omitempty"` + CoverURL string `json:"cover_url,omitempty"` + BigCoverURL string `json:"big_cover_url,omitempty"` + PreviewVideoURL string `json:"preview_video_url,omitempty"` + PreviewVideoHLSURL string `json:"preview_video_hls_url,omitempty"` + PreviewImages []string `json:"preview_images,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// MetaTubeActorInfo 表示 MetaTube 演员详情。 +type MetaTubeActorInfo struct { + ID string `json:"id"` + Name string `json:"name"` + JapaneseName string `json:"japanese_name,omitempty"` + Aliases []string `json:"aliases,omitempty"` + Provider string `json:"provider,omitempty"` + Images []string `json:"images,omitempty"` + Summary string `json:"summary,omitempty"` + Hobby string `json:"hobby,omitempty"` + Skill string `json:"skill,omitempty"` + Birthday string `json:"birthday,omitempty"` + BloodType string `json:"blood_type,omitempty"` + CupSize string `json:"cup_size,omitempty"` + Measurements string `json:"measurements,omitempty"` + Height int `json:"height,omitempty"` +} + +// MetaTubeProvidersResponse 表示 /v1/providers 响应。 +type MetaTubeProvidersResponse struct { + Data struct { + MovieProviders map[string]string `json:"movie_providers"` + ActorProviders map[string]string `json:"actor_providers"` + } `json:"data"` +} + +// MetaTubeConfig 表示 MetaTube 客户端运行配置。 +type MetaTubeConfig struct { + ServerURL string `json:"server_url"` + Token string `json:"token"` + DefaultProvider string `json:"default_provider,omitempty"` + EnableActor bool `json:"enable_actor"` + EnableTrailer bool `json:"enable_trailer"` + CropCover bool `json:"crop_cover"` + Badge bool `json:"badge"` +} + +// MetaTubeTestResult 表示 MetaTube 连通性测试结果。 +type MetaTubeTestResult struct { + Success bool `json:"success"` + LatencyMs int64 `json:"latency_ms"` + Providers []string `json:"providers,omitempty"` + Error string `json:"error,omitempty"` +} diff --git a/internal/service/scraper_test.go b/internal/service/scraper_test.go index f9bb5ad..87ca082 100644 --- a/internal/service/scraper_test.go +++ b/internal/service/scraper_test.go @@ -2,6 +2,8 @@ package service import ( "errors" + "net/http" + "net/http/httptest" "path/filepath" "strings" "testing" @@ -370,7 +372,53 @@ func TestApplyManualMatchSkipsTMDbEpisodeStillWhenDisabled(t *testing.T) { if !strings.HasSuffix(got.PosterURL, "/images/w500/poster.jpg") { t.Fatalf("series poster should still be saved when manual episode artwork is disabled: got %q", got.PosterURL) } - if !strings.HasSuffix(got.BackdropURL, "/images/w1280/backdrop.jpg") { - t.Fatalf("series backdrop should still be saved when manual episode artwork is disabled: got %q", got.BackdropURL) + if !strings.HasSuffix(got.BackdropURL, "/images/w1280/backdrop.jpg") { + t.Fatalf("series backdrop should still be saved when manual episode artwork is disabled: got %q", got.BackdropURL) + } + } + +func TestEnrichOneAdultScrapesArtwork(t *testing.T) { + scraper, repos, closeServer := newTestScraper(t) + defer closeServer() + + lib := model.Library{Name: "小姐姐", Path: t.TempDir(), Type: "movie", Enabled: true} + if err := repos.DB.Create(&lib).Error; err != nil { + t.Fatal(err) + } + media := model.Media{ + LibraryID: lib.ID, + Title: "IPX-235-C", + OriginalName: "IPX-235-C", + Path: filepath.Join(lib.Path, "IPX-235-C.strm"), + ScrapeStatus: "pending", + } + if err := repos.DB.Create(&media).Error; err != nil { + t.Fatal(err) + } + + // Mock adult provider + adultServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(`

IPX-235 测试番号封面

`)) + })) + defer adultServer.Close() + + scraper.adult = &AdultProvider{ + client: adultServer.Client(), + } + + if err := scraper.EnrichOne(t.Context(), &media); err != nil { + t.Fatal(err) + } + + var got model.Media + if err := repos.DB.First(&got, "id = ?", media.ID).Error; err != nil { + t.Fatal(err) + } + if got.ScrapeStatus != "matched" { + t.Fatalf("ScrapeStatus = %q, want 'matched'", got.ScrapeStatus) + } + if got.PosterURL != "https://pics.dmm.co.jp/digital/video/ipx00235/ipx00235pl.jpg" && got.PosterURL != adultServer.URL+"/pics/cover/ipx235.jpg" { + t.Fatalf("unexpected PosterURL = %q", got.PosterURL) } } diff --git a/internal/service/service_builder.go b/internal/service/service_builder.go index fc9fafd..d3dbbe7 100644 --- a/internal/service/service_builder.go +++ b/internal/service/service_builder.go @@ -67,7 +67,7 @@ func (b *serviceContainerBuilder) initProviderServices() { b.c.Fanart = NewFanartProvider(b.cfg, b.log) b.c.RecognitionWords = NewRecognitionWordsService(b.log, b.repos) - adult := NewAdultProvider(b.log, b.c.APIConfig) + adult := NewAdultProvider(b.log, b.c.APIConfig, b.repos) b.c.Scraper = NewScraperService( b.cfg, b.log, b.repos, b.c.TMDb, b.c.Bangumi, b.c.TheTVDB, b.c.Fanart, diff --git a/web/src/api/admin.ts b/web/src/api/admin.ts index 6763f93..36fbf16 100644 --- a/web/src/api/admin.ts +++ b/web/src/api/admin.ts @@ -54,4 +54,22 @@ export const adminAPI = { systemUpdateCheck: () => api.post('/admin/system/update/check').then((r) => r.data), systemUpdateApply: () => api.post('/admin/system/update/apply').then((r) => r.data), + + testAdultScraper: (payload: { + engine?: string + server_url?: string + token?: string + javdb_url?: string + javbus_url?: string + cookie?: string + }) => + api + .post<{ + success: boolean + latency_ms?: number + providers?: string[] + message?: string + error?: string + }>('/admin/adult/test-scraper', payload) + .then((r) => r.data), } diff --git a/web/src/api/api_configs.ts b/web/src/api/api_configs.ts index b909d52..d7afc05 100644 --- a/web/src/api/api_configs.ts +++ b/web/src/api/api_configs.ts @@ -27,4 +27,6 @@ export const apiConfigsAPI = { update: (provider: string, patch: APIConfigPatch) => api.put(`/admin/api-configs/${provider}`, patch).then((r) => r.data), remove: (provider: string) => api.delete(`/admin/api-configs/${provider}`).then((r) => r.data), + test: (provider: string) => + api.post<{ result: string; error?: string }>(`/admin/api-configs/${provider}/test`).then((r) => r.data), } diff --git a/web/src/components/APIConfigsPanel.tsx b/web/src/components/APIConfigsPanel.tsx index ca45b4b..fd9cc9c 100644 --- a/web/src/components/APIConfigsPanel.tsx +++ b/web/src/components/APIConfigsPanel.tsx @@ -181,18 +181,24 @@ function EditingRow({
{item.provider} - {!isAdult && ( - - )} +