From d53d01b419adca6ff14dc7e8f1d6f19727d4714d Mon Sep 17 00:00:00 2001 From: ShukeBta <272197458+ShukeBta@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:14:38 +0800 Subject: [PATCH] split helper http utilities --- internal/helper/cloudflare.go | 37 +++ internal/helper/flaresolverr.go | 128 ++++++++++ internal/helper/http.go | 340 --------------------------- internal/helper/page_source.go | 48 ++++ internal/helper/site_auth.go | 50 ++++ internal/helper/site_connectivity.go | 96 ++++++++ 6 files changed, 359 insertions(+), 340 deletions(-) create mode 100644 internal/helper/cloudflare.go create mode 100644 internal/helper/flaresolverr.go create mode 100644 internal/helper/page_source.go create mode 100644 internal/helper/site_auth.go create mode 100644 internal/helper/site_connectivity.go diff --git a/internal/helper/cloudflare.go b/internal/helper/cloudflare.go new file mode 100644 index 0000000..e52b1bb --- /dev/null +++ b/internal/helper/cloudflare.go @@ -0,0 +1,37 @@ +package helper + +import "strings" + +// IsCloudflareChallenge checks if the HTML content is a Cloudflare challenge page. +func IsCloudflareChallenge(html string) bool { + challengeTitles := []string{ + "Just a moment...", + "请稍候…", + "DDOS-GUARD", + } + challengeSelectors := []string{ + "#cf-challenge-running", + ".ray_id", + ".attack-box", + "#cf-please-wait", + "#challenge-spinner", + "#trk_jschal_js", + } + + lowerHTML := strings.ToLower(html) + for _, title := range challengeTitles { + titleLower := strings.ToLower(title) + if strings.Contains(lowerHTML, strings.ToLower(""+title)) || + strings.Contains(lowerHTML, titleLower) { + return true + } + } + + for _, selector := range challengeSelectors { + if strings.Contains(lowerHTML, strings.ToLower(selector)) { + return true + } + } + + return false +} diff --git a/internal/helper/flaresolverr.go b/internal/helper/flaresolverr.go new file mode 100644 index 0000000..d001c24 --- /dev/null +++ b/internal/helper/flaresolverr.go @@ -0,0 +1,128 @@ +package helper + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "go.uber.org/zap" +) + +// FlareSolverrRequest represents a request to FlareSolverr. +type FlareSolverrRequest struct { + Cmd string `json:"cmd"` + URL string `json:"url"` + Session string `json:"session,omitempty"` + MaxTimeout int `json:"maxTimeout,omitempty"` + Proxy *FlareSolverrProxy `json:"proxy,omitempty"` + Cookies []FlareSolverrCookie `json:"cookies,omitempty"` +} + +// FlareSolverrProxy represents proxy config for FlareSolverr. +type FlareSolverrProxy struct { + URL string `json:"url"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` +} + +// FlareSolverrCookie represents a cookie for FlareSolverr. +type FlareSolverrCookie struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain,omitempty"` + Path string `json:"path,omitempty"` +} + +// FlareSolverrResponse represents FlareSolverr's response. +type FlareSolverrResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Solution *FlareSolverrSolution `json:"solution,omitempty"` +} + +// FlareSolverrSolution contains the solved challenge result. +type FlareSolverrSolution struct { + URL string `json:"url"` + Status int `json:"status"` + Headers map[string]string `json:"headers"` + Cookies []FlareSolverrCookie `json:"cookies"` + UserAgent string `json:"userAgent"` + Response string `json:"response"` +} + +// FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL, +// bypassing Cloudflare/WAF challenges. +func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) { + if flareSolverrURL == "" { + return "", fmt.Errorf("FlareSolverr URL not configured") + } + if timeout <= 0 { + timeout = 60 + } + + var cookies []FlareSolverrCookie + if cookieStr != "" { + cookies = parseCookiesForFlareSolverr(cookieStr) + } + + reqBody := FlareSolverrRequest{ + Cmd: "request.get", + URL: targetURL, + MaxTimeout: timeout * 1000, + Cookies: cookies, + } + if proxyURL != "" { + reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL} + } + + jsonBody, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err) + } + + client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second} + resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody))) + if err != nil { + return "", fmt.Errorf("FlareSolverr request failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read FlareSolverr response: %w", err) + } + + var fsResp FlareSolverrResponse + if err := json.Unmarshal(body, &fsResp); err != nil { + return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err) + } + + if fsResp.Status != "ok" { + return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message) + } + + if fsResp.Solution != nil { + return fsResp.Solution.Response, nil + } + return "", fmt.Errorf("FlareSolverr returned no solution") +} + +// parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format. +func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie { + var cookies []FlareSolverrCookie + parts := strings.Split(cookieStr, ";") + for _, part := range parts { + part = strings.TrimSpace(part) + kv := strings.SplitN(part, "=", 2) + if len(kv) == 2 { + cookies = append(cookies, FlareSolverrCookie{ + Name: kv[0], + Value: kv[1], + }) + } + } + return cookies +} diff --git a/internal/helper/http.go b/internal/helper/http.go index c4abc65..4d85d41 100644 --- a/internal/helper/http.go +++ b/internal/helper/http.go @@ -3,16 +3,11 @@ package helper import ( - "encoding/json" - "fmt" - "io" "net/http" "net/url" - "strings" "time" "github.com/ShukeBta/MediaStationGo/internal/model" - "go.uber.org/zap" ) // NewSiteHTTPClient builds an http.Client honoring per-site policies: @@ -60,338 +55,3 @@ func HTTPHeaderPresets() map[string]string { "Cache-Control": "max-age=0", } } - -// ─── FlareSolverr Support ─────────────────────────────────────────────── - -// FlareSolverrRequest represents a request to FlareSolverr. -type FlareSolverrRequest struct { - Cmd string `json:"cmd"` - URL string `json:"url"` - Session string `json:"session,omitempty"` - MaxTimeout int `json:"maxTimeout,omitempty"` - Proxy *FlareSolverrProxy `json:"proxy,omitempty"` - Cookies []FlareSolverrCookie `json:"cookies,omitempty"` -} - -// FlareSolverrProxy represents proxy config for FlareSolverr. -type FlareSolverrProxy struct { - URL string `json:"url"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` -} - -// FlareSolverrCookie represents a cookie for FlareSolverr. -type FlareSolverrCookie struct { - Name string `json:"name"` - Value string `json:"value"` - Domain string `json:"domain,omitempty"` - Path string `json:"path,omitempty"` -} - -// FlareSolverrResponse represents FlareSolverr's response. -type FlareSolverrResponse struct { - Status string `json:"status"` - Message string `json:"message"` - Solution *FlareSolverrSolution `json:"solution,omitempty"` -} - -// FlareSolverrSolution contains the solved challenge result. -type FlareSolverrSolution struct { - URL string `json:"url"` - Status int `json:"status"` - Headers map[string]string `json:"headers"` - Cookies []FlareSolverrCookie `json:"cookies"` - UserAgent string `json:"userAgent"` - Response string `json:"response"` -} - -// FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL, -// bypassing Cloudflare/WAF challenges. -func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) { - if flareSolverrURL == "" { - return "", fmt.Errorf("FlareSolverr URL not configured") - } - if timeout <= 0 { - timeout = 60 - } - - // Parse cookies - var cookies []FlareSolverrCookie - if cookieStr != "" { - cookies = parseCookiesForFlareSolverr(cookieStr) - } - - // Build request - reqBody := FlareSolverrRequest{ - Cmd: "request.get", - URL: targetURL, - MaxTimeout: timeout * 1000, - Cookies: cookies, - } - if proxyURL != "" { - reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL} - } - - jsonBody, err := json.Marshal(reqBody) - if err != nil { - return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err) - } - - // Send request to FlareSolverr - client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second} - resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody))) - if err != nil { - return "", fmt.Errorf("FlareSolverr request failed: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read FlareSolverr response: %w", err) - } - - var fsResp FlareSolverrResponse - if err := json.Unmarshal(body, &fsResp); err != nil { - return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err) - } - - if fsResp.Status != "ok" { - return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message) - } - - if fsResp.Solution != nil { - return fsResp.Solution.Response, nil - } - return "", fmt.Errorf("FlareSolverr returned no solution") -} - -// parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format. -func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie { - var cookies []FlareSolverrCookie - parts := strings.Split(cookieStr, ";") - for _, part := range parts { - part = strings.TrimSpace(part) - kv := strings.SplitN(part, "=", 2) - if len(kv) == 2 { - cookies = append(cookies, FlareSolverrCookie{ - Name: kv[0], - Value: kv[1], - }) - } - } - return cookies -} - -// ─── Cloudflare Challenge Detection ───────────────────────────────────── - -// isCloudflareChallenge checks if the HTML content is a Cloudflare challenge page. -func IsCloudflareChallenge(html string) bool { - challengeTitles := []string{ - "Just a moment...", - "请稍候…", - "DDOS-GUARD", - } - challengeSelectors := []string{ - "#cf-challenge-running", - ".ray_id", - ".attack-box", - "#cf-please-wait", - "#challenge-spinner", - "#trk_jschal_js", - } - - lowerHTML := strings.ToLower(html) - for _, title := range challengeTitles { - // Check for <title>... with the challenge title - titleLower := strings.ToLower(title) - if strings.Contains(lowerHTML, strings.ToLower(""+title)) || - strings.Contains(lowerHTML, titleLower) { - return true - } - } - - for _, selector := range challengeSelectors { - if strings.Contains(lowerHTML, strings.ToLower(selector)) { - return true - } - } - - return false -} - -// ─── Site Connectivity Test ───────────────────────────────────────────── - -// TestSiteConnectivity performs a site connectivity test with browser-like headers. -// If flareSolverrURL is non-empty AND the site has BrowserEmulation turned on, -// it will attempt to use FlareSolverr first. -// Returns (ok, message, error). -func TestSiteConnectivity(site *model.Site, flareSolverrURL string, timeout int, log *zap.Logger) (bool, string, error) { - // Try FlareSolverr first when (a) globally enabled and (b) the site - // asked for browser emulation. This matches the contract used by the - // search path (see service.SiteService.siteModelToConfig). - useFlare := flareSolverrURL != "" && site.BrowserEmulation - if useFlare { - log.Info("Trying FlareSolverr for site test", zap.String("url", site.URL)) - body, err := FetchURLWithFlareSolverr(flareSolverrURL, site.URL, site.Cookie, timeout, "", log) - if err == nil { - // Successfully got page via FlareSolverr - if IsCloudflareChallenge(body) { - return false, "站点被 Cloudflare/WAF 拦截,但 FlareSolverr 未能完全解决", nil - } - return true, "连接成功 (via FlareSolverr)", nil - } - log.Warn("FlareSolverr failed, falling back to direct request", zap.Error(err)) - // Fall through to direct request - } - - // Direct HTTP request with browser-like headers. Honors HTTP(S)_PROXY - // when the site has UseProxy enabled — this makes the "use proxy" - // checkbox in the UI actually do something. - client := NewSiteHTTPClient(timeout, site.UseProxy) - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - if len(via) >= 10 { - return fmt.Errorf("too many redirects") - } - return nil - } - - req, err := http.NewRequest("GET", site.URL, nil) - if err != nil { - return false, err.Error(), nil - } - - // Apply browser-like headers - headers := HTTPHeaderPresets() - for k, v := range headers { - req.Header.Set(k, v) - } - - // Apply auth headers - ApplySiteAuthHeaders(req, site) - - // Execute request - resp, err := client.Do(req) - if err != nil { - return false, err.Error(), nil - } - defer resp.Body.Close() - - // Read response body for Cloudflare challenge detection - body, _ := io.ReadAll(resp.Body) - bodyStr := string(body) - - // Check for Cloudflare challenge - if IsCloudflareChallenge(bodyStr) { - log.Warn("Cloudflare challenge detected", zap.String("url", site.URL)) - return false, "站点被 Cloudflare/WAF 拦截,请配置 FlareSolverr 或浏览器模拟", nil - } - - // Evaluate status code (mirror the reference Python project's semantics: - // 200 → success, 3xx → success/redirect-to-self, 401/403 → failure with - // hint to check credentials, 4xx/5xx → failure with raw status text). - switch { - case resp.StatusCode >= 200 && resp.StatusCode < 300: - return true, fmt.Sprintf("连接成功 (%s)", resp.Status), nil - case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308: - loc := resp.Header.Get("Location") - if loc == "" { - loc = "(unknown)" - } - // Most PT sites redirect logged-out users to login; treat as failure. - return false, fmt.Sprintf("未登录或 Cookie 失效(重定向至 %s)", loc), nil - case resp.StatusCode == 401: - return false, "未授权(HTTP 401),请检查 API Key / Cookie", nil - case resp.StatusCode == 403: - return false, "认证失败(HTTP 403),请检查 Cookie / API Key 或站点是否需要浏览器模拟", nil - case resp.StatusCode == 429: - return false, "请求被限流(HTTP 429),请稍后再试", nil - case resp.StatusCode == 503: - return false, "服务暂时不可用(HTTP 503)", nil - default: - return false, resp.Status, nil - } -} - -// ApplySiteAuthHeaders applies authentication headers based on site config. -func ApplySiteAuthHeaders(req *http.Request, site *model.Site) { - switch site.AuthType { - case "cookie": - if site.Cookie != "" { - req.Header.Set("Cookie", site.Cookie) - } - case "api_key": - if site.APIKey != "" { - if isYemaPTSite(site) { - req.Header.Set("Authorization", site.APIKey) - } else { - req.Header.Set("x-api-key", site.APIKey) - } - } - case "auth_header": - if site.AuthHeader != "" { - req.Header.Set("Authorization", site.AuthHeader) - } - } - - // Apply custom User-Agent if configured - if site.UserAgent != "" { - req.Header.Set("User-Agent", site.UserAgent) - } -} - -func isYemaPTSite(site *model.Site) bool { - if site == nil { - return false - } - if strings.EqualFold(strings.TrimSpace(site.Type), "yemapt") { - return true - } - u, err := url.Parse(strings.TrimSpace(site.URL)) - if err != nil { - return false - } - host := strings.ToLower(u.Hostname()) - return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org") -} - -// GetPageSource fetches a page with browser-like headers. -// Returns (pageSource, cookies, error). -func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) { - client := NewSiteHTTPClient(timeout, site.UseProxy) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", "", err - } - - // Apply browser-like headers - headers := HTTPHeaderPresets() - for k, v := range headers { - req.Header.Set(k, v) - } - - // Apply auth - ApplySiteAuthHeaders(req, site) - - resp, err := client.Do(req) - if err != nil { - return "", "", err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", "", err - } - - // Extract cookies from response - cookies := "" - for _, c := range resp.Cookies() { - if cookies != "" { - cookies += "; " - } - cookies += c.Name + "=" + c.Value - } - - return string(body), cookies, nil -} diff --git a/internal/helper/page_source.go b/internal/helper/page_source.go new file mode 100644 index 0000000..7adb265 --- /dev/null +++ b/internal/helper/page_source.go @@ -0,0 +1,48 @@ +package helper + +import ( + "io" + "net/http" + + "github.com/ShukeBta/MediaStationGo/internal/model" + "go.uber.org/zap" +) + +// GetPageSource fetches a page with browser-like headers. +// Returns (pageSource, cookies, error). +func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) { + client := NewSiteHTTPClient(timeout, site.UseProxy) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", "", err + } + + headers := HTTPHeaderPresets() + for k, v := range headers { + req.Header.Set(k, v) + } + + ApplySiteAuthHeaders(req, site) + + resp, err := client.Do(req) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", "", err + } + + cookies := "" + for _, c := range resp.Cookies() { + if cookies != "" { + cookies += "; " + } + cookies += c.Name + "=" + c.Value + } + + return string(body), cookies, nil +} diff --git a/internal/helper/site_auth.go b/internal/helper/site_auth.go new file mode 100644 index 0000000..bb7c96f --- /dev/null +++ b/internal/helper/site_auth.go @@ -0,0 +1,50 @@ +package helper + +import ( + "net/http" + "net/url" + "strings" + + "github.com/ShukeBta/MediaStationGo/internal/model" +) + +// ApplySiteAuthHeaders applies authentication headers based on site config. +func ApplySiteAuthHeaders(req *http.Request, site *model.Site) { + switch site.AuthType { + case "cookie": + if site.Cookie != "" { + req.Header.Set("Cookie", site.Cookie) + } + case "api_key": + if site.APIKey != "" { + if isYemaPTSite(site) { + req.Header.Set("Authorization", site.APIKey) + } else { + req.Header.Set("x-api-key", site.APIKey) + } + } + case "auth_header": + if site.AuthHeader != "" { + req.Header.Set("Authorization", site.AuthHeader) + } + } + + if site.UserAgent != "" { + req.Header.Set("User-Agent", site.UserAgent) + } +} + +func isYemaPTSite(site *model.Site) bool { + if site == nil { + return false + } + if strings.EqualFold(strings.TrimSpace(site.Type), "yemapt") { + return true + } + u, err := url.Parse(strings.TrimSpace(site.URL)) + if err != nil { + return false + } + host := strings.ToLower(u.Hostname()) + return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org") +} diff --git a/internal/helper/site_connectivity.go b/internal/helper/site_connectivity.go new file mode 100644 index 0000000..0626ffe --- /dev/null +++ b/internal/helper/site_connectivity.go @@ -0,0 +1,96 @@ +package helper + +import ( + "fmt" + "io" + "net/http" + + "github.com/ShukeBta/MediaStationGo/internal/model" + "go.uber.org/zap" +) + +// TestSiteConnectivity performs a site connectivity test with browser-like headers. +// If flareSolverrURL is non-empty AND the site has BrowserEmulation turned on, +// it will attempt to use FlareSolverr first. +// Returns (ok, message, error). +func TestSiteConnectivity(site *model.Site, flareSolverrURL string, timeout int, log *zap.Logger) (bool, string, error) { + // Try FlareSolverr first when (a) globally enabled and (b) the site + // asked for browser emulation. This matches the contract used by the + // search path (see service.SiteService.siteModelToConfig). + useFlare := flareSolverrURL != "" && site.BrowserEmulation + if useFlare { + log.Info("Trying FlareSolverr for site test", zap.String("url", site.URL)) + body, err := FetchURLWithFlareSolverr(flareSolverrURL, site.URL, site.Cookie, timeout, "", log) + if err == nil { + // Successfully got page via FlareSolverr + if IsCloudflareChallenge(body) { + return false, "站点被 Cloudflare/WAF 拦截,但 FlareSolverr 未能完全解决", nil + } + return true, "连接成功 (via FlareSolverr)", nil + } + log.Warn("FlareSolverr failed, falling back to direct request", zap.Error(err)) + // Fall through to direct request + } + + // Direct HTTP request with browser-like headers. Honors HTTP(S)_PROXY + // when the site has UseProxy enabled — this makes the "use proxy" + // checkbox in the UI actually do something. + client := NewSiteHTTPClient(timeout, site.UseProxy) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("too many redirects") + } + return nil + } + + req, err := http.NewRequest("GET", site.URL, nil) + if err != nil { + return false, err.Error(), nil + } + + headers := HTTPHeaderPresets() + for k, v := range headers { + req.Header.Set(k, v) + } + + ApplySiteAuthHeaders(req, site) + + resp, err := client.Do(req) + if err != nil { + return false, err.Error(), nil + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + bodyStr := string(body) + + if IsCloudflareChallenge(bodyStr) { + log.Warn("Cloudflare challenge detected", zap.String("url", site.URL)) + return false, "站点被 Cloudflare/WAF 拦截,请配置 FlareSolverr 或浏览器模拟", nil + } + + // Evaluate status code (mirror the reference Python project's semantics: + // 200 → success, 3xx → success/redirect-to-self, 401/403 → failure with + // hint to check credentials, 4xx/5xx → failure with raw status text). + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return true, fmt.Sprintf("连接成功 (%s)", resp.Status), nil + case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308: + loc := resp.Header.Get("Location") + if loc == "" { + loc = "(unknown)" + } + // Most PT sites redirect logged-out users to login; treat as failure. + return false, fmt.Sprintf("未登录或 Cookie 失效(重定向至 %s)", loc), nil + case resp.StatusCode == 401: + return false, "未授权(HTTP 401),请检查 API Key / Cookie", nil + case resp.StatusCode == 403: + return false, "认证失败(HTTP 403),请检查 Cookie / API Key 或站点是否需要浏览器模拟", nil + case resp.StatusCode == 429: + return false, "请求被限流(HTTP 429),请稍后再试", nil + case resp.StatusCode == 503: + return false, "服务暂时不可用(HTTP 503)", nil + default: + return false, resp.Status, nil + } +}