// Package helper provides shared HTTP client utilities // with browser-like headers and Cloudflare/WAF bypass support. package helper import ( "encoding/json" "fmt" "io" "net/http" "net/url" "strings" "time" "github.com/ShukeBta/MediaStationGo/internal/model" "go.uber.org/zap" ) // NewSiteHTTPClient builds an http.Client honoring per-site policies: // - timeout (seconds, defaults to 15) // - proxy via HTTP(S)_PROXY environment variables when site.UseProxy is on // // When useProxy is false, the client is created without proxy plumbing so // the request goes out direct, matching the user's checkbox intent. func NewSiteHTTPClient(timeoutSeconds int, useProxy bool) *http.Client { if timeoutSeconds <= 0 { timeoutSeconds = 15 } tr := &http.Transport{ MaxIdleConns: 16, MaxIdleConnsPerHost: 4, IdleConnTimeout: 60 * time.Second, TLSHandshakeTimeout: 15 * time.Second, ExpectContinueTimeout: 1 * time.Second, } if useProxy { tr.Proxy = func(r *http.Request) (*url.URL, error) { return ProxyFromEnvironmentOrSystem(r) } } return &http.Client{ Timeout: time.Duration(timeoutSeconds) * time.Second, Transport: tr, } } // HTTPHeaderPresets returns a map of realistic browser HTTP headers. // These mimic a real Chrome browser to avoid WAF/bot detection. func HTTPHeaderPresets() map[string]string { return map[string]string{ "User-Agent": model.DefaultUserAgent, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", "Cache-Control": "max-age=0", } } // ─── FlareSolverr Support ─────────────────────────────────────────────── // FlareSolverrRequest represents a request to FlareSolverr. type FlareSolverrRequest struct { Cmd string `json:"cmd"` URL string `json:"url"` Session string `json:"session,omitempty"` MaxTimeout int `json:"maxTimeout,omitempty"` Proxy *FlareSolverrProxy `json:"proxy,omitempty"` Cookies []FlareSolverrCookie `json:"cookies,omitempty"` } // FlareSolverrProxy represents proxy config for FlareSolverr. type FlareSolverrProxy struct { URL string `json:"url"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` } // FlareSolverrCookie represents a cookie for FlareSolverr. type FlareSolverrCookie struct { Name string `json:"name"` Value string `json:"value"` Domain string `json:"domain,omitempty"` Path string `json:"path,omitempty"` } // FlareSolverrResponse represents FlareSolverr's response. type FlareSolverrResponse struct { Status string `json:"status"` Message string `json:"message"` Solution *FlareSolverrSolution `json:"solution,omitempty"` } // FlareSolverrSolution contains the solved challenge result. type FlareSolverrSolution struct { URL string `json:"url"` Status int `json:"status"` Headers map[string]string `json:"headers"` Cookies []FlareSolverrCookie `json:"cookies"` UserAgent string `json:"userAgent"` Response string `json:"response"` } // FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL, // bypassing Cloudflare/WAF challenges. func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) { if flareSolverrURL == "" { return "", fmt.Errorf("FlareSolverr URL not configured") } if timeout <= 0 { timeout = 60 } // Parse cookies var cookies []FlareSolverrCookie if cookieStr != "" { cookies = parseCookiesForFlareSolverr(cookieStr) } // Build request reqBody := FlareSolverrRequest{ Cmd: "request.get", URL: targetURL, MaxTimeout: timeout * 1000, Cookies: cookies, } if proxyURL != "" { reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL} } jsonBody, err := json.Marshal(reqBody) if err != nil { return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err) } // Send request to FlareSolverr client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second} resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody))) if err != nil { return "", fmt.Errorf("FlareSolverr request failed: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("failed to read FlareSolverr response: %w", err) } var fsResp FlareSolverrResponse if err := json.Unmarshal(body, &fsResp); err != nil { return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err) } if fsResp.Status != "ok" { return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message) } if fsResp.Solution != nil { return fsResp.Solution.Response, nil } return "", fmt.Errorf("FlareSolverr returned no solution") } // parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format. func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie { var cookies []FlareSolverrCookie parts := strings.Split(cookieStr, ";") for _, part := range parts { part = strings.TrimSpace(part) kv := strings.SplitN(part, "=", 2) if len(kv) == 2 { cookies = append(cookies, FlareSolverrCookie{ Name: kv[0], Value: kv[1], }) } } return cookies } // ─── Cloudflare Challenge Detection ───────────────────────────────────── // isCloudflareChallenge checks if the HTML content is a Cloudflare challenge page. func IsCloudflareChallenge(html string) bool { challengeTitles := []string{ "Just a moment...", "请稍候…", "DDOS-GUARD", } challengeSelectors := []string{ "#cf-challenge-running", ".ray_id", ".attack-box", "#cf-please-wait", "#challenge-spinner", "#trk_jschal_js", } lowerHTML := strings.ToLower(html) for _, title := range challengeTitles { // Check for