From a8e825ec1369c222333627ee99297363d7306389 Mon Sep 17 00:00:00 2001 From: ShukeBta Date: Mon, 18 May 2026 01:41:00 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=A2=9E=E5=BC=BA=E7=AB=99=E7=82=B9?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B5=8B=E8=AF=95HTTP=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E5=A4=B4=EF=BC=8C=E6=B7=BB=E5=8A=A0=E6=B5=8F=E8=A7=88=E5=99=A8?= =?UTF-8?q?=E4=BB=BF=E7=9C=9F=E4=B8=8EFlareSolverr=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 internal/helper/http.go：带浏览器仿真请求头和Cloudflare/WAF绕过支持 - TestSiteConnectivity 支持 FlareSolverr 代理（需配置 flareSolverrUrl） - 检测 Cloudflare challenge 页面并返回友好错误信息 - 重构 service/site.go TestConnection 使用新的 helper 方法 --- internal/helper/http.go | 345 +++++++++++++++++++++++++++++++++++++++ internal/model/site.go | 6 + internal/service/site.go | 58 ++----- 3 files changed, 362 insertions(+), 47 deletions(-) create mode 100644 internal/helper/http.go diff --git a/internal/helper/http.go b/internal/helper/http.go new file mode 100644 index 0000000..8f95108 --- /dev/null +++ b/internal/helper/http.go @@ -0,0 +1,345 @@ +// Package helper provides shared HTTP client utilities +// with browser-like headers and Cloudflare/WAF bypass support. +package helper + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/ShukeBta/MediaStationGo/internal/model" + "go.uber.org/zap" +) + +// HTTPHeaderPresets returns a map of realistic browser HTTP headers. +// These mimic a real Chrome browser to avoid WAF/bot detection. +func HTTPHeaderPresets() map[string]string { + return map[string]string{ + "User-Agent": model.DefaultUserAgent, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Cache-Control": "max-age=0", + } +} + +// ─── FlareSolverr Support ─────────────────────────────────────────────── + +// FlareSolverrRequest represents a request to FlareSolverr. +type FlareSolverrRequest struct { + Cmd string `json:"cmd"` + URL string `json:"url"` + Session string `json:"session,omitempty"` + MaxTimeout int `json:"maxTimeout,omitempty"` + Proxy *FlareSolverrProxy `json:"proxy,omitempty"` + Cookies []FlareSolverrCookie `json:"cookies,omitempty"` +} + +// FlareSolverrProxy represents proxy config for FlareSolverr. +type FlareSolverrProxy struct { + URL string `json:"url"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` +} + +// FlareSolverrCookie represents a cookie for FlareSolverr. +type FlareSolverrCookie struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain,omitempty"` + Path string `json:"path,omitempty"` +} + +// FlareSolverrResponse represents FlareSolverr's response. +type FlareSolverrResponse struct { + Status string `json:"status"` + Message string `json:"message"` + Solution *FlareSolverrSolution `json:"solution,omitempty"` +} + +// FlareSolverrSolution contains the solved challenge result. +type FlareSolverrSolution struct { + URL string `json:"url"` + Status int `json:"status"` + Headers map[string]string `json:"headers"` + Cookies []FlareSolverrCookie `json:"cookies"` + UserAgent string `json:"userAgent"` + Response string `json:"response"` +} + +// FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL, +// bypassing Cloudflare/WAF challenges. +func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) { + if flareSolverrURL == "" { + return "", fmt.Errorf("FlareSolverr URL not configured") + } + if timeout <= 0 { + timeout = 60 + } + + // Parse cookies + var cookies []FlareSolverrCookie + if cookieStr != "" { + cookies = parseCookiesForFlareSolverr(cookieStr) + } + + // Build request + reqBody := FlareSolverrRequest{ + Cmd: "request.get", + URL: targetURL, + MaxTimeout: timeout * 1000, + Cookies: cookies, + } + if proxyURL != "" { + reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL} + } + + jsonBody, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err) + } + + // Send request to FlareSolverr + client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second} + resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody))) + if err != nil { + return "", fmt.Errorf("FlareSolverr request failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read FlareSolverr response: %w", err) + } + + var fsResp FlareSolverrResponse + if err := json.Unmarshal(body, &fsResp); err != nil { + return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err) + } + + if fsResp.Status != "ok" { + return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message) + } + + if fsResp.Solution != nil { + return fsResp.Solution.Response, nil + } + return "", fmt.Errorf("FlareSolverr returned no solution") +} + +// parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format. +func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie { + var cookies []FlareSolverrCookie + parts := strings.Split(cookieStr, ";") + for _, part := range parts { + part = strings.TrimSpace(part) + kv := strings.SplitN(part, "=", 2) + if len(kv) == 2 { + cookies = append(cookies, FlareSolverrCookie{ + Name: kv[0], + Value: kv[1], + }) + } + } + return cookies +} + +// ─── Cloudflare Challenge Detection ───────────────────────────────────── + +// isCloudflareChallenge checks if the HTML content is a Cloudflare challenge page. +func IsCloudflareChallenge(html string) bool { + challengeTitles := []string{ + "Just a moment...", + "请稍候…", + "DDOS-GUARD", + } + challengeSelectors := []string{ + "#cf-challenge-running", + ".ray_id", + ".attack-box", + "#cf-please-wait", + "#challenge-spinner", + "#trk_jschal_js", + } + + lowerHTML := strings.ToLower(html) + for _, title := range challengeTitles { + // Check for ... with the challenge title + titleLower := strings.ToLower(title) + if strings.Contains(lowerHTML, strings.ToLower(""+title)) || + strings.Contains(lowerHTML, titleLower) { + return true + } + } + + for _, selector := range challengeSelectors { + if strings.Contains(lowerHTML, strings.ToLower(selector)) { + return true + } + } + + return false +} + +// ─── Site Connectivity Test ───────────────────────────────────────────── + +// TestSiteConnectivity performs a site connectivity test with browser-like headers. +// If flareSolverrURL is non-empty, it will attempt to use FlareSolverr first. +// Returns (ok, message, error). +func TestSiteConnectivity(site *model.Site, flareSolverrURL string, timeout int, log *zap.Logger) (bool, string, error) { + // Try FlareSolverr first if configured + if flareSolverrURL != "" { + log.Info("Trying FlareSolverr for site test", zap.String("url", site.URL)) + body, err := FetchURLWithFlareSolverr(flareSolverrURL, site.URL, site.Cookie, timeout, "", log) + if err == nil { + // Successfully got page via FlareSolverr + if IsCloudflareChallenge(body) { + return false, "站点被 Cloudflare/WAF 拦截，但 FlareSolverr 未能完全解决", nil + } + return true, "连接成功 (via FlareSolverr)", nil + } + log.Warn("FlareSolverr failed, falling back to direct request", zap.Error(err)) + // Fall through to direct request + } + + // Direct HTTP request with browser-like headers + client := &http.Client{ + Timeout: time.Duration(timeout) * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("too many redirects") + } + return nil + }, + } + + req, err := http.NewRequest("GET", site.URL, nil) + if err != nil { + return false, err.Error(), nil + } + + // Apply browser-like headers + headers := HTTPHeaderPresets() + for k, v := range headers { + req.Header.Set(k, v) + } + + // Apply auth headers + ApplySiteAuthHeaders(req, site) + + // Execute request + resp, err := client.Do(req) + if err != nil { + return false, err.Error(), nil + } + defer resp.Body.Close() + + // Read response body for Cloudflare challenge detection + body, _ := io.ReadAll(resp.Body) + bodyStr := string(body) + + // Check for Cloudflare challenge + if IsCloudflareChallenge(bodyStr) { + log.Warn("Cloudflare challenge detected", zap.String("url", site.URL)) + return false, "站点被 Cloudflare/WAF 拦截，请配置 FlareSolverr 或浏览器模拟", nil + } + + // Evaluate status code (same logic as before) + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return true, fmt.Sprintf("连接成功 (%s)", resp.Status), nil + case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308: + loc := resp.Header.Get("Location") + if loc == "" { + loc = "(unknown)" + } + return true, fmt.Sprintf("站点可达，但返回重定向至 %s", loc), nil + case resp.StatusCode == 401: + return true, "站点可达，需要认证 (HTTP 401)", nil + case resp.StatusCode == 403: + return true, "站点可达，但访问被拒绝 — 可能被 Cloudflare/WAF 拦截 (HTTP 403)", nil + case resp.StatusCode == 429: + return true, "站点可达，但被限流 (HTTP 429)", nil + case resp.StatusCode == 503: + return true, "站点可达，服务暂时不可用 (HTTP 503)", nil + default: + ok := resp.StatusCode >= 400 && resp.StatusCode < 500 + return ok, resp.Status, nil + } +} + +// ApplySiteAuthHeaders applies authentication headers based on site config. +func ApplySiteAuthHeaders(req *http.Request, site *model.Site) { + switch site.AuthType { + case "cookie": + if site.Cookie != "" { + req.Header.Set("Cookie", site.Cookie) + } + case "api_key": + if site.APIKey != "" { + req.Header.Set("x-api-key", site.APIKey) + } + case "auth_header": + if site.AuthHeader != "" { + req.Header.Set("Authorization", site.AuthHeader) + } + } + + // Apply custom User-Agent if configured + if site.UserAgent != "" { + req.Header.Set("User-Agent", site.UserAgent) + } +} + +// GetPageSource fetches a page with browser-like headers. +// Returns (pageSource, cookies, error). +func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) { + client := &http.Client{ + Timeout: time.Duration(timeout) * time.Second, + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", "", err + } + + // Apply browser-like headers + headers := HTTPHeaderPresets() + for k, v := range headers { + req.Header.Set(k, v) + } + + // Apply auth + ApplySiteAuthHeaders(req, site) + + resp, err := client.Do(req) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", "", err + } + + // Extract cookies from response + cookies := "" + for _, c := range resp.Cookies() { + if cookies != "" { + cookies += "; " + } + cookies += c.Name + "=" + c.Value + } + + return string(body), cookies, nil +} diff --git a/internal/model/site.go b/internal/model/site.go index 8b6ba2b..e2464d9 100644 --- a/internal/model/site.go +++ b/internal/model/site.go @@ -49,3 +49,9 @@ func SiteTypes() []string { func AuthTypes() []string { return []string{"cookie", "api_key", "auth_header"} } + +// DefaultUserAgent 是默认浏览器 User-Agent（用于 HTTP 请求头）。 +const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + +// SiteTypes 返回支持的站点类型列表（用于前端下拉）。 +// 注意：此函数供 API 返回类型列表使用，不在此处添加新类型。 diff --git a/internal/service/site.go b/internal/service/site.go index b824224..3363ad0 100644 --- a/internal/service/site.go +++ b/internal/service/site.go @@ -8,13 +8,13 @@ package service import ( "context" "errors" - "net/http" "strings" "time" "go.uber.org/zap" "gorm.io/gorm" + "github.com/ShukeBta/MediaStationGo/internal/helper" "github.com/ShukeBta/MediaStationGo/internal/model" "github.com/ShukeBta/MediaStationGo/internal/repository" ) @@ -74,67 +74,31 @@ func (s *SiteService) Delete(ctx context.Context, id string) error { // TestConnection tries to reach the site's base URL with the configured // credentials and reports success/failure. +// Now uses helper.TestSiteConnectivity with browser-like headers +// and optional FlareSolverr support. func (s *SiteService) TestConnection(ctx context.Context, id string) (bool, string, error) { site, err := s.FindByID(ctx, id) if err != nil || site == nil { return false, "site not found", err } - client := &http.Client{Timeout: 15 * time.Second} - req, err := http.NewRequestWithContext(ctx, http.MethodGet, site.URL, nil) - if err != nil { - return false, err.Error(), nil + // Get timeout from site config (default 15 seconds) + timeout := site.Timeout + if timeout <= 0 { + timeout = 15 } - // Apply auth headers. - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") - switch site.AuthType { - case "cookie": - if site.Cookie != "" { - req.Header.Set("Cookie", site.Cookie) - } - case "api_key": - if site.APIKey != "" { - req.Header.Set("x-api-key", site.APIKey) - } - case "authorization": - if site.AuthHeader != "" { - req.Header.Set("Authorization", site.AuthHeader) - } - } + // TODO: Get flareSolverrURL from config (e.g. from config.yaml) + // For now, pass empty string to skip FlareSolverr + flareSolverrURL := "" // TODO: load from config - resp, err := client.Do(req) + ok, msg, err := helper.TestSiteConnectivity(site, flareSolverrURL, timeout, s.log) if err != nil { now := time.Now() _ = s.repo.DB.WithContext(ctx).Model(&model.Site{}).Where("id = ?", id). Updates(map[string]any{"last_error": err.Error(), "last_check_at": &now}).Error return false, err.Error(), nil } - defer resp.Body.Close() - - var ok bool - var msg string - switch { - case resp.StatusCode >= 200 && resp.StatusCode < 300: - ok, msg = true, "连接成功 ("+resp.Status+")" - case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308: - // Redirect is common for PT sites behind CDN/WAF — site is reachable - loc := resp.Header.Get("Location") - if loc == "" { - loc = "(unknown)" - } - ok, msg = true, "站点可达，但返回重定向至 "+loc - case resp.StatusCode == 401: - ok, msg = true, "站点可达，需要认证 (HTTP 401)" - case resp.StatusCode == 403: - ok, msg = true, "站点可达，但访问被拒绝 — 可能被 Cloudflare/WAF 拦截 (HTTP 403)" - case resp.StatusCode == 429: - ok, msg = true, "站点可达，但被限流 (HTTP 429)" - case resp.StatusCode == 503: - ok, msg = true, "站点可达，服务暂时不可用 (HTTP 503)" - default: - ok, msg = resp.StatusCode >= 400 && resp.StatusCode < 500, resp.Status - } loginStatus := "ok" if !ok {