mirror of
https://github.com/truewhile/MeBox.git
synced 2026-09-28 11:16:37 +08:00
split helper http utilities
This commit is contained in:
ShukeBta
@@ -0,0 +1,37 @@
|
||||
package helper
|
||||
|
||||
import "strings"
|
||||
|
||||
// IsCloudflareChallenge checks if the HTML content is a Cloudflare challenge page.
|
||||
func IsCloudflareChallenge(html string) bool {
|
||||
challengeTitles := []string{
|
||||
"Just a moment...",
|
||||
"请稍候…",
|
||||
"DDOS-GUARD",
|
||||
}
|
||||
challengeSelectors := []string{
|
||||
"#cf-challenge-running",
|
||||
".ray_id",
|
||||
".attack-box",
|
||||
"#cf-please-wait",
|
||||
"#challenge-spinner",
|
||||
"#trk_jschal_js",
|
||||
}
|
||||
|
||||
lowerHTML := strings.ToLower(html)
|
||||
for _, title := range challengeTitles {
|
||||
titleLower := strings.ToLower(title)
|
||||
if strings.Contains(lowerHTML, strings.ToLower("<title>"+title)) ||
|
||||
strings.Contains(lowerHTML, titleLower) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
for _, selector := range challengeSelectors {
|
||||
if strings.Contains(lowerHTML, strings.ToLower(selector)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// FlareSolverrRequest represents a request to FlareSolverr.
|
||||
type FlareSolverrRequest struct {
|
||||
Cmd string `json:"cmd"`
|
||||
URL string `json:"url"`
|
||||
Session string `json:"session,omitempty"`
|
||||
MaxTimeout int `json:"maxTimeout,omitempty"`
|
||||
Proxy *FlareSolverrProxy `json:"proxy,omitempty"`
|
||||
Cookies []FlareSolverrCookie `json:"cookies,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrProxy represents proxy config for FlareSolverr.
|
||||
type FlareSolverrProxy struct {
|
||||
URL string `json:"url"`
|
||||
Username string `json:"username,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrCookie represents a cookie for FlareSolverr.
|
||||
type FlareSolverrCookie struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrResponse represents FlareSolverr's response.
|
||||
type FlareSolverrResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Solution *FlareSolverrSolution `json:"solution,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrSolution contains the solved challenge result.
|
||||
type FlareSolverrSolution struct {
|
||||
URL string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
Cookies []FlareSolverrCookie `json:"cookies"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
Response string `json:"response"`
|
||||
}
|
||||
|
||||
// FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL,
|
||||
// bypassing Cloudflare/WAF challenges.
|
||||
func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) {
|
||||
if flareSolverrURL == "" {
|
||||
return "", fmt.Errorf("FlareSolverr URL not configured")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 60
|
||||
}
|
||||
|
||||
var cookies []FlareSolverrCookie
|
||||
if cookieStr != "" {
|
||||
cookies = parseCookiesForFlareSolverr(cookieStr)
|
||||
}
|
||||
|
||||
reqBody := FlareSolverrRequest{
|
||||
Cmd: "request.get",
|
||||
URL: targetURL,
|
||||
MaxTimeout: timeout * 1000,
|
||||
Cookies: cookies,
|
||||
}
|
||||
if proxyURL != "" {
|
||||
reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL}
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second}
|
||||
resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody)))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("FlareSolverr request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read FlareSolverr response: %w", err)
|
||||
}
|
||||
|
||||
var fsResp FlareSolverrResponse
|
||||
if err := json.Unmarshal(body, &fsResp); err != nil {
|
||||
return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err)
|
||||
}
|
||||
|
||||
if fsResp.Status != "ok" {
|
||||
return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message)
|
||||
}
|
||||
|
||||
if fsResp.Solution != nil {
|
||||
return fsResp.Solution.Response, nil
|
||||
}
|
||||
return "", fmt.Errorf("FlareSolverr returned no solution")
|
||||
}
|
||||
|
||||
// parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format.
|
||||
func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie {
|
||||
var cookies []FlareSolverrCookie
|
||||
parts := strings.Split(cookieStr, ";")
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
kv := strings.SplitN(part, "=", 2)
|
||||
if len(kv) == 2 {
|
||||
cookies = append(cookies, FlareSolverrCookie{
|
||||
Name: kv[0],
|
||||
Value: kv[1],
|
||||
})
|
||||
}
|
||||
}
|
||||
return cookies
|
||||
}
|
||||
@@ -3,16 +3,11 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// NewSiteHTTPClient builds an http.Client honoring per-site policies:
|
||||
@@ -60,338 +55,3 @@ func HTTPHeaderPresets() map[string]string {
|
||||
"Cache-Control": "max-age=0",
|
||||
}
|
||||
}
|
||||
|
||||
// ─── FlareSolverr Support ───────────────────────────────────────────────
|
||||
|
||||
// FlareSolverrRequest represents a request to FlareSolverr.
|
||||
type FlareSolverrRequest struct {
|
||||
Cmd string `json:"cmd"`
|
||||
URL string `json:"url"`
|
||||
Session string `json:"session,omitempty"`
|
||||
MaxTimeout int `json:"maxTimeout,omitempty"`
|
||||
Proxy *FlareSolverrProxy `json:"proxy,omitempty"`
|
||||
Cookies []FlareSolverrCookie `json:"cookies,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrProxy represents proxy config for FlareSolverr.
|
||||
type FlareSolverrProxy struct {
|
||||
URL string `json:"url"`
|
||||
Username string `json:"username,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrCookie represents a cookie for FlareSolverr.
|
||||
type FlareSolverrCookie struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrResponse represents FlareSolverr's response.
|
||||
type FlareSolverrResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Solution *FlareSolverrSolution `json:"solution,omitempty"`
|
||||
}
|
||||
|
||||
// FlareSolverrSolution contains the solved challenge result.
|
||||
type FlareSolverrSolution struct {
|
||||
URL string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
Cookies []FlareSolverrCookie `json:"cookies"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
Response string `json:"response"`
|
||||
}
|
||||
|
||||
// FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL,
|
||||
// bypassing Cloudflare/WAF challenges.
|
||||
func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) {
|
||||
if flareSolverrURL == "" {
|
||||
return "", fmt.Errorf("FlareSolverr URL not configured")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 60
|
||||
}
|
||||
|
||||
// Parse cookies
|
||||
var cookies []FlareSolverrCookie
|
||||
if cookieStr != "" {
|
||||
cookies = parseCookiesForFlareSolverr(cookieStr)
|
||||
}
|
||||
|
||||
// Build request
|
||||
reqBody := FlareSolverrRequest{
|
||||
Cmd: "request.get",
|
||||
URL: targetURL,
|
||||
MaxTimeout: timeout * 1000,
|
||||
Cookies: cookies,
|
||||
}
|
||||
if proxyURL != "" {
|
||||
reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL}
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err)
|
||||
}
|
||||
|
||||
// Send request to FlareSolverr
|
||||
client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second}
|
||||
resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody)))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("FlareSolverr request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read FlareSolverr response: %w", err)
|
||||
}
|
||||
|
||||
var fsResp FlareSolverrResponse
|
||||
if err := json.Unmarshal(body, &fsResp); err != nil {
|
||||
return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err)
|
||||
}
|
||||
|
||||
if fsResp.Status != "ok" {
|
||||
return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message)
|
||||
}
|
||||
|
||||
if fsResp.Solution != nil {
|
||||
return fsResp.Solution.Response, nil
|
||||
}
|
||||
return "", fmt.Errorf("FlareSolverr returned no solution")
|
||||
}
|
||||
|
||||
// parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format.
|
||||
func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie {
|
||||
var cookies []FlareSolverrCookie
|
||||
parts := strings.Split(cookieStr, ";")
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
kv := strings.SplitN(part, "=", 2)
|
||||
if len(kv) == 2 {
|
||||
cookies = append(cookies, FlareSolverrCookie{
|
||||
Name: kv[0],
|
||||
Value: kv[1],
|
||||
})
|
||||
}
|
||||
}
|
||||
return cookies
|
||||
}
|
||||
|
||||
// ─── Cloudflare Challenge Detection ─────────────────────────────────────
|
||||
|
||||
// isCloudflareChallenge checks if the HTML content is a Cloudflare challenge page.
|
||||
func IsCloudflareChallenge(html string) bool {
|
||||
challengeTitles := []string{
|
||||
"Just a moment...",
|
||||
"请稍候…",
|
||||
"DDOS-GUARD",
|
||||
}
|
||||
challengeSelectors := []string{
|
||||
"#cf-challenge-running",
|
||||
".ray_id",
|
||||
".attack-box",
|
||||
"#cf-please-wait",
|
||||
"#challenge-spinner",
|
||||
"#trk_jschal_js",
|
||||
}
|
||||
|
||||
lowerHTML := strings.ToLower(html)
|
||||
for _, title := range challengeTitles {
|
||||
// Check for <title>...</title> with the challenge title
|
||||
titleLower := strings.ToLower(title)
|
||||
if strings.Contains(lowerHTML, strings.ToLower("<title>"+title)) ||
|
||||
strings.Contains(lowerHTML, titleLower) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
for _, selector := range challengeSelectors {
|
||||
if strings.Contains(lowerHTML, strings.ToLower(selector)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// ─── Site Connectivity Test ─────────────────────────────────────────────
|
||||
|
||||
// TestSiteConnectivity performs a site connectivity test with browser-like headers.
|
||||
// If flareSolverrURL is non-empty AND the site has BrowserEmulation turned on,
|
||||
// it will attempt to use FlareSolverr first.
|
||||
// Returns (ok, message, error).
|
||||
func TestSiteConnectivity(site *model.Site, flareSolverrURL string, timeout int, log *zap.Logger) (bool, string, error) {
|
||||
// Try FlareSolverr first when (a) globally enabled and (b) the site
|
||||
// asked for browser emulation. This matches the contract used by the
|
||||
// search path (see service.SiteService.siteModelToConfig).
|
||||
useFlare := flareSolverrURL != "" && site.BrowserEmulation
|
||||
if useFlare {
|
||||
log.Info("Trying FlareSolverr for site test", zap.String("url", site.URL))
|
||||
body, err := FetchURLWithFlareSolverr(flareSolverrURL, site.URL, site.Cookie, timeout, "", log)
|
||||
if err == nil {
|
||||
// Successfully got page via FlareSolverr
|
||||
if IsCloudflareChallenge(body) {
|
||||
return false, "站点被 Cloudflare/WAF 拦截，但 FlareSolverr 未能完全解决", nil
|
||||
}
|
||||
return true, "连接成功 (via FlareSolverr)", nil
|
||||
}
|
||||
log.Warn("FlareSolverr failed, falling back to direct request", zap.Error(err))
|
||||
// Fall through to direct request
|
||||
}
|
||||
|
||||
// Direct HTTP request with browser-like headers. Honors HTTP(S)_PROXY
|
||||
// when the site has UseProxy enabled — this makes the "use proxy"
|
||||
// checkbox in the UI actually do something.
|
||||
client := NewSiteHTTPClient(timeout, site.UseProxy)
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 {
|
||||
return fmt.Errorf("too many redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", site.URL, nil)
|
||||
if err != nil {
|
||||
return false, err.Error(), nil
|
||||
}
|
||||
|
||||
// Apply browser-like headers
|
||||
headers := HTTPHeaderPresets()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
// Apply auth headers
|
||||
ApplySiteAuthHeaders(req, site)
|
||||
|
||||
// Execute request
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err.Error(), nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response body for Cloudflare challenge detection
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
bodyStr := string(body)
|
||||
|
||||
// Check for Cloudflare challenge
|
||||
if IsCloudflareChallenge(bodyStr) {
|
||||
log.Warn("Cloudflare challenge detected", zap.String("url", site.URL))
|
||||
return false, "站点被 Cloudflare/WAF 拦截，请配置 FlareSolverr 或浏览器模拟", nil
|
||||
}
|
||||
|
||||
// Evaluate status code (mirror the reference Python project's semantics:
|
||||
// 200 → success, 3xx → success/redirect-to-self, 401/403 → failure with
|
||||
// hint to check credentials, 4xx/5xx → failure with raw status text).
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
return true, fmt.Sprintf("连接成功 (%s)", resp.Status), nil
|
||||
case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308:
|
||||
loc := resp.Header.Get("Location")
|
||||
if loc == "" {
|
||||
loc = "(unknown)"
|
||||
}
|
||||
// Most PT sites redirect logged-out users to login; treat as failure.
|
||||
return false, fmt.Sprintf("未登录或 Cookie 失效（重定向至 %s）", loc), nil
|
||||
case resp.StatusCode == 401:
|
||||
return false, "未授权（HTTP 401），请检查 API Key / Cookie", nil
|
||||
case resp.StatusCode == 403:
|
||||
return false, "认证失败（HTTP 403），请检查 Cookie / API Key 或站点是否需要浏览器模拟", nil
|
||||
case resp.StatusCode == 429:
|
||||
return false, "请求被限流（HTTP 429），请稍后再试", nil
|
||||
case resp.StatusCode == 503:
|
||||
return false, "服务暂时不可用（HTTP 503）", nil
|
||||
default:
|
||||
return false, resp.Status, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ApplySiteAuthHeaders applies authentication headers based on site config.
|
||||
func ApplySiteAuthHeaders(req *http.Request, site *model.Site) {
|
||||
switch site.AuthType {
|
||||
case "cookie":
|
||||
if site.Cookie != "" {
|
||||
req.Header.Set("Cookie", site.Cookie)
|
||||
}
|
||||
case "api_key":
|
||||
if site.APIKey != "" {
|
||||
if isYemaPTSite(site) {
|
||||
req.Header.Set("Authorization", site.APIKey)
|
||||
} else {
|
||||
req.Header.Set("x-api-key", site.APIKey)
|
||||
}
|
||||
}
|
||||
case "auth_header":
|
||||
if site.AuthHeader != "" {
|
||||
req.Header.Set("Authorization", site.AuthHeader)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply custom User-Agent if configured
|
||||
if site.UserAgent != "" {
|
||||
req.Header.Set("User-Agent", site.UserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
func isYemaPTSite(site *model.Site) bool {
|
||||
if site == nil {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(site.Type), "yemapt") {
|
||||
return true
|
||||
}
|
||||
u, err := url.Parse(strings.TrimSpace(site.URL))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org")
|
||||
}
|
||||
|
||||
// GetPageSource fetches a page with browser-like headers.
|
||||
// Returns (pageSource, cookies, error).
|
||||
func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) {
|
||||
client := NewSiteHTTPClient(timeout, site.UseProxy)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Apply browser-like headers
|
||||
headers := HTTPHeaderPresets()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
// Apply auth
|
||||
ApplySiteAuthHeaders(req, site)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Extract cookies from response
|
||||
cookies := ""
|
||||
for _, c := range resp.Cookies() {
|
||||
if cookies != "" {
|
||||
cookies += "; "
|
||||
}
|
||||
cookies += c.Name + "=" + c.Value
|
||||
}
|
||||
|
||||
return string(body), cookies, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// GetPageSource fetches a page with browser-like headers.
|
||||
// Returns (pageSource, cookies, error).
|
||||
func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) {
|
||||
client := NewSiteHTTPClient(timeout, site.UseProxy)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
headers := HTTPHeaderPresets()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
ApplySiteAuthHeaders(req, site)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
cookies := ""
|
||||
for _, c := range resp.Cookies() {
|
||||
if cookies != "" {
|
||||
cookies += "; "
|
||||
}
|
||||
cookies += c.Name + "=" + c.Value
|
||||
}
|
||||
|
||||
return string(body), cookies, nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
)
|
||||
|
||||
// ApplySiteAuthHeaders applies authentication headers based on site config.
|
||||
func ApplySiteAuthHeaders(req *http.Request, site *model.Site) {
|
||||
switch site.AuthType {
|
||||
case "cookie":
|
||||
if site.Cookie != "" {
|
||||
req.Header.Set("Cookie", site.Cookie)
|
||||
}
|
||||
case "api_key":
|
||||
if site.APIKey != "" {
|
||||
if isYemaPTSite(site) {
|
||||
req.Header.Set("Authorization", site.APIKey)
|
||||
} else {
|
||||
req.Header.Set("x-api-key", site.APIKey)
|
||||
}
|
||||
}
|
||||
case "auth_header":
|
||||
if site.AuthHeader != "" {
|
||||
req.Header.Set("Authorization", site.AuthHeader)
|
||||
}
|
||||
}
|
||||
|
||||
if site.UserAgent != "" {
|
||||
req.Header.Set("User-Agent", site.UserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
func isYemaPTSite(site *model.Site) bool {
|
||||
if site == nil {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(site.Type), "yemapt") {
|
||||
return true
|
||||
}
|
||||
u, err := url.Parse(strings.TrimSpace(site.URL))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org")
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// TestSiteConnectivity performs a site connectivity test with browser-like headers.
|
||||
// If flareSolverrURL is non-empty AND the site has BrowserEmulation turned on,
|
||||
// it will attempt to use FlareSolverr first.
|
||||
// Returns (ok, message, error).
|
||||
func TestSiteConnectivity(site *model.Site, flareSolverrURL string, timeout int, log *zap.Logger) (bool, string, error) {
|
||||
// Try FlareSolverr first when (a) globally enabled and (b) the site
|
||||
// asked for browser emulation. This matches the contract used by the
|
||||
// search path (see service.SiteService.siteModelToConfig).
|
||||
useFlare := flareSolverrURL != "" && site.BrowserEmulation
|
||||
if useFlare {
|
||||
log.Info("Trying FlareSolverr for site test", zap.String("url", site.URL))
|
||||
body, err := FetchURLWithFlareSolverr(flareSolverrURL, site.URL, site.Cookie, timeout, "", log)
|
||||
if err == nil {
|
||||
// Successfully got page via FlareSolverr
|
||||
if IsCloudflareChallenge(body) {
|
||||
return false, "站点被 Cloudflare/WAF 拦截，但 FlareSolverr 未能完全解决", nil
|
||||
}
|
||||
return true, "连接成功 (via FlareSolverr)", nil
|
||||
}
|
||||
log.Warn("FlareSolverr failed, falling back to direct request", zap.Error(err))
|
||||
// Fall through to direct request
|
||||
}
|
||||
|
||||
// Direct HTTP request with browser-like headers. Honors HTTP(S)_PROXY
|
||||
// when the site has UseProxy enabled — this makes the "use proxy"
|
||||
// checkbox in the UI actually do something.
|
||||
client := NewSiteHTTPClient(timeout, site.UseProxy)
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 {
|
||||
return fmt.Errorf("too many redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", site.URL, nil)
|
||||
if err != nil {
|
||||
return false, err.Error(), nil
|
||||
}
|
||||
|
||||
headers := HTTPHeaderPresets()
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
ApplySiteAuthHeaders(req, site)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err.Error(), nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
bodyStr := string(body)
|
||||
|
||||
if IsCloudflareChallenge(bodyStr) {
|
||||
log.Warn("Cloudflare challenge detected", zap.String("url", site.URL))
|
||||
return false, "站点被 Cloudflare/WAF 拦截，请配置 FlareSolverr 或浏览器模拟", nil
|
||||
}
|
||||
|
||||
// Evaluate status code (mirror the reference Python project's semantics:
|
||||
// 200 → success, 3xx → success/redirect-to-self, 401/403 → failure with
|
||||
// hint to check credentials, 4xx/5xx → failure with raw status text).
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
return true, fmt.Sprintf("连接成功 (%s)", resp.Status), nil
|
||||
case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308:
|
||||
loc := resp.Header.Get("Location")
|
||||
if loc == "" {
|
||||
loc = "(unknown)"
|
||||
}
|
||||
// Most PT sites redirect logged-out users to login; treat as failure.
|
||||
return false, fmt.Sprintf("未登录或 Cookie 失效（重定向至 %s）", loc), nil
|
||||
case resp.StatusCode == 401:
|
||||
return false, "未授权（HTTP 401），请检查 API Key / Cookie", nil
|
||||
case resp.StatusCode == 403:
|
||||
return false, "认证失败（HTTP 403），请检查 Cookie / API Key 或站点是否需要浏览器模拟", nil
|
||||
case resp.StatusCode == 429:
|
||||
return false, "请求被限流（HTTP 429），请稍后再试", nil
|
||||
case resp.StatusCode == 503:
|
||||
return false, "服务暂时不可用（HTTP 503）", nil
|
||||
default:
|
||||
return false, resp.Status, nil
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user