tcjlj/MeBox
1
0
Fork 0
mirror of https://github.com/truewhile/MeBox.git synced 2026-10-06 13:26:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

split helper http utilities

Browse Source
This commit is contained in:
ShukeBta
2026-06-27 03:14:38 +08:00
parent a0b8ab3807
commit d53d01b419
6 changed files with 359 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/helper/cloudflare.go
+37
View File
@@ -0,0 +1,37 @@
package helper
import "strings"
// IsCloudflareChallenge checks if the HTML content is a Cloudflare challenge page.
func IsCloudflareChallenge(html string) bool {
challengeTitles := []string{
"Just a moment...",
"请稍候…",
"DDOS-GUARD",
}
challengeSelectors := []string{
"#cf-challenge-running",
".ray_id",
".attack-box",
"#cf-please-wait",
"#challenge-spinner",
"#trk_jschal_js",
}
lowerHTML := strings.ToLower(html)
for _, title := range challengeTitles {
titleLower := strings.ToLower(title)
if strings.Contains(lowerHTML, strings.ToLower("<title>"+title)) ||
strings.Contains(lowerHTML, titleLower) {
return true
}
}
for _, selector := range challengeSelectors {
if strings.Contains(lowerHTML, strings.ToLower(selector)) {
return true
}
}
return false
}
internal/helper/flaresolverr.go
+128
View File
@@ -0,0 +1,128 @@
package helper
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"go.uber.org/zap"
)
// FlareSolverrRequest represents a request to FlareSolverr.
type FlareSolverrRequest struct {
Cmd string `json:"cmd"`
URL string `json:"url"`
Session string `json:"session,omitempty"`
MaxTimeout int `json:"maxTimeout,omitempty"`
Proxy *FlareSolverrProxy `json:"proxy,omitempty"`
Cookies []FlareSolverrCookie `json:"cookies,omitempty"`
}
// FlareSolverrProxy represents proxy config for FlareSolverr.
type FlareSolverrProxy struct {
URL string `json:"url"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
}
// FlareSolverrCookie represents a cookie for FlareSolverr.
type FlareSolverrCookie struct {
Name string `json:"name"`
Value string `json:"value"`
Domain string `json:"domain,omitempty"`
Path string `json:"path,omitempty"`
}
// FlareSolverrResponse represents FlareSolverr's response.
type FlareSolverrResponse struct {
Status string `json:"status"`
Message string `json:"message"`
Solution *FlareSolverrSolution `json:"solution,omitempty"`
}
// FlareSolverrSolution contains the solved challenge result.
type FlareSolverrSolution struct {
URL string `json:"url"`
Status int `json:"status"`
Headers map[string]string `json:"headers"`
Cookies []FlareSolverrCookie `json:"cookies"`
UserAgent string `json:"userAgent"`
Response string `json:"response"`
}
// FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL,
// bypassing Cloudflare/WAF challenges.
func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) {
if flareSolverrURL == "" {
return "", fmt.Errorf("FlareSolverr URL not configured")
}
if timeout <= 0 {
timeout = 60
}
var cookies []FlareSolverrCookie
if cookieStr != "" {
cookies = parseCookiesForFlareSolverr(cookieStr)
}
reqBody := FlareSolverrRequest{
Cmd: "request.get",
URL: targetURL,
MaxTimeout: timeout * 1000,
Cookies: cookies,
}
if proxyURL != "" {
reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL}
}
jsonBody, err := json.Marshal(reqBody)
if err != nil {
return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err)
}
client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second}
resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody)))
if err != nil {
return "", fmt.Errorf("FlareSolverr request failed: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("failed to read FlareSolverr response: %w", err)
}
var fsResp FlareSolverrResponse
if err := json.Unmarshal(body, &fsResp); err != nil {
return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err)
}
if fsResp.Status != "ok" {
return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message)
}
if fsResp.Solution != nil {
return fsResp.Solution.Response, nil
}
return "", fmt.Errorf("FlareSolverr returned no solution")
}
// parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format.
func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie {
var cookies []FlareSolverrCookie
parts := strings.Split(cookieStr, ";")
for _, part := range parts {
part = strings.TrimSpace(part)
kv := strings.SplitN(part, "=", 2)
if len(kv) == 2 {
cookies = append(cookies, FlareSolverrCookie{
Name: kv[0],
Value: kv[1],
})
}
}
return cookies
}
internal/helper/http.go
-340
View File
@@ -3,16 +3,11 @@
package helper package helper
import ( import (
"encoding/json"
"fmt"
"io"
"net/http" "net/http"
"net/url" "net/url"
"strings"
"time" "time"
"github.com/ShukeBta/MediaStationGo/internal/model" "github.com/ShukeBta/MediaStationGo/internal/model"
"go.uber.org/zap"
) )
// NewSiteHTTPClient builds an http.Client honoring per-site policies: // NewSiteHTTPClient builds an http.Client honoring per-site policies:
@@ -60,338 +55,3 @@ func HTTPHeaderPresets() map[string]string {
"Cache-Control": "max-age=0", "Cache-Control": "max-age=0",
} }
} }
// ─── FlareSolverr Support ───────────────────────────────────────────────
// FlareSolverrRequest represents a request to FlareSolverr.
type FlareSolverrRequest struct {
Cmd string `json:"cmd"`
URL string `json:"url"`
Session string `json:"session,omitempty"`
MaxTimeout int `json:"maxTimeout,omitempty"`
Proxy *FlareSolverrProxy `json:"proxy,omitempty"`
Cookies []FlareSolverrCookie `json:"cookies,omitempty"`
}
// FlareSolverrProxy represents proxy config for FlareSolverr.
type FlareSolverrProxy struct {
URL string `json:"url"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
}
// FlareSolverrCookie represents a cookie for FlareSolverr.
type FlareSolverrCookie struct {
Name string `json:"name"`
Value string `json:"value"`
Domain string `json:"domain,omitempty"`
Path string `json:"path,omitempty"`
}
// FlareSolverrResponse represents FlareSolverr's response.
type FlareSolverrResponse struct {
Status string `json:"status"`
Message string `json:"message"`
Solution *FlareSolverrSolution `json:"solution,omitempty"`
}
// FlareSolverrSolution contains the solved challenge result.
type FlareSolverrSolution struct {
URL string `json:"url"`
Status int `json:"status"`
Headers map[string]string `json:"headers"`
Cookies []FlareSolverrCookie `json:"cookies"`
UserAgent string `json:"userAgent"`
Response string `json:"response"`
}
// FetchURLWithFlareSolverr uses FlareSolverr to fetch a URL,
// bypassing Cloudflare/WAF challenges.
func FetchURLWithFlareSolverr(flareSolverrURL string, targetURL string, cookieStr string, timeout int, proxyURL string, log *zap.Logger) (string, error) {
if flareSolverrURL == "" {
return "", fmt.Errorf("FlareSolverr URL not configured")
}
if timeout <= 0 {
timeout = 60
}
// Parse cookies
var cookies []FlareSolverrCookie
if cookieStr != "" {
cookies = parseCookiesForFlareSolverr(cookieStr)
}
// Build request
reqBody := FlareSolverrRequest{
Cmd: "request.get",
URL: targetURL,
MaxTimeout: timeout * 1000,
Cookies: cookies,
}
if proxyURL != "" {
reqBody.Proxy = &FlareSolverrProxy{URL: proxyURL}
}
jsonBody, err := json.Marshal(reqBody)
if err != nil {
return "", fmt.Errorf("failed to marshal FlareSolverr request: %w", err)
}
// Send request to FlareSolverr
client := &http.Client{Timeout: time.Duration(timeout+10) * time.Second}
resp, err := client.Post(flareSolverrURL, "application/json", strings.NewReader(string(jsonBody)))
if err != nil {
return "", fmt.Errorf("FlareSolverr request failed: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("failed to read FlareSolverr response: %w", err)
}
var fsResp FlareSolverrResponse
if err := json.Unmarshal(body, &fsResp); err != nil {
return "", fmt.Errorf("failed to parse FlareSolverr response: %w", err)
}
if fsResp.Status != "ok" {
return "", fmt.Errorf("FlareSolverr error: %s", fsResp.Message)
}
if fsResp.Solution != nil {
return fsResp.Solution.Response, nil
}
return "", fmt.Errorf("FlareSolverr returned no solution")
}
// parseCookiesForFlareSolverr converts a cookie header string to FlareSolverr format.
func parseCookiesForFlareSolverr(cookieStr string) []FlareSolverrCookie {
var cookies []FlareSolverrCookie
parts := strings.Split(cookieStr, ";")
for _, part := range parts {
part = strings.TrimSpace(part)
kv := strings.SplitN(part, "=", 2)
if len(kv) == 2 {
cookies = append(cookies, FlareSolverrCookie{
Name: kv[0],
Value: kv[1],
})
}
}
return cookies
}
// ─── Cloudflare Challenge Detection ─────────────────────────────────────
// isCloudflareChallenge checks if the HTML content is a Cloudflare challenge page.
func IsCloudflareChallenge(html string) bool {
challengeTitles := []string{
"Just a moment...",
"请稍候…",
"DDOS-GUARD",
}
challengeSelectors := []string{
"#cf-challenge-running",
".ray_id",
".attack-box",
"#cf-please-wait",
"#challenge-spinner",
"#trk_jschal_js",
}
lowerHTML := strings.ToLower(html)
for _, title := range challengeTitles {
// Check for <title>...</title> with the challenge title
titleLower := strings.ToLower(title)
if strings.Contains(lowerHTML, strings.ToLower("<title>"+title)) ||
strings.Contains(lowerHTML, titleLower) {
return true
}
}
for _, selector := range challengeSelectors {
if strings.Contains(lowerHTML, strings.ToLower(selector)) {
return true
}
}
return false
}
// ─── Site Connectivity Test ─────────────────────────────────────────────
// TestSiteConnectivity performs a site connectivity test with browser-like headers.
// If flareSolverrURL is non-empty AND the site has BrowserEmulation turned on,
// it will attempt to use FlareSolverr first.
// Returns (ok, message, error).
func TestSiteConnectivity(site *model.Site, flareSolverrURL string, timeout int, log *zap.Logger) (bool, string, error) {
// Try FlareSolverr first when (a) globally enabled and (b) the site
// asked for browser emulation. This matches the contract used by the
// search path (see service.SiteService.siteModelToConfig).
useFlare := flareSolverrURL != "" && site.BrowserEmulation
if useFlare {
log.Info("Trying FlareSolverr for site test", zap.String("url", site.URL))
body, err := FetchURLWithFlareSolverr(flareSolverrURL, site.URL, site.Cookie, timeout, "", log)
if err == nil {
// Successfully got page via FlareSolverr
if IsCloudflareChallenge(body) {
return false, "站点被 Cloudflare/WAF 拦截，但 FlareSolverr 未能完全解决", nil
}
return true, "连接成功 (via FlareSolverr)", nil
}
log.Warn("FlareSolverr failed, falling back to direct request", zap.Error(err))
// Fall through to direct request
}
// Direct HTTP request with browser-like headers. Honors HTTP(S)_PROXY
// when the site has UseProxy enabled — this makes the "use proxy"
// checkbox in the UI actually do something.
client := NewSiteHTTPClient(timeout, site.UseProxy)
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("too many redirects")
}
return nil
}
req, err := http.NewRequest("GET", site.URL, nil)
if err != nil {
return false, err.Error(), nil
}
// Apply browser-like headers
headers := HTTPHeaderPresets()
for k, v := range headers {
req.Header.Set(k, v)
}
// Apply auth headers
ApplySiteAuthHeaders(req, site)
// Execute request
resp, err := client.Do(req)
if err != nil {
return false, err.Error(), nil
}
defer resp.Body.Close()
// Read response body for Cloudflare challenge detection
body, _ := io.ReadAll(resp.Body)
bodyStr := string(body)
// Check for Cloudflare challenge
if IsCloudflareChallenge(bodyStr) {
log.Warn("Cloudflare challenge detected", zap.String("url", site.URL))
return false, "站点被 Cloudflare/WAF 拦截，请配置 FlareSolverr 或浏览器模拟", nil
}
// Evaluate status code (mirror the reference Python project's semantics:
// 200 → success, 3xx → success/redirect-to-self, 401/403 → failure with
// hint to check credentials, 4xx/5xx → failure with raw status text).
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 300:
return true, fmt.Sprintf("连接成功 (%s)", resp.Status), nil
case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308:
loc := resp.Header.Get("Location")
if loc == "" {
loc = "(unknown)"
}
// Most PT sites redirect logged-out users to login; treat as failure.
return false, fmt.Sprintf("未登录或 Cookie 失效（重定向至 %s）", loc), nil
case resp.StatusCode == 401:
return false, "未授权（HTTP 401），请检查 API Key / Cookie", nil
case resp.StatusCode == 403:
return false, "认证失败（HTTP 403），请检查 Cookie / API Key 或站点是否需要浏览器模拟", nil
case resp.StatusCode == 429:
return false, "请求被限流（HTTP 429），请稍后再试", nil
case resp.StatusCode == 503:
return false, "服务暂时不可用（HTTP 503）", nil
default:
return false, resp.Status, nil
}
}
// ApplySiteAuthHeaders applies authentication headers based on site config.
func ApplySiteAuthHeaders(req *http.Request, site *model.Site) {
switch site.AuthType {
case "cookie":
if site.Cookie != "" {
req.Header.Set("Cookie", site.Cookie)
}
case "api_key":
if site.APIKey != "" {
if isYemaPTSite(site) {
req.Header.Set("Authorization", site.APIKey)
} else {
req.Header.Set("x-api-key", site.APIKey)
}
}
case "auth_header":
if site.AuthHeader != "" {
req.Header.Set("Authorization", site.AuthHeader)
}
}
// Apply custom User-Agent if configured
if site.UserAgent != "" {
req.Header.Set("User-Agent", site.UserAgent)
}
}
func isYemaPTSite(site *model.Site) bool {
if site == nil {
return false
}
if strings.EqualFold(strings.TrimSpace(site.Type), "yemapt") {
return true
}
u, err := url.Parse(strings.TrimSpace(site.URL))
if err != nil {
return false
}
host := strings.ToLower(u.Hostname())
return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org")
}
// GetPageSource fetches a page with browser-like headers.
// Returns (pageSource, cookies, error).
func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) {
client := NewSiteHTTPClient(timeout, site.UseProxy)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", "", err
}
// Apply browser-like headers
headers := HTTPHeaderPresets()
for k, v := range headers {
req.Header.Set(k, v)
}
// Apply auth
ApplySiteAuthHeaders(req, site)
resp, err := client.Do(req)
if err != nil {
return "", "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", "", err
}
// Extract cookies from response
cookies := ""
for _, c := range resp.Cookies() {
if cookies != "" {
cookies += "; "
}
cookies += c.Name + "=" + c.Value
}
return string(body), cookies, nil
}
internal/helper/page_source.go
+48
View File
@@ -0,0 +1,48 @@
package helper
import (
"io"
"net/http"
"github.com/ShukeBta/MediaStationGo/internal/model"
"go.uber.org/zap"
)
// GetPageSource fetches a page with browser-like headers.
// Returns (pageSource, cookies, error).
func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) {
client := NewSiteHTTPClient(timeout, site.UseProxy)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", "", err
}
headers := HTTPHeaderPresets()
for k, v := range headers {
req.Header.Set(k, v)
}
ApplySiteAuthHeaders(req, site)
resp, err := client.Do(req)
if err != nil {
return "", "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", "", err
}
cookies := ""
for _, c := range resp.Cookies() {
if cookies != "" {
cookies += "; "
}
cookies += c.Name + "=" + c.Value
}
return string(body), cookies, nil
}
internal/helper/site_auth.go
+50
View File
@@ -0,0 +1,50 @@
package helper
import (
"net/http"
"net/url"
"strings"
"github.com/ShukeBta/MediaStationGo/internal/model"
)
// ApplySiteAuthHeaders applies authentication headers based on site config.
func ApplySiteAuthHeaders(req *http.Request, site *model.Site) {
switch site.AuthType {
case "cookie":
if site.Cookie != "" {
req.Header.Set("Cookie", site.Cookie)
}
case "api_key":
if site.APIKey != "" {
if isYemaPTSite(site) {
req.Header.Set("Authorization", site.APIKey)
} else {
req.Header.Set("x-api-key", site.APIKey)
}
}
case "auth_header":
if site.AuthHeader != "" {
req.Header.Set("Authorization", site.AuthHeader)
}
}
if site.UserAgent != "" {
req.Header.Set("User-Agent", site.UserAgent)
}
}
func isYemaPTSite(site *model.Site) bool {
if site == nil {
return false
}
if strings.EqualFold(strings.TrimSpace(site.Type), "yemapt") {
return true
}
u, err := url.Parse(strings.TrimSpace(site.URL))
if err != nil {
return false
}
host := strings.ToLower(u.Hostname())
return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org")
}
internal/helper/site_connectivity.go
+96
View File
@@ -0,0 +1,96 @@
package helper
import (
"fmt"
"io"
"net/http"
"github.com/ShukeBta/MediaStationGo/internal/model"
"go.uber.org/zap"
)
// TestSiteConnectivity performs a site connectivity test with browser-like headers.
// If flareSolverrURL is non-empty AND the site has BrowserEmulation turned on,
// it will attempt to use FlareSolverr first.
// Returns (ok, message, error).
func TestSiteConnectivity(site *model.Site, flareSolverrURL string, timeout int, log *zap.Logger) (bool, string, error) {
// Try FlareSolverr first when (a) globally enabled and (b) the site
// asked for browser emulation. This matches the contract used by the
// search path (see service.SiteService.siteModelToConfig).
useFlare := flareSolverrURL != "" && site.BrowserEmulation
if useFlare {
log.Info("Trying FlareSolverr for site test", zap.String("url", site.URL))
body, err := FetchURLWithFlareSolverr(flareSolverrURL, site.URL, site.Cookie, timeout, "", log)
if err == nil {
// Successfully got page via FlareSolverr
if IsCloudflareChallenge(body) {
return false, "站点被 Cloudflare/WAF 拦截，但 FlareSolverr 未能完全解决", nil
}
return true, "连接成功 (via FlareSolverr)", nil
}
log.Warn("FlareSolverr failed, falling back to direct request", zap.Error(err))
// Fall through to direct request
}
// Direct HTTP request with browser-like headers. Honors HTTP(S)_PROXY
// when the site has UseProxy enabled — this makes the "use proxy"
// checkbox in the UI actually do something.
client := NewSiteHTTPClient(timeout, site.UseProxy)
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("too many redirects")
}
return nil
}
req, err := http.NewRequest("GET", site.URL, nil)
if err != nil {
return false, err.Error(), nil
}
headers := HTTPHeaderPresets()
for k, v := range headers {
req.Header.Set(k, v)
}
ApplySiteAuthHeaders(req, site)
resp, err := client.Do(req)
if err != nil {
return false, err.Error(), nil
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
bodyStr := string(body)
if IsCloudflareChallenge(bodyStr) {
log.Warn("Cloudflare challenge detected", zap.String("url", site.URL))
return false, "站点被 Cloudflare/WAF 拦截，请配置 FlareSolverr 或浏览器模拟", nil
}
// Evaluate status code (mirror the reference Python project's semantics:
// 200 → success, 3xx → success/redirect-to-self, 401/403 → failure with
// hint to check credentials, 4xx/5xx → failure with raw status text).
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 300:
return true, fmt.Sprintf("连接成功 (%s)", resp.Status), nil
case resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307 || resp.StatusCode == 308:
loc := resp.Header.Get("Location")
if loc == "" {
loc = "(unknown)"
}
// Most PT sites redirect logged-out users to login; treat as failure.
return false, fmt.Sprintf("未登录或 Cookie 失效（重定向至 %s）", loc), nil
case resp.StatusCode == 401:
return false, "未授权（HTTP 401），请检查 API Key / Cookie", nil
case resp.StatusCode == 403:
return false, "认证失败（HTTP 403），请检查 Cookie / API Key 或站点是否需要浏览器模拟", nil
case resp.StatusCode == 429:
return false, "请求被限流（HTTP 429），请稍后再试", nil
case resp.StatusCode == 503:
return false, "服务暂时不可用（HTTP 503）", nil
default:
return false, resp.Status, nil
}
}