From d675c4e8ed166eb041347fea01aa3b35f7042f9b Mon Sep 17 00:00:00 2001 From: ShukeBta <272197458+ShukeBta@users.noreply.github.com> Date: Sat, 27 Jun 2026 01:54:42 +0800 Subject: [PATCH] split adult scraper helpers --- internal/service/adult_scraper.go | 232 ----------------------- internal/service/adult_scraper_code.go | 41 ++++ internal/service/adult_scraper_config.go | 67 +++++++ internal/service/adult_scraper_parse.go | 142 ++++++++++++++ 4 files changed, 250 insertions(+), 232 deletions(-) create mode 100644 internal/service/adult_scraper_code.go create mode 100644 internal/service/adult_scraper_config.go create mode 100644 internal/service/adult_scraper_parse.go diff --git a/internal/service/adult_scraper.go b/internal/service/adult_scraper.go index a279f8c..f7f1b45 100644 --- a/internal/service/adult_scraper.go +++ b/internal/service/adult_scraper.go @@ -4,13 +4,10 @@ import ( "context" "errors" "fmt" - "html" "io" "net/http" "net/url" - "path/filepath" "regexp" - "strconv" "strings" "time" @@ -120,49 +117,6 @@ func (p *AdultProvider) resolveBases(ctx context.Context) []string { return dedupeStrings(out) } -func adultConfiguredBases(value string) []string { - parts := strings.FieldsFunc(value, func(r rune) bool { - switch r { - case ',', ';', '\n', '\r', '\t', ' ': - return true - default: - return false - } - }) - out := make([]string, 0, len(parts)) - for _, part := range parts { - part = strings.TrimSpace(part) - if part == "" { - continue - } - if !strings.HasPrefix(part, "http://") && !strings.HasPrefix(part, "https://") { - part = "https://" + part - } - out = append(out, strings.TrimRight(part, "/")) - } - return out -} - -func adultSourceKind(base string) string { - u, err := url.Parse(strings.TrimSpace(base)) - host := "" - if err == nil { - host = strings.ToLower(u.Hostname()) - } - if host == "" { - host = strings.ToLower(base) - } - if strings.Contains(host, "javdb") { - return "javdb" - } - for _, needle := range []string{"javbus", "cdnbus", "javsee", "busjav"} { - if strings.Contains(host, needle) { - return "javbus" - } - } - return "javdb" -} - func (p *AdultProvider) scrapeJavDB(ctx context.Context, base, code string) (*Match, error) { searchURL := base + "/search?q=" + url.QueryEscape(code) + "&f=all" body, err := p.fetchText(ctx, searchURL, base) @@ -225,54 +179,6 @@ func (p *AdultProvider) fetchText(ctx context.Context, targetURL, referer string return string(body), nil } -func parseAdultDetailHTML(body, code, source, detailURL string) *Match { - match := &Match{ - OriginalName: code, - MediaType: "adult", - NSFW: true, - Genres: []string{"Adult", source}, - } - if title := firstAdultTitle(body, code); title != "" { - match.Title = title - } - if match.Title == "" { - return nil - } - if source == "javbus" { - if m := adultJavBusCoverPattern.FindStringSubmatch(body); len(m) > 1 { - match.PosterURL = absolutizeURL(detailURL, m[1]) - } - } else if cover := firstAdultImage(body, "video-cover", "cover", "column-video-cover"); cover != "" { - match.PosterURL = absolutizeURL(detailURL, cover) - } - if m := adultSamplePattern.FindStringSubmatch(body); len(m) > 1 { - match.BackdropURL = absolutizeURL(detailURL, m[1]) - } - if dmmPoster := adultDMMPosterFromSampleURL(match.BackdropURL); dmmPoster != "" { - match.PosterURL = dmmPoster - } - match.Year = firstYearInText(body) - match.Rating = firstRatingInText(body) - return match -} - -func firstAdultTitle(body, code string) string { - for _, found := range adultTitlePattern.FindAllStringSubmatch(body, -1) { - if len(found) < 2 { - continue - } - title := strings.TrimSpace(stripAdultHTML(found[1])) - if title == "" { - continue - } - title = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(title, code), strings.ToUpper(code))) - if title != "" { - return title - } - } - return "" -} - func applyAdultHeaders(req *http.Request, referer string) { req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") @@ -281,141 +187,3 @@ func applyAdultHeaders(req *http.Request, referer string) { req.Header.Set("Referer", referer) } } - -func AdultCodeFromMediaPath(path string) string { - if code := normalizeAdultCode(filepath.Base(path)); code != "" { - return code - } - return normalizeAdultCode(path) -} - -func normalizeAdultCode(input string) string { - input = strings.ToUpper(strings.TrimSpace(input)) - if input == "" { - return "" - } - input = strings.ReplaceAll(input, "_", "-") - if m := adultFC2Pattern.FindStringSubmatch(input); len(m) > 1 { - return "FC2-PPV-" + m[1] - } - if m := adultHEYZOPattern.FindStringSubmatch(input); len(m) > 1 { - return "HEYZO-" + m[1] - } - if m := adultUncensoredPattern.FindStringSubmatch(input); len(m) > 2 { - return m[1] + "-" + m[2] - } - for _, m := range adultStandardPattern.FindAllStringSubmatch(input, -1) { - if len(m) < 3 { - continue - } - prefix := strings.TrimSpace(m[1]) - if _, excluded := adultExcludedPrefixes[prefix]; excluded { - continue - } - return prefix + "-" + m[2] - } - return "" -} - -func stripAdultHTML(value string) string { - value = adultTagPattern.ReplaceAllString(value, " ") - return strings.Join(strings.Fields(html.UnescapeString(value)), " ") -} - -func firstAdultImage(body string, classNeedles ...string) string { - for _, found := range adultImagePattern.FindAllStringSubmatch(body, -1) { - if len(found) < 2 { - continue - } - attrs := adultAttrs(found[1]) - class := strings.ToLower(attrs["class"]) - for _, needle := range classNeedles { - if strings.Contains(class, strings.ToLower(needle)) { - if attrs["src"] != "" { - return attrs["src"] - } - if attrs["data-src"] != "" { - return attrs["data-src"] - } - } - } - } - return "" -} - -func adultDMMPosterFromSampleURL(raw string) string { - u, err := url.Parse(strings.TrimSpace(raw)) - if err != nil || u.Host == "" || !strings.Contains(strings.ToLower(u.Host), "dmm.co.jp") { - return "" - } - lowerPath := strings.ToLower(u.Path) - for _, suffix := range []string{"jp-1.jpg", "jp.jpg"} { - if strings.HasSuffix(lowerPath, suffix) { - u.Path = u.Path[:len(u.Path)-len(suffix)] + "pl.jpg" - return u.String() - } - } - return "" -} - -func adultAttrs(raw string) map[string]string { - out := map[string]string{} - for _, found := range adultAttrPattern.FindAllStringSubmatch(raw, -1) { - if len(found) >= 3 { - out[strings.ToLower(found[1])] = html.UnescapeString(found[2]) - } - } - return out -} - -func absolutizeURL(base, raw string) string { - raw = strings.TrimSpace(html.UnescapeString(raw)) - if raw == "" { - return "" - } - u, err := url.Parse(raw) - if err == nil && u.IsAbs() { - return raw - } - b, err := url.Parse(base) - if err != nil { - return raw - } - return b.ResolveReference(u).String() -} - -func firstYearInText(body string) int { - m := regexp.MustCompile(`(?:19|20)\d{2}[-/.]\d{1,2}[-/.]\d{1,2}`).FindString(body) - if len(m) >= 4 { - year, _ := strconv.Atoi(m[:4]) - return year - } - return 0 -} - -func firstRatingInText(body string) float32 { - m := regexp.MustCompile(`(?i)(?:score|rating|評分|评分)[^0-9]{0,20}([0-9](?:\.[0-9])?)`).FindStringSubmatch(body) - if len(m) > 1 { - v, _ := strconv.ParseFloat(m[1], 32) - return float32(v) - } - return 0 -} - -func dedupeStrings(values []string) []string { - seen := map[string]struct{}{} - out := make([]string, 0, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - key := strings.ToLower(value) - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - out = append(out, value) - } - return out -} diff --git a/internal/service/adult_scraper_code.go b/internal/service/adult_scraper_code.go new file mode 100644 index 0000000..a05db53 --- /dev/null +++ b/internal/service/adult_scraper_code.go @@ -0,0 +1,41 @@ +package service + +import ( + "path/filepath" + "strings" +) + +func AdultCodeFromMediaPath(path string) string { + if code := normalizeAdultCode(filepath.Base(path)); code != "" { + return code + } + return normalizeAdultCode(path) +} + +func normalizeAdultCode(input string) string { + input = strings.ToUpper(strings.TrimSpace(input)) + if input == "" { + return "" + } + input = strings.ReplaceAll(input, "_", "-") + if m := adultFC2Pattern.FindStringSubmatch(input); len(m) > 1 { + return "FC2-PPV-" + m[1] + } + if m := adultHEYZOPattern.FindStringSubmatch(input); len(m) > 1 { + return "HEYZO-" + m[1] + } + if m := adultUncensoredPattern.FindStringSubmatch(input); len(m) > 2 { + return m[1] + "-" + m[2] + } + for _, m := range adultStandardPattern.FindAllStringSubmatch(input, -1) { + if len(m) < 3 { + continue + } + prefix := strings.TrimSpace(m[1]) + if _, excluded := adultExcludedPrefixes[prefix]; excluded { + continue + } + return prefix + "-" + m[2] + } + return "" +} diff --git a/internal/service/adult_scraper_config.go b/internal/service/adult_scraper_config.go new file mode 100644 index 0000000..f6499c4 --- /dev/null +++ b/internal/service/adult_scraper_config.go @@ -0,0 +1,67 @@ +package service + +import ( + "net/url" + "strings" +) + +func adultConfiguredBases(value string) []string { + parts := strings.FieldsFunc(value, func(r rune) bool { + switch r { + case ',', ';', '\n', '\r', '\t', ' ': + return true + default: + return false + } + }) + out := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if !strings.HasPrefix(part, "http://") && !strings.HasPrefix(part, "https://") { + part = "https://" + part + } + out = append(out, strings.TrimRight(part, "/")) + } + return out +} + +func adultSourceKind(base string) string { + u, err := url.Parse(strings.TrimSpace(base)) + host := "" + if err == nil { + host = strings.ToLower(u.Hostname()) + } + if host == "" { + host = strings.ToLower(base) + } + if strings.Contains(host, "javdb") { + return "javdb" + } + for _, needle := range []string{"javbus", "cdnbus", "javsee", "busjav"} { + if strings.Contains(host, needle) { + return "javbus" + } + } + return "javdb" +} + +func dedupeStrings(values []string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + key := strings.ToLower(value) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, value) + } + return out +} diff --git a/internal/service/adult_scraper_parse.go b/internal/service/adult_scraper_parse.go new file mode 100644 index 0000000..f72edbe --- /dev/null +++ b/internal/service/adult_scraper_parse.go @@ -0,0 +1,142 @@ +package service + +import ( + "html" + "net/url" + "regexp" + "strconv" + "strings" +) + +func parseAdultDetailHTML(body, code, source, detailURL string) *Match { + match := &Match{ + OriginalName: code, + MediaType: "adult", + NSFW: true, + Genres: []string{"Adult", source}, + } + if title := firstAdultTitle(body, code); title != "" { + match.Title = title + } + if match.Title == "" { + return nil + } + if source == "javbus" { + if m := adultJavBusCoverPattern.FindStringSubmatch(body); len(m) > 1 { + match.PosterURL = absolutizeURL(detailURL, m[1]) + } + } else if cover := firstAdultImage(body, "video-cover", "cover", "column-video-cover"); cover != "" { + match.PosterURL = absolutizeURL(detailURL, cover) + } + if m := adultSamplePattern.FindStringSubmatch(body); len(m) > 1 { + match.BackdropURL = absolutizeURL(detailURL, m[1]) + } + if dmmPoster := adultDMMPosterFromSampleURL(match.BackdropURL); dmmPoster != "" { + match.PosterURL = dmmPoster + } + match.Year = firstYearInText(body) + match.Rating = firstRatingInText(body) + return match +} + +func firstAdultTitle(body, code string) string { + for _, found := range adultTitlePattern.FindAllStringSubmatch(body, -1) { + if len(found) < 2 { + continue + } + title := strings.TrimSpace(stripAdultHTML(found[1])) + if title == "" { + continue + } + title = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(title, code), strings.ToUpper(code))) + if title != "" { + return title + } + } + return "" +} + +func stripAdultHTML(value string) string { + value = adultTagPattern.ReplaceAllString(value, " ") + return strings.Join(strings.Fields(html.UnescapeString(value)), " ") +} + +func firstAdultImage(body string, classNeedles ...string) string { + for _, found := range adultImagePattern.FindAllStringSubmatch(body, -1) { + if len(found) < 2 { + continue + } + attrs := adultAttrs(found[1]) + class := strings.ToLower(attrs["class"]) + for _, needle := range classNeedles { + if strings.Contains(class, strings.ToLower(needle)) { + if attrs["src"] != "" { + return attrs["src"] + } + if attrs["data-src"] != "" { + return attrs["data-src"] + } + } + } + } + return "" +} + +func adultDMMPosterFromSampleURL(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" || !strings.Contains(strings.ToLower(u.Host), "dmm.co.jp") { + return "" + } + lowerPath := strings.ToLower(u.Path) + for _, suffix := range []string{"jp-1.jpg", "jp.jpg"} { + if strings.HasSuffix(lowerPath, suffix) { + u.Path = u.Path[:len(u.Path)-len(suffix)] + "pl.jpg" + return u.String() + } + } + return "" +} + +func adultAttrs(raw string) map[string]string { + out := map[string]string{} + for _, found := range adultAttrPattern.FindAllStringSubmatch(raw, -1) { + if len(found) >= 3 { + out[strings.ToLower(found[1])] = html.UnescapeString(found[2]) + } + } + return out +} + +func absolutizeURL(base, raw string) string { + raw = strings.TrimSpace(html.UnescapeString(raw)) + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err == nil && u.IsAbs() { + return raw + } + b, err := url.Parse(base) + if err != nil { + return raw + } + return b.ResolveReference(u).String() +} + +func firstYearInText(body string) int { + m := regexp.MustCompile(`(?:19|20)\d{2}[-/.]\d{1,2}[-/.]\d{1,2}`).FindString(body) + if len(m) >= 4 { + year, _ := strconv.Atoi(m[:4]) + return year + } + return 0 +} + +func firstRatingInText(body string) float32 { + m := regexp.MustCompile(`(?i)(?:score|rating|評分|评分)[^0-9]{0,20}([0-9](?:\.[0-9])?)`).FindStringSubmatch(body) + if len(m) > 1 { + v, _ := strconv.ParseFloat(m[1], 32) + return float32(v) + } + return 0 +}