Fix scraping workflows subscriptions bot and recycle bin

This commit is contained in:
ShukeBta
2026-06-17 19:48:17 +08:00
parent 91afbe54a2
commit 850d3db811
43 changed files with 2782 additions and 76 deletions
+155
View File
@@ -132,6 +132,161 @@ func (d *DoubanProvider) SearchMatch(ctx context.Context, query string) (*Match,
return match, nil
}
func (d *DoubanProvider) GetMatchByID(ctx context.Context, doubanID string) (*Match, error) {
doubanID = strings.TrimSpace(doubanID)
if doubanID == "" {
return nil, nil
}
u := "https://movie.douban.com/j/subject_abstract?subject_id=" + url.QueryEscape(doubanID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
d.setHeaders(req)
resp, err := d.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return nil, fmt.Errorf("douban detail: %d", resp.StatusCode)
}
var raw map[string]any
if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
return nil, err
}
subject := raw
if nested, ok := raw["subject"].(map[string]any); ok {
subject = nested
} else if nested, ok := raw["data"].(map[string]any); ok {
subject = nested
}
title := firstStringFromMap(subject, "title", "name")
year := 0
if y := firstStringFromMap(subject, "year"); len(y) >= 4 {
_, _ = fmt.Sscanf(y[:4], "%d", &year)
}
m := &Match{
DoubanID: doubanID,
Title: title,
Overview: firstStringFromMap(subject, "short_comment", "intro", "summary", "abstract"),
PosterURL: firstStringFromMap(subject, "pic", "img", "cover", "cover_url"),
Year: year,
Rating: float32FromMap(subject, "rate", "rating"),
}
if m.Title == "" {
m.Title = firstStringFromMap(raw, "title")
}
return m, nil
}
func (d *DoubanProvider) GetEpisodeCount(ctx context.Context, query string) (int, error) {
match, err := d.Search(ctx, query)
if err != nil || match == nil || strings.TrimSpace(match.DoubanID) == "" {
return 0, err
}
return d.GetEpisodeCountByID(ctx, match.DoubanID)
}
func (d *DoubanProvider) GetEpisodeCountByID(ctx context.Context, doubanID string) (int, error) {
doubanID = strings.TrimSpace(doubanID)
if doubanID == "" {
return 0, nil
}
u := "https://movie.douban.com/j/subject_abstract?subject_id=" + url.QueryEscape(doubanID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return 0, err
}
d.setHeaders(req)
resp, err := d.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return 0, fmt.Errorf("douban detail: %d", resp.StatusCode)
}
var raw map[string]any
if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
return 0, err
}
for _, key := range []string{"episode_count", "episodes_count", "episodes", "eps"} {
if count := doubanEpisodeCountFromValue(raw[key]); count > 0 {
return count, nil
}
}
for _, key := range []string{"subject", "data"} {
if nested, ok := raw[key].(map[string]any); ok {
for _, field := range []string{"episode_count", "episodes_count", "episodes", "eps"} {
if count := doubanEpisodeCountFromValue(nested[field]); count > 0 {
return count, nil
}
}
}
}
return 0, nil
}
func firstStringFromMap(values map[string]any, keys ...string) string {
for _, key := range keys {
switch v := values[key].(type) {
case string:
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
case map[string]any:
if s := firstStringFromMap(v, "normal", "large", "small", "url"); s != "" {
return s
}
}
}
return ""
}
func float32FromMap(values map[string]any, keys ...string) float32 {
for _, key := range keys {
switch v := values[key].(type) {
case float64:
return float32(v)
case string:
var out float32
if _, err := fmt.Sscanf(strings.TrimSpace(v), "%f", &out); err == nil {
return out
}
case map[string]any:
if out := float32FromMap(v, "value", "score"); out > 0 {
return out
}
}
}
return 0
}
func doubanEpisodeCountFromValue(value any) int {
switch v := value.(type) {
case float64:
if v > 0 {
return int(v)
}
case int:
if v > 0 {
return v
}
case string:
var n int
if _, err := fmt.Sscanf(strings.TrimSpace(v), "%d", &n); err == nil && n > 0 {
return n
}
case []any:
if len(v) > 0 {
return len(v)
}
}
return 0
}
func (d *DoubanProvider) setHeaders(req *http.Request) {
req.Header.Set("User-Agent", userAgents[secureRandomIntn(len(userAgents))])
req.Header.Set("Referer", "https://movie.douban.com/")