mirror of
https://github.com/truewhile/MeBox.git
synced 2026-10-04 20:46:37 +08:00
refactor: split modules and harden scraping workflows
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rssFeed is the minimal RSS subset we need to decode.
|
||||
type rssFeed struct {
|
||||
XMLName xml.Name `xml:"rss"`
|
||||
Channel struct {
|
||||
Items []rssItem `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
|
||||
type rssItem struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
GUID string `xml:"guid"`
|
||||
Description string `xml:"description"`
|
||||
Enclosure struct {
|
||||
URL string `xml:"url,attr"`
|
||||
} `xml:"enclosure"`
|
||||
}
|
||||
|
||||
func (s *SubscriptionService) fetch(ctx context.Context, feedURL string) (*rssFeed, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "MediaStationGo/0.1")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("rss %s: %d", feedURL, resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var f rssFeed
|
||||
if err := xml.Unmarshal(body, &f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func compileFilter(pat string) *regexp.Regexp {
|
||||
pat = strings.TrimSpace(pat)
|
||||
if pat == "" {
|
||||
return nil
|
||||
}
|
||||
if r, err := regexp.Compile("(?i)" + pat); err == nil {
|
||||
return r
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitNonEmpty(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0)
|
||||
for _, p := range strings.Split(s, "\n") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user