refactor: split modules and harden scraping workflows

internal/service/image_proxy.go

@@ -13,60 +13,16 @@
 package service
 
 import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
-	"io"
-	"net"
 	"net/http"
-	"net/url"
-	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
 	"go.uber.org/zap"
 
 	"github.com/ShukeBta/MediaStationGo/internal/config"
-	"github.com/ShukeBta/MediaStationGo/internal/service/cloud"
 )
 
-// transparent1x1PNG is a baseline 67-byte PNG used as a fallback when the
-// upstream image cannot be retrieved, so browser layouts never collapse.
-var transparent1x1PNG = []byte{
-	0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-	0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
-	0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
-	0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4,
-	0x89, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x44, 0x41,
-	0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00,
-	0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00,
-	0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae,
-	0x42, 0x60, 0x82,
-}
-
-// knownImageHosts are hosts we explicitly recognize. The list is no longer
-// a hard allow-list — it only short-circuits cases where we can be 100%
-// sure the destination is a public image CDN. Other hosts are accepted as
-// long as the scheme is http/https; this is required so users behind GFW
-// can configure their own TMDb mirror via secrets.tmdb_image_proxy.
-var knownImageHosts = map[string]struct{}{
-	"image.tmdb.org":       {},
-	"www.themoviedb.org":   {},
-	"lain.bgm.tv":          {},
-	"img.bgm.tv":           {},
-	"webdav.bgm.tv":        {},
-	"img1.doubanio.com":    {},
-	"img2.doubanio.com":    {},
-	"img3.doubanio.com":    {},
-	"img9.doubanio.com":    {},
-	"assets.fanart.tv":     {},
-	"artworks.thetvdb.com": {},
-}
-
 // ImageProxy fetches and caches remote images on behalf of the browser.
 type ImageProxy struct {
 	cfg      *config.Config
@@ -129,515 +85,3 @@ func (p *ImageProxy) libraryRoots() []string {
 	p.libRootsAt = time.Now()
 	return p.libRootsCache
 }
-
-// validateURL parses raw and ensures the scheme is http/https and the
-// target host is not a private/loopback/link-local address (SSRF guard).
-func (p *ImageProxy) validateURL(raw string) (*url.URL, error) {
-	if raw == "" {
-		return nil, errors.New("missing url")
-	}
-	u, err := url.Parse(raw)
-	if err != nil || u.Host == "" {
-		return nil, errors.New("invalid url")
-	}
-	scheme := strings.ToLower(u.Scheme)
-	if scheme != "http" && scheme != "https" {
-		return nil, errors.New("unsupported scheme")
-	}
-	if isPrivateHost(u.Hostname()) {
-		return nil, errors.New("requests to private/internal hosts are not allowed")
-	}
-	return u, nil
-}
-
-// isPrivateHost returns true only when host is a *literal* loopback, private,
-// link-local or unspecified IP address. This blocks the obvious SSRF vectors
-// (e.g. http://127.0.0.1/… or the cloud metadata IP 169.254.169.254) while
-// NOT blocking hostnames.
-//
-// We deliberately do not resolve hostnames here: under GFW DNS poisoning,
-// public image CDNs such as image.tmdb.org are frequently resolved to
-// loopback/private/bogus IPs. Blocking on resolved addresses would therefore
-// wrongly drop legitimate posters for exactly the users this proxy exists to
-// serve, which is what caused posters to stop displaying.
-func isPrivateHost(host string) bool {
-	if host == "" {
-		return true
-	}
-	ip := net.ParseIP(host)
-	if ip != nil {
-		return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified()
-	}
-	return false
-}
-
-// isAllowedLocalPath restricts local file reads to known-safe roots to
-// prevent arbitrary file reads via path traversal. Allowed roots are the
-// data dir, cache dir, the configured movies/tv/anime dirs, and — crucially —
-// every configured media library root, because sidecar posters/artwork are
-// stored alongside media under those (arbitrary, user-defined) paths.
-func (p *ImageProxy) isAllowedLocalPath(abs string) bool {
-	roots := []string{p.cfg.App.DataDir, p.cfg.Cache.CacheDir, p.cfg.Media.MoviesDir, p.cfg.Media.TVDir, p.cfg.Media.AnimeDir}
-	roots = append(roots, p.libraryRoots()...)
-	for _, root := range roots {
-		if strings.TrimSpace(root) == "" {
-			continue
-		}
-		rootAbs, err := filepath.Abs(root)
-		if err != nil {
-			continue
-		}
-		if strings.HasPrefix(abs, rootAbs+string(filepath.Separator)) || abs == rootAbs {
-			return true
-		}
-	}
-	return false
-}
-
-func isLocalImagePath(raw string) bool {
-	raw = strings.TrimSpace(raw)
-	if raw == "" || isHTTPish(raw) {
-		return false
-	}
-	ext := strings.ToLower(filepath.Ext(raw))
-	switch ext {
-	case ".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp":
-		return true
-	default:
-		return false
-	}
-}
-
-func isHTTPish(raw string) bool {
-	return strings.HasPrefix(strings.ToLower(raw), "http://") || strings.HasPrefix(strings.ToLower(raw), "https://")
-}
-
-// detectContentType returns the MIME type of data using the first 512 bytes.
-func detectContentType(data []byte) string {
-	if len(data) > 512 {
-		return http.DetectContentType(data[:512])
-	}
-	return http.DetectContentType(data)
-}
-
-// servePlaceholder writes a 1×1 transparent PNG to w. Used as a fallback
-// when upstream fetch fails so the browser layout stays intact.
-func servePlaceholder(w http.ResponseWriter) {
-	w.Header().Set("Content-Type", "image/png")
-	w.Header().Set("Cache-Control", "no-store")
-	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write(transparent1x1PNG)
-}
-
-func serveCachedPlaceholder(w http.ResponseWriter) {
-	w.Header().Set("Content-Type", "image/png")
-	w.Header().Set("Cache-Control", imagePlaceholderCacheControl)
-	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write(transparent1x1PNG)
-}
-
-func (p *ImageProxy) cloudImageCachePaths(stableKey string) (string, string, string) {
-	stableKey = strings.TrimSpace(stableKey)
-	if stableKey == "" {
-		stableKey = "unknown"
-	}
-	sum := sha256.Sum256([]byte("cloud-image:" + stableKey))
-	key := "cloud-" + hex.EncodeToString(sum[:])
-	cachePath := filepath.Join(p.cacheDir, key)
-	return key, cachePath, cachePath + ".fail"
-}
-
-func serveCachedImageFile(w http.ResponseWriter, r *http.Request, key, cachePath string) bool {
-	data, err := os.ReadFile(cachePath) // #nosec G304 -- cachePath is derived from a SHA-256 cache key under the configured cache directory.
-	if err != nil || len(data) == 0 {
-		return false
-	}
-	w.Header().Set("Content-Type", detectContentType(data))
-	w.Header().Set("Cache-Control", imageBrowserCacheControl)
-	stat, _ := os.Stat(cachePath)
-	modTime := time.Now()
-	if stat != nil {
-		modTime = stat.ModTime()
-	}
-	http.ServeContent(w, r, key, modTime, bytes.NewReader(data))
-	return true
-}
-
-func freshNegativeImageCache(failPath string) bool {
-	stat, err := os.Stat(failPath)
-	if err != nil {
-		return false
-	}
-	if time.Since(stat.ModTime()) < imageNegativeCacheTTL {
-		return true
-	}
-	_ = os.Remove(failPath)
-	return false
-}
-
-// CloudImageCached reports whether a stable cloud-image ref already has a
-// usable positive or short-lived negative cache entry. Scanner pre-warm uses it
-// to avoid repeatedly resolving the same cloud sidecar image.
-func (p *ImageProxy) CloudImageCached(stableKey string) bool {
-	if p == nil {
-		return false
-	}
-	_, cachePath, failPath := p.cloudImageCachePaths(stableKey)
-	if stat, err := os.Stat(cachePath); err == nil && stat.Size() > 0 {
-		return true
-	}
-	return freshNegativeImageCache(failPath)
-}
-
-// ServeCloudCached serves an already-local cloud sidecar image without asking
-// the cloud provider for a fresh direct link. It returns true when it wrote a
-// response, including a fresh negative-cache placeholder.
-func (p *ImageProxy) ServeCloudCached(w http.ResponseWriter, r *http.Request, stableKey string) bool {
-	if p == nil {
-		return false
-	}
-	key, cachePath, failPath := p.cloudImageCachePaths(stableKey)
-	if serveCachedImageFile(w, r, key, cachePath) {
-		return true
-	}
-	if freshNegativeImageCache(failPath) {
-		serveCachedPlaceholder(w)
-		return true
-	}
-	return false
-}
-
-// Serve writes the requested image to w. Caller is expected to validate
-// the JWT before invoking it.
-func (p *ImageProxy) Serve(ctx context.Context, w http.ResponseWriter, r *http.Request, raw string) error {
-	if isLocalImagePath(raw) {
-		path := filepath.Clean(raw)
-		abs, err := filepath.Abs(path)
-		if err != nil || !p.isAllowedLocalPath(abs) {
-			servePlaceholder(w)
-			return nil
-		}
-		path = abs
-		data, err := os.ReadFile(path)
-		if err != nil || len(data) == 0 {
-			servePlaceholder(w)
-			return nil
-		}
-		stat, _ := os.Stat(path)
-		modTime := time.Now()
-		if stat != nil {
-			modTime = stat.ModTime()
-		}
-		w.Header().Set("Content-Type", detectContentType(data))
-		w.Header().Set("Cache-Control", imageBrowserCacheControl)
-		http.ServeContent(w, r, filepath.Base(path), modTime, bytes.NewReader(data))
-		return nil
-	}
-
-	u, err := p.validateURL(raw)
-	if err != nil {
-		// Bad URL is the only request-side error; everything else falls
-		// through to the placeholder so the UI stays clean.
-		return err
-	}
-	host := strings.ToLower(u.Host)
-
-	// Cache key = sha256(url)
-	sum := sha256.Sum256([]byte(raw))
-	key := hex.EncodeToString(sum[:])
-	cachePath := filepath.Join(p.cacheDir, key)
-	failPath := cachePath + ".fail"
-
-	// Cache hit.
-	if data, err := os.ReadFile(cachePath); err == nil && len(data) > 0 { // #nosec G304 -- cachePath is derived from a SHA-256 cache key under the configured cache directory.
-		w.Header().Set("Content-Type", detectContentType(data))
-		w.Header().Set("Cache-Control", imageBrowserCacheControl)
-		stat, _ := os.Stat(cachePath)
-		modTime := time.Now()
-		if stat != nil {
-			modTime = stat.ModTime()
-		}
-		http.ServeContent(w, r, key, modTime, bytes.NewReader(data))
-		return nil
-	}
-	if stat, err := os.Stat(failPath); err == nil && time.Since(stat.ModTime()) < imageNegativeCacheTTL {
-		serveCachedPlaceholder(w)
-		return nil
-	} else if err == nil {
-		_ = os.Remove(failPath)
-	}
-
-	// Cache miss → fetch upstream.
-	if err := os.MkdirAll(p.cacheDir, 0o750); err != nil {
-		p.log.Warn("imageproxy: mkdir failed", zap.String("dir", p.cacheDir), zap.Error(err))
-		servePlaceholder(w)
-		return nil
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
-	if err != nil {
-		p.log.Warn("imageproxy: build request failed", zap.String("url", raw), zap.Error(err))
-		servePlaceholder(w)
-		return nil
-	}
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36")
-	req.Header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
-	if strings.Contains(host, "doubanio.com") {
-		req.Header.Set("Referer", "https://movie.douban.com/")
-	}
-
-	resp, err := p.client.Do(req)
-	if err != nil {
-		p.log.Warn("imageproxy: upstream fetch failed",
-			zap.String("host", host), zap.Error(err))
-		p.markImageFetchFailed(failPath)
-		serveCachedPlaceholder(w)
-		return nil
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		p.log.Warn("imageproxy: upstream returned non-OK",
-			zap.String("host", host), zap.String("status", resp.Status))
-		p.markImageFetchFailed(failPath)
-		serveCachedPlaceholder(w)
-		return nil
-	}
-
-	data, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) // 32 MiB cap
-	if err != nil || len(data) == 0 {
-		p.log.Warn("imageproxy: read upstream body failed",
-			zap.String("host", host), zap.Error(err))
-		p.markImageFetchFailed(failPath)
-		serveCachedPlaceholder(w)
-		return nil
-	}
-
-	// Write to a temp file then rename for atomicity.
-	p.mu.Lock()
-	tmp, tmpErr := os.CreateTemp(p.cacheDir, "img-*.tmp")
-	if tmpErr == nil {
-		if _, werr := tmp.Write(data); werr == nil {
-			_ = tmp.Close()
-			if rerr := os.Rename(tmp.Name(), cachePath); rerr != nil {
-				_ = os.Remove(tmp.Name())
-			} else {
-				_ = os.Remove(failPath)
-			}
-		} else {
-			_ = tmp.Close()
-			_ = os.Remove(tmp.Name())
-		}
-	}
-	p.mu.Unlock()
-
-	ctype := resp.Header.Get("Content-Type")
-	if ctype == "" {
-		ctype = detectContentType(data)
-	}
-	w.Header().Set("Content-Type", ctype)
-	if v := resp.Header.Get("Content-Length"); v != "" {
-		w.Header().Set("Content-Length", v)
-	}
-	if v := resp.Header.Get("ETag"); v != "" {
-		w.Header().Set("ETag", v)
-	}
-	if v := resp.Header.Get("Last-Modified"); v != "" {
-		w.Header().Set("Last-Modified", v)
-	}
-	w.Header().Set("Cache-Control", imageBrowserCacheControl)
-	http.ServeContent(w, r, key, time.Now(), bytes.NewReader(data))
-	return nil
-}
-
-// ServeCloudResolved stores a cloud sidecar image in the same disk cache used
-// by remote posters, then serves it with long browser-cache headers. Cloud
-// direct links are often short-lived, so caching by the stable provider/ref
-// avoids re-resolving and re-downloading artwork every time the web UI or an
-// Emby-compatible client opens a library.
-func (p *ImageProxy) ServeCloudResolved(ctx context.Context, w http.ResponseWriter, r *http.Request, stableKey string, link *cloud.DirectLink) error {
-	if p == nil || link == nil || strings.TrimSpace(link.URL) == "" {
-		servePlaceholder(w)
-		return nil
-	}
-	stableKey = strings.TrimSpace(stableKey)
-	if stableKey == "" {
-		stableKey = link.URL
-	}
-	key, cachePath, failPath := p.cloudImageCachePaths(stableKey)
-
-	if serveCachedImageFile(w, r, key, cachePath) {
-		return nil
-	}
-	if freshNegativeImageCache(failPath) {
-		serveCachedPlaceholder(w)
-		return nil
-	}
-
-	data, ctype, err := p.fetchAndCacheCloudImage(ctx, stableKey, link, r.UserAgent())
-	if err != nil {
-		p.log.Warn("imageproxy: cloud image fetch failed", zap.String("url", link.URL), zap.Error(err))
-		serveCachedPlaceholder(w)
-		return nil
-	}
-
-	w.Header().Set("Content-Type", ctype)
-	w.Header().Set("Cache-Control", imageBrowserCacheControl)
-	http.ServeContent(w, r, key, time.Now(), bytes.NewReader(data))
-	return nil
-}
-
-// PrefetchCloudResolved downloads a cloud sidecar image into the local cache
-// without writing an HTTP response. It is intentionally best-effort; callers
-// should queue it with low concurrency so large cloud libraries do not overload
-// small NAS devices.
-func (p *ImageProxy) PrefetchCloudResolved(ctx context.Context, stableKey string, link *cloud.DirectLink) error {
-	if p == nil || link == nil || strings.TrimSpace(link.URL) == "" {
-		return nil
-	}
-	if p.CloudImageCached(stableKey) {
-		return nil
-	}
-	_, _, err := p.fetchAndCacheCloudImage(ctx, stableKey, link, "MediaStationGo/0.1")
-	return err
-}
-
-func (p *ImageProxy) fetchAndCacheCloudImage(ctx context.Context, stableKey string, link *cloud.DirectLink, userAgent string) ([]byte, string, error) {
-	if err := os.MkdirAll(p.cacheDir, 0o750); err != nil {
-		return nil, "", err
-	}
-	_, cachePath, failPath := p.cloudImageCachePaths(stableKey)
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
-	if err != nil {
-		return nil, "", err
-	}
-	for k, v := range link.Headers {
-		req.Header.Set(k, v)
-	}
-	if req.Header.Get("User-Agent") == "" {
-		if strings.TrimSpace(userAgent) != "" {
-			req.Header.Set("User-Agent", userAgent)
-		} else {
-			req.Header.Set("User-Agent", "MediaStationGo/0.1")
-		}
-	}
-	req.Header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
-
-	resp, err := p.client.Do(req)
-	if err != nil {
-		p.markImageFetchFailed(failPath)
-		return nil, "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		p.markImageFetchFailed(failPath)
-		return nil, "", errors.New("cloud image returned " + resp.Status)
-	}
-	data, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
-	if err != nil {
-		p.markImageFetchFailed(failPath)
-		return nil, "", err
-	}
-	if len(data) == 0 {
-		p.markImageFetchFailed(failPath)
-		return nil, "", errors.New("cloud image body is empty")
-	}
-
-	p.mu.Lock()
-	tmp, tmpErr := os.CreateTemp(p.cacheDir, "img-cloud-*.tmp")
-	if tmpErr == nil {
-		if _, werr := tmp.Write(data); werr == nil {
-			_ = tmp.Close()
-			if rerr := os.Rename(tmp.Name(), cachePath); rerr != nil {
-				_ = os.Remove(tmp.Name())
-			} else {
-				_ = os.Remove(failPath)
-			}
-		} else {
-			_ = tmp.Close()
-			_ = os.Remove(tmp.Name())
-		}
-	}
-	p.mu.Unlock()
-
-	ctype := resp.Header.Get("Content-Type")
-	if ctype == "" {
-		ctype = detectContentType(data)
-	}
-	return data, ctype, nil
-}
-
-func (p *ImageProxy) markImageFetchFailed(failPath string) {
-	if err := os.MkdirAll(filepath.Dir(failPath), 0o750); err != nil {
-		return
-	}
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	_ = os.WriteFile(failPath, []byte(time.Now().Format(time.RFC3339Nano)), 0o600)
-}
-
-// Fetch 拉取远程图片并返回字节和 Content-Type（带缓存）。
-func (p *ImageProxy) Fetch(ctx context.Context, raw string) ([]byte, string, error) {
-	u, err := p.validateURL(raw)
-	if err != nil {
-		return nil, "", err
-	}
-
-	// Cache lookup
-	sum := sha256.Sum256([]byte(raw))
-	key := hex.EncodeToString(sum[:])
-	cachePath := filepath.Join(p.cacheDir, key)
-
-	if data, err := os.ReadFile(cachePath); err == nil && len(data) > 0 { // #nosec G304 -- cachePath is derived from a SHA-256 cache key under the configured cache directory.
-		return data, detectContentType(data), nil
-	}
-
-	// Fetch upstream
-	if err := os.MkdirAll(p.cacheDir, 0o750); err != nil {
-		return nil, "", err
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)
-	if err != nil {
-		return nil, "", err
-	}
-	req.Header.Set("User-Agent", "MediaStationGo/0.1")
-
-	resp, err := p.client.Do(req)
-	if err != nil {
-		return nil, "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return nil, "", errors.New("upstream returned " + resp.Status)
-	}
-
-	data, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
-	if err != nil {
-		return nil, "", err
-	}
-
-	// Write to cache
-	p.mu.Lock()
-	tmp, terr := os.CreateTemp(p.cacheDir, "img-*.tmp")
-	if terr == nil {
-		if _, werr := tmp.Write(data); werr == nil {
-			_ = tmp.Close()
-			if rerr := os.Rename(tmp.Name(), cachePath); rerr != nil {
-				_ = os.Remove(tmp.Name())
-			}
-		} else {
-			_ = tmp.Close()
-			_ = os.Remove(tmp.Name())
-		}
-	}
-	p.mu.Unlock()
-
-	ctype := resp.Header.Get("Content-Type")
-	if ctype == "" {
-		ctype = detectContentType(data)
-	}
-	// host is unused here but referenced for log clarity in the future.
-	_ = u
-	return data, ctype, nil
-}