refactor: split modules and harden scraping workflows

internal/service/storage_config.go

@@ -1,4 +1,4 @@
-// Package service — Alist / S3 / WebDAV configuration management.
+// Package service — external storage configuration management.
 //
 // StorageConfigService stores connection settings encrypted at rest
 // (via CryptoService). It also exposes a Test() probe so the React UI
@@ -10,9 +10,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
-	"net/url"
 	"strconv"
 	"strings"
 	"sync"
@@ -36,24 +34,6 @@ type StorageConfigService struct {
 	resolveFlight map[string]*cloudResolveCall
 }
 
-type cloudResolveCacheEntry struct {
-	link      *cloud.DirectLink
-	expiresAt time.Time
-	hits      int
-	lastHit   time.Time
-}
-
-type cloudResolveCall struct {
-	done chan struct{}
-	link *cloud.DirectLink
-	err  error
-}
-
-const (
-	cloudResolveHotHitThreshold      = 3
-	cloudResolveBackgroundRefreshMax = 30 * time.Second
-)
-
 // NewStorageConfigService is the constructor.
 func NewStorageConfigService(log *zap.Logger, repo *repository.Container, crypto *CryptoService) *StorageConfigService {
 	return &StorageConfigService{
@@ -108,6 +88,9 @@ func (s *StorageConfigService) List(ctx context.Context) ([]StorageView, error)
 	}
 	out := make([]StorageView, 0, len(rows))
 	for _, r := range rows {
+		if !IsAdminStorageConfigurable(r.Type) {
+			continue
+		}
 		plain := s.crypto.Decrypt(r.Config)
 		var cfg map[string]any
 		_ = json.Unmarshal([]byte(plain), &cfg)
@@ -334,7 +317,7 @@ func (s *StorageConfigService) Test(ctx context.Context, in StorageInput) error
 		}
 		defer resp.Body.Close()
 		return nil
-	case cloud.TypeQuark, cloud.Type115, cloud.TypeCloudDrive2:
+	case cloud.Type115, cloud.TypeCloudDrive2:
 		p, err := cloud.New(in.Type, cfg, client)
 		if err != nil {
 			return err
@@ -373,287 +356,40 @@ func (s *StorageConfigService) CloudList(ctx context.Context, typ, dirID string)
 	return p.List(ctx, dirID)
 }
 
-// CloudResolve resolves a cloud file reference to a direct link.
-//
-// clientUA is the User-Agent of the playback client that will follow the 302
-// redirect. 115/夸克 CDN links are bound to the UA used to request them, so we
-// resolve with the client's own UA — that way the pure 302 the host issues
-// points at a link the client can fetch directly (true offload). When clientUA
-// is empty the provider's default UA is used.
-func (s *StorageConfigService) CloudResolve(ctx context.Context, typ, fileRef, clientUA string) (*cloud.DirectLink, error) {
-	if s == nil {
-		return nil, errors.New("storage config service unavailable")
-	}
-	cacheKey := s.resolveCacheKey(typ, fileRef, clientUA)
-	if link, ok, refresh := s.cachedResolve(cacheKey, typ); ok {
-		if refresh {
-			s.refreshResolveInBackground(cacheKey, typ, fileRef, clientUA)
-		}
-		return link, nil
-	}
-	if call, owner := s.beginResolve(cacheKey); !owner {
-		select {
-		case <-call.done:
-			if call.err != nil {
-				return nil, call.err
-			}
-			return cloneDirectLink(call.link), nil
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-	} else {
-		defer s.finishResolve(cacheKey, call)
-		p, err := s.cloudProviderWithUA(ctx, typ, clientUA)
-		if err != nil {
-			call.err = err
-			return nil, err
-		}
-		link, err := p.Resolve(ctx, fileRef)
-		if err != nil {
-			call.err = err
-			return nil, err
-		}
-		call.link = cloneDirectLink(link)
-		s.storeResolvedLink(cacheKey, typ, link)
-		return cloneDirectLink(link), nil
-	}
-}
-
-func (s *StorageConfigService) resolveCacheKey(typ, fileRef, clientUA string) string {
-	return strings.TrimSpace(typ) + "\x00" + strings.TrimSpace(fileRef) + "\x00" + strings.TrimSpace(clientUA)
-}
-
-func (s *StorageConfigService) cachedResolve(key, typ string) (*cloud.DirectLink, bool, bool) {
-	s.resolveMu.Lock()
-	defer s.resolveMu.Unlock()
-	if s.resolveCache == nil {
-		s.resolveCache = make(map[string]cloudResolveCacheEntry)
-		return nil, false, false
-	}
-	entry, ok := s.resolveCache[key]
-	now := time.Now()
-	if !ok || now.After(entry.expiresAt) {
-		if ok {
-			delete(s.resolveCache, key)
-		}
-		return nil, false, false
-	}
-	entry.hits++
-	entry.lastHit = now
-	s.resolveCache[key] = entry
-	refreshWindow := cloudResolveHotRefreshWindow(cloudResolveCacheTTL(typ))
-	shouldRefresh := entry.hits >= cloudResolveHotHitThreshold &&
-		refreshWindow > 0 &&
-		now.Add(refreshWindow).After(entry.expiresAt)
-	return cloneDirectLink(entry.link), true, shouldRefresh
-}
-
-func (s *StorageConfigService) beginResolve(key string) (*cloudResolveCall, bool) {
-	s.resolveMu.Lock()
-	defer s.resolveMu.Unlock()
-	if s.resolveFlight == nil {
-		s.resolveFlight = make(map[string]*cloudResolveCall)
-	}
-	if call := s.resolveFlight[key]; call != nil {
-		return call, false
-	}
-	call := &cloudResolveCall{done: make(chan struct{})}
-	s.resolveFlight[key] = call
-	return call, true
-}
-
-func (s *StorageConfigService) finishResolve(key string, call *cloudResolveCall) {
-	s.resolveMu.Lock()
-	if current := s.resolveFlight[key]; current == call {
-		delete(s.resolveFlight, key)
-	}
-	s.resolveMu.Unlock()
-	close(call.done)
-}
-
-func (s *StorageConfigService) refreshResolveInBackground(key, typ, fileRef, clientUA string) {
-	if s == nil {
-		return
-	}
-	go func() {
-		call, owner := s.beginResolve(key)
-		if !owner {
-			return
-		}
-		defer s.finishResolve(key, call)
-		ctx, cancel := context.WithTimeout(context.Background(), cloudResolveBackgroundRefreshMax)
-		defer cancel()
-		p, err := s.cloudProviderWithUA(ctx, typ, clientUA)
-		if err != nil {
-			call.err = err
-			if s.log != nil {
-				s.log.Debug("refresh cloud direct link failed", zap.String("provider", typ), zap.Error(err))
-			}
-			return
-		}
-		link, err := p.Resolve(ctx, fileRef)
-		if err != nil {
-			call.err = err
-			if s.log != nil {
-				s.log.Debug("refresh cloud direct link failed", zap.String("provider", typ), zap.Error(err))
-			}
-			return
-		}
-		call.link = cloneDirectLink(link)
-		s.storeResolvedLink(key, typ, link)
-	}()
-}
-
-func (s *StorageConfigService) storeResolvedLink(key, typ string, link *cloud.DirectLink) {
-	if link == nil || strings.TrimSpace(link.URL) == "" {
-		return
-	}
-	ttl := cloudResolveCacheTTL(typ)
-	if ttl <= 0 {
-		return
-	}
-	s.resolveMu.Lock()
-	defer s.resolveMu.Unlock()
-	if s.resolveCache == nil {
-		s.resolveCache = make(map[string]cloudResolveCacheEntry)
-	}
-	now := time.Now()
-	hits := 0
-	if existing, ok := s.resolveCache[key]; ok {
-		hits = existing.hits
-	}
-	s.resolveCache[key] = cloudResolveCacheEntry{link: cloneDirectLink(link), expiresAt: now.Add(ttl), hits: hits, lastHit: now}
-}
-
-func cloudResolveHotRefreshWindow(ttl time.Duration) time.Duration {
-	if ttl <= 0 {
-		return 0
-	}
-	window := ttl / 4
-	if window < 15*time.Second {
-		window = 15 * time.Second
-	}
-	if window > 2*time.Minute {
-		window = 2 * time.Minute
-	}
-	return window
-}
-
-func cloudResolveCacheTTL(typ string) time.Duration {
-	switch typ {
-	case cloud.TypeQuark, cloud.Type115, cloud.TypeCloudDrive2, cloud.TypeOpenList:
-		return 2 * time.Minute
-	default:
-		return 5 * time.Minute
-	}
-}
-
-func cloneDirectLink(link *cloud.DirectLink) *cloud.DirectLink {
-	if link == nil {
-		return nil
-	}
-	out := &cloud.DirectLink{
-		URL:     link.URL,
-		Headers: make(map[string]string, len(link.Headers)),
-		Proxy:   link.Proxy,
-	}
-	for k, v := range link.Headers {
-		out.Headers[k] = v
-	}
-	return out
-}
-
-func (s *StorageConfigService) clearResolveCacheForType(typ string) {
-	typ = strings.TrimSpace(typ)
-	if typ == "" {
-		return
-	}
-	prefix := typ + "\x00"
-	s.resolveMu.Lock()
-	defer s.resolveMu.Unlock()
-	for key := range s.resolveCache {
-		if strings.HasPrefix(key, prefix) {
-			delete(s.resolveCache, key)
-		}
-	}
-	for key, call := range s.resolveFlight {
-		if strings.HasPrefix(key, prefix) && call != nil {
-			call.err = fmt.Errorf("%s storage config changed", typ)
-		}
-	}
-}
-
-func (s *StorageConfigService) CloudResolveUncached(ctx context.Context, typ, fileRef, clientUA string) (*cloud.DirectLink, error) {
-	p, err := s.cloudProviderWithUA(ctx, typ, clientUA)
+func (s *StorageConfigService) CloudMkdir(ctx context.Context, typ, parentDir, name string) (*cloud.FileEntry, error) {
+	p, err := s.CloudProvider(ctx, typ)
 	if err != nil {
 		return nil, err
 	}
-	return p.Resolve(ctx, fileRef)
+	mutable, ok := p.(cloud.MutableProvider)
+	if !ok {
+		return nil, fmt.Errorf("%s does not support folder creation", typ)
+	}
+	return mutable.Mkdir(ctx, parentDir, name)
 }
 
-// CloudReadText resolves a small cloud file and returns its text payload. It is
-// used for cloud-hosted .strm files: the scanner reads the STRM target once and
-// stores the real playback URL, while the media bytes still stay in the cloud.
-func (s *StorageConfigService) CloudReadText(ctx context.Context, typ, fileRef string, limit int64) (string, error) {
-	if limit <= 0 {
-		limit = 64 << 10
-	}
-	link, err := s.CloudResolve(ctx, typ, fileRef, "")
-	if err != nil {
-		return "", err
-	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil)
-	if err != nil {
-		return "", err
-	}
-	for k, v := range link.Headers {
-		req.Header.Set(k, v)
-	}
-	resp, err := s.client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return "", fmt.Errorf("%s: read strm returned http %d", typ, resp.StatusCode)
-	}
-	body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
-	if err != nil {
-		return "", err
-	}
-	if int64(len(body)) > limit {
-		return "", fmt.Errorf("%s: strm file is too large", typ)
-	}
-	return strings.TrimSpace(strings.TrimPrefix(string(body), "\ufeff")), nil
-}
-
-// cloudProviderWithUA builds a provider, overriding the request UA when a
-// non-empty clientUA is supplied.
-func (s *StorageConfigService) cloudProviderWithUA(ctx context.Context, typ, clientUA string) (cloud.Provider, error) {
-	if !cloud.IsCloudType(typ) {
-		return nil, fmt.Errorf("not a cloud provider: %q", typ)
-	}
-	view, err := s.Get(ctx, typ)
+func (s *StorageConfigService) CloudRename(ctx context.Context, typ, ref, name string) (*cloud.FileEntry, error) {
+	p, err := s.CloudProvider(ctx, typ)
 	if err != nil {
 		return nil, err
 	}
-	if view == nil {
-		return nil, fmt.Errorf("%s storage not configured", typ)
+	mutable, ok := p.(cloud.MutableProvider)
+	if !ok {
+		return nil, fmt.Errorf("%s does not support rename", typ)
 	}
-	if !view.Enabled {
-		return nil, fmt.Errorf("%s storage disabled", typ)
+	return mutable.Rename(ctx, ref, name)
+}
+
+func (s *StorageConfigService) CloudMove(ctx context.Context, typ, ref, targetDir, name string) (*cloud.FileEntry, error) {
+	p, err := s.CloudProvider(ctx, typ)
+	if err != nil {
+		return nil, err
 	}
-	cfg := view.Config
-	if strings.TrimSpace(clientUA) != "" {
-		// Copy so we never mutate the cached view config.
-		cp := make(map[string]any, len(cfg)+1)
-		for k, v := range cfg {
-			cp[k] = v
-		}
-		cp["ua"] = clientUA
-		cfg = cp
+	movable, ok := p.(cloud.MovableProvider)
+	if !ok {
+		return nil, fmt.Errorf("%s does not support move", typ)
 	}
-	return cloud.New(typ, cfg, s.clientForConfig(cfg))
+	return movable.Move(ctx, ref, targetDir, name)
 }
 
 func (s *StorageConfigService) clientForConfig(cfg map[string]any) *http.Client {
@@ -704,8 +440,6 @@ func storageTimeoutFromConfig(cfg map[string]any, fallback time.Duration) time.D
 // cloudLibraryName maps a provider type to a friendly Chinese library name.
 func cloudLibraryName(typ string) string {
 	switch typ {
-	case cloud.TypeQuark:
-		return "夸克网盘"
 	case cloud.Type115:
 		return "115 网盘"
 	case cloud.TypeCloudDrive2:
@@ -766,7 +500,7 @@ func (s *StorageConfigService) CloudImport(ctx context.Context, typ, fileRef, na
 		Path:         cloudMediaPath(typ, fileRef),
 		SizeBytes:    size,
 		Container:    container,
-		STRMURL:      BuildPublicAPIURL(ctx, s.repo, nil, "/api/cloud/play/"+typ, url.Values{"ref": []string{fileRef}}),
+		STRMURL:      BuildRelativeCloudPlayURL(typ, fileRef),
 		ScrapeStatus: "pending",
 	}
 	if err := s.repo.Media.Upsert(ctx, m); err != nil {
@@ -777,7 +511,7 @@ func (s *StorageConfigService) CloudImport(ctx context.Context, typ, fileRef, na
 
 func validStorageType(t string) bool {
 	switch t {
-	case "alist", "s3", "webdav", cloud.TypeQuark, cloud.Type115, cloud.TypeCloudDrive2, cloud.TypeOpenList:
+	case "alist", "s3", "webdav", cloud.Type115, cloud.TypeCloudDrive2, cloud.TypeOpenList:
 		return true
 	}
 	return false