fix cloud library scale and playback compatibility

Root cause: display-only cloud library filtering was reused by scan jobs, merged cloud mounts could be skipped, and OpenList listing relied on WebDAV/first-page behavior that could cap huge directories around 100 items. Cloud scans also let new items consume probe budget before existing rows with missing track/artwork metadata.\n\nChanges:\n- split display filtering from scannable cloud filtering so merged cloud mounts still scan\n- add OpenList API pagination with WebDAV fallback\n- prioritize existing cloud media missing metadata before new imports\n- restrict automatic cloud sync to one successful 19:00-21:00 daily window and keep manual scan immediate\n- disable startup cloud full-scan by default, with an explicit opt-in setting\n- improve Emby/cloud playback compatibility and cache/search/test coverage from the continued work
2026-10-08 06:16:37 +08:00 · 2026-06-11 18:31:01 +08:00
parent c22d8271ae
commit 23d1fbfddb
35 changed files with 2797 additions and 139 deletions
@@ -15,6 +15,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"go.uber.org/zap"
@@ -26,19 +27,42 @@ import (

 // StorageConfigService encrypts + persists external storage configs.
 type StorageConfigService struct {
-	log    *zap.Logger
-	repo   *repository.Container
-	crypto *CryptoService
-	client *http.Client
+	log           *zap.Logger
+	repo          *repository.Container
+	crypto        *CryptoService
+	client        *http.Client
+	resolveMu     sync.Mutex
+	resolveCache  map[string]cloudResolveCacheEntry
+	resolveFlight map[string]*cloudResolveCall
 }

+type cloudResolveCacheEntry struct {
+	link      *cloud.DirectLink
+	expiresAt time.Time
+	hits      int
+	lastHit   time.Time
+}
+
+type cloudResolveCall struct {
+	done chan struct{}
+	link *cloud.DirectLink
+	err  error
+}
+
+const (
+	cloudResolveHotHitThreshold      = 3
+	cloudResolveBackgroundRefreshMax = 30 * time.Second
+)
+
 // NewStorageConfigService is the constructor.
 func NewStorageConfigService(log *zap.Logger, repo *repository.Container, crypto *CryptoService) *StorageConfigService {
 	return &StorageConfigService{
-		log:    log,
-		repo:   repo,
-		crypto: crypto,
-		client: &http.Client{Timeout: 120 * time.Second},
+		log:           log,
+		repo:          repo,
+		crypto:        crypto,
+		client:        &http.Client{Timeout: 120 * time.Second},
+		resolveCache:  make(map[string]cloudResolveCacheEntry),
+		resolveFlight: make(map[string]*cloudResolveCall),
 	}
 }

@@ -119,6 +143,7 @@ func (s *StorageConfigService) Save(ctx context.Context, in StorageInput) (*Stor
 	if err := s.repo.StorageConfig.Upsert(ctx, row); err != nil {
 		return nil, err
 	}
+	s.clearResolveCacheForType(in.Type)
 	return s.Get(ctx, in.Type)
 }

@@ -263,6 +288,211 @@ func (s *StorageConfigService) CloudList(ctx context.Context, typ, dirID string)
 // points at a link the client can fetch directly (true offload). When clientUA
 // is empty the provider's default UA is used.
 func (s *StorageConfigService) CloudResolve(ctx context.Context, typ, fileRef, clientUA string) (*cloud.DirectLink, error) {
+	if s == nil {
+		return nil, errors.New("storage config service unavailable")
+	}
+	cacheKey := s.resolveCacheKey(typ, fileRef, clientUA)
+	if link, ok, refresh := s.cachedResolve(cacheKey, typ); ok {
+		if refresh {
+			s.refreshResolveInBackground(cacheKey, typ, fileRef, clientUA)
+		}
+		return link, nil
+	}
+	if call, owner := s.beginResolve(cacheKey); !owner {
+		select {
+		case <-call.done:
+			if call.err != nil {
+				return nil, call.err
+			}
+			return cloneDirectLink(call.link), nil
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	} else {
+		defer s.finishResolve(cacheKey, call)
+		p, err := s.cloudProviderWithUA(ctx, typ, clientUA)
+		if err != nil {
+			call.err = err
+			return nil, err
+		}
+		link, err := p.Resolve(ctx, fileRef)
+		if err != nil {
+			call.err = err
+			return nil, err
+		}
+		call.link = cloneDirectLink(link)
+		s.storeResolvedLink(cacheKey, typ, link)
+		return cloneDirectLink(link), nil
+	}
+}
+
+func (s *StorageConfigService) resolveCacheKey(typ, fileRef, clientUA string) string {
+	return strings.TrimSpace(typ) + "\x00" + strings.TrimSpace(fileRef) + "\x00" + strings.TrimSpace(clientUA)
+}
+
+func (s *StorageConfigService) cachedResolve(key, typ string) (*cloud.DirectLink, bool, bool) {
+	s.resolveMu.Lock()
+	defer s.resolveMu.Unlock()
+	if s.resolveCache == nil {
+		s.resolveCache = make(map[string]cloudResolveCacheEntry)
+		return nil, false, false
+	}
+	entry, ok := s.resolveCache[key]
+	now := time.Now()
+	if !ok || now.After(entry.expiresAt) {
+		if ok {
+			delete(s.resolveCache, key)
+		}
+		return nil, false, false
+	}
+	entry.hits++
+	entry.lastHit = now
+	s.resolveCache[key] = entry
+	refreshWindow := cloudResolveHotRefreshWindow(cloudResolveCacheTTL(typ))
+	shouldRefresh := entry.hits >= cloudResolveHotHitThreshold &&
+		refreshWindow > 0 &&
+		now.Add(refreshWindow).After(entry.expiresAt)
+	return cloneDirectLink(entry.link), true, shouldRefresh
+}
+
+func (s *StorageConfigService) beginResolve(key string) (*cloudResolveCall, bool) {
+	s.resolveMu.Lock()
+	defer s.resolveMu.Unlock()
+	if s.resolveFlight == nil {
+		s.resolveFlight = make(map[string]*cloudResolveCall)
+	}
+	if call := s.resolveFlight[key]; call != nil {
+		return call, false
+	}
+	call := &cloudResolveCall{done: make(chan struct{})}
+	s.resolveFlight[key] = call
+	return call, true
+}
+
+func (s *StorageConfigService) finishResolve(key string, call *cloudResolveCall) {
+	s.resolveMu.Lock()
+	if current := s.resolveFlight[key]; current == call {
+		delete(s.resolveFlight, key)
+	}
+	s.resolveMu.Unlock()
+	close(call.done)
+}
+
+func (s *StorageConfigService) refreshResolveInBackground(key, typ, fileRef, clientUA string) {
+	if s == nil {
+		return
+	}
+	go func() {
+		call, owner := s.beginResolve(key)
+		if !owner {
+			return
+		}
+		defer s.finishResolve(key, call)
+		ctx, cancel := context.WithTimeout(context.Background(), cloudResolveBackgroundRefreshMax)
+		defer cancel()
+		p, err := s.cloudProviderWithUA(ctx, typ, clientUA)
+		if err != nil {
+			call.err = err
+			if s.log != nil {
+				s.log.Debug("refresh cloud direct link failed", zap.String("provider", typ), zap.Error(err))
+			}
+			return
+		}
+		link, err := p.Resolve(ctx, fileRef)
+		if err != nil {
+			call.err = err
+			if s.log != nil {
+				s.log.Debug("refresh cloud direct link failed", zap.String("provider", typ), zap.Error(err))
+			}
+			return
+		}
+		call.link = cloneDirectLink(link)
+		s.storeResolvedLink(key, typ, link)
+	}()
+}
+
+func (s *StorageConfigService) storeResolvedLink(key, typ string, link *cloud.DirectLink) {
+	if link == nil || strings.TrimSpace(link.URL) == "" {
+		return
+	}
+	ttl := cloudResolveCacheTTL(typ)
+	if ttl <= 0 {
+		return
+	}
+	s.resolveMu.Lock()
+	defer s.resolveMu.Unlock()
+	if s.resolveCache == nil {
+		s.resolveCache = make(map[string]cloudResolveCacheEntry)
+	}
+	now := time.Now()
+	hits := 0
+	if existing, ok := s.resolveCache[key]; ok {
+		hits = existing.hits
+	}
+	s.resolveCache[key] = cloudResolveCacheEntry{link: cloneDirectLink(link), expiresAt: now.Add(ttl), hits: hits, lastHit: now}
+}
+
+func cloudResolveHotRefreshWindow(ttl time.Duration) time.Duration {
+	if ttl <= 0 {
+		return 0
+	}
+	window := ttl / 4
+	if window < 15*time.Second {
+		window = 15 * time.Second
+	}
+	if window > 2*time.Minute {
+		window = 2 * time.Minute
+	}
+	return window
+}
+
+func cloudResolveCacheTTL(typ string) time.Duration {
+	switch typ {
+	case cloud.TypeQuark, cloud.Type115:
+		return 2 * time.Minute
+	case cloud.TypeCloudDrive2, cloud.TypeOpenList:
+		return 15 * time.Minute
+	default:
+		return 5 * time.Minute
+	}
+}
+
+func cloneDirectLink(link *cloud.DirectLink) *cloud.DirectLink {
+	if link == nil {
+		return nil
+	}
+	out := &cloud.DirectLink{
+		URL:     link.URL,
+		Headers: make(map[string]string, len(link.Headers)),
+		Proxy:   link.Proxy,
+	}
+	for k, v := range link.Headers {
+		out.Headers[k] = v
+	}
+	return out
+}
+
+func (s *StorageConfigService) clearResolveCacheForType(typ string) {
+	typ = strings.TrimSpace(typ)
+	if typ == "" {
+		return
+	}
+	prefix := typ + "\x00"
+	s.resolveMu.Lock()
+	defer s.resolveMu.Unlock()
+	for key := range s.resolveCache {
+		if strings.HasPrefix(key, prefix) {
+			delete(s.resolveCache, key)
+		}
+	}
+	for key, call := range s.resolveFlight {
+		if strings.HasPrefix(key, prefix) && call != nil {
+			call.err = fmt.Errorf("%s storage config changed", typ)
+		}
+	}
+}
+
+func (s *StorageConfigService) CloudResolveUncached(ctx context.Context, typ, fileRef, clientUA string) (*cloud.DirectLink, error) {
 	p, err := s.cloudProviderWithUA(ctx, typ, clientUA)
 	if err != nil {
 		return nil, err
@@ -496,7 +726,6 @@ func strr(v any) string {
 	return strings.TrimSpace(fmt.Sprint(v))
 }

-
 // DeleteStorage 删除存储配置并清理关联数据
 func (s *StorageConfigService) DeleteStorage(ctx context.Context, storageType string) error {
 	// 查找配置