refactor: split cloud scan candidate collection

2026-10-05 04:56:37 +08:00 · 2026-06-24 20:28:01 +08:00
parent a7c8d46589
commit 68e6b5050b
2 changed files with 235 additions and 137 deletions
@@ -3,9 +3,6 @@ package service
 import (
 	"context"
 	"fmt"
-	"path/filepath"
-	"strings"
-	"sync"

 	"go.uber.org/zap"

@@ -31,142 +28,17 @@ func (s *ScannerService) scanCloudLibrary(ctx context.Context, lib *model.Librar
 	autoCategoryRoot := cloudRootMountNeedsAutoCategory(mount)
 	scopeIDs := s.cloudScanLibraryScopeIDs(ctx, lib, mount)
 	seen := make(map[string]struct{})
-	seenRefs := make(map[string]struct{})
-	candidates := make([]cloudCandidate, 0, 256)
-	candidateByKey := make(map[string]int)
-	visitedDirs := map[string]struct{}{}
 	progress := newCloudScanProgressState()
-	var stateMu sync.Mutex
 	progress.publish(s, lib.ID, res, "listing", true)
-	var walkWG sync.WaitGroup
-	var walkErr error
-	var walkErrOnce sync.Once
-	setWalkErr := func(err error) {
-		if err != nil {
-			walkErrOnce.Do(func() {
-				walkErr = err
-			})
-		}
-	}
-	listSlots := make(chan struct{}, s.cloudScanWorkerCount())
-	var walkCloud func(dirID, displayDir string, inheritedMeta *LocalMetadata) error
-	walkCloud = func(dirID, displayDir string, inheritedMeta *LocalMetadata) error {
-		defer walkWG.Done()
-		if err := ctx.Err(); err != nil {
-			setWalkErr(err)
-			return err
-		}
-		stateMu.Lock()
-		if _, ok := visitedDirs[dirID]; ok {
-			stateMu.Unlock()
-			return nil
-		}
-		visitedDirs[dirID] = struct{}{}
-		stateMu.Unlock()
-
-		select {
-		case listSlots <- struct{}{}:
-			defer func() { <-listSlots }()
-		case <-ctx.Done():
-			setWalkErr(ctx.Err())
-			return ctx.Err()
-		}
-		entries, err := s.storage.CloudList(ctx, typ, dirID)
-		if err != nil {
-			if dirID != rootDir {
-				progress.addSkipped(res)
-				s.log.Warn("skip inaccessible cloud directory",
-					zap.String("library_id", lib.ID),
-					zap.String("provider", typ),
-					zap.String("dir", dirID),
-					zap.Error(err))
-				return nil
-			}
-			setWalkErr(err)
-			return err
-		}
-		progress.publish(s, lib.ID, res, "listing", progress.markDirVisited())
-		sidecars := newCloudSidecarSet(typ, entries)
-		dirMeta := s.cloudDirectoryMetadata(ctx, typ, displayDir, sidecars, inheritedMeta)
-		s.cacheCloudMetadataArtworkNow(ctx, dirMeta)
-		for _, entry := range entries {
-			select {
-			case <-ctx.Done():
-				setWalkErr(ctx.Err())
-				return ctx.Err()
-			default:
-			}
-			if entry.IsDir {
-				if strings.TrimSpace(entry.ID) != "" {
-					walkWG.Add(1)
-					go func(childID, childDisplay string, childMeta *LocalMetadata) {
-						_ = walkCloud(childID, childDisplay, childMeta)
-					}(entry.ID, joinCloudDisplayPath(displayDir, entry.Name), dirMeta)
-				}
-				continue
-			}
-			ext := strings.ToLower(filepath.Ext(entry.Name))
-			if _, ok := videoExtensions[ext]; !ok {
-				continue
-			}
-			ref := cloudEntryRef(typ, entry.ID, entry.PickCode)
-			if ref == "" {
-				progress.addSkipped(res)
-				continue
-			}
-			stateMu.Lock()
-			if _, ok := seenRefs[ref]; ok {
-				stateMu.Unlock()
-				progress.addSkipped(res)
-				continue
-			}
-			seenRefs[ref] = struct{}{}
-			stateMu.Unlock()
-			progress.publish(s, lib.ID, res, "listing", progress.markFileDiscovered())
-			displayPath := joinCloudDisplayPath(displayDir, entry.Name)
-			path := cloudMediaPath(typ, displayPath)
-			localMeta := s.cloudFileMetadata(ctx, typ, displayPath, entry.Name, sidecars, dirMeta, librarySupportsSeasons(lib))
-			localMeta = s.enrichCloudMetadataFromExternalIDs(ctx, lib, path, localMeta)
-			if localMeta != nil {
-				s.cacheCloudMetadataArtworkNow(ctx, localMeta)
-			}
-			candidate := cloudCandidate{
-				ref:       ref,
-				name:      entry.Name,
-				size:      entry.Size,
-				path:      path,
-				localMeta: localMeta,
-			}
-			if autoCategoryRoot {
-				candidate.categoryDisplayDir = cloudAutoCategoryDisplayDirForMediaPath(path)
-			}
-			key := cloudMediaDedupeKey(lib, displayDir, entry.Name, entry.Size)
-			stateMu.Lock()
-			if key != "" {
-				if prevIndex, ok := candidateByKey[key]; ok {
-					if candidate.size > candidates[prevIndex].size {
-						candidates[prevIndex] = candidate
-					}
-					stateMu.Unlock()
-					progress.addSkipped(res)
-					continue
-				}
-				candidateByKey[key] = len(candidates)
-			}
-			candidates = append(candidates, candidate)
-			stateMu.Unlock()
-		}
-		return nil
-	}
-	walkWG.Add(1)
-	go func() {
-		_ = walkCloud(rootDir, rootDisplayDir, nil)
-	}()
-	walkWG.Wait()
-	if walkErr != nil {
-		return res, walkErr
-	}
-	if err := ctx.Err(); err != nil {
+	candidates, err := s.collectCloudScanCandidates(ctx, lib, cloudScanCandidateRequest{
+		provider:         typ,
+		rootDir:          rootDir,
+		rootDisplayDir:   rootDisplayDir,
+		autoCategoryRoot: autoCategoryRoot,
+		progress:         progress,
+		result:           res,
+	})
+	if err != nil {
 		return res, err
 	}
 	existingMedia, err := s.existingCloudMediaSnapshotForLibraries(ctx, scopeIDs)