Improve large library storage backends and scans

2026-10-08 14:26:37 +08:00 · 2026-06-15 18:16:34 +08:00
parent c9e6adf041
commit a5bf4bfdd4
36 changed files with 2340 additions and 93 deletions
@@ -3,11 +3,15 @@ package service

 import (
 	"context"
+	"crypto/sha1"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"

 	"go.uber.org/zap"

@@ -18,9 +22,10 @@ import (

 // MediaService offers high-level CRUD over libraries and media items.
 type MediaService struct {
-	cfg  *config.Config
-	log  *zap.Logger
-	repo *repository.Container
+	cfg   *config.Config
+	log   *zap.Logger
+	repo  *repository.Container
+	cache *RuntimeCacheService
 }

 type MediaVisibility struct {
@@ -29,6 +34,11 @@ type MediaVisibility struct {
 	HiddenLibraryIDs  []string
 }

+type MediaItem struct {
+	model.Media
+	Versions []model.Media `json:"versions,omitempty"`
+}
+
 const maxMediaSearchLimit = 50000
 const maxMediaSearchPageSize = 2000

@@ -60,6 +70,13 @@ func NewMediaService(cfg *config.Config, log *zap.Logger, repo *repository.Conta
 	return &MediaService{cfg: cfg, log: log, repo: repo}
 }

+func (s *MediaService) SetRuntimeCache(cache *RuntimeCacheService) *MediaService {
+	if s != nil {
+		s.cache = cache
+	}
+	return s
+}
+
 // CreateLibrary persists a library after validating that its path exists.
 func (s *MediaService) CreateLibrary(ctx context.Context, name, path, kind string) (*model.Library, error) {
 	if name == "" || path == "" {
@@ -74,6 +91,7 @@ func (s *MediaService) CreateLibrary(ctx context.Context, name, path, kind strin
 	if err := s.repo.Library.Create(ctx, lib); err != nil {
 		return nil, err
 	}
+	s.invalidateMediaCache(ctx)
 	return lib, nil
 }

@@ -306,13 +324,21 @@ func (s *MediaService) DeleteLibrary(ctx context.Context, id string) error {
 			if err := s.repo.Media.PurgeByLibrary(ctx, id); err != nil {
 				return err
 			}
-			return s.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.Library{}).Error
+			err := s.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.Library{}).Error
+			if err == nil {
+				s.invalidateMediaCache(ctx)
+			}
+			return err
 		}
 	}
 	if err := s.repo.Media.DeleteByLibrary(ctx, id); err != nil {
 		return err
 	}
-	return s.repo.Library.Delete(ctx, id)
+	err = s.repo.Library.Delete(ctx, id)
+	if err == nil {
+		s.invalidateMediaCache(ctx)
+	}
+	return err
 }

 // ListMedia paginates media items inside a library.
@@ -335,11 +361,194 @@ func (s *MediaService) ListMediaVisible(ctx context.Context, libraryID string, p
 	if err != nil {
 		return nil, 0, err
 	}
-	return s.repo.Media.ListByLibrariesFiltered(ctx, libraryIDs, (page-1)*pageSize, pageSize, repository.MediaQueryFilter{
+	filter := repository.MediaQueryFilter{
 		IncludeNSFW:       visibility.IncludeNSFW,
 		AllowedLibraryIDs: visibility.AllowedLibraryIDs,
 		HiddenLibraryIDs:  visibility.HiddenLibraryIDs,
+	}
+	cacheKey := s.mediaListCacheKey(libraryID, libraryIDs, page, pageSize, filter)
+	var cached mediaListCacheValue
+	if s.cache != nil && s.cache.GetJSON(ctx, cacheKey, &cached) {
+		return cached.Items, cached.Total, nil
+	}
+	items, total, err := s.repo.Media.ListByLibrariesFiltered(ctx, libraryIDs, (page-1)*pageSize, pageSize, filter)
+	if err != nil {
+		return nil, 0, err
+	}
+	if s.cache != nil {
+		s.cache.SetJSON(ctx, cacheKey, mediaListCacheValue{Items: items, Total: total}, time.Duration(s.mediaCacheTTLSeconds())*time.Second)
+	}
+	return items, total, nil
+}
+
+func (s *MediaService) ListMediaVisibleGrouped(ctx context.Context, libraryID string, page, pageSize int, visibility MediaVisibility) ([]MediaItem, int64, error) {
+	items, _, err := s.ListMediaVisible(ctx, libraryID, page, pageSize, visibility)
+	if err != nil {
+		return nil, 0, err
+	}
+	grouped := groupMediaVersions(items)
+	return grouped, int64(len(grouped)), nil
+}
+
+type mediaListCacheValue struct {
+	Items []model.Media `json:"items"`
+	Total int64         `json:"total"`
+}
+
+func (s *MediaService) mediaListCacheKey(libraryID string, libraryIDs []string, page, pageSize int, filter repository.MediaQueryFilter) string {
+	allowed := append([]string(nil), filter.AllowedLibraryIDs...)
+	hidden := append([]string(nil), filter.HiddenLibraryIDs...)
+	libs := append([]string(nil), libraryIDs...)
+	sort.Strings(allowed)
+	sort.Strings(hidden)
+	sort.Strings(libs)
+	sum := sha1.Sum([]byte(strings.Join([]string{
+		libraryID,
+		strings.Join(libs, ","),
+		fmt.Sprintf("%d:%d:%t", page, pageSize, filter.IncludeNSFW),
+		strings.Join(allowed, ","),
+		strings.Join(hidden, ","),
+	}, "|")))
+	return "media:list:" + hex.EncodeToString(sum[:])
+}
+
+func (s *MediaService) mediaCacheTTLSeconds() int {
+	if s == nil || s.cfg == nil || s.cfg.Cache.MediaTTLSeconds < 1 {
+		return 15
+	}
+	return s.cfg.Cache.MediaTTLSeconds
+}
+
+func (s *MediaService) invalidateMediaCache(ctx context.Context) {
+	if s != nil && s.cache != nil {
+		s.cache.DeletePrefix(ctx, "media:")
+		s.cache.DeletePrefix(ctx, "stats:")
+	}
+}
+
+func groupMediaVersions(items []model.Media) []MediaItem {
+	if len(items) == 0 {
+		return nil
+	}
+	type group struct {
+		key     string
+		primary model.Media
+		rows    []model.Media
+	}
+	groups := make([]group, 0, len(items))
+	byKey := make(map[string]int, len(items))
+	for _, item := range items {
+		key := mediaVersionGroupKey(item)
+		if key == "" {
+			groups = append(groups, group{primary: item, rows: []model.Media{item}})
+			continue
+		}
+		if idx, ok := byKey[key]; ok {
+			groups[idx].rows = append(groups[idx].rows, item)
+			if betterMediaVersion(item, groups[idx].primary) {
+				groups[idx].primary = item
+			}
+			continue
+		}
+		byKey[key] = len(groups)
+		groups = append(groups, group{key: key, primary: item, rows: []model.Media{item}})
+	}
+	out := make([]MediaItem, 0, len(groups))
+	for _, g := range groups {
+		sort.SliceStable(g.rows, func(i, j int) bool {
+			return betterMediaVersion(g.rows[i], g.rows[j])
+		})
+		item := MediaItem{Media: g.primary}
+		if len(g.rows) > 1 {
+			item.Versions = g.rows
+		}
+		out = append(out, item)
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		return out[i].CreatedAt.After(out[j].CreatedAt)
 	})
+	return out
+}
+
+func mediaVersionGroupKey(m model.Media) string {
+	if m.SeasonNum > 0 || m.EpisodeNum > 0 {
+		title := firstNonEmpty(m.OriginalName, m.Title)
+		if title == "" {
+			title, _ = CleanQuery(m.Path)
+		}
+		title = normalizeMediaVersionText(title)
+		if title == "" {
+			return ""
+		}
+		return strings.Join([]string{
+			"episode",
+			strings.ToLower(strings.TrimSpace(m.LibraryID)),
+			title,
+			fmt.Sprintf("%d:%d", m.SeasonNum, m.EpisodeNum),
+		}, "|")
+	}
+	switch {
+	case m.TMDbID > 0:
+		return fmt.Sprintf("tmdb:%d", m.TMDbID)
+	case m.BangumiID > 0:
+		return fmt.Sprintf("bangumi:%d", m.BangumiID)
+	case strings.TrimSpace(m.DoubanID) != "":
+		return "douban:" + strings.ToLower(strings.TrimSpace(m.DoubanID))
+	case strings.TrimSpace(m.TheTVDBID) != "":
+		return "thetvdb:" + strings.ToLower(strings.TrimSpace(m.TheTVDBID))
+	}
+	title := firstNonEmpty(m.OriginalName, m.Title)
+	if title == "" {
+		title, _ = CleanQuery(m.Path)
+	}
+	title = normalizeMediaVersionText(title)
+	if title == "" {
+		return ""
+	}
+	year := m.Year
+	if year <= 0 {
+		_, year = CleanQuery(m.Path)
+	}
+	return fmt.Sprintf("movie:%s:%d", title, year)
+}
+
+func normalizeMediaVersionText(value string) string {
+	value = strings.ToLower(strings.TrimSpace(value))
+	if value == "" {
+		return ""
+	}
+	fields := strings.FieldsFunc(value, func(r rune) bool {
+		switch r {
+		case '.', '_', '-', ' ', '\t', '/', '\\', '[', ']', '(', ')', '（', '）', '【', '】':
+			return true
+		default:
+			return false
+		}
+	})
+	out := fields[:0]
+	for _, field := range fields {
+		field = strings.TrimSpace(field)
+		if field == "" {
+			continue
+		}
+		if _, noise := noiseTokenSet[field]; noise {
+			continue
+		}
+		out = append(out, field)
+	}
+	return strings.Join(out, " ")
+}
+
+func betterMediaVersion(candidate, current model.Media) bool {
+	candidatePixels := candidate.Width * candidate.Height
+	currentPixels := current.Width * current.Height
+	if candidatePixels != currentPixels {
+		return candidatePixels > currentPixels
+	}
+	if candidate.SizeBytes != current.SizeBytes {
+		return candidate.SizeBytes > current.SizeBytes
+	}
+	return candidate.CreatedAt.After(current.CreatedAt)
 }

 // SearchMedia performs a simple LIKE search across titles.
@@ -361,6 +570,14 @@ func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, lim
 	})
 }

+func (s *MediaService) SearchMediaVisibleGrouped(ctx context.Context, query string, limit int, visibility MediaVisibility) ([]MediaItem, error) {
+	items, err := s.SearchMediaVisible(ctx, query, limit, visibility)
+	if err != nil {
+		return nil, err
+	}
+	return groupMediaVersions(items), nil
+}
+
 func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility) ([]model.Media, int64, error) {
 	if pageSize <= 0 {
 		pageSize = 50
@@ -379,6 +596,15 @@ func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string,
 	})
 }

+func (s *MediaService) SearchMediaVisiblePageGrouped(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility) ([]MediaItem, int64, error) {
+	items, _, err := s.SearchMediaVisiblePage(ctx, query, page, pageSize, visibility)
+	if err != nil {
+		return nil, 0, err
+	}
+	grouped := groupMediaVersions(items)
+	return grouped, int64(len(grouped)), nil
+}
+
 // GetMedia returns a single media row.
 func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) {
 	return s.repo.Media.FindByID(ctx, id)
@@ -392,15 +618,27 @@ func (s *MediaService) SoftDelete(ctx context.Context, id string) error {
 		return err
 	}
 	if media != nil && isCloudMediaPath(media.Path) {
-		return s.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.Media{}).Error
+		err := s.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.Media{}).Error
+		if err == nil {
+			s.invalidateMediaCache(ctx)
+		}
+		return err
 	}
-	return s.repo.DB.Where("id = ?", id).Delete(&model.Media{}).Error
+	err = s.repo.DB.WithContext(ctx).Where("id = ?", id).Delete(&model.Media{}).Error
+	if err == nil {
+		s.invalidateMediaCache(ctx)
+	}
+	return err
 }

 // RestoreDeleted unsets DeletedAt for a single media row.
 func (s *MediaService) RestoreDeleted(ctx context.Context, id string) error {
-	return s.repo.DB.Unscoped().Model(&model.Media{}).
+	err := s.repo.DB.WithContext(ctx).Unscoped().Model(&model.Media{}).
 		Where("id = ?", id).Update("deleted_at", nil).Error
+	if err == nil {
+		s.invalidateMediaCache(ctx)
+	}
+	return err
 }

 // ListRecycleBin returns every soft-deleted row, newest first.
@@ -419,5 +657,9 @@ func (s *MediaService) ListRecycleBin(ctx context.Context, limit int) ([]model.M

 // PurgeDeleted permanently removes a soft-deleted row from the database.
 func (s *MediaService) PurgeDeleted(ctx context.Context, id string) error {
-	return s.repo.DB.Unscoped().Where("id = ?", id).Delete(&model.Media{}).Error
+	err := s.repo.DB.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.Media{}).Error
+	if err == nil {
+		s.invalidateMediaCache(ctx)
+	}
+	return err
 }