mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-16 12:54:22 +08:00
fix media event access, cache isolation and indexing
This commit is contained in:
@@ -12,13 +12,14 @@ import (
|
||||
|
||||
// MediaService offers high-level CRUD over libraries and media items.
|
||||
type MediaService struct {
|
||||
cfg *config.Config
|
||||
log *zap.Logger
|
||||
repo *repository.Container
|
||||
cache *RuntimeCacheService
|
||||
groupedMediaFlight singleflight.Group
|
||||
libraryRowsFlight singleflight.Group
|
||||
recentSeriesFlight singleflight.Group
|
||||
cfg *config.Config
|
||||
log *zap.Logger
|
||||
repo *repository.Container
|
||||
cache *RuntimeCacheService
|
||||
groupedMediaFlight singleflight.Group
|
||||
mediaSortFlight singleflight.Group
|
||||
libraryRowsFlight singleflight.Group
|
||||
recentSeriesFlight singleflight.Group
|
||||
}
|
||||
|
||||
type MediaVisibility struct {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
@@ -129,6 +129,26 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
visibility MediaVisibility,
|
||||
filters MediaListFilters,
|
||||
) ([]MediaItem, error) {
|
||||
value, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return value.Items, nil
|
||||
}
|
||||
|
||||
// Each immutable grouping has its own revision. Lightweight sort indices can
|
||||
// reference that revision without retaining another copy of the full rows.
|
||||
type groupedMediaCacheValue struct {
|
||||
Items []MediaItem
|
||||
Revision string
|
||||
}
|
||||
|
||||
func (s *MediaService) groupedMediaVisibleFiltered(
|
||||
ctx context.Context,
|
||||
libraryID string,
|
||||
visibility MediaVisibility,
|
||||
filters MediaListFilters,
|
||||
) (*groupedMediaCacheValue, error) {
|
||||
visibility = ExpandMediaVisibilityForMergedCloudLibraries(ctx, s.repo, visibility)
|
||||
libraryIDs, err := MergedLibraryIDsForLibrary(ctx, s.repo, libraryID)
|
||||
if err != nil {
|
||||
@@ -143,7 +163,7 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
value, err, _ := s.groupedMediaFlight.Do(itemsCacheKey, func() (any, error) {
|
||||
if s.cache != nil {
|
||||
if cachedObj, ok := s.cache.GetObject(itemsCacheKey); ok {
|
||||
if cached, ok := cachedObj.([]MediaItem); ok {
|
||||
if cached, ok := cachedObj.(*groupedMediaCacheValue); ok {
|
||||
return cached, nil
|
||||
}
|
||||
}
|
||||
@@ -154,8 +174,8 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grouped := groupMediaVersions(items)
|
||||
if s.cache != nil && len(grouped) > 0 {
|
||||
grouped := &groupedMediaCacheValue{Items: groupMediaVersions(items), Revision: uuid.NewString()}
|
||||
if s.cache != nil && len(grouped.Items) > 0 {
|
||||
s.cache.SetObject(itemsCacheKey, grouped, s.mediaObjectTTL())
|
||||
}
|
||||
return grouped, nil
|
||||
@@ -163,10 +183,10 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if grouped, ok := value.([]MediaItem); ok {
|
||||
if grouped, ok := value.(*groupedMediaCacheValue); ok {
|
||||
return grouped, nil
|
||||
}
|
||||
return nil, nil
|
||||
return &groupedMediaCacheValue{}, nil
|
||||
}
|
||||
|
||||
func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryID string, visibility MediaVisibility, filters MediaListFilters) ([]model.Media, error) {
|
||||
@@ -180,24 +200,19 @@ func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryI
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
})
|
||||
// 版本分组的 URL 分页发生在 Go 进程内,响应里的 total 是分组后的数量,
|
||||
// 不需要数据库再为原始行做一次 COUNT(*)。全量 COUNT 在超大媒体库上
|
||||
// 会重复扫描整个 library_id 范围,而这里只关心是否存在截断风险。
|
||||
items, err := s.repo.Media.ListByLibrariesFilteredNoCount(ctx, libraryIDs, 0, maxMediaSearchLimit, filter)
|
||||
// Group before paginating so versions stay together. The search safety limit
|
||||
// must not truncate a library listing or its reported grouped total.
|
||||
items, err := s.repo.Media.ListAllByLibrariesFilteredNoCount(ctx, libraryIDs, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) >= maxMediaSearchLimit && s.log != nil {
|
||||
s.log.Warn("media version grouping may be truncated by safety limit",
|
||||
zap.String("library_id", libraryID),
|
||||
zap.Int("limit", maxMediaSearchLimit))
|
||||
}
|
||||
s.attachLibraryMetadata(ctx, items)
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// GetMedia returns a single media row.
|
||||
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) { media, err := s.repo.Media.FindByID(ctx, id)
|
||||
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) {
|
||||
media, err := s.repo.Media.FindByID(ctx, id)
|
||||
if err != nil || media == nil {
|
||||
return media, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ListMediaVisibleSortedPage reuses grouped rows and a small position index.
|
||||
// Only the requested page is copied; pagination does not sort the full library
|
||||
// again or retain a second full-size MediaItem slice for every sort order.
|
||||
func (s *MediaService) ListMediaVisibleSortedPage(
|
||||
ctx context.Context,
|
||||
libraryID string,
|
||||
page, pageSize int,
|
||||
visibility MediaVisibility,
|
||||
filters MediaListFilters,
|
||||
spec MediaSortSpec,
|
||||
history map[string]time.Time,
|
||||
) ([]MediaItem, int64, error) {
|
||||
grouped, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
spec = NormalizeMediaSort(spec.Field, spec.Order)
|
||||
indices := s.sortedMediaIndices(grouped, spec, history)
|
||||
page, pageSize = normalizeGroupedMediaPage(page, pageSize)
|
||||
total := len(grouped.Items)
|
||||
// Division avoids overflowing (page-1)*pageSize on an excessive page value.
|
||||
if total == 0 || page-1 > (total-1)/pageSize {
|
||||
return []MediaItem{}, int64(total), nil
|
||||
}
|
||||
start := (page - 1) * pageSize
|
||||
end := min(start+pageSize, total)
|
||||
items := make([]MediaItem, 0, end-start)
|
||||
for _, index := range indices[start:end] {
|
||||
items = append(items, grouped.Items[index])
|
||||
}
|
||||
return items, int64(total), nil
|
||||
}
|
||||
|
||||
func (s *MediaService) sortedMediaIndices(grouped *groupedMediaCacheValue, spec MediaSortSpec, history map[string]time.Time) []int {
|
||||
// Playback history changes independently of the media cache and is personal.
|
||||
// Keep this explicit sort live rather than caching another user's history.
|
||||
if s.cache == nil || spec.Field == "last_played" || len(grouped.Items) == 0 {
|
||||
return sortMediaIndices(grouped.Items, spec, history)
|
||||
}
|
||||
key := fmt.Sprintf("media:obj:sort:%s:%s:%s", grouped.Revision, spec.Field, spec.Order)
|
||||
value, _, _ := s.mediaSortFlight.Do(key, func() (any, error) {
|
||||
if cached, ok := s.cache.GetObject(key); ok {
|
||||
if indices, ok := cached.([]int); ok {
|
||||
return indices, nil
|
||||
}
|
||||
}
|
||||
indices := sortMediaIndices(grouped.Items, spec, history)
|
||||
s.cache.SetObject(key, indices, s.mediaObjectTTL())
|
||||
return indices, nil
|
||||
})
|
||||
return value.([]int)
|
||||
}
|
||||
|
||||
func sortMediaIndices(items []MediaItem, spec MediaSortSpec, history map[string]time.Time) []int {
|
||||
indices := make([]int, len(items))
|
||||
for i := range indices {
|
||||
indices[i] = i
|
||||
}
|
||||
sort.SliceStable(indices, func(i, j int) bool {
|
||||
a, b := items[indices[i]], items[indices[j]]
|
||||
cmp := compareMediaByField(a, b, spec, history)
|
||||
if cmp == 0 {
|
||||
cmp = compareID(a.ID, b.ID)
|
||||
}
|
||||
return cmp < 0
|
||||
})
|
||||
return indices
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
func TestSortedMediaPageIncludesRowsBeyondSearchLimit(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "Large", Path: "/media/large", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// This must cross the historical 50,000-row boundary: a smaller fixture
|
||||
// cannot detect truncation before grouping, counting, and sorting.
|
||||
now := time.Now()
|
||||
for start := 0; start <= maxMediaSearchLimit; start += 500 {
|
||||
var rows []model.Media
|
||||
for i := start; i < min(start+500, maxMediaSearchLimit+1); i++ {
|
||||
rows = append(rows, model.Media{
|
||||
Base: model.Base{ID: fmt.Sprintf("large-%05d", i), CreatedAt: now, UpdatedAt: now},
|
||||
LibraryID: lib.ID, Path: fmt.Sprintf("/media/large/%05d.mkv", i),
|
||||
Title: fmt.Sprintf("Movie %05d", i), TMDbID: i + 1,
|
||||
})
|
||||
}
|
||||
if err := db.CreateInBatches(&rows, 100).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos)
|
||||
items, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, maxMediaSearchLimit/100+1, 100,
|
||||
MediaVisibility{IncludeNSFW: true}, MediaListFilters{}, MediaSortSpec{Field: "title", Order: "asc"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != int64(maxMediaSearchLimit+1) || len(items) != 1 || items[0].ID != "large-50000" {
|
||||
t.Fatalf("total=%d, page length=%d, want 50001 rows and final item large-50000", total, len(items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortedMediaPagesReuseIndicesAndRefreshAfterInvalidation(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "Movies", Path: "/media/movies", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := []model.Media{
|
||||
{Base: model.Base{ID: "alpha"}, LibraryID: lib.ID, Path: "/media/movies/a.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 100},
|
||||
{Base: model.Base{ID: "alpha-alt"}, LibraryID: lib.ID, Path: "/media/movies/a-alt.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 50},
|
||||
{Base: model.Base{ID: "beta"}, LibraryID: lib.ID, Path: "/media/movies/b.mkv", Title: "Beta", TMDbID: 2},
|
||||
{Base: model.Base{ID: "hidden"}, LibraryID: lib.ID, Path: "/media/movies/hidden.mkv", Title: "Hidden", TMDbID: 3, NSFW: true},
|
||||
}
|
||||
if err := db.Create(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos).
|
||||
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
|
||||
visibility := MediaVisibility{IncludeNSFW: false}
|
||||
spec := MediaSortSpec{Field: "title", Order: "asc"}
|
||||
first, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || total != 2 || len(first) != 1 || first[0].ID != "alpha" || len(first[0].Versions) != 2 {
|
||||
t.Fatalf("first=%#v, total=%d, error=%v", first, total, err)
|
||||
}
|
||||
grouped, err := svc.groupedMediaVisibleFiltered(t.Context(), lib.ID, visibility, MediaListFilters{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
index := svc.sortedMediaIndices(grouped, spec, nil)
|
||||
second, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 2, 1, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || len(second) != 1 || second[0].ID != "beta" {
|
||||
t.Fatalf("second=%#v, error=%v", second, err)
|
||||
}
|
||||
cachedIndex := svc.sortedMediaIndices(grouped, spec, nil)
|
||||
if &index[0] != &cachedIndex[0] {
|
||||
t.Fatal("pagination rebuilt the sorted position index")
|
||||
}
|
||||
// The returned page is independently writable without corrupting cached rows.
|
||||
first[0].Title = "Modified by caller"
|
||||
if err := db.Model(&model.Media{}).Where("id = ?", "beta").Update("title", "Aardvark").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc.invalidateMediaCache(t.Context())
|
||||
fresh, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || len(fresh) != 1 || fresh[0].ID != "beta" {
|
||||
t.Fatalf("updated page=%#v, error=%v", fresh, err)
|
||||
}
|
||||
empty, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, math.MaxInt, 100, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || empty == nil || len(empty) != 0 {
|
||||
t.Fatalf("out-of-range page=%#v, error=%v", empty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaSortIndexDoesNotCachePersonalPlaybackHistory(t *testing.T) {
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
|
||||
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
|
||||
grouped := &groupedMediaCacheValue{Revision: "history-test", Items: []MediaItem{
|
||||
{Media: model.Media{Base: model.Base{ID: "a"}}},
|
||||
{Media: model.Media{Base: model.Base{ID: "b"}}},
|
||||
}}
|
||||
spec := MediaSortSpec{Field: "last_played", Order: "desc"}
|
||||
first := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"a": time.Now()})
|
||||
second := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"b": time.Now()})
|
||||
if first[0] != 0 || second[0] != 1 {
|
||||
t.Fatalf("history sorts = %v, %v", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMediaSortIndex(b *testing.B) {
|
||||
items := make([]MediaItem, 10000)
|
||||
for i := range items {
|
||||
items[i].ID = fmt.Sprint(i)
|
||||
items[i].Title = fmt.Sprintf("Movie %05d", len(items)-i)
|
||||
}
|
||||
spec := MediaSortSpec{Field: "title", Order: "asc"}
|
||||
b.Run("sort-every-page", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
SortMediaItems(items, spec.Field, spec.Order, nil)
|
||||
}
|
||||
})
|
||||
b.Run("cached-position-index", func(b *testing.B) {
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
|
||||
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
|
||||
grouped := &groupedMediaCacheValue{Items: items, Revision: "benchmark"}
|
||||
svc.sortedMediaIndices(grouped, spec, nil)
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
svc.sortedMediaIndices(grouped, spec, nil)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user