fix media event access, cache isolation and indexing

This commit is contained in:
jason jia
2026-10-20 12:39:13 +00:00
parent 0b68397265
commit 3cf898756b
18 changed files with 926 additions and 141 deletions
+8 -7
View File
@@ -12,13 +12,14 @@ import (
// MediaService offers high-level CRUD over libraries and media items.
type MediaService struct {
cfg *config.Config
log *zap.Logger
repo *repository.Container
cache *RuntimeCacheService
groupedMediaFlight singleflight.Group
libraryRowsFlight singleflight.Group
recentSeriesFlight singleflight.Group
cfg *config.Config
log *zap.Logger
repo *repository.Container
cache *RuntimeCacheService
groupedMediaFlight singleflight.Group
mediaSortFlight singleflight.Group
libraryRowsFlight singleflight.Group
recentSeriesFlight singleflight.Group
}
type MediaVisibility struct {
+31 -16
View File
@@ -5,7 +5,7 @@ import (
"strings"
"time"
"go.uber.org/zap"
"github.com/google/uuid"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
@@ -129,6 +129,26 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
visibility MediaVisibility,
filters MediaListFilters,
) ([]MediaItem, error) {
value, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
if err != nil {
return nil, err
}
return value.Items, nil
}
// Each immutable grouping has its own revision. Lightweight sort indices can
// reference that revision without retaining another copy of the full rows.
type groupedMediaCacheValue struct {
Items []MediaItem
Revision string
}
func (s *MediaService) groupedMediaVisibleFiltered(
ctx context.Context,
libraryID string,
visibility MediaVisibility,
filters MediaListFilters,
) (*groupedMediaCacheValue, error) {
visibility = ExpandMediaVisibilityForMergedCloudLibraries(ctx, s.repo, visibility)
libraryIDs, err := MergedLibraryIDsForLibrary(ctx, s.repo, libraryID)
if err != nil {
@@ -143,7 +163,7 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
value, err, _ := s.groupedMediaFlight.Do(itemsCacheKey, func() (any, error) {
if s.cache != nil {
if cachedObj, ok := s.cache.GetObject(itemsCacheKey); ok {
if cached, ok := cachedObj.([]MediaItem); ok {
if cached, ok := cachedObj.(*groupedMediaCacheValue); ok {
return cached, nil
}
}
@@ -154,8 +174,8 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
if err != nil {
return nil, err
}
grouped := groupMediaVersions(items)
if s.cache != nil && len(grouped) > 0 {
grouped := &groupedMediaCacheValue{Items: groupMediaVersions(items), Revision: uuid.NewString()}
if s.cache != nil && len(grouped.Items) > 0 {
s.cache.SetObject(itemsCacheKey, grouped, s.mediaObjectTTL())
}
return grouped, nil
@@ -163,10 +183,10 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
if err != nil {
return nil, err
}
if grouped, ok := value.([]MediaItem); ok {
if grouped, ok := value.(*groupedMediaCacheValue); ok {
return grouped, nil
}
return nil, nil
return &groupedMediaCacheValue{}, nil
}
func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryID string, visibility MediaVisibility, filters MediaListFilters) ([]model.Media, error) {
@@ -180,24 +200,19 @@ func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryI
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
})
// 版本分组的 URL 分页发生在 Go 进程内,响应里的 total 是分组后的数量,
// 不需要数据库再为原始行做一次 COUNT(*)。全量 COUNT 在超大媒体库上
// 会重复扫描整个 library_id 范围,而这里只关心是否存在截断风险。
items, err := s.repo.Media.ListByLibrariesFilteredNoCount(ctx, libraryIDs, 0, maxMediaSearchLimit, filter)
// Group before paginating so versions stay together. The search safety limit
// must not truncate a library listing or its reported grouped total.
items, err := s.repo.Media.ListAllByLibrariesFilteredNoCount(ctx, libraryIDs, filter)
if err != nil {
return nil, err
}
if len(items) >= maxMediaSearchLimit && s.log != nil {
s.log.Warn("media version grouping may be truncated by safety limit",
zap.String("library_id", libraryID),
zap.Int("limit", maxMediaSearchLimit))
}
s.attachLibraryMetadata(ctx, items)
return items, nil
}
// GetMedia returns a single media row.
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) { media, err := s.repo.Media.FindByID(ctx, id)
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) {
media, err := s.repo.Media.FindByID(ctx, id)
if err != nil || media == nil {
return media, err
}
+77
View File
@@ -0,0 +1,77 @@
package service
import (
"context"
"fmt"
"sort"
"time"
)
// ListMediaVisibleSortedPage reuses grouped rows and a small position index.
// Only the requested page is copied; pagination does not sort the full library
// again or retain a second full-size MediaItem slice for every sort order.
func (s *MediaService) ListMediaVisibleSortedPage(
ctx context.Context,
libraryID string,
page, pageSize int,
visibility MediaVisibility,
filters MediaListFilters,
spec MediaSortSpec,
history map[string]time.Time,
) ([]MediaItem, int64, error) {
grouped, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
if err != nil {
return nil, 0, err
}
spec = NormalizeMediaSort(spec.Field, spec.Order)
indices := s.sortedMediaIndices(grouped, spec, history)
page, pageSize = normalizeGroupedMediaPage(page, pageSize)
total := len(grouped.Items)
// Division avoids overflowing (page-1)*pageSize on an excessive page value.
if total == 0 || page-1 > (total-1)/pageSize {
return []MediaItem{}, int64(total), nil
}
start := (page - 1) * pageSize
end := min(start+pageSize, total)
items := make([]MediaItem, 0, end-start)
for _, index := range indices[start:end] {
items = append(items, grouped.Items[index])
}
return items, int64(total), nil
}
func (s *MediaService) sortedMediaIndices(grouped *groupedMediaCacheValue, spec MediaSortSpec, history map[string]time.Time) []int {
// Playback history changes independently of the media cache and is personal.
// Keep this explicit sort live rather than caching another user's history.
if s.cache == nil || spec.Field == "last_played" || len(grouped.Items) == 0 {
return sortMediaIndices(grouped.Items, spec, history)
}
key := fmt.Sprintf("media:obj:sort:%s:%s:%s", grouped.Revision, spec.Field, spec.Order)
value, _, _ := s.mediaSortFlight.Do(key, func() (any, error) {
if cached, ok := s.cache.GetObject(key); ok {
if indices, ok := cached.([]int); ok {
return indices, nil
}
}
indices := sortMediaIndices(grouped.Items, spec, history)
s.cache.SetObject(key, indices, s.mediaObjectTTL())
return indices, nil
})
return value.([]int)
}
func sortMediaIndices(items []MediaItem, spec MediaSortSpec, history map[string]time.Time) []int {
indices := make([]int, len(items))
for i := range indices {
indices[i] = i
}
sort.SliceStable(indices, func(i, j int) bool {
a, b := items[indices[i]], items[indices[j]]
cmp := compareMediaByField(a, b, spec, history)
if cmp == 0 {
cmp = compareID(a.ID, b.ID)
}
return cmp < 0
})
return indices
}
+141
View File
@@ -0,0 +1,141 @@
package service
import (
"fmt"
"math"
"testing"
"time"
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
)
func TestSortedMediaPageIncludesRowsBeyondSearchLimit(t *testing.T) {
db := newServiceTestDB(t)
repos := repository.New(db)
lib := model.Library{Name: "Large", Path: "/media/large", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
// This must cross the historical 50,000-row boundary: a smaller fixture
// cannot detect truncation before grouping, counting, and sorting.
now := time.Now()
for start := 0; start <= maxMediaSearchLimit; start += 500 {
var rows []model.Media
for i := start; i < min(start+500, maxMediaSearchLimit+1); i++ {
rows = append(rows, model.Media{
Base: model.Base{ID: fmt.Sprintf("large-%05d", i), CreatedAt: now, UpdatedAt: now},
LibraryID: lib.ID, Path: fmt.Sprintf("/media/large/%05d.mkv", i),
Title: fmt.Sprintf("Movie %05d", i), TMDbID: i + 1,
})
}
if err := db.CreateInBatches(&rows, 100).Error; err != nil {
t.Fatal(err)
}
}
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos)
items, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, maxMediaSearchLimit/100+1, 100,
MediaVisibility{IncludeNSFW: true}, MediaListFilters{}, MediaSortSpec{Field: "title", Order: "asc"}, nil)
if err != nil {
t.Fatal(err)
}
if total != int64(maxMediaSearchLimit+1) || len(items) != 1 || items[0].ID != "large-50000" {
t.Fatalf("total=%d, page length=%d, want 50001 rows and final item large-50000", total, len(items))
}
}
func TestSortedMediaPagesReuseIndicesAndRefreshAfterInvalidation(t *testing.T) {
db := newServiceTestDB(t)
repos := repository.New(db)
lib := model.Library{Name: "Movies", Path: "/media/movies", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
rows := []model.Media{
{Base: model.Base{ID: "alpha"}, LibraryID: lib.ID, Path: "/media/movies/a.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 100},
{Base: model.Base{ID: "alpha-alt"}, LibraryID: lib.ID, Path: "/media/movies/a-alt.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 50},
{Base: model.Base{ID: "beta"}, LibraryID: lib.ID, Path: "/media/movies/b.mkv", Title: "Beta", TMDbID: 2},
{Base: model.Base{ID: "hidden"}, LibraryID: lib.ID, Path: "/media/movies/hidden.mkv", Title: "Hidden", TMDbID: 3, NSFW: true},
}
if err := db.Create(&rows).Error; err != nil {
t.Fatal(err)
}
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos).
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
visibility := MediaVisibility{IncludeNSFW: false}
spec := MediaSortSpec{Field: "title", Order: "asc"}
first, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
if err != nil || total != 2 || len(first) != 1 || first[0].ID != "alpha" || len(first[0].Versions) != 2 {
t.Fatalf("first=%#v, total=%d, error=%v", first, total, err)
}
grouped, err := svc.groupedMediaVisibleFiltered(t.Context(), lib.ID, visibility, MediaListFilters{})
if err != nil {
t.Fatal(err)
}
index := svc.sortedMediaIndices(grouped, spec, nil)
second, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 2, 1, visibility, MediaListFilters{}, spec, nil)
if err != nil || len(second) != 1 || second[0].ID != "beta" {
t.Fatalf("second=%#v, error=%v", second, err)
}
cachedIndex := svc.sortedMediaIndices(grouped, spec, nil)
if &index[0] != &cachedIndex[0] {
t.Fatal("pagination rebuilt the sorted position index")
}
// The returned page is independently writable without corrupting cached rows.
first[0].Title = "Modified by caller"
if err := db.Model(&model.Media{}).Where("id = ?", "beta").Update("title", "Aardvark").Error; err != nil {
t.Fatal(err)
}
svc.invalidateMediaCache(t.Context())
fresh, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
if err != nil || len(fresh) != 1 || fresh[0].ID != "beta" {
t.Fatalf("updated page=%#v, error=%v", fresh, err)
}
empty, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, math.MaxInt, 100, visibility, MediaListFilters{}, spec, nil)
if err != nil || empty == nil || len(empty) != 0 {
t.Fatalf("out-of-range page=%#v, error=%v", empty, err)
}
}
func TestMediaSortIndexDoesNotCachePersonalPlaybackHistory(t *testing.T) {
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
grouped := &groupedMediaCacheValue{Revision: "history-test", Items: []MediaItem{
{Media: model.Media{Base: model.Base{ID: "a"}}},
{Media: model.Media{Base: model.Base{ID: "b"}}},
}}
spec := MediaSortSpec{Field: "last_played", Order: "desc"}
first := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"a": time.Now()})
second := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"b": time.Now()})
if first[0] != 0 || second[0] != 1 {
t.Fatalf("history sorts = %v, %v", first, second)
}
}
func BenchmarkMediaSortIndex(b *testing.B) {
items := make([]MediaItem, 10000)
for i := range items {
items[i].ID = fmt.Sprint(i)
items[i].Title = fmt.Sprintf("Movie %05d", len(items)-i)
}
spec := MediaSortSpec{Field: "title", Order: "asc"}
b.Run("sort-every-page", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
SortMediaItems(items, spec.Field, spec.Order, nil)
}
})
b.Run("cached-position-index", func(b *testing.B) {
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
grouped := &groupedMediaCacheValue{Items: items, Revision: "benchmark"}
svc.sortedMediaIndices(grouped, spec, nil)
b.ReportAllocs()
for b.Loop() {
svc.sortedMediaIndices(grouped, spec, nil)
}
})
}