优化查询

This commit is contained in:
truewhile
2026-10-20 23:29:51 +08:00
parent 5615806c2c
commit 5ad71b9421
5 changed files with 265 additions and 11 deletions
+100 -6
View File
@@ -4,6 +4,7 @@ package handler
import (
"context"
"errors"
"fmt"
"math"
"net/http"
"sort"
@@ -705,6 +706,8 @@ func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
}
paginated := c.Query("page") != "" || c.Query("page_size") != ""
sortBy := normalizeMediaSearchSort(c.Query("sort"))
// 远程搜索与本地查询并行发起:本地结果不再排在远程超时之后才返回。
remoteCh := startRemoteMediaSearch(ctx, svc, query, visibility, filter)
// Sorting and grouping must operate on the same candidate set before
// either a page size or the caller's output limit is applied.
items, err := svc.Media.SearchMediaVisibleFiltered(ctx, query, 50000, visibility, filter)
@@ -712,12 +715,9 @@ func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if svc.EmbyRemote != nil && query != "" && (filter.LibraryID == "" || service.IsEmbyRemoteID(filter.LibraryID)) {
remote, _ := svc.EmbyRemote.RemoteSearchMedia(ctx, query, 50000, visibility)
for _, item := range remote {
if remoteSearchMatchesFilter(item, filter) {
items = append(items, item)
}
if remoteCh != nil {
for _, item := range <-remoteCh {
items = append(items, item)
}
}
if c.DefaultQuery("group_series", "0") != "0" {
@@ -741,6 +741,100 @@ func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
}
}
const (
// remoteSearchItemLimit 是每个远程 Emby 挂载点返回的条目上限。此前这里传
// 50000(等于让上游做一次全库查询),上游稍慢就会一路跑到 5 秒超时,把每次
// 搜索都拖成秒级;搜索场景只需要少量候选,多出来的结果也会被分页丢掉。
remoteSearchItemLimit = 100
// remoteSearchBudget 限制远程结果最多给一次请求增加多少延迟。
remoteSearchBudget = 2 * time.Second
// remoteSearchCacheTTL 让同一关键词的翻页与重复搜索复用远程结果,避免每翻
// 一页都重新打一遍远程挂载点。
remoteSearchCacheTTL = 30 * time.Second
// remoteSearchFailureCacheTTL 用于上游超时/失败时的短缓存:部分结果只保留
// 几秒,避免反复吃满 remoteSearchBudget,又不至于长期压住后续重试。
remoteSearchFailureCacheTTL = 5 * time.Second
)
// startRemoteMediaSearch 并发发起远程搜索,返回一个只投递一次结果的通道。
// 不满足远程搜索条件时返回 nil。通道带缓冲,调用方即使提前返回也不会泄漏
// goroutine;请求结束时请求上下文被取消,远程调用会随之退出。
func startRemoteMediaSearch(
ctx context.Context,
svc *service.Container,
query string,
visibility service.MediaVisibility,
filter service.MediaSearchFilter,
) <-chan []model.Media {
if svc == nil || svc.EmbyRemote == nil || query == "" {
return nil
}
if filter.LibraryID != "" && !service.IsEmbyRemoteID(filter.LibraryID) {
return nil
}
results := make(chan []model.Media, 1)
go func() {
results <- remoteMediaSearchItems(ctx, svc, query, visibility, filter)
}()
return results
}
func remoteMediaSearchItems(
ctx context.Context,
svc *service.Container,
query string,
visibility service.MediaVisibility,
filter service.MediaSearchFilter,
) []model.Media {
cacheKey := remoteSearchCacheKey(query, visibility)
if svc.Cache != nil {
if cached, ok := svc.Cache.GetObject(cacheKey); ok {
if rows, ok := cached.([]model.Media); ok {
return filterRemoteSearchItems(rows, filter)
}
}
}
searchCtx, cancel := context.WithTimeout(ctx, remoteSearchBudget)
defer cancel()
remote, _ := svc.EmbyRemote.RemoteSearchMedia(searchCtx, query, remoteSearchItemLimit, visibility)
if svc.Cache != nil {
// 空结果也要缓存:否则查不到片名时每次都会重新等到 remoteSearchBudget。
if remote == nil {
remote = []model.Media{}
}
ttl := remoteSearchCacheTTL
if searchCtx.Err() != nil {
ttl = remoteSearchFailureCacheTTL
}
svc.Cache.SetObject(cacheKey, remote, ttl)
}
return filterRemoteSearchItems(remote, filter)
}
func filterRemoteSearchItems(remote []model.Media, filter service.MediaSearchFilter) []model.Media {
if len(remote) == 0 {
return nil
}
out := make([]model.Media, 0, len(remote))
for _, item := range remote {
if remoteSearchMatchesFilter(item, filter) {
out = append(out, item)
}
}
return out
}
// remoteSearchCacheKey 按可见范围隔离远程搜索结果,避免把某个用户可见挂载
// 点的结果复用给可见范围不同的用户。
func remoteSearchCacheKey(query string, visibility service.MediaVisibility) string {
allowed := append([]string(nil), visibility.AllowedLibraryIDs...)
hidden := append([]string(nil), visibility.HiddenLibraryIDs...)
sort.Strings(allowed)
sort.Strings(hidden)
return fmt.Sprintf("media:obj:remote-search:%t|%s|%s|%s",
visibility.IncludeNSFW, strings.Join(allowed, ","), strings.Join(hidden, ","), query)
}
func writeMediaSearchResults[T any](c *gin.Context, items []T, page, size, limit int, paginated bool) {
if items == nil {
items = []T{}
@@ -0,0 +1,153 @@
package handler
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"github.com/gin-gonic/gin"
"github.com/glebarez/sqlite"
"go.uber.org/zap"
"gorm.io/gorm"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/middleware"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
"github.com/truewhile/MeBox/internal/service"
)
// TestSearchMediaRemoteSearchBoundedAndCached 防止远程搜索再次退化:
// 1. 每个挂载点的 Limit 必须是 remoteSearchItemLimit,而不是历史全库查询的 50000;
// 2. 同一关键词的重复搜索应命中缓存,不再打上游;
// 3. 指定本地媒体库时不应枚举远程挂载点。
func TestSearchMediaRemoteSearchBoundedAndCached(t *testing.T) {
gin.SetMode(gin.TestMode)
var mu sync.Mutex
searchCalls := 0
searchLimits := make([]string, 0, 4)
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if term := r.URL.Query().Get("SearchTerm"); term != "" {
mu.Lock()
searchCalls++
searchLimits = append(searchLimits, r.URL.Query().Get("Limit"))
mu.Unlock()
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"TotalRecordCount": 1,
"Items": []map[string]any{{
"Id": "remote-1",
"Name": "远方剧集",
"Type": "Series",
"ProductionYear": 2024,
}},
})
}))
defer upstream.Close()
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.StrmAccount{}, &model.EmbyMount{}, &model.Setting{}, &model.User{}, &model.PlayProfile{}); err != nil {
t.Fatal(err)
}
repos := repository.New(db)
if err := repos.DB.Create(&model.User{Base: model.Base{ID: "user-1"}, Username: "admin", Role: "admin"}).Error; err != nil {
t.Fatal(err)
}
lib := model.Library{Name: "本地电影", Path: "/media/movies", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
if err := db.Create(&model.Media{
Base: model.Base{ID: "local-1"}, LibraryID: lib.ID, Title: "流浪地球", Path: "/media/movies/流浪地球.mkv",
}).Error; err != nil {
t.Fatal(err)
}
rawConfig, _ := json.Marshal(map[string]string{"url": upstream.URL, "token": "fake-token"})
acct := model.StrmAccount{
Base: model.Base{ID: "acct"}, Name: "远程Emby", Provider: model.StrmProviderEmbyRemote,
Config: string(rawConfig), Enabled: true,
}
if err := repos.StrmAccount.Create(t.Context(), &acct); err != nil {
t.Fatal(err)
}
mount := model.EmbyMount{
Base: model.Base{ID: "mount-1"}, AccountID: acct.ID, RemoteViewID: "view-1",
RemoteViewName: "远程剧集", CollectionType: "tvshows", Enabled: true,
}
if err := repos.EmbyMount.Create(t.Context(), &mount); err != nil {
t.Fatal(err)
}
svc := &service.Container{
Repo: repos,
Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos),
EmbyRemote: service.NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, service.NewCryptoService("", zap.NewNop())),
Cache: service.NewRuntimeCacheService(&config.Config{}, zap.NewNop()),
}
search := func(params string) []service.MediaItem {
t.Helper()
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Set(middleware.CtxUserID, "user-1")
c.Set(middleware.CtxUserRole, "admin")
c.Request = httptest.NewRequest(http.MethodGet, "/api/media?"+params, nil)
searchMediaHandler(svc)(c)
if w.Code != http.StatusOK {
t.Fatalf("search status=%d, body=%s", w.Code, w.Body.String())
}
var response struct {
Items []service.MediaItem `json:"items"`
}
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
return response.Items
}
callCount := func() int {
mu.Lock()
defer mu.Unlock()
return searchCalls
}
if items := search("q=远方&page=1&page_size=48"); len(items) != 1 || items[0].Title != "远方剧集" {
t.Fatalf("expected remote result, got %#v", items)
}
if calls := callCount(); calls != 1 {
t.Fatalf("expected exactly one upstream search, got %d", calls)
}
mu.Lock()
limit := searchLimits[0]
mu.Unlock()
if limit != "100" {
t.Fatalf("remote search Limit=%q, want bounded 100 (50000 means a full-library query)", limit)
}
// 同一关键词再来一次(例如刷新/重复搜索)应复用缓存,不再请求上游。
if items := search("q=远方&page=1&page_size=48"); len(items) != 1 || items[0].Title != "远方剧集" {
t.Fatalf("cached search lost the remote result: %#v", items)
}
if calls := callCount(); calls != 1 {
t.Fatalf("expected cached remote search to skip upstream, got %d calls", calls)
}
// 指定本地媒体库时不应枚举远程挂载点。
if items := search("q=流浪&page=1&page_size=48&library_id=" + lib.ID); len(items) != 1 || items[0].ID != "local-1" {
t.Fatalf("expected local-only result, got %#v", items)
}
if calls := callCount(); calls != 1 {
t.Fatalf("local library filter must not hit upstream, got %d calls", calls)
}
}
+5 -1
View File
@@ -6,6 +6,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/gin-gonic/gin"
@@ -35,7 +36,10 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
}))
defer upstream.Close()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
+4 -1
View File
@@ -559,7 +559,10 @@ func TestSearchMediaHandlerIncludesEmbyRemote(t *testing.T) {
}))
defer server.Close()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
+3 -3
View File
@@ -2611,14 +2611,14 @@ export function PlayerPage() {
<button
type="button"
onClick={() => setDiagnosticsOpen((open) => !open)}
className="fixed bottom-5 right-4 z-[80] inline-flex items-center gap-2 rounded-full border border-white/15 bg-black/75 px-3 py-2 text-xs font-semibold text-white shadow-lg backdrop-blur hover:bg-black/90"
className="absolute right-2.5 top-14 z-[80] inline-flex items-center gap-2 rounded-full border border-white/15 bg-black/75 px-3 py-2 text-xs font-semibold text-white shadow-lg backdrop-blur hover:bg-black/90 sm:right-3.5 sm:top-16"
title="播放诊断"
aria-label="播放诊断"
>
<Activity size={15} />诊断
</button>
{diagnosticsOpen && (
<aside className="fixed bottom-16 right-4 z-[80] max-h-[70vh] w-[min(92vw,420px)] overflow-hidden rounded-2xl border border-white/15 bg-[#111827]/95 text-white shadow-2xl backdrop-blur">
<aside className="absolute right-2.5 top-[6.75rem] z-[80] flex max-h-[calc(100%-8.5rem)] w-[min(92vw,420px)] flex-col overflow-hidden rounded-2xl border border-white/15 bg-[#111827]/95 text-white shadow-2xl backdrop-blur sm:right-3.5 sm:top-[7.25rem]">
<div className="flex items-center justify-between border-b border-white/10 px-4 py-3">
<div><h2 className="text-sm font-bold">播放诊断</h2><p className="mt-1 text-[11px] text-white/60">记录最近 40 条本地播放事件</p></div>
<div className="flex gap-2">
@@ -2626,7 +2626,7 @@ export function PlayerPage() {
<button type="button" onClick={() => setDiagnosticsOpen(false)} className="rounded-lg p-1.5 hover:bg-white/10" aria-label="关闭"><X size={16} /></button>
</div>
</div>
<div className="max-h-[55vh] space-y-2 overflow-y-auto p-3">
<div className="min-h-0 flex-1 space-y-2 overflow-y-auto p-3">
<div className="rounded-lg bg-white/5 px-3 py-2 text-xs text-white/70">模式:{mode === 'hls' ? `HLS / ${hlsSource}` : '直连'} · 来源:{isRemoteEmbyID(media?.id) ? '远程 Emby' : isStrmMedia(media) ? 'STRM' : '本地文件'}</div>
{diagnosticEvents.length === 0 ? <p className="py-6 text-center text-xs text-white/50">等待播放事件…</p> : [...diagnosticEvents].reverse().map((event, index) => (
<div key={`${event.at}-${index}`} className="rounded-lg bg-white/5 px-3 py-2">