Compare commits

..

7 Commits

Author SHA1 Message Date
truewhile e34e81cce0 Merge pull request #48 from truewhile/fix/media-access-cache-search
修复实时事件权限和媒体缓存隔离,移除大库列表截断并缓存排序索引,批量同步 OpenSearch 并处理逐项失败。
2026-10-20 20:40:49 +08:00
jason jia 3cf898756b fix media event access, cache isolation and indexing 2026-10-20 12:39:13 +00:00
truewhile 0b68397265 Merge pull request #46 from truewhile/fix/introdb-api-v4
迁移 TheIntroDB 接口到 v4
2026-10-19 22:26:33 +08:00
truewhile 90bfac3b09 迁移 TheIntroDB 接口到 v4
v1 已删除、v2/v3 已弃用,仅保留只读的 GET /media。本次把 base URL 切到 v4,并适配响应以 tvdb_id 为身份(tmdb_id 字段已移除)。请求仍用 tmdb_id 寻址,v4 仍接受并由服务端解析季集映射。
2026-10-19 22:23:06 +08:00
truewhile f1974a179e 优化 2026-10-19 21:38:55 +08:00
truewhile f2357200be 优化 2026-10-19 21:11:35 +08:00
truewhile ca9a80b862 优化 2026-10-18 22:30:02 +08:00
33 changed files with 1611 additions and 188 deletions
+2 -4
View File
@@ -556,19 +556,17 @@ func listMediaHandler(svc *service.Container) gin.HandlerFunc {
})
return
}
grouped, err := svc.Media.GroupedMediaVisibleFiltered(ctx, id, mediaVisibilityForRequest(c, svc), filters)
items, total, err := svc.Media.ListMediaVisibleSortedPage(ctx, id, page, size, mediaVisibilityForRequest(c, svc), filters, sortSpec, history)
if err != nil {
writeInternalOrCanceled(c, err)
return
}
grouped = service.SortMediaItems(grouped, sortSpec.Field, sortSpec.Order, history)
items := service.PaginateMediaItems(grouped, page, size)
if items == nil {
items = []service.MediaItem{}
}
c.JSON(http.StatusOK, gin.H{
"items": items,
"total": len(grouped),
"total": total,
"page": page,
"page_size": size,
})
+5
View File
@@ -80,6 +80,11 @@ func playbackInfoHandler(svc *service.Container) gin.HandlerFunc {
// stream_url 跳转,行为与改动前一致。
directURL, hasDirectURL := "", false
if svc.Stream != nil {
// `?refresh=1`:客户端手上的直链已失效(CDN 403),要求丢掉缓存
// 重新换链,别把同一条死链再发一次。
if strings.EqualFold(strings.TrimSpace(c.Query("refresh")), "1") {
service.InvalidateMediaLinkCache(m)
}
directURL, hasDirectURL = svc.Stream.ResolveDirectPlayURL(c.Request, m)
}
payload := gin.H{
@@ -21,6 +21,8 @@ func registerAuthedDLNARoutes(authed *gin.RouterGroup, svc *service.Container) {
}
func registerAuthedRealtimeRoutes(authed *gin.RouterGroup, svc *service.Container) {
authed.GET("/ws", wsHandler(svc))
authed.GET("/events", sseHandler(svc))
// These streams carry operator tasks, including filesystem paths and scan
// errors. Their payloads are not scoped to an individual user's libraries.
authed.GET("/ws", middleware.AdminRequired(), wsHandler(svc))
authed.GET("/events", middleware.AdminRequired(), sseHandler(svc))
}
+8 -5
View File
@@ -1,9 +1,7 @@
// Package handler — WebSocket endpoint.
//
// Clients connect to /api/ws?token=... (the token is the same JWT used for
// REST calls). The first message they send is a JSON {"action":"subscribe",
// "topics":["scan","scrape","transcode"]}. Subsequent server-pushed events
// arrive as {"topic":"...","payload":{...}}.
// Administrators connect to /api/ws?token=... with their REST access token.
// Operator events arrive as {"topic":"...","payload":{...}}.
package handler
import (
@@ -52,8 +50,11 @@ func wsHandler(svc *service.Container) gin.HandlerFunc {
sub := svc.WSHub.Subscribe(id, nil)
defer svc.WSHub.Unsubscribe(id)
// Reader: accept ping + subscription updates.
// Reading processes WebSocket control frames and detects disconnects.
done := make(chan struct{})
conn.SetReadLimit(64 << 10)
go func() {
defer close(done)
for {
if _, _, err := conn.NextReader(); err != nil {
_ = conn.Close()
@@ -67,6 +68,8 @@ func wsHandler(svc *service.Container) gin.HandlerFunc {
defer ticker.Stop()
for {
select {
case <-done:
return
case ev, ok := <-sub.Out:
if !ok {
return
+93
View File
@@ -0,0 +1,93 @@
package handler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/golang-jwt/jwt/v5"
"github.com/gorilla/websocket"
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/middleware"
"github.com/truewhile/MeBox/internal/service"
)
func realtimeTestRouter(hub *service.Hub) *gin.Engine {
gin.SetMode(gin.TestMode)
router := gin.New()
authed := router.Group("/api")
authed.Use(middleware.AuthRequired("realtime-test"))
registerAuthedRealtimeRoutes(authed, &service.Container{WSHub: hub})
return router
}
func realtimeTestToken(t *testing.T, role string) string {
t.Helper()
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, middleware.Claims{
UserID: "viewer", Role: role,
RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour))},
}).SignedString([]byte("realtime-test"))
if err != nil {
t.Fatal(err)
}
return token
}
func TestRealtimeStreamsRejectOrdinaryUsers(t *testing.T) {
router := realtimeTestRouter(nil)
for _, path := range []string{"/api/ws", "/api/events"} {
for _, role := range []string{"", "user"} {
req := httptest.NewRequest(http.MethodGet, path, nil)
want := http.StatusUnauthorized
if role != "" {
req.Header.Set("Authorization", "Bearer "+realtimeTestToken(t, role))
want = http.StatusForbidden
}
recorder := httptest.NewRecorder()
router.ServeHTTP(recorder, req)
if recorder.Code != want {
t.Fatalf("%s role %q: status %d, want %d", path, role, recorder.Code, want)
}
}
}
}
func TestAdminWebSocketReceivesTaskEvents(t *testing.T) {
hub := service.NewHub(zap.NewNop())
go hub.Run()
defer hub.Stop()
server := httptest.NewServer(realtimeTestRouter(hub))
defer server.Close()
url := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/ws?token=" + realtimeTestToken(t, "admin")
conn, _, err := websocket.DefaultDialer.Dial(url, nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
stop := make(chan struct{})
defer close(stop)
go func() {
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
hub.Publish("task", service.BackgroundTask{ID: "task-1", SourcePath: "/media/private"})
}
}
}()
_ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
var event service.Event
if err := conn.ReadJSON(&event); err != nil {
t.Fatal(err)
}
if event.Topic != "task" || event.Payload.(map[string]any)["source_path"] != "/media/private" {
t.Fatalf("event = %#v", event)
}
}
-8
View File
@@ -178,14 +178,6 @@ func (r *MediaRepository) YearRange(ctx context.Context, filter MediaQueryFilter
return min, max, nil
}
func (r *MediaRepository) indexMediaBestEffort(ctx context.Context, media model.Media) {
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
if !ok {
return
}
_ = backend.IndexMedia(ctx, []model.Media{media})
}
// FindByID returns the media row or (nil, nil).
func (r *MediaRepository) FindByID(ctx context.Context, id string) (*model.Media, error) {
var m model.Media
+22 -3
View File
@@ -96,15 +96,34 @@ func (r *MediaRepository) UpsertBatchWithAliases(ctx context.Context, items []Me
// indexByIDBestEffort 在事务提交后按 ID 回读最新行并同步搜索索引。
func (r *MediaRepository) indexByIDBestEffort(ctx context.Context, ids []string) {
if _, ok := r.searchBackend.(MediaSearchSyncBackend); !ok {
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
if !ok {
return
}
unique := make([]string, 0, len(ids))
seen := make(map[string]struct{}, len(ids))
for _, id := range ids {
if id == "" {
continue
}
if fresh, err := r.FindByID(ctx, id); err == nil && fresh != nil {
r.indexMediaBestEffort(ctx, *fresh)
if _, exists := seen[id]; !exists {
seen[id] = struct{}{}
unique = append(unique, id)
}
}
// Read committed rows and submit one bulk per bounded batch. Keep network
// work outside the transaction and below SQLite's parameter limit.
const batchSize = 500
for start := 0; start < len(unique); start += batchSize {
batch := unique[start:min(start+batchSize, len(unique))]
rows, err := r.ListMediaByIDs(ctx, batch)
if err == nil {
err = backend.IndexMedia(ctx, rows)
}
if err != nil {
// The primary write has committed; do not report a failed upsert and
// make scanners repeat it. Surface the separate index failure instead.
r.db.Logger.Error(ctx, "media saved but search index sync failed for %d items: %v", len(batch), err)
}
}
}
+119 -4
View File
@@ -4,8 +4,10 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
@@ -135,14 +137,113 @@ func (b *OpenSearchMediaBackend) EnsureIndex(ctx context.Context) error {
}
func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Media) error {
const batchSize = 500
var failures []error
for start := 0; start < len(rows); start += batchSize {
if err := b.indexMediaBatch(ctx, rows[start:min(start+batchSize, len(rows))]); err != nil {
failures = append(failures, err)
if ctx.Err() != nil {
break
}
}
}
return errors.Join(failures...)
}
type openSearchBulkResponse struct {
Errors bool `json:"errors"`
Items []struct {
Index struct {
ID string `json:"_id"`
Status int `json:"status"`
Error *struct {
Type string `json:"type"`
} `json:"error"`
} `json:"index"`
} `json:"items"`
}
func (b *OpenSearchMediaBackend) indexMediaBatch(ctx context.Context, rows []model.Media) error {
if len(rows) == 0 {
return nil
}
pending := rows
var failures []error
const attempts = 3
for attempt := 0; attempt < attempts; attempt++ {
if err := ctx.Err(); err != nil {
return errors.Join(append(failures, err)...)
}
if attempt > 0 {
timer := time.NewTimer(time.Duration(100*(1<<(attempt-1))) * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
return errors.Join(append(failures, ctx.Err())...)
case <-timer.C:
}
}
response, err := b.submitMediaBulk(ctx, pending)
if err != nil {
if retryableOpenSearchError(err) && attempt < attempts-1 {
continue
}
return errors.Join(append(failures, err)...)
}
if len(response.Items) != len(pending) {
return errors.Join(append(failures, fmt.Errorf("opensearch bulk returned %d item results for %d documents", len(response.Items), len(pending)))...)
}
var retry []model.Media
failed := false
for i, item := range response.Items {
result := item.Index
if result.Status >= 200 && result.Status < 300 && result.Error == nil {
continue
}
failed = true
if retryableOpenSearchStatus(result.Status) && attempt < attempts-1 {
retry = append(retry, pending[i])
continue
}
errorType := "unknown"
if result.Error != nil {
errorType = result.Error.Type
}
failures = append(failures, fmt.Errorf("opensearch bulk document %q failed: status %d (%s)", pending[i].ID, result.Status, errorType))
}
if response.Errors && !failed {
failures = append(failures, errors.New("opensearch bulk reported errors without failed item results"))
}
if len(retry) == 0 {
break
}
pending = retry
}
return errors.Join(failures...)
}
func retryableOpenSearchStatus(status int) bool {
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
}
func retryableOpenSearchError(err error) bool {
var statusError *openSearchHTTPError
if errors.As(err, &statusError) {
return retryableOpenSearchStatus(statusError.status)
}
var networkError net.Error
return errors.As(err, &networkError)
}
func (b *OpenSearchMediaBackend) submitMediaBulk(ctx context.Context, rows []model.Media) (*openSearchBulkResponse, error) {
if len(rows) == 0 {
return &openSearchBulkResponse{}, nil
}
var bulk bytes.Buffer
enc := json.NewEncoder(&bulk)
for _, row := range rows {
if err := enc.Encode(map[string]any{"index": map[string]any{"_index": b.index, "_id": row.ID}}); err != nil {
return err
return nil, err
}
if err := enc.Encode(map[string]any{
"id": row.ID,
@@ -155,10 +256,24 @@ func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Me
"deleted": row.DeletedAt.Valid,
"created_at": row.CreatedAt,
}); err != nil {
return err
return nil, err
}
}
return b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", nil)
var response openSearchBulkResponse
if err := b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", &response); err != nil {
return nil, err
}
return &response, nil
}
type openSearchHTTPError struct {
method string
path string
status int
}
func (e *openSearchHTTPError) Error() string {
return fmt.Sprintf("opensearch %s %s returned %d", e.method, e.path, e.status)
}
func (b *OpenSearchMediaBackend) doJSON(ctx context.Context, method, path string, body any, out any) error {
@@ -190,7 +305,7 @@ func (b *OpenSearchMediaBackend) do(ctx context.Context, method, path string, bo
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("opensearch %s %s returned %d", method, path, resp.StatusCode)
return &openSearchHTTPError{method: method, path: path, status: resp.StatusCode}
}
if out == nil {
_, _ = io.Copy(io.Discard, resp.Body)
+164
View File
@@ -1,12 +1,23 @@
package repository
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/model"
)
func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
@@ -55,3 +66,156 @@ func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
t.Fatalf("ids=%#v total=%d", ids, total)
}
}
func bulkRequestIDs(r *http.Request) []string {
var ids []string
scanner := bufio.NewScanner(r.Body)
for scanner.Scan() {
var action struct {
Index struct {
ID string `json:"_id"`
} `json:"index"`
}
_ = json.Unmarshal(scanner.Bytes(), &action)
ids = append(ids, action.Index.ID)
if !scanner.Scan() {
break
}
}
return ids
}
func writeBulkResults(w http.ResponseWriter, ids []string, statuses map[string]int) {
items := make([]any, 0, len(ids))
failed := false
for _, id := range ids {
status := statuses[id]
if status == 0 {
status = http.StatusCreated
}
result := map[string]any{"_id": id, "status": status}
if status >= 400 {
failed = true
result["error"] = map[string]any{"type": "test_failure"}
}
items = append(items, map[string]any{"index": result})
}
_ = json.NewEncoder(w).Encode(map[string]any{"errors": failed, "items": items})
}
func TestOpenSearchBulkRetriesOnlyTransientFailedDocuments(t *testing.T) {
var requests [][]string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ids := bulkRequestIDs(r)
requests = append(requests, ids)
statuses := map[string]int{}
if len(requests) == 1 {
statuses["retry"] = http.StatusTooManyRequests
statuses["invalid"] = http.StatusBadRequest
}
writeBulkResults(w, ids, statuses)
}))
defer upstream.Close()
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
rows := []model.Media{{Base: model.Base{ID: "ok"}}, {Base: model.Base{ID: "retry"}}, {Base: model.Base{ID: "invalid"}}}
err := backend.IndexMedia(t.Context(), rows)
if err == nil || !strings.Contains(err.Error(), `"invalid"`) {
t.Fatalf("error = %v, want permanent document failure", err)
}
if len(requests) != 2 || strings.Join(requests[1], ",") != "retry" {
t.Fatalf("requests = %#v; successful/permanent documents must not be retried", requests)
}
}
func TestOpenSearchBulkBoundsRetriesAndRejectsMalformedResponses(t *testing.T) {
for _, scenario := range []string{"transient-item", "transient-http", "invalid-http", "missing-results"} {
t.Run(scenario, func(t *testing.T) {
calls := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
switch scenario {
case "transient-item":
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusServiceUnavailable})
case "transient-http":
w.WriteHeader(http.StatusServiceUnavailable)
case "invalid-http":
w.WriteHeader(http.StatusBadRequest)
case "missing-results":
fmt.Fprint(w, `{"errors":false,"items":[]}`)
}
}))
defer upstream.Close()
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
if err := backend.IndexMedia(t.Context(), []model.Media{{Base: model.Base{ID: "m1"}}}); err == nil {
t.Fatal("failed indexing was reported as successful")
}
want := 1
if strings.HasPrefix(scenario, "transient") {
want = 3
}
if calls != want {
t.Fatalf("calls = %d, want %d", calls, want)
}
})
}
}
func TestOpenSearchBulkHonorsCancellationDuringRetry(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
calls := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusTooManyRequests})
cancel()
}))
defer upstream.Close()
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
err := backend.IndexMedia(ctx, []model.Media{{Base: model.Base{ID: "m1"}}})
if !errors.Is(err, context.Canceled) || calls != 1 {
t.Fatalf("error = %v, calls = %d", err, calls)
}
}
func TestMediaUpsertBatchIndexesCommittedRowsTogether(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatal(err)
}
sqlDB, _ := db.DB()
sqlDB.SetMaxOpenConns(1)
defer sqlDB.Close()
if err := db.AutoMigrate(&model.Media{}); err != nil {
t.Fatal(err)
}
repos := New(db)
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
calls := 0
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
ids := bulkRequestIDs(r)
if len(ids) != 5 {
t.Errorf("bulk contains %d documents, want 5", len(ids))
}
// With only one DB connection this also detects network work inside the
// transaction: a separate read could not observe the committed batch.
var count int64
if err := db.WithContext(ctx).Model(&model.Media{}).Count(&count).Error; err != nil || count != 5 {
t.Errorf("committed rows = %d, error = %v", count, err)
}
writeBulkResults(w, ids, nil)
}))
defer upstream.Close()
repos.Media.SetSearchBackend(NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL}))
var rows []*model.Media
for i := 0; i < 5; i++ {
rows = append(rows, &model.Media{LibraryID: "lib", Path: fmt.Sprintf("/media/%d.mkv", i)})
}
if err := repos.Media.UpsertBatch(ctx, rows); err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("index requests = %d, want one bulk", calls)
}
}
@@ -46,16 +46,20 @@ func (r *RefreshTokenRepository) RevokeByUserID(ctx context.Context, userID stri
})
}
// RevokeOldestActiveByUserID keeps at most limit active refresh tokens for a
// user by revoking the oldest non-expired, non-revoked tokens.
func (r *RefreshTokenRepository) RevokeOldestActiveByUserID(ctx context.Context, userID string, limit int) error {
// RevokeOldestActiveLegacyByUserID keeps at most limit active refresh tokens
// for a user among tokens that are not bound to a device auth session
// (legacy web clients). Device sessions are independent: trimming one
// device's token because another device logged in would silently sign the
// first device out even though its session is still valid.
func (r *RefreshTokenRepository) RevokeOldestActiveLegacyByUserID(ctx context.Context, userID string, limit int) error {
if limit < 1 {
limit = 1
}
return withSQLiteBusyRetry(ctx, func() error {
var tokens []model.RefreshToken
if err := r.db.WithContext(ctx).
Where("user_id = ? AND revoked = ? AND expires_at > ?", userID, false, time.Now()).
Where("user_id = ? AND revoked = ? AND expires_at > ? AND (auth_session_id IS NULL OR auth_session_id = '')",
userID, false, time.Now()).
Order("created_at desc, id desc").
Find(&tokens).Error; err != nil {
return err
+47
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"errors"
"fmt"
"testing"
"time"
@@ -163,6 +164,52 @@ func TestRefreshKeepsSessionAssociation(t *testing.T) {
}
}
// TestMobileDeviceSessionsDoNotTrimEachOther 覆盖同一账号多设备登录:
// 每台设备有独立认证会话与刷新令牌,新设备登录或任一设备刷新都不应撤销
// 其他设备的刷新令牌。历史实现把「最大同时登录终端」直接作用在刷新令牌
// 上,第 N+1 台设备登录会把最早那台设备的刷新令牌标记为已撤销,那台设备
// 下一次刷新就会收到 SESSION_REVOKED,被强制退回登录页。
func TestMobileDeviceSessionsDoNotTrimEachOther(t *testing.T) {
sessions, tokens, repos := newAuthSessionTestEnv(t)
ctx := context.Background()
user := seedSessionUser(t, repos)
// 默认上限是 3,这里故意登录 4 台设备。
const devices = 4
pairs := make([]*TokenPair, 0, devices)
for i := 0; i < devices; i++ {
session, err := sessions.Create(ctx, CreateAuthSessionInput{
UserID: user.ID,
DeviceID: fmt.Sprintf("device-%02d", i),
DeviceName: fmt.Sprintf("Phone %d", i+1),
})
if err != nil {
t.Fatalf("create session %d: %v", i+1, err)
}
pair, err := tokens.IssuePairForSession(ctx, user.ID, user.Role, user.Tier, session.ID)
if err != nil {
t.Fatalf("issue pair %d: %v", i+1, err)
}
pairs = append(pairs, pair)
}
for i, pair := range pairs {
row, err := repos.RefreshToken.FindByHash(ctx, repository.HashToken(pair.RefreshToken))
if err != nil {
t.Fatalf("load device %d token: %v", i+1, err)
}
if row == nil {
t.Fatalf("device %d refresh token missing", i+1)
}
if row.Revoked {
t.Fatalf("device %d refresh token was revoked by another device", i+1)
}
if _, err := tokens.Refresh(ctx, pair.RefreshToken); err != nil {
t.Fatalf("device %d refresh failed: %v", i+1, err)
}
}
}
func TestServerIdentityPersistsAcrossInstances(t *testing.T) {
db := newServiceTestDB(t, &model.Setting{})
repos := repository.New(db)
+89 -10
View File
@@ -1,26 +1,34 @@
// 下载直链进程级缓存。
//
// 115 的 /open/ufile/downurl 换取结果在一段时间内有效,且换取请求是
// WAF 风控重点关注的密集接口。缓存按 pickcode 存直链、避免对同一文件
// 反复换取;下载失败(http 403/404 等)时由调用方主动清除对应条目,
// 下一轮重试会重新换取链接。
// 115 的 /open/ufile/downurl 换取结果带 `t=` 过期时间戳(实测为「换取时刻 +
// 1800 秒」,即 30 分钟),且换取请求是 WAF 风控重点关注的密集接口。缓存按
// pickcode 存直链、避免对同一文件反复换取;下载失败(http 403/404 等)时由
// 调用方主动清除对应条目,下一轮重试会重新换取链接。
package cloud115
import (
"net/url"
"strconv"
"strings"
"sync"
"time"
)
// downloadURLCacheTTL 直链缓存时长。115 直链一般有效 1 小时左右,
// 取 45 分钟留出余量。QMediaSync 同类缓存使用 50 分钟。
const downloadURLCacheTTL = 45 * time.Minute
// downloadURLCacheMaxTTL 直链缓存上限:仅在解析不出直链自带过期时间(`t=`)
// 时使用。115 直链实际有效 30 分钟,这里取 45 分钟只是兜底上限,正常情况下
// 由 linkExpirySafety 提前失效,避免把已经过期的直链发给客户端。
const downloadURLCacheMaxTTL = 45 * time.Minute
// linkExpirySafety 提前量:直链自身的过期时间戳到达前就停止使用,留出客户端
// 建连与传输的余量,防止出现「服务端认为还有效、CDN 已经 403」的窗口。
const linkExpirySafety = 2 * time.Minute
// maxCachedURLs 超过该数量时顺带清理过期条目,防止 map 无限膨胀。
const maxCachedURLs = 10000
type urlCacheEntry struct {
url string
createdAt time.Time
expiresAt time.Time
}
@@ -37,6 +45,43 @@ func urlCacheKey(pickCode, ua string) string {
return pickCode + "@" + ua
}
// LinkExpiry 解析 115 直链自带的过期时间戳(`t=` 参数,单位秒)。
// 解析不出时返回零值,调用方按兜底 TTL 处理。
func LinkExpiry(rawURL string) time.Time {
u, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil || u == nil {
return time.Time{}
}
raw := strings.TrimSpace(u.Query().Get("t"))
if raw == "" {
return time.Time{}
}
sec, err := strconv.ParseInt(raw, 10, 64)
if err != nil || sec <= 0 {
return time.Time{}
}
return time.Unix(sec, 0)
}
// cacheExpiry 计算写入缓存时应使用的过期时刻:跟随直链自身的 `t=` 并提前
// linkExpirySafety,解析不出时退回 downloadURLCacheMaxTTL。返回零值表示这条
// 直链已经过期,不应写入缓存。
func cacheExpiry(rawURL string, now time.Time) time.Time {
maxExpiry := now.Add(downloadURLCacheMaxTTL)
linkExpiry := LinkExpiry(rawURL)
if linkExpiry.IsZero() {
return maxExpiry
}
expiry := linkExpiry.Add(-linkExpirySafety)
if !expiry.After(now) {
return time.Time{}
}
if expiry.After(maxExpiry) {
return maxExpiry
}
return expiry
}
// GetDownloadURLCache 返回未过期的缓存直链;不存在或已过期返回空串。
func GetDownloadURLCache(pickCode string, uas ...string) string {
if pickCode == "" {
@@ -59,11 +104,17 @@ func GetDownloadURLCache(pickCode string, uas ...string) string {
return entry.url
}
// SetDownloadURLCache 写入直链缓存。
// SetDownloadURLCache 写入直链缓存。过期时刻跟随直链自身的 `t=`,已经过期的
// 直链直接丢弃,避免把死链缓存下来。
func SetDownloadURLCache(pickCode string, url string, uas ...string) {
if pickCode == "" || url == "" {
return
}
now := time.Now()
expiresAt := cacheExpiry(url, now)
if expiresAt.IsZero() {
return
}
ua := ""
if len(uas) > 0 {
ua = uas[0]
@@ -71,7 +122,6 @@ func SetDownloadURLCache(pickCode string, url string, uas ...string) {
urlCacheMu.Lock()
defer urlCacheMu.Unlock()
if len(urlCache) >= maxCachedURLs {
now := time.Now()
for k, e := range urlCache {
if now.After(e.expiresAt) {
delete(urlCache, k)
@@ -79,7 +129,7 @@ func SetDownloadURLCache(pickCode string, url string, uas ...string) {
}
}
key := urlCacheKey(pickCode, ua)
urlCache[key] = urlCacheEntry{url: url, expiresAt: time.Now().Add(downloadURLCacheTTL)}
urlCache[key] = urlCacheEntry{url: url, createdAt: now, expiresAt: expiresAt}
}
// ClearDownloadURLCache 删除指定 pickcode 的所有缓存(下载得到非 2xx 时调用)。
@@ -95,3 +145,32 @@ func ClearDownloadURLCache(pickCode string) {
}
}
}
// ClearDownloadURLCacheIfOlderThan 删除指定 pickcode 下「已经存在超过 minAge」
// 的缓存条目,返回是否删除了至少一条。
//
// 客户端上报「直链被 CDN 拒绝」时用它强制换链:既让下一次解析拿到新地址,
// 又不会让同一秒内的重试风暴把 115 换链接口打出突发。
func ClearDownloadURLCacheIfOlderThan(pickCode string, minAge time.Duration) bool {
if pickCode == "" {
return false
}
if minAge < 0 {
minAge = 0
}
now := time.Now()
removed := false
urlCacheMu.Lock()
defer urlCacheMu.Unlock()
for k, e := range urlCache {
if k != pickCode && !strings.HasPrefix(k, pickCode+"@") {
continue
}
if now.Sub(e.createdAt) < minAge {
continue
}
delete(urlCache, k)
removed = true
}
return removed
}
@@ -1,6 +1,7 @@
package cloud115
import (
"fmt"
"testing"
"time"
)
@@ -29,3 +30,72 @@ func TestDownloadURLCache(t *testing.T) {
t.Fatalf("cleared entry should be a miss, got %q", got)
}
}
// 115 直链自带 `t=` 过期时间戳(实测为换取时刻 + 1800 秒)。缓存必须跟随它,
// 否则会出现「服务端认为还有效、CDN 已经 403」的窗口:直链被缓存到 45 分钟,
// 但第 30 分钟起客户端就必然拿到 403。
func TestDownloadURLCacheExpiryFollowsLinkTimestamp(t *testing.T) {
now := time.Now()
link := fmt.Sprintf("https://cdn/1/a.mkv?t=%d&u=1&k=abc", now.Add(30*time.Minute).Unix())
got := cacheExpiry(link, now)
want := now.Add(30*time.Minute - linkExpirySafety)
if diff := got.Sub(want); diff > time.Second || diff < -time.Second {
t.Fatalf("cacheExpiry = %v, want %v", got, want)
}
// 解析不出 t= 时退回兜底 TTL,保持旧行为。
fallback := cacheExpiry("https://cdn/1/a.mkv", now)
if diff := fallback.Sub(now.Add(downloadURLCacheMaxTTL)); diff > time.Second || diff < -time.Second {
t.Fatalf("fallback cacheExpiry = %v, want %v", fallback, now.Add(downloadURLCacheMaxTTL))
}
// 直链本身已经过期(或只剩不足提前量的寿命)时不进缓存。
if got := cacheExpiry(fmt.Sprintf("https://cdn/1/a.mkv?t=%d", now.Add(-time.Minute).Unix()), now); !got.IsZero() {
t.Fatalf("expired link must not be cached, got expiry %v", got)
}
if got := cacheExpiry(fmt.Sprintf("https://cdn/1/a.mkv?t=%d", now.Add(time.Minute).Unix()), now); !got.IsZero() {
t.Fatalf("link expiring inside the safety margin must not be cached, got %v", got)
}
}
// 客户端上报「直链被 CDN 拒绝」时按 pickcode 清缓存,但要挡住刚写入的条目,
// 避免重试风暴把 115 换链接口打出突发。
func TestClearDownloadURLCacheIfOlderThan(t *testing.T) {
urlCacheMu.Lock()
urlCache = map[string]urlCacheEntry{}
urlCacheMu.Unlock()
t.Cleanup(func() {
urlCacheMu.Lock()
urlCache = map[string]urlCacheEntry{}
urlCacheMu.Unlock()
})
future := time.Now().Add(30 * time.Minute)
link := fmt.Sprintf("https://cdn/1/a.mkv?t=%d", future.Unix())
SetDownloadURLCache("pick-fresh", link, "ua-a")
if ClearDownloadURLCacheIfOlderThan("pick-fresh", time.Minute) {
t.Fatal("fresh entry must survive the min-age guard")
}
if got := GetDownloadURLCache("pick-fresh", "ua-a"); got != link {
t.Fatalf("fresh entry lost, got %q", got)
}
SetDownloadURLCache("pick-old", link, "ua-a")
urlCacheMu.Lock()
entry := urlCache[urlCacheKey("pick-old", "ua-a")]
entry.createdAt = time.Now().Add(-10 * time.Minute)
urlCache[urlCacheKey("pick-old", "ua-a")] = entry
urlCacheMu.Unlock()
if !ClearDownloadURLCacheIfOlderThan("pick-old", time.Minute) {
t.Fatal("stale entry should be cleared")
}
if got := GetDownloadURLCache("pick-old", "ua-a"); got != "" {
t.Fatalf("stale entry should be gone, got %q", got)
}
// 无关 pickcode 不受影响。
if got := GetDownloadURLCache("pick-fresh", "ua-a"); got != link {
t.Fatalf("unrelated pickcode lost, got %q", got)
}
}
+6 -3
View File
@@ -22,8 +22,9 @@ import (
const (
// IntroDBBaseURL is the public read endpoint. Overridable on the service
// for tests and for pointing at a mirror.
IntroDBBaseURL = "https://api.theintrodb.org/v3"
// for tests and for pointing at a mirror. v1 已删除,v2/v3 冻结在只读的
// GET /media 上并被官方标记为弃用,所以这里直接跟 v4。
IntroDBBaseURL = "https://api.theintrodb.org/v4"
// IntroDBSource tags rows that came from this provider.
IntroDBSource = "theintrodb"
@@ -92,8 +93,10 @@ type introDBRange struct {
EndMs *int64 `json:"end_ms"`
}
// introDBResponse 对应 v4 的 MediaResponse。v4 以 TVDB 为身份,响应只带
// tvdb_id —— tmdb_id 字段已被官方移除,需要 tmdb_id 只能从请求侧自己记。
type introDBResponse struct {
TMDbID int `json:"tmdb_id"`
TVDBID int `json:"tvdb_id"`
Type string `json:"type"`
Intro []introDBRange `json:"intro"`
Recap []introDBRange `json:"recap"`
+9 -8
View File
@@ -11,11 +11,12 @@ import (
"go.uber.org/zap"
)
// 这两段响应是从 api.theintrodb.org/v3/media 实测抓下来的原文,
// 用来锁住 null 语义:start_ms: null = 从片头开始,end_ms: null = 一直到片尾。
// 这两段响应是从 api.theintrodb.org/v4/media 实测抓下来的原文(v4 以 TVDB 为身份,
// 响应里只有 tvdb_id),用来锁住 null 语义:start_ms: null = 从片头开始,
// end_ms: null = 一直到片尾。
const (
introDBTVPayload = `{"tmdb_id":1396,"type":"tv","season":1,"episode":1,"intro":[{"start_ms":228664,"end_ms":246143}],"credits":[{"start_ms":3431000,"end_ms":null}]}`
introDBMoviePayload = `{"tmdb_id":27205,"type":"movie","intro":[{"start_ms":null,"end_ms":38000}]}`
introDBTVPayload = `{"tvdb_id":81189,"type":"tv","season":1,"episode":1,"intro":[{"start_ms":228664,"end_ms":246143}],"credits":[{"start_ms":3431000,"end_ms":null}]}`
introDBMoviePayload = `{"tvdb_id":113,"type":"movie","intro":[{"start_ms":null,"end_ms":38000}]}`
)
func TestParseIntroDBResponseResolvesNullBounds(t *testing.T) {
@@ -48,7 +49,7 @@ func TestParseIntroDBResponseResolvesNullBounds(t *testing.T) {
}
func TestParseIntroDBResponseDropsEmptyRanges(t *testing.T) {
body := `{"tmdb_id":1,"type":"movie",
body := `{"tvdb_id":1,"type":"movie",
"intro":[{"start_ms":5000,"end_ms":5000},{"start_ms":9000,"end_ms":8000},{"start_ms":1000,"end_ms":2000}],
"recap":[],"credits":[],"preview":[]}`
spans, err := parseIntroDBResponse([]byte(body))
@@ -62,7 +63,7 @@ func TestParseIntroDBResponseDropsEmptyRanges(t *testing.T) {
}
func TestParseIntroDBResponseOrdersByType(t *testing.T) {
body := `{"tmdb_id":1,"type":"tv","credits":[{"start_ms":900,"end_ms":1000}],
body := `{"tvdb_id":1,"type":"tv","credits":[{"start_ms":900,"end_ms":1000}],
"intro":[{"start_ms":100,"end_ms":200}],"recap":[{"start_ms":50,"end_ms":60}]}`
spans, err := parseIntroDBResponse([]byte(body))
if err != nil {
@@ -82,12 +83,12 @@ func TestParseIntroDBResponseOrdersByType(t *testing.T) {
func TestIntroDBMediaURLOnlyAddsSeasonEpisodeForTV(t *testing.T) {
svc := NewIntroDBService(zap.NewNop())
if got, want := svc.mediaURL(1396, 1, 1),
"https://api.theintrodb.org/v3/media?tmdb_id=1396&season=1&episode=1"; got != want {
"https://api.theintrodb.org/v4/media?tmdb_id=1396&season=1&episode=1"; got != want {
t.Fatalf("tv url = %q, want %q", got, want)
}
// 电影(season/episode 为 0)不能带季集参数,否则会被当成剧集查不到。
if got, want := svc.mediaURL(27205, 0, 0),
"https://api.theintrodb.org/v3/media?tmdb_id=27205"; got != want {
"https://api.theintrodb.org/v4/media?tmdb_id=27205"; got != want {
t.Fatalf("movie url = %q, want %q", got, want)
}
}
@@ -184,6 +184,85 @@ func TestReadAdultMetadataDerivesDMMPosterFromRemoteFanart(t *testing.T) {
}
}
func TestReadAdultArtworkIgnoresLoosePosterInLibraryRoot(t *testing.T) {
root := t.TempDir()
// 云下载这类库把不同影片平铺在同一个根目录里,根目录中残留的松散海报
// 属于别的影片,不能被抓来当作当前影片的封面。
if err := os.WriteFile(filepath.Join(root, "AAA-001-poster.jpg"), []byte("jpg"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "AAA-001-fanart.jpg"), []byte("jpg"), 0o644); err != nil {
t.Fatal(err)
}
mediaPath := filepath.Join(root, "BBB-002.mp4")
if err := os.WriteFile(mediaPath, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
got, err := ReadLocalMetadata(mediaPath, root, false)
if err != nil {
t.Fatal(err)
}
if got != nil && (got.PosterURL != "" || got.BackdropURL != "") {
t.Fatalf("library-root artwork must not be inherited: poster=%q backdrop=%q", got.PosterURL, got.BackdropURL)
}
}
func TestReadAdultArtworkIgnoresLoosePosterInSharedFolder(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "shared")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
for _, name := range []string{"AAA-001.mp4", "BBB-002.mp4"} {
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(dir, "AAA-001-poster.jpg"), []byte("jpg"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "AAA-001-fanart.jpg"), []byte("jpg"), 0o644); err != nil {
t.Fatal(err)
}
got, err := ReadLocalMetadata(filepath.Join(dir, "BBB-002.mp4"), root, false)
if err != nil {
t.Fatal(err)
}
if got != nil && (got.PosterURL != "" || got.BackdropURL != "") {
t.Fatalf("sibling artwork must not be inherited in a shared folder: poster=%q backdrop=%q", got.PosterURL, got.BackdropURL)
}
}
func TestReadAdultArtworkKeepsLooseFallbackInItemFolder(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "BBB-002")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
mediaPath := filepath.Join(dir, "video.mp4")
if err := os.WriteFile(mediaPath, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
poster := filepath.Join(dir, "japanese-title-cover.jpg")
fanart := filepath.Join(dir, "japanese-title-fanart.jpg")
if err := os.WriteFile(poster, []byte("jpg"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fanart, []byte("jpg"), 0o644); err != nil {
t.Fatal(err)
}
got, err := ReadLocalMetadata(mediaPath, root, false)
if err != nil {
t.Fatal(err)
}
if got == nil || got.PosterURL != poster || got.BackdropURL != fanart {
t.Fatalf("single-item folder should keep loose artwork fallback: %+v", got)
}
}
func TestReadAdultArtworkByCodeWithoutNFO(t *testing.T) {
root := t.TempDir()
mediaPath := filepath.Join(root, "SSIS-001-CD1.mp4")
+47 -5
View File
@@ -11,23 +11,23 @@ import (
"strings"
)
func metadataFromArtwork(mediaPath, showBaseDir string) *LocalMetadata {
func metadataFromArtwork(mediaPath, showBaseDir, libraryRoot string) *LocalMetadata {
meta := &LocalMetadata{}
mergeArtworkMetadata(meta, mediaPath, showBaseDir)
mergeArtworkMetadata(meta, mediaPath, showBaseDir, libraryRoot)
if meta.PosterURL == "" && meta.BackdropURL == "" {
return nil
}
return meta
}
func mergeArtworkMetadata(meta *LocalMetadata, mediaPath, showBaseDir string) {
func mergeArtworkMetadata(meta *LocalMetadata, mediaPath, showBaseDir, libraryRoot string) {
if meta == nil {
return
}
mediaDir := filepath.Dir(mediaPath)
if localPoster := firstLocalPoster(mediaPath, showBaseDir); localPoster != "" {
meta.PosterURL = localPoster
} else if meta.PosterURL == "" {
} else if meta.PosterURL == "" && looseArtworkFallbackAllowed(mediaDir, libraryRoot) {
meta.PosterURL = firstAdultLooseImage(mediaDir, "poster")
}
dirs := []string{mediaDir, showBaseDir}
@@ -39,7 +39,7 @@ func mergeArtworkMetadata(meta *LocalMetadata, mediaPath, showBaseDir string) {
meta.BackdropURL = img
break
}
if meta.BackdropURL == "" {
if meta.BackdropURL == "" && looseArtworkFallbackAllowed(dir, libraryRoot) {
meta.BackdropURL = firstAdultLooseImage(dir, "backdrop")
}
}
@@ -245,6 +245,48 @@ func firstExistingPosterImage(dir string, names ...string) string {
return ""
}
// looseArtworkFallbackAllowed reports whether an unmatched image sitting in dir
// can be attributed to mediaPath. Such an image carries no filename relation to
// a specific video, so it is only trustworthy when the folder holds a single
// media file and that folder is not the library root. Download tools routinely
// drop unrelated titles flat into one directory (a 115 cloud-download library,
// for example); claiming the first "*-poster.jpg" there made every item show the
// same artwork. Mirrors the root-level guard findMovieNFO already applies to
// loose NFO files.
func looseArtworkFallbackAllowed(dir, libraryRoot string) bool {
if dir == "" {
return false
}
if libraryRoot != "" && samePath(dir, libraryRoot) {
return false
}
return countMediaFilesInDir(dir) <= 1
}
// countMediaFilesInDir counts the video files directly inside dir, stopping once
// more than one is found. It walks the real directory rather than reusing the
// loose-image glob because a shared folder may contain no images at all.
func countMediaFilesInDir(dir string) int {
entries, err := os.ReadDir(dir)
if err != nil {
return 0
}
count := 0
for _, entry := range entries {
if entry.IsDir() {
continue
}
if _, ok := videoExtensions[strings.ToLower(filepath.Ext(entry.Name()))]; !ok {
continue
}
count++
if count > 1 {
return count
}
}
return count
}
func firstAdultLooseImage(dir, kind string) string {
if dir == "" {
return ""
+4 -4
View File
@@ -17,12 +17,12 @@ func ReadLocalMetadata(mediaPath, libraryRoot string, seriesLike bool) (*LocalMe
doc, path, err := findMovieNFO(mediaPath, libraryRoot)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return metadataFromArtwork(mediaPath, ""), nil
return metadataFromArtwork(mediaPath, "", libraryRoot), nil
}
return nil, err
}
meta := metadataFromDoc(doc, filepath.Dir(path), false)
mergeArtworkMetadata(meta, mediaPath, filepath.Dir(path))
mergeArtworkMetadata(meta, mediaPath, filepath.Dir(path), libraryRoot)
// A sidecar written while resolution tokens were misread as SxxExx must not
// reintroduce the bogus season/episode on rescan.
dropResolutionArtifactEpisodeIdentity(meta, mediaPath)
@@ -115,9 +115,9 @@ func readSeriesMetadata(mediaPath, libraryRoot string) (*LocalMetadata, error) {
}
}
if meta == nil {
meta = metadataFromArtwork(mediaPath, showBaseDir)
meta = metadataFromArtwork(mediaPath, showBaseDir, libraryRoot)
} else {
mergeArtworkMetadata(meta, mediaPath, showBaseDir)
mergeArtworkMetadata(meta, mediaPath, showBaseDir, libraryRoot)
}
// A sidecar written while resolution tokens were misread as SxxExx must not
// reintroduce the bogus season/episode on rescan.
+8 -7
View File
@@ -12,13 +12,14 @@ import (
// MediaService offers high-level CRUD over libraries and media items.
type MediaService struct {
cfg *config.Config
log *zap.Logger
repo *repository.Container
cache *RuntimeCacheService
groupedMediaFlight singleflight.Group
libraryRowsFlight singleflight.Group
recentSeriesFlight singleflight.Group
cfg *config.Config
log *zap.Logger
repo *repository.Container
cache *RuntimeCacheService
groupedMediaFlight singleflight.Group
mediaSortFlight singleflight.Group
libraryRowsFlight singleflight.Group
recentSeriesFlight singleflight.Group
}
type MediaVisibility struct {
+31 -16
View File
@@ -5,7 +5,7 @@ import (
"strings"
"time"
"go.uber.org/zap"
"github.com/google/uuid"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
@@ -129,6 +129,26 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
visibility MediaVisibility,
filters MediaListFilters,
) ([]MediaItem, error) {
value, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
if err != nil {
return nil, err
}
return value.Items, nil
}
// Each immutable grouping has its own revision. Lightweight sort indices can
// reference that revision without retaining another copy of the full rows.
type groupedMediaCacheValue struct {
Items []MediaItem
Revision string
}
func (s *MediaService) groupedMediaVisibleFiltered(
ctx context.Context,
libraryID string,
visibility MediaVisibility,
filters MediaListFilters,
) (*groupedMediaCacheValue, error) {
visibility = ExpandMediaVisibilityForMergedCloudLibraries(ctx, s.repo, visibility)
libraryIDs, err := MergedLibraryIDsForLibrary(ctx, s.repo, libraryID)
if err != nil {
@@ -143,7 +163,7 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
value, err, _ := s.groupedMediaFlight.Do(itemsCacheKey, func() (any, error) {
if s.cache != nil {
if cachedObj, ok := s.cache.GetObject(itemsCacheKey); ok {
if cached, ok := cachedObj.([]MediaItem); ok {
if cached, ok := cachedObj.(*groupedMediaCacheValue); ok {
return cached, nil
}
}
@@ -154,8 +174,8 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
if err != nil {
return nil, err
}
grouped := groupMediaVersions(items)
if s.cache != nil && len(grouped) > 0 {
grouped := &groupedMediaCacheValue{Items: groupMediaVersions(items), Revision: uuid.NewString()}
if s.cache != nil && len(grouped.Items) > 0 {
s.cache.SetObject(itemsCacheKey, grouped, s.mediaObjectTTL())
}
return grouped, nil
@@ -163,10 +183,10 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
if err != nil {
return nil, err
}
if grouped, ok := value.([]MediaItem); ok {
if grouped, ok := value.(*groupedMediaCacheValue); ok {
return grouped, nil
}
return nil, nil
return &groupedMediaCacheValue{}, nil
}
func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryID string, visibility MediaVisibility, filters MediaListFilters) ([]model.Media, error) {
@@ -180,24 +200,19 @@ func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryI
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
})
// 版本分组的 URL 分页发生在 Go 进程内,响应里的 total 是分组后的数量,
// 不需要数据库再为原始行做一次 COUNT(*)。全量 COUNT 在超大媒体库上
// 会重复扫描整个 library_id 范围,而这里只关心是否存在截断风险。
items, err := s.repo.Media.ListByLibrariesFilteredNoCount(ctx, libraryIDs, 0, maxMediaSearchLimit, filter)
// Group before paginating so versions stay together. The search safety limit
// must not truncate a library listing or its reported grouped total.
items, err := s.repo.Media.ListAllByLibrariesFilteredNoCount(ctx, libraryIDs, filter)
if err != nil {
return nil, err
}
if len(items) >= maxMediaSearchLimit && s.log != nil {
s.log.Warn("media version grouping may be truncated by safety limit",
zap.String("library_id", libraryID),
zap.Int("limit", maxMediaSearchLimit))
}
s.attachLibraryMetadata(ctx, items)
return items, nil
}
// GetMedia returns a single media row.
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) { media, err := s.repo.Media.FindByID(ctx, id)
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) {
media, err := s.repo.Media.FindByID(ctx, id)
if err != nil || media == nil {
return media, err
}
+77
View File
@@ -0,0 +1,77 @@
package service
import (
"context"
"fmt"
"sort"
"time"
)
// ListMediaVisibleSortedPage reuses grouped rows and a small position index.
// Only the requested page is copied; pagination does not sort the full library
// again or retain a second full-size MediaItem slice for every sort order.
func (s *MediaService) ListMediaVisibleSortedPage(
ctx context.Context,
libraryID string,
page, pageSize int,
visibility MediaVisibility,
filters MediaListFilters,
spec MediaSortSpec,
history map[string]time.Time,
) ([]MediaItem, int64, error) {
grouped, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
if err != nil {
return nil, 0, err
}
spec = NormalizeMediaSort(spec.Field, spec.Order)
indices := s.sortedMediaIndices(grouped, spec, history)
page, pageSize = normalizeGroupedMediaPage(page, pageSize)
total := len(grouped.Items)
// Division avoids overflowing (page-1)*pageSize on an excessive page value.
if total == 0 || page-1 > (total-1)/pageSize {
return []MediaItem{}, int64(total), nil
}
start := (page - 1) * pageSize
end := min(start+pageSize, total)
items := make([]MediaItem, 0, end-start)
for _, index := range indices[start:end] {
items = append(items, grouped.Items[index])
}
return items, int64(total), nil
}
func (s *MediaService) sortedMediaIndices(grouped *groupedMediaCacheValue, spec MediaSortSpec, history map[string]time.Time) []int {
// Playback history changes independently of the media cache and is personal.
// Keep this explicit sort live rather than caching another user's history.
if s.cache == nil || spec.Field == "last_played" || len(grouped.Items) == 0 {
return sortMediaIndices(grouped.Items, spec, history)
}
key := fmt.Sprintf("media:obj:sort:%s:%s:%s", grouped.Revision, spec.Field, spec.Order)
value, _, _ := s.mediaSortFlight.Do(key, func() (any, error) {
if cached, ok := s.cache.GetObject(key); ok {
if indices, ok := cached.([]int); ok {
return indices, nil
}
}
indices := sortMediaIndices(grouped.Items, spec, history)
s.cache.SetObject(key, indices, s.mediaObjectTTL())
return indices, nil
})
return value.([]int)
}
func sortMediaIndices(items []MediaItem, spec MediaSortSpec, history map[string]time.Time) []int {
indices := make([]int, len(items))
for i := range indices {
indices[i] = i
}
sort.SliceStable(indices, func(i, j int) bool {
a, b := items[indices[i]], items[indices[j]]
cmp := compareMediaByField(a, b, spec, history)
if cmp == 0 {
cmp = compareID(a.ID, b.ID)
}
return cmp < 0
})
return indices
}
+141
View File
@@ -0,0 +1,141 @@
package service
import (
"fmt"
"math"
"testing"
"time"
"go.uber.org/zap"
"github.com/truewhile/MeBox/internal/config"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
)
func TestSortedMediaPageIncludesRowsBeyondSearchLimit(t *testing.T) {
db := newServiceTestDB(t)
repos := repository.New(db)
lib := model.Library{Name: "Large", Path: "/media/large", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
// This must cross the historical 50,000-row boundary: a smaller fixture
// cannot detect truncation before grouping, counting, and sorting.
now := time.Now()
for start := 0; start <= maxMediaSearchLimit; start += 500 {
var rows []model.Media
for i := start; i < min(start+500, maxMediaSearchLimit+1); i++ {
rows = append(rows, model.Media{
Base: model.Base{ID: fmt.Sprintf("large-%05d", i), CreatedAt: now, UpdatedAt: now},
LibraryID: lib.ID, Path: fmt.Sprintf("/media/large/%05d.mkv", i),
Title: fmt.Sprintf("Movie %05d", i), TMDbID: i + 1,
})
}
if err := db.CreateInBatches(&rows, 100).Error; err != nil {
t.Fatal(err)
}
}
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos)
items, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, maxMediaSearchLimit/100+1, 100,
MediaVisibility{IncludeNSFW: true}, MediaListFilters{}, MediaSortSpec{Field: "title", Order: "asc"}, nil)
if err != nil {
t.Fatal(err)
}
if total != int64(maxMediaSearchLimit+1) || len(items) != 1 || items[0].ID != "large-50000" {
t.Fatalf("total=%d, page length=%d, want 50001 rows and final item large-50000", total, len(items))
}
}
func TestSortedMediaPagesReuseIndicesAndRefreshAfterInvalidation(t *testing.T) {
db := newServiceTestDB(t)
repos := repository.New(db)
lib := model.Library{Name: "Movies", Path: "/media/movies", Type: "movie", Enabled: true}
if err := repos.Library.Create(t.Context(), &lib); err != nil {
t.Fatal(err)
}
rows := []model.Media{
{Base: model.Base{ID: "alpha"}, LibraryID: lib.ID, Path: "/media/movies/a.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 100},
{Base: model.Base{ID: "alpha-alt"}, LibraryID: lib.ID, Path: "/media/movies/a-alt.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 50},
{Base: model.Base{ID: "beta"}, LibraryID: lib.ID, Path: "/media/movies/b.mkv", Title: "Beta", TMDbID: 2},
{Base: model.Base{ID: "hidden"}, LibraryID: lib.ID, Path: "/media/movies/hidden.mkv", Title: "Hidden", TMDbID: 3, NSFW: true},
}
if err := db.Create(&rows).Error; err != nil {
t.Fatal(err)
}
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos).
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
visibility := MediaVisibility{IncludeNSFW: false}
spec := MediaSortSpec{Field: "title", Order: "asc"}
first, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
if err != nil || total != 2 || len(first) != 1 || first[0].ID != "alpha" || len(first[0].Versions) != 2 {
t.Fatalf("first=%#v, total=%d, error=%v", first, total, err)
}
grouped, err := svc.groupedMediaVisibleFiltered(t.Context(), lib.ID, visibility, MediaListFilters{})
if err != nil {
t.Fatal(err)
}
index := svc.sortedMediaIndices(grouped, spec, nil)
second, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 2, 1, visibility, MediaListFilters{}, spec, nil)
if err != nil || len(second) != 1 || second[0].ID != "beta" {
t.Fatalf("second=%#v, error=%v", second, err)
}
cachedIndex := svc.sortedMediaIndices(grouped, spec, nil)
if &index[0] != &cachedIndex[0] {
t.Fatal("pagination rebuilt the sorted position index")
}
// The returned page is independently writable without corrupting cached rows.
first[0].Title = "Modified by caller"
if err := db.Model(&model.Media{}).Where("id = ?", "beta").Update("title", "Aardvark").Error; err != nil {
t.Fatal(err)
}
svc.invalidateMediaCache(t.Context())
fresh, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
if err != nil || len(fresh) != 1 || fresh[0].ID != "beta" {
t.Fatalf("updated page=%#v, error=%v", fresh, err)
}
empty, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, math.MaxInt, 100, visibility, MediaListFilters{}, spec, nil)
if err != nil || empty == nil || len(empty) != 0 {
t.Fatalf("out-of-range page=%#v, error=%v", empty, err)
}
}
func TestMediaSortIndexDoesNotCachePersonalPlaybackHistory(t *testing.T) {
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
grouped := &groupedMediaCacheValue{Revision: "history-test", Items: []MediaItem{
{Media: model.Media{Base: model.Base{ID: "a"}}},
{Media: model.Media{Base: model.Base{ID: "b"}}},
}}
spec := MediaSortSpec{Field: "last_played", Order: "desc"}
first := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"a": time.Now()})
second := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"b": time.Now()})
if first[0] != 0 || second[0] != 1 {
t.Fatalf("history sorts = %v, %v", first, second)
}
}
func BenchmarkMediaSortIndex(b *testing.B) {
items := make([]MediaItem, 10000)
for i := range items {
items[i].ID = fmt.Sprint(i)
items[i].Title = fmt.Sprintf("Movie %05d", len(items)-i)
}
spec := MediaSortSpec{Field: "title", Order: "asc"}
b.Run("sort-every-page", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
SortMediaItems(items, spec.Field, spec.Order, nil)
}
})
b.Run("cached-position-index", func(b *testing.B) {
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
grouped := &groupedMediaCacheValue{Items: items, Revision: "benchmark"}
svc.sortedMediaIndices(grouped, spec, nil)
b.ReportAllocs()
for b.Loop() {
svc.sortedMediaIndices(grouped, spec, nil)
}
})
}
@@ -3,11 +3,13 @@ package service
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"go.uber.org/zap"
@@ -15,6 +17,7 @@ import (
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
"github.com/truewhile/MeBox/internal/service/cloud"
"github.com/truewhile/MeBox/internal/service/cloud115"
)
// directRedirectTestRepo 建一个带网盘账号表的库:normalizeCloudPlayTarget 需要
@@ -145,6 +148,53 @@ func TestServeFileFallsBackWhenDirectResolveUnavailable(t *testing.T) {
}
}
// `?refresh=1`:客户端手上的直链已被 CDN 403,服务端必须丢掉缓存重新换链,
// 不能把同一条死链再发一次;不带该参数时缓存照旧命中,保住快路径。
func TestServeFileRefreshDropsCachedDirectLink(t *testing.T) {
repos := directRedirectTestRepo(t)
seedCloudSTRMMedia(t, repos, "cloud-refresh", "/api/strm/play/cloud115/video.mkv?acct=a1&pickcode=pc-refresh")
cached := fmt.Sprintf("https://cdn.example/stale.mkv?t=%d", time.Now().Add(30*time.Minute).Unix())
fresh := "https://cdn.example/fresh.mkv?t=2"
cloud115.SetDownloadURLCache("pc-refresh", cached, "RodelPlayer/2.0")
t.Cleanup(func() { cloud115.ClearDownloadURLCache("pc-refresh") })
oldMinAge := directLinkRefreshMinAge
directLinkRefreshMinAge = 0
defer func() { directLinkRefreshMinAge = oldMinAge }()
run := func(t *testing.T, query string) (cachedAtResolve, location string) {
t.Helper()
svc := NewStreamService(&config.Config{}, zap.NewNop(), repos, nil).
SetStrmPlayTargetResolver(func(_ context.Context, _ string, userAgent string) (*StrmPlayResult, error) {
cachedAtResolve = cloud115.GetDownloadURLCache("pc-refresh", userAgent)
return &StrmPlayResult{
RedirectURL: fresh,
Link: &cloud.DirectLink{URL: fresh, Headers: map[string]string{"User-Agent": userAgent}},
}, nil
})
req := httptest.NewRequest(http.MethodGet, "http://nas.local:18080/api/stream/cloud-refresh?token=jwt123"+query, nil)
req.Header.Set("User-Agent", "RodelPlayer/2.0")
w := httptest.NewRecorder()
if err := svc.ServeFile(w, req, "cloud-refresh"); err != nil {
t.Fatalf("ServeFile: %v", err)
}
return cachedAtResolve, w.Header().Get("Location")
}
// 正常播放:缓存里的直链照旧命中,服务端不该白跑一次换链。
if got, loc := run(t, ""); got != cached {
t.Fatalf("cached link = %q, want the cached %q", got, cached)
} else if loc != fresh {
t.Fatalf("Location = %q, want the resolved direct link", loc)
}
// 客户端报告直链失效:换链前必须先清掉缓存。
if got, loc := run(t, "&refresh=1"); got != "" {
t.Fatalf("cached link must be dropped before resolving, got %q", got)
} else if loc != fresh {
t.Fatalf("Location = %q, want the freshly resolved direct link", loc)
}
}
// 没有注入解析器(测试/精简部署)时保持原有跳转,不受本次优化影响。
func TestServeFileKeepsSTRMEndpointWithoutResolver(t *testing.T) {
repos := directRedirectTestRepo(t)
+47
View File
@@ -10,6 +10,7 @@ import (
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
"github.com/truewhile/MeBox/internal/service/cloud115"
)
// ServeFile streams the file backing the given media ID using
@@ -22,6 +23,47 @@ func (s *StreamService) ServeFile(w http.ResponseWriter, r *http.Request, mediaI
return s.ServeFileWithCloudMode(w, r, mediaID, "")
}
// PickCodeFromPlayTarget 从 /api/strm/play/... 形式的播放目标里取出 pickcode。
// 115 直链缓存按 pickcode 分键,客户端要求强制换链时用它定位缓存条目。
func PickCodeFromPlayTarget(raw string) string {
u, err := url.Parse(strings.TrimSpace(raw))
if err != nil || u == nil {
return ""
}
return strings.TrimSpace(u.Query().Get("pickcode"))
}
// directLinkRefreshMinAge 是「强制换链」的最小间隔:缓存条目刚写入不久就
// 收到换链请求时忽略它,避免客户端重试风暴把 115 换链接口打出突发。
// 测试会临时改成 0,让「CDN 拒绝 → 立刻换链」的路径可验证。
var directLinkRefreshMinAge = 30 * time.Second
// wantsFreshLink 判断客户端是否要求跳过直链缓存重新换链(`?refresh=1`)。
//
// 115 直链只有 30 分钟有效期,客户端拿到过期直链被 CDN 403 后需要一条新链;
// 正常路径下缓存会先于直链过期,这里是客户端上报失败时的兜底。
func wantsFreshLink(r *http.Request) bool {
if r == nil {
return false
}
switch strings.ToLower(strings.TrimSpace(r.URL.Query().Get("refresh"))) {
case "1", "true", "yes":
return true
default:
return false
}
}
// InvalidateMediaLinkCache 清除媒体播放目标对应的云盘直链缓存,让下一次解析
// 重新换链。非云盘/无 pickcode 的媒体是空操作。
func InvalidateMediaLinkCache(m *model.Media) {
pickCode := PickCodeFromPlayTarget(MediaSTRMTarget(m))
if pickCode == "" {
return
}
cloud115.ClearDownloadURLCacheIfOlderThan(pickCode, directLinkRefreshMinAge)
}
// MediaSTRMTarget 返回媒体行固化的 STRM 播放目标。STRMURL 为空时回读本地
// .strm 文件内容兜底(扫描时内容解析失败的行只剩 Container=strm + Path),
// 仍拿不到返回空串。
@@ -57,6 +99,11 @@ func (s *StreamService) ServeFileWithCloudMode(w http.ResponseWriter, r *http.Re
// 能在服务端换到最终直链就直接 302 过去:客户端少跟随一次 302,等于
// 省掉一次「DNS+TCP+TLS+请求」的往返。解析结果同时写进 strm 层直链
// 缓存,后续 /api/strm/play 请求直接命中。
// `?refresh=1` 表示客户端手上的直链已经失效(被 CDN 403),要求丢掉
// 缓存重新换链。
if wantsFreshLink(r) {
InvalidateMediaLinkCache(m)
}
if direct, ok := s.resolveDirectPlayTargetURL(r, strmURL); ok {
setCloudRedirectNoStore(w)
http.Redirect(w, r, direct, http.StatusFound)
+82 -11
View File
@@ -16,6 +16,7 @@ import (
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/service/cloud"
"github.com/truewhile/MeBox/internal/service/cloud115"
)
// ErrStrmPlayNotFound 表示 strm 播放目标不存在(handler 返回 404)。
@@ -250,21 +251,87 @@ func (s *StrmService) ProxyMediaDirect(ctx context.Context, w http.ResponseWrite
if err != nil {
return err
}
switch {
case result.Link != nil && result.Link.URL != "":
return s.ProxyDirect(ctx, w, r, result.Link)
case result.RedirectURL != "":
return s.ProxyDirect(ctx, w, r, &cloud.DirectLink{URL: result.RedirectURL})
default:
link := directLinkOfPlayResult(result)
if link == nil {
// 本地文件(LocalPath)由静态文件处理器提供,本身就是同源。
return ErrStrmProxyNotApplicable
}
resp, method, err := s.openDirectUpstream(ctx, r, link)
if err != nil {
return err
}
// 直链被 CDN 拒绝:115 直链带 30 分钟过期时间戳,缓存命中也可能拿到已经
// 失效的地址。这里清掉缓存重新换链再试一次,避免客户端只看到 403。
if directLinkRejected(resp.StatusCode) {
_ = resp.Body.Close()
if fresh := s.refreshDirectLink(ctx, raw); fresh != nil {
if retry, retryMethod, retryErr := s.openDirectUpstream(ctx, r, fresh); retryErr == nil {
resp, method = retry, retryMethod
}
}
}
defer resp.Body.Close()
return writeDirectUpstream(w, resp, method)
}
// directLinkOfPlayResult 取出可反向代理的直链;本地文件返回 nil。
func directLinkOfPlayResult(result *StrmPlayResult) *cloud.DirectLink {
if result == nil {
return nil
}
switch {
case result.Link != nil && result.Link.URL != "":
return result.Link
case result.RedirectURL != "":
return &cloud.DirectLink{URL: result.RedirectURL}
default:
return nil
}
}
// directLinkRejected 判断上游状态码是否属于「直链本身失效」(而不是文件/权限
// 之类与链接无关的问题)。
func directLinkRejected(status int) bool {
switch status {
case http.StatusForbidden, http.StatusNotFound, http.StatusGone:
return true
default:
return false
}
}
// refreshDirectLink 丢弃当前直链缓存并重新换链,返回新的直链;无法换链时返回
// nil,调用方继续用原来那条。
func (s *StrmService) refreshDirectLink(ctx context.Context, raw string) *cloud.DirectLink {
pickCode := PickCodeFromPlayTarget(raw)
if pickCode == "" {
return nil
}
if !cloud115.ClearDownloadURLCacheIfOlderThan(pickCode, directLinkRefreshMinAge) {
// 缓存刚写入不久:说明失败与链接过期无关,重换一次也会拿到同一条。
return nil
}
fresh, err := s.ResolvePlayTarget(ctx, raw)
if err != nil {
return nil
}
return directLinkOfPlayResult(fresh)
}
// ProxyDirect 反向代理渲染直链内容(保留 Range 请求头以支持拖动播放)。
func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r *http.Request, link *cloud.DirectLink) error {
resp, method, err := s.openDirectUpstream(ctx, r, link)
if err != nil {
return err
}
defer resp.Body.Close()
return writeDirectUpstream(w, resp, method)
}
// openDirectUpstream 向网盘直链发起一次请求,保留 Range 与 UA 语义。
func (s *StrmService) openDirectUpstream(ctx context.Context, r *http.Request, link *cloud.DirectLink) (*http.Response, string, error) {
if link == nil || link.URL == "" {
return errors.New("空直链")
return nil, "", errors.New("空直链")
}
method := http.MethodGet
if r != nil && r.Method == http.MethodHead {
@@ -272,7 +339,7 @@ func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r
}
req, err := http.NewRequestWithContext(ctx, method, link.URL, nil)
if err != nil {
return err
return nil, "", err
}
for k, v := range link.Headers {
req.Header.Set(k, v)
@@ -288,9 +355,13 @@ func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r
}
resp, err := s.streamHTTP.Do(req)
if err != nil {
return err
return nil, "", err
}
defer resp.Body.Close()
return resp, method, nil
}
// writeDirectUpstream 把上游响应原样写给客户端。
func writeDirectUpstream(w http.ResponseWriter, resp *http.Response, method string) error {
for _, header := range []string{"Content-Type", "Content-Length", "Content-Range", "Accept-Ranges", "ETag", "Last-Modified"} {
if value := resp.Header.Get(header); value != "" {
w.Header().Set(header, value)
@@ -310,7 +381,7 @@ func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r
if flusher, ok := w.(http.Flusher); ok {
writer = &flushWriter{writer: w, flusher: flusher}
}
_, err = io.Copy(writer, resp.Body)
_, err := io.Copy(writer, resp.Body)
return err
}
+132
View File
@@ -3,12 +3,16 @@ package service
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"sync"
"testing"
"time"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/service/cloud115"
)
// 同源转发必须原样保留 Range 语义:把 206 改写成 200 会让浏览器误判响应长度,
@@ -55,6 +59,134 @@ func TestProxyMediaDirectForwardsRangeAndStatus(t *testing.T) {
}
}
// 115 直链只有 30 分钟寿命,缓存命中也可能拿到已经失效的地址。同源转发遇到
// CDN 403 时必须丢掉缓存重新换链再试一次,而不是把 403 直接透给客户端。
func TestProxyMediaDirectRetriesWithFreshLinkWhenCDNRejects(t *testing.T) {
svc := testStrmService(t)
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
t.Fatal(err)
}
cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/fresh.mkv" {
w.WriteHeader(http.StatusForbidden)
return
}
w.Header().Set("Content-Type", "video/x-matroska")
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write([]byte("fresh-bytes"))
}))
defer cdn.Close()
// 第一次换链给出「已过期」的旧地址,重新换链才给新地址。
var mu sync.Mutex
var calls int
expiry := time.Now().Add(30 * time.Minute).Unix()
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
calls++
attempt := calls
mu.Unlock()
link := cdn.URL + "/old.mkv"
if attempt > 1 {
link = cdn.URL + "/fresh.mkv"
}
_, _ = fmt.Fprintf(w, `{"state":true,"data":{"1":{"pick_code":"pc-retry","url":{"url":%q}}}}`,
fmt.Sprintf("%s?t=%d&u=1&k=abc", link, expiry))
}))
defer api.Close()
oldPro := cloud115.ProAPIBase
cloud115.ProAPIBase = api.URL
defer func() { cloud115.ProAPIBase = oldPro }()
defer cloud115.ClearDownloadURLCache("pc-retry")
oldMinAge := directLinkRefreshMinAge
directLinkRefreshMinAge = 0
defer func() { directLinkRefreshMinAge = oldMinAge }()
media := &model.Media{
STRMURL: "/api/strm/play/cloud115/video.mkv?acct=" + acct.ID + "&pickcode=pc-retry",
}
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/stream/media-1?proxy=1", nil)
req.Header.Set("User-Agent", "MeBoxTest/1.0")
if err := svc.ProxyMediaDirect(context.Background(), rec, req, media); err != nil {
t.Fatalf("ProxyMediaDirect: %v", err)
}
if rec.Code != http.StatusPartialContent {
t.Fatalf("status = %d, want 206 (body=%s)", rec.Code, rec.Body.String())
}
if rec.Body.String() != "fresh-bytes" {
t.Fatalf("body = %q, want the fresh link payload", rec.Body.String())
}
mu.Lock()
defer mu.Unlock()
if calls != 2 {
t.Fatalf("downurl calls = %d, want 2 (one retry after the CDN rejected the stale link)", calls)
}
}
// 换链后依然是同一条失效直链时,保持原有行为:原样透传 CDN 的状态码。
func TestProxyMediaDirectPassesRejectedStatusWhenRefreshCannotHelp(t *testing.T) {
svc := testStrmService(t)
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
t.Fatal(err)
}
cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
}))
defer cdn.Close()
expiry := time.Now().Add(30 * time.Minute).Unix()
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, `{"state":true,"data":{"1":{"pick_code":"pc-dead","url":{"url":%q}}}}`,
fmt.Sprintf("%s/dead.mkv?t=%d&u=1&k=abc", cdn.URL, expiry))
}))
defer api.Close()
oldPro := cloud115.ProAPIBase
cloud115.ProAPIBase = api.URL
defer func() { cloud115.ProAPIBase = oldPro }()
defer cloud115.ClearDownloadURLCache("pc-dead")
oldMinAge := directLinkRefreshMinAge
directLinkRefreshMinAge = 0
defer func() { directLinkRefreshMinAge = oldMinAge }()
media := &model.Media{
STRMURL: "/api/strm/play/cloud115/video.mkv?acct=" + acct.ID + "&pickcode=pc-dead",
}
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/stream/media-1?proxy=1", nil)
if err := svc.ProxyMediaDirect(context.Background(), rec, req, media); err != nil {
t.Fatalf("ProxyMediaDirect: %v", err)
}
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want the upstream 403 passed through", rec.Code)
}
}
// 强制换链只认 /api/strm/play 形式播放目标里的 pickcode。
func TestPickCodeFromPlayTarget(t *testing.T) {
cases := []struct {
raw string
want string
}{
{"/api/strm/play/cloud115/video.mkv?acct=a1&pickcode=pc1", "pc1"},
{"https://nas.local/api/strm/play/cloud115/video.mkv?pickcode=pc2&acct=a1", "pc2"},
{"/api/strm/play/cloud115/video.mkv?acct=a1", ""},
{"https://cdn.example.test/Movie.mkv?sign=1", ""},
{"", ""},
}
for _, tc := range cases {
if got := PickCodeFromPlayTarget(tc.raw); got != tc.want {
t.Fatalf("PickCodeFromPlayTarget(%q) = %q, want %q", tc.raw, got, tc.want)
}
}
}
// 本地文件本身就是同源资源,不应该被代理(调用方按原静态文件逻辑处理)。
func TestProxyMediaDirectSkipsLocalFile(t *testing.T) {
svc := testStrmService(t)
+10 -2
View File
@@ -170,8 +170,16 @@ func (s *TokenService) storeRefreshToken(ctx context.Context, rt *model.RefreshT
if err := s.repo.RefreshToken.Create(ctx, rt); err != nil {
return err
}
if err := s.repo.RefreshToken.RevokeOldestActiveByUserID(ctx, rt.UserID, s.maxActiveRefreshTokens(ctx)); err != nil && s.log != nil {
s.log.Warn("failed to enforce refresh token session limit", zap.String("user_id", rt.UserID), zap.Error(err))
// 「最大同时登录终端」只约束未绑定设备会话的旧客户端。
//
// 移动端每台设备有独立认证会话,刷新令牌归属该会话;如果仍按用户统一
// 裁剪活跃令牌,第 N+1 台设备登录(或任一设备刷新)就会把最早那台设备
// 的刷新令牌标记为已撤销,那台设备下次刷新收到 SESSION_REVOKED,被迫
// 重新登录。设备数量由设备策略单独约束,不能靠撤销刷新令牌实现。
if rt.AuthSessionID == "" {
if err := s.repo.RefreshToken.RevokeOldestActiveLegacyByUserID(ctx, rt.UserID, s.maxActiveRefreshTokens(ctx)); err != nil && s.log != nil {
s.log.Warn("failed to enforce refresh token session limit", zap.String("user_id", rt.UserID), zap.Error(err))
}
}
return nil
}
+9 -2
View File
@@ -1,13 +1,20 @@
import { useEffect, type ReactNode } from 'react'
import { Fragment, useEffect, type ReactNode } from 'react'
import { Navigate, useLocation } from 'react-router-dom'
import { useAuthStore } from '../stores/auth'
import { usePlayProfileStore } from '../stores/playProfile'
import { libraryCacheScope } from '../utils/libraryCache'
import { hydrateReaderSettings, startReaderSettingsSync } from '../utils/readerSettingsSync'
// Route guard: redirects to /login when no token is present.
export function RequireAuth({ children }: { children: ReactNode }) {
const token = useAuthStore((s) => s.token)
const userId = useAuthStore((s) => s.user?.id ?? null)
// Drop local page state and in-flight callbacks when visibility changes.
useAuthStore((s) => s.user)
usePlayProfileStore((s) => s.activeProfileId)
usePlayProfileStore((s) => s.activeProfilePinToken)
const visibilityKey = libraryCacheScope()
const location = useLocation()
// 阅读器偏好按账号存在服务端:一进入受保护区域就开启回写订阅,并在
@@ -23,7 +30,7 @@ export function RequireAuth({ children }: { children: ReactNode }) {
if (!token) {
return <Navigate to="/login" replace state={{ from: location }} />
}
return <>{children}</>
return <Fragment key={visibilityKey}>{children}</Fragment>
}
// Route guard: only allows users with role === "admin".
+3 -2
View File
@@ -15,6 +15,7 @@ const SLOW_RECONNECT_INTERVAL = 60_000
export function useWebSocket(onEvent: (topic: string, payload: unknown) => void) {
const ref = useRef<WebSocket | null>(null)
const token = useAuthStore((s) => s.token)
const isAdmin = useAuthStore((s) => s.user?.role === 'admin')
const onEventRef = useRef(onEvent)
useEffect(() => {
@@ -22,7 +23,7 @@ export function useWebSocket(onEvent: (topic: string, payload: unknown) => void)
}, [onEvent])
useEffect(() => {
if (!token) return
if (!token || !isAdmin) return
let closed = false
let timer: number | undefined
let reconnectAttempts = 0
@@ -88,5 +89,5 @@ export function useWebSocket(onEvent: (topic: string, payload: unknown) => void)
if (timer) window.clearTimeout(timer)
ref.current?.close()
}
}, [token])
}, [token, isAdmin])
}
+3 -5
View File
@@ -158,11 +158,9 @@ export function useLibraryData(
try {
const resolved = await resolveLibrary(libraryID)
if (cancelled || seq !== requestSeqRef.current) return
if (!lib) {
lib = resolved
libraryRef.current = lib
setLibrary(lib)
}
lib = resolved
libraryRef.current = lib
setLibrary(lib)
} catch {
if (!cancelled && seq === requestSeqRef.current) {
setLoading(false)
+29 -83
View File
@@ -1,89 +1,35 @@
import { libraryAPI } from '../api/library'
import type { Library } from '../types'
import { useAuthStore } from '../stores/auth'
import { usePlayProfileStore } from '../stores/playProfile'
import { createLibraryCache } from './libraryCacheState'
// 会话级媒体库元数据缓存(不落 localStorage)。
// 首页 / 媒体库页 / 媒体库详情页都要先拿库列表或库信息才能渲染,
// 同一会话内反复进出这些页面时直接命中缓存,避免每次都等一整轮请求。
// TTL 内直接返回缓存;过期后重新请求并刷新。并发调用共享同一个在途请求。
const LIST_TTL_MS = 30_000
const LIBRARY_TTL_MS = 30_000
let listData: Library[] | null = null
let listAt = 0
let listInflight: Promise<Library[]> | null = null
const byId = new Map<string, { at: number; data: Library }>()
const byIdInflight = new Map<string, Promise<Library>>()
function fresh(at: number, ttl: number): boolean {
return Date.now() - at < ttl
export function libraryCacheScope(): string {
const { token, user } = useAuthStore.getState()
const profile = usePlayProfileStore.getState()
return JSON.stringify([
Boolean(token),
user?.id,
user?.role,
user?.hide_adult,
user?.allowed_library_ids,
profile.activeProfileId,
profile.activeProfilePinToken,
])
}
function rememberList(data: Library[]) {
listData = data
listAt = Date.now()
for (const lib of data) {
byId.set(lib.id, { at: listAt, data: lib })
}
}
const cache = createLibraryCache({
list: () => libraryAPI.list(),
get: (id) => libraryAPI.get(id),
readScope: libraryCacheScope,
})
// fetchLibraries 拉取库列表:TTL 内命中缓存立即返回(含 peek 先行渲染),
// 过期或未缓存时发起请求,并发调用共享在途请求。
export function fetchLibraries(): Promise<Library[]> {
if (listData && fresh(listAt, LIST_TTL_MS)) return Promise.resolve(listData)
if (!listInflight) {
listInflight = libraryAPI
.list()
.then((rows) => {
const data = Array.isArray(rows) ? rows : []
rememberList(data)
return data
})
.finally(() => {
listInflight = null
})
}
return listInflight
}
// Invalidate synchronously, including logout/login between two reads and auth
// rehydration from another tab. Refreshing a token alone preserves the cache.
useAuthStore.subscribe(() => cache.syncScope())
usePlayProfileStore.subscribe(() => cache.syncScope())
// peekLibraries 同步返回当前缓存的库列表(可能过期),没有则 null。
// 页面用它先行渲染上一次的数据,再由 fetchLibraries 在后台补一次刷新。
export function peekLibraries(): Library[] | null {
return listData
}
// peekLibrary 同步返回单库缓存(库列表缓存也会回填单库索引)。
export function peekLibrary(id: string): Library | null {
const entry = byId.get(id)
return entry?.data ?? null
}
// resolveLibrary 拿单个库信息:命中缓存直接返回;未命中走 /libraries/:id
// 并回填缓存。远程 Emby 挂载库的详情接口同样返回 is_remote_emby,缓存可用。
export function resolveLibrary(id: string): Promise<Library> {
const entry = byId.get(id)
if (entry && fresh(entry.at, LIBRARY_TTL_MS)) return Promise.resolve(entry.data)
const inflight = byIdInflight.get(id)
if (inflight) return inflight
const request = libraryAPI
.get(id)
.then((lib) => {
byId.set(id, { at: Date.now(), data: lib })
return lib
})
.finally(() => {
byIdInflight.delete(id)
})
byIdInflight.set(id, request)
return request
}
// invalidate 清空缓存。管理页对库做过增删改后调用,确保下次拉到新数据。
export function invalidateLibraries() {
listData = null
listAt = 0
listInflight = null
byId.clear()
byIdInflight.clear()
}
export const fetchLibraries = cache.fetchLibraries
export const peekLibraries = cache.peekLibraries
export const peekLibrary = cache.peekLibrary
export const resolveLibrary = cache.resolveLibrary
export const invalidateLibraries = cache.invalidate
+107
View File
@@ -0,0 +1,107 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { createLibraryCache } from './libraryCacheState.ts'
import type { Library } from '../types/library.ts'
const library = (id: string) => ({ id, name: id } as Library)
function deferred<T>() {
let resolve!: (value: T) => void
const promise = new Promise<T>((done) => { resolve = done })
return { promise, resolve }
}
test('list requests share a flight and populate the detail cache', async () => {
const request = deferred<Library[]>()
let calls = 0
const cache = createLibraryCache({
readScope: () => 'account-a:profile-a',
list: () => { calls++; return request.promise },
get: async () => { throw new Error('detail should be cached') },
})
const first = cache.fetchLibraries()
assert.equal(first, cache.fetchLibraries())
request.resolve([library('a')])
await first
assert.equal((await cache.resolveLibrary('a')).id, 'a')
assert.equal((await cache.fetchLibraries())[0].id, 'a')
assert.equal(calls, 1)
})
test('account and profile changes discard both list and detail data', async () => {
let scope = 'account-a:profile-a'
const cache = createLibraryCache({
readScope: () => scope,
list: async () => [library(scope)],
get: async () => library(scope),
})
await cache.fetchLibraries()
scope = 'account-b:profile-a'
assert.equal(cache.peekLibraries(), null)
assert.equal(cache.peekLibrary('account-a:profile-a'), null)
assert.equal((await cache.fetchLibraries())[0].id, scope)
scope = 'account-b:profile-b'
assert.equal(cache.peekLibraries(), null)
assert.equal((await cache.resolveLibrary('shared-id')).id, scope)
})
test('invalidation rejects old list results without clearing the new flight', async () => {
const old = deferred<Library[]>()
const fresh = deferred<Library[]>()
let calls = 0
const cache = createLibraryCache({
readScope: () => 'a',
list: () => ++calls === 1 ? old.promise : fresh.promise,
get: async (id) => library(id),
})
const oldRequest = cache.fetchLibraries()
const rejected = assert.rejects(oldRequest, { name: 'AbortError' })
cache.invalidate()
const current = cache.fetchLibraries()
old.resolve([library('deleted')])
await rejected
assert.equal(cache.peekLibraries(), null)
assert.equal(cache.fetchLibraries(), current)
fresh.resolve([library('current')])
await current
assert.equal(cache.peekLibraries()?.[0].id, 'current')
assert.equal(calls, 2)
})
test('invalidation rejects old detail results without clearing the new flight', async () => {
const old = deferred<Library>()
const fresh = deferred<Library>()
let calls = 0
const cache = createLibraryCache({
readScope: () => 'a',
list: async () => [],
get: () => ++calls === 1 ? old.promise : fresh.promise,
})
const oldRequest = cache.resolveLibrary('same-id')
const rejected = assert.rejects(oldRequest, { name: 'AbortError' })
cache.invalidate()
const current = cache.resolveLibrary('same-id')
old.resolve(library('old'))
await rejected
assert.equal(cache.peekLibrary('same-id'), null)
assert.equal(cache.resolveLibrary('same-id'), current)
fresh.resolve(library('current'))
await current
assert.equal(cache.peekLibrary('same-id')?.id, 'current')
})
test('logout followed by login to the same account invalidates old requests', async () => {
let scope = 'account-a'
const request = deferred<Library[]>()
const cache = createLibraryCache({ readScope: () => scope, list: () => request.promise, get: async libraryID => library(libraryID) })
const pending = cache.fetchLibraries()
const rejected = assert.rejects(pending, { name: 'AbortError' })
scope = 'logged-out'
cache.syncScope()
scope = 'account-a'
cache.syncScope()
request.resolve([library('old-session')])
await rejected
assert.equal(cache.peekLibraries(), null)
})
+106
View File
@@ -0,0 +1,106 @@
import type { Library } from '../types/library.ts'
const TTL_MS = 30_000
// Kept independent of the API and auth stores so races can be tested with
// deferred requests. Only the active visibility scope is retained in memory.
export function createLibraryCache({
list,
get,
readScope,
}: {
list: () => Promise<Library[]>
get: (id: string) => Promise<Library>
readScope: () => string
}) {
let scope = readScope()
let generation = 0
let listData: Library[] | null = null
let listAt = 0
let listInflight: Promise<Library[]> | null = null
const byId = new Map<string, { at: number; data: Library }>()
const byIdInflight = new Map<string, Promise<Library>>()
function invalidate() {
generation += 1
listData = null
listAt = 0
listInflight = null
byId.clear()
byIdInflight.clear()
}
function syncScope() {
const next = readScope()
if (scope !== next) {
scope = next
invalidate()
}
}
function checkGeneration(expected: number) {
syncScope()
if (generation !== expected) {
const error = new Error('Library request belongs to an invalidated cache')
error.name = 'AbortError'
throw error
}
}
function fetchLibraries(): Promise<Library[]> {
syncScope()
if (listData && Date.now() - listAt < TTL_MS) return Promise.resolve(listData)
if (listInflight) return listInflight
const expected = generation
const request = list()
.then((rows) => {
checkGeneration(expected)
const data = Array.isArray(rows) ? rows : []
listData = data
listAt = Date.now()
byId.clear()
for (const lib of data) byId.set(lib.id, { at: listAt, data: lib })
return data
})
.finally(() => {
if (listInflight === request) listInflight = null
})
listInflight = request
return request
}
function resolveLibrary(id: string): Promise<Library> {
syncScope()
const entry = byId.get(id)
if (entry && Date.now() - entry.at < TTL_MS) return Promise.resolve(entry.data)
const inflight = byIdInflight.get(id)
if (inflight) return inflight
const expected = generation
const request = get(id)
.then((lib) => {
checkGeneration(expected)
byId.set(id, { at: Date.now(), data: lib })
return lib
})
.finally(() => {
if (byIdInflight.get(id) === request) byIdInflight.delete(id)
})
byIdInflight.set(id, request)
return request
}
return {
fetchLibraries,
resolveLibrary,
invalidate,
syncScope,
peekLibraries: () => {
syncScope()
return listData
},
peekLibrary: (id: string) => {
syncScope()
return byId.get(id)?.data ?? null
},
}
}