mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-15 20:34:23 +08:00
Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 74487359bf | |||
| 55854a0adb | |||
| 121a0a3cc5 | |||
| 3057664a48 | |||
| 7793d50906 | |||
| 5ad71b9421 | |||
| 5615806c2c | |||
| 1e8fe2396e | |||
| e34e81cce0 | |||
| 3cf898756b | |||
| 0b68397265 | |||
| 90bfac3b09 | |||
| f1974a179e | |||
| f2357200be | |||
| ca9a80b862 | |||
| 4f19e2210e |
@@ -79,6 +79,9 @@ func (a *application) start() error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := database.ApplyPendingSQLiteRestore(context.Background(), a.cfg, a.logger); err != nil {
|
||||
return fmt.Errorf("apply pending database restore: %w", err)
|
||||
}
|
||||
db, err := database.Open(a.cfg, a.logger)
|
||||
if err != nil {
|
||||
a.logger.Error("database open failed", zap.Error(err))
|
||||
|
||||
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,46 @@
|
||||
# Playback Correctness and Isolation Implementation Plan
|
||||
|
||||
**Goal:** Fix the five playback defects from the 2026-10-21 review and make playback capability queries side-effect-free, without replacing the player or adding dependencies.
|
||||
|
||||
**Scope:** The user approved implementing the reviewed fixes. Work in the current clean checkout; do not commit, push, deploy, or alter unrelated files.
|
||||
|
||||
## Design and invariants
|
||||
|
||||
- Keep existing media IDs, stream routes, direct playback and external-player URLs compatible.
|
||||
- Add optional `session_id` to local HLS requests. Authenticated handlers supply the actual user identity; a client parameter never supplies the owner. New web playback sessions use random IDs. Legacy clients are isolated by user and, where available, Emby play-session/device identity.
|
||||
- Keep transcoder media lookup/logging separate from its internal task key. A task key incorporates user, media and playback session. Each actual ffmpeg run writes to a fresh generation directory, so delayed segment requests never read a replacement run's output.
|
||||
- Stop only the authenticated caller's playback session. Check media visibility and scoped playback tokens on stop, just as on playlist/segment requests. Segment and playlist queries must match the running generation.
|
||||
- Cloud child requests resolve their real session media first, compare query media, enforce token scope and visibility against the real media, and reject mismatches before upstream fetch.
|
||||
- Treat player business positions as absolute source seconds. Only local HLS converts to/from a relative ffmpeg timeline.
|
||||
- Invalidate outstanding cloud-transcode requests on media/unmount, explicit source/quality changes, and newer requests. Guard both fulfilled and rejected responses.
|
||||
- Select local fallbacks only from available local qualities. Keep the existing 115-first preference when a cloud quality is available.
|
||||
- `PlaybackInfo` only reads capabilities/status. Explicit `StartTranscode` requests enqueue cloud work. Retain pending/ready response contracts, polling, and retries.
|
||||
|
||||
## Tasks
|
||||
|
||||
1. Add regression tests for local-scope isolation, stale generations, stop authorization, cloud media binding, pure capability reads, timeline conversion, available-local selection and stale async results. Run tests before implementation and verify the intended failures.
|
||||
2. Add tested playback helpers; route skip/undo/progress/quality/subtitle calculations through the common absolute timeline; guard cloud transcode results and fix local fallback choice.
|
||||
3. Extend existing transcoder/stream methods with an optional scope, preserve legacy internal calls, isolate task keys/output directories, and pass caller scope from web and Emby handlers. Bind web local HLS URLs and stop requests to captured session IDs.
|
||||
4. Enforce real-media authorization on cloud child requests and authenticated ownership on local stop.
|
||||
5. Split read-only cloud capability lookup from explicit cloud enqueue. Update existing cloud tests to use the explicit start operation and test no push during reads.
|
||||
6. Run focused and full Go tests, frontend tests, TypeScript/Vite build, lint diagnostics and review the diff for security, lifecycle and API compatibility. Fix regressions before reporting completion.
|
||||
|
||||
## Verification commands
|
||||
|
||||
- `node --experimental-strip-types --test src/pages/hlsPlayback.test.ts src/pages/directPlaySeek.test.ts src/pages/playerPageModel.test.ts src/pages/playbackQuality.test.ts src/pages/playbackRequestGuard.test.ts src/utils/skipSegments.test.ts`
|
||||
- `go test ./internal/service ./internal/handler ./internal/middleware -count=1`
|
||||
- `go test ./...`
|
||||
- `npm run build` and targeted ESLint for changed frontend files.
|
||||
|
||||
## Out of scope
|
||||
|
||||
New remux/audio-only ffmpeg pipelines, speculative browser codec policy changes, player replacement, broad UI refactors, performance improvement claims without measurements, and production deployment.
|
||||
|
||||
## Verification outcome
|
||||
|
||||
- Six frontend playback regression scripts pass; targeted ESLint and the TypeScript/Vite production build pass. The existing bundle-size warning remains.
|
||||
- Playback/stream/transcode/progress tests in service, handler and middleware pass, including scoped output reads, stale generation rejection, scoped stops, device-to-segment propagation, read-only capability queries and concurrent cloud start deduplication.
|
||||
- Whole-repository tests pass with only `TestQueueLibraryRootScanImportsNewLibraryMedia` and `TestSQLiteRestoreStagesWithoutChangingLiveDatabase` excluded. Both were run independently and reproduce Windows file-lock / SQLite errors in unmodified scan/database code. They are not silently disabled in source.
|
||||
- FFmpeg is not installed in this environment, so the existing real-transcode lifecycle test skips. Actual video, upstream 115 and browser end-to-end playback remain unverified.
|
||||
- Clients with no playback-session or device identifier retain per-user legacy isolation; new web requests always carry an independent session ID.
|
||||
- No commit, push or deployment was performed.
|
||||
@@ -0,0 +1,189 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
var sqliteRestoreMu sync.Mutex
|
||||
|
||||
func sqliteRestorePath(cfg *config.Config) (string, error) {
|
||||
if cfg == nil || strings.TrimSpace(cfg.Database.DBPath) == "" || cfg.Database.DBPath == ":memory:" || strings.HasPrefix(cfg.Database.DBPath, "file:") {
|
||||
return "", errors.New("恢复需要配置有效的 SQLite 数据库文件路径")
|
||||
}
|
||||
return filepath.Abs(cfg.Database.DBPath)
|
||||
}
|
||||
|
||||
func sqliteRestoreDSN(path, mode string) string {
|
||||
u := url.URL{Scheme: "file", Path: filepath.ToSlash(path)}
|
||||
q := url.Values{"mode": {mode}, "_pragma": {"busy_timeout(5000)"}}
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func validateSQLiteRestore(ctx context.Context, path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return errors.New("备份必须是常规文件")
|
||||
}
|
||||
db, err := sql.Open("sqlite", sqliteRestoreDSN(path, "ro"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
var result string
|
||||
if err := db.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&result); err != nil {
|
||||
return fmt.Errorf("SQLite 备份校验失败: %w", err)
|
||||
}
|
||||
if result != "ok" {
|
||||
return fmt.Errorf("SQLite 备份校验失败: %s", result)
|
||||
}
|
||||
var tables int
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('users', 'media', 'settings')").Scan(&tables); err != nil {
|
||||
return err
|
||||
}
|
||||
if tables == 0 {
|
||||
return errors.New("备份不包含 MeBox 数据表")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StageSQLiteRestore leaves the live database and its WAL untouched. The
|
||||
// validated copy is applied on the next startup, before any database is opened.
|
||||
func StageSQLiteRestore(ctx context.Context, cfg *config.Config, source string) error {
|
||||
sqliteRestoreMu.Lock()
|
||||
defer sqliteRestoreMu.Unlock()
|
||||
dbPath, err := sqliteRestorePath(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pending := dbPath + ".restore-pending"
|
||||
if _, err := os.Lstat(pending); err == nil {
|
||||
return errors.New("已有等待恢复的备份,请重启 MeBox 后再操作")
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, source); err != nil {
|
||||
return err
|
||||
}
|
||||
in, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.CreateTemp(filepath.Dir(dbPath), ".mebox-restore-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := out.Name()
|
||||
defer os.Remove(tmp)
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, pending)
|
||||
}
|
||||
|
||||
// ApplyPendingSQLiteRestore must run before database.Open and before workers or
|
||||
// HTTP listeners start. The previous database is retained alongside the file.
|
||||
func ApplyPendingSQLiteRestore(ctx context.Context, cfg *config.Config, log *zap.Logger) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
dialect := normalizeDatabaseType(cfg.Database.Type)
|
||||
if dialect == "auto" {
|
||||
dialect = effectiveAutoDatabaseType(cfg)
|
||||
}
|
||||
if dialect != "sqlite" {
|
||||
return nil
|
||||
}
|
||||
sqliteRestoreMu.Lock()
|
||||
defer sqliteRestoreMu.Unlock()
|
||||
dbPath, err := sqliteRestorePath(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pending := dbPath + ".restore-pending"
|
||||
if _, err := os.Lstat(pending); errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, pending); err != nil {
|
||||
return err
|
||||
}
|
||||
previous := ""
|
||||
if _, err := os.Lstat(dbPath); err == nil {
|
||||
// A clean checkpoint prevents an old WAL from being replayed into the
|
||||
// replacement database. A busy checkpoint leaves all files untouched.
|
||||
live, err := sql.Open("sqlite", sqliteRestoreDSN(dbPath, "rw"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var busy, pages, copied int
|
||||
err = live.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &pages, &copied)
|
||||
closeErr := live.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if busy != 0 {
|
||||
return errors.New("SQLite 数据库仍被占用,恢复已保留等待下次启动")
|
||||
}
|
||||
old, err := os.CreateTemp(filepath.Dir(dbPath), filepath.Base(dbPath)+".before-restore-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
previous = old.Name()
|
||||
if err := old.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(previous); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(dbPath, previous); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(pending, dbPath); err != nil {
|
||||
if previous != "" {
|
||||
if rollbackErr := os.Rename(previous, dbPath); rollbackErr != nil {
|
||||
return fmt.Errorf("恢复失败: %w;回退失败: %v,原数据库保留于 %s", err, rollbackErr, previous)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
if log != nil {
|
||||
log.Warn("SQLite backup restored before startup", zap.String("previous_database", previous))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
func TestSQLiteRestoreStagesWithoutChangingLiveDatabase(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.Database.Type = "sqlite"
|
||||
cfg.Database.DBPath = filepath.Join(root, "live.db")
|
||||
live, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "rwc"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer live.Close()
|
||||
for _, statement := range []string{
|
||||
"PRAGMA journal_mode=WAL", "CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)",
|
||||
"INSERT INTO settings VALUES ('state', 'backup')",
|
||||
} {
|
||||
if _, err := live.Exec(statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
source := filepath.Join(root, "backup.db")
|
||||
if _, err := live.Exec("VACUUM INTO ?", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := live.Exec("UPDATE settings SET value = 'live'"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value string
|
||||
if err := live.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "live" {
|
||||
t.Fatalf("staging changed live database: value=%q err=%v", value, err)
|
||||
}
|
||||
if _, err := live.Exec("UPDATE settings SET value = 'after-stage'"); err != nil {
|
||||
t.Fatalf("database could not write after staging: %v", err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
|
||||
t.Fatal("second staging should not replace the pending restore")
|
||||
}
|
||||
if err := os.Remove(source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := live.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restored, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "ro"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer restored.Close()
|
||||
if err := restored.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "backup" {
|
||||
t.Fatalf("restore failed: value=%q err=%v", value, err)
|
||||
}
|
||||
previous, err := filepath.Glob(cfg.Database.DBPath + ".before-restore-*")
|
||||
if err != nil || len(previous) != 1 {
|
||||
t.Fatalf("previous database not preserved: files=%v err=%v", previous, err)
|
||||
}
|
||||
old, err := sql.Open("sqlite", sqliteRestoreDSN(previous[0], "ro"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer old.Close()
|
||||
if err := old.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "after-stage" {
|
||||
t.Fatalf("old database lost latest writes: value=%q err=%v", value, err)
|
||||
}
|
||||
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
|
||||
t.Fatalf("repeat startup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteRestoreRejectsCorruptAndSymlinkBackups(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.Database.DBPath = filepath.Join(root, "live.db")
|
||||
source := filepath.Join(root, "corrupt.db")
|
||||
if err := os.WriteFile(source, []byte("not a SQLite database"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
|
||||
t.Fatal("corrupt backup accepted")
|
||||
}
|
||||
link := filepath.Join(root, "link.db")
|
||||
if err := os.Symlink(source, link); err == nil {
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, link); err == nil {
|
||||
t.Fatal("symlink backup accepted")
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(cfg.Database.DBPath + ".restore-pending"); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed validation left pending data: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func backgroundTasksHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if svc == nil || svc.Tasks == nil {
|
||||
c.JSON(http.StatusOK, service.TaskSnapshot{Active: []service.BackgroundTask{}, Recent: []service.BackgroundTask{}})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, svc.Tasks.Snapshot())
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
@@ -41,12 +42,28 @@ func deleteBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func downloadBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
filename := c.Query("filename")
|
||||
path, err := svc.Backup.ResolvePath(filename)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.FileAttachment(path, filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
func restoreBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.Backup.Restore(c.Request.Context(), c.Query("filename")); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"message": "restored — restart the server to apply"})
|
||||
message := "数据库已恢复,请重启 MeBox 以刷新运行时配置"
|
||||
if svc.Repo.DB.Dialector.Name() == "sqlite" {
|
||||
message = "备份已校验并暂存,请重启 MeBox 后应用恢复"
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"message": message, "restart_required": true})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,10 +124,32 @@ func cloud115HLSSessionHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "missing media_id"})
|
||||
return
|
||||
}
|
||||
if !enforceScopedPlaybackToken(c, mediaID) {
|
||||
if c.Request.Context().Err() != nil {
|
||||
c.AbortWithStatus(499)
|
||||
return
|
||||
}
|
||||
err := svc.Cloud115.HLSProxy().ServeChild(
|
||||
actualMediaID, err := svc.Cloud115.HLSProxy().SessionMediaID(c.Param("session"), c.Param("key"))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"code": "hls_session_not_found", "error": "hls session not found"})
|
||||
return
|
||||
}
|
||||
if mediaID != actualMediaID {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "playback session media mismatch"})
|
||||
return
|
||||
}
|
||||
if !enforceScopedPlaybackToken(c, actualMediaID) {
|
||||
return
|
||||
}
|
||||
if svc.Repo == nil || svc.Repo.Media == nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "media not found"})
|
||||
return
|
||||
}
|
||||
m, findErr := svc.Repo.Media.FindByID(c.Request.Context(), actualMediaID)
|
||||
if findErr != nil || m == nil || !mediaVisibleForRequest(c, svc, m) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "media not found"})
|
||||
return
|
||||
}
|
||||
err = svc.Cloud115.HLSProxy().ServeChild(
|
||||
c.Request.Context(),
|
||||
c.Writer,
|
||||
c.Request,
|
||||
|
||||
@@ -418,7 +418,11 @@ func embyVideoHLSPlaylistHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"))
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"), scope)
|
||||
if errors.Is(err, service.ErrTranscodeDisabled) {
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "transcode disabled"})
|
||||
return
|
||||
@@ -444,7 +448,11 @@ func embyVideoHLSSegmentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if err := svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg")); err != nil {
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg"), scope); err != nil {
|
||||
c.Status(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func TestStopHLSChecksMediaVisibility(t *testing.T) {
|
||||
router, svc, secret := newPlaybackScopeTestRouter(t)
|
||||
svc.Transcoder = service.NewTranscoderService(svc.Cfg, zap.NewNop(), svc.Repo, nil)
|
||||
router.DELETE("/api/hls/:id", middleware.AuthRequired(secret), stopTranscodeHandler(svc))
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/hls/nonexistent?session_id=player-a", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+signedTestToken(t, secret))
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("stop of a nonexistent/invisible media: status=%d body=%s, want 404", rec.Code, rec.Body.String())
|
||||
}
|
||||
profile := &model.PlayProfile{UserID: "user-1", Name: "restricted", IsDefault: true, AllowedLibraryIDs: `["other-library"]`}
|
||||
if err := svc.Repo.DB.Create(profile).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodDelete, "/api/hls/media-1?session_id=player-a", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+signedTestToken(t, secret))
|
||||
rec = httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("invisible existing media can be stopped: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
+257
-113
@@ -4,8 +4,10 @@ package handler
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -40,6 +42,11 @@ type webLibraryPayload struct {
|
||||
Cards []service.SeriesCardView `json:"cards,omitempty"`
|
||||
}
|
||||
|
||||
// remoteLibraryPreviewBudget 限制 /api/libraries?with_preview=1 里远程 Emby
|
||||
// 库预览(总条数 + 最新卡片)的总耗时。这些预览只用于首页卡片,值得为它
|
||||
// 排队等待的量很小;此前它跟随请求上下文,上游超时会让整页等 15 秒以上。
|
||||
var remoteLibraryPreviewBudget = 3 * time.Second
|
||||
|
||||
// remoteLibraryItemTypes 远程库内容拉取时按 CollectionType 过滤直属条目,
|
||||
// 避免电影库里的合集文件夹(Folder) 漏出为电影卡片。
|
||||
func remoteLibraryItemTypes(collectionType string) string {
|
||||
@@ -176,6 +183,12 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
remotePayloads[i] = webLibraryPayload{Library: v.Library, IsRemoteEmby: true, RemoteSource: v.AccountName}
|
||||
}
|
||||
if withPreview && len(allowedViews) > 0 {
|
||||
// 远程库预览只是首页卡片上的「最新几条」,绝不能拖住整个
|
||||
// 库列表:上游 Emby 一旦超时,每个远程库都要等满
|
||||
// embyRemoteHTTPTimeout(15s),串起来就是 17–30 秒。
|
||||
// 给整段预览一个总预算,超时就只返回库名,卡片留给下一次刷新。
|
||||
previewCtx, cancelPreview := context.WithTimeout(ctx, remoteLibraryPreviewBudget)
|
||||
defer cancelPreview()
|
||||
const maxRemotePreviewWorkers = 6
|
||||
sem := make(chan struct{}, maxRemotePreviewWorkers)
|
||||
var wg sync.WaitGroup
|
||||
@@ -187,11 +200,11 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
defer func() { <-sem }()
|
||||
case <-ctx.Done():
|
||||
case <-previewCtx.Done():
|
||||
return
|
||||
}
|
||||
helper.Run(svc.Log, "media.remotePreview", func() {
|
||||
acct := svc.EmbyRemote.AccountByID(ctx, v.AccountID)
|
||||
acct := svc.EmbyRemote.AccountByID(previewCtx, v.AccountID)
|
||||
if acct == nil {
|
||||
return
|
||||
}
|
||||
@@ -203,10 +216,10 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
Name: v.Library.Name,
|
||||
}
|
||||
itemTypes := remoteLibraryItemTypes(v.CollectionType)
|
||||
if _, total, err := svc.EmbyRemote.RemoteLibraryMedia(ctx, tmpMount, acct, v.RemoteID, itemTypes, 0, 1); err == nil {
|
||||
if _, total, err := svc.EmbyRemote.RemoteLibraryMedia(previewCtx, tmpMount, acct, v.RemoteID, itemTypes, 0, 1); err == nil {
|
||||
remotePayloads[i].Total = total
|
||||
}
|
||||
if cards, err := svc.EmbyRemote.RemoteLatestCards(ctx, tmpMount, acct, v.RemoteID, limit); err == nil {
|
||||
if cards, err := svc.EmbyRemote.RemoteLatestCards(previewCtx, tmpMount, acct, v.RemoteID, limit); err == nil {
|
||||
remotePayloads[i].Cards = service.NewSeriesCardViews(cards)
|
||||
}
|
||||
})
|
||||
@@ -556,19 +569,17 @@ func listMediaHandler(svc *service.Container) gin.HandlerFunc {
|
||||
})
|
||||
return
|
||||
}
|
||||
grouped, err := svc.Media.GroupedMediaVisibleFiltered(ctx, id, mediaVisibilityForRequest(c, svc), filters)
|
||||
items, total, err := svc.Media.ListMediaVisibleSortedPage(ctx, id, page, size, mediaVisibilityForRequest(c, svc), filters, sortSpec, history)
|
||||
if err != nil {
|
||||
writeInternalOrCanceled(c, err)
|
||||
return
|
||||
}
|
||||
grouped = service.SortMediaItems(grouped, sortSpec.Field, sortSpec.Order, history)
|
||||
items := service.PaginateMediaItems(grouped, page, size)
|
||||
if items == nil {
|
||||
items = []service.MediaItem{}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": items,
|
||||
"total": len(grouped),
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
@@ -656,6 +667,9 @@ func paginateSlice[T any](items []T, page, size int) []T {
|
||||
if len(items) == 0 {
|
||||
return []T{}
|
||||
}
|
||||
if page-1 > len(items)/size {
|
||||
return []T{}
|
||||
}
|
||||
start := (page - 1) * size
|
||||
if start >= len(items) {
|
||||
return []T{}
|
||||
@@ -670,127 +684,257 @@ func paginateSlice[T any](items []T, page, size int) []T {
|
||||
func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
q := c.Query("q")
|
||||
query := strings.TrimSpace(c.Query("q"))
|
||||
visibility := mediaVisibilityForRequest(c, svc)
|
||||
groupVersions := c.DefaultQuery("group_versions", "1") != "0"
|
||||
|
||||
fetchRemote := func(limit int) []model.Media {
|
||||
if svc.EmbyRemote == nil || strings.TrimSpace(q) == "" {
|
||||
return nil
|
||||
}
|
||||
remoteItems, _ := svc.EmbyRemote.RemoteSearchMedia(ctx, q, limit, visibility)
|
||||
return remoteItems
|
||||
filter := service.MediaSearchFilter{
|
||||
LibraryID: strings.TrimSpace(c.Query("library_id")),
|
||||
YearMin: boundedSearchYear(c.Query("year_min")),
|
||||
YearMax: boundedSearchYear(c.Query("year_max")),
|
||||
}
|
||||
|
||||
if c.DefaultQuery("group_series", "0") != "0" {
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
for _, genre := range c.QueryArray("genre") {
|
||||
if genre = strings.TrimSpace(genre); genre != "" && len(filter.Genres) < 8 {
|
||||
filter.Genres = append(filter.Genres, genre)
|
||||
}
|
||||
remoteItems := fetchRemote(50000)
|
||||
all := service.GroupMediaSeriesItems(append(localItems, remoteItems...))
|
||||
|
||||
if c.Query("page") != "" || c.Query("page_size") != "" {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
paged := paginateSlice(all, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(all),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": all})
|
||||
return
|
||||
}
|
||||
|
||||
if c.Query("page") != "" || c.Query("page_size") != "" {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if !groupVersions {
|
||||
localItems, _, err := svc.Media.SearchMediaVisiblePage(ctx, q, 1, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(size * 2)
|
||||
all := append(localItems, remoteItems...)
|
||||
paged := paginateSlice(all, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(all),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(size * 2)
|
||||
all := append(localItems, remoteItems...)
|
||||
grouped := service.GroupMediaVersions(all)
|
||||
paged := service.PaginateMediaItems(grouped, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(grouped),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
if filter.YearMin > 0 && filter.YearMax > 0 && filter.YearMin > filter.YearMax {
|
||||
filter.YearMin, filter.YearMax = filter.YearMax, filter.YearMin
|
||||
}
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if size < 1 {
|
||||
size = 50
|
||||
} else if size > 2000 {
|
||||
size = 2000
|
||||
}
|
||||
|
||||
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||
if limit <= 0 {
|
||||
if limit < 1 {
|
||||
limit = 50
|
||||
} else if limit > 50000 {
|
||||
limit = 50000
|
||||
}
|
||||
if !groupVersions {
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, limit, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(limit)
|
||||
all := append(localItems, remoteItems...)
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
if all == nil {
|
||||
all = []model.Media{}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": all})
|
||||
return
|
||||
}
|
||||
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
paginated := c.Query("page") != "" || c.Query("page_size") != ""
|
||||
sortBy := normalizeMediaSearchSort(c.Query("sort"))
|
||||
// 远程搜索与本地查询并行发起:本地结果不再排在远程超时之后才返回。
|
||||
remoteCh := startRemoteMediaSearch(ctx, svc, query, visibility, filter)
|
||||
// Sorting and grouping must operate on the same candidate set before
|
||||
// either a page size or the caller's output limit is applied.
|
||||
items, err := svc.Media.SearchMediaVisibleFiltered(ctx, query, 50000, visibility, filter)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(limit)
|
||||
all := append(localItems, remoteItems...)
|
||||
grouped := service.GroupMediaVersions(all)
|
||||
items := service.FirstMediaItems(grouped, limit)
|
||||
if items == nil {
|
||||
items = []service.MediaItem{}
|
||||
if remoteCh != nil {
|
||||
for _, item := range <-remoteCh {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items})
|
||||
if c.DefaultQuery("group_series", "0") != "0" {
|
||||
items = service.GroupMediaSeriesItems(items)
|
||||
sortSearchMedia(items, sortBy)
|
||||
writeMediaSearchResults(c, items, page, size, limit, paginated)
|
||||
return
|
||||
}
|
||||
if c.DefaultQuery("group_versions", "1") != "0" {
|
||||
grouped := service.GroupMediaVersions(items)
|
||||
if sortBy != "" {
|
||||
sort.SliceStable(grouped, func(i, j int) bool {
|
||||
return mediaSearchSortLess(grouped[i].Media, grouped[j].Media, sortBy)
|
||||
})
|
||||
}
|
||||
writeMediaSearchResults(c, grouped, page, size, limit, paginated)
|
||||
return
|
||||
}
|
||||
sortSearchMedia(items, sortBy)
|
||||
writeMediaSearchResults(c, items, page, size, limit, paginated)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
// remoteSearchItemLimit 是每个远程 Emby 挂载点返回的条目上限。此前这里传
|
||||
// 50000(等于让上游做一次全库查询),上游稍慢就会一路跑到 5 秒超时,把每次
|
||||
// 搜索都拖成秒级;搜索场景只需要少量候选,多出来的结果也会被分页丢掉。
|
||||
remoteSearchItemLimit = 100
|
||||
// remoteSearchBudget 限制远程结果最多给一次请求增加多少延迟。
|
||||
remoteSearchBudget = 2 * time.Second
|
||||
// remoteSearchCacheTTL 让同一关键词的翻页与重复搜索复用远程结果,避免每翻
|
||||
// 一页都重新打一遍远程挂载点。
|
||||
remoteSearchCacheTTL = 30 * time.Second
|
||||
// remoteSearchFailureCacheTTL 用于上游超时/失败时的短缓存:部分结果只保留
|
||||
// 几秒,避免反复吃满 remoteSearchBudget,又不至于长期压住后续重试。
|
||||
remoteSearchFailureCacheTTL = 5 * time.Second
|
||||
)
|
||||
|
||||
// startRemoteMediaSearch 并发发起远程搜索,返回一个只投递一次结果的通道。
|
||||
// 不满足远程搜索条件时返回 nil。通道带缓冲,调用方即使提前返回也不会泄漏
|
||||
// goroutine;请求结束时请求上下文被取消,远程调用会随之退出。
|
||||
func startRemoteMediaSearch(
|
||||
ctx context.Context,
|
||||
svc *service.Container,
|
||||
query string,
|
||||
visibility service.MediaVisibility,
|
||||
filter service.MediaSearchFilter,
|
||||
) <-chan []model.Media {
|
||||
if svc == nil || svc.EmbyRemote == nil || query == "" {
|
||||
return nil
|
||||
}
|
||||
if filter.LibraryID != "" && !service.IsEmbyRemoteID(filter.LibraryID) {
|
||||
return nil
|
||||
}
|
||||
results := make(chan []model.Media, 1)
|
||||
go func() {
|
||||
results <- remoteMediaSearchItems(ctx, svc, query, visibility, filter)
|
||||
}()
|
||||
return results
|
||||
}
|
||||
|
||||
func remoteMediaSearchItems(
|
||||
ctx context.Context,
|
||||
svc *service.Container,
|
||||
query string,
|
||||
visibility service.MediaVisibility,
|
||||
filter service.MediaSearchFilter,
|
||||
) []model.Media {
|
||||
cacheKey := remoteSearchCacheKey(query, visibility)
|
||||
if svc.Cache != nil {
|
||||
if cached, ok := svc.Cache.GetObject(cacheKey); ok {
|
||||
if rows, ok := cached.([]model.Media); ok {
|
||||
return filterRemoteSearchItems(rows, filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
searchCtx, cancel := context.WithTimeout(ctx, remoteSearchBudget)
|
||||
defer cancel()
|
||||
remote, _ := svc.EmbyRemote.RemoteSearchMedia(searchCtx, query, remoteSearchItemLimit, visibility)
|
||||
if svc.Cache != nil {
|
||||
// 空结果也要缓存:否则查不到片名时每次都会重新等到 remoteSearchBudget。
|
||||
if remote == nil {
|
||||
remote = []model.Media{}
|
||||
}
|
||||
ttl := remoteSearchCacheTTL
|
||||
if searchCtx.Err() != nil {
|
||||
ttl = remoteSearchFailureCacheTTL
|
||||
}
|
||||
svc.Cache.SetObject(cacheKey, remote, ttl)
|
||||
}
|
||||
return filterRemoteSearchItems(remote, filter)
|
||||
}
|
||||
|
||||
func filterRemoteSearchItems(remote []model.Media, filter service.MediaSearchFilter) []model.Media {
|
||||
if len(remote) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]model.Media, 0, len(remote))
|
||||
for _, item := range remote {
|
||||
if remoteSearchMatchesFilter(item, filter) {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// remoteSearchCacheKey 按可见范围隔离远程搜索结果,避免把某个用户可见挂载
|
||||
// 点的结果复用给可见范围不同的用户。
|
||||
func remoteSearchCacheKey(query string, visibility service.MediaVisibility) string {
|
||||
allowed := append([]string(nil), visibility.AllowedLibraryIDs...)
|
||||
hidden := append([]string(nil), visibility.HiddenLibraryIDs...)
|
||||
sort.Strings(allowed)
|
||||
sort.Strings(hidden)
|
||||
return fmt.Sprintf("media:obj:remote-search:%t|%s|%s|%s",
|
||||
visibility.IncludeNSFW, strings.Join(allowed, ","), strings.Join(hidden, ","), query)
|
||||
}
|
||||
|
||||
func writeMediaSearchResults[T any](c *gin.Context, items []T, page, size, limit int, paginated bool) {
|
||||
if items == nil {
|
||||
items = []T{}
|
||||
}
|
||||
if paginated {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paginateSlice(items, page, size),
|
||||
"total": len(items), "page": page, "page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items})
|
||||
}
|
||||
|
||||
func remoteSearchMatchesFilter(item model.Media, filter service.MediaSearchFilter) bool {
|
||||
if filter.LibraryID != "" && item.LibraryID != filter.LibraryID && item.DisplayLibraryID != filter.LibraryID {
|
||||
return false
|
||||
}
|
||||
if filter.YearMin > 0 && item.Year < filter.YearMin || filter.YearMax > 0 && item.Year > filter.YearMax {
|
||||
return false
|
||||
}
|
||||
if len(filter.Genres) == 0 {
|
||||
return true
|
||||
}
|
||||
normalize := func(value string) string {
|
||||
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), " ", ""))
|
||||
}
|
||||
for _, genre := range strings.Split(strings.ReplaceAll(item.Genres, ",", ","), ",") {
|
||||
for _, wanted := range filter.Genres {
|
||||
if normalize(genre) == normalize(wanted) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeMediaSearchSort(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
switch value {
|
||||
case "title_asc", "year_desc", "rating_desc":
|
||||
return value
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func sortSearchMedia(items []model.Media, sortBy string) {
|
||||
if sortBy == "" {
|
||||
return
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
return mediaSearchSortLess(items[i], items[j], sortBy)
|
||||
})
|
||||
}
|
||||
|
||||
func mediaSearchSortLess(a, b model.Media, sortBy string) bool {
|
||||
switch sortBy {
|
||||
case "title_asc":
|
||||
left, right := strings.ToLower(strings.TrimSpace(a.Title)), strings.ToLower(strings.TrimSpace(b.Title))
|
||||
if left != right {
|
||||
return left < right
|
||||
}
|
||||
case "year_desc":
|
||||
if a.Year != b.Year {
|
||||
return a.Year > b.Year
|
||||
}
|
||||
case "rating_desc":
|
||||
if a.Rating != b.Rating {
|
||||
return a.Rating > b.Rating
|
||||
}
|
||||
}
|
||||
if !a.CreatedAt.Equal(b.CreatedAt) {
|
||||
return a.CreatedAt.After(b.CreatedAt)
|
||||
}
|
||||
return a.ID < b.ID
|
||||
}
|
||||
|
||||
func boundedSearchYear(value string) int {
|
||||
year, err := strconv.Atoi(strings.TrimSpace(value))
|
||||
if err != nil || year < 1870 || year > 2200 {
|
||||
return 0
|
||||
}
|
||||
return year
|
||||
}
|
||||
|
||||
func streamHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// TestListLibrariesRemotePreviewIsBounded 锁定远程库预览的总预算:
|
||||
// 上游 Emby 卡住时,/api/libraries?with_preview=1 必须按时返回远程库条目
|
||||
// (只是没有卡片),而不是跟着上游一起等十几秒。线上实测这个接口因此出现过
|
||||
// 17.4 秒、30 秒(前端放弃)的响应,设置页也跟着整页空转。
|
||||
func TestListLibrariesRemotePreviewIsBounded(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
oldBudget := remoteLibraryPreviewBudget
|
||||
remoteLibraryPreviewBudget = 200 * time.Millisecond
|
||||
t.Cleanup(func() { remoteLibraryPreviewBudget = oldBudget })
|
||||
|
||||
// Views / 用户列表正常返回,Preview(Items 查询)永远不返回:模拟上游半挂。
|
||||
release := make(chan struct{})
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
path := r.URL.Path
|
||||
switch {
|
||||
case strings.HasSuffix(path, "/Views"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Items": []map[string]any{{
|
||||
"Id": "view-1", "Name": "远程动画", "CollectionType": "tvshows",
|
||||
}},
|
||||
})
|
||||
return
|
||||
case strings.HasSuffix(path, "/Users"):
|
||||
// api_key 账号要先解析出远程用户 Id,否则请求路径会落到兜底 "0"。
|
||||
_ = json.NewEncoder(w).Encode([]map[string]any{{"Id": "uid-1", "Name": "admin"}})
|
||||
return
|
||||
case r.Method == http.MethodPost:
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"AccessToken": "fake-token",
|
||||
"User": map[string]any{"Id": "uid-1"},
|
||||
})
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-release:
|
||||
case <-r.Context().Done():
|
||||
case <-time.After(30 * time.Second):
|
||||
}
|
||||
w.WriteHeader(http.StatusGatewayTimeout)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
defer close(release)
|
||||
|
||||
// 唯一库名 + cache=shared:预览协程会并发读库(账号解析)。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.StrmAccount{}, &model.EmbyMount{}, &model.Setting{}, &model.User{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
if err := db.Create(&model.User{Base: model.Base{ID: "admin-1"}, Username: "admin", Role: "admin"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawConfig, _ := json.Marshal(map[string]string{"url": upstream.URL, "token": "fake-token"})
|
||||
acct := model.StrmAccount{
|
||||
Base: model.Base{ID: "acct-1"}, Name: "远程Emby", Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig), Enabled: true,
|
||||
}
|
||||
if err := repos.StrmAccount.Create(t.Context(), &acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mount := model.EmbyMount{
|
||||
Base: model.Base{ID: "mount-1"}, AccountID: acct.ID, RemoteViewID: "view-1",
|
||||
RemoteViewName: "远程动画", CollectionType: "tvshows", Enabled: true,
|
||||
}
|
||||
if err := repos.EmbyMount.Create(t.Context(), &mount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := &service.Container{
|
||||
Repo: repos,
|
||||
Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos),
|
||||
EmbyRemote: service.NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, service.NewCryptoService("", zap.NewNop())),
|
||||
Cache: service.NewRuntimeCacheService(&config.Config{}, zap.NewNop()),
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Set(middleware.CtxUserID, "admin-1")
|
||||
c.Set(middleware.CtxUserRole, "admin")
|
||||
c.Request = httptest.NewRequest(http.MethodGet, "/api/libraries?with_preview=1&include_hidden=1", nil)
|
||||
|
||||
start := time.Now()
|
||||
listLibrariesHandler(svc)(c)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if elapsed > 2*time.Second {
|
||||
t.Fatalf("request took %v with a hung upstream; the preview budget must bound it", elapsed)
|
||||
}
|
||||
var payload []webLibraryPayload
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("decode response: %v (%s)", err, w.Body.String())
|
||||
}
|
||||
var remote *webLibraryPayload
|
||||
for i := range payload {
|
||||
if payload[i].IsRemoteEmby {
|
||||
remote = &payload[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if remote == nil {
|
||||
t.Fatalf("remote library missing from the bounded response: %#v", payload)
|
||||
}
|
||||
if len(remote.Cards) != 0 || remote.Total != 0 {
|
||||
t.Fatalf("expected the preview to be skipped on timeout, got total=%d cards=%d", remote.Total, len(remote.Cards))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func mediaQualityHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
} else if page > 1000000 {
|
||||
page = 1000000
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 50
|
||||
} else if pageSize > 100 {
|
||||
pageSize = 100
|
||||
}
|
||||
issue := c.DefaultQuery("issue", "unmatched")
|
||||
if issue != "unmatched" && issue != "poster" && issue != "missing" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "未知检查分类"})
|
||||
return
|
||||
}
|
||||
items, total, err := svc.Media.QualityIssues(c.Request.Context(), issue, page, pageSize)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items, "total": total, "page": page, "page_size": pageSize})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func TestSearchMediaSortLimitFiltersAndDeepPagination(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
defer sqlDB.Close()
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.User{}, &model.Setting{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "Review", Path: "/review", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := make([]model.Media, 2001)
|
||||
for i := range rows {
|
||||
rows[i] = model.Media{
|
||||
Base: model.Base{ID: fmt.Sprintf("review-%04d", i)}, LibraryID: lib.ID,
|
||||
Title: fmt.Sprintf("Review %04d", i), Path: fmt.Sprintf("/review/%04d.mp4", i), Year: 2020, Genres: "Comedy",
|
||||
}
|
||||
}
|
||||
rows[2000].Year = 2025
|
||||
rows[2000].Genres = "Drama"
|
||||
if err := db.CreateInBatches(rows, 100).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := &service.Container{Repo: repos, Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos)}
|
||||
cases := []struct {
|
||||
name, query, expectedID string
|
||||
count, total, page, size int
|
||||
}{
|
||||
{name: "sorted output limit", query: "sort=year_desc&limit=1", expectedID: "review-2000", count: 1},
|
||||
{name: "page past old 2000 cap", query: "sort=title_asc&page=2001&page_size=1", expectedID: "review-2000", count: 1, total: 2001, page: 2001, size: 1},
|
||||
{name: "genre filtering", query: "genre=Drama&library_id=" + lib.ID, expectedID: "review-2000", count: 1},
|
||||
{name: "normalized page", query: "page=0&page_size=0", count: 50, total: 2001, page: 1, size: 50},
|
||||
{name: "bounded page size", query: "page=1&page_size=99999", count: 2000, total: 2001, page: 1, size: 2000},
|
||||
{name: "huge page does not overflow", query: "page=" + strconv.Itoa(int(^uint(0)>>1)) + "&page_size=2000", count: 0, total: 2001, page: int(^uint(0) >> 1), size: 2000},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(w)
|
||||
ctx.Set(middleware.CtxUserID, "admin")
|
||||
ctx.Set(middleware.CtxUserRole, "admin")
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/media?q=Review&group_versions=0&"+tc.query, nil)
|
||||
searchMediaHandler(svc)(ctx)
|
||||
var response struct {
|
||||
Items []model.Media `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
Size int `json:"page_size"`
|
||||
}
|
||||
if w.Code != http.StatusOK || json.Unmarshal(w.Body.Bytes(), &response) != nil {
|
||||
t.Fatalf("search failed: status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if len(response.Items) != tc.count || response.Total != tc.total || response.Page != tc.page || response.Size != tc.size {
|
||||
t.Fatalf("response count=%d total=%d page=%d size=%d, want %#v", len(response.Items), response.Total, response.Page, response.Size, tc)
|
||||
}
|
||||
if tc.expectedID != "" && response.Items[0].ID != tc.expectedID {
|
||||
t.Fatalf("first ID=%q want %q", response.Items[0].ID, tc.expectedID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// TestSearchMediaRemoteSearchBoundedAndCached 防止远程搜索再次退化:
|
||||
// 1. 每个挂载点的 Limit 必须是 remoteSearchItemLimit,而不是历史全库查询的 50000;
|
||||
// 2. 同一关键词的重复搜索应命中缓存,不再打上游;
|
||||
// 3. 指定本地媒体库时不应枚举远程挂载点。
|
||||
func TestSearchMediaRemoteSearchBoundedAndCached(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
var mu sync.Mutex
|
||||
searchCalls := 0
|
||||
searchLimits := make([]string, 0, 4)
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if term := r.URL.Query().Get("SearchTerm"); term != "" {
|
||||
mu.Lock()
|
||||
searchCalls++
|
||||
searchLimits = append(searchLimits, r.URL.Query().Get("Limit"))
|
||||
mu.Unlock()
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"TotalRecordCount": 1,
|
||||
"Items": []map[string]any{{
|
||||
"Id": "remote-1",
|
||||
"Name": "远方剧集",
|
||||
"Type": "Series",
|
||||
"ProductionYear": 2024,
|
||||
}},
|
||||
})
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
|
||||
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.StrmAccount{}, &model.EmbyMount{}, &model.Setting{}, &model.User{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
if err := repos.DB.Create(&model.User{Base: model.Base{ID: "user-1"}, Username: "admin", Role: "admin"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lib := model.Library{Name: "本地电影", Path: "/media/movies", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.Media{
|
||||
Base: model.Base{ID: "local-1"}, LibraryID: lib.ID, Title: "流浪地球", Path: "/media/movies/流浪地球.mkv",
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawConfig, _ := json.Marshal(map[string]string{"url": upstream.URL, "token": "fake-token"})
|
||||
acct := model.StrmAccount{
|
||||
Base: model.Base{ID: "acct"}, Name: "远程Emby", Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig), Enabled: true,
|
||||
}
|
||||
if err := repos.StrmAccount.Create(t.Context(), &acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mount := model.EmbyMount{
|
||||
Base: model.Base{ID: "mount-1"}, AccountID: acct.ID, RemoteViewID: "view-1",
|
||||
RemoteViewName: "远程剧集", CollectionType: "tvshows", Enabled: true,
|
||||
}
|
||||
if err := repos.EmbyMount.Create(t.Context(), &mount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc := &service.Container{
|
||||
Repo: repos,
|
||||
Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos),
|
||||
EmbyRemote: service.NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, service.NewCryptoService("", zap.NewNop())),
|
||||
Cache: service.NewRuntimeCacheService(&config.Config{}, zap.NewNop()),
|
||||
}
|
||||
|
||||
search := func(params string) []service.MediaItem {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Set(middleware.CtxUserID, "user-1")
|
||||
c.Set(middleware.CtxUserRole, "admin")
|
||||
c.Request = httptest.NewRequest(http.MethodGet, "/api/media?"+params, nil)
|
||||
searchMediaHandler(svc)(c)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("search status=%d, body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var response struct {
|
||||
Items []service.MediaItem `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return response.Items
|
||||
}
|
||||
callCount := func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return searchCalls
|
||||
}
|
||||
|
||||
if items := search("q=远方&page=1&page_size=48"); len(items) != 1 || items[0].Title != "远方剧集" {
|
||||
t.Fatalf("expected remote result, got %#v", items)
|
||||
}
|
||||
if calls := callCount(); calls != 1 {
|
||||
t.Fatalf("expected exactly one upstream search, got %d", calls)
|
||||
}
|
||||
mu.Lock()
|
||||
limit := searchLimits[0]
|
||||
mu.Unlock()
|
||||
if limit != "100" {
|
||||
t.Fatalf("remote search Limit=%q, want bounded 100 (50000 means a full-library query)", limit)
|
||||
}
|
||||
|
||||
// 同一关键词再来一次(例如刷新/重复搜索)应复用缓存,不再请求上游。
|
||||
if items := search("q=远方&page=1&page_size=48"); len(items) != 1 || items[0].Title != "远方剧集" {
|
||||
t.Fatalf("cached search lost the remote result: %#v", items)
|
||||
}
|
||||
if calls := callCount(); calls != 1 {
|
||||
t.Fatalf("expected cached remote search to skip upstream, got %d calls", calls)
|
||||
}
|
||||
|
||||
// 指定本地媒体库时不应枚举远程挂载点。
|
||||
if items := search("q=流浪&page=1&page_size=48&library_id=" + lib.ID); len(items) != 1 || items[0].ID != "local-1" {
|
||||
t.Fatalf("expected local-only result, got %#v", items)
|
||||
}
|
||||
if calls := callCount(); calls != 1 {
|
||||
t.Fatalf("local library filter must not hit upstream, got %d calls", calls)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -27,13 +29,17 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
|
||||
"TotalRecordCount": 1,
|
||||
"Items": []map[string]any{{
|
||||
"Id": "same-series", "Name": "龙珠改", "Type": "Series",
|
||||
"ProviderIds": map[string]string{"Tmdb": "61709"},
|
||||
"ProviderIds": map[string]string{"Tmdb": "61709"},
|
||||
"ProductionYear": 2024, "Genres": []string{"Science Fiction"},
|
||||
}},
|
||||
})
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
|
||||
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -108,4 +114,9 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
|
||||
}
|
||||
seen[item.DisplayLibraryID] = true
|
||||
}
|
||||
remoteLibraryID := service.EncodeEmbyRemoteID("mount-a", "view-a")
|
||||
items, total := request("page=1&page_size=8&library_id=" + url.QueryEscape(remoteLibraryID) + "&year_min=2020&genre=Science+Fiction")
|
||||
if total != 1 || len(items) != 1 || items[0].DisplayLibraryID != remoteLibraryID {
|
||||
t.Fatalf("remote library/year/genre filters lost results: total=%d items=%#v", total, items)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -559,7 +559,10 @@ func TestSearchMediaHandlerIncludesEmbyRemote(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
|
||||
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -80,6 +80,11 @@ func playbackInfoHandler(svc *service.Container) gin.HandlerFunc {
|
||||
// stream_url 跳转,行为与改动前一致。
|
||||
directURL, hasDirectURL := "", false
|
||||
if svc.Stream != nil {
|
||||
// `?refresh=1`:客户端手上的直链已失效(CDN 403),要求丢掉缓存
|
||||
// 重新换链,别把同一条死链再发一次。
|
||||
if strings.EqualFold(strings.TrimSpace(c.Query("refresh")), "1") {
|
||||
service.InvalidateMediaLinkCache(m)
|
||||
}
|
||||
directURL, hasDirectURL = svc.Stream.ResolveDirectPlayURL(c.Request, m)
|
||||
}
|
||||
payload := gin.H{
|
||||
|
||||
@@ -149,10 +149,10 @@ func readerDeleteSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
|
||||
func readerDebugSourceHandler(svc *service.Container) gin.HandlerFunc {
|
||||
var body struct {
|
||||
Key string `json:"key" binding:"required"`
|
||||
}
|
||||
return func(c *gin.Context) {
|
||||
var body struct {
|
||||
Key string `json:"key" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
|
||||
@@ -13,6 +13,8 @@ func registerAdminRoutes(api *gin.RouterGroup, cfg *config.Config, svc *service.
|
||||
admin := api.Group("/admin")
|
||||
admin.Use(middleware.AuthRequired(cfg.Secrets.JWTSecret), middleware.AdminRequired())
|
||||
registerAdminUserRoutes(admin, svc)
|
||||
admin.GET("/tasks", backgroundTasksHandler(svc))
|
||||
admin.GET("/quality", mediaQualityHandler(svc))
|
||||
registerAdminPermissionRoutes(admin, svc)
|
||||
registerAdminSystemRoutes(admin, svc)
|
||||
registerAdminBackupRoutes(admin, svc)
|
||||
@@ -143,6 +145,7 @@ func registerAdminSystemRoutes(admin *gin.RouterGroup, svc *service.Container) {
|
||||
func registerAdminBackupRoutes(admin *gin.RouterGroup, svc *service.Container) {
|
||||
admin.GET("/backups", listBackupsHandler(svc))
|
||||
admin.POST("/backups", createBackupHandler(svc))
|
||||
admin.GET("/backups/download", downloadBackupHandler(svc))
|
||||
admin.DELETE("/backups", deleteBackupHandler(svc))
|
||||
admin.POST("/backups/restore", restoreBackupHandler(svc))
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ func registerAuthedDLNARoutes(authed *gin.RouterGroup, svc *service.Container) {
|
||||
}
|
||||
|
||||
func registerAuthedRealtimeRoutes(authed *gin.RouterGroup, svc *service.Container) {
|
||||
authed.GET("/ws", wsHandler(svc))
|
||||
authed.GET("/events", sseHandler(svc))
|
||||
// These streams carry operator tasks, including filesystem paths and scan
|
||||
// errors. Their payloads are not scoped to an individual user's libraries.
|
||||
authed.GET("/ws", middleware.AdminRequired(), wsHandler(svc))
|
||||
authed.GET("/events", middleware.AdminRequired(), sseHandler(svc))
|
||||
}
|
||||
|
||||
@@ -27,7 +27,15 @@ func hlsPlaylistHandler(svc *service.Container) gin.HandlerFunc {
|
||||
if !enforceScopedPlaybackToken(c, m.ID) {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"))
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"), scope)
|
||||
if errors.Is(err, service.ErrHLSGenerationSuperseded) {
|
||||
c.JSON(http.StatusConflict, gin.H{"code": "hls_generation_superseded", "error": err.Error()})
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrMediaNotFound) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
|
||||
return
|
||||
@@ -62,7 +70,11 @@ func hlsSegmentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
if !enforceScopedPlaybackToken(c, m.ID) {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg"))
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg"), scope)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -72,11 +84,39 @@ func hlsSegmentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
|
||||
func stopTranscodeHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
svc.Transcoder.StopJob(c.Param("id"))
|
||||
m, err := findMediaForPlaybackEndpoint(c, svc, c.Param("id"))
|
||||
if err != nil || m == nil || !mediaVisibleForRequest(c, svc, m) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "media not found"})
|
||||
return
|
||||
}
|
||||
if !enforceScopedPlaybackToken(c, m.ID) {
|
||||
return
|
||||
}
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if svc.Transcoder != nil {
|
||||
svc.Transcoder.StopPlayback(m.ID, scope)
|
||||
}
|
||||
c.Status(http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
|
||||
func localHLSPlaybackScope(c *gin.Context) (service.HLSPlaybackScope, bool) {
|
||||
userID := currentUserID(c)
|
||||
if userID == "" {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "missing playback owner"})
|
||||
return service.HLSPlaybackScope{}, false
|
||||
}
|
||||
scope, err := service.HLSPlaybackScopeForRequest(c.Request, userID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return service.HLSPlaybackScope{}, false
|
||||
}
|
||||
return scope, true
|
||||
}
|
||||
|
||||
func imageProxyHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
raw := c.Query("url")
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
// Package handler — WebSocket endpoint.
|
||||
//
|
||||
// Clients connect to /api/ws?token=... (the token is the same JWT used for
|
||||
// REST calls). The first message they send is a JSON {"action":"subscribe",
|
||||
// "topics":["scan","scrape","transcode"]}. Subsequent server-pushed events
|
||||
// arrive as {"topic":"...","payload":{...}}.
|
||||
// Administrators connect to /api/ws?token=... with their REST access token.
|
||||
// Operator events arrive as {"topic":"...","payload":{...}}.
|
||||
package handler
|
||||
|
||||
import (
|
||||
@@ -52,8 +50,11 @@ func wsHandler(svc *service.Container) gin.HandlerFunc {
|
||||
sub := svc.WSHub.Subscribe(id, nil)
|
||||
defer svc.WSHub.Unsubscribe(id)
|
||||
|
||||
// Reader: accept ping + subscription updates.
|
||||
// Reading processes WebSocket control frames and detects disconnects.
|
||||
done := make(chan struct{})
|
||||
conn.SetReadLimit(64 << 10)
|
||||
go func() {
|
||||
defer close(done)
|
||||
for {
|
||||
if _, _, err := conn.NextReader(); err != nil {
|
||||
_ = conn.Close()
|
||||
@@ -67,6 +68,8 @@ func wsHandler(svc *service.Container) gin.HandlerFunc {
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case ev, ok := <-sub.Out:
|
||||
if !ok {
|
||||
return
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/gorilla/websocket"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func realtimeTestRouter(hub *service.Hub) *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
authed := router.Group("/api")
|
||||
authed.Use(middleware.AuthRequired("realtime-test"))
|
||||
registerAuthedRealtimeRoutes(authed, &service.Container{WSHub: hub})
|
||||
return router
|
||||
}
|
||||
|
||||
func realtimeTestToken(t *testing.T, role string) string {
|
||||
t.Helper()
|
||||
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, middleware.Claims{
|
||||
UserID: "viewer", Role: role,
|
||||
RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour))},
|
||||
}).SignedString([]byte("realtime-test"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
func TestRealtimeStreamsRejectOrdinaryUsers(t *testing.T) {
|
||||
router := realtimeTestRouter(nil)
|
||||
for _, path := range []string{"/api/ws", "/api/events"} {
|
||||
for _, role := range []string{"", "user"} {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
want := http.StatusUnauthorized
|
||||
if role != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+realtimeTestToken(t, role))
|
||||
want = http.StatusForbidden
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
router.ServeHTTP(recorder, req)
|
||||
if recorder.Code != want {
|
||||
t.Fatalf("%s role %q: status %d, want %d", path, role, recorder.Code, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminWebSocketReceivesTaskEvents(t *testing.T) {
|
||||
hub := service.NewHub(zap.NewNop())
|
||||
go hub.Run()
|
||||
defer hub.Stop()
|
||||
server := httptest.NewServer(realtimeTestRouter(hub))
|
||||
defer server.Close()
|
||||
url := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/ws?token=" + realtimeTestToken(t, "admin")
|
||||
conn, _, err := websocket.DefaultDialer.Dial(url, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go func() {
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
hub.Publish("task", service.BackgroundTask{ID: "task-1", SourcePath: "/media/private"})
|
||||
}
|
||||
}
|
||||
}()
|
||||
_ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
var event service.Event
|
||||
if err := conn.ReadJSON(&event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if event.Topic != "task" || event.Payload.(map[string]any)["source_path"] != "/media/private" {
|
||||
t.Fatalf("event = %#v", event)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// BackgroundTaskRecord persists the operator-facing lifecycle of scan,
|
||||
// organize, and other long-running application tasks.
|
||||
type BackgroundTaskRecord struct {
|
||||
ID string `gorm:"primaryKey;size:36" json:"id"`
|
||||
Kind string `gorm:"index;size:32;not null" json:"kind"`
|
||||
Name string `gorm:"size:255;not null" json:"name"`
|
||||
Status string `gorm:"index;size:24;not null" json:"status"`
|
||||
Stage string `gorm:"size:64" json:"stage"`
|
||||
SourcePath string `gorm:"size:2048" json:"source_path"`
|
||||
DestPath string `gorm:"size:2048" json:"dest_path"`
|
||||
Message string `gorm:"type:text" json:"message"`
|
||||
Error string `gorm:"type:text" json:"error"`
|
||||
Details string `gorm:"type:text" json:"details"`
|
||||
Metrics string `gorm:"type:text" json:"metrics"`
|
||||
Items string `gorm:"type:text" json:"items"`
|
||||
StartedAt time.Time `gorm:"index" json:"started_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// Library 表示一个逻辑媒体库。Path 保留为兼容字段,指向第一条 LibraryRoot。
|
||||
type Library struct {
|
||||
Base
|
||||
@@ -26,12 +28,15 @@ type LibraryRoot struct {
|
||||
// Media 是单个可播放项。剧集链接到 SeriesID;电影 SeriesID == ""。
|
||||
type Media struct {
|
||||
Base
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
AvailabilityState string `gorm:"index;size:16;not null;default:available" json:"availability_state"`
|
||||
MissingSince *time.Time `json:"missing_since,omitempty"`
|
||||
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
// Path 的唯一性由 database 包按方言显式建索引,这里刻意不用 uniqueIndex:
|
||||
// PostgreSQL 的 btree 单条索引项上限 2704 字节,而 varchar(1024) 在中文路径
|
||||
// 下可达 3072 字节,直接对整列建唯一索引会让长路径的扫描插入直接失败
|
||||
|
||||
@@ -61,6 +61,7 @@ func AllModels() []interface{} {
|
||||
&StrmUploadTask{},
|
||||
&StrmDirCache{},
|
||||
&ScrapeTask{},
|
||||
&BackgroundTaskRecord{},
|
||||
&EmbyMount{},
|
||||
&ReaderBookSource{},
|
||||
&ReaderSourceState{},
|
||||
|
||||
@@ -178,14 +178,6 @@ func (r *MediaRepository) YearRange(ctx context.Context, filter MediaQueryFilter
|
||||
return min, max, nil
|
||||
}
|
||||
|
||||
func (r *MediaRepository) indexMediaBestEffort(ctx context.Context, media model.Media) {
|
||||
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
_ = backend.IndexMedia(ctx, []model.Media{media})
|
||||
}
|
||||
|
||||
// FindByID returns the media row or (nil, nil).
|
||||
func (r *MediaRepository) FindByID(ctx context.Context, id string) (*model.Media, error) {
|
||||
var m model.Media
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
@@ -96,15 +97,34 @@ func (r *MediaRepository) UpsertBatchWithAliases(ctx context.Context, items []Me
|
||||
|
||||
// indexByIDBestEffort 在事务提交后按 ID 回读最新行并同步搜索索引。
|
||||
func (r *MediaRepository) indexByIDBestEffort(ctx context.Context, ids []string) {
|
||||
if _, ok := r.searchBackend.(MediaSearchSyncBackend); !ok {
|
||||
backend, ok := r.searchBackend.(MediaSearchSyncBackend)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
unique := make([]string, 0, len(ids))
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if fresh, err := r.FindByID(ctx, id); err == nil && fresh != nil {
|
||||
r.indexMediaBestEffort(ctx, *fresh)
|
||||
if _, exists := seen[id]; !exists {
|
||||
seen[id] = struct{}{}
|
||||
unique = append(unique, id)
|
||||
}
|
||||
}
|
||||
// Read committed rows and submit one bulk per bounded batch. Keep network
|
||||
// work outside the transaction and below SQLite's parameter limit.
|
||||
const batchSize = 500
|
||||
for start := 0; start < len(unique); start += batchSize {
|
||||
batch := unique[start:min(start+batchSize, len(unique))]
|
||||
rows, err := r.ListMediaByIDs(ctx, batch)
|
||||
if err == nil {
|
||||
err = backend.IndexMedia(ctx, rows)
|
||||
}
|
||||
if err != nil {
|
||||
// The primary write has committed; do not report a failed upsert and
|
||||
// make scanners repeat it. Surface the separate index failure instead.
|
||||
r.db.Logger.Error(ctx, "media saved but search index sync failed for %d items: %v", len(batch), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -144,6 +164,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
|
||||
existing.Path = m.Path
|
||||
existing.DeletedAt = gorm.DeletedAt{}
|
||||
}
|
||||
if _, ok := updates["availability_state"]; ok {
|
||||
existing.AvailabilityState = "available"
|
||||
existing.MissingSince = nil
|
||||
now := updates["last_seen_at"].(time.Time)
|
||||
existing.LastSeenAt = &now
|
||||
}
|
||||
*m = existing
|
||||
return existing.ID, nil
|
||||
}
|
||||
@@ -197,6 +223,10 @@ func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.
|
||||
if m.ScrapeStatus == "" {
|
||||
m.ScrapeStatus = "pending"
|
||||
}
|
||||
now := time.Now()
|
||||
m.AvailabilityState = "available"
|
||||
m.MissingSince = nil
|
||||
m.LastSeenAt = &now
|
||||
// PostgreSQL 唯一键异常会中止整个事务,之后重查也会失败。
|
||||
// DO NOTHING 保持事务可用,并兼容 path 的表达式唯一索引。
|
||||
result := db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(m)
|
||||
@@ -246,6 +276,11 @@ func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, cur
|
||||
|
||||
func mediaUpsertUpdates(existing, incoming model.Media, reclaimLibrary bool) map[string]any {
|
||||
updates := map[string]any{}
|
||||
if existing.AvailabilityState != "available" || existing.MissingSince != nil {
|
||||
updates["availability_state"] = "available"
|
||||
updates["missing_since"] = nil
|
||||
updates["last_seen_at"] = time.Now()
|
||||
}
|
||||
addMediaFileScanUpdates(updates, existing, incoming)
|
||||
addMediaTitleUpdates(updates, existing, incoming)
|
||||
addMediaExternalIDUpdates(updates, existing, incoming)
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -135,14 +137,113 @@ func (b *OpenSearchMediaBackend) EnsureIndex(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Media) error {
|
||||
const batchSize = 500
|
||||
var failures []error
|
||||
for start := 0; start < len(rows); start += batchSize {
|
||||
if err := b.indexMediaBatch(ctx, rows[start:min(start+batchSize, len(rows))]); err != nil {
|
||||
failures = append(failures, err)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
|
||||
type openSearchBulkResponse struct {
|
||||
Errors bool `json:"errors"`
|
||||
Items []struct {
|
||||
Index struct {
|
||||
ID string `json:"_id"`
|
||||
Status int `json:"status"`
|
||||
Error *struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"error"`
|
||||
} `json:"index"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) indexMediaBatch(ctx context.Context, rows []model.Media) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
pending := rows
|
||||
var failures []error
|
||||
const attempts = 3
|
||||
for attempt := 0; attempt < attempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return errors.Join(append(failures, err)...)
|
||||
}
|
||||
if attempt > 0 {
|
||||
timer := time.NewTimer(time.Duration(100*(1<<(attempt-1))) * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return errors.Join(append(failures, ctx.Err())...)
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
response, err := b.submitMediaBulk(ctx, pending)
|
||||
if err != nil {
|
||||
if retryableOpenSearchError(err) && attempt < attempts-1 {
|
||||
continue
|
||||
}
|
||||
return errors.Join(append(failures, err)...)
|
||||
}
|
||||
if len(response.Items) != len(pending) {
|
||||
return errors.Join(append(failures, fmt.Errorf("opensearch bulk returned %d item results for %d documents", len(response.Items), len(pending)))...)
|
||||
}
|
||||
var retry []model.Media
|
||||
failed := false
|
||||
for i, item := range response.Items {
|
||||
result := item.Index
|
||||
if result.Status >= 200 && result.Status < 300 && result.Error == nil {
|
||||
continue
|
||||
}
|
||||
failed = true
|
||||
if retryableOpenSearchStatus(result.Status) && attempt < attempts-1 {
|
||||
retry = append(retry, pending[i])
|
||||
continue
|
||||
}
|
||||
errorType := "unknown"
|
||||
if result.Error != nil {
|
||||
errorType = result.Error.Type
|
||||
}
|
||||
failures = append(failures, fmt.Errorf("opensearch bulk document %q failed: status %d (%s)", pending[i].ID, result.Status, errorType))
|
||||
}
|
||||
if response.Errors && !failed {
|
||||
failures = append(failures, errors.New("opensearch bulk reported errors without failed item results"))
|
||||
}
|
||||
if len(retry) == 0 {
|
||||
break
|
||||
}
|
||||
pending = retry
|
||||
}
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
|
||||
func retryableOpenSearchStatus(status int) bool {
|
||||
return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500
|
||||
}
|
||||
|
||||
func retryableOpenSearchError(err error) bool {
|
||||
var statusError *openSearchHTTPError
|
||||
if errors.As(err, &statusError) {
|
||||
return retryableOpenSearchStatus(statusError.status)
|
||||
}
|
||||
var networkError net.Error
|
||||
return errors.As(err, &networkError)
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) submitMediaBulk(ctx context.Context, rows []model.Media) (*openSearchBulkResponse, error) {
|
||||
if len(rows) == 0 {
|
||||
return &openSearchBulkResponse{}, nil
|
||||
}
|
||||
var bulk bytes.Buffer
|
||||
enc := json.NewEncoder(&bulk)
|
||||
for _, row := range rows {
|
||||
if err := enc.Encode(map[string]any{"index": map[string]any{"_index": b.index, "_id": row.ID}}); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if err := enc.Encode(map[string]any{
|
||||
"id": row.ID,
|
||||
@@ -155,10 +256,24 @@ func (b *OpenSearchMediaBackend) IndexMedia(ctx context.Context, rows []model.Me
|
||||
"deleted": row.DeletedAt.Valid,
|
||||
"created_at": row.CreatedAt,
|
||||
}); err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", nil)
|
||||
var response openSearchBulkResponse
|
||||
if err := b.do(ctx, http.MethodPost, "/_bulk", &bulk, "application/x-ndjson", &response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
type openSearchHTTPError struct {
|
||||
method string
|
||||
path string
|
||||
status int
|
||||
}
|
||||
|
||||
func (e *openSearchHTTPError) Error() string {
|
||||
return fmt.Sprintf("opensearch %s %s returned %d", e.method, e.path, e.status)
|
||||
}
|
||||
|
||||
func (b *OpenSearchMediaBackend) doJSON(ctx context.Context, method, path string, body any, out any) error {
|
||||
@@ -190,7 +305,7 @@ func (b *OpenSearchMediaBackend) do(ctx context.Context, method, path string, bo
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("opensearch %s %s returned %d", method, path, resp.StatusCode)
|
||||
return &openSearchHTTPError{method: method, path: path, status: resp.StatusCode}
|
||||
}
|
||||
if out == nil {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
@@ -1,12 +1,23 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
|
||||
@@ -55,3 +66,156 @@ func TestOpenSearchMediaBackendSearchesIDs(t *testing.T) {
|
||||
t.Fatalf("ids=%#v total=%d", ids, total)
|
||||
}
|
||||
}
|
||||
|
||||
func bulkRequestIDs(r *http.Request) []string {
|
||||
var ids []string
|
||||
scanner := bufio.NewScanner(r.Body)
|
||||
for scanner.Scan() {
|
||||
var action struct {
|
||||
Index struct {
|
||||
ID string `json:"_id"`
|
||||
} `json:"index"`
|
||||
}
|
||||
_ = json.Unmarshal(scanner.Bytes(), &action)
|
||||
ids = append(ids, action.Index.ID)
|
||||
if !scanner.Scan() {
|
||||
break
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func writeBulkResults(w http.ResponseWriter, ids []string, statuses map[string]int) {
|
||||
items := make([]any, 0, len(ids))
|
||||
failed := false
|
||||
for _, id := range ids {
|
||||
status := statuses[id]
|
||||
if status == 0 {
|
||||
status = http.StatusCreated
|
||||
}
|
||||
result := map[string]any{"_id": id, "status": status}
|
||||
if status >= 400 {
|
||||
failed = true
|
||||
result["error"] = map[string]any{"type": "test_failure"}
|
||||
}
|
||||
items = append(items, map[string]any{"index": result})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"errors": failed, "items": items})
|
||||
}
|
||||
|
||||
func TestOpenSearchBulkRetriesOnlyTransientFailedDocuments(t *testing.T) {
|
||||
var requests [][]string
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ids := bulkRequestIDs(r)
|
||||
requests = append(requests, ids)
|
||||
statuses := map[string]int{}
|
||||
if len(requests) == 1 {
|
||||
statuses["retry"] = http.StatusTooManyRequests
|
||||
statuses["invalid"] = http.StatusBadRequest
|
||||
}
|
||||
writeBulkResults(w, ids, statuses)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
|
||||
rows := []model.Media{{Base: model.Base{ID: "ok"}}, {Base: model.Base{ID: "retry"}}, {Base: model.Base{ID: "invalid"}}}
|
||||
err := backend.IndexMedia(t.Context(), rows)
|
||||
if err == nil || !strings.Contains(err.Error(), `"invalid"`) {
|
||||
t.Fatalf("error = %v, want permanent document failure", err)
|
||||
}
|
||||
if len(requests) != 2 || strings.Join(requests[1], ",") != "retry" {
|
||||
t.Fatalf("requests = %#v; successful/permanent documents must not be retried", requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSearchBulkBoundsRetriesAndRejectsMalformedResponses(t *testing.T) {
|
||||
for _, scenario := range []string{"transient-item", "transient-http", "invalid-http", "missing-results"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
calls := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
switch scenario {
|
||||
case "transient-item":
|
||||
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusServiceUnavailable})
|
||||
case "transient-http":
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
case "invalid-http":
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
case "missing-results":
|
||||
fmt.Fprint(w, `{"errors":false,"items":[]}`)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
|
||||
if err := backend.IndexMedia(t.Context(), []model.Media{{Base: model.Base{ID: "m1"}}}); err == nil {
|
||||
t.Fatal("failed indexing was reported as successful")
|
||||
}
|
||||
want := 1
|
||||
if strings.HasPrefix(scenario, "transient") {
|
||||
want = 3
|
||||
}
|
||||
if calls != want {
|
||||
t.Fatalf("calls = %d, want %d", calls, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSearchBulkHonorsCancellationDuringRetry(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
writeBulkResults(w, bulkRequestIDs(r), map[string]int{"m1": http.StatusTooManyRequests})
|
||||
cancel()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
backend := NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL})
|
||||
err := backend.IndexMedia(ctx, []model.Media{{Base: model.Base{ID: "m1"}}})
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("error = %v, calls = %d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaUpsertBatchIndexesCommittedRowsTogether(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
defer sqlDB.Close()
|
||||
if err := db.AutoMigrate(&model.Media{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := New(db)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
calls := 0
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
ids := bulkRequestIDs(r)
|
||||
if len(ids) != 5 {
|
||||
t.Errorf("bulk contains %d documents, want 5", len(ids))
|
||||
}
|
||||
// With only one DB connection this also detects network work inside the
|
||||
// transaction: a separate read could not observe the committed batch.
|
||||
var count int64
|
||||
if err := db.WithContext(ctx).Model(&model.Media{}).Count(&count).Error; err != nil || count != 5 {
|
||||
t.Errorf("committed rows = %d, error = %v", count, err)
|
||||
}
|
||||
writeBulkResults(w, ids, nil)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
repos.Media.SetSearchBackend(NewOpenSearchMediaBackend(config.SearchConfig{Backend: "opensearch", OpenSearchURL: upstream.URL}))
|
||||
var rows []*model.Media
|
||||
for i := 0; i < 5; i++ {
|
||||
rows = append(rows, &model.Media{LibraryID: "lib", Path: fmt.Sprintf("/media/%d.mkv", i)})
|
||||
}
|
||||
if err := repos.Media.UpsertBatch(ctx, rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("index requests = %d, want one bulk", calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,16 +46,20 @@ func (r *RefreshTokenRepository) RevokeByUserID(ctx context.Context, userID stri
|
||||
})
|
||||
}
|
||||
|
||||
// RevokeOldestActiveByUserID keeps at most limit active refresh tokens for a
|
||||
// user by revoking the oldest non-expired, non-revoked tokens.
|
||||
func (r *RefreshTokenRepository) RevokeOldestActiveByUserID(ctx context.Context, userID string, limit int) error {
|
||||
// RevokeOldestActiveLegacyByUserID keeps at most limit active refresh tokens
|
||||
// for a user among tokens that are not bound to a device auth session
|
||||
// (legacy web clients). Device sessions are independent: trimming one
|
||||
// device's token because another device logged in would silently sign the
|
||||
// first device out even though its session is still valid.
|
||||
func (r *RefreshTokenRepository) RevokeOldestActiveLegacyByUserID(ctx context.Context, userID string, limit int) error {
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
return withSQLiteBusyRetry(ctx, func() error {
|
||||
var tokens []model.RefreshToken
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("user_id = ? AND revoked = ? AND expires_at > ?", userID, false, time.Now()).
|
||||
Where("user_id = ? AND revoked = ? AND expires_at > ? AND (auth_session_id IS NULL OR auth_session_id = '')",
|
||||
userID, false, time.Now()).
|
||||
Order("created_at desc, id desc").
|
||||
Find(&tokens).Error; err != nil {
|
||||
return err
|
||||
|
||||
@@ -24,6 +24,14 @@ func (s *ApiConfigService) TestConnection(ctx context.Context, provider string)
|
||||
switch provider {
|
||||
case "tmdb":
|
||||
return s.testTMDb(cfg)
|
||||
case "bangumi":
|
||||
return s.testBangumi(ctx, cfg)
|
||||
case "thetvdb":
|
||||
return s.testTheTVDB(ctx, cfg)
|
||||
case "fanart":
|
||||
return s.testFanart(ctx, cfg)
|
||||
case "douban":
|
||||
return s.testDouban(ctx, cfg)
|
||||
case "openai":
|
||||
return s.testOpenAI(cfg)
|
||||
case "deepseek":
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// Bangumi's public metadata endpoints work without a token. If one was saved,
|
||||
// also check /v0/me: public endpoints alone may accept an expired token.
|
||||
func (s *ApiConfigService) testBangumi(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://api.bgm.tv", "/v0/subjects/253", http.MethodGet, nil)
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
if token := strings.TrimSpace(cfg.APIKey); token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
identityReq := req.Clone(ctx)
|
||||
identityURL := *req.URL
|
||||
identityURL.Path = strings.TrimSuffix(identityURL.Path, "/subjects/253") + "/me"
|
||||
identityReq.URL = &identityURL
|
||||
var identity struct {
|
||||
ID int `json:"id"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(identityReq, "Bangumi", &identity); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if identity.ID <= 0 {
|
||||
return "error", errors.New("Bangumi returned an invalid user response")
|
||||
}
|
||||
}
|
||||
var subject struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "Bangumi", &subject); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if subject.ID != 253 || strings.TrimSpace(subject.Name) == "" {
|
||||
return "error", errors.New("Bangumi returned an invalid subject response")
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func (s *ApiConfigService) testTheTVDB(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
key := strings.TrimSpace(cfg.APIKey)
|
||||
if key == "" {
|
||||
return "error", errors.New("TheTVDB API key is required")
|
||||
}
|
||||
body, err := json.Marshal(map[string]string{"apikey": key})
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://api4.thetvdb.com/v4", "/login", http.MethodPost, strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
var response struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "TheTVDB", &response); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if response.Status != "success" || strings.TrimSpace(response.Data.Token) == "" {
|
||||
return "error", errors.New("TheTVDB login did not return a valid token")
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func (s *ApiConfigService) testFanart(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
key := strings.TrimSpace(cfg.APIKey)
|
||||
if key == "" {
|
||||
return "error", errors.New("Fanart API key is required")
|
||||
}
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://webservice.fanart.tv/v3", "/movies/550", http.MethodGet, nil)
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
query := req.URL.Query()
|
||||
query.Set("api_key", key)
|
||||
req.URL.RawQuery = query.Encode()
|
||||
var response struct {
|
||||
TMDbID json.Number `json:"tmdb_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "Fanart", &response); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if response.TMDbID.String() != "550" || strings.TrimSpace(response.Name) == "" {
|
||||
return "error", errors.New("Fanart returned an invalid movie response")
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func (s *ApiConfigService) testDouban(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://movie.douban.com", "/j/subject_suggest", http.MethodGet, nil)
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
query := req.URL.Query()
|
||||
query.Set("q", "Inception")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.Header.Set("User-Agent", userAgents[0])
|
||||
req.Header.Set("Referer", "https://movie.douban.com/")
|
||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
||||
if cookie := strings.TrimSpace(cfg.APIKey); cookie != "" {
|
||||
req.Header.Set("Cookie", cookie)
|
||||
}
|
||||
var suggestions []struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "Douban", &suggestions); err != nil {
|
||||
return result, err
|
||||
}
|
||||
// An empty array is a valid search result; null or an error object is not.
|
||||
if suggestions == nil {
|
||||
return "error", errors.New("Douban returned an invalid search response (login or verification may be required)")
|
||||
}
|
||||
for _, suggestion := range suggestions {
|
||||
if suggestion.ID == "" || strings.TrimSpace(suggestion.Title) == "" {
|
||||
return "error", errors.New("Douban returned an invalid search suggestion")
|
||||
}
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func newMetadataProbeRequest(ctx context.Context, cfg *model.APIConfig, defaultBase, path, method string, body io.Reader) (*http.Request, error) {
|
||||
base := strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/")
|
||||
if base == "" {
|
||||
base = defaultBase
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, base+path, body)
|
||||
if err != nil {
|
||||
// The saved base URL may contain credentials. Never echo it to the UI.
|
||||
return nil, fmt.Errorf("%s: invalid base URL", cfg.Provider)
|
||||
}
|
||||
req.Header.Set("User-Agent", "MeBox/0.1 (https://github.com/truewhile/MeBox)")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// Decode a bounded, complete JSON document rather than accepting any HTTP 200
|
||||
// (reverse proxies and anti-bot/login pages often return 200 as well).
|
||||
func testMetadataProbeJSON(req *http.Request, provider string, out any) (string, error) {
|
||||
client := NewExternalHTTPClient(10 * time.Second)
|
||||
defer client.CloseIdleConnections()
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
// url.Error includes the full request URL, including Fanart's API key.
|
||||
var urlErr *url.Error
|
||||
if errors.As(err, &urlErr) {
|
||||
err = urlErr.Err
|
||||
}
|
||||
return "error", fmt.Errorf("%s connection failed: %w", provider, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
return "invalid", fmt.Errorf("%s authentication failed (HTTP 401): check configured credentials", provider)
|
||||
}
|
||||
if resp.StatusCode == http.StatusForbidden {
|
||||
return "error", fmt.Errorf("%s access denied (HTTP 403): check credentials, permissions or anti-bot verification", provider)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "error", fmt.Errorf("%s API returned status %d", provider, resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, (1<<20)+1))
|
||||
if err != nil {
|
||||
return "error", fmt.Errorf("%s response read failed: %w", provider, err)
|
||||
}
|
||||
if len(body) > 1<<20 {
|
||||
return "error", fmt.Errorf("%s response exceeds 1 MiB", provider)
|
||||
}
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
// Do not include the response body: it may echo credentials.
|
||||
return "error", fmt.Errorf("%s returned invalid JSON (login or verification page may have intercepted the request)", provider)
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
type metadataProbeFixture struct {
|
||||
provider string
|
||||
path string
|
||||
key string
|
||||
body string
|
||||
}
|
||||
|
||||
var metadataProbeFixtures = []metadataProbeFixture{
|
||||
{"bangumi", "/v0/subjects/253", "", `{"id":253,"name":"カウボーイビバップ","type":2}`},
|
||||
{"thetvdb", "/login", "tvdb-test-secret", `{"status":"success","data":{"token":"test-jwt"}}`},
|
||||
{"fanart", "/movies/550", "fanart+test&secret", `{"name":"Fight Club","tmdb_id":"550","imdb_id":"tt0137523","movieposter":[]}`},
|
||||
{"douban", "/j/subject_suggest", "bid=test-cookie", `[{"id":"3541415","title":"盗梦空间","year":"2010","type":"movie"}]`},
|
||||
}
|
||||
|
||||
func newMetadataConnectionTestService(t *testing.T) *ApiConfigService {
|
||||
t.Helper()
|
||||
log := zap.NewNop()
|
||||
return NewApiConfigService(&config.Config{}, log, repository.New(newServiceTestDB(t)), NewCryptoService("metadata-test-secret", log))
|
||||
}
|
||||
|
||||
func saveMetadataProbeConfig(t *testing.T, svc *ApiConfigService, provider, baseURL, key string) {
|
||||
t.Helper()
|
||||
if _, err := svc.Upsert(t.Context(), provider, key, baseURL, "", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Removing a dispatch branch, ignoring the saved URL/key, or using the wrong
|
||||
// endpoint must fail this integration test (real database and HTTP server).
|
||||
func TestMetadataConnectionUsesSavedConfig(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, fixture := range metadataProbeFixtures {
|
||||
t.Run(fixture.provider, func(t *testing.T) {
|
||||
calls := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
if r.URL.Path != "/proxy"+fixture.path {
|
||||
t.Errorf("request path = %q", r.URL.Path)
|
||||
}
|
||||
if r.Header.Get("User-Agent") == "" {
|
||||
t.Error("missing User-Agent")
|
||||
}
|
||||
if fixture.provider == "thetvdb" {
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if r.Method != http.MethodPost || body["apikey"] != fixture.key || r.Header.Get("Content-Type") != "application/json" {
|
||||
t.Error("TheTVDB login must POST the saved, decrypted API key as JSON")
|
||||
}
|
||||
} else if r.Method != http.MethodGet {
|
||||
t.Errorf("method = %s, want GET", r.Method)
|
||||
}
|
||||
if fixture.provider == "fanart" && r.URL.Query().Get("api_key") != fixture.key {
|
||||
t.Error("Fanart API key was not escaped/passed correctly")
|
||||
}
|
||||
if fixture.provider == "douban" {
|
||||
if r.Header.Get("Cookie") != fixture.key || r.Header.Get("Referer") != "https://movie.douban.com/" || r.URL.Query().Get("q") == "" {
|
||||
t.Error("Douban search must carry the configured cookie, referer and query")
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(fixture.body))
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, fixture.provider, " "+server.URL+"/proxy/ ", fixture.key)
|
||||
result, err := svc.TestConnection(t.Context(), fixture.provider)
|
||||
if err != nil || result != "success" {
|
||||
t.Fatalf("TestConnection = %q, %v, want success", result, err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("HTTP calls = %d, want 1", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionRejectsBadResponses(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, fixture := range metadataProbeFixtures {
|
||||
for _, response := range []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
result string
|
||||
}{
|
||||
{"unauthorized", 401, `{"error":"denied"}`, "invalid"},
|
||||
{"forbidden", 403, `<html>Access denied</html>`, "error"},
|
||||
{"rate limited", 429, `{}`, "error"},
|
||||
{"upstream error", 503, `{}`, "error"},
|
||||
{"challenge page", 200, `<html>Please log in or complete verification</html>`, "error"},
|
||||
{"empty body", 200, ``, "error"},
|
||||
{"wrong schema", 200, `{"error":"intercepted"}`, "error"},
|
||||
{"null", 200, `null`, "error"},
|
||||
{"trailing garbage", 200, fixture.body + `<html>blocked</html>`, "error"},
|
||||
} {
|
||||
t.Run(fixture.provider+"/"+response.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(response.status)
|
||||
_, _ = w.Write([]byte(response.body))
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, fixture.provider, server.URL, fixture.key)
|
||||
result, err := svc.TestConnection(t.Context(), fixture.provider)
|
||||
if err == nil || result != response.result {
|
||||
t.Fatalf("TestConnection = %q, %v, want %q and error", result, err, response.result)
|
||||
}
|
||||
if fixture.key != "" && strings.Contains(err.Error(), fixture.key) {
|
||||
t.Error("error exposes configured credentials")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionRequiresAPIKeys(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, provider := range []string{"thetvdb", "fanart"} {
|
||||
t.Run(provider, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("must reject missing key before making a request")
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, provider, server.URL, " ")
|
||||
result, err := svc.TestConnection(t.Context(), provider)
|
||||
if result != "error" || err == nil || !strings.Contains(err.Error(), "API key") {
|
||||
t.Fatalf("TestConnection = %q, %v, want missing API key error", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionBangumiValidatesConfiguredToken(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, valid := range []bool{true, false} {
|
||||
name := "valid"
|
||||
if !valid {
|
||||
name = "invalid"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
meCalls, subjectCalls := 0, 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer bangumi-test-token" {
|
||||
t.Error("missing saved Bangumi bearer token")
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/v0/me":
|
||||
meCalls++
|
||||
if !valid {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"id":42,"username":"test-user"}`))
|
||||
case "/v0/subjects/253":
|
||||
subjectCalls++
|
||||
_, _ = w.Write([]byte(`{"id":253,"name":"カウボーイビバップ"}`))
|
||||
default:
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, "bangumi", server.URL, "bangumi-test-token")
|
||||
result, err := svc.TestConnection(t.Context(), "bangumi")
|
||||
if valid && (result != "success" || err != nil || subjectCalls != 1) {
|
||||
t.Fatalf("valid token: result=%q err=%v subjectCalls=%d", result, err, subjectCalls)
|
||||
}
|
||||
if !valid && (result != "invalid" || err == nil || subjectCalls != 0) {
|
||||
t.Fatalf("invalid token: result=%q err=%v subjectCalls=%d", result, err, subjectCalls)
|
||||
}
|
||||
if meCalls != 1 {
|
||||
t.Errorf("identity requests = %d, want 1", meCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionDoubanDoesNotRequireCookie(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Cookie") != "" {
|
||||
t.Error("unexpected Cookie for public search")
|
||||
}
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, "douban", server.URL, "")
|
||||
result, err := svc.TestConnection(t.Context(), "douban")
|
||||
if result != "success" || err != nil {
|
||||
t.Fatalf("public search with empty results = %q, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionHonorsContext(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, fixture := range metadataProbeFixtures {
|
||||
t.Run(fixture.provider, func(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
close(started)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, fixture.provider, server.URL, fixture.key)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-started:
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
result, err := svc.TestConnection(ctx, fixture.provider)
|
||||
if result != "error" || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled request = %q, %v, want context.Canceled", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionTransportErrorsHideKeys(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
server := httptest.NewServer(http.NotFoundHandler())
|
||||
server.Close()
|
||||
key := "sensitive-fanart-key"
|
||||
saveMetadataProbeConfig(t, svc, "fanart", server.URL, key)
|
||||
result, err := svc.TestConnection(t.Context(), "fanart")
|
||||
if result != "error" || err == nil || strings.Contains(err.Error(), key) {
|
||||
t.Fatalf("transport failure = %q, %v, want error without credentials", result, err)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -163,6 +164,52 @@ func TestRefreshKeepsSessionAssociation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestMobileDeviceSessionsDoNotTrimEachOther 覆盖同一账号多设备登录:
|
||||
// 每台设备有独立认证会话与刷新令牌,新设备登录或任一设备刷新都不应撤销
|
||||
// 其他设备的刷新令牌。历史实现把「最大同时登录终端」直接作用在刷新令牌
|
||||
// 上,第 N+1 台设备登录会把最早那台设备的刷新令牌标记为已撤销,那台设备
|
||||
// 下一次刷新就会收到 SESSION_REVOKED,被强制退回登录页。
|
||||
func TestMobileDeviceSessionsDoNotTrimEachOther(t *testing.T) {
|
||||
sessions, tokens, repos := newAuthSessionTestEnv(t)
|
||||
ctx := context.Background()
|
||||
user := seedSessionUser(t, repos)
|
||||
|
||||
// 默认上限是 3,这里故意登录 4 台设备。
|
||||
const devices = 4
|
||||
pairs := make([]*TokenPair, 0, devices)
|
||||
for i := 0; i < devices; i++ {
|
||||
session, err := sessions.Create(ctx, CreateAuthSessionInput{
|
||||
UserID: user.ID,
|
||||
DeviceID: fmt.Sprintf("device-%02d", i),
|
||||
DeviceName: fmt.Sprintf("Phone %d", i+1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create session %d: %v", i+1, err)
|
||||
}
|
||||
pair, err := tokens.IssuePairForSession(ctx, user.ID, user.Role, user.Tier, session.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("issue pair %d: %v", i+1, err)
|
||||
}
|
||||
pairs = append(pairs, pair)
|
||||
}
|
||||
|
||||
for i, pair := range pairs {
|
||||
row, err := repos.RefreshToken.FindByHash(ctx, repository.HashToken(pair.RefreshToken))
|
||||
if err != nil {
|
||||
t.Fatalf("load device %d token: %v", i+1, err)
|
||||
}
|
||||
if row == nil {
|
||||
t.Fatalf("device %d refresh token missing", i+1)
|
||||
}
|
||||
if row.Revoked {
|
||||
t.Fatalf("device %d refresh token was revoked by another device", i+1)
|
||||
}
|
||||
if _, err := tokens.Refresh(ctx, pair.RefreshToken); err != nil {
|
||||
t.Fatalf("device %d refresh failed: %v", i+1, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerIdentityPersistsAcrossInstances(t *testing.T) {
|
||||
db := newServiceTestDB(t, &model.Setting{})
|
||||
repos := repository.New(db)
|
||||
|
||||
+81
-37
@@ -5,7 +5,7 @@
|
||||
//
|
||||
// 两种方言的备份形态不同:
|
||||
// - SQLite:用 "VACUUM INTO"(3.27.0 起可用)产出一个一致的库文件,
|
||||
// 扩展名 .db;恢复即「替换库文件 + 重启」。
|
||||
// 扩展名 .db;恢复先暂存,在下次启动打开数据库前替换文件。
|
||||
// - PostgreSQL:没有单文件语义,用逻辑备份(见 backup_postgres.go),
|
||||
// 扩展名 .json.gz;恢复是在单个事务内重建全部表。
|
||||
package service
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/database"
|
||||
)
|
||||
|
||||
// BackupService manages database backups.
|
||||
@@ -44,6 +45,7 @@ type BackupInfo struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Automatic bool `json:"automatic"`
|
||||
}
|
||||
|
||||
func (b *BackupService) backupDir() string {
|
||||
@@ -69,6 +71,23 @@ func (b *BackupService) backupFilePath(filename string) (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// ResolvePath returns the absolute path of a listed backup after applying the
|
||||
// same filename and directory checks used by restore and delete operations.
|
||||
func (b *BackupService) ResolvePath(filename string) (string, error) {
|
||||
path, err := b.backupFilePath(filename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", errors.New("backup is not a regular file")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// isPostgres reports whether the service is backed by PostgreSQL.
|
||||
//
|
||||
// PostgreSQL 不支持 SQLite 的 "VACUUM INTO",按文件替换恢复也会写错目标,
|
||||
@@ -80,11 +99,23 @@ func (b *BackupService) isPostgres() bool {
|
||||
// Create produces a new hot backup. SQLite uses "VACUUM INTO"; PostgreSQL uses
|
||||
// a logical dump.
|
||||
func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, false)
|
||||
}
|
||||
|
||||
func (b *BackupService) CreateAutomatic(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, true)
|
||||
}
|
||||
|
||||
func (b *BackupService) create(ctx context.Context, automatic bool) (*BackupInfo, error) {
|
||||
dir := b.backupDir()
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := time.Now().Format("20060102_150405")
|
||||
now := time.Now()
|
||||
ts := fmt.Sprintf("%s_%09d", now.Format("20060102_150405"), now.Nanosecond())
|
||||
if automatic {
|
||||
ts = "auto_" + ts
|
||||
}
|
||||
|
||||
var (
|
||||
dst string
|
||||
@@ -108,15 +139,24 @@ func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
Path: dst,
|
||||
Size: stat.Size(),
|
||||
CreatedAt: stat.ModTime(),
|
||||
Automatic: automatic,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *BackupService) createSQLiteBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), fmt.Sprintf("mebox_%s.db", stamp))
|
||||
// VACUUM INTO creates a clean, non-WAL copy atomically.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
|
||||
tmp := dst + ".tmp"
|
||||
defer os.Remove(tmp)
|
||||
// Publish only completed backups, with credentials protected by file mode.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", tmp).Error; err != nil {
|
||||
return "", fmt.Errorf("VACUUM INTO failed: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmp, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -132,7 +172,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
}
|
||||
out := make([]BackupInfo, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !isBackupFilename(e.Name()) {
|
||||
if !e.Type().IsRegular() || !isBackupFilename(e.Name()) {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
@@ -144,6 +184,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
Path: filepath.Join(dir, e.Name()),
|
||||
Size: info.Size(),
|
||||
CreatedAt: info.ModTime(),
|
||||
Automatic: strings.HasPrefix(e.Name(), "mebox_auto_"),
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
@@ -152,6 +193,36 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Prune keeps maxCount automatic backups for the current database dialect.
|
||||
// Manual backups and backups from another dialect are never pruned.
|
||||
// A value below one disables pruning so an invalid setting cannot erase all
|
||||
// recoverable copies.
|
||||
func (b *BackupService) Prune(maxCount int) error {
|
||||
if maxCount < 1 {
|
||||
return nil
|
||||
}
|
||||
items, err := b.List()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible := items[:0]
|
||||
for _, item := range items {
|
||||
if item.Automatic && strings.HasSuffix(item.Filename, postgresBackupSuffix) == b.isPostgres() {
|
||||
eligible = append(eligible, item)
|
||||
}
|
||||
}
|
||||
items = eligible
|
||||
if maxCount >= len(items) {
|
||||
return nil
|
||||
}
|
||||
for _, item := range items[maxCount:] {
|
||||
if err := b.Delete(item.Filename); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a single backup file.
|
||||
func (b *BackupService) Delete(filename string) error {
|
||||
path, err := b.backupFilePath(filename)
|
||||
@@ -161,12 +232,10 @@ func (b *BackupService) Delete(filename string) error {
|
||||
return os.Remove(path)
|
||||
}
|
||||
|
||||
// Restore replaces the live database with the given backup.
|
||||
// WARNING: this is destructive. SQLite restores by swapping the database file
|
||||
// (the caller should shut down the server afterwards); PostgreSQL restores by
|
||||
// rebuilding every table inside a single transaction.
|
||||
// Restore stages SQLite backups for the next startup. PostgreSQL restores
|
||||
// rebuild every table inside a single transaction.
|
||||
func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
src, err := b.backupFilePath(filename)
|
||||
src, err := b.ResolvePath(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -182,35 +251,10 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
if !strings.HasSuffix(filename, ".db") {
|
||||
return errors.New("该备份是 PostgreSQL 逻辑备份,当前运行在 SQLite 上,无法恢复")
|
||||
}
|
||||
return b.restoreSQLiteBackup(filename, src)
|
||||
}
|
||||
|
||||
func (b *BackupService) restoreSQLiteBackup(filename, src string) error {
|
||||
dbPath, err := filepath.Abs(b.cfg.Database.DBPath)
|
||||
if err != nil || dbPath == "" {
|
||||
return errors.New("invalid database path")
|
||||
}
|
||||
// Strategy: rename live → live.old, copy backup → live, delete old.
|
||||
old := dbPath + ".before_restore"
|
||||
if err := os.Rename(dbPath, old); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
if err := database.StageSQLiteRestore(ctx, b.cfg, src); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(src) // #nosec G304 -- src is constrained by backupFilePath to the configured backups directory.
|
||||
if err != nil {
|
||||
// Revert
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dbPath, data, 0o600); err != nil { // #nosec G703 -- database path is trusted process config and normalized before restore.
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(old)
|
||||
// Also remove WAL/SHM so SQLite opens the fresh file cleanly.
|
||||
_ = os.Remove(dbPath + "-wal")
|
||||
_ = os.Remove(dbPath + "-shm")
|
||||
b.log.Warn("database restored from backup — restart the server",
|
||||
zap.String("backup", filename))
|
||||
b.log.Warn("SQLite backup staged; restart to apply", zap.String("backup", filename))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -87,7 +88,8 @@ func postgresBackupModelSchemas(db *gorm.DB) (map[string]*schema.Schema, map[str
|
||||
// createPostgresBackup 把全库导出成一个 gzip 压缩的 JSON Lines 文件。
|
||||
func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), "mebox_"+stamp+postgresBackupSuffix)
|
||||
file, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
tmp := dst + ".tmp"
|
||||
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -96,7 +98,7 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
_ = file.Close()
|
||||
if !complete {
|
||||
// 半截备份文件比没有备份更危险:留着会让人以为有可用的还原点。
|
||||
_ = os.Remove(dst)
|
||||
_ = os.Remove(tmp)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -109,17 +111,30 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("写入备份头失败: %w", err)
|
||||
}
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(b.db, m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !b.db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := b.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return "", err
|
||||
var options []*sql.TxOptions
|
||||
if b.isPostgres() {
|
||||
options = append(options, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||
}
|
||||
// Every table must come from the same snapshot, including while background
|
||||
// workers are inserting related rows during an automatic backup.
|
||||
if err := b.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
snapshot := *b
|
||||
snapshot.db = tx
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(tx, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !tx.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := snapshot.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, options...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return "", fmt.Errorf("压缩备份失败: %w", err)
|
||||
@@ -127,6 +142,12 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
if err := file.Sync(); err != nil {
|
||||
return "", fmt.Errorf("写入备份失败: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
complete = true
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBackupRetentionPreservesManualAndOtherDialectBackups(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
names := []string{"mebox_auto_new.db", "mebox_auto_old.db", "mebox_manual.db", "mebox_auto_postgres.json.gz"}
|
||||
for i, name := range names {
|
||||
path := filepath.Join(svc.backupDir(), name)
|
||||
if err := os.WriteFile(path, []byte("fixture"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
when := time.Now().Add(-time.Duration(i) * time.Hour)
|
||||
if err := os.Chtimes(path, when, when); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := svc.Prune(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range names {
|
||||
_, err := os.Stat(filepath.Join(svc.backupDir(), name))
|
||||
if name == "mebox_auto_old.db" {
|
||||
if !os.IsNotExist(err) {
|
||||
t.Fatalf("old automatic backup was retained: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("backup %s should have been preserved: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupCreateUsesUniquePrivateCompletedFiles(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
manual, err := svc.Create(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
automatic, err := svc.CreateAutomatic(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manual.Filename == automatic.Filename || manual.Automatic || !automatic.Automatic {
|
||||
t.Fatalf("invalid backup identities: manual=%#v automatic=%#v", manual, automatic)
|
||||
}
|
||||
items, err := svc.List()
|
||||
if err != nil || len(items) != 2 {
|
||||
t.Fatalf("completed backup list: items=%#v err=%v", items, err)
|
||||
}
|
||||
for _, item := range items {
|
||||
info, err := os.Stat(item.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("backup permissions are not private: %s err=%v", item.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,34 @@
|
||||
// 下载直链进程级缓存。
|
||||
//
|
||||
// 115 的 /open/ufile/downurl 换取结果在一段时间内有效,且换取请求是
|
||||
// WAF 风控重点关注的密集接口。缓存按 pickcode 存直链、避免对同一文件
|
||||
// 反复换取;下载失败(http 403/404 等)时由调用方主动清除对应条目,
|
||||
// 下一轮重试会重新换取链接。
|
||||
// 115 的 /open/ufile/downurl 换取结果带 `t=` 过期时间戳(实测为「换取时刻 +
|
||||
// 1800 秒」,即 30 分钟),且换取请求是 WAF 风控重点关注的密集接口。缓存按
|
||||
// pickcode 存直链、避免对同一文件反复换取;下载失败(http 403/404 等)时由
|
||||
// 调用方主动清除对应条目,下一轮重试会重新换取链接。
|
||||
package cloud115
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// downloadURLCacheTTL 直链缓存时长。115 直链一般有效 1 小时左右,
|
||||
// 取 45 分钟留出余量。QMediaSync 同类缓存使用 50 分钟。
|
||||
const downloadURLCacheTTL = 45 * time.Minute
|
||||
// downloadURLCacheMaxTTL 直链缓存上限:仅在解析不出直链自带过期时间(`t=`)
|
||||
// 时使用。115 直链实际有效 30 分钟,这里取 45 分钟只是兜底上限,正常情况下
|
||||
// 由 linkExpirySafety 提前失效,避免把已经过期的直链发给客户端。
|
||||
const downloadURLCacheMaxTTL = 45 * time.Minute
|
||||
|
||||
// linkExpirySafety 提前量:直链自身的过期时间戳到达前就停止使用,留出客户端
|
||||
// 建连与传输的余量,防止出现「服务端认为还有效、CDN 已经 403」的窗口。
|
||||
const linkExpirySafety = 2 * time.Minute
|
||||
|
||||
// maxCachedURLs 超过该数量时顺带清理过期条目,防止 map 无限膨胀。
|
||||
const maxCachedURLs = 10000
|
||||
|
||||
type urlCacheEntry struct {
|
||||
url string
|
||||
createdAt time.Time
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
@@ -37,6 +45,43 @@ func urlCacheKey(pickCode, ua string) string {
|
||||
return pickCode + "@" + ua
|
||||
}
|
||||
|
||||
// LinkExpiry 解析 115 直链自带的过期时间戳(`t=` 参数,单位秒)。
|
||||
// 解析不出时返回零值,调用方按兜底 TTL 处理。
|
||||
func LinkExpiry(rawURL string) time.Time {
|
||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil || u == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
raw := strings.TrimSpace(u.Query().Get("t"))
|
||||
if raw == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
sec, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil || sec <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(sec, 0)
|
||||
}
|
||||
|
||||
// cacheExpiry 计算写入缓存时应使用的过期时刻:跟随直链自身的 `t=` 并提前
|
||||
// linkExpirySafety,解析不出时退回 downloadURLCacheMaxTTL。返回零值表示这条
|
||||
// 直链已经过期,不应写入缓存。
|
||||
func cacheExpiry(rawURL string, now time.Time) time.Time {
|
||||
maxExpiry := now.Add(downloadURLCacheMaxTTL)
|
||||
linkExpiry := LinkExpiry(rawURL)
|
||||
if linkExpiry.IsZero() {
|
||||
return maxExpiry
|
||||
}
|
||||
expiry := linkExpiry.Add(-linkExpirySafety)
|
||||
if !expiry.After(now) {
|
||||
return time.Time{}
|
||||
}
|
||||
if expiry.After(maxExpiry) {
|
||||
return maxExpiry
|
||||
}
|
||||
return expiry
|
||||
}
|
||||
|
||||
// GetDownloadURLCache 返回未过期的缓存直链;不存在或已过期返回空串。
|
||||
func GetDownloadURLCache(pickCode string, uas ...string) string {
|
||||
if pickCode == "" {
|
||||
@@ -59,11 +104,17 @@ func GetDownloadURLCache(pickCode string, uas ...string) string {
|
||||
return entry.url
|
||||
}
|
||||
|
||||
// SetDownloadURLCache 写入直链缓存。
|
||||
// SetDownloadURLCache 写入直链缓存。过期时刻跟随直链自身的 `t=`,已经过期的
|
||||
// 直链直接丢弃,避免把死链缓存下来。
|
||||
func SetDownloadURLCache(pickCode string, url string, uas ...string) {
|
||||
if pickCode == "" || url == "" {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
expiresAt := cacheExpiry(url, now)
|
||||
if expiresAt.IsZero() {
|
||||
return
|
||||
}
|
||||
ua := ""
|
||||
if len(uas) > 0 {
|
||||
ua = uas[0]
|
||||
@@ -71,7 +122,6 @@ func SetDownloadURLCache(pickCode string, url string, uas ...string) {
|
||||
urlCacheMu.Lock()
|
||||
defer urlCacheMu.Unlock()
|
||||
if len(urlCache) >= maxCachedURLs {
|
||||
now := time.Now()
|
||||
for k, e := range urlCache {
|
||||
if now.After(e.expiresAt) {
|
||||
delete(urlCache, k)
|
||||
@@ -79,7 +129,7 @@ func SetDownloadURLCache(pickCode string, url string, uas ...string) {
|
||||
}
|
||||
}
|
||||
key := urlCacheKey(pickCode, ua)
|
||||
urlCache[key] = urlCacheEntry{url: url, expiresAt: time.Now().Add(downloadURLCacheTTL)}
|
||||
urlCache[key] = urlCacheEntry{url: url, createdAt: now, expiresAt: expiresAt}
|
||||
}
|
||||
|
||||
// ClearDownloadURLCache 删除指定 pickcode 的所有缓存(下载得到非 2xx 时调用)。
|
||||
@@ -95,3 +145,32 @@ func ClearDownloadURLCache(pickCode string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ClearDownloadURLCacheIfOlderThan 删除指定 pickcode 下「已经存在超过 minAge」
|
||||
// 的缓存条目,返回是否删除了至少一条。
|
||||
//
|
||||
// 客户端上报「直链被 CDN 拒绝」时用它强制换链:既让下一次解析拿到新地址,
|
||||
// 又不会让同一秒内的重试风暴把 115 换链接口打出突发。
|
||||
func ClearDownloadURLCacheIfOlderThan(pickCode string, minAge time.Duration) bool {
|
||||
if pickCode == "" {
|
||||
return false
|
||||
}
|
||||
if minAge < 0 {
|
||||
minAge = 0
|
||||
}
|
||||
now := time.Now()
|
||||
removed := false
|
||||
urlCacheMu.Lock()
|
||||
defer urlCacheMu.Unlock()
|
||||
for k, e := range urlCache {
|
||||
if k != pickCode && !strings.HasPrefix(k, pickCode+"@") {
|
||||
continue
|
||||
}
|
||||
if now.Sub(e.createdAt) < minAge {
|
||||
continue
|
||||
}
|
||||
delete(urlCache, k)
|
||||
removed = true
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cloud115
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -29,3 +30,72 @@ func TestDownloadURLCache(t *testing.T) {
|
||||
t.Fatalf("cleared entry should be a miss, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 115 直链自带 `t=` 过期时间戳(实测为换取时刻 + 1800 秒)。缓存必须跟随它,
|
||||
// 否则会出现「服务端认为还有效、CDN 已经 403」的窗口:直链被缓存到 45 分钟,
|
||||
// 但第 30 分钟起客户端就必然拿到 403。
|
||||
func TestDownloadURLCacheExpiryFollowsLinkTimestamp(t *testing.T) {
|
||||
now := time.Now()
|
||||
link := fmt.Sprintf("https://cdn/1/a.mkv?t=%d&u=1&k=abc", now.Add(30*time.Minute).Unix())
|
||||
|
||||
got := cacheExpiry(link, now)
|
||||
want := now.Add(30*time.Minute - linkExpirySafety)
|
||||
if diff := got.Sub(want); diff > time.Second || diff < -time.Second {
|
||||
t.Fatalf("cacheExpiry = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// 解析不出 t= 时退回兜底 TTL,保持旧行为。
|
||||
fallback := cacheExpiry("https://cdn/1/a.mkv", now)
|
||||
if diff := fallback.Sub(now.Add(downloadURLCacheMaxTTL)); diff > time.Second || diff < -time.Second {
|
||||
t.Fatalf("fallback cacheExpiry = %v, want %v", fallback, now.Add(downloadURLCacheMaxTTL))
|
||||
}
|
||||
|
||||
// 直链本身已经过期(或只剩不足提前量的寿命)时不进缓存。
|
||||
if got := cacheExpiry(fmt.Sprintf("https://cdn/1/a.mkv?t=%d", now.Add(-time.Minute).Unix()), now); !got.IsZero() {
|
||||
t.Fatalf("expired link must not be cached, got expiry %v", got)
|
||||
}
|
||||
if got := cacheExpiry(fmt.Sprintf("https://cdn/1/a.mkv?t=%d", now.Add(time.Minute).Unix()), now); !got.IsZero() {
|
||||
t.Fatalf("link expiring inside the safety margin must not be cached, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 客户端上报「直链被 CDN 拒绝」时按 pickcode 清缓存,但要挡住刚写入的条目,
|
||||
// 避免重试风暴把 115 换链接口打出突发。
|
||||
func TestClearDownloadURLCacheIfOlderThan(t *testing.T) {
|
||||
urlCacheMu.Lock()
|
||||
urlCache = map[string]urlCacheEntry{}
|
||||
urlCacheMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
urlCacheMu.Lock()
|
||||
urlCache = map[string]urlCacheEntry{}
|
||||
urlCacheMu.Unlock()
|
||||
})
|
||||
|
||||
future := time.Now().Add(30 * time.Minute)
|
||||
link := fmt.Sprintf("https://cdn/1/a.mkv?t=%d", future.Unix())
|
||||
SetDownloadURLCache("pick-fresh", link, "ua-a")
|
||||
if ClearDownloadURLCacheIfOlderThan("pick-fresh", time.Minute) {
|
||||
t.Fatal("fresh entry must survive the min-age guard")
|
||||
}
|
||||
if got := GetDownloadURLCache("pick-fresh", "ua-a"); got != link {
|
||||
t.Fatalf("fresh entry lost, got %q", got)
|
||||
}
|
||||
|
||||
SetDownloadURLCache("pick-old", link, "ua-a")
|
||||
urlCacheMu.Lock()
|
||||
entry := urlCache[urlCacheKey("pick-old", "ua-a")]
|
||||
entry.createdAt = time.Now().Add(-10 * time.Minute)
|
||||
urlCache[urlCacheKey("pick-old", "ua-a")] = entry
|
||||
urlCacheMu.Unlock()
|
||||
|
||||
if !ClearDownloadURLCacheIfOlderThan("pick-old", time.Minute) {
|
||||
t.Fatal("stale entry should be cleared")
|
||||
}
|
||||
if got := GetDownloadURLCache("pick-old", "ua-a"); got != "" {
|
||||
t.Fatalf("stale entry should be gone, got %q", got)
|
||||
}
|
||||
// 无关 pickcode 不受影响。
|
||||
if got := GetDownloadURLCache("pick-fresh", "ua-a"); got != link {
|
||||
t.Fatalf("unrelated pickcode lost, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCloud115HLSChildRejectsMismatchedSessionMediaBeforeFetch(t *testing.T) {
|
||||
fetched := false
|
||||
proxy := &Cloud115HLSProxy{sessions: map[string]*cloud115HLSSession{}}
|
||||
proxy.client = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
fetched = true
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"video/mp2t"}}, Body: io.NopCloser(strings.NewReader("private-segment")), Request: req}, nil
|
||||
})}
|
||||
proxy.sessions["session-b"] = &cloud115HLSSession{
|
||||
ID: "session-b", MediaID: "media-b", ExpiresAt: time.Now().Add(time.Hour),
|
||||
entries: map[string]string{"segment": "https://cpats01.115.com/seg.ts"},
|
||||
}
|
||||
for _, query := range []string{"?media_id=media-a", ""} {
|
||||
fetched = false
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/cloud115/hls/session-b/segment"+query, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
err := proxy.ServeChild(context.Background(), rec, req, "session-b", "segment")
|
||||
if err == nil || fetched {
|
||||
t.Errorf("query %q must reject the wrong/missing media before fetch; err=%v fetched=%v", query, err, fetched)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,8 +20,9 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCloud115HLSSessionNotFound = errors.New("115 hls session not found")
|
||||
ErrCloud115HLSUpstreamExpired = errors.New("115 hls upstream expired")
|
||||
ErrCloud115HLSSessionNotFound = errors.New("115 hls session not found")
|
||||
ErrCloud115HLSUpstreamExpired = errors.New("115 hls upstream expired")
|
||||
ErrCloud115HLSSessionMediaMismatch = errors.New("115 hls session media mismatch")
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -131,6 +132,9 @@ func (p *Cloud115HLSProxy) ServeChild(ctx context.Context, w http.ResponseWriter
|
||||
if !ok {
|
||||
return ErrCloud115HLSSessionNotFound
|
||||
}
|
||||
if r == nil || strings.TrimSpace(r.URL.Query().Get("media_id")) != session.MediaID {
|
||||
return ErrCloud115HLSSessionMediaMismatch
|
||||
}
|
||||
resp, err := p.fetchUpstream(ctx, upstream, r.Header.Get("Range"))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -186,6 +190,15 @@ func (p *Cloud115HLSProxy) ServeChild(ctx context.Context, w http.ResponseWriter
|
||||
return err
|
||||
}
|
||||
|
||||
// SessionMediaID resolves the server-owned media identity before handler authorization.
|
||||
func (p *Cloud115HLSProxy) SessionMediaID(sessionID, key string) (string, error) {
|
||||
session, _, ok := p.lookup(sessionID, key)
|
||||
if !ok {
|
||||
return "", ErrCloud115HLSSessionNotFound
|
||||
}
|
||||
return session.MediaID, nil
|
||||
}
|
||||
|
||||
func (p *Cloud115HLSProxy) newSession(mediaID string, definition int) *cloud115HLSSession {
|
||||
now := time.Now()
|
||||
return &cloud115HLSSession{
|
||||
|
||||
@@ -104,11 +104,13 @@ func (s *Cloud115PlaybackService) HLSProxy() *Cloud115HLSProxy {
|
||||
return s.proxy
|
||||
}
|
||||
|
||||
// PlaybackInfo 返回媒体在 direct/115-cloud/local 三种模式下的能力描述。
|
||||
//
|
||||
// requestedDefinition 为空或 0 时默认选择 1080P;该档位不可用时返回
|
||||
// transcoding 状态(并只触发一次加速转码,避免轮询风暴)。
|
||||
// PlaybackInfo only reads direct/115-cloud/local playback capabilities.
|
||||
// Cloud work is queued exclusively by StartTranscode, never by polling.
|
||||
func (s *Cloud115PlaybackService) PlaybackInfo(ctx context.Context, mediaID string, requestedDefinition int) (*PlaybackInfo, error) {
|
||||
return s.playbackInfo(ctx, mediaID, requestedDefinition, false)
|
||||
}
|
||||
|
||||
func (s *Cloud115PlaybackService) playbackInfo(ctx context.Context, mediaID string, requestedDefinition int, startTranscode bool) (*PlaybackInfo, error) {
|
||||
if s == nil || s.repo == nil || s.repo.Media == nil {
|
||||
return nil, ErrMediaNotFound
|
||||
}
|
||||
@@ -179,13 +181,26 @@ func (s *Cloud115PlaybackService) PlaybackInfo(ctx context.Context, mediaID stri
|
||||
return info, nil
|
||||
}
|
||||
|
||||
info.Transcode = s.ensureCloudTranscode(ctx, account.ID, pickCode, selected, client)
|
||||
if startTranscode {
|
||||
info.Transcode = s.ensureCloudTranscode(ctx, account.ID, pickCode, selected, client)
|
||||
} else {
|
||||
info.Transcode = Cloud115TranscodeState{
|
||||
State: "idle", Definition: selected.ID, Message: "该清晰度尚未就绪,可请求 115 云端转码", RetryAfterSec: 5,
|
||||
}
|
||||
s.mu.Lock()
|
||||
if last, ok := s.pushState[account.ID+":"+pickCode]; ok {
|
||||
info.Transcode.State = "transcoding"
|
||||
info.Transcode.Message = last.message
|
||||
info.Transcode.StartedAt = last.at.Unix()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// StartTranscode 显式触发某个清晰度的云端转码,返回最新状态。
|
||||
func (s *Cloud115PlaybackService) StartTranscode(ctx context.Context, mediaID string, definition int) (*PlaybackInfo, error) {
|
||||
return s.PlaybackInfo(ctx, mediaID, definition)
|
||||
return s.playbackInfo(ctx, mediaID, definition, true)
|
||||
}
|
||||
|
||||
// ResolveCloud115URL 返回指定清晰度的 115 云端 m3u8 地址。
|
||||
@@ -258,6 +273,9 @@ func (s *Cloud115PlaybackService) ensureCloudTranscode(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Reserve the in-flight attempt before releasing the lock. Two simultaneous
|
||||
// explicit starts must not both submit a push while the first awaits 115.
|
||||
s.pushState[key] = cloud115PushAttempt{at: now, message: "正在提交 115 云端转码请求…"}
|
||||
s.mu.Unlock()
|
||||
|
||||
err := client.SubmitVideoPush(ctx, pickCode, "vip_push")
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
func TestCloud115PlaybackInfoTriggersTranscodeOnce(t *testing.T) {
|
||||
func TestCloud115PlaybackInfoIsReadOnlyAndStartTriggersTranscodeOnce(t *testing.T) {
|
||||
var playCalls int32
|
||||
var pushCalls int32
|
||||
var ready int32
|
||||
@@ -93,9 +93,16 @@ func TestCloud115PlaybackInfoTriggersTranscodeOnce(t *testing.T) {
|
||||
}
|
||||
svc := NewCloud115PlaybackService(nil, zap.NewNop(), repos, strm)
|
||||
|
||||
info, err := svc.PlaybackInfo(ctx, media.ID, 0)
|
||||
if _, err := svc.PlaybackInfo(ctx, media.ID, 0); err != nil {
|
||||
t.Fatalf("read playback info: %v", err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&pushCalls); got != 0 {
|
||||
t.Fatalf("read-only playback info triggered %d cloud transcode pushes", got)
|
||||
}
|
||||
|
||||
info, err := svc.StartTranscode(ctx, media.ID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("first playback info: %v", err)
|
||||
t.Fatalf("start transcode: %v", err)
|
||||
}
|
||||
if info.Provider != model.StrmProvider115 || info.DefaultQuality != "4" {
|
||||
t.Fatalf("unexpected playback info: %#v", info)
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
func TestCloud115ConcurrentStartsDoNotDuplicateInFlightPush(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
var pushCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if pushCalls.Add(1) == 1 {
|
||||
close(started)
|
||||
}
|
||||
<-release
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"state": true, "code": 0, "data": []any{}})
|
||||
}))
|
||||
defer server.Close()
|
||||
defer releaseOnce.Do(func() { close(release) })
|
||||
oldBase := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = server.URL
|
||||
defer func() { cloud115.ProAPIBase = oldBase }()
|
||||
client := cloud115.NewOpenClient("review-app", "access", "refresh")
|
||||
svc := NewCloud115PlaybackService(nil, zap.NewNop(), nil, nil)
|
||||
quality := PlaybackQuality{ID: "4", Label: "1080P", Source: "cloud"}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
first := make(chan struct{})
|
||||
go func() { defer close(first); svc.ensureCloudTranscode(ctx, "account", "pick", quality, client) }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("first push did not reach the local server")
|
||||
}
|
||||
second := make(chan Cloud115TranscodeState, 1)
|
||||
go func() { second <- svc.ensureCloudTranscode(ctx, "account", "pick", quality, client) }()
|
||||
secondReceived := false
|
||||
select {
|
||||
case state := <-second:
|
||||
secondReceived = true
|
||||
if state.State != "transcoding" {
|
||||
t.Errorf("in-flight request should be reported as transcoding: %#v", state)
|
||||
}
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
t.Error("second start issued another request instead of reusing the in-flight push")
|
||||
}
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
select {
|
||||
case <-first:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("first push did not finish")
|
||||
}
|
||||
// The duplicate path is awaited before restoring the package-wide fake API.
|
||||
if !secondReceived {
|
||||
select {
|
||||
case <-second:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("second start did not finish")
|
||||
}
|
||||
}
|
||||
if got := pushCalls.Load(); got != 1 {
|
||||
t.Fatalf("pushes=%d, want 1", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var ErrInvalidHLSPlaybackScope = errors.New("invalid HLS playback scope")
|
||||
var ErrHLSGenerationSuperseded = errors.New("HLS playback generation superseded")
|
||||
|
||||
// HLSPlaybackScope separates a media's independent users and player instances.
|
||||
// UserID must come from server authentication, never from a request parameter.
|
||||
type HLSPlaybackScope struct {
|
||||
UserID string
|
||||
SessionID string
|
||||
}
|
||||
|
||||
func HLSPlaybackScopeForRequest(r *http.Request, userID string) (HLSPlaybackScope, error) {
|
||||
scope := HLSPlaybackScope{UserID: strings.TrimSpace(userID)}
|
||||
if scope.UserID == "" || r == nil {
|
||||
return scope, ErrInvalidHLSPlaybackScope
|
||||
}
|
||||
for _, key := range []string{"session_id", "PlaySessionId", "playSessionId", "DeviceId", "deviceId"} {
|
||||
if value := strings.TrimSpace(r.URL.Query().Get(key)); value != "" {
|
||||
scope.SessionID = value
|
||||
break
|
||||
}
|
||||
}
|
||||
if scope.SessionID == "" {
|
||||
if device := strings.TrimSpace(r.Header.Get("X-Emby-Device-Id")); device != "" {
|
||||
scope.SessionID = device
|
||||
}
|
||||
}
|
||||
if len(scope.SessionID) > 256 || strings.IndexFunc(scope.SessionID, unicode.IsControl) >= 0 {
|
||||
return scope, ErrInvalidHLSPlaybackScope
|
||||
}
|
||||
// Clients without a session/device identifier remain compatible but only
|
||||
// share a legacy task with the same authenticated user, never other users.
|
||||
return scope, nil
|
||||
}
|
||||
|
||||
func (scope HLSPlaybackScope) jobKey(mediaID string) string {
|
||||
if scope.UserID == "" && scope.SessionID == "" {
|
||||
return mediaID // Existing internal callers/tests retain their legacy API.
|
||||
}
|
||||
sum := sha256.Sum256([]byte(scope.UserID + "\x00" + mediaID + "\x00" + scope.SessionID))
|
||||
return "player-" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func hlsPlaybackJobKey(mediaID string, scopes []HLSPlaybackScope) string {
|
||||
if len(scopes) == 0 {
|
||||
return mediaID
|
||||
}
|
||||
return scopes[0].jobKey(mediaID)
|
||||
}
|
||||
|
||||
func (job *hlsJob) key() string {
|
||||
if job.jobKey != "" {
|
||||
return job.jobKey
|
||||
}
|
||||
return job.mediaID
|
||||
}
|
||||
|
||||
func (t *TranscoderService) jobPlaylistPath(job *hlsJob) string {
|
||||
dir := job.outputDir
|
||||
if dir == "" {
|
||||
dir = t.HLSDir(job.key())
|
||||
}
|
||||
return filepath.Join(dir, "index.m3u8")
|
||||
}
|
||||
|
||||
// hlsOutputDir captures one immutable run directory. Even if a new run starts
|
||||
// after this check, the request cannot accidentally open its replacement files.
|
||||
func (t *TranscoderService) hlsOutputDir(key string, generation int64) (string, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
job := t.jobs[key]
|
||||
if job == nil {
|
||||
return "", ErrMediaNotFound
|
||||
}
|
||||
if job.seekGen != generation {
|
||||
return "", ErrHLSGenerationSuperseded
|
||||
}
|
||||
t.touchJobLocked(key)
|
||||
if job.outputDir != "" {
|
||||
return job.outputDir, nil
|
||||
}
|
||||
return t.HLSDir(key), nil
|
||||
}
|
||||
|
||||
func (t *TranscoderService) StopPlayback(mediaID string, scope HLSPlaybackScope) {
|
||||
key := scope.jobKey(mediaID)
|
||||
unlock := t.lockJobStart(key)
|
||||
defer unlock()
|
||||
t.mu.Lock()
|
||||
job := t.detachJobLocked(key)
|
||||
t.mu.Unlock()
|
||||
waitJobExit(job, 12*time.Second)
|
||||
// Keep cleanup under the same gate as start: a delayed stop must not wipe
|
||||
// an output directory created by a concurrent replacement request.
|
||||
_ = os.RemoveAll(t.HLSDir(key))
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
func cachedScopedHLS(t *testing.T, scopes ...HLSPlaybackScope) (*StreamService, *TranscoderService, map[string]string) {
|
||||
t.Helper()
|
||||
cfg := &config.Config{}
|
||||
cfg.Cache.CacheDir = t.TempDir()
|
||||
cfg.Transcoder.Enabled = true
|
||||
db := newServiceTestDB(t)
|
||||
if err := db.Create(&model.Media{Base: model.Base{ID: "movie"}, Path: "unused-cached-source", DurationSec: 60}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder := NewTranscoderService(cfg, zap.NewNop(), repository.New(db), nil)
|
||||
paths := map[string]string{}
|
||||
for _, scope := range scopes {
|
||||
key := scope.jobKey("movie")
|
||||
dir := filepath.Join(transcoder.HLSDir(key), "run-cached")
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
playlist := filepath.Join(dir, "index.m3u8")
|
||||
if err := os.WriteFile(playlist, []byte("#EXTM3U\n#EXTINF:4,\nseg_00000.ts\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "seg_00000.ts"), []byte(scope.UserID+"/"+scope.SessionID), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder.jobs[key] = &hlsJob{mediaID: "movie", jobKey: key, outputDir: dir, subtitleStream: -1, startedAt: time.Now(), cancel: func() {}}
|
||||
paths[key] = playlist
|
||||
}
|
||||
return NewStreamService(cfg, zap.NewNop(), repository.New(db), transcoder), transcoder, paths
|
||||
}
|
||||
|
||||
func TestHLSPlaybackEnsureReusesOnlyMatchingOwnerAndSession(t *testing.T) {
|
||||
a := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-b"}
|
||||
_, transcoder, paths := cachedScopedHLS(t, a, b)
|
||||
for _, scope := range []HLSPlaybackScope{a, b} {
|
||||
playlist, err := transcoder.EnsureJobFromSubtitleQuality(t.Context(), "movie", 0, 0, -1, "", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if playlist != paths[scope.jobKey("movie")] {
|
||||
t.Fatalf("scope %#v reused another session's playlist %q", scope, playlist)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackStopRemovesOnlyItsSessionOutput(t *testing.T) {
|
||||
a := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-b"}
|
||||
_, transcoder, paths := cachedScopedHLS(t, a, b)
|
||||
transcoder.StopPlayback("movie", a)
|
||||
if _, err := os.Stat(paths[a.jobKey("movie")]); !os.IsNotExist(err) {
|
||||
t.Fatalf("stopped session output must be cleaned up, err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(paths[b.jobKey("movie")]); err != nil {
|
||||
t.Fatalf("another player's output was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackIdleCleanupRemovesExpiredSessionOutput(t *testing.T) {
|
||||
a := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-b"}
|
||||
_, transcoder, paths := cachedScopedHLS(t, a, b)
|
||||
job := transcoder.jobs[a.jobKey("movie")]
|
||||
job.lastAccess = time.Now().Add(-2 * time.Minute)
|
||||
if !transcoder.stopIdleJob(job, time.Minute) {
|
||||
t.Fatal("expired session was not stopped")
|
||||
}
|
||||
if _, err := os.Stat(paths[a.jobKey("movie")]); !os.IsNotExist(err) {
|
||||
t.Fatalf("expired output was retained: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(paths[b.jobKey("movie")]); err != nil {
|
||||
t.Fatalf("active session output was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackPropagatesHeaderDeviceIdentityToSegments(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/hls/movie/index.m3u8?token=test", nil)
|
||||
req.Header.Set("X-Emby-Device-Id", "legacy-device")
|
||||
scope, err := HLSPlaybackScopeForRequest(req, "viewer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc, _, _ := cachedScopedHLS(t, scope)
|
||||
rec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSPlaylist(rec, req, "movie", scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var segment string
|
||||
for _, line := range strings.Split(rec.Body.String(), "\n") {
|
||||
if strings.HasPrefix(line, "seg_") {
|
||||
segment = line
|
||||
break
|
||||
}
|
||||
}
|
||||
u, err := url.Parse(segment)
|
||||
if err != nil || u.Query().Get("session_id") == "" {
|
||||
t.Fatalf("playlist must carry the header-only device scope to child URLs: %q err=%v", segment, err)
|
||||
}
|
||||
child := httptest.NewRequest(http.MethodGet, "/api/hls/movie/"+segment, nil)
|
||||
childScope, err := HLSPlaybackScopeForRequest(child, "viewer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if childScope.jobKey("movie") != scope.jobKey("movie") {
|
||||
t.Fatal("child request changed its owner's device scope")
|
||||
}
|
||||
childRec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSSegment(childRec, child, "movie", "seg_00000.ts", childScope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if childRec.Body.String() != "viewer/"+scope.SessionID {
|
||||
t.Fatalf("wrong scoped segment: %s", childRec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
func TestHLSPlaybackScopesIsolateSegmentReadsAndStop(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
cfg.Cache.CacheDir = t.TempDir()
|
||||
transcoder := NewTranscoderService(cfg, zap.NewNop(), nil, nil)
|
||||
svc := NewStreamService(cfg, zap.NewNop(), nil, transcoder)
|
||||
a := HLSPlaybackScope{UserID: "user-a", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "user-b", SessionID: "tab-a"}
|
||||
c := HLSPlaybackScope{UserID: "user-a", SessionID: "tab-c"}
|
||||
for _, scope := range []HLSPlaybackScope{a, b, c} {
|
||||
key := scope.jobKey("movie")
|
||||
dir := filepath.Join(cfg.Cache.CacheDir, key, "run-current")
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "seg_00000.ts"), []byte(scope.UserID+"/"+scope.SessionID), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder.jobs[key] = &hlsJob{mediaID: "movie", jobKey: key, outputDir: dir, seekGen: 100, cancel: func() {}}
|
||||
}
|
||||
for _, scope := range []HLSPlaybackScope{a, b, c} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/hls/movie/seg_00000.ts?_seek=100", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSSegment(rec, req, "movie", "seg_00000.ts", scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := rec.Body.String(), scope.UserID+"/"+scope.SessionID; got != want {
|
||||
t.Fatalf("segment=%q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
transcoder.StopPlayback("movie", a)
|
||||
if transcoder.jobs[a.jobKey("movie")] != nil {
|
||||
t.Fatal("stopped scope is still active")
|
||||
}
|
||||
if transcoder.jobs[b.jobKey("movie")] == nil || transcoder.jobs[c.jobKey("movie")] == nil {
|
||||
t.Fatal("stopping one player affected another user/tab")
|
||||
}
|
||||
// A delayed stop from a previous local-playback session must not affect a replacement.
|
||||
transcoder.StopPlayback("movie", a)
|
||||
if transcoder.jobs[c.jobKey("movie")] == nil {
|
||||
t.Fatal("old stop cancelled a new session")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackRejectsStaleSegmentGeneration(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
cfg.Cache.CacheDir = t.TempDir()
|
||||
transcoder := NewTranscoderService(cfg, zap.NewNop(), nil, nil)
|
||||
svc := NewStreamService(cfg, zap.NewNop(), nil, transcoder)
|
||||
scope := HLSPlaybackScope{UserID: "viewer", SessionID: "tab"}
|
||||
key := scope.jobKey("movie")
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "seg_00000.ts"), []byte("new-generation"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder.jobs[key] = &hlsJob{mediaID: "movie", jobKey: key, outputDir: dir, seekGen: 200, cancel: func() {}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/hls/movie/seg_00000.ts?_seek=100", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSSegment(rec, req, "movie", "seg_00000.ts", scope); err == nil {
|
||||
t.Fatal("stale segment request read a replacement generation")
|
||||
}
|
||||
if rec.Body.Len() != 0 {
|
||||
t.Fatal("stale request exposed replacement bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackScopeRequiresServerOwner(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/?session_id=tab&user_id=attacker-controlled", nil)
|
||||
if _, err := HLSPlaybackScopeForRequest(req, ""); err == nil {
|
||||
t.Fatal("missing authenticated owner was accepted")
|
||||
}
|
||||
a, err := HLSPlaybackScopeForRequest(req, "real-owner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.UserID != "real-owner" {
|
||||
t.Fatal("client parameter supplied the owner")
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodGet, "/?PlaySessionId=emby-session", nil)
|
||||
b, err := HLSPlaybackScopeForRequest(req, "real-owner")
|
||||
if err != nil || b.SessionID == "" {
|
||||
t.Fatalf("Emby session must stay scoped: %#v %v", b, err)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,10 @@ type ImageProxy struct {
|
||||
resizeSemMu sync.Mutex
|
||||
resizeSem chan struct{}
|
||||
|
||||
// resizeQueueWait 是一次请求最多为一格解码槽排队的时长。为 0 时用
|
||||
// defaultResizeQueueWait;测试会把它调小以免真的等待两秒。
|
||||
resizeQueueWait time.Duration
|
||||
|
||||
// libraryRootsFn returns the configured media library roots so that
|
||||
// sidecar poster/artwork files stored alongside media (under arbitrary
|
||||
// per-library paths) are allowed by isAllowedLocalPath. It is provided
|
||||
@@ -86,6 +90,12 @@ const (
|
||||
// imageRenditionCacheSubdir 存放挂载 Emby 直接按尺寸产出的成品图:它们
|
||||
// 已经是客户端要的最终尺寸,和派生缩略图一样属于长期保留的热路径缓存。
|
||||
imageRenditionCacheSubdir = "renditions"
|
||||
|
||||
// defaultResizeQueueWait 是请求为一格解码槽排队的最长时间。一张 3MP
|
||||
// 海报的解码+缩放约 0.15s,2 核机器两格槽位足以在几秒内消化一整屏
|
||||
// 海报;等待远好于「槽位忙就回退多兆字节原图」(那样每张图在外部网络
|
||||
// 下要十几秒)。超时后仍回退原图,保证请求不会被无限期挂住。
|
||||
defaultResizeQueueWait = 3 * time.Second
|
||||
)
|
||||
|
||||
// NewImageProxy is the constructor.
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -138,8 +139,8 @@ func fitSize(srcW, srcH, maxW, maxH int) (int, int) {
|
||||
|
||||
var errImageResizeTooLarge = errors.New("image source exceeds resize pixel budget")
|
||||
|
||||
// resizeImageData 按选项缩放图片并重新编码。返回 unchanged=true 表示原图
|
||||
// 本身已满足目标尺寸,调用方应直接输出原始字节(不重复编码、不损失质量)。
|
||||
// resizeImageData 按选项缩放图片并重新编码。返回 unchanged=true 表示源图
|
||||
// 已经是可直接下发的紧凑字节,调用方应直接输出原始字节(不重复编码)。
|
||||
func resizeImageData(data []byte, o imageResizeOptions) (out []byte, ctype string, unchanged bool, err error) {
|
||||
if len(data) == 0 || !o.active() {
|
||||
return data, detectContentType(data), true, nil
|
||||
@@ -151,12 +152,21 @@ func resizeImageData(data []byte, o imageResizeOptions) (out []byte, ctype strin
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return nil, "", false, errors.New("invalid image dimensions")
|
||||
}
|
||||
// 先用 DecodeConfig 判断是否需要解码整图:图已够小就零成本返回原字节。
|
||||
// 先用 DecodeConfig 判断是否需要缩放。尺寸已在目标框内时仍要重编码:
|
||||
// 刮削来源的海报常是 1000×1500、1MB 以上的 PNG,直接原样下发会让一处
|
||||
// 海报墙就从「几十 KB 缩略图」变成「几十 MB 原图」——外部网络下每张
|
||||
// 都要十几秒。只有本来就是紧凑 Web 格式(JPEG/WebP)且体积很小的源图
|
||||
// 才值得零成本直出。
|
||||
dstW, dstH := fitSize(cfg.Width, cfg.Height, o.MaxWidth, o.MaxHeight)
|
||||
if dstW == cfg.Width && dstH == cfg.Height {
|
||||
unchangedSize := dstW == cfg.Width && dstH == cfg.Height
|
||||
if unchangedSize && isCompactWebFormat(format) && len(data) <= compactImageSkipBytes {
|
||||
return data, detectContentType(data), true, nil
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > imageResizeMaxSourcePixels {
|
||||
if unchangedSize {
|
||||
// 尺寸已满足但像素量超出解码预算:宁可直出原图,也不冒 OOM 风险。
|
||||
return data, detectContentType(data), true, nil
|
||||
}
|
||||
return nil, "", false, errImageResizeTooLarge
|
||||
}
|
||||
|
||||
@@ -165,7 +175,12 @@ func resizeImageData(data []byte, o imageResizeOptions) (out []byte, ctype strin
|
||||
return nil, "", false, err
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dstW, dstH))
|
||||
resizeScaled(dst, src, cfg.Width, cfg.Height)
|
||||
if unchangedSize {
|
||||
// 不缩放时省掉一次全图重采样,只做格式转换。
|
||||
draw.Draw(dst, dst.Bounds(), src, src.Bounds().Min, draw.Src)
|
||||
} else {
|
||||
resizeScaled(dst, src, cfg.Width, cfg.Height)
|
||||
}
|
||||
|
||||
// 只有可能带透明的源格式才需要逐像素确认,避免 JPEG 的无谓遍历。
|
||||
// 写实海报的 PNG 通常比等价 JPEG 大一个数量级,因此在确认不含透明
|
||||
@@ -231,9 +246,12 @@ func isOpaqueImage(img *image.RGBA) bool {
|
||||
// resizeSourceKey 生成源图身份键:只覆盖源文件身份(路径 + 大小 + mtime),不含
|
||||
// 目标尺寸。同一源图的所有尺寸档位共用这个前缀,才能互相列举与派生(小图直接
|
||||
// 从已缓存的大图缩小,而不是重新解码原图)。
|
||||
//
|
||||
// 前缀里的 v3 是缓存格式版本:v2 时期「尺寸已在目标框内」的 PNG 会原样直出、
|
||||
// 不写缓存,而现在同样条件会重编码成 JPEG,旧缓存与新结果语义不同,必须换键。
|
||||
func (o imageResizeOptions) resizeSourceKey(sourceID string, stat os.FileInfo) string {
|
||||
h := sha256.New()
|
||||
_, _ = fmt.Fprintf(h, "v2|%s", sourceID)
|
||||
_, _ = fmt.Fprintf(h, "v3|%s", sourceID)
|
||||
if stat != nil {
|
||||
_, _ = fmt.Fprintf(h, "|%d|%d", stat.Size(), stat.ModTime().UnixNano())
|
||||
}
|
||||
@@ -356,13 +374,53 @@ func (p *ImageProxy) tryAcquireResizeSlot() (func(), bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// acquireResizeSlotFor 对中等体积的图只尝试一次槽位。忙则放弃缩放,
|
||||
// 避免背景图墙在 2 核机器上排成数秒。超大原图仍等待,以免直出多兆字节。
|
||||
func (p *ImageProxy) resizeQueueTimeout() time.Duration {
|
||||
if p != nil && p.resizeQueueWait > 0 {
|
||||
return p.resizeQueueWait
|
||||
}
|
||||
return defaultResizeQueueWait
|
||||
}
|
||||
|
||||
// acquireResizeSlotFor 对紧凑小图只尝试一次槽位,其余情况最多排队
|
||||
// resizeQueueTimeout。
|
||||
//
|
||||
// 从前这里对 1.5MB 以下的图一律「槽位忙就跳过缩放」,结果是并发加载海报墙
|
||||
// 时几乎每张图都回退成 1MB 上下的 PNG 原图:一页要传几十 MB,外部网络下单张
|
||||
// 图就要二十多秒。缩略图本身只有几十 KB,排队几百毫秒明显更划算;只有真正
|
||||
// 大到会拖慢页面的原图才值得在超时后直出。
|
||||
func (p *ImageProxy) acquireResizeSlotFor(ctx context.Context, size int64) (func(), bool) {
|
||||
if size > 0 && size <= resizeQueueBypassBytes {
|
||||
if size > 0 && size <= compactImageSkipBytes {
|
||||
return p.tryAcquireResizeSlot()
|
||||
}
|
||||
return p.acquireResizeSlot(ctx)
|
||||
return p.acquireResizeSlotWithin(ctx, p.resizeQueueTimeout())
|
||||
}
|
||||
|
||||
// acquireResizeSlotWithin 在 timeout 内等待一格解码槽;超时或请求取消返回
|
||||
// false,调用方回退原图。
|
||||
func (p *ImageProxy) acquireResizeSlotWithin(ctx context.Context, timeout time.Duration) (func(), bool) {
|
||||
if p == nil {
|
||||
return func() {}, true
|
||||
}
|
||||
if timeout <= 0 {
|
||||
return p.tryAcquireResizeSlot()
|
||||
}
|
||||
p.resizeSemMu.Lock()
|
||||
if p.resizeSem == nil {
|
||||
p.resizeSem = make(chan struct{}, imageResizeConcurrency())
|
||||
}
|
||||
sem := p.resizeSem
|
||||
p.resizeSemMu.Unlock()
|
||||
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
return func() { <-sem }, true
|
||||
case <-timer.C:
|
||||
return nil, false
|
||||
case <-ctx.Done():
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// serveResizedFromFile 从 srcPath 读取图片,按选项缩放后写出,并把结果缓存
|
||||
@@ -472,10 +530,6 @@ func (p *ImageProxy) ensureResizeCache(ctx context.Context, srcPath string, o im
|
||||
// ponytail: 200KB 覆盖这台机器上的典型海报(平均约 100KB);更大的图仍尝试缩放。
|
||||
const compactImageSkipBytes = 200 * 1024
|
||||
|
||||
// resizeQueueBypassBytes 是“槽位忙就放弃缩放”的上限。超过它的原图继续排队,
|
||||
// 以免把未压缩的剧照直接发给客户端。
|
||||
const resizeQueueBypassBytes = 1536 * 1024
|
||||
|
||||
// serveCompactOriginal 在源文件已经很小且是浏览器可直接显示的 JPEG/WebP 时
|
||||
// 跳过解码。返回 false 表示仍应走缩放路径。
|
||||
func serveCompactOriginal(w http.ResponseWriter, r *http.Request, srcPath string, stat os.FileInfo) bool {
|
||||
@@ -504,6 +558,17 @@ func isCompactWebImage(header []byte) bool {
|
||||
string(header[8:12]) == "WEBP"
|
||||
}
|
||||
|
||||
// isCompactWebFormat 判断 image.DecodeConfig 报告的格式是否已经适合直接下发
|
||||
// 给浏览器。PNG/BMP/GIF 即使尺寸很小也可能有几百 KB,需要重编码成 JPEG。
|
||||
func isCompactWebFormat(format string) bool {
|
||||
switch strings.ToLower(format) {
|
||||
case "jpeg", "jpg", "webp":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// writeResizeCache 原子写入缩放结果;失败只记日志,不影响本次响应。
|
||||
// 与 writeImageCache 一样不再持有全局锁:临时文件名唯一、rename 原子,
|
||||
// 持锁只会把缩略图的写盘和原图的写盘串成一条队。
|
||||
|
||||
@@ -123,17 +123,58 @@ func TestResizeImageDataScalesDownAndReencodesAsJPEG(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResizeImageDataLeavesImagesWithinBoundsUntouched(t *testing.T) {
|
||||
// 尺寸已经在目标框内的 PNG 也必须转成 JPEG。刮削来的海报常是 1000×1500、
|
||||
// 1MB 级的 PNG;此前这种情况原样直出,一处海报墙就要下载几十 MB,外部网络下
|
||||
// 每张图要十几秒(线上实测单张 1.3MB / 20 多秒)。
|
||||
func TestResizeImageDataReencodesPNGWithinBounds(t *testing.T) {
|
||||
data := encodeTestPNG(t, 200, 300, 255)
|
||||
out, _, unchanged, err := resizeImageData(data, imageResizeOptions{MaxWidth: 400, MaxHeight: 600})
|
||||
out, ctype, unchanged, err := resizeImageData(data, imageResizeOptions{MaxWidth: 400, MaxHeight: 600, Quality: 80})
|
||||
if err != nil {
|
||||
t.Fatalf("resize: %v", err)
|
||||
}
|
||||
if unchanged {
|
||||
t.Fatal("PNG within the target box must still be re-encoded as a compact web format")
|
||||
}
|
||||
if ctype != "image/jpeg" {
|
||||
t.Fatalf("content type = %q, want image/jpeg", ctype)
|
||||
}
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(out))
|
||||
if err != nil {
|
||||
t.Fatalf("decode re-encoded image: %v", err)
|
||||
}
|
||||
if format != "jpeg" {
|
||||
t.Fatalf("format = %q, want jpeg", format)
|
||||
}
|
||||
if cfg.Width != 200 || cfg.Height != 300 {
|
||||
t.Fatalf("re-encoded to %dx%d, want the original 200x300 (no upscaling)", cfg.Width, cfg.Height)
|
||||
}
|
||||
}
|
||||
|
||||
// 已经是紧凑 Web 格式且体积很小的源图仍然零成本直出。
|
||||
func TestResizeImageDataLeavesSmallCompactSourceUntouched(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
img := image.NewRGBA(image.Rect(0, 0, 200, 300))
|
||||
for y := 0; y < 300; y++ {
|
||||
for x := 0; x < 200; x++ {
|
||||
img.Set(x, y, color.RGBA{R: uint8(x % 16 * 16), G: uint8(y % 16 * 16), B: 200, A: 255})
|
||||
}
|
||||
}
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 80}); err != nil {
|
||||
t.Fatalf("encode test jpeg: %v", err)
|
||||
}
|
||||
data := buf.Bytes()
|
||||
if len(data) > compactImageSkipBytes {
|
||||
t.Fatalf("test jpeg is %d bytes, expected a compact source", len(data))
|
||||
}
|
||||
out, ctype, unchanged, err := resizeImageData(data, imageResizeOptions{MaxWidth: 400, MaxHeight: 600})
|
||||
if err != nil {
|
||||
t.Fatalf("resize: %v", err)
|
||||
}
|
||||
if !unchanged {
|
||||
t.Fatal("expected an image already within bounds to be returned unchanged")
|
||||
t.Fatal("a compact JPEG already within bounds should be served unchanged")
|
||||
}
|
||||
if !bytes.Equal(out, data) {
|
||||
t.Fatal("expected the original bytes to be returned verbatim")
|
||||
if !bytes.Equal(out, data) || ctype != "image/jpeg" {
|
||||
t.Fatalf("expected the original jpeg bytes back, got %d bytes of %q", len(out), ctype)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,7 +343,9 @@ func TestServeResizedFromFileSkipsDecodeForCompactJPEG(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeResizedFromFileDoesNotQueueWhenResizeBusy(t *testing.T) {
|
||||
// 排队上限:槽位一直不空时,请求最多等 resizeQueueWait,然后回退原图,
|
||||
// 不会被无限期挂住。这条约束保护的是请求本身,而不是「不等就直出原图」。
|
||||
func TestServeResizedFromFileFallsBackWhenResizeQueueStaysBusy(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := dir + string(os.PathSeparator) + "backdrop.jpg"
|
||||
data := make([]byte, compactImageSkipBytes+1024)
|
||||
@@ -310,7 +353,7 @@ func TestServeResizedFromFileDoesNotQueueWhenResizeBusy(t *testing.T) {
|
||||
if err := os.WriteFile(src, data, 0o644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
proxy := &ImageProxy{cacheDir: dir + string(os.PathSeparator) + "cache"}
|
||||
proxy := &ImageProxy{cacheDir: dir + string(os.PathSeparator) + "cache", resizeQueueWait: 50 * time.Millisecond}
|
||||
var releases []func()
|
||||
for i := 0; i < imageResizeConcurrency(); i++ {
|
||||
release, ok := proxy.acquireResizeSlot(context.Background())
|
||||
@@ -327,10 +370,77 @@ func TestServeResizedFromFileDoesNotQueueWhenResizeBusy(t *testing.T) {
|
||||
|
||||
start := time.Now()
|
||||
handled := proxy.serveResizedFromFile(httptest.NewRecorder(), httptest.NewRequest("GET", "/x?maxWidth=480", nil), src, imageResizeOptions{MaxWidth: 480})
|
||||
if time.Since(start) > 200*time.Millisecond {
|
||||
t.Fatal("resize slot was busy but the request still waited in the queue")
|
||||
if elapsed := time.Since(start); elapsed > 500*time.Millisecond {
|
||||
t.Fatalf("request waited %v with every slot held; the queue wait must be bounded", elapsed)
|
||||
}
|
||||
if handled {
|
||||
t.Fatal("expected a busy resize slot to fall back so the caller can serve the original")
|
||||
t.Fatal("expected a permanently busy queue to fall back so the caller can serve the original")
|
||||
}
|
||||
}
|
||||
|
||||
// 槽位只是短暂忙碌时必须排队等待并产出缩略图,而不是直出多兆字节原图。
|
||||
// 这是线上「海报墙每张图 20 多秒」的根因:并发加载时槽位全忙,旧逻辑把
|
||||
// 1MB 级 PNG 原样下发。
|
||||
// encodeNoisyPNG 生成难以压缩的 PNG:规则色块会被压到 1KB 级别,测不出
|
||||
// 「大体积 PNG 被原样下发」这条回归。
|
||||
func encodeNoisyPNG(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
seed := uint32(0x9e3779b9)
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
seed = seed*1664525 + 1013904223
|
||||
img.Set(x, y, color.RGBA{R: uint8(seed >> 24), G: uint8(seed >> 16), B: uint8(seed >> 8), A: 255})
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatalf("encode noisy png: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestServeResizedFromFileWaitsForSlotInsteadOfServingOriginal(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := dir + string(os.PathSeparator) + "poster.png"
|
||||
original := encodeNoisyPNG(t, 378, 540)
|
||||
if len(original) <= compactImageSkipBytes {
|
||||
t.Fatalf("test poster is %d bytes, want a source above the compact threshold", len(original))
|
||||
}
|
||||
if err := os.WriteFile(src, original, 0o644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
proxy := &ImageProxy{cacheDir: dir + string(os.PathSeparator) + "cache"}
|
||||
var releases []func()
|
||||
for i := 0; i < imageResizeConcurrency(); i++ {
|
||||
release, ok := proxy.acquireResizeSlot(context.Background())
|
||||
if !ok {
|
||||
t.Fatal("expected a resize slot")
|
||||
}
|
||||
releases = append(releases, release)
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
releases[0]()
|
||||
}()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
if !proxy.serveResizedFromFile(rec, httptest.NewRequest("GET", "/x?maxWidth=480&maxHeight=600", nil), src, imageResizeOptions{MaxWidth: 480, MaxHeight: 600}) {
|
||||
t.Fatal("expected the request to wait for the slot and serve a thumbnail")
|
||||
}
|
||||
body := rec.Body.Bytes()
|
||||
if bytes.Equal(body, original) {
|
||||
t.Fatal("served the multi-hundred-KB PNG original instead of a re-encoded thumbnail")
|
||||
}
|
||||
if rec.Header().Get("Content-Type") != "image/jpeg" {
|
||||
t.Fatalf("content type = %q, want image/jpeg", rec.Header().Get("Content-Type"))
|
||||
}
|
||||
// 同一请求第二次必须直接命中缩放缓存。
|
||||
rec2 := httptest.NewRecorder()
|
||||
if !proxy.serveResizedFromFile(rec2, httptest.NewRequest("GET", "/x?maxWidth=480&maxHeight=600", nil), src, imageResizeOptions{MaxWidth: 480, MaxHeight: 600}) {
|
||||
t.Fatal("expected the cached thumbnail to be reused")
|
||||
}
|
||||
if !bytes.Equal(rec2.Body.Bytes(), body) {
|
||||
t.Fatal("expected the cached thumbnail bytes to be identical")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,8 +22,9 @@ import (
|
||||
|
||||
const (
|
||||
// IntroDBBaseURL is the public read endpoint. Overridable on the service
|
||||
// for tests and for pointing at a mirror.
|
||||
IntroDBBaseURL = "https://api.theintrodb.org/v3"
|
||||
// for tests and for pointing at a mirror. v1 已删除,v2/v3 冻结在只读的
|
||||
// GET /media 上并被官方标记为弃用,所以这里直接跟 v4。
|
||||
IntroDBBaseURL = "https://api.theintrodb.org/v4"
|
||||
// IntroDBSource tags rows that came from this provider.
|
||||
IntroDBSource = "theintrodb"
|
||||
|
||||
@@ -92,8 +93,10 @@ type introDBRange struct {
|
||||
EndMs *int64 `json:"end_ms"`
|
||||
}
|
||||
|
||||
// introDBResponse 对应 v4 的 MediaResponse。v4 以 TVDB 为身份,响应只带
|
||||
// tvdb_id —— tmdb_id 字段已被官方移除,需要 tmdb_id 只能从请求侧自己记。
|
||||
type introDBResponse struct {
|
||||
TMDbID int `json:"tmdb_id"`
|
||||
TVDBID int `json:"tvdb_id"`
|
||||
Type string `json:"type"`
|
||||
Intro []introDBRange `json:"intro"`
|
||||
Recap []introDBRange `json:"recap"`
|
||||
|
||||
@@ -11,11 +11,12 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// 这两段响应是从 api.theintrodb.org/v3/media 实测抓下来的原文,
|
||||
// 用来锁住 null 语义:start_ms: null = 从片头开始,end_ms: null = 一直到片尾。
|
||||
// 这两段响应是从 api.theintrodb.org/v4/media 实测抓下来的原文(v4 以 TVDB 为身份,
|
||||
// 响应里只有 tvdb_id),用来锁住 null 语义:start_ms: null = 从片头开始,
|
||||
// end_ms: null = 一直到片尾。
|
||||
const (
|
||||
introDBTVPayload = `{"tmdb_id":1396,"type":"tv","season":1,"episode":1,"intro":[{"start_ms":228664,"end_ms":246143}],"credits":[{"start_ms":3431000,"end_ms":null}]}`
|
||||
introDBMoviePayload = `{"tmdb_id":27205,"type":"movie","intro":[{"start_ms":null,"end_ms":38000}]}`
|
||||
introDBTVPayload = `{"tvdb_id":81189,"type":"tv","season":1,"episode":1,"intro":[{"start_ms":228664,"end_ms":246143}],"credits":[{"start_ms":3431000,"end_ms":null}]}`
|
||||
introDBMoviePayload = `{"tvdb_id":113,"type":"movie","intro":[{"start_ms":null,"end_ms":38000}]}`
|
||||
)
|
||||
|
||||
func TestParseIntroDBResponseResolvesNullBounds(t *testing.T) {
|
||||
@@ -48,7 +49,7 @@ func TestParseIntroDBResponseResolvesNullBounds(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseIntroDBResponseDropsEmptyRanges(t *testing.T) {
|
||||
body := `{"tmdb_id":1,"type":"movie",
|
||||
body := `{"tvdb_id":1,"type":"movie",
|
||||
"intro":[{"start_ms":5000,"end_ms":5000},{"start_ms":9000,"end_ms":8000},{"start_ms":1000,"end_ms":2000}],
|
||||
"recap":[],"credits":[],"preview":[]}`
|
||||
spans, err := parseIntroDBResponse([]byte(body))
|
||||
@@ -62,7 +63,7 @@ func TestParseIntroDBResponseDropsEmptyRanges(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseIntroDBResponseOrdersByType(t *testing.T) {
|
||||
body := `{"tmdb_id":1,"type":"tv","credits":[{"start_ms":900,"end_ms":1000}],
|
||||
body := `{"tvdb_id":1,"type":"tv","credits":[{"start_ms":900,"end_ms":1000}],
|
||||
"intro":[{"start_ms":100,"end_ms":200}],"recap":[{"start_ms":50,"end_ms":60}]}`
|
||||
spans, err := parseIntroDBResponse([]byte(body))
|
||||
if err != nil {
|
||||
@@ -82,12 +83,12 @@ func TestParseIntroDBResponseOrdersByType(t *testing.T) {
|
||||
func TestIntroDBMediaURLOnlyAddsSeasonEpisodeForTV(t *testing.T) {
|
||||
svc := NewIntroDBService(zap.NewNop())
|
||||
if got, want := svc.mediaURL(1396, 1, 1),
|
||||
"https://api.theintrodb.org/v3/media?tmdb_id=1396&season=1&episode=1"; got != want {
|
||||
"https://api.theintrodb.org/v4/media?tmdb_id=1396&season=1&episode=1"; got != want {
|
||||
t.Fatalf("tv url = %q, want %q", got, want)
|
||||
}
|
||||
// 电影(season/episode 为 0)不能带季集参数,否则会被当成剧集查不到。
|
||||
if got, want := svc.mediaURL(27205, 0, 0),
|
||||
"https://api.theintrodb.org/v3/media?tmdb_id=27205"; got != want {
|
||||
"https://api.theintrodb.org/v4/media?tmdb_id=27205"; got != want {
|
||||
t.Fatalf("movie url = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,6 +184,85 @@ func TestReadAdultMetadataDerivesDMMPosterFromRemoteFanart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAdultArtworkIgnoresLoosePosterInLibraryRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
// 云下载这类库把不同影片平铺在同一个根目录里,根目录中残留的松散海报
|
||||
// 属于别的影片,不能被抓来当作当前影片的封面。
|
||||
if err := os.WriteFile(filepath.Join(root, "AAA-001-poster.jpg"), []byte("jpg"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "AAA-001-fanart.jpg"), []byte("jpg"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mediaPath := filepath.Join(root, "BBB-002.mp4")
|
||||
if err := os.WriteFile(mediaPath, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := ReadLocalMetadata(mediaPath, root, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != nil && (got.PosterURL != "" || got.BackdropURL != "") {
|
||||
t.Fatalf("library-root artwork must not be inherited: poster=%q backdrop=%q", got.PosterURL, got.BackdropURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAdultArtworkIgnoresLoosePosterInSharedFolder(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "shared")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"AAA-001.mp4", "BBB-002.mp4"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "AAA-001-poster.jpg"), []byte("jpg"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "AAA-001-fanart.jpg"), []byte("jpg"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := ReadLocalMetadata(filepath.Join(dir, "BBB-002.mp4"), root, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != nil && (got.PosterURL != "" || got.BackdropURL != "") {
|
||||
t.Fatalf("sibling artwork must not be inherited in a shared folder: poster=%q backdrop=%q", got.PosterURL, got.BackdropURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAdultArtworkKeepsLooseFallbackInItemFolder(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "BBB-002")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mediaPath := filepath.Join(dir, "video.mp4")
|
||||
if err := os.WriteFile(mediaPath, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
poster := filepath.Join(dir, "japanese-title-cover.jpg")
|
||||
fanart := filepath.Join(dir, "japanese-title-fanart.jpg")
|
||||
if err := os.WriteFile(poster, []byte("jpg"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(fanart, []byte("jpg"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := ReadLocalMetadata(mediaPath, root, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got == nil || got.PosterURL != poster || got.BackdropURL != fanart {
|
||||
t.Fatalf("single-item folder should keep loose artwork fallback: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadAdultArtworkByCodeWithoutNFO(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mediaPath := filepath.Join(root, "SSIS-001-CD1.mp4")
|
||||
|
||||
@@ -11,23 +11,23 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
func metadataFromArtwork(mediaPath, showBaseDir string) *LocalMetadata {
|
||||
func metadataFromArtwork(mediaPath, showBaseDir, libraryRoot string) *LocalMetadata {
|
||||
meta := &LocalMetadata{}
|
||||
mergeArtworkMetadata(meta, mediaPath, showBaseDir)
|
||||
mergeArtworkMetadata(meta, mediaPath, showBaseDir, libraryRoot)
|
||||
if meta.PosterURL == "" && meta.BackdropURL == "" {
|
||||
return nil
|
||||
}
|
||||
return meta
|
||||
}
|
||||
|
||||
func mergeArtworkMetadata(meta *LocalMetadata, mediaPath, showBaseDir string) {
|
||||
func mergeArtworkMetadata(meta *LocalMetadata, mediaPath, showBaseDir, libraryRoot string) {
|
||||
if meta == nil {
|
||||
return
|
||||
}
|
||||
mediaDir := filepath.Dir(mediaPath)
|
||||
if localPoster := firstLocalPoster(mediaPath, showBaseDir); localPoster != "" {
|
||||
meta.PosterURL = localPoster
|
||||
} else if meta.PosterURL == "" {
|
||||
} else if meta.PosterURL == "" && looseArtworkFallbackAllowed(mediaDir, libraryRoot) {
|
||||
meta.PosterURL = firstAdultLooseImage(mediaDir, "poster")
|
||||
}
|
||||
dirs := []string{mediaDir, showBaseDir}
|
||||
@@ -39,7 +39,7 @@ func mergeArtworkMetadata(meta *LocalMetadata, mediaPath, showBaseDir string) {
|
||||
meta.BackdropURL = img
|
||||
break
|
||||
}
|
||||
if meta.BackdropURL == "" {
|
||||
if meta.BackdropURL == "" && looseArtworkFallbackAllowed(dir, libraryRoot) {
|
||||
meta.BackdropURL = firstAdultLooseImage(dir, "backdrop")
|
||||
}
|
||||
}
|
||||
@@ -245,6 +245,48 @@ func firstExistingPosterImage(dir string, names ...string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// looseArtworkFallbackAllowed reports whether an unmatched image sitting in dir
|
||||
// can be attributed to mediaPath. Such an image carries no filename relation to
|
||||
// a specific video, so it is only trustworthy when the folder holds a single
|
||||
// media file and that folder is not the library root. Download tools routinely
|
||||
// drop unrelated titles flat into one directory (a 115 cloud-download library,
|
||||
// for example); claiming the first "*-poster.jpg" there made every item show the
|
||||
// same artwork. Mirrors the root-level guard findMovieNFO already applies to
|
||||
// loose NFO files.
|
||||
func looseArtworkFallbackAllowed(dir, libraryRoot string) bool {
|
||||
if dir == "" {
|
||||
return false
|
||||
}
|
||||
if libraryRoot != "" && samePath(dir, libraryRoot) {
|
||||
return false
|
||||
}
|
||||
return countMediaFilesInDir(dir) <= 1
|
||||
}
|
||||
|
||||
// countMediaFilesInDir counts the video files directly inside dir, stopping once
|
||||
// more than one is found. It walks the real directory rather than reusing the
|
||||
// loose-image glob because a shared folder may contain no images at all.
|
||||
func countMediaFilesInDir(dir string) int {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
count := 0
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if _, ok := videoExtensions[strings.ToLower(filepath.Ext(entry.Name()))]; !ok {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
if count > 1 {
|
||||
return count
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func firstAdultLooseImage(dir, kind string) string {
|
||||
if dir == "" {
|
||||
return ""
|
||||
|
||||
@@ -17,12 +17,12 @@ func ReadLocalMetadata(mediaPath, libraryRoot string, seriesLike bool) (*LocalMe
|
||||
doc, path, err := findMovieNFO(mediaPath, libraryRoot)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return metadataFromArtwork(mediaPath, ""), nil
|
||||
return metadataFromArtwork(mediaPath, "", libraryRoot), nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
meta := metadataFromDoc(doc, filepath.Dir(path), false)
|
||||
mergeArtworkMetadata(meta, mediaPath, filepath.Dir(path))
|
||||
mergeArtworkMetadata(meta, mediaPath, filepath.Dir(path), libraryRoot)
|
||||
// A sidecar written while resolution tokens were misread as SxxExx must not
|
||||
// reintroduce the bogus season/episode on rescan.
|
||||
dropResolutionArtifactEpisodeIdentity(meta, mediaPath)
|
||||
@@ -115,9 +115,9 @@ func readSeriesMetadata(mediaPath, libraryRoot string) (*LocalMetadata, error) {
|
||||
}
|
||||
}
|
||||
if meta == nil {
|
||||
meta = metadataFromArtwork(mediaPath, showBaseDir)
|
||||
meta = metadataFromArtwork(mediaPath, showBaseDir, libraryRoot)
|
||||
} else {
|
||||
mergeArtworkMetadata(meta, mediaPath, showBaseDir)
|
||||
mergeArtworkMetadata(meta, mediaPath, showBaseDir, libraryRoot)
|
||||
}
|
||||
// A sidecar written while resolution tokens were misread as SxxExx must not
|
||||
// reintroduce the bogus season/episode on rescan.
|
||||
|
||||
@@ -12,13 +12,14 @@ import (
|
||||
|
||||
// MediaService offers high-level CRUD over libraries and media items.
|
||||
type MediaService struct {
|
||||
cfg *config.Config
|
||||
log *zap.Logger
|
||||
repo *repository.Container
|
||||
cache *RuntimeCacheService
|
||||
groupedMediaFlight singleflight.Group
|
||||
libraryRowsFlight singleflight.Group
|
||||
recentSeriesFlight singleflight.Group
|
||||
cfg *config.Config
|
||||
log *zap.Logger
|
||||
repo *repository.Container
|
||||
cache *RuntimeCacheService
|
||||
groupedMediaFlight singleflight.Group
|
||||
mediaSortFlight singleflight.Group
|
||||
libraryRowsFlight singleflight.Group
|
||||
recentSeriesFlight singleflight.Group
|
||||
}
|
||||
|
||||
type MediaVisibility struct {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
@@ -129,6 +129,26 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
visibility MediaVisibility,
|
||||
filters MediaListFilters,
|
||||
) ([]MediaItem, error) {
|
||||
value, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return value.Items, nil
|
||||
}
|
||||
|
||||
// Each immutable grouping has its own revision. Lightweight sort indices can
|
||||
// reference that revision without retaining another copy of the full rows.
|
||||
type groupedMediaCacheValue struct {
|
||||
Items []MediaItem
|
||||
Revision string
|
||||
}
|
||||
|
||||
func (s *MediaService) groupedMediaVisibleFiltered(
|
||||
ctx context.Context,
|
||||
libraryID string,
|
||||
visibility MediaVisibility,
|
||||
filters MediaListFilters,
|
||||
) (*groupedMediaCacheValue, error) {
|
||||
visibility = ExpandMediaVisibilityForMergedCloudLibraries(ctx, s.repo, visibility)
|
||||
libraryIDs, err := MergedLibraryIDsForLibrary(ctx, s.repo, libraryID)
|
||||
if err != nil {
|
||||
@@ -143,7 +163,7 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
value, err, _ := s.groupedMediaFlight.Do(itemsCacheKey, func() (any, error) {
|
||||
if s.cache != nil {
|
||||
if cachedObj, ok := s.cache.GetObject(itemsCacheKey); ok {
|
||||
if cached, ok := cachedObj.([]MediaItem); ok {
|
||||
if cached, ok := cachedObj.(*groupedMediaCacheValue); ok {
|
||||
return cached, nil
|
||||
}
|
||||
}
|
||||
@@ -154,8 +174,8 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grouped := groupMediaVersions(items)
|
||||
if s.cache != nil && len(grouped) > 0 {
|
||||
grouped := &groupedMediaCacheValue{Items: groupMediaVersions(items), Revision: uuid.NewString()}
|
||||
if s.cache != nil && len(grouped.Items) > 0 {
|
||||
s.cache.SetObject(itemsCacheKey, grouped, s.mediaObjectTTL())
|
||||
}
|
||||
return grouped, nil
|
||||
@@ -163,10 +183,10 @@ func (s *MediaService) GroupedMediaVisibleFiltered(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if grouped, ok := value.([]MediaItem); ok {
|
||||
if grouped, ok := value.(*groupedMediaCacheValue); ok {
|
||||
return grouped, nil
|
||||
}
|
||||
return nil, nil
|
||||
return &groupedMediaCacheValue{}, nil
|
||||
}
|
||||
|
||||
func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryID string, visibility MediaVisibility, filters MediaListFilters) ([]model.Media, error) {
|
||||
@@ -180,24 +200,19 @@ func (s *MediaService) listMediaVisibleForGrouping(ctx context.Context, libraryI
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
})
|
||||
// 版本分组的 URL 分页发生在 Go 进程内,响应里的 total 是分组后的数量,
|
||||
// 不需要数据库再为原始行做一次 COUNT(*)。全量 COUNT 在超大媒体库上
|
||||
// 会重复扫描整个 library_id 范围,而这里只关心是否存在截断风险。
|
||||
items, err := s.repo.Media.ListByLibrariesFilteredNoCount(ctx, libraryIDs, 0, maxMediaSearchLimit, filter)
|
||||
// Group before paginating so versions stay together. The search safety limit
|
||||
// must not truncate a library listing or its reported grouped total.
|
||||
items, err := s.repo.Media.ListAllByLibrariesFilteredNoCount(ctx, libraryIDs, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) >= maxMediaSearchLimit && s.log != nil {
|
||||
s.log.Warn("media version grouping may be truncated by safety limit",
|
||||
zap.String("library_id", libraryID),
|
||||
zap.Int("limit", maxMediaSearchLimit))
|
||||
}
|
||||
s.attachLibraryMetadata(ctx, items)
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// GetMedia returns a single media row.
|
||||
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) { media, err := s.repo.Media.FindByID(ctx, id)
|
||||
func (s *MediaService) GetMedia(ctx context.Context, id string) (*model.Media, error) {
|
||||
media, err := s.repo.Media.FindByID(ctx, id)
|
||||
if err != nil || media == nil {
|
||||
return media, err
|
||||
}
|
||||
|
||||
@@ -2,17 +2,71 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
type MediaQualityItem struct {
|
||||
Issue string `json:"issue"`
|
||||
Media model.Media `json:"media"`
|
||||
}
|
||||
|
||||
func (s *MediaService) QualityIssues(ctx context.Context, issue string, page, pageSize int) ([]MediaQualityItem, int64, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 50
|
||||
}
|
||||
if pageSize > 100 {
|
||||
pageSize = 100
|
||||
}
|
||||
q := s.repo.DB.WithContext(ctx).Model(&model.Media{})
|
||||
switch issue {
|
||||
case "missing":
|
||||
q = q.Where("availability_state = ?", "missing")
|
||||
case "unmatched":
|
||||
q = q.Where("scrape_status IN ? OR scrape_status = ''", []string{"pending", "no_match"})
|
||||
case "poster":
|
||||
q = q.Where("poster_url IS NULL OR TRIM(poster_url) = ''")
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unknown quality issue: %s", issue)
|
||||
}
|
||||
var total int64
|
||||
if err := q.Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var rows []model.Media
|
||||
if err := q.Order("updated_at DESC, id ASC").Offset((page - 1) * pageSize).Limit(pageSize).Find(&rows).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
s.attachLibraryMetadata(ctx, rows)
|
||||
items := make([]MediaQualityItem, 0, len(rows))
|
||||
for _, media := range rows {
|
||||
items = append(items, MediaQualityItem{Issue: issue, Media: media})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
type MediaSearchFilter struct {
|
||||
LibraryID string
|
||||
YearMin int
|
||||
YearMax int
|
||||
Genres []string
|
||||
}
|
||||
|
||||
// SearchMedia performs a simple LIKE search across titles.
|
||||
func (s *MediaService) SearchMedia(ctx context.Context, query string, limit int) ([]model.Media, error) {
|
||||
return s.SearchMediaVisible(ctx, query, limit, MediaVisibility{IncludeNSFW: true})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, limit int, visibility MediaVisibility) ([]model.Media, error) {
|
||||
return s.SearchMediaVisibleFiltered(ctx, query, limit, visibility, MediaSearchFilter{})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisibleFiltered(ctx context.Context, query string, limit int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
} else if limit > maxMediaSearchLimit {
|
||||
@@ -23,6 +77,10 @@ func (s *MediaService) SearchMediaVisible(ctx context.Context, query string, lim
|
||||
IncludeNSFW: visibility.IncludeNSFW,
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
LibraryID: filter.LibraryID,
|
||||
YearMin: filter.YearMin,
|
||||
YearMax: filter.YearMax,
|
||||
Genres: filter.Genres,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -45,6 +103,10 @@ func (s *MediaService) SearchMediaVisibleGrouped(ctx context.Context, query stri
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility) ([]model.Media, int64, error) {
|
||||
return s.SearchMediaVisiblePageFiltered(ctx, query, page, pageSize, visibility, MediaSearchFilter{})
|
||||
}
|
||||
|
||||
func (s *MediaService) SearchMediaVisiblePageFiltered(ctx context.Context, query string, page, pageSize int, visibility MediaVisibility, filter MediaSearchFilter) ([]model.Media, int64, error) {
|
||||
if pageSize <= 0 {
|
||||
pageSize = 50
|
||||
}
|
||||
@@ -59,6 +121,10 @@ func (s *MediaService) SearchMediaVisiblePage(ctx context.Context, query string,
|
||||
IncludeNSFW: visibility.IncludeNSFW,
|
||||
AllowedLibraryIDs: visibility.AllowedLibraryIDs,
|
||||
HiddenLibraryIDs: visibility.HiddenLibraryIDs,
|
||||
LibraryID: filter.LibraryID,
|
||||
YearMin: filter.YearMin,
|
||||
YearMax: filter.YearMax,
|
||||
Genres: filter.Genres,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ListMediaVisibleSortedPage reuses grouped rows and a small position index.
|
||||
// Only the requested page is copied; pagination does not sort the full library
|
||||
// again or retain a second full-size MediaItem slice for every sort order.
|
||||
func (s *MediaService) ListMediaVisibleSortedPage(
|
||||
ctx context.Context,
|
||||
libraryID string,
|
||||
page, pageSize int,
|
||||
visibility MediaVisibility,
|
||||
filters MediaListFilters,
|
||||
spec MediaSortSpec,
|
||||
history map[string]time.Time,
|
||||
) ([]MediaItem, int64, error) {
|
||||
grouped, err := s.groupedMediaVisibleFiltered(ctx, libraryID, visibility, filters)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
spec = NormalizeMediaSort(spec.Field, spec.Order)
|
||||
indices := s.sortedMediaIndices(grouped, spec, history)
|
||||
page, pageSize = normalizeGroupedMediaPage(page, pageSize)
|
||||
total := len(grouped.Items)
|
||||
// Division avoids overflowing (page-1)*pageSize on an excessive page value.
|
||||
if total == 0 || page-1 > (total-1)/pageSize {
|
||||
return []MediaItem{}, int64(total), nil
|
||||
}
|
||||
start := (page - 1) * pageSize
|
||||
end := min(start+pageSize, total)
|
||||
items := make([]MediaItem, 0, end-start)
|
||||
for _, index := range indices[start:end] {
|
||||
items = append(items, grouped.Items[index])
|
||||
}
|
||||
return items, int64(total), nil
|
||||
}
|
||||
|
||||
func (s *MediaService) sortedMediaIndices(grouped *groupedMediaCacheValue, spec MediaSortSpec, history map[string]time.Time) []int {
|
||||
// Playback history changes independently of the media cache and is personal.
|
||||
// Keep this explicit sort live rather than caching another user's history.
|
||||
if s.cache == nil || spec.Field == "last_played" || len(grouped.Items) == 0 {
|
||||
return sortMediaIndices(grouped.Items, spec, history)
|
||||
}
|
||||
key := fmt.Sprintf("media:obj:sort:%s:%s:%s", grouped.Revision, spec.Field, spec.Order)
|
||||
value, _, _ := s.mediaSortFlight.Do(key, func() (any, error) {
|
||||
if cached, ok := s.cache.GetObject(key); ok {
|
||||
if indices, ok := cached.([]int); ok {
|
||||
return indices, nil
|
||||
}
|
||||
}
|
||||
indices := sortMediaIndices(grouped.Items, spec, history)
|
||||
s.cache.SetObject(key, indices, s.mediaObjectTTL())
|
||||
return indices, nil
|
||||
})
|
||||
return value.([]int)
|
||||
}
|
||||
|
||||
func sortMediaIndices(items []MediaItem, spec MediaSortSpec, history map[string]time.Time) []int {
|
||||
indices := make([]int, len(items))
|
||||
for i := range indices {
|
||||
indices[i] = i
|
||||
}
|
||||
sort.SliceStable(indices, func(i, j int) bool {
|
||||
a, b := items[indices[i]], items[indices[j]]
|
||||
cmp := compareMediaByField(a, b, spec, history)
|
||||
if cmp == 0 {
|
||||
cmp = compareID(a.ID, b.ID)
|
||||
}
|
||||
return cmp < 0
|
||||
})
|
||||
return indices
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
func TestSortedMediaPageIncludesRowsBeyondSearchLimit(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "Large", Path: "/media/large", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// This must cross the historical 50,000-row boundary: a smaller fixture
|
||||
// cannot detect truncation before grouping, counting, and sorting.
|
||||
now := time.Now()
|
||||
for start := 0; start <= maxMediaSearchLimit; start += 500 {
|
||||
var rows []model.Media
|
||||
for i := start; i < min(start+500, maxMediaSearchLimit+1); i++ {
|
||||
rows = append(rows, model.Media{
|
||||
Base: model.Base{ID: fmt.Sprintf("large-%05d", i), CreatedAt: now, UpdatedAt: now},
|
||||
LibraryID: lib.ID, Path: fmt.Sprintf("/media/large/%05d.mkv", i),
|
||||
Title: fmt.Sprintf("Movie %05d", i), TMDbID: i + 1,
|
||||
})
|
||||
}
|
||||
if err := db.CreateInBatches(&rows, 100).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos)
|
||||
items, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, maxMediaSearchLimit/100+1, 100,
|
||||
MediaVisibility{IncludeNSFW: true}, MediaListFilters{}, MediaSortSpec{Field: "title", Order: "asc"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != int64(maxMediaSearchLimit+1) || len(items) != 1 || items[0].ID != "large-50000" {
|
||||
t.Fatalf("total=%d, page length=%d, want 50001 rows and final item large-50000", total, len(items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortedMediaPagesReuseIndicesAndRefreshAfterInvalidation(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "Movies", Path: "/media/movies", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := []model.Media{
|
||||
{Base: model.Base{ID: "alpha"}, LibraryID: lib.ID, Path: "/media/movies/a.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 100},
|
||||
{Base: model.Base{ID: "alpha-alt"}, LibraryID: lib.ID, Path: "/media/movies/a-alt.mkv", Title: "Alpha", TMDbID: 1, SizeBytes: 50},
|
||||
{Base: model.Base{ID: "beta"}, LibraryID: lib.ID, Path: "/media/movies/b.mkv", Title: "Beta", TMDbID: 2},
|
||||
{Base: model.Base{ID: "hidden"}, LibraryID: lib.ID, Path: "/media/movies/hidden.mkv", Title: "Hidden", TMDbID: 3, NSFW: true},
|
||||
}
|
||||
if err := db.Create(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), repos).
|
||||
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
|
||||
visibility := MediaVisibility{IncludeNSFW: false}
|
||||
spec := MediaSortSpec{Field: "title", Order: "asc"}
|
||||
first, total, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || total != 2 || len(first) != 1 || first[0].ID != "alpha" || len(first[0].Versions) != 2 {
|
||||
t.Fatalf("first=%#v, total=%d, error=%v", first, total, err)
|
||||
}
|
||||
grouped, err := svc.groupedMediaVisibleFiltered(t.Context(), lib.ID, visibility, MediaListFilters{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
index := svc.sortedMediaIndices(grouped, spec, nil)
|
||||
second, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 2, 1, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || len(second) != 1 || second[0].ID != "beta" {
|
||||
t.Fatalf("second=%#v, error=%v", second, err)
|
||||
}
|
||||
cachedIndex := svc.sortedMediaIndices(grouped, spec, nil)
|
||||
if &index[0] != &cachedIndex[0] {
|
||||
t.Fatal("pagination rebuilt the sorted position index")
|
||||
}
|
||||
// The returned page is independently writable without corrupting cached rows.
|
||||
first[0].Title = "Modified by caller"
|
||||
if err := db.Model(&model.Media{}).Where("id = ?", "beta").Update("title", "Aardvark").Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc.invalidateMediaCache(t.Context())
|
||||
fresh, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, 1, 1, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || len(fresh) != 1 || fresh[0].ID != "beta" {
|
||||
t.Fatalf("updated page=%#v, error=%v", fresh, err)
|
||||
}
|
||||
empty, _, err := svc.ListMediaVisibleSortedPage(t.Context(), lib.ID, math.MaxInt, 100, visibility, MediaListFilters{}, spec, nil)
|
||||
if err != nil || empty == nil || len(empty) != 0 {
|
||||
t.Fatalf("out-of-range page=%#v, error=%v", empty, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMediaSortIndexDoesNotCachePersonalPlaybackHistory(t *testing.T) {
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
|
||||
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
|
||||
grouped := &groupedMediaCacheValue{Revision: "history-test", Items: []MediaItem{
|
||||
{Media: model.Media{Base: model.Base{ID: "a"}}},
|
||||
{Media: model.Media{Base: model.Base{ID: "b"}}},
|
||||
}}
|
||||
spec := MediaSortSpec{Field: "last_played", Order: "desc"}
|
||||
first := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"a": time.Now()})
|
||||
second := svc.sortedMediaIndices(grouped, spec, map[string]time.Time{"b": time.Now()})
|
||||
if first[0] != 0 || second[0] != 1 {
|
||||
t.Fatalf("history sorts = %v, %v", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMediaSortIndex(b *testing.B) {
|
||||
items := make([]MediaItem, 10000)
|
||||
for i := range items {
|
||||
items[i].ID = fmt.Sprint(i)
|
||||
items[i].Title = fmt.Sprintf("Movie %05d", len(items)-i)
|
||||
}
|
||||
spec := MediaSortSpec{Field: "title", Order: "asc"}
|
||||
b.Run("sort-every-page", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
SortMediaItems(items, spec.Field, spec.Order, nil)
|
||||
}
|
||||
})
|
||||
b.Run("cached-position-index", func(b *testing.B) {
|
||||
svc := NewMediaService(&config.Config{}, zap.NewNop(), nil).
|
||||
SetRuntimeCache(NewRuntimeCacheService(&config.Config{}, zap.NewNop()))
|
||||
grouped := &groupedMediaCacheValue{Items: items, Revision: "benchmark"}
|
||||
svc.sortedMediaIndices(grouped, spec, nil)
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
svc.sortedMediaIndices(grouped, spec, nil)
|
||||
}
|
||||
})
|
||||
}
|
||||
+29
-26
@@ -14,6 +14,7 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -144,32 +145,34 @@ type localMediaProbeTask struct {
|
||||
}
|
||||
|
||||
type existingLocalMedia struct {
|
||||
LibraryRootID string
|
||||
RelativePath string
|
||||
Title string
|
||||
OriginalName string
|
||||
EpisodeTitle string
|
||||
SizeBytes int64
|
||||
DurationSec int
|
||||
Width int
|
||||
Height int
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
Container string
|
||||
STRMURL string
|
||||
FileID string
|
||||
PosterURL string
|
||||
BackdropURL string
|
||||
Overview string
|
||||
Year int
|
||||
ReleaseDate string
|
||||
Rating float32
|
||||
TMDbID int
|
||||
BangumiID int
|
||||
DoubanID string
|
||||
TheTVDBID string
|
||||
SeasonNum int
|
||||
EpisodeNum int
|
||||
AvailabilityState string
|
||||
MissingSince *time.Time
|
||||
LibraryRootID string
|
||||
RelativePath string
|
||||
Title string
|
||||
OriginalName string
|
||||
EpisodeTitle string
|
||||
SizeBytes int64
|
||||
DurationSec int
|
||||
Width int
|
||||
Height int
|
||||
VideoCodec string
|
||||
AudioCodec string
|
||||
Container string
|
||||
STRMURL string
|
||||
FileID string
|
||||
PosterURL string
|
||||
BackdropURL string
|
||||
Overview string
|
||||
Year int
|
||||
ReleaseDate string
|
||||
Rating float32
|
||||
TMDbID int
|
||||
BangumiID int
|
||||
DoubanID string
|
||||
TheTVDBID string
|
||||
SeasonNum int
|
||||
EpisodeNum int
|
||||
// EpisodeFraction 是集号的小数部分(S01E11.5 → 0.5);变化同样要触发刷新,
|
||||
// 否则已有行上的「半集」标记不会被写回。
|
||||
EpisodeFraction float64
|
||||
|
||||
@@ -33,7 +33,7 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
|
||||
query = query.Where("(library_root_id = ? OR library_root_id = '' OR library_root_id IS NULL)", rootID)
|
||||
}
|
||||
rows, err := query.
|
||||
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status").
|
||||
Select("path", "library_root_id", "relative_path", "title", "original_name", "episode_title", "size_bytes", "duration_sec", "width", "height", "video_codec", "audio_codec", "container", "strm_url", "file_id", "poster_url", "backdrop_url", "overview", "year", "release_date", "rating", "tm_db_id", "bangumi_id", "douban_id", "thetvdb_id", "season_num", "episode_num", "episode_fraction", "genres", "countries", "languages", "nsfw", "scrape_status", "availability_state", "missing_since").
|
||||
Rows()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -80,6 +80,8 @@ func (s *ScannerService) existingLocalMediaSnapshotForRoot(ctx context.Context,
|
||||
&row.Languages,
|
||||
&row.NSFW,
|
||||
&row.ScrapeStatus,
|
||||
&row.AvailabilityState,
|
||||
&row.MissingSince,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -548,7 +548,7 @@ func TestScanLibraryMapsPersistedHostLibraryPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovePathDeletesVanishedMedia(t *testing.T) {
|
||||
func TestRemovePathRetainsVanishedMedia(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
cache := NewRuntimeCacheService(&config.Config{}, zap.NewNop())
|
||||
sc.SetRuntimeCache(cache)
|
||||
@@ -589,8 +589,15 @@ func TestRemovePathDeletesVanishedMedia(t *testing.T) {
|
||||
if removed != 1 {
|
||||
t.Fatalf("removed = %d, want 1", removed)
|
||||
}
|
||||
if countMedia(t, repos) != 0 {
|
||||
t.Fatal("expected 0 media after removal")
|
||||
if countMedia(t, repos) != 1 {
|
||||
t.Fatal("missing media metadata must be retained")
|
||||
}
|
||||
var missing model.Media
|
||||
if err := repos.DB.Where("path = ?", file).First(&missing).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.AvailabilityState != "missing" || missing.MissingSince == nil {
|
||||
t.Fatalf("missing state not recorded: %#v", missing)
|
||||
}
|
||||
if cache.GetJSON(t.Context(), "media:list:test", &cached) {
|
||||
t.Fatal("vanished media removal should invalidate media cache")
|
||||
|
||||
@@ -67,8 +67,16 @@ func TestScanLibraryScansMultipleRootsAndPrunesPerRoot(t *testing.T) {
|
||||
if err := db.Find(&rows).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].Path != fileB {
|
||||
t.Fatalf("remaining rows = %#v, want offline root media preserved", rows)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("remaining rows = %#v, want both media records preserved", rows)
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.Path == fileA && row.AvailabilityState != "missing" {
|
||||
t.Fatalf("vanished file was not marked missing: %#v", row)
|
||||
}
|
||||
if row.Path == fileB && row.AvailabilityState != "available" {
|
||||
t.Fatalf("offline root status was changed: %#v", row)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ func (s *ScannerService) localMediaScanState(in localMediaScanStateInput) (bool,
|
||||
}
|
||||
existing, exists := in.existingMedia[in.cleanPath]
|
||||
isNewMedia := !exists
|
||||
if exists && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
|
||||
if exists && existing.AvailabilityState != "missing" && existing.MissingSince == nil && in.ext != ".strm" && existing.SizeBytes == in.size && !localMetadataNeedsRefresh(existing, in.localMeta) && !localDerivedMetadataNeedsRefresh(existing, in.incoming) {
|
||||
// Unchanged rows are skipped, but a row that never received ffprobe data
|
||||
// still needs a backfill probe so installing ffmpeg later repairs the library.
|
||||
s.queueProbeBackfillIfNeeded(in.path, in.ext, existing)
|
||||
|
||||
@@ -372,7 +372,14 @@ func TestScanLibraryPrunesMissingMedia(t *testing.T) {
|
||||
if err := db.Model(&model.Media{}).Where("library_id = ?", lib.ID).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("media count = %d, want 1", count)
|
||||
if count != 2 {
|
||||
t.Fatalf("media count = %d, want current and missing records preserved", count)
|
||||
}
|
||||
var missing model.Media
|
||||
if err := db.First(&missing, "id = ?", stale.ID).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if missing.AvailabilityState != "missing" {
|
||||
t.Fatalf("stale media state = %q, want missing", missing.AvailabilityState)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestScanRestoresUnchangedMissingFileAndMetadata(t *testing.T) {
|
||||
scanner, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
library := model.Library{Name: "Movies", Path: root, Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &library); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(root, "Example (2024).mkv")
|
||||
writeTestFile(t, path, "unchanged content")
|
||||
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var original model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&original).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := repos.DB.Model(&original).Updates(map[string]any{
|
||||
"title": "Manually matched title", "scrape_status": "matched", "tm_db_id": 42,
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 1 {
|
||||
t.Fatalf("mark missing: changed=%d err=%v", changed, err)
|
||||
}
|
||||
if changed, err := scanner.RemovePath(t.Context(), path); err != nil || changed != 0 {
|
||||
t.Fatalf("repeated missing event: changed=%d err=%v", changed, err)
|
||||
}
|
||||
writeTestFile(t, path, "unchanged content")
|
||||
if _, err := scanner.ScanLibrary(t.Context(), library.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restored model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&restored).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restored.ID != original.ID || restored.TMDbID != 42 || restored.Title != "Manually matched title" {
|
||||
t.Fatalf("restored file lost identity or metadata: %#v", restored)
|
||||
}
|
||||
if restored.AvailabilityState != "available" || restored.MissingSince != nil || restored.LastSeenAt == nil {
|
||||
t.Fatalf("unchanged restored file stayed missing: %#v", restored)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -12,11 +13,15 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// RemovePath 移除磁盘上已不存在的媒体记录。普通的删除仍做物理删除;
|
||||
// 可确认是 STRM 扩展名改名的路径则先保留软删除墓碑,供后续 ingest 继承元数据。
|
||||
// RemovePath 将确认不存在的媒体标记为失联;只有可确认的 STRM 扩展名改名
|
||||
// 才保留软删除墓碑,供后续 ingest 继承元数据。
|
||||
func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, error) {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return 0, nil // still exists; nothing to remove
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
// Permission, I/O, and mount errors are not evidence that a file was
|
||||
// removed. Preserve records until the path can be inspected reliably.
|
||||
return 0, nil
|
||||
}
|
||||
// 目录整体消失(删除/改名离开):连同其子树下的媒体行一并移除。
|
||||
// 此前只删 path 精确匹配的行——目录本身通常没有 media 行,导致目录
|
||||
@@ -33,7 +38,7 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
return 0, err
|
||||
}
|
||||
softIDs := make([]string, 0, len(rows))
|
||||
hardIDs := make([]string, 0, len(rows))
|
||||
missingIDs := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
// LIKE 里的 % _ 是通配符(候选集只会偏大),用 Go 前缀精确过滤,
|
||||
// 避免对含 % / _ 的路径误删。
|
||||
@@ -43,12 +48,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
// A vanished STRM with a live sibling (foo.strm <-> foo.mkv.strm)
|
||||
// is a rename. Keep a soft-deleted tombstone so the later ingest can
|
||||
// adopt its ID and scraped metadata even if fsnotify delivers the
|
||||
// remove event before the create event. A true deletion stays hard.
|
||||
// remove event before the create event.
|
||||
if row.Path == path && liveSTRMPathAlias(path) {
|
||||
softIDs = append(softIDs, row.ID)
|
||||
continue
|
||||
}
|
||||
hardIDs = append(hardIDs, row.ID)
|
||||
missingIDs = append(missingIDs, row.ID)
|
||||
}
|
||||
var removed int64
|
||||
if len(softIDs) > 0 {
|
||||
@@ -60,14 +65,12 @@ func (s *ScannerService) RemovePath(ctx context.Context, path string) (int64, er
|
||||
}
|
||||
removed += res.RowsAffected
|
||||
}
|
||||
if len(hardIDs) > 0 {
|
||||
res := s.repo.DB.WithContext(ctx).Unscoped().
|
||||
Where("id IN ?", hardIDs).
|
||||
Delete(&model.Media{})
|
||||
if res.Error != nil {
|
||||
return removed, res.Error
|
||||
if len(missingIDs) > 0 {
|
||||
count, err := s.markMediaMissingByIDs(ctx, missingIDs)
|
||||
if err != nil {
|
||||
return removed, err
|
||||
}
|
||||
removed += res.RowsAffected
|
||||
removed += count
|
||||
}
|
||||
if removed > 0 {
|
||||
s.invalidateMediaCache(ctx)
|
||||
@@ -82,7 +85,7 @@ const strmRenameTombstoneRetention = 7 * 24 * time.Hour
|
||||
|
||||
// purgeExpiredSTRMTombstones keeps the soft-delete grace period bounded.
|
||||
// It is deliberately restricted to deleted media rows under a STRM alias
|
||||
// workflow; normal deletions are still hard-deleted immediately.
|
||||
// workflow; ordinary missing files retain their metadata for recovery.
|
||||
func (s *ScannerService) purgeExpiredSTRMTombstones(ctx context.Context) {
|
||||
cutoff := time.Now().Add(-strmRenameTombstoneRetention)
|
||||
if err := s.repo.DB.WithContext(ctx).Unscoped().
|
||||
@@ -121,7 +124,7 @@ func (s *ScannerService) pruneMissingMedia(ctx context.Context, libraryID string
|
||||
}
|
||||
stale = append(stale, row.ID)
|
||||
}
|
||||
return s.deleteMediaByIDs(ctx, stale, true)
|
||||
return s.markMediaMissingByIDs(ctx, stale)
|
||||
}
|
||||
|
||||
func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID, rootID, rootPath string, seen map[string]struct{}) (int64, error) {
|
||||
@@ -158,7 +161,33 @@ func (s *ScannerService) pruneMissingMediaForRoot(ctx context.Context, libraryID
|
||||
}
|
||||
stale = append(stale, row.ID)
|
||||
}
|
||||
return s.deleteMediaByIDs(ctx, stale, true)
|
||||
return s.markMediaMissingByIDs(ctx, stale)
|
||||
}
|
||||
|
||||
func (s *ScannerService) markMediaMissingByIDs(ctx context.Context, ids []string) (int64, error) {
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
const batch = 500
|
||||
var changed int64
|
||||
now := time.Now()
|
||||
for i := 0; i < len(ids); i += batch {
|
||||
end := i + batch
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
result := s.repo.DB.WithContext(ctx).Model(&model.Media{}).
|
||||
Where("id IN ? AND (availability_state IS NULL OR availability_state <> ?)", ids[i:end], "missing").
|
||||
Updates(map[string]any{"availability_state": "missing", "missing_since": now})
|
||||
if result.Error != nil {
|
||||
return changed, result.Error
|
||||
}
|
||||
changed += result.RowsAffected
|
||||
}
|
||||
if changed > 0 {
|
||||
s.invalidateMediaCache(ctx)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func pathBelongsToRoot(pathValue, rootPath string) bool {
|
||||
|
||||
@@ -202,7 +202,7 @@ func (s *ScannerService) scanLocalLibraryFiles(ctx context.Context, lib *model.L
|
||||
func existingLocalMediaFileIDs(existingMedia map[string]existingLocalMedia) map[string]string {
|
||||
seenInodes := make(map[string]string)
|
||||
for path, existing := range existingMedia {
|
||||
if existing.FileID != "" {
|
||||
if existing.FileID != "" && existing.AvailabilityState != "missing" {
|
||||
seenInodes[existing.FileID] = path
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,7 +175,7 @@ func TestIngestPathAdoptsSoftDeletedSTRMRenameTombstone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
|
||||
func TestRemovePathRetainsTrueDeletionWithoutAlias(t *testing.T) {
|
||||
sc, repos := newScannerTestEnv(t)
|
||||
root := t.TempDir()
|
||||
lib := model.Library{Name: "Anime", Path: root, Type: "anime", Enabled: true}
|
||||
@@ -197,8 +197,15 @@ func TestRemovePathStillHardDeletesTrueDeletionWithoutAlias(t *testing.T) {
|
||||
if err := repos.DB.Unscoped().Model(&model.Media{}).Where("path = ?", path).Count(&count).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("true deletion left %d tombstone row(s)", count)
|
||||
if count != 1 {
|
||||
t.Fatalf("missing media count = %d, want metadata retained", count)
|
||||
}
|
||||
var media model.Media
|
||||
if err := repos.DB.Where("path = ?", path).First(&media).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if media.AvailabilityState != "missing" || media.MissingSince == nil {
|
||||
t.Fatalf("true deletion did not record missing state: %#v", media)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ type SchedulerService struct {
|
||||
readerFilesTTLHours int
|
||||
|
||||
segments *MediaSegmentService
|
||||
backup *BackupService
|
||||
|
||||
mu sync.Mutex
|
||||
stopCh chan struct{}
|
||||
@@ -65,6 +66,10 @@ func (s *SchedulerService) SetTaskTracker(tasks *TaskTrackerService) {
|
||||
s.tasks = tasks
|
||||
}
|
||||
|
||||
func (s *SchedulerService) SetBackupService(backup *BackupService) {
|
||||
s.backup = backup
|
||||
}
|
||||
|
||||
func (s *SchedulerService) SetOrganizePipeline(pipeline *OrganizePipelineService) {
|
||||
s.organizePipeline = pipeline
|
||||
}
|
||||
@@ -196,6 +201,13 @@ func (s *SchedulerService) Start(ctx context.Context) {
|
||||
run: s.jobTelegramExpiryWarning,
|
||||
})
|
||||
}
|
||||
if s.backup != nil {
|
||||
s.jobs = append(s.jobs, &scheduledJob{
|
||||
name: "database_backup",
|
||||
interval: 5 * time.Minute,
|
||||
run: s.jobDatabaseBackup,
|
||||
})
|
||||
}
|
||||
for _, j := range s.jobs {
|
||||
initialDelay := 15 * time.Second
|
||||
if j.name == "library_scan" || j.name == "organize_source" || j.name == "segment_prewarm" {
|
||||
|
||||
@@ -11,6 +11,53 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const lastAutomaticBackupDateKey = "backup.last_automatic_date"
|
||||
|
||||
func (s *SchedulerService) jobDatabaseBackup(ctx context.Context) error {
|
||||
if s == nil || s.backup == nil || s.repo == nil || s.repo.Setting == nil {
|
||||
return nil
|
||||
}
|
||||
enabled, _ := s.repo.Setting.Get(ctx, "backup.auto_enabled")
|
||||
if !parseBoolSetting(enabled, false) {
|
||||
return nil
|
||||
}
|
||||
clock, _ := s.repo.Setting.Get(ctx, "backup.auto_time")
|
||||
if clock == "" {
|
||||
clock = "03:00"
|
||||
}
|
||||
parsed, err := time.Parse("15:04", clock)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
now := s.currentTime().In(time.Local)
|
||||
dueAt := time.Date(now.Year(), now.Month(), now.Day(), parsed.Hour(), parsed.Minute(), 0, 0, now.Location())
|
||||
if now.Before(dueAt) {
|
||||
return nil
|
||||
}
|
||||
date := now.Format("2006-01-02")
|
||||
last, _ := s.repo.Setting.Get(ctx, lastAutomaticBackupDateKey)
|
||||
if last == date {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.backup.CreateAutomatic(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// Record the successful backup before pruning. If retention cleanup fails,
|
||||
// the next scheduler tick must not create another copy for the same day.
|
||||
if err := s.repo.Setting.Set(ctx, lastAutomaticBackupDateKey, date); err != nil {
|
||||
return err
|
||||
}
|
||||
retentionValue, _ := s.repo.Setting.Get(ctx, "backup.retention")
|
||||
retention, err := strconv.Atoi(strings.TrimSpace(retentionValue))
|
||||
if err != nil || retention < 1 {
|
||||
retention = 7
|
||||
}
|
||||
if err := s.backup.Prune(retention); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jobScanLibraries re-walks every enabled library.
|
||||
//
|
||||
// 默认关闭:文件变更由 WatcherService 增量入库,无需周期性全量重扫。
|
||||
|
||||
@@ -63,6 +63,9 @@ func (b *serviceContainerBuilder) startRealtimeServices() {
|
||||
b.c.WSHub = NewHub(b.log)
|
||||
helper.Go(b.log, "ws.hub", b.c.WSHub.Run)
|
||||
b.c.Tasks = NewTaskTrackerService(b.log, b.c.WSHub)
|
||||
if b.repos != nil {
|
||||
b.c.Tasks.SetPersistence(b.repos.DB)
|
||||
}
|
||||
b.c.SystemUpdate = NewSystemUpdateService(b.cfg, b.log, b.repos, b.c.Tasks, b.version)
|
||||
|
||||
b.c.SSEHub = NewSSEHub(b.log)
|
||||
@@ -206,6 +209,7 @@ func (b *serviceContainerBuilder) initAccessAndStorageServices() {
|
||||
b.c.Organizer, b.c.WSHub, b.cfg.Cache.CacheDir,
|
||||
)
|
||||
b.c.Scheduler.SetTaskTracker(b.c.Tasks)
|
||||
b.c.Scheduler.SetBackupService(b.c.Backup)
|
||||
b.c.Scheduler.SetOrganizePipeline(b.c.OrganizePipeline)
|
||||
b.c.Scheduler.SetSegments(b.c.Segments)
|
||||
b.c.Scheduler.SetImageCachePolicyProvider(func() ImageCachePolicy {
|
||||
|
||||
@@ -3,11 +3,13 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -15,6 +17,7 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
// directRedirectTestRepo 建一个带网盘账号表的库:normalizeCloudPlayTarget 需要
|
||||
@@ -145,6 +148,53 @@ func TestServeFileFallsBackWhenDirectResolveUnavailable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// `?refresh=1`:客户端手上的直链已被 CDN 403,服务端必须丢掉缓存重新换链,
|
||||
// 不能把同一条死链再发一次;不带该参数时缓存照旧命中,保住快路径。
|
||||
func TestServeFileRefreshDropsCachedDirectLink(t *testing.T) {
|
||||
repos := directRedirectTestRepo(t)
|
||||
seedCloudSTRMMedia(t, repos, "cloud-refresh", "/api/strm/play/cloud115/video.mkv?acct=a1&pickcode=pc-refresh")
|
||||
cached := fmt.Sprintf("https://cdn.example/stale.mkv?t=%d", time.Now().Add(30*time.Minute).Unix())
|
||||
fresh := "https://cdn.example/fresh.mkv?t=2"
|
||||
cloud115.SetDownloadURLCache("pc-refresh", cached, "RodelPlayer/2.0")
|
||||
t.Cleanup(func() { cloud115.ClearDownloadURLCache("pc-refresh") })
|
||||
oldMinAge := directLinkRefreshMinAge
|
||||
directLinkRefreshMinAge = 0
|
||||
defer func() { directLinkRefreshMinAge = oldMinAge }()
|
||||
|
||||
run := func(t *testing.T, query string) (cachedAtResolve, location string) {
|
||||
t.Helper()
|
||||
svc := NewStreamService(&config.Config{}, zap.NewNop(), repos, nil).
|
||||
SetStrmPlayTargetResolver(func(_ context.Context, _ string, userAgent string) (*StrmPlayResult, error) {
|
||||
cachedAtResolve = cloud115.GetDownloadURLCache("pc-refresh", userAgent)
|
||||
return &StrmPlayResult{
|
||||
RedirectURL: fresh,
|
||||
Link: &cloud.DirectLink{URL: fresh, Headers: map[string]string{"User-Agent": userAgent}},
|
||||
}, nil
|
||||
})
|
||||
req := httptest.NewRequest(http.MethodGet, "http://nas.local:18080/api/stream/cloud-refresh?token=jwt123"+query, nil)
|
||||
req.Header.Set("User-Agent", "RodelPlayer/2.0")
|
||||
w := httptest.NewRecorder()
|
||||
if err := svc.ServeFile(w, req, "cloud-refresh"); err != nil {
|
||||
t.Fatalf("ServeFile: %v", err)
|
||||
}
|
||||
return cachedAtResolve, w.Header().Get("Location")
|
||||
}
|
||||
|
||||
// 正常播放:缓存里的直链照旧命中,服务端不该白跑一次换链。
|
||||
if got, loc := run(t, ""); got != cached {
|
||||
t.Fatalf("cached link = %q, want the cached %q", got, cached)
|
||||
} else if loc != fresh {
|
||||
t.Fatalf("Location = %q, want the resolved direct link", loc)
|
||||
}
|
||||
|
||||
// 客户端报告直链失效:换链前必须先清掉缓存。
|
||||
if got, loc := run(t, "&refresh=1"); got != "" {
|
||||
t.Fatalf("cached link must be dropped before resolving, got %q", got)
|
||||
} else if loc != fresh {
|
||||
t.Fatalf("Location = %q, want the freshly resolved direct link", loc)
|
||||
}
|
||||
}
|
||||
|
||||
// 没有注入解析器(测试/精简部署)时保持原有跳转,不受本次优化影响。
|
||||
func TestServeFileKeepsSTRMEndpointWithoutResolver(t *testing.T) {
|
||||
repos := directRedirectTestRepo(t)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
// ServeFile streams the file backing the given media ID using
|
||||
@@ -22,6 +23,47 @@ func (s *StreamService) ServeFile(w http.ResponseWriter, r *http.Request, mediaI
|
||||
return s.ServeFileWithCloudMode(w, r, mediaID, "")
|
||||
}
|
||||
|
||||
// PickCodeFromPlayTarget 从 /api/strm/play/... 形式的播放目标里取出 pickcode。
|
||||
// 115 直链缓存按 pickcode 分键,客户端要求强制换链时用它定位缓存条目。
|
||||
func PickCodeFromPlayTarget(raw string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || u == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(u.Query().Get("pickcode"))
|
||||
}
|
||||
|
||||
// directLinkRefreshMinAge 是「强制换链」的最小间隔:缓存条目刚写入不久就
|
||||
// 收到换链请求时忽略它,避免客户端重试风暴把 115 换链接口打出突发。
|
||||
// 测试会临时改成 0,让「CDN 拒绝 → 立刻换链」的路径可验证。
|
||||
var directLinkRefreshMinAge = 30 * time.Second
|
||||
|
||||
// wantsFreshLink 判断客户端是否要求跳过直链缓存重新换链(`?refresh=1`)。
|
||||
//
|
||||
// 115 直链只有 30 分钟有效期,客户端拿到过期直链被 CDN 403 后需要一条新链;
|
||||
// 正常路径下缓存会先于直链过期,这里是客户端上报失败时的兜底。
|
||||
func wantsFreshLink(r *http.Request) bool {
|
||||
if r == nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(r.URL.Query().Get("refresh"))) {
|
||||
case "1", "true", "yes":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// InvalidateMediaLinkCache 清除媒体播放目标对应的云盘直链缓存,让下一次解析
|
||||
// 重新换链。非云盘/无 pickcode 的媒体是空操作。
|
||||
func InvalidateMediaLinkCache(m *model.Media) {
|
||||
pickCode := PickCodeFromPlayTarget(MediaSTRMTarget(m))
|
||||
if pickCode == "" {
|
||||
return
|
||||
}
|
||||
cloud115.ClearDownloadURLCacheIfOlderThan(pickCode, directLinkRefreshMinAge)
|
||||
}
|
||||
|
||||
// MediaSTRMTarget 返回媒体行固化的 STRM 播放目标。STRMURL 为空时回读本地
|
||||
// .strm 文件内容兜底(扫描时内容解析失败的行只剩 Container=strm + Path),
|
||||
// 仍拿不到返回空串。
|
||||
@@ -57,6 +99,11 @@ func (s *StreamService) ServeFileWithCloudMode(w http.ResponseWriter, r *http.Re
|
||||
// 能在服务端换到最终直链就直接 302 过去:客户端少跟随一次 302,等于
|
||||
// 省掉一次「DNS+TCP+TLS+请求」的往返。解析结果同时写进 strm 层直链
|
||||
// 缓存,后续 /api/strm/play 请求直接命中。
|
||||
// `?refresh=1` 表示客户端手上的直链已经失效(被 CDN 403),要求丢掉
|
||||
// 缓存重新换链。
|
||||
if wantsFreshLink(r) {
|
||||
InvalidateMediaLinkCache(m)
|
||||
}
|
||||
if direct, ok := s.resolveDirectPlayTargetURL(r, strmURL); ok {
|
||||
setCloudRedirectNoStore(w)
|
||||
http.Redirect(w, r, direct, http.StatusFound)
|
||||
|
||||
@@ -17,7 +17,8 @@ import (
|
||||
// Optional query `start` (seconds) restarts ffmpeg from that source offset so
|
||||
// the web player can scrub the full timeline without waiting for a full
|
||||
// head-to-tail transcode.
|
||||
func (s *StreamService) ServeHLSPlaylist(w http.ResponseWriter, r *http.Request, mediaID string) error {
|
||||
func (s *StreamService) ServeHLSPlaylist(w http.ResponseWriter, r *http.Request, mediaID string, scopes ...HLSPlaybackScope) error {
|
||||
jobKey := hlsPlaybackJobKey(mediaID, scopes)
|
||||
// 「客户端直连解码」模式下宿主机不提供转码,HLS 一律拒绝,
|
||||
// 迫使播放器走 direct play 本地解码。
|
||||
if s.directPlayOnly(r.Context()) {
|
||||
@@ -27,19 +28,23 @@ func (s *StreamService) ServeHLSPlaylist(w http.ResponseWriter, r *http.Request,
|
||||
seekGen := parseHLSSeekGen(r)
|
||||
subtitleStream := parseHLSSubtitleStream(r)
|
||||
quality := parseHLSQuality(r)
|
||||
if _, err := s.transcoder.EnsureJobFromSubtitleQuality(r.Context(), mediaID, startSec, seekGen, subtitleStream, quality); err != nil {
|
||||
if _, err := s.transcoder.EnsureJobFromSubtitleQuality(r.Context(), mediaID, startSec, seekGen, subtitleStream, quality, scopes...); err != nil {
|
||||
return err
|
||||
}
|
||||
s.transcoder.TouchJob(mediaID)
|
||||
s.transcoder.TouchJob(jobKey)
|
||||
readyTimeout := 45 * time.Second
|
||||
if startSec > 0.05 {
|
||||
// Mid-file HTTP seeks (esp. WMV) need longer before the first segment appears.
|
||||
readyTimeout = 120 * time.Second
|
||||
}
|
||||
if !s.transcoder.WaitReady(r.Context(), mediaID, readyTimeout) {
|
||||
if !s.transcoder.WaitReady(r.Context(), jobKey, readyTimeout) {
|
||||
return errors.New("hls playlist not ready")
|
||||
}
|
||||
playlist := s.transcoder.PlaylistPath(mediaID)
|
||||
outputDir, err := s.transcoder.hlsOutputDir(jobKey, seekGen)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
playlist := filepath.Join(outputDir, "index.m3u8")
|
||||
f, err := os.Open(playlist) // #nosec G304 -- playlist path is generated under the transcoder cache directory for this media ID.
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -49,12 +54,18 @@ func (s *StreamService) ServeHLSPlaylist(w http.ResponseWriter, r *http.Request,
|
||||
w.Header().Set("Content-Type", "application/vnd.apple.mpegurl")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Content-Disposition", "inline")
|
||||
if r.URL.RawQuery != "" {
|
||||
rawQuery := r.URL.RawQuery
|
||||
if len(scopes) > 0 && scopes[0].SessionID != "" {
|
||||
query := r.URL.Query()
|
||||
query.Set("session_id", scopes[0].SessionID)
|
||||
rawQuery = query.Encode()
|
||||
}
|
||||
if rawQuery != "" {
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
playlist := appendQueryToHLSSegments(string(data), r.URL.RawQuery)
|
||||
playlist := appendQueryToHLSSegments(string(data), rawQuery)
|
||||
_, err = io.WriteString(w, playlist)
|
||||
return err
|
||||
}
|
||||
@@ -171,19 +182,26 @@ func filterHLSSegmentQuery(rawQuery string) string {
|
||||
}
|
||||
|
||||
// ServeHLSSegment writes a single .ts segment from the on-disk cache.
|
||||
func (s *StreamService) ServeHLSSegment(w http.ResponseWriter, r *http.Request, mediaID, segment string) error {
|
||||
s.transcoder.TouchJob(mediaID)
|
||||
func (s *StreamService) ServeHLSSegment(w http.ResponseWriter, r *http.Request, mediaID, segment string, scopes ...HLSPlaybackScope) error {
|
||||
jobKey := hlsPlaybackJobKey(mediaID, scopes)
|
||||
outputDir, err := s.transcoder.hlsOutputDir(jobKey, parseHLSSeekGen(r))
|
||||
if err != nil {
|
||||
if len(scopes) > 0 || parseHLSSeekGen(r) > 0 {
|
||||
return err
|
||||
}
|
||||
outputDir = s.transcoder.HLSDir(jobKey) // Legacy internal static-file callers.
|
||||
}
|
||||
// Only allow segments that look like seg_NNNNN.ts so we cannot be tricked
|
||||
// into reading arbitrary files via path traversal.
|
||||
if !strings.HasPrefix(segment, "seg_") || !strings.HasSuffix(segment, ".ts") {
|
||||
return errors.New("bad segment")
|
||||
}
|
||||
full := filepath.Join(s.transcoder.HLSDir(mediaID), segment)
|
||||
full := filepath.Join(outputDir, segment)
|
||||
abs, err := filepath.Abs(full)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, _ := filepath.Abs(s.transcoder.HLSDir(mediaID))
|
||||
dir, _ := filepath.Abs(outputDir)
|
||||
if !pathWithin(abs, dir) {
|
||||
return errors.New("path escape")
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
// ErrStrmPlayNotFound 表示 strm 播放目标不存在(handler 返回 404)。
|
||||
@@ -250,21 +251,87 @@ func (s *StrmService) ProxyMediaDirect(ctx context.Context, w http.ResponseWrite
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case result.Link != nil && result.Link.URL != "":
|
||||
return s.ProxyDirect(ctx, w, r, result.Link)
|
||||
case result.RedirectURL != "":
|
||||
return s.ProxyDirect(ctx, w, r, &cloud.DirectLink{URL: result.RedirectURL})
|
||||
default:
|
||||
link := directLinkOfPlayResult(result)
|
||||
if link == nil {
|
||||
// 本地文件(LocalPath)由静态文件处理器提供,本身就是同源。
|
||||
return ErrStrmProxyNotApplicable
|
||||
}
|
||||
resp, method, err := s.openDirectUpstream(ctx, r, link)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 直链被 CDN 拒绝:115 直链带 30 分钟过期时间戳,缓存命中也可能拿到已经
|
||||
// 失效的地址。这里清掉缓存重新换链再试一次,避免客户端只看到 403。
|
||||
if directLinkRejected(resp.StatusCode) {
|
||||
_ = resp.Body.Close()
|
||||
if fresh := s.refreshDirectLink(ctx, raw); fresh != nil {
|
||||
if retry, retryMethod, retryErr := s.openDirectUpstream(ctx, r, fresh); retryErr == nil {
|
||||
resp, method = retry, retryMethod
|
||||
}
|
||||
}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return writeDirectUpstream(w, resp, method)
|
||||
}
|
||||
|
||||
// directLinkOfPlayResult 取出可反向代理的直链;本地文件返回 nil。
|
||||
func directLinkOfPlayResult(result *StrmPlayResult) *cloud.DirectLink {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case result.Link != nil && result.Link.URL != "":
|
||||
return result.Link
|
||||
case result.RedirectURL != "":
|
||||
return &cloud.DirectLink{URL: result.RedirectURL}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// directLinkRejected 判断上游状态码是否属于「直链本身失效」(而不是文件/权限
|
||||
// 之类与链接无关的问题)。
|
||||
func directLinkRejected(status int) bool {
|
||||
switch status {
|
||||
case http.StatusForbidden, http.StatusNotFound, http.StatusGone:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// refreshDirectLink 丢弃当前直链缓存并重新换链,返回新的直链;无法换链时返回
|
||||
// nil,调用方继续用原来那条。
|
||||
func (s *StrmService) refreshDirectLink(ctx context.Context, raw string) *cloud.DirectLink {
|
||||
pickCode := PickCodeFromPlayTarget(raw)
|
||||
if pickCode == "" {
|
||||
return nil
|
||||
}
|
||||
if !cloud115.ClearDownloadURLCacheIfOlderThan(pickCode, directLinkRefreshMinAge) {
|
||||
// 缓存刚写入不久:说明失败与链接过期无关,重换一次也会拿到同一条。
|
||||
return nil
|
||||
}
|
||||
fresh, err := s.ResolvePlayTarget(ctx, raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return directLinkOfPlayResult(fresh)
|
||||
}
|
||||
|
||||
// ProxyDirect 反向代理渲染直链内容(保留 Range 请求头以支持拖动播放)。
|
||||
func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r *http.Request, link *cloud.DirectLink) error {
|
||||
resp, method, err := s.openDirectUpstream(ctx, r, link)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return writeDirectUpstream(w, resp, method)
|
||||
}
|
||||
|
||||
// openDirectUpstream 向网盘直链发起一次请求,保留 Range 与 UA 语义。
|
||||
func (s *StrmService) openDirectUpstream(ctx context.Context, r *http.Request, link *cloud.DirectLink) (*http.Response, string, error) {
|
||||
if link == nil || link.URL == "" {
|
||||
return errors.New("空直链")
|
||||
return nil, "", errors.New("空直链")
|
||||
}
|
||||
method := http.MethodGet
|
||||
if r != nil && r.Method == http.MethodHead {
|
||||
@@ -272,7 +339,7 @@ func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, link.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, "", err
|
||||
}
|
||||
for k, v := range link.Headers {
|
||||
req.Header.Set(k, v)
|
||||
@@ -288,9 +355,13 @@ func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r
|
||||
}
|
||||
resp, err := s.streamHTTP.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return resp, method, nil
|
||||
}
|
||||
|
||||
// writeDirectUpstream 把上游响应原样写给客户端。
|
||||
func writeDirectUpstream(w http.ResponseWriter, resp *http.Response, method string) error {
|
||||
for _, header := range []string{"Content-Type", "Content-Length", "Content-Range", "Accept-Ranges", "ETag", "Last-Modified"} {
|
||||
if value := resp.Header.Get(header); value != "" {
|
||||
w.Header().Set(header, value)
|
||||
@@ -310,7 +381,7 @@ func (s *StrmService) ProxyDirect(ctx context.Context, w http.ResponseWriter, r
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
writer = &flushWriter{writer: w, flusher: flusher}
|
||||
}
|
||||
_, err = io.Copy(writer, resp.Body)
|
||||
_, err := io.Copy(writer, resp.Body)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -3,12 +3,16 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
// 同源转发必须原样保留 Range 语义:把 206 改写成 200 会让浏览器误判响应长度,
|
||||
@@ -55,6 +59,134 @@ func TestProxyMediaDirectForwardsRangeAndStatus(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 115 直链只有 30 分钟寿命,缓存命中也可能拿到已经失效的地址。同源转发遇到
|
||||
// CDN 403 时必须丢掉缓存重新换链再试一次,而不是把 403 直接透给客户端。
|
||||
func TestProxyMediaDirectRetriesWithFreshLinkWhenCDNRejects(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
|
||||
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/fresh.mkv" {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "video/x-matroska")
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write([]byte("fresh-bytes"))
|
||||
}))
|
||||
defer cdn.Close()
|
||||
|
||||
// 第一次换链给出「已过期」的旧地址,重新换链才给新地址。
|
||||
var mu sync.Mutex
|
||||
var calls int
|
||||
expiry := time.Now().Add(30 * time.Minute).Unix()
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
calls++
|
||||
attempt := calls
|
||||
mu.Unlock()
|
||||
link := cdn.URL + "/old.mkv"
|
||||
if attempt > 1 {
|
||||
link = cdn.URL + "/fresh.mkv"
|
||||
}
|
||||
_, _ = fmt.Fprintf(w, `{"state":true,"data":{"1":{"pick_code":"pc-retry","url":{"url":%q}}}}`,
|
||||
fmt.Sprintf("%s?t=%d&u=1&k=abc", link, expiry))
|
||||
}))
|
||||
defer api.Close()
|
||||
oldPro := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = api.URL
|
||||
defer func() { cloud115.ProAPIBase = oldPro }()
|
||||
defer cloud115.ClearDownloadURLCache("pc-retry")
|
||||
oldMinAge := directLinkRefreshMinAge
|
||||
directLinkRefreshMinAge = 0
|
||||
defer func() { directLinkRefreshMinAge = oldMinAge }()
|
||||
|
||||
media := &model.Media{
|
||||
STRMURL: "/api/strm/play/cloud115/video.mkv?acct=" + acct.ID + "&pickcode=pc-retry",
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/stream/media-1?proxy=1", nil)
|
||||
req.Header.Set("User-Agent", "MeBoxTest/1.0")
|
||||
|
||||
if err := svc.ProxyMediaDirect(context.Background(), rec, req, media); err != nil {
|
||||
t.Fatalf("ProxyMediaDirect: %v", err)
|
||||
}
|
||||
if rec.Code != http.StatusPartialContent {
|
||||
t.Fatalf("status = %d, want 206 (body=%s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
if rec.Body.String() != "fresh-bytes" {
|
||||
t.Fatalf("body = %q, want the fresh link payload", rec.Body.String())
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if calls != 2 {
|
||||
t.Fatalf("downurl calls = %d, want 2 (one retry after the CDN rejected the stale link)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 换链后依然是同一条失效直链时,保持原有行为:原样透传 CDN 的状态码。
|
||||
func TestProxyMediaDirectPassesRejectedStatusWhenRefreshCannotHelp(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
|
||||
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
}))
|
||||
defer cdn.Close()
|
||||
|
||||
expiry := time.Now().Add(30 * time.Minute).Unix()
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, `{"state":true,"data":{"1":{"pick_code":"pc-dead","url":{"url":%q}}}}`,
|
||||
fmt.Sprintf("%s/dead.mkv?t=%d&u=1&k=abc", cdn.URL, expiry))
|
||||
}))
|
||||
defer api.Close()
|
||||
oldPro := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = api.URL
|
||||
defer func() { cloud115.ProAPIBase = oldPro }()
|
||||
defer cloud115.ClearDownloadURLCache("pc-dead")
|
||||
oldMinAge := directLinkRefreshMinAge
|
||||
directLinkRefreshMinAge = 0
|
||||
defer func() { directLinkRefreshMinAge = oldMinAge }()
|
||||
|
||||
media := &model.Media{
|
||||
STRMURL: "/api/strm/play/cloud115/video.mkv?acct=" + acct.ID + "&pickcode=pc-dead",
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/stream/media-1?proxy=1", nil)
|
||||
|
||||
if err := svc.ProxyMediaDirect(context.Background(), rec, req, media); err != nil {
|
||||
t.Fatalf("ProxyMediaDirect: %v", err)
|
||||
}
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want the upstream 403 passed through", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// 强制换链只认 /api/strm/play 形式播放目标里的 pickcode。
|
||||
func TestPickCodeFromPlayTarget(t *testing.T) {
|
||||
cases := []struct {
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{"/api/strm/play/cloud115/video.mkv?acct=a1&pickcode=pc1", "pc1"},
|
||||
{"https://nas.local/api/strm/play/cloud115/video.mkv?pickcode=pc2&acct=a1", "pc2"},
|
||||
{"/api/strm/play/cloud115/video.mkv?acct=a1", ""},
|
||||
{"https://cdn.example.test/Movie.mkv?sign=1", ""},
|
||||
{"", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := PickCodeFromPlayTarget(tc.raw); got != tc.want {
|
||||
t.Fatalf("PickCodeFromPlayTarget(%q) = %q, want %q", tc.raw, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 本地文件本身就是同源资源,不应该被代理(调用方按原静态文件逻辑处理)。
|
||||
func TestProxyMediaDirectSkipsLocalFile(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
|
||||
@@ -182,6 +182,77 @@ func (s *SubtitleService) discoverExternalUncached(ctx context.Context, mediaID
|
||||
return discoverExternalSubtitleTracks(m), nil
|
||||
}
|
||||
|
||||
// subtitleLanguageNames 把文件名后缀/容器里的语言标记折成可读名称。
|
||||
//
|
||||
// 外挂字幕只能从文件名拿到语言(`movie.chs.ass` → `chs`),直接展示给用户
|
||||
// 就是「chs」这种看不懂的值;内嵌轨没有 title 标签时同理。认不出来的标记
|
||||
// 原样保留,交给调用方(例如 Emby 的「字幕 (ASS)」兜底)处理。
|
||||
var subtitleLanguageNames = map[string]string{
|
||||
"zh": "中文",
|
||||
"chi": "中文",
|
||||
"zho": "中文",
|
||||
"cn": "中文",
|
||||
|
||||
"chs": "简体中文",
|
||||
"sc": "简体中文",
|
||||
"zh-cn": "简体中文",
|
||||
"zh-hans": "简体中文",
|
||||
"zh-sg": "简体中文",
|
||||
|
||||
"cht": "繁体中文",
|
||||
"tc": "繁体中文",
|
||||
"zh-tw": "繁体中文",
|
||||
"zh-hk": "繁体中文",
|
||||
"zh-mo": "繁体中文",
|
||||
"zh-hant": "繁体中文",
|
||||
|
||||
"jpn": "日本語",
|
||||
"ja": "日本語",
|
||||
"jp": "日本語",
|
||||
|
||||
"eng": "English",
|
||||
"en": "English",
|
||||
|
||||
"kor": "한국어",
|
||||
"ko": "한국어",
|
||||
|
||||
"yue": "粤语",
|
||||
}
|
||||
|
||||
// subtitleLanguageName 返回语言标记的可读名称,认不出来时返回空串。
|
||||
func subtitleLanguageName(raw string) string {
|
||||
tag := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(raw, "_", "-")))
|
||||
if tag == "" {
|
||||
return ""
|
||||
}
|
||||
if name, ok := subtitleLanguageNames[tag]; ok {
|
||||
return name
|
||||
}
|
||||
// `chs[bilibiliid_danmu]`、`chs.ass` 这类带附加信息的标记先取前缀。
|
||||
head := tag
|
||||
if i := strings.IndexAny(head, "[.@"); i > 0 {
|
||||
head = strings.TrimSpace(head[:i])
|
||||
}
|
||||
return subtitleLanguageNames[head]
|
||||
}
|
||||
|
||||
// subtitleDisplayLabel 把语言标记换成可读名称;认不出来时保留原值(含 und)。
|
||||
func subtitleDisplayLabel(raw string) string {
|
||||
if name := subtitleLanguageName(raw); name != "" {
|
||||
return name
|
||||
}
|
||||
return strings.TrimSpace(raw)
|
||||
}
|
||||
|
||||
// isDanmakuSubtitle 报告字幕文件是否是弹幕转存(`xxx[BilibiliID_danmu].ass`)。
|
||||
// 这类轨道是飘过的弹幕而不是对白字幕,名称里要标出来,避免和对白字幕混淆。
|
||||
func isDanmakuSubtitle(name string) bool {
|
||||
lower := strings.ToLower(name)
|
||||
return strings.Contains(lower, "danmu") ||
|
||||
strings.Contains(lower, "danmaku") ||
|
||||
strings.Contains(name, "弹幕")
|
||||
}
|
||||
|
||||
func discoverExternalSubtitleTracks(m *model.Media) []SubtitleTrack {
|
||||
dir := filepath.Dir(m.Path)
|
||||
bases := mediaSidecarBaseVariants(m.Path)
|
||||
@@ -228,9 +299,13 @@ func discoverExternalSubtitleTracks(m *model.Media) []SubtitleTrack {
|
||||
matchedBase = bases[0]
|
||||
}
|
||||
lang := detectLang(fullName, matchedBase)
|
||||
label := subtitleDisplayLabel(lang)
|
||||
if isDanmakuSubtitle(e.Name()) {
|
||||
label += " · 弹幕"
|
||||
}
|
||||
tracks = append(tracks, SubtitleTrack{
|
||||
Lang: lang,
|
||||
Label: lang,
|
||||
Label: label,
|
||||
Path: filepath.Join(c, e.Name()),
|
||||
Codec: codec,
|
||||
Source: "external",
|
||||
@@ -319,7 +394,11 @@ func subtitleTracksFromProbe(probe embeddedSubtitleProbe) []SubtitleTrack {
|
||||
}
|
||||
label := strings.TrimSpace(stream.Tags.Title)
|
||||
if label == "" {
|
||||
label = lang
|
||||
// 没有 title 标签时用可读语言名(chi → 中文)代替裸语言码。
|
||||
label = subtitleDisplayLabel(lang)
|
||||
}
|
||||
if label == "" {
|
||||
label = "字幕"
|
||||
}
|
||||
if stream.Disposition.Forced != 0 {
|
||||
label += "(强制)"
|
||||
|
||||
@@ -202,3 +202,103 @@ func TestSubtitleServeRawWritesSourceBytes(t *testing.T) {
|
||||
t.Fatal("ServeRaw accepted a non-subtitle file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitleLanguageNameMapsCommonTags(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"chs": "简体中文",
|
||||
"CHS": "简体中文",
|
||||
"zh-Hans": "简体中文",
|
||||
"cht": "繁体中文",
|
||||
"chi": "中文",
|
||||
"zh": "中文",
|
||||
"jpn": "日本語",
|
||||
"eng": "English",
|
||||
"kor": "한국어",
|
||||
"chs[bilibiliid_danmu]": "简体中文",
|
||||
"und": "",
|
||||
"": "",
|
||||
}
|
||||
for raw, want := range cases {
|
||||
if got := subtitleLanguageName(raw); got != want {
|
||||
t.Errorf("subtitleLanguageName(%q) = %q, want %q", raw, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubtitleDisplayLabelKeepsUnknownTags(t *testing.T) {
|
||||
// 认不出来的标记原样返回:Emby 的 DisplayTitle 依赖 `und` 走「字幕 (ASS)」兜底。
|
||||
if got := subtitleDisplayLabel("und"); got != "und" {
|
||||
t.Fatalf("subtitleDisplayLabel(und) = %q, want und", got)
|
||||
}
|
||||
if got := subtitleDisplayLabel("weird"); got != "weird" {
|
||||
t.Fatalf("subtitleDisplayLabel(weird) = %q, want weird", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsDanmakuSubtitle(t *testing.T) {
|
||||
if !isDanmakuSubtitle("S01E01.chs[BilibiliID_danmu].ass") {
|
||||
t.Fatal("danmaku sidecar was not detected")
|
||||
}
|
||||
if !isDanmakuSubtitle("弹幕.ass") {
|
||||
t.Fatal("Chinese danmaku name was not detected")
|
||||
}
|
||||
if isDanmakuSubtitle("S01E01.chs.ass") {
|
||||
t.Fatal("normal subtitle was misdetected as danmaku")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverExternalSubtitleLabelsAreReadable(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
videoPath := filepath.Join(dir, "S01E01.mkv")
|
||||
files := map[string]string{
|
||||
"S01E01.chs.ass": "[Script Info]\n",
|
||||
"S01E01.chs[BilibiliID_danmu].ass": "[Script Info]\n",
|
||||
}
|
||||
for name, body := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
m := &model.Media{Path: videoPath}
|
||||
tracks := discoverExternalSubtitleTracks(m)
|
||||
if len(tracks) != 2 {
|
||||
t.Fatalf("len(tracks) = %d, want 2: %#v", len(tracks), tracks)
|
||||
}
|
||||
labels := map[string]string{}
|
||||
for _, track := range tracks {
|
||||
labels[filepath.Base(track.Path)] = track.Label
|
||||
}
|
||||
if got := labels["S01E01.chs.ass"]; got != "简体中文" {
|
||||
t.Fatalf("chs.ass label = %q, want 简体中文", got)
|
||||
}
|
||||
if got := labels["S01E01.chs[BilibiliID_danmu].ass"]; got != "简体中文 · 弹幕" {
|
||||
t.Fatalf("danmaku label = %q, want 简体中文 · 弹幕", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedSubtitleProbeUsesReadableLanguage(t *testing.T) {
|
||||
var probe embeddedSubtitleProbe
|
||||
raw := []byte(`{"streams":[
|
||||
{"index":2,"codec_name":"hdmv_pgs_subtitle","tags":{"language":"chi"},"disposition":{"default":1}},
|
||||
{"index":3,"codec_name":"subrip","tags":{"language":"jpn"}},
|
||||
{"index":4,"codec_name":"subrip","tags":{"language":"chi","title":"简日双语"}}
|
||||
]}`)
|
||||
if err := json.Unmarshal(raw, &probe); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tracks := subtitleTracksFromProbe(probe)
|
||||
if len(tracks) != 3 {
|
||||
t.Fatalf("len(tracks) = %d, want 3", len(tracks))
|
||||
}
|
||||
if got := tracks[0].Label; got != "中文(默认)(内嵌·图片)" {
|
||||
t.Fatalf("bitmap label = %q", got)
|
||||
}
|
||||
if got := tracks[1].Label; got != "日本語(内嵌)" {
|
||||
t.Fatalf("text label = %q", got)
|
||||
}
|
||||
// 容器自带 title 时优先用 title。
|
||||
if got := tracks[2].Label; got != "简日双语(内嵌)" {
|
||||
t.Fatalf("titled label = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"html"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -9,12 +10,16 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
TaskStatusRunning = "running"
|
||||
TaskStatusCompleted = "completed"
|
||||
TaskStatusFailed = "failed"
|
||||
TaskStatusRunning = "running"
|
||||
TaskStatusCompleted = "completed"
|
||||
TaskStatusFailed = "failed"
|
||||
TaskStatusInterrupted = "interrupted"
|
||||
|
||||
TaskKindOrganize = "organize"
|
||||
TaskKindScan = "scan"
|
||||
@@ -70,6 +75,26 @@ type TaskTrackerService struct {
|
||||
recent []BackgroundTask
|
||||
maxRecent int
|
||||
now func() time.Time
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// SetPersistence enables durable task history and marks tasks left running by
|
||||
// a previous process as interrupted. Actual task execution is never resumed
|
||||
// automatically; this record is an operator-visible recovery point.
|
||||
func (t *TaskTrackerService) SetPersistence(db *gorm.DB) {
|
||||
if t == nil || db == nil {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.db = db
|
||||
now := t.currentTime()
|
||||
t.mu.Unlock()
|
||||
err := db.Model(&model.BackgroundTaskRecord{}).
|
||||
Where("status = ?", TaskStatusRunning).
|
||||
Updates(map[string]any{"status": TaskStatusInterrupted, "error": "服务在任务完成前重启;请检查文件状态后重新执行", "updated_at": now, "finished_at": now}).Error
|
||||
if err != nil && t.log != nil {
|
||||
t.log.Warn("recover interrupted background tasks failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
type TaskHandle struct {
|
||||
@@ -105,9 +130,11 @@ func (t *TaskTrackerService) Start(kind, name string, update TaskUpdate) *TaskHa
|
||||
StartedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
applyTaskUpdate(task, update)
|
||||
t.mu.Lock()
|
||||
t.active[task.ID] = task
|
||||
snapshot := cloneBackgroundTask(*task)
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
return &TaskHandle{tracker: t, id: task.ID}
|
||||
@@ -132,18 +159,47 @@ func (t *TaskTrackerService) Snapshot() TaskSnapshot {
|
||||
return TaskSnapshot{}
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
active := make([]BackgroundTask, 0, len(t.active))
|
||||
for _, task := range t.active {
|
||||
active = append(active, cloneBackgroundTask(*task))
|
||||
}
|
||||
recent := make([]BackgroundTask, 0, len(t.recent))
|
||||
db := t.db
|
||||
limit := t.maxRecent
|
||||
if limit < 1 {
|
||||
limit = 30
|
||||
}
|
||||
for _, task := range t.recent {
|
||||
recent = append(recent, cloneBackgroundTask(task))
|
||||
}
|
||||
t.mu.Unlock()
|
||||
if db != nil {
|
||||
var rows []model.BackgroundTaskRecord
|
||||
if err := db.Where("status <> ?", TaskStatusRunning).Order("started_at DESC").Limit(limit).Find(&rows).Error; err == nil {
|
||||
recent = make([]BackgroundTask, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
recent = append(recent, backgroundTaskFromRecord(row))
|
||||
}
|
||||
} else if t.log != nil {
|
||||
t.log.Warn("load background task history failed; using memory history", zap.Error(err))
|
||||
}
|
||||
}
|
||||
return TaskSnapshot{Active: active, Recent: recent}
|
||||
}
|
||||
|
||||
func backgroundTaskFromRecord(row model.BackgroundTaskRecord) BackgroundTask {
|
||||
task := BackgroundTask{
|
||||
ID: row.ID, Kind: row.Kind, Name: row.Name, Status: row.Status,
|
||||
Stage: row.Stage, SourcePath: row.SourcePath, DestPath: row.DestPath,
|
||||
Message: row.Message, Error: row.Error, StartedAt: row.StartedAt,
|
||||
UpdatedAt: row.UpdatedAt, FinishedAt: row.FinishedAt,
|
||||
}
|
||||
_ = json.Unmarshal([]byte(row.Details), &task.Details)
|
||||
_ = json.Unmarshal([]byte(row.Metrics), &task.Metrics)
|
||||
_ = json.Unmarshal([]byte(row.Items), &task.Items)
|
||||
return task
|
||||
}
|
||||
|
||||
func (t *TaskTrackerService) update(id string, update TaskUpdate) {
|
||||
now := t.currentTime()
|
||||
t.mu.Lock()
|
||||
@@ -155,6 +211,7 @@ func (t *TaskTrackerService) update(id string, update TaskUpdate) {
|
||||
applyTaskUpdate(task, update)
|
||||
task.UpdatedAt = now
|
||||
snapshot := cloneBackgroundTask(*task)
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
}
|
||||
@@ -186,6 +243,7 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
|
||||
t.recent = t.recent[:t.maxRecent]
|
||||
}
|
||||
notifier := t.failureNotifier
|
||||
t.persist(snapshot)
|
||||
t.mu.Unlock()
|
||||
t.publish(snapshot)
|
||||
|
||||
@@ -195,6 +253,26 @@ func (t *TaskTrackerService) finish(id string, err error, update TaskUpdate) {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TaskTrackerService) persist(task BackgroundTask) {
|
||||
if t == nil || t.db == nil {
|
||||
return
|
||||
}
|
||||
encode := func(value any) string {
|
||||
data, _ := json.Marshal(value)
|
||||
return string(data)
|
||||
}
|
||||
row := model.BackgroundTaskRecord{
|
||||
ID: task.ID, Kind: task.Kind, Name: task.Name, Status: task.Status,
|
||||
Stage: task.Stage, SourcePath: task.SourcePath, DestPath: task.DestPath,
|
||||
Message: task.Message, Error: task.Error, Details: encode(task.Details),
|
||||
Metrics: encode(task.Metrics), Items: encode(task.Items), StartedAt: task.StartedAt,
|
||||
UpdatedAt: task.UpdatedAt, FinishedAt: task.FinishedAt,
|
||||
}
|
||||
if err := t.db.Save(&row).Error; err != nil && t.log != nil {
|
||||
t.log.Warn("persist background task failed", zap.String("task_id", task.ID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// SetFailureNotifier 注入任务失败时的管理员通知回调。
|
||||
func (t *TaskTrackerService) SetFailureNotifier(fn func(ctx context.Context, text string)) {
|
||||
if t == nil {
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
func TestTaskTrackerPersistsHistoryAcrossRestart(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
task := tracker.Start(TaskKindScan, "scan", TaskUpdate{Details: []string{"initial detail"}})
|
||||
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 42}})
|
||||
task.Finish(nil, TaskUpdate{Message: "done"})
|
||||
restarted := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
restarted.SetPersistence(db)
|
||||
snapshot := restarted.Snapshot()
|
||||
if len(snapshot.Active) != 0 || len(snapshot.Recent) != 1 {
|
||||
t.Fatalf("unexpected history: %#v", snapshot)
|
||||
}
|
||||
recent := snapshot.Recent[0]
|
||||
if recent.Status != TaskStatusCompleted || recent.Metrics["visited"] != 42 || len(recent.Details) != 1 || recent.FinishedAt == nil {
|
||||
t.Fatalf("task history lost fields: %#v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskTrackerConcurrentUpdatesDoNotRegressFinishedTasks(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
var workers sync.WaitGroup
|
||||
for i := 0; i < 12; i++ {
|
||||
workers.Add(1)
|
||||
go func(index int) {
|
||||
defer workers.Done()
|
||||
task := tracker.Start(TaskKindScan, fmt.Sprintf("scan-%d", index), TaskUpdate{})
|
||||
task.Update(TaskUpdate{Metrics: map[string]int64{"visited": 1}})
|
||||
task.Finish(nil, TaskUpdate{})
|
||||
task.Update(TaskUpdate{Message: "late update"})
|
||||
_ = tracker.Snapshot()
|
||||
}(i)
|
||||
}
|
||||
workers.Wait()
|
||||
var completed int64
|
||||
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusCompleted).Count(&completed).Error; err != nil || completed != 12 {
|
||||
t.Fatalf("finished task status regressed: completed=%d err=%v", completed, err)
|
||||
}
|
||||
if snapshot := tracker.Snapshot(); len(snapshot.Active) != 0 || len(snapshot.Recent) != 12 {
|
||||
t.Fatalf("inconsistent task snapshot: %#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskTrackerMarksAbandonedTasksInterrupted(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
tracker := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
tracker.SetPersistence(db)
|
||||
tracker.Start(TaskKindOrganize, "unfinished", TaskUpdate{})
|
||||
restarted := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
restarted.SetPersistence(db)
|
||||
snapshot := restarted.Snapshot()
|
||||
if len(snapshot.Recent) != 1 || snapshot.Recent[0].Status != TaskStatusInterrupted || snapshot.Recent[0].FinishedAt == nil {
|
||||
t.Fatalf("abandoned task not marked interrupted: %#v", snapshot)
|
||||
}
|
||||
var running int64
|
||||
if err := db.Model(&model.BackgroundTaskRecord{}).Where("status = ?", TaskStatusRunning).Count(&running).Error; err != nil || running != 0 {
|
||||
t.Fatalf("stale running tasks: %d err=%v", running, err)
|
||||
}
|
||||
}
|
||||
@@ -170,8 +170,16 @@ func (s *TokenService) storeRefreshToken(ctx context.Context, rt *model.RefreshT
|
||||
if err := s.repo.RefreshToken.Create(ctx, rt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.repo.RefreshToken.RevokeOldestActiveByUserID(ctx, rt.UserID, s.maxActiveRefreshTokens(ctx)); err != nil && s.log != nil {
|
||||
s.log.Warn("failed to enforce refresh token session limit", zap.String("user_id", rt.UserID), zap.Error(err))
|
||||
// 「最大同时登录终端」只约束未绑定设备会话的旧客户端。
|
||||
//
|
||||
// 移动端每台设备有独立认证会话,刷新令牌归属该会话;如果仍按用户统一
|
||||
// 裁剪活跃令牌,第 N+1 台设备登录(或任一设备刷新)就会把最早那台设备
|
||||
// 的刷新令牌标记为已撤销,那台设备下次刷新收到 SESSION_REVOKED,被迫
|
||||
// 重新登录。设备数量由设备策略单独约束,不能靠撤销刷新令牌实现。
|
||||
if rt.AuthSessionID == "" {
|
||||
if err := s.repo.RefreshToken.RevokeOldestActiveLegacyByUserID(ctx, rt.UserID, s.maxActiveRefreshTokens(ctx)); err != nil && s.log != nil {
|
||||
s.log.Warn("failed to enforce refresh token session limit", zap.String("user_id", rt.UserID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
// Package service — HLS on-demand transcoder.
|
||||
//
|
||||
// TranscoderService spawns ffmpeg processes that segment a source media file
|
||||
// into HLS (.m3u8 + .ts). The output lives under cache.cache_dir/hls/<id>.
|
||||
// into HLS (.m3u8 + .ts). Each playback scope owns a cache directory and
|
||||
// every ffmpeg run writes to its own run-* subdirectory.
|
||||
// The HTTP layer serves these files directly with a normal http.FileServer.
|
||||
//
|
||||
// Encoder selection (read once at startup from the config):
|
||||
@@ -15,9 +16,11 @@
|
||||
// plus the kernel module loaded)
|
||||
//
|
||||
// Concurrency model:
|
||||
// - Each Media has at most one active ffmpeg job.
|
||||
// - jobs[mediaID] tracks the running goroutine + cancel func.
|
||||
// - Calling Start while a job already exists is a no-op.
|
||||
// - Each authenticated user/player scope has at most one active ffmpeg job.
|
||||
// - jobs[jobKey] tracks the running goroutine + cancel func; mediaID remains
|
||||
// the actual media identity for database access and activity reporting.
|
||||
// - Legacy internal callers keep media-keyed behaviour.
|
||||
// - Equivalent requests within one scope reuse the current output.
|
||||
// - When the playlist file appears on disk we consider the job "ready"
|
||||
// and unblock the HTTP handler that was waiting on it.
|
||||
package service
|
||||
@@ -49,9 +52,10 @@ type TranscoderService struct {
|
||||
|
||||
mu sync.Mutex
|
||||
jobs map[string]*hlsJob
|
||||
// startGates serializes EnsureJobFrom / StopJob per media so concurrent
|
||||
// playlist hits cannot spawn multiple ffmpeg writers into one HLS dir.
|
||||
startGates sync.Map // mediaID -> *sync.Mutex
|
||||
// Start gates are reference-counted so random playback session keys do not
|
||||
// accumulate after their last starter/stopper has left.
|
||||
startGateMu sync.Mutex
|
||||
startGates map[string]*hlsStartGate
|
||||
strmResolve func(ctx context.Context, raw string) (*StrmPlayResult, error)
|
||||
probe *FFprobeService
|
||||
|
||||
@@ -66,6 +70,7 @@ type TranscoderService struct {
|
||||
|
||||
// hlsJob holds the live state of one ffmpeg run.
|
||||
type hlsJob struct {
|
||||
jobKey string
|
||||
mediaID string
|
||||
outputDir string
|
||||
cancel context.CancelFunc
|
||||
@@ -156,14 +161,19 @@ func (t *TranscoderService) transcodeConfig() *config.Config {
|
||||
}
|
||||
}
|
||||
|
||||
// HLSDir is the per-media directory that holds index.m3u8 + segment files.
|
||||
func (t *TranscoderService) HLSDir(mediaID string) string {
|
||||
return filepath.Join(t.cfg.Cache.CacheDir, "hls", mediaID)
|
||||
// HLSDir is the cache root for one internal playback job key.
|
||||
func (t *TranscoderService) HLSDir(jobKey string) string {
|
||||
return filepath.Join(t.cfg.Cache.CacheDir, "hls", jobKey)
|
||||
}
|
||||
|
||||
// PlaylistPath returns the absolute path of the m3u8 playlist for a media.
|
||||
func (t *TranscoderService) PlaylistPath(mediaID string) string {
|
||||
return filepath.Join(t.HLSDir(mediaID), "index.m3u8")
|
||||
func (t *TranscoderService) PlaylistPath(jobKey string) string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if job := t.jobs[jobKey]; job != nil {
|
||||
return t.jobPlaylistPath(job)
|
||||
}
|
||||
return filepath.Join(t.HLSDir(jobKey), "index.m3u8")
|
||||
}
|
||||
|
||||
// EnsureJob makes sure a transcode is running for mediaID from the start of
|
||||
@@ -194,7 +204,8 @@ func (t *TranscoderService) EnsureJobFromSubtitle(ctx context.Context, mediaID s
|
||||
|
||||
// EnsureJobFromSubtitleQuality is EnsureJobFromSubtitle with an explicit local
|
||||
// HLS quality profile. Empty qualityID keeps the global transcoder config.
|
||||
func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, mediaID string, startSec float64, seekGen int64, subtitleStream int, qualityID string) (string, error) {
|
||||
func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, mediaID string, startSec float64, seekGen int64, subtitleStream int, qualityID string, scopes ...HLSPlaybackScope) (string, error) {
|
||||
jobKey := hlsPlaybackJobKey(mediaID, scopes)
|
||||
cfg := t.transcodeConfig()
|
||||
if !cfg.Transcoder.Enabled {
|
||||
return "", ErrTranscodeDisabled
|
||||
@@ -219,28 +230,31 @@ func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, me
|
||||
return "", ErrMediaNotFound
|
||||
}
|
||||
|
||||
gate := t.mediaStartGate(mediaID)
|
||||
gate.Lock()
|
||||
defer gate.Unlock()
|
||||
unlock := t.lockJobStart(jobKey)
|
||||
defer unlock()
|
||||
|
||||
t.mu.Lock()
|
||||
if existing, ok := t.jobs[mediaID]; ok {
|
||||
if existing, ok := t.jobs[jobKey]; ok {
|
||||
reusable := true
|
||||
playlist := t.jobPlaylistPath(existing)
|
||||
if existing.completed {
|
||||
info, err := os.Stat(t.PlaylistPath(mediaID))
|
||||
info, err := os.Stat(playlist)
|
||||
reusable = err == nil && info.Size() > 0
|
||||
}
|
||||
if reusable && sameHLSConfiguration(existing, startSec, subtitleStream, qualityID) {
|
||||
t.touchJobLocked(mediaID)
|
||||
if seekGen > existing.seekGen {
|
||||
existing.seekGen = seekGen
|
||||
}
|
||||
t.touchJobLocked(jobKey)
|
||||
t.mu.Unlock()
|
||||
return t.PlaylistPath(mediaID), nil
|
||||
return playlist, nil
|
||||
}
|
||||
if reusable && !shouldReplaceHLSJobConfigurationQuality(existing, startSec, seekGen, subtitleStream, qualityID) {
|
||||
t.touchJobLocked(mediaID)
|
||||
t.touchJobLocked(jobKey)
|
||||
t.mu.Unlock()
|
||||
return t.PlaylistPath(mediaID), nil
|
||||
return playlist, nil
|
||||
}
|
||||
prev := t.detachJobLocked(mediaID)
|
||||
prev := t.detachJobLocked(jobKey)
|
||||
t.mu.Unlock()
|
||||
waitJobExit(prev, 12*time.Second)
|
||||
} else {
|
||||
@@ -258,6 +272,9 @@ func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, me
|
||||
input.Quality = quality
|
||||
t.maybeFillDuration(ctx, m, input)
|
||||
settings := ffmpegArgSettingsFromConfig(cfg, quality)
|
||||
// 「原画」档位遇到浏览器可解的 H.264 源时只换封装,避免慢机器上重编码
|
||||
// 追不上播放(两核软件编码实测 0.1~0.2 倍速)。其它档位要改分辨率,仍重编码。
|
||||
input.CopyVideo = t.remuxEligible(ctx, m, quality, subtitleStream)
|
||||
jobEncoder := settings.encoder
|
||||
if settings.noScale || subtitleStream >= 0 {
|
||||
jobEncoder = ""
|
||||
@@ -267,12 +284,23 @@ func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, me
|
||||
return "", err
|
||||
}
|
||||
|
||||
outDir := t.HLSDir(mediaID)
|
||||
// Wipe prior segments so a mid-file restart cannot serve stale early chunks.
|
||||
// Only safe after the previous ffmpeg has exited (waited above / via gate).
|
||||
if err := resetHLSDir(outDir); err != nil {
|
||||
rootDir := t.HLSDir(jobKey)
|
||||
// Runs use distinct directories even when Windows keeps an old segment
|
||||
// open. A delayed request can never read a newer run's seg_00000.ts.
|
||||
if err := resetHLSDir(rootDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
outDir, err := os.MkdirTemp(rootDir, "run-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
registered := false
|
||||
defer func() {
|
||||
if !registered {
|
||||
_ = os.RemoveAll(outDir)
|
||||
_ = os.Remove(rootDir)
|
||||
}
|
||||
}()
|
||||
|
||||
t.mu.Lock()
|
||||
if max := t.maxConcurrent(); max > 0 && t.runningJobCountLocked() >= max {
|
||||
@@ -287,7 +315,11 @@ func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, me
|
||||
if subtitleStream >= 0 {
|
||||
jobEncoder = "software (subtitle burn-in)"
|
||||
}
|
||||
if input.CopyVideo {
|
||||
jobEncoder = remuxEncoderLabel
|
||||
}
|
||||
job := &hlsJob{
|
||||
jobKey: jobKey,
|
||||
mediaID: mediaID,
|
||||
outputDir: outDir,
|
||||
cancel: cancel,
|
||||
@@ -302,7 +334,8 @@ func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, me
|
||||
ffmpegPath: bin,
|
||||
config: cfg,
|
||||
}
|
||||
t.jobs[mediaID] = job
|
||||
t.jobs[jobKey] = job
|
||||
registered = true
|
||||
t.mu.Unlock()
|
||||
|
||||
helper.Go(t.log, "transcoder.monitorIdle", func() { t.monitorIdle(jobCtx, job) })
|
||||
@@ -310,7 +343,7 @@ func (t *TranscoderService) EnsureJobFromSubtitleQuality(ctx context.Context, me
|
||||
defer close(job.done)
|
||||
t.runFFmpeg(jobCtx, job, input)
|
||||
})
|
||||
return t.PlaylistPath(mediaID), nil
|
||||
return filepath.Join(outDir, "index.m3u8"), nil
|
||||
}
|
||||
|
||||
// shouldReplaceHLSJob reports whether an incoming playlist request may cancel
|
||||
@@ -357,9 +390,33 @@ func sameHLSConfiguration(existing *hlsJob, startSec float64, subtitleStream int
|
||||
sameHLSStart(existing.startSec, startSec)
|
||||
}
|
||||
|
||||
func (t *TranscoderService) mediaStartGate(mediaID string) *sync.Mutex {
|
||||
v, _ := t.startGates.LoadOrStore(mediaID, &sync.Mutex{})
|
||||
return v.(*sync.Mutex)
|
||||
type hlsStartGate struct {
|
||||
mu sync.Mutex
|
||||
users int
|
||||
}
|
||||
|
||||
func (t *TranscoderService) lockJobStart(key string) func() {
|
||||
t.startGateMu.Lock()
|
||||
if t.startGates == nil {
|
||||
t.startGates = make(map[string]*hlsStartGate)
|
||||
}
|
||||
gate := t.startGates[key]
|
||||
if gate == nil {
|
||||
gate = &hlsStartGate{}
|
||||
t.startGates[key] = gate
|
||||
}
|
||||
gate.users++
|
||||
t.startGateMu.Unlock()
|
||||
gate.mu.Lock()
|
||||
return func() {
|
||||
gate.mu.Unlock()
|
||||
t.startGateMu.Lock()
|
||||
gate.users--
|
||||
if gate.users == 0 {
|
||||
delete(t.startGates, key)
|
||||
}
|
||||
t.startGateMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// detachJobLocked cancels and removes a job from the map without waiting.
|
||||
|
||||
@@ -16,6 +16,9 @@ type transcodeInput struct {
|
||||
StartSec float64
|
||||
SubtitleStream *int
|
||||
Quality *LocalHLSQuality
|
||||
// CopyVideo 为 true 时视频流只换封装(-c:v copy),不重编码。
|
||||
// 仅当源视频是浏览器可解的老三样(H.264/8bit/4:2:0)且不需要缩放时启用。
|
||||
CopyVideo bool
|
||||
}
|
||||
|
||||
type ffmpegArgSettings struct {
|
||||
@@ -52,6 +55,7 @@ func buildFFmpegArgsForInput(cfg *config.Config, input transcodeInput, playlist,
|
||||
// 原画档位不做硬件缩放的兼容处理,直接走软件编码,保持源分辨率。
|
||||
settings.encoder = ""
|
||||
}
|
||||
copyVideo := input.CopyVideo && input.SubtitleStream == nil
|
||||
video := ffmpegVideoPlanForSettings(settings)
|
||||
if input.SubtitleStream != nil {
|
||||
// Bitmap subtitles must be composited in software. Keeping CUDA/QSV/
|
||||
@@ -60,13 +64,17 @@ func buildFFmpegArgsForInput(cfg *config.Config, input transcodeInput, playlist,
|
||||
settings.encoder = ""
|
||||
video = ffmpegVideoPlanForSettings(settings)
|
||||
}
|
||||
if copyVideo {
|
||||
// 直通:不做缩放、不挂滤镜、不选编码器,整条视频流原样塞进 TS。
|
||||
video = ffmpegVideoPlan{codec: "copy"}
|
||||
}
|
||||
|
||||
// Mid-file restarts must not use -re: output -ss would otherwise crawl to the
|
||||
// seek point at 1x wall-clock before emitting the first HLS segment.
|
||||
realtime := settings.realtime && input.StartSec <= 0.05
|
||||
args := baseFFmpegArgs(video.preInput, realtime)
|
||||
args = appendInputAndVideoArgs(args, input, settings, video)
|
||||
args = appendOutputHLSArgs(args, settings, segments, playlist)
|
||||
args = appendInputAndVideoArgs(args, input, settings, video, copyVideo)
|
||||
args = appendOutputHLSArgs(args, settings, segments, playlist, copyVideo)
|
||||
return args
|
||||
}
|
||||
|
||||
@@ -161,7 +169,7 @@ func baseFFmpegArgs(preInput string, realtime bool) []string {
|
||||
return args
|
||||
}
|
||||
|
||||
func appendInputAndVideoArgs(args []string, input transcodeInput, settings ffmpegArgSettings, video ffmpegVideoPlan) []string {
|
||||
func appendInputAndVideoArgs(args []string, input transcodeInput, settings ffmpegArgSettings, video ffmpegVideoPlan, copyVideo bool) []string {
|
||||
args = append(args, ffmpegHTTPInputArgs(input)...)
|
||||
ss := ""
|
||||
if input.StartSec > 0.05 {
|
||||
@@ -172,6 +180,11 @@ func appendInputAndVideoArgs(args []string, input transcodeInput, settings ffmpe
|
||||
// scrubbing; CDNs with Range support jump via demuxer seek instead.
|
||||
if ss != "" {
|
||||
args = append(args, "-ss", ss)
|
||||
if copyVideo {
|
||||
// 复制视频流时 ffmpeg 不能解码,精确 seek 会把关键帧之前的分组丢掉,
|
||||
// 输出从 P 帧开始(花屏)。退到前一个关键帧,最多提前一两秒。
|
||||
args = append(args, "-noaccurate_seek")
|
||||
}
|
||||
}
|
||||
args = append(args, "-i", input.Source)
|
||||
if input.SubtitleStream != nil {
|
||||
@@ -197,7 +210,23 @@ func appendInputAndVideoArgs(args []string, input transcodeInput, settings ffmpe
|
||||
return args
|
||||
}
|
||||
|
||||
func appendOutputHLSArgs(args []string, settings ffmpegArgSettings, segments, playlist string) []string {
|
||||
func appendOutputHLSArgs(args []string, settings ffmpegArgSettings, segments, playlist string, copyVideo bool) []string {
|
||||
if copyVideo {
|
||||
// 视频是复制进来的,编码参数一律不适用;只保留音频转 AAC 与切片参数。
|
||||
// -avoid_negative_ts 让带 B 帧的源在 -ss 之后不会出现负时间戳。
|
||||
return append(args,
|
||||
"-avoid_negative_ts", "make_zero",
|
||||
"-c:a", "aac",
|
||||
"-ar", "48000",
|
||||
"-b:a", "128k",
|
||||
"-ac", "2",
|
||||
"-f", "hls",
|
||||
"-hls_time", fmt.Sprintf("%d", settings.segmentSeconds),
|
||||
"-hls_list_size", "0",
|
||||
"-hls_segment_filename", segments,
|
||||
playlist,
|
||||
)
|
||||
}
|
||||
return append(args,
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-b:v", settings.bitrate,
|
||||
|
||||
@@ -49,16 +49,18 @@ func (t *TranscoderService) runFFmpeg(ctx context.Context, job *hlsJob, input tr
|
||||
|
||||
t.mu.Lock()
|
||||
// Only update the map entry if we are still the registered generation.
|
||||
if cur, ok := t.jobs[job.mediaID]; ok && cur == job {
|
||||
if cur, ok := t.jobs[job.key()]; ok && cur == job {
|
||||
if err == nil && ctx.Err() == nil {
|
||||
job.completed = true
|
||||
} else {
|
||||
delete(t.jobs, job.mediaID)
|
||||
delete(t.jobs, job.key())
|
||||
}
|
||||
}
|
||||
t.mu.Unlock()
|
||||
if err != nil || ctx.Err() != nil {
|
||||
job.cancel()
|
||||
_ = os.RemoveAll(job.outputDir)
|
||||
_ = os.Remove(filepath.Dir(job.outputDir)) // Remove the scope root only if empty.
|
||||
}
|
||||
|
||||
t.hub.Publish("transcode", map[string]any{
|
||||
|
||||
@@ -26,16 +26,18 @@ func (t *TranscoderService) WaitReady(ctx context.Context, mediaID string, timeo
|
||||
job, ok := t.jobs[mediaID]
|
||||
var started time.Time
|
||||
var completed bool
|
||||
var playlist string
|
||||
if ok {
|
||||
started = job.startedAt
|
||||
completed = job.completed
|
||||
playlist = t.jobPlaylistPath(job)
|
||||
}
|
||||
t.mu.Unlock()
|
||||
if !ok {
|
||||
return false // A failed/exited job cannot become ready later.
|
||||
}
|
||||
if ok {
|
||||
if info, err := os.Stat(t.PlaylistPath(mediaID)); err == nil {
|
||||
if info, err := os.Stat(playlist); err == nil {
|
||||
// Allow a small clock skew; reject anything older than this job.
|
||||
if !info.ModTime().Before(started.Add(-2 * time.Second)) {
|
||||
t.mu.Lock()
|
||||
@@ -66,9 +68,8 @@ func (t *TranscoderService) WaitReady(ctx context.Context, mediaID string, timeo
|
||||
// briefly for it to exit so a subsequent EnsureJobFrom cannot race-write the
|
||||
// same HLS directory.
|
||||
func (t *TranscoderService) StopJob(mediaID string) {
|
||||
gate := t.mediaStartGate(mediaID)
|
||||
gate.Lock()
|
||||
defer gate.Unlock()
|
||||
unlock := t.lockJobStart(mediaID)
|
||||
defer unlock()
|
||||
|
||||
t.mu.Lock()
|
||||
prev := t.detachJobLocked(mediaID)
|
||||
@@ -170,7 +171,7 @@ func (t *TranscoderService) monitorIdle(ctx context.Context, job *hlsJob) {
|
||||
return
|
||||
case <-ticker.C:
|
||||
t.mu.Lock()
|
||||
current, ok := t.jobs[job.mediaID]
|
||||
current, ok := t.jobs[job.key()]
|
||||
if !ok || current != job {
|
||||
t.mu.Unlock()
|
||||
return
|
||||
@@ -185,11 +186,10 @@ func (t *TranscoderService) monitorIdle(ctx context.Context, job *hlsJob) {
|
||||
}
|
||||
|
||||
func (t *TranscoderService) stopIdleJob(job *hlsJob, timeout time.Duration) bool {
|
||||
gate := t.mediaStartGate(job.mediaID)
|
||||
gate.Lock()
|
||||
defer gate.Unlock()
|
||||
unlock := t.lockJobStart(job.key())
|
||||
defer unlock()
|
||||
t.mu.Lock()
|
||||
current := t.jobs[job.mediaID]
|
||||
current := t.jobs[job.key()]
|
||||
if current != job {
|
||||
t.mu.Unlock()
|
||||
return true
|
||||
@@ -199,9 +199,12 @@ func (t *TranscoderService) stopIdleJob(job *hlsJob, timeout time.Duration) bool
|
||||
t.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
t.detachJobLocked(job.mediaID)
|
||||
t.detachJobLocked(job.key())
|
||||
t.mu.Unlock()
|
||||
waitJobExit(job, 12*time.Second)
|
||||
if job.jobKey != "" {
|
||||
_ = os.RemoveAll(t.HLSDir(job.key()))
|
||||
}
|
||||
t.log.Info("transcode idle timeout", zap.String("media_id", job.mediaID),
|
||||
zap.Duration("idle_for", idleFor), zap.Duration("timeout", timeout))
|
||||
return true
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// 蓝光原盘(m2ts,40 Mbps 级别)在两核小机器上重编码只有 0.1~0.2 倍速,HLS
|
||||
// 永远追不上播放,播放器等不到切片就报「转码失败」。当源视频本身就是浏览器
|
||||
// 能解的 H.264 / 8bit / 4:2:0,且用户选的是「原画」档位(不改分辨率)时,
|
||||
// 这里改走只换封装(-c:v copy)的直通路径:视频不重编码,音频转 AAC,
|
||||
// CPU 占用接近于零,起播时间从几十秒降到几秒。
|
||||
//
|
||||
// 直通只对「原画」生效。1080P/720P/480P 档位意味着要改分辨率,必须重新编码;
|
||||
// 那类请求仍然按原来的 libx264 路径处理(慢,但码率可控)。
|
||||
|
||||
// remuxEncoderLabel 是直通任务在日志与任务面板里的标签。
|
||||
const remuxEncoderLabel = "remux"
|
||||
|
||||
// remuxEligible 判断这次请求能否复制视频流而不是重编码。
|
||||
func (t *TranscoderService) remuxEligible(ctx context.Context, m *model.Media, quality *LocalHLSQuality, subtitleStream int) bool {
|
||||
if t == nil || t.repo == nil || m == nil {
|
||||
return false
|
||||
}
|
||||
// 烧录图形字幕要把字幕叠加进画面,必须解码+重编码。
|
||||
if subtitleStream >= 0 {
|
||||
return false
|
||||
}
|
||||
if !remuxAppliesToQuality(quality) {
|
||||
return false
|
||||
}
|
||||
probe, err := t.repo.MediaProbe.Get(ctx, m.ID)
|
||||
if err != nil || probe == nil {
|
||||
// 没有探测缓存就无法确认档次/像素格式,保守走重编码。
|
||||
return false
|
||||
}
|
||||
return remuxCopyableHeight(probe) > 0
|
||||
}
|
||||
|
||||
// remuxAppliesToQuality 只放行「原画」档位(Height 为 0,即不缩放)。
|
||||
// quality 为 nil 表示调用方没有指定档位(走全局 max_height),同样不直通,
|
||||
// 否则会绕过配置的分辨率上限给出一个远超预期的码流。
|
||||
func remuxAppliesToQuality(quality *LocalHLSQuality) bool {
|
||||
return quality != nil && quality.Height <= 0
|
||||
}
|
||||
|
||||
// remuxCopyableHeight 在源视频可直接复制进 HLS 时返回它的高度,否则返回 0。
|
||||
func remuxCopyableHeight(probe *model.MediaProbe) int {
|
||||
if probe == nil || probe.Height <= 0 {
|
||||
return 0
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(probe.VideoCodec), "h264") {
|
||||
return 0
|
||||
}
|
||||
if !browserSafeH264Payload(probe.Payload) {
|
||||
return 0
|
||||
}
|
||||
return probe.Height
|
||||
}
|
||||
|
||||
// probePayloadStreams 只解析落库 payload 里与直通判断有关的字段。
|
||||
type probePayloadStreams struct {
|
||||
Streams []struct {
|
||||
Type string `json:"type"`
|
||||
Profile string `json:"profile"`
|
||||
PixFmt string `json:"pix_fmt"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
|
||||
// browserSafeH264Payload 只放行 8bit 4:2:0 的 Baseline/Main/High。
|
||||
// Hi10P(High 10)和 4:2:2 母带浏览器解不出来,复制过去只会黑屏或无声。
|
||||
func browserSafeH264Payload(payload string) bool {
|
||||
payload = strings.TrimSpace(payload)
|
||||
if payload == "" {
|
||||
// 老数据没有 payload:已经有 codec + 分辨率,按 8bit 常规情况处理。
|
||||
return true
|
||||
}
|
||||
var parsed probePayloadStreams
|
||||
if err := json.Unmarshal([]byte(payload), &parsed); err != nil {
|
||||
return false
|
||||
}
|
||||
for _, stream := range parsed.Streams {
|
||||
if !strings.EqualFold(strings.TrimSpace(stream.Type), "video") {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(stream.Profile)) {
|
||||
case "", "constrained baseline", "baseline", "main", "high":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(stream.PixFmt)) {
|
||||
case "", "yuv420p", "yuvj420p":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// 直通档位必须真的把视频流复制过去:一旦又带上 libx264 / -vf / 码率参数,
|
||||
// 慢机器上的「原画」就还是几十秒起播。
|
||||
func TestBuildFFmpegArgsRemuxCopiesVideoStream(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
cfg.Transcoder.SegmentSeconds = 4
|
||||
cfg.Transcoder.Realtime = true
|
||||
cfg.Transcoder.Threads = 2
|
||||
cfg.Transcoder.MaxHeight = 720
|
||||
|
||||
quality, ok := LocalHLSQualityByID("source")
|
||||
if !ok {
|
||||
t.Fatal("source quality missing")
|
||||
}
|
||||
args := buildFFmpegArgsForInput(cfg, transcodeInput{
|
||||
Source: "https://cdn.example/x.m2ts",
|
||||
Headers: map[string]string{"User-Agent": "MeBox-115-GoClient/1.0"},
|
||||
StartSec: 52.4,
|
||||
Quality: &quality,
|
||||
CopyVideo: true,
|
||||
}, "/cache/hls/k/run-1/index.m3u8", "/cache/hls/k/run-1/seg_%05d.ts")
|
||||
joined := strings.Join(args, " ")
|
||||
|
||||
for _, want := range []string{
|
||||
"-c:v copy",
|
||||
"-c:a aac",
|
||||
"-ar 48000",
|
||||
"-b:a 128k",
|
||||
"-ac 2",
|
||||
"-avoid_negative_ts make_zero",
|
||||
"-ss 52.400",
|
||||
"-noaccurate_seek",
|
||||
"-hls_time 4",
|
||||
"-hls_segment_filename /cache/hls/k/run-1/seg_%05d.ts",
|
||||
} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("remux args missing %q: %s", want, joined)
|
||||
}
|
||||
}
|
||||
for _, unwanted := range []string{
|
||||
"libx264",
|
||||
"-vf ",
|
||||
"-b:v",
|
||||
"-maxrate",
|
||||
"-bufsize",
|
||||
"-force_key_frames",
|
||||
"-preset",
|
||||
"-threads",
|
||||
"-pix_fmt",
|
||||
} {
|
||||
if strings.Contains(joined, unwanted) {
|
||||
t.Errorf("remux args must not contain %q: %s", unwanted, joined)
|
||||
}
|
||||
}
|
||||
// 中途起播不能加 -re,否则会先按 1 倍速爬到目标位置才出第一个切片。
|
||||
if strings.Contains(joined, " -re ") {
|
||||
t.Errorf("mid-file remux must not use -re: %s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// 烧录图形字幕要把字幕叠进画面,必须解码重编码,不能被直通标志绕过。
|
||||
func TestBuildFFmpegArgsRemuxIgnoredForSubtitleBurnIn(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
cfg.Transcoder.MaxHeight = 720
|
||||
cfg.Transcoder.SegmentSeconds = 4
|
||||
|
||||
quality, _ := LocalHLSQualityByID("source")
|
||||
subtitle := 4
|
||||
args := buildFFmpegArgsForInput(cfg, transcodeInput{
|
||||
Source: "/media/ep.mkv",
|
||||
Quality: &quality,
|
||||
CopyVideo: true,
|
||||
SubtitleStream: &subtitle,
|
||||
}, "/o/x.m3u8", "/o/seg_%05d.ts")
|
||||
joined := strings.Join(args, " ")
|
||||
|
||||
if strings.Contains(joined, "-c:v copy") {
|
||||
t.Fatalf("subtitle burn-in must not copy the video stream: %s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "libx264") || !strings.Contains(joined, "-filter_complex") {
|
||||
t.Fatalf("subtitle burn-in should re-encode: %s", joined)
|
||||
}
|
||||
if strings.Contains(joined, "-noaccurate_seek") {
|
||||
t.Fatalf("encoded path should keep accurate seek: %s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// 只有「原画」档位(不缩放)可以直通;其它档位要改分辨率。
|
||||
func TestRemuxAppliesToQuality(t *testing.T) {
|
||||
if remuxAppliesToQuality(nil) {
|
||||
t.Fatal("nil quality must not remux: it falls back to the configured max height")
|
||||
}
|
||||
source, _ := LocalHLSQualityByID("source")
|
||||
if !remuxAppliesToQuality(&source) {
|
||||
t.Fatal("source quality should remux")
|
||||
}
|
||||
for _, id := range []string{"1080", "720", "480"} {
|
||||
quality, ok := LocalHLSQualityByID(id)
|
||||
if !ok {
|
||||
t.Fatalf("quality %s missing", id)
|
||||
}
|
||||
if remuxAppliesToQuality(&quality) {
|
||||
t.Fatalf("quality %s must re-encode", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 只有 8bit 4:2:0 的 H.264 能安全复制;Hi10P / 4:2:2 / HEVC 复制过去浏览器黑屏。
|
||||
func TestRemuxCopyableHeight(t *testing.T) {
|
||||
const blurayPayload = `{"container":"mpegts","duration_sec":1451,"bit_rate":42609934,"streams":[{"index":0,"type":"video","codec":"h264","profile":"High","width":1920,"height":1080,"pix_fmt":"yuv420p","frame_rate":"23.976"},{"index":1,"type":"audio","codec":"pcm_bluray","channels":2}]}`
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
probe *model.MediaProbe
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "bluray h264 8bit 420",
|
||||
probe: &model.MediaProbe{VideoCodec: "h264", Height: 1080, Payload: blurayPayload},
|
||||
want: 1080,
|
||||
},
|
||||
{
|
||||
name: "no payload falls back to codec and height",
|
||||
probe: &model.MediaProbe{VideoCodec: "h264", Height: 720},
|
||||
want: 720,
|
||||
},
|
||||
{
|
||||
name: "hevc cannot be copied into browser hls",
|
||||
probe: &model.MediaProbe{VideoCodec: "hevc", Height: 1080, Payload: `{"streams":[{"type":"video","profile":"Main 10","pix_fmt":"yuv420p10le"}]}`},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "hi10p h264 is unplayable in browsers",
|
||||
probe: &model.MediaProbe{VideoCodec: "h264", Height: 1080, Payload: `{"streams":[{"type":"video","profile":"High 10","pix_fmt":"yuv420p10le"}]}`},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "4:2:2 h264 is unplayable in browsers",
|
||||
probe: &model.MediaProbe{VideoCodec: "h264", Height: 1080, Payload: `{"streams":[{"type":"video","profile":"High 4:2:2","pix_fmt":"yuv422p"}]}`},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "missing probe",
|
||||
probe: nil,
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "unknown height",
|
||||
probe: &model.MediaProbe{VideoCodec: "h264", Height: 0, Payload: blurayPayload},
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := remuxCopyableHeight(tc.probe); got != tc.want {
|
||||
t.Fatalf("remuxCopyableHeight() = %d, want %d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { api, LONG_REQUEST_TIMEOUT } from './client'
|
||||
|
||||
export interface BackupInfo {
|
||||
filename: string
|
||||
size: number
|
||||
created_at: string
|
||||
automatic: boolean
|
||||
}
|
||||
|
||||
export const backupAPI = {
|
||||
list: () => api.get<{ items: BackupInfo[] }>('/admin/backups').then((r) => r.data.items ?? []),
|
||||
create: () => api.post<BackupInfo>('/admin/backups', {}, { timeout: LONG_REQUEST_TIMEOUT }).then((r) => r.data),
|
||||
remove: (filename: string) => api.delete('/admin/backups', { params: { filename } }),
|
||||
restore: (filename: string) => api.post<{ message: string; restart_required: boolean }>('/admin/backups/restore', null, { params: { filename }, timeout: LONG_REQUEST_TIMEOUT }).then((r) => r.data),
|
||||
download: async (filename: string) => {
|
||||
const response = await api.get<Blob>('/admin/backups/download', {
|
||||
params: { filename },
|
||||
responseType: 'blob',
|
||||
timeout: LONG_REQUEST_TIMEOUT,
|
||||
})
|
||||
const url = URL.createObjectURL(response.data)
|
||||
const anchor = document.createElement('a')
|
||||
anchor.href = url
|
||||
anchor.download = filename
|
||||
anchor.click()
|
||||
URL.revokeObjectURL(url)
|
||||
},
|
||||
}
|
||||
@@ -175,7 +175,7 @@ export function streamURL(mediaId: string, options: { proxy?: boolean } = {}): s
|
||||
|
||||
// hlsURL returns the m3u8 playlist URL fed into hls.js.
|
||||
// startSec > 0 asks the server to (re)start ffmpeg from that source offset.
|
||||
export function hlsURL(mediaId: string, startSec = 0, subtitleStream?: number, quality?: string): string {
|
||||
export function hlsURL(mediaId: string, startSec = 0, subtitleStream?: number, quality?: string, sessionID?: string): string {
|
||||
const safeStart = Math.max(0, Math.round(startSec * 1000) / 1000)
|
||||
// Always send start= (including 0) so the server can tell an intentional
|
||||
// restart-from-head apart from a missing query on a stale refresh.
|
||||
@@ -188,7 +188,8 @@ export function hlsURL(mediaId: string, startSec = 0, subtitleStream?: number, q
|
||||
? `&subtitle=${encodeURIComponent(String(subtitleStream))}`
|
||||
: ''
|
||||
const qualityQuery = quality ? `&quality=${encodeURIComponent(quality)}` : ''
|
||||
return `/api/hls/${encodeURIComponent(mediaId)}/index.m3u8?${tokenQuery()}${profileQuery()}${start}${subtitle}${qualityQuery}${bust}`
|
||||
const session = sessionID ? `&session_id=${encodeURIComponent(sessionID)}` : ''
|
||||
return `/api/hls/${encodeURIComponent(mediaId)}/index.m3u8?${tokenQuery()}${profileQuery()}${start}${subtitle}${qualityQuery}${bust}${session}`
|
||||
}
|
||||
|
||||
// cloudHlsURL returns the MeBox-proxied 115 cloud HLS master playlist URL.
|
||||
@@ -202,8 +203,9 @@ export function cloudHlsURL(mediaId: string, definition: string): string {
|
||||
|
||||
// Stop an on-demand HLS job. keepalive makes the request survive page
|
||||
// navigation/tab close, where an axios promise can be discarded by browsers.
|
||||
export function stopHLSJob(mediaId: string): void {
|
||||
const url = `/api/hls/${encodeURIComponent(mediaId)}?${tokenQuery()}${profileQuery()}`
|
||||
export function stopHLSJob(mediaId: string, sessionID?: string): void {
|
||||
const session = sessionID ? `&session_id=${encodeURIComponent(sessionID)}` : ''
|
||||
const url = `/api/hls/${encodeURIComponent(mediaId)}?${tokenQuery()}${profileQuery()}${session}`
|
||||
void fetch(url, {
|
||||
method: 'DELETE',
|
||||
credentials: 'same-origin',
|
||||
|
||||
+14
-1
@@ -262,7 +262,15 @@ export const mediaAPI = {
|
||||
})
|
||||
.then((r) => r.data),
|
||||
|
||||
searchPage: (q: string, page = 1, pageSize = 50, options?: { groupVersions?: boolean; groupSeries?: boolean }) =>
|
||||
searchPage: (q: string, page = 1, pageSize = 50, options?: {
|
||||
groupVersions?: boolean
|
||||
groupSeries?: boolean
|
||||
libraryId?: string
|
||||
yearMin?: number
|
||||
yearMax?: number
|
||||
genres?: string[]
|
||||
sort?: 'title_asc' | 'year_desc' | 'rating_desc'
|
||||
}) =>
|
||||
api
|
||||
.get<MediaSearchPage>('/media', {
|
||||
params: {
|
||||
@@ -271,6 +279,11 @@ export const mediaAPI = {
|
||||
page_size: pageSize,
|
||||
group_versions: options?.groupVersions === false ? 0 : undefined,
|
||||
...(options?.groupSeries ? { group_series: 1 } : {}),
|
||||
...(options?.libraryId ? { library_id: options.libraryId } : {}),
|
||||
...(options?.yearMin ? { year_min: options.yearMin } : {}),
|
||||
...(options?.yearMax ? { year_max: options.yearMax } : {}),
|
||||
...(options?.genres?.length ? { genre: options.genres } : {}),
|
||||
...(options?.sort ? { sort: options.sort } : {}),
|
||||
},
|
||||
timeout: LONG_REQUEST_TIMEOUT,
|
||||
})
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { api } from './client'
|
||||
import type { Media } from '../types'
|
||||
|
||||
export type MediaQualityIssue = 'missing' | 'unmatched' | 'poster'
|
||||
|
||||
export interface MediaQualityRow {
|
||||
issue: MediaQualityIssue
|
||||
media: Media
|
||||
}
|
||||
|
||||
export const qualityAPI = {
|
||||
list: (issue: MediaQualityIssue, page = 1, pageSize = 50) =>
|
||||
api.get<{ items: MediaQualityRow[]; total: number; page: number; page_size: number }>('/admin/quality', {
|
||||
params: { issue, page, page_size: pageSize },
|
||||
}).then((r) => r.data),
|
||||
}
|
||||
@@ -249,6 +249,8 @@ export const readerAPI = {
|
||||
deleteSource: (id: string) => api.delete(`/reader/sources/${id}`),
|
||||
debugSource: (id: string, key: string) =>
|
||||
api.post<{ logs: string[] }>(`/reader/sources/${id}/debug`, { key }, longOpts).then((r) => r.data.logs),
|
||||
smokeSource: (id: string, key: string) =>
|
||||
api.post<{ logs: string[] }>(`/reader/sources/${id}/debug`, { key }, { timeout: 60_000 }).then((r) => r.data.logs),
|
||||
|
||||
// ── 书源登录与源变量 ──
|
||||
// 取登录界面描述(loginUi 控件 + 已保存值 + 当前 Cookie 状态)
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { api } from './client'
|
||||
|
||||
export interface BackgroundTask {
|
||||
id: string
|
||||
kind: string
|
||||
name: string
|
||||
status: 'running' | 'completed' | 'failed' | 'interrupted' | string
|
||||
stage?: string
|
||||
source_path?: string
|
||||
dest_path?: string
|
||||
message?: string
|
||||
error?: string
|
||||
details?: string[]
|
||||
metrics?: Record<string, number>
|
||||
started_at: string
|
||||
updated_at: string
|
||||
finished_at?: string
|
||||
}
|
||||
|
||||
export interface TaskSnapshot {
|
||||
active: BackgroundTask[]
|
||||
recent: BackgroundTask[]
|
||||
}
|
||||
|
||||
export const tasksAPI = {
|
||||
snapshot: () => api.get<TaskSnapshot>('/admin/tasks').then((r) => r.data),
|
||||
}
|
||||
@@ -50,6 +50,8 @@ const StrmUploadQueuePage = lazy(() =>
|
||||
)
|
||||
const ScraperQueuePage = lazy(() => import('./pages/ScraperQueuePage').then((m) => ({ default: m.ScraperQueuePage })))
|
||||
const TaskQueuePage = lazy(() => import('./pages/TaskQueuePage').then((m) => ({ default: m.TaskQueuePage })))
|
||||
const MediaQualityPage = lazy(() => import('./pages/MediaQualityPage').then((m) => ({ default: m.MediaQualityPage })))
|
||||
const SetupWizardPage = lazy(() => import('./pages/SetupWizardPage').then((m) => ({ default: m.SetupWizardPage })))
|
||||
|
||||
// 常用页面的路由 chunk 空闲预取:应用加载完成后浏览器一空闲就把浏览路径
|
||||
// (库列表/库详情/媒体详情/收藏/历史等)的 chunk 拉下来,
|
||||
@@ -125,5 +127,7 @@ export const appRoutes: AppRoute[] = [
|
||||
{ path: 'strm/uploads', element: <StrmUploadQueuePage />, adminOnly: true },
|
||||
{ path: 'scraper/queue', element: <ScraperQueuePage />, adminOnly: true },
|
||||
{ path: 'queue', element: <TaskQueuePage />, adminOnly: true },
|
||||
{ path: 'quality', element: <MediaQualityPage />, adminOnly: true },
|
||||
{ path: 'setup', element: <SetupWizardPage />, adminOnly: true },
|
||||
{ path: 'admin', element: <AdminPage />, adminOnly: true },
|
||||
]
|
||||
|
||||
@@ -89,6 +89,11 @@ export const MediaCard = memo(function MediaCard({
|
||||
<span>{displayRating.toFixed(1)}</span>
|
||||
</span>
|
||||
)}
|
||||
{media.availability_state === 'missing' && (
|
||||
<span className="absolute left-2 top-2 rounded-lg border border-amber-300/30 bg-[#111827]/90 px-2 py-1 text-[10px] font-bold text-amber-300 shadow-sm">
|
||||
文件失联
|
||||
</span>
|
||||
)}
|
||||
|
||||
{/* Premium Hover Overlay(CSS 过渡替代 framer-motion 逐卡动画实例) */}
|
||||
<div className={`absolute inset-0 bg-gradient-to-t from-[#111827]/90 via-[#111827]/30 to-transparent opacity-0 group-hover:opacity-100 transition-opacity duration-300 flex flex-col justify-end ${
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
import { useEffect, type ReactNode } from 'react'
|
||||
import { Fragment, useEffect, type ReactNode } from 'react'
|
||||
import { Navigate, useLocation } from 'react-router-dom'
|
||||
|
||||
import { useAuthStore } from '../stores/auth'
|
||||
import { usePlayProfileStore } from '../stores/playProfile'
|
||||
import { libraryCacheScope } from '../utils/libraryCache'
|
||||
import { hydrateReaderSettings, startReaderSettingsSync } from '../utils/readerSettingsSync'
|
||||
|
||||
// Route guard: redirects to /login when no token is present.
|
||||
export function RequireAuth({ children }: { children: ReactNode }) {
|
||||
const token = useAuthStore((s) => s.token)
|
||||
const userId = useAuthStore((s) => s.user?.id ?? null)
|
||||
// Drop local page state and in-flight callbacks when visibility changes.
|
||||
useAuthStore((s) => s.user)
|
||||
usePlayProfileStore((s) => s.activeProfileId)
|
||||
usePlayProfileStore((s) => s.activeProfilePinToken)
|
||||
const visibilityKey = libraryCacheScope()
|
||||
const location = useLocation()
|
||||
|
||||
// 阅读器偏好按账号存在服务端:一进入受保护区域就开启回写订阅,并在
|
||||
@@ -23,7 +30,7 @@ export function RequireAuth({ children }: { children: ReactNode }) {
|
||||
if (!token) {
|
||||
return <Navigate to="/login" replace state={{ from: location }} />
|
||||
}
|
||||
return <>{children}</>
|
||||
return <Fragment key={visibilityKey}>{children}</Fragment>
|
||||
}
|
||||
|
||||
// Route guard: only allows users with role === "admin".
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user