mirror of
https://github.com/truewhile/MeBox.git
synced 2026-11-13 03:44:23 +08:00
Compare commits
29 Commits
mebox-v0.2.4
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 08d521b65d | |||
| 5806839601 | |||
| d7c150ffd9 | |||
| 79943d9067 | |||
| 9ffac320c4 | |||
| 9b8ca33279 | |||
| 34e61a5994 | |||
| d91ded9d55 | |||
| 8458b8060a | |||
| 623d87f12b | |||
| d820214abc | |||
| 0ab4693b92 | |||
| db516ded61 | |||
| 416d4c8c3b | |||
| 78cfe0aba2 | |||
| 60ed34d5b1 | |||
| 31e775b9ba | |||
| bf35bd3d02 | |||
| 0a84031d06 | |||
| 9ccf60427a | |||
| 54b0e33968 | |||
| 74487359bf | |||
| 55854a0adb | |||
| 121a0a3cc5 | |||
| 3057664a48 | |||
| 7793d50906 | |||
| 5ad71b9421 | |||
| 5615806c2c | |||
| 1e8fe2396e |
@@ -79,6 +79,9 @@ func (a *application) start() error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := database.ApplyPendingSQLiteRestore(context.Background(), a.cfg, a.logger); err != nil {
|
||||
return fmt.Errorf("apply pending database restore: %w", err)
|
||||
}
|
||||
db, err := database.Open(a.cfg, a.logger)
|
||||
if err != nil {
|
||||
a.logger.Error("database open failed", zap.Error(err))
|
||||
@@ -110,6 +113,14 @@ func (a *application) start() error {
|
||||
a.logger.Info("polluted episode metadata cleanup completed", zap.Int("media_count", cleaned))
|
||||
}
|
||||
|
||||
// 一次性删除旧版解析误写的「整集片头/回顾」区间(end_ms <= start_ms),
|
||||
// 否则升级后播放器仍会一开播就弹跳过按钮。
|
||||
if cleaned, err := a.services.NormalizeImpossibleSegments(context.Background()); err != nil {
|
||||
a.logger.Warn("impossible skip segment cleanup failed", zap.Error(err))
|
||||
} else if cleaned > 0 {
|
||||
a.logger.Info("impossible skip segments removed", zap.Int("rows", cleaned))
|
||||
}
|
||||
|
||||
if err := a.services.Auth.SeedAdmin(context.Background()); err != nil {
|
||||
a.logger.Warn("seed admin failed", zap.Error(err))
|
||||
}
|
||||
|
||||
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,636 @@
|
||||
# 115 云下载 → STRM 同步 → 自动刮削 · 实施方案
|
||||
|
||||
| 项 | 内容 |
|
||||
| :--- | :--- |
|
||||
| 状态 | 设计已定稿,待实现(P0 未开工) |
|
||||
| 适用版本 | MeBox 当前主线(`internal/service/strm_*`、`internal/service/cloud115/*`) |
|
||||
| 关联文档 | `115doc/115开放平台/API列表/云下载/*`(仓库内离线文档) |
|
||||
| 最后核对 | 本文所有代码事实均按仓库当前代码逐条核对,行号仅供定位参考 |
|
||||
|
||||
---
|
||||
|
||||
## 1. 目标与范围
|
||||
|
||||
### 1.1 目标
|
||||
|
||||
在「115 媒体库映射」(即 STRM 管理中的一条 `StrmSyncPath`:115 目录 → 本地目录 .strm)之上提供云下载入口,用户提交磁力 / BT 种子 / ed2k / HTTP 链接后:
|
||||
|
||||
1. 任务提交到 115 离线(云下载);
|
||||
2. 后台轮询 115 任务状态;
|
||||
3. 任务完成后**自动触发**该同步目录的 STRM 增量同步;
|
||||
4. 同步完成后**自动扫描**对应媒体库,生成 Media 行;
|
||||
5. **自动刮削**新增媒体,无需用户再点一次。
|
||||
|
||||
全程无需离开 MeBox,链路上的每个阶段在任务列表里可见。
|
||||
|
||||
### 1.2 明确不做(本期范围外)
|
||||
|
||||
| 项 | 决定 | 原因 |
|
||||
| :--- | :--- | :--- |
|
||||
| 磁力链接选择部分文件下载 | **不做** | 开放平台 `/open/offline/add_task_urls` 无文件选择参数,且 `/open/offline/torrent` 只接受已上传种子的 `torrent_sha1` + `pick_code`,无法解析 magnet。见第 6 节 |
|
||||
| `.torrent` 勾选文件 UI | **不做** | 用户已确认本期不做选文件;种子按全选提交 |
|
||||
| 磁力转种子(DHT 抓 metadata) | **不做** | 需引入 torrent 依赖与常驻 DHT 网络行为,本期不引入新依赖 |
|
||||
| 非 115 网盘的离线下载 | **不做** | CD2 / OpenList / 本地目录无对应能力 |
|
||||
| 普通用户自助使用 | **不做** | 首期仅管理员(沿用 `AdminRequired` + `can_manage_strm` 语义),权限位见 9.3 |
|
||||
| 下载完成后自动整理/重命名(Organize) | **不做** | 与 STRM 映射模型冲突,`.strm` 必须与远端路径一一对应 |
|
||||
|
||||
### 1.3 非目标(但需保证不被破坏)
|
||||
|
||||
- 现有 STRM 手动同步 / Cron 同步 / 下载队列 / 上传队列行为不变;
|
||||
- 115 令牌自动刷新机制不得被第二个客户端实例破坏(见 3.3 关键约束 K1);
|
||||
- 刮削不重复刷已 `matched` 的媒体(沿用 `EnqueueLibrary` 的候选过滤)。
|
||||
|
||||
---
|
||||
|
||||
## 2. 现状核实
|
||||
|
||||
### 2.1 已具备
|
||||
|
||||
| 能力 | 位置 | 说明 |
|
||||
| :--- | :--- | :--- |
|
||||
| 115 OpenAPI 客户端 | `internal/service/cloud115/client.go`、`const.go:16` | `OpenClient`,`ProAPIBase = https://proapi.115.com`,`doJSON` 内建重试与令牌刷新 |
|
||||
| 115 授权(PKCE/扫码)与令牌落库 | `cloud115/oauth.go`、`service/strm_115_oauth.go` | `OnTokenRefreshed` → `persist115Tokens` 写回 `StrmAccount.Config` |
|
||||
| 115 文件上传 / 秒传 | `cloud115/upload.go:217` | `Upload(ctx, filePath, parentCID, signKey, signVal)` → `UploadCompleteResult{FileId, PickCode, Sha1}` |
|
||||
| 115 目录创建 / 查找 | `cloud115/upload.go:318`(`MkDir`)、`cloud115/open.go:126`(`FindNamedContentInParent`) | 可用于建「云下载/种子文件」暂存目录 |
|
||||
| 115 全局限流/熔断 | `cloud115/throttle_manager.go` | `GetGlobalThrottleManager()` |
|
||||
| STRM 同步(增量/全量) | `service/strm_sync.go:82` `StartSync` | 已运行时返回「该目录正在同步中」,不排队 |
|
||||
| STRM 目录/记录/队列数据层 | `model/strm.go`、`repository/strm_repository.go` | — |
|
||||
| 扫描 + 自动刮削入口 | `service/scanner_scan.go`、`scanner_post_scan.go:19` | `startAutoScrape` 受 `scrape.auto_on_scan` 全局开关约束 |
|
||||
| 刮削入队 | `service/scraper_queue.go:219` `EnqueueLibrary` | 仅挑 `scrape_status` 为空的 `pending` 行,天然幂等、不会重刷已匹配 |
|
||||
| 周期任务框架 | `service/scheduler.go:118` `scheduledJob` | 新增 job 只需在 `Start` 里 append |
|
||||
| 服务装配 | `service/service_builder.go:148` | `NewStrmService` 之后才有 `Cloud115`,可作为依赖注入范式 |
|
||||
|
||||
### 2.2 缺口(本方案要补的)
|
||||
|
||||
| 缺口 | 影响 |
|
||||
| :--- | :--- |
|
||||
| 全仓无任何离线/云下载代码(`grep -r "offline" internal/service/cloud115/` 无命中) | 需从零封装 4 个 API |
|
||||
| `finishSync` 结束后**没有任何回调**(`strm_sync.go:323`) | 「同步完成 → 扫描」链路是断的,必须新增 hook |
|
||||
| 无「本地路径 → 媒体库」反查 | 需新增 repository 查询 |
|
||||
| 无 `.torrent` multipart 上传端点(现有仅 reader 书源、DB 恢复用到 `FormFile`) | 需新增端点 + 上传大小限制 |
|
||||
| 无云下载任务表 | 需新增模型 + 仓储 + 迁移注册 |
|
||||
|
||||
---
|
||||
|
||||
## 3. 总体设计
|
||||
|
||||
### 3.1 链路时序
|
||||
|
||||
```
|
||||
[前端] 提交(多行链接 / .torrent 文件)+ 选择目标 STRM 同步目录
|
||||
│
|
||||
▼
|
||||
[CloudDownloadService.Submit]
|
||||
│ 1) 校验账号可用、同步目录启用
|
||||
│ 2) 配额预检 GET /open/offline/get_quota_info
|
||||
│ 3) .torrent: 上传到「云下载/种子文件」→ POST /open/offline/torrent 拿 info_hash
|
||||
│ 4) POST /open/offline/add_task_urls(批量 ≤115/批)或 /open/offline/add_task_bt
|
||||
▼
|
||||
[DB] CloudDownloadTask: status=submitted(逐条记录每个链接的受理结果)
|
||||
│
|
||||
▼
|
||||
[Scheduler: cloud_download_poll 每 60s]
|
||||
│ GET /open/offline/get_task_list?page=N 逐页
|
||||
│ 按 info_hash(优先)/ url(回退)匹配本地任务
|
||||
│ status 0/1 → downloading(percentDone) | 2 → succeeded | -1 → failed
|
||||
▼
|
||||
[Pipeline: succeeded 任务]
|
||||
│ A. 收集受影响 syncPathID(30s 去抖合并)
|
||||
│ B. StrmService.StartSync(syncPathID, incremental)
|
||||
▼
|
||||
[StrmService.finishSync] → SetSyncFinishHook(ok)
|
||||
│ C. 反查 LocalPath 对应的媒体库 → Scanner.ScanLibrary(libraryID)
|
||||
│ D. Scraper.EnqueueLibrary(libraryID, 默认 options)(仅 pending 行)
|
||||
▼
|
||||
[DB] 回写 Stage=done;前端列表展示各阶段结果
|
||||
```
|
||||
|
||||
### 3.2 模块职责
|
||||
|
||||
| 模块 | 职责 | 明确不负责 |
|
||||
| :--- | :--- | :--- |
|
||||
| `cloud115/offline.go` | 纯 API 封装:参数拼装、响应解析、错误码映射 | 不碰数据库、不做编排 |
|
||||
| `CloudDownloadService` | 输入解析、校验、提交、状态机、轮询、编排 | 不直接读写 .strm 文件 |
|
||||
| `StrmService` | 现有同步逻辑 + 新增完成回调 | 不感知云下载概念 |
|
||||
| `ScannerService` / `ScraperService` | 现有扫描/刮削逻辑 | 不感知云下载概念 |
|
||||
|
||||
### 3.3 关键约束(实现时必须遵守)
|
||||
|
||||
**K1 — 必须复用 StrmService 已缓存的 115 客户端,禁止自行 `NewOpenClient`。**
|
||||
`strm_service.go:524` 的 `newProvider` 里给 `OpenClient` 挂了 `OnTokenRefreshed` → `persist115Tokens`。115 的 `refresh_token` 是一次性轮转的:若云下载服务另建一个客户端,两边各自刷新会互相作废,最终把账号刷成「授权已失效」。
|
||||
|
||||
落地方式:给 `StrmService` 增加导出方法
|
||||
|
||||
```go
|
||||
// Open115Client 返回账号已缓存的 115 客户端(含令牌自动刷新与落库)。
|
||||
func (s *StrmService) Open115Client(ctx context.Context, accountID string) (*cloud115.OpenClient, error)
|
||||
```
|
||||
|
||||
内部复用 `providerFor` + `cloud.OpenAPI115Provider` 断言(与 `strm_sync.go:379` 的用法一致)。
|
||||
|
||||
**K2 — 去抖。** `walk115Flat`(`strm_sync.go:682`)是整树自适应拉取,不是只看新目录。多个任务在短时间内完成时,同一 syncPath 只应同步一次。
|
||||
|
||||
**K3 — 阶段失败隔离。** STRM 同步失败不得把 115 下载状态改成失败;反之亦然。下载状态与后处理阶段分别落在不同字段(见 4.1)。
|
||||
|
||||
**K4 — 同步不排队。** `StartSync` 在运行中直接报错,Pipeline 必须自己实现「等待 + 重投」,不能靠重试打满日志。
|
||||
|
||||
**K5 — 所有 115 调用走全局 throttle。** 轮询在无活跃任务时不得发起请求。
|
||||
|
||||
---
|
||||
|
||||
## 4. 数据模型
|
||||
|
||||
### 4.1 `model.CloudDownloadTask`(新表 `cloud_download_tasks`)
|
||||
|
||||
```go
|
||||
// internal/model/cloud_download.go
|
||||
type CloudDownloadTask struct {
|
||||
Base
|
||||
AccountID string `gorm:"size:36;index" json:"account_id"`
|
||||
SyncPathID string `gorm:"size:36;index" json:"sync_path_id"` // 绑定的 STRM 同步目录
|
||||
LibraryID string `gorm:"size:36;index" json:"library_id"` // 由 LocalPath 反查得到,可为空
|
||||
Provider string `gorm:"size:32" json:"provider"` // 固定 cloud115
|
||||
|
||||
SourceType string `gorm:"size:16;index" json:"source_type"` // magnet / torrent / ed2k / http / ftp
|
||||
SourceURL string `gorm:"type:text" json:"source_url"` // 原始链接(.torrent 时存文件名)
|
||||
SourceKey string `gorm:"size:512;index" json:"source_key"` // 归一化后的匹配键:大写 info_hash 或 trim 后的 url
|
||||
InfoHash string `gorm:"size:64;index" json:"info_hash"`
|
||||
TorrentName string `gorm:"size:512" json:"torrent_name,omitempty"`
|
||||
TorrentSha1 string `gorm:"size:64" json:"torrent_sha1,omitempty"` // .torrent 路径用
|
||||
PickCode string `gorm:"size:128" json:"pick_code,omitempty"` // .torrent 路径用
|
||||
|
||||
WpPathID string `gorm:"size:64" json:"wp_path_id"` // 115 目标目录 ID(= syncPath.RemotePath)
|
||||
SavePath string `gorm:"size:512" json:"save_path,omitempty"` // 相对 wp_path_id 的子路径(本期留空)
|
||||
|
||||
Status string `gorm:"size:16;index" json:"status"` // queued/submitted/downloading/succeeded/failed/canceled
|
||||
Percent int `json:"percent"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Error string `gorm:"size:1024" json:"error"`
|
||||
|
||||
Stage string `gorm:"size:16;index" json:"stage"` // pending/waiting_sync/syncing/scanning/scraping/done/na
|
||||
StageError string `gorm:"size:1024" json:"stage_error"`
|
||||
StrmRecordID string `gorm:"size:36" json:"strm_record_id,omitempty"`
|
||||
ScanAdded int `json:"scan_added"`
|
||||
ScrapeQueued int `json:"scrape_queued"`
|
||||
|
||||
RetryCount int `json:"retry_count"`
|
||||
LastPollAt *time.Time `json:"last_poll_at,omitempty"`
|
||||
NextPollAt *time.Time `gorm:"index" json:"next_poll_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
注册点:`model.AllModels()`(`internal/model/model.go:37`)、`repository.Container`(`internal/repository/repository.go:11`)、`service.Container`(`internal/service/service.go:19`)。
|
||||
|
||||
> **不复用 `StrmDownloadTask`**:那是「远端元数据 → 本地」的文件搬运队列(`model/strm.go`),语义完全不同,混用会让 `/admin/strm/downloads` 列表出现云下载任务。
|
||||
|
||||
### 4.2 状态机
|
||||
|
||||
**Status(115 下载侧)**
|
||||
|
||||
| 值 | 进入条件 | 终态 |
|
||||
| :--- | :--- | :--- |
|
||||
| `queued` | 本地已建行,尚未提交 115 | 否 |
|
||||
| `submitted` | 115 已受理(`data[].state=true`) | 否 |
|
||||
| `downloading` | `get_task_list` 返回 status 0 或 1 | 否 |
|
||||
| `succeeded` | status 2 | **是**,进入 Pipeline |
|
||||
| `failed` | status -1 / 提交被拒 / 超时(默认 72h) | **是** |
|
||||
| `canceled` | 用户取消 | **是** |
|
||||
|
||||
**Stage(后处理侧)**
|
||||
|
||||
| 值 | 含义 |
|
||||
| :--- | :--- |
|
||||
| `pending` | 尚未开始(未成功) |
|
||||
| `waiting_sync` | 已成功,等待去抖窗口结束或等待同一目录的同步结束 |
|
||||
| `syncing` | STRM 增量同步执行中 |
|
||||
| `scanning` | 媒体库扫描中 |
|
||||
| `scraping` | 刮削已入队 |
|
||||
| `done` | 全链路完成 |
|
||||
| `na` | 同步目录已删除/禁用,跳过后续(不算失败) |
|
||||
| `failed` | 阶段失败,`StageError` 记录原因 |
|
||||
|
||||
> Stage 失败**不回滚** Status;用户可对 Stage 失败的任务点「重做后处理」。
|
||||
|
||||
---
|
||||
|
||||
## 5. 115 开放平台对接
|
||||
|
||||
### 5.1 接口清单
|
||||
|
||||
| 用途 | 方法 | 路径 | 关键参数 | 关键返回 |
|
||||
| :--- | :--- | :--- | :--- | :--- |
|
||||
| 添加链接任务 | POST | `/open/offline/add_task_urls` | `urls`(换行分隔,≤115 条)、`wp_path_id` | `data[]{state,code,message,info_hash,url}` |
|
||||
| 解析 BT 种子 | POST | `/open/offline/torrent` | `torrent_sha1`、`pick_code` | `data{file_size,torrent_name,file_count,info_hash,torrent_filelist[]{size,path,wanted}}` |
|
||||
| 添加 BT 任务 | POST | `/open/offline/add_task_bt` | `info_hash`、`wanted`、`save_path`、`torrent_sha1`、`pick_code`、`wp_path_id` | `data[]`(成功为空数组) |
|
||||
| 任务列表 | GET | `/open/offline/get_task_list` | `page` | `data{page,page_count,count,tasks[]{...}}` |
|
||||
| 配额信息 | GET | `/open/offline/get_quota_info` | 无 | `data{package[]{surplus,used,count,name,expire_info[]},count,surplus,used}` |
|
||||
| 删除任务(P2) | POST | `/open/offline/del_task` | `info_hash`、`del_source_file` | `data[]` |
|
||||
| 清空任务(P2) | POST | `/open/offline/clear_task` | `flag`(0~5) | `data[]` |
|
||||
|
||||
请求形态:`add_task_*` / `del_task` / `clear_task` / `torrent` 均为 **multipart/form-data**;`get_*` 为 GET query。全部走 `Authorization: Bearer <access_token>`,即现有 `doAuthJSON` 通道。
|
||||
|
||||
### 5.2 任务列表字段(轮询用)
|
||||
|
||||
| 字段 | 类型 | 用途 |
|
||||
| :--- | :--- | :--- |
|
||||
| `info_hash` | string | 首选匹配键 |
|
||||
| `url` | string | 回退匹配键(链接任务) |
|
||||
| `status` | int | -1 失败 / 0 分配中 / 1 下载中 / 2 成功 |
|
||||
| `percentDone` | int | 进度(**注意驼峰**,见 5.4) |
|
||||
| `size` | int | 任务总字节 |
|
||||
| `name` | string | 任务名,用于 UI |
|
||||
| `file_id` / `delete_file_id` | string | 完成后源文件/文件夹 ID(P2 裁剪用) |
|
||||
| `wp_path_id` | string | 落盘父目录,可用于校验目录一致性 |
|
||||
| `add_time` / `last_update` | int | 超时判断 |
|
||||
|
||||
### 5.3 错误码映射(要落到用户可读文案)
|
||||
|
||||
| 码 | 115 文案 | MeBox 提示 |
|
||||
| :--- | :--- | :--- |
|
||||
| `1000012` | 云下载配额已用完 | 「115 云下载配额已用尽,请在 115 端购买/领取配额后重试」 |
|
||||
| `91006` | 存储空间不足 | 「115 存储空间不足」 |
|
||||
| `20018` | 文件不存在或已删除 | 「种子文件找不到(可能已被清理),请重新上传 .torrent」 |
|
||||
| `1000011` | 链接数量超过 115 个 | 本地已按 115 条切批,正常不应触发;触发则记 warning 并重切 |
|
||||
| `990002` | 参数错误 | 「提交参数不被 115 接受」,附原始 message |
|
||||
| `980004` | 操作失败 | 可重试,走退避 |
|
||||
| 无权限/未开通 | —— | 「当前 115 应用未开通云下载权限」,并提示到开放平台申请(见风险 R1) |
|
||||
|
||||
### 5.4 实现陷阱(必须按此实现)
|
||||
|
||||
1. **`percentDone` 是驼峰**,`get_task_list` 其余字段是 snake_case。Go struct tag 必须写 `json:"percentDone"`,写错会永远 0 进度。
|
||||
2. **`data[].info_hash` 仅任务成功受理时返回**,可能为空 → 必须同时保存提交时的原始 `url` 做回退匹配,并对 url 做归一化(去首尾空白、统一小写协议头、去掉 `&` 后跟踪参数?**不做过度归一化**,仅 trim,避免误合并)。
|
||||
3. **`wanted` 索引基准未确认**(见 11 节实测项)。本期不做选文件,实现统一传 `wanted="0"`(官方文档唯一明确允许的取值),待真机确认后再决定是否改为全索引拼接。
|
||||
4. **`add_task_urls` 单批 ≤115**,超了服务端报 `1000011`;本地必须切批。
|
||||
5. **`wp_path_id=0` 表示根目录**,绝不能把「未选择目录」当 0 提交,否则文件落到 115 根目录,STRM 永远扫不到 → 提交前强校验 `WpPathID != "" && != "0"`。
|
||||
6. **`get_task_list` 只支持 page,无 page_size**,需按 `page_count` 翻页;轮询设页数上限(默认 10 页)防止账号任务堆积把一轮轮询拖爆。
|
||||
|
||||
---
|
||||
|
||||
## 6. 为什么磁力不能选文件(结论留档)
|
||||
|
||||
| 接口 | 能否选择文件 | 依据 |
|
||||
| :--- | :--- | :--- |
|
||||
| `add_task_urls`(磁力/ed2k/http) | ❌ 无 wanted 参数 | `115doc/.../添加云下载链接任务.md` 请求参数仅 `urls`、`wp_path_id` |
|
||||
| `torrent`(解析种子) | ✅ 返回 `torrent_filelist[].wanted` | `解析BT种子.md` 返回字段表 |
|
||||
| `add_task_bt`(提交 BT) | ✅ `wanted` 必填 | `添加云下载BT任务.md` 请求参数表 |
|
||||
|
||||
`torrent` 的入参是 **`torrent_sha1` + `pick_code`**,只能解析「已上传到本人 115 空间的 .torrent 文件」,不接受 magnet / info_hash。开放平台没有「解析磁力 → 文件列表」这一步,因此磁力链路从入口就没有选文件能力。
|
||||
|
||||
**若要支持磁力选文件**,唯一路径是服务端自行把 magnet 抓成 .torrent(DHT metadata),属新增外部依赖与常驻网络行为,本期不做,记录为 P3 候选。
|
||||
|
||||
---
|
||||
|
||||
## 7. 后端实现
|
||||
|
||||
### 7.1 文件清单
|
||||
|
||||
**新增**
|
||||
|
||||
| 文件 | 内容 |
|
||||
| :--- | :--- |
|
||||
| `internal/model/cloud_download.go` | `CloudDownloadTask` 模型 + 状态常量 |
|
||||
| `internal/repository/cloud_download_repository.go` | CRUD、按状态取活跃任务、按阶段取待推进任务、统计 |
|
||||
| `internal/service/cloud115/offline.go` | 4+2 个 API 封装、响应结构体、错误码常量与 `OfflineError` 类型 |
|
||||
| `internal/service/cloud_download.go` | `CloudDownloadService` 主体:构造、提交、取消、重试、列表、配额 |
|
||||
| `internal/service/cloud_download_input.go` | 输入解析(多行/类型判定/去重/切批 ≤115)、匹配键归一化 |
|
||||
| `internal/service/cloud_download_submit.go` | 链接提交 + `.torrent` 上传解析 → `add_task_bt`、暂存目录查找/创建 |
|
||||
| `internal/service/cloud_download_poll.go` | 轮询、状态映射、超时熔断、远程任务与本地记录对账 |
|
||||
| `internal/service/cloud_download_pipeline.go` | Stage 编排、去抖、等待同步、扫描、刮削、失败隔离 |
|
||||
| `internal/service/scheduler_cloud_download.go` | `jobCloudDownloadPoll` |
|
||||
| `internal/handler/cloud_download.go` | REST handler + multipart 种子上传 |
|
||||
| `internal/handler/routes_cloud_download.go` | 路由注册(在 `registerAdminStrmRoutes` 同级新增 `registerAdminCloudDownloadRoutes`) |
|
||||
| `web/src/types/cloudDownload.ts` | 前端类型 |
|
||||
| `web/src/api/cloudDownload.ts` | 前端 API |
|
||||
| `web/src/pages/CloudDownloadPage.tsx` | 页面(提交面板 + 任务列表 + 配额) |
|
||||
|
||||
**修改**
|
||||
|
||||
| 文件 | 改动 | 风险 |
|
||||
| :--- | :--- | :--- |
|
||||
| `internal/model/model.go` | `AllModels()` 追加 `&CloudDownloadTask{}` | 低(AutoMigrate 新增表) |
|
||||
| `internal/repository/repository.go` | `Container` 增加 `CloudDownload *CloudDownloadRepository` | 低 |
|
||||
| `internal/service/service.go` | `Container` 增加 `CloudDownload *CloudDownloadService`,`Start/Stop` 联动 | 低 |
|
||||
| `internal/service/strm_service.go` | 新增 `SetSyncFinishHook`、`Open115Client` | 中(新增导出方法,不改现有逻辑) |
|
||||
| `internal/service/strm_sync.go` | `finishSync` 末尾触发 hook(异步) | **中**:hook 里必须自带 recover + 超时,绝不能阻塞或 panic 影响同步协程 |
|
||||
| `internal/service/service_builder.go` | 构造 `CloudDownload`(在 `Strm` 之后),注入 Strm/Scan/Scraper/Tasks/Hub;回调 hook | 低 |
|
||||
| `internal/service/scheduler.go` | `Start()` 里 append `cloud_download_poll` job(仅在 `CloudDownload != nil` 时注册) | 低 |
|
||||
| `internal/repository/library_repository.go` | 新增 `FindEnabledRootsByLocalPath(ctx, localPath)`(归一化 + 前缀匹配) | 低 |
|
||||
| `web/src/appRoutes.tsx`、`web/src/components/layoutNavigation.ts` | 路由 + 侧边栏入口 | 低 |
|
||||
|
||||
### 7.2 服务对外 API
|
||||
|
||||
```go
|
||||
type CloudDownloadService struct{ /* cfg, log, repos, strm, scanner, scraper, tasks, hub */ }
|
||||
|
||||
func (s *CloudDownloadService) Submit(ctx context.Context, req CloudDownloadSubmit) (*CloudDownloadSubmitResult, error)
|
||||
func (s *CloudDownloadService) SubmitTorrent(ctx context.Context, req CloudDownloadTorrentSubmit) (*CloudDownloadSubmitResult, error)
|
||||
func (s *CloudDownloadService) List(ctx context.Context, q CloudDownloadQuery) (*CloudDownloadPage, error)
|
||||
func (s *CloudDownloadService) Cancel(ctx context.Context, id string) error
|
||||
func (s *CloudDownloadService) Retry(ctx context.Context, id string) error // 失败任务重新提交
|
||||
func (s *CloudDownloadService) RetryPipeline(ctx context.Context, id string) error // 仅重做后处理
|
||||
func (s *CloudDownloadService) Delete(ctx context.Context, id string) error
|
||||
func (s *CloudDownloadService) ClearFinished(ctx context.Context) (int64, error)
|
||||
func (s *CloudDownloadService) Quota(ctx context.Context, accountID string) (*OfflineQuota, error)
|
||||
func (s *CloudDownloadService) Start(ctx context.Context) // 恢复中断任务
|
||||
func (s *CloudDownloadService) Stop()
|
||||
```
|
||||
|
||||
`CloudDownloadSubmit`:`AccountID`、`SyncPathID`、`URLs []string`(一次提交多条)。
|
||||
`CloudDownloadSubmitResult`:逐条 `{source, task_id, ok, error}`,**部分成功必须返回 200 + 明细**。
|
||||
|
||||
### 7.3 提交校验(顺序不可乱)
|
||||
|
||||
1. `SyncPathID` 存在、`Enabled`、`Provider == cloud115`、`AccountID` 非空、`RemotePath` 为有效目录 ID 且 ≠ `"0"`;
|
||||
2. 账号存在且 `Enabled`,`Open115Client` 可取得;
|
||||
3. 配额预检:`get_quota_info` → `surplus <= 0` 时直接拒绝(批量多链接时按条数预估,无法精确预估大小,只做「配额是否耗尽」闸门);
|
||||
4. 输入解析:按行拆分 → 类型判定 `magnet:?xt=urn:btih:` / `ed2k://` / `http(s)://` / `ftp://` → trim → 去重(同批内 + 与库内 `submitted/downloading` 任务去重);
|
||||
5. 切批:每批 ≤115 条,逐批调用 `add_task_urls`;
|
||||
6. 逐条落库:`data[].state=true` → `submitted`;失败 → `failed` + 错误码映射文案。
|
||||
|
||||
### 7.4 轮询实现要点
|
||||
|
||||
```
|
||||
每 tick(默认 60s):
|
||||
1. 取所有 status IN (submitted, downloading) 的任务,按 account_id 分组
|
||||
2. 无活跃任务 → 直接返回(不打 115 接口)
|
||||
3. 对每个账号:
|
||||
翻页取 get_task_list(page=1..min(page_count, 10))
|
||||
构建索引:info_hash(大写) → task,url(trim) → task
|
||||
逐条匹配本地记录:
|
||||
- info_hash 命中优先
|
||||
- 否则 url 命中
|
||||
- 都未命中 → 记 LastPollAt,未超时前不判失败(115 受理有延迟)
|
||||
状态映射 + 更新 Percent/SizeBytes/Name
|
||||
status=2 → succeeded + CompletedAt + Stage=pending
|
||||
status=-1 → failed(Error 用 115 文案)
|
||||
4. 超时:submitted/downloading 且 add_time 早于 now-72h → failed「下载超时」
|
||||
```
|
||||
|
||||
### 7.5 Pipeline 实现要点
|
||||
|
||||
```
|
||||
推进循环(可挂在同一 tick 之后,也可独立每 15s):
|
||||
A. 取 status=succeeded AND stage IN (pending, waiting_sync, failed) 的任务,按 sync_path_id 分组
|
||||
- 每组内若有任务 completed_at 在去抖窗口内(默认 30s)→ 本轮跳过(等凑批)
|
||||
B. 对每个 syncPath:
|
||||
- 若 IsSyncRunning(pathID) → 全部置 waiting_sync,下一轮再看
|
||||
- 否则 StartSync(pathID, incremental),置 syncing,记录 StrmRecordID(取该 path 最新记录)
|
||||
C. 同步完成后(hook):
|
||||
- 置 scanning → 反查 library(LocalPath 与 LibraryRoot.Path 归一化后相等或为其子目录)
|
||||
- 无匹配库 → Stage=na + StageError「本地目录未加入任何媒体库」
|
||||
- ScanLibrary(libraryID) → ScanAdded
|
||||
- 置 scraping → EnqueueLibrary(libraryID, ScrapeOptions{}) → ScrapeQueued
|
||||
- 置 done
|
||||
D. 任一步失败:Stage=failed + StageError,不重试(等待用户点「重做后处理」)
|
||||
E. 进程重启:Start() 时把 syncing/scanning/scraping 的僵尸阶段重置为 pending(参考 StrmService.recoverInterruptedSyncs 的写法)
|
||||
```
|
||||
|
||||
去抖细节:以 `syncPathID` 为单位,服务内存里维护 `lastTrigger[pathID] = time`;窗口内到达的任务全部 `waiting_sync`,窗口结束时由首个触发者发起同步。
|
||||
|
||||
### 7.6 STRM 完成回调
|
||||
|
||||
```go
|
||||
// strm_service.go
|
||||
func (s *StrmService) SetSyncFinishHook(fn func(ctx context.Context, syncPathID, status string)) { ... }
|
||||
```
|
||||
|
||||
在 `finishSync`(`strm_sync.go:323`)**落库完成之后**调用:
|
||||
|
||||
```go
|
||||
if hook := s.syncFinishHook(); hook != nil {
|
||||
helper.Go(s.log, "strm.syncFinishHook", func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
hook(ctx, p.ID, status)
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
要点:**异步 + 自带 recover + 超时**,绝不能让 hook 的 panic 或阻塞影响同步协程;也不要在 `finishSync` 里同步调用扫描(同步耗时长,会拖住同步收尾与 Pending 记录)。
|
||||
|
||||
### 7.7 调度与配置项
|
||||
|
||||
设置项(沿用 `repo.Setting`,键名与现有 `scrape.*` / `strm.*` 风格一致):
|
||||
|
||||
| 键 | 默认 | 说明 |
|
||||
| :--- | :--- | :--- |
|
||||
| `cloud_download.poll_interval_sec` | `60` | 轮询间隔(30~600) |
|
||||
| `cloud_download.sync_debounce_sec` | `30` | 同一同步目录的去抖窗口(0~300) |
|
||||
| `cloud_download.auto_scrape` | `true` | 同步后是否自动刮削 |
|
||||
| `cloud_download.task_timeout_hours` | `72` | 下载超时判定 |
|
||||
| `cloud_download.torrent_max_mb` | `20` | `.torrent` 上传大小上限 |
|
||||
| `cloud_download.max_pages` | `10` | 单轮轮询最大翻页数 |
|
||||
|
||||
`auto_scrape=true` 时**不受** `scrape.auto_on_scan` 全局开关约束(用户显式提交了下载,期望链路闭环);但若 `Scraper.AnyEnabled()==false`(无任何刮削源),Stage 直接置 `done` 并在 `StageError` 记「未配置刮削源」。
|
||||
|
||||
### 7.8 HTTP 路由与鉴权
|
||||
|
||||
挂在 `registerAdminStrmRoutes` 同组(`admin`,已带 `AuthRequired` + `AdminRequired`):
|
||||
|
||||
```go
|
||||
admin.GET("/cloud-downloads", listCloudDownloadsHandler(svc))
|
||||
admin.POST("/cloud-downloads", createCloudDownloadsHandler(svc))
|
||||
admin.POST("/cloud-downloads/torrent", uploadCloudDownloadTorrentHandler(svc)) // multipart
|
||||
admin.GET("/cloud-downloads/quota", cloudDownloadQuotaHandler(svc))
|
||||
admin.POST("/cloud-downloads/:id/retry", retryCloudDownloadHandler(svc))
|
||||
admin.POST("/cloud-downloads/:id/retry-pipeline", retryCloudDownloadPipelineHandler(svc))
|
||||
admin.POST("/cloud-downloads/:id/cancel", cancelCloudDownloadHandler(svc))
|
||||
admin.DELETE("/cloud-downloads/:id", deleteCloudDownloadHandler(svc))
|
||||
admin.POST("/cloud-downloads/clear-finished", clearFinishedCloudDownloadsHandler(svc))
|
||||
```
|
||||
|
||||
multipart 端点:`file`(.torrent)+ `account_id` + `sync_path_id`;实现需 `c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxMB<<20)`,并校验文件头(BT 种子是 bencode,以 `d` 开头,含 `announce` / `info`)。
|
||||
|
||||
---
|
||||
|
||||
## 8. 前端实现
|
||||
|
||||
### 8.1 页面结构(`CloudDownloadPage.tsx`)
|
||||
|
||||
**顶部:提交面板**
|
||||
- 目标同步目录下拉:复用 `GET /admin/strm/paths`,只列 `provider=cloud115 && enabled`,展示 `name + remote_display_path + local_path`;
|
||||
- 账号显示:由所选同步目录自动带出(`account_id`),无需二次选择;
|
||||
- 链接文本框:多行粘贴,右下角实时显示「识别到 N 条:磁力 x / ed2k y / http z」;
|
||||
- `.torrent` 拖拽/选择:单文件,提交时走 multipart 端点;
|
||||
- 配额条:`surplus/count`,耗尽时提交按钮禁用并提示;
|
||||
- 提交结果:逐条成功/失败明细(不合并成一句「提交失败」)。
|
||||
|
||||
**下方:任务列表**
|
||||
- 列:来源(截断显示 + tooltip 全量)、目标目录、Size、下载进度、**阶段徽标**(下载中 / 等待同步 / 同步中 / 扫描中 / 刮削中 / 完成 / 失败)、时间、操作;
|
||||
- 操作:取消(仅下载中)、重试(失败)、重做后处理(Stage 失败)、删除(终态);
|
||||
- 批量:清空已完成;
|
||||
- 刷新:复用现有 WS Hub(`hub.Publish("cloud_download", …)`),断线时降级为 5s 轮询。
|
||||
|
||||
### 8.2 路由与导航
|
||||
|
||||
- `appRoutes.tsx`:`{ path: 'cloud-downloads', element: <CloudDownloadPage />, adminOnly: true }`
|
||||
- `layoutNavigation.ts`:`LAYOUT_NAV_ITEMS` 中 `STRM 管理` 之后加 `{ to: '/cloud-downloads', label: '云下载', icon: CloudDownload, adminOnly: true }`
|
||||
- `StrmManagePage` 顶部工具区加一个「云下载」跳转按钮,形成入口闭环。
|
||||
|
||||
---
|
||||
|
||||
## 9. 权限与安全
|
||||
|
||||
### 9.1 鉴权
|
||||
|
||||
首期仅管理员:路由挂在 `admin` 组,`AdminRequired` 中间件已覆盖。
|
||||
|
||||
### 9.2 输入安全
|
||||
|
||||
- 链接只做**字符串拼接转发**,服务端绝不主动发起对这些 URL 的请求(无 SSRF 面);
|
||||
- `.torrent` 文件仅落临时目录 → 上传 115 → 删除,不本地解析 bencode(P1 若要预览文件名需加 bencode 解析,届时必须限大小与深度);
|
||||
- 上传文件名禁用 `..` / 路径分隔符;
|
||||
- `save_path` 本期固定为空(留字段),避免路径穿越争议。
|
||||
|
||||
### 9.3 后续(P2)
|
||||
|
||||
`model.UserPermission` 增加 `CanUseCloudDownload bool`,配套 handler 从 `AdminRequired` 换成权限中间件,并给普通用户做配额上限(否则一个用户可以把账号的 115 云下载配额打空)。
|
||||
|
||||
---
|
||||
|
||||
## 10. 测试计划
|
||||
|
||||
### 10.1 单元测试(必须)
|
||||
|
||||
| 目标 | 用例 |
|
||||
| :--- | :--- |
|
||||
| 输入解析 | 多行混合、空行、重复链接、首尾空白、大小写协议头、>115 条切批、非法行跳过并计数 |
|
||||
| 状态映射 | status -1/0/1/2 → failed/downloading/downloading/succeeded;缺失 info_hash 时 url 回退命中;大小写 info_hash |
|
||||
| 错误码映射 | 1000012 / 91006 / 20018 / 990002 文案与是否可重试 |
|
||||
| 提交校验 | `RemotePath="0"`、账号禁用、同步目录禁用、配额耗尽 → 全部拒绝且不产生 115 请求 |
|
||||
| 去抖 | 30s 内 3 个任务完成 → 只触发 1 次 `StartSync` |
|
||||
| 等待同步 | `IsSyncRunning=true` → 全部 waiting_sync;结束后成功发起 |
|
||||
| 阶段隔离 | 同步失败不改变 Status;扫描失败不改变 syncing 记录 |
|
||||
| 僵尸恢复 | 重启后 syncing/scanning/scraping → pending |
|
||||
|
||||
测试风格沿用仓库现状:同包 `*_test.go`、`t.Context()`、httptest 模拟 115(参考 `internal/service/cloud115/cloud115_test.go` 的 mock 模式)。
|
||||
|
||||
### 10.2 集成测试
|
||||
|
||||
- `httptest` 起伪 115(`/open/offline/*`):验证提交 → 轮询 → 状态流转 → pipeline 触发(Strm 用 fake hook,Scanner/Scraper 用桩)。
|
||||
- 现有测试回归:`go test ./internal/service/... ./internal/repository/... ./internal/handler/...`。
|
||||
|
||||
### 10.3 真机验收(不可省)
|
||||
|
||||
| # | 步骤 | 期望 |
|
||||
| :--- | :--- | :--- |
|
||||
| 1 | 提交一条小体积磁力(<200MB)绑定某 115 同步目录 | 任务 5s 内变 `submitted` |
|
||||
| 2 | 等 115 完成 | 任务变 `succeeded`,随后 `syncing` |
|
||||
| 3 | 同步完成 | 本地目录出现新 `.strm` 文件 |
|
||||
| 4 | 自动扫描 | 媒体库新出现对应条目 |
|
||||
| 5 | 自动刮削 | 条目 `scrape_status` 变 `matched`,海报正确 |
|
||||
| 6 | 提交 `.torrent` 文件 | 同上,且 115 端任务名与种子内任务名一致 |
|
||||
| 7 | 提交 ed2k 链接 | 同上(ed2k 无 info_hash,验证 url 回退匹配) |
|
||||
| 8 | 提交 120 条链接 | 自动切 2 批,逐条结果完整 |
|
||||
|
||||
---
|
||||
|
||||
## 11. 待实测确认项
|
||||
|
||||
| # | 项 | 影响 | 处理 |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| 1 | `add_task_bt` 的 `wanted` 是 0-based 还是 1-based | 后续若开放选文件,索引拼接依赖此语义 | 真机用一个 3 文件种子验证;当前实现统一传 `"0"`,不受此影响 |
|
||||
| 2 | `wanted="0"` 是「全部」还是「第 0 个」 | 决定 `.torrent` 任务是否只下一个文件 | **必测**:当前实现依赖它表示「全部」 |
|
||||
| 3 | `MkDir` 对已存在目录的返回 | 暂存目录重复创建 | 先 `FindNamedContentInParent` 查找,未命中再 `MkDir`;两者都失败则回退到目标同步目录 |
|
||||
| 4 | 当前 115 应用是否已开通云下载权限 | 全功能可用性 | 提交前 `get_quota_info` 探测;无权限时给出「去开放平台申请」指引 |
|
||||
| 5 | 磁力整体提交时 115 默认勾选哪些文件 | 是否漏下样片/字幕 | 不影响链路,只在文档说明 |
|
||||
| 6 | `get_task_list` 单页实际返回条数 | 翻页策略 | 首轮实测后调整 `max_pages` 默认值 |
|
||||
|
||||
---
|
||||
|
||||
## 11.5 实现状态(P0 已落地)
|
||||
|
||||
已完成(后端 + 前端最小可用):
|
||||
|
||||
| 层 | 文件 | 说明 |
|
||||
| :--- | :--- | :--- |
|
||||
| 模型 | `internal/model/cloud_download.go` | `CloudDownloadTask` + 状态/阶段常量,已注册 `AllModels()` |
|
||||
| 仓储 | `internal/repository/cloud_download_repository.go` | 分页查询、活跃任务去重、阶段推进、僵尸阶段重置 |
|
||||
| 115 协议 | `internal/service/cloud115/offline.go` | 7 个接口封装;`client.go` 新增 multipart 请求构造并复用限流/刷新/错误封装 |
|
||||
| 服务 | `internal/service/cloud_download*.go` | 输入解析、提交、轮询、去抖、Pipeline(同步→扫描→刮削) |
|
||||
| 调度 | `internal/service/scheduler_cloud_download.go` | `cloud_download_poll` 周期任务 |
|
||||
| 打通 | `internal/service/strm_service.go` / `strm_sync.go` | `Open115Client` 复用缓存客户端;`SetSyncFinishHook` 异步回调 |
|
||||
| HTTP | `internal/handler/cloud_download.go` | 9 个管理员接口 + multipart 种子上传 |
|
||||
| 前端 | `web/src/pages/CloudDownloadPage.tsx` 等 | 提交面板、配额、任务列表、重试/取消/删除、5s 自动刷新 |
|
||||
|
||||
P0 范围外(保留为 P1/P2):
|
||||
|
||||
- 设置项 UI(当前通过 `Setting` 表键值生效,默认轮询 60s / 去抖 30s / 超时 72h);
|
||||
- WebSocket 实时进度(当前为 5s 轮询);
|
||||
- 普通用户权限位与单用户配额上限;
|
||||
- 远端任务清理(`del_task` / `clear_task`)在 UI 上无入口(取消任务时会尽力调用 `del_task`)。
|
||||
|
||||
验证证据:
|
||||
|
||||
- `go build ./...`、`go vet` 通过;
|
||||
- 新增单测:`internal/service/cloud115/offline_test.go`(7 项,含 multipart 格式与 `percentDone` 解析)、`internal/service/cloud_download_test.go`(9 项,含解析/去抖/目录匹配/校验);
|
||||
- 前端 `tsc --noEmit`、`eslint`、`vite build` 通过;
|
||||
- 3 个既有测试在本机 HEAD 上同样失败(Windows 文件占用与 SQLite `VACUUM INTO` 环境问题),已用独立 worktree 复核,与本次改动无关。
|
||||
|
||||
## 12. 分期与验收
|
||||
|
||||
### P0(可跑通,约 1 个迭代)
|
||||
|
||||
- 模型 + 仓储 + 迁移注册
|
||||
- `cloud115/offline.go`(4 个接口:add_task_urls / get_task_list / get_quota_info / torrent + add_task_bt)
|
||||
- 提交(磁力/ed2k/http)+ 轮询 + 状态机
|
||||
- STRM finish hook + Pipeline(同步 → 扫描 → 刮削)
|
||||
- 调度 job + 设置项
|
||||
- 前端最小页:提交面板 + 任务列表(阶段徽标、重试、删除)
|
||||
- **验收**:10.3 的步骤 1~5、7、8 通过
|
||||
|
||||
### P1(体验完善)
|
||||
|
||||
- `.torrent` 上传端点 + 全选提交(10.3 步骤 6)
|
||||
- 配额条、提交结果逐条明细
|
||||
- 去抖、僵尸恢复、超时熔断
|
||||
- WS 实时进度
|
||||
- 分页列表 + 筛选(状态/目录)
|
||||
|
||||
### P2(可选)
|
||||
|
||||
- 远端任务清理(`del_task` / `clear_task`)
|
||||
- 完成后删除暂存 `.torrent`
|
||||
- 普通用户权限位 + 单用户配额上限
|
||||
- Telegram 完成通知
|
||||
|
||||
### P3(本期不做,记录候选)
|
||||
|
||||
- 磁力选文件(服务端 DHT 抓 .torrent)
|
||||
- 完成后按 file_id 裁剪不需要的文件
|
||||
|
||||
---
|
||||
|
||||
## 13. 风险与回滚
|
||||
|
||||
| # | 风险 | 影响 | 缓解 |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| R1 | 115 应用未开通云下载权限 | 功能不可用 | 提交前探测 + 明确文案;不影响现有 STRM 功能 |
|
||||
| R2 | 令牌双实例互相作废 | 账号被标「授权已失效」 | **强约束 K1**:只复用 StrmService 缓存的客户端 |
|
||||
| R3 | 频繁同步打爆 CPU/115 限流 | 服务卡顿、风控 | 去抖 30s + 最小间隔 + 同步运行中等待 + throttle_manager |
|
||||
| R4 | 目录错配(`wp_path_id` 不在同步目录) | 下了但永远扫不到 | 提交时强制 `RemotePath` 非空非 0;UI 只允许选同步目录,不放开自由选目录 |
|
||||
| R5 | finish hook panic 拖垮同步 | 同步记录永远 running | hook 异步 + `helper.Go` + recover + 独立超时 |
|
||||
| R6 | 新增表迁移失败 | 启动失败 | `AutoMigrate` 加表无破坏性;出问题回滚只需删表 + 移除注册 |
|
||||
| R7 | 单页页数过多导致轮询耗时 | 轮询堆积 | `max_pages` 上限;超限记 warning 并下轮继续 |
|
||||
|
||||
**回滚方式**:移除路由与侧边栏入口即可(功能面隐藏);表可保留(不影响其它子系统),或手动 `DROP TABLE cloud_download_tasks`。
|
||||
|
||||
---
|
||||
|
||||
## 14. 附录:核对依据
|
||||
|
||||
| 结论 | 依据位置 |
|
||||
| :--- | :--- |
|
||||
| 115 开放平台离线接口形态与参数 | `115doc/115开放平台/API列表/云下载/*.md` |
|
||||
| STRM 同步入口与运行互斥 | `internal/service/strm_sync.go:82-138` |
|
||||
| 同步收尾无回调 | `internal/service/strm_sync.go:323`(`finishSync`) |
|
||||
| 115 目录整树拉取成本 | `internal/service/strm_sync.go:682`(`walk115Flat`) |
|
||||
| 令牌刷新落库与单实例要求 | `internal/service/strm_service.go:524-556`、`strm_115_oauth.go:438` |
|
||||
| 115 provider 断言复用范式 | `internal/service/strm_sync.go:378`(`cloud.OpenAPI115Provider`) |
|
||||
| 上传种子可拿 PickCode | `internal/service/cloud115/upload.go:95-102,217` |
|
||||
| 刮削候选只挑 pending | `internal/service/scraper_library.go:183-207` |
|
||||
| 扫描自动刮削开关 | `internal/service/scanner_prune.go:279`、`scanner_post_scan.go:19` |
|
||||
| 周期任务注册范式 | `internal/service/scheduler.go:118-210` |
|
||||
| 模型迁移注册 | `internal/model/model.go:37`、`internal/database/schema_migration.go:34` |
|
||||
| 服务装配顺序 | `internal/service/service_builder.go:148` |
|
||||
@@ -0,0 +1,46 @@
|
||||
# Playback Correctness and Isolation Implementation Plan
|
||||
|
||||
**Goal:** Fix the five playback defects from the 2026-10-21 review and make playback capability queries side-effect-free, without replacing the player or adding dependencies.
|
||||
|
||||
**Scope:** The user approved implementing the reviewed fixes. Work in the current clean checkout; do not commit, push, deploy, or alter unrelated files.
|
||||
|
||||
## Design and invariants
|
||||
|
||||
- Keep existing media IDs, stream routes, direct playback and external-player URLs compatible.
|
||||
- Add optional `session_id` to local HLS requests. Authenticated handlers supply the actual user identity; a client parameter never supplies the owner. New web playback sessions use random IDs. Legacy clients are isolated by user and, where available, Emby play-session/device identity.
|
||||
- Keep transcoder media lookup/logging separate from its internal task key. A task key incorporates user, media and playback session. Each actual ffmpeg run writes to a fresh generation directory, so delayed segment requests never read a replacement run's output.
|
||||
- Stop only the authenticated caller's playback session. Check media visibility and scoped playback tokens on stop, just as on playlist/segment requests. Segment and playlist queries must match the running generation.
|
||||
- Cloud child requests resolve their real session media first, compare query media, enforce token scope and visibility against the real media, and reject mismatches before upstream fetch.
|
||||
- Treat player business positions as absolute source seconds. Only local HLS converts to/from a relative ffmpeg timeline.
|
||||
- Invalidate outstanding cloud-transcode requests on media/unmount, explicit source/quality changes, and newer requests. Guard both fulfilled and rejected responses.
|
||||
- Select local fallbacks only from available local qualities. Keep the existing 115-first preference when a cloud quality is available.
|
||||
- `PlaybackInfo` only reads capabilities/status. Explicit `StartTranscode` requests enqueue cloud work. Retain pending/ready response contracts, polling, and retries.
|
||||
|
||||
## Tasks
|
||||
|
||||
1. Add regression tests for local-scope isolation, stale generations, stop authorization, cloud media binding, pure capability reads, timeline conversion, available-local selection and stale async results. Run tests before implementation and verify the intended failures.
|
||||
2. Add tested playback helpers; route skip/undo/progress/quality/subtitle calculations through the common absolute timeline; guard cloud transcode results and fix local fallback choice.
|
||||
3. Extend existing transcoder/stream methods with an optional scope, preserve legacy internal calls, isolate task keys/output directories, and pass caller scope from web and Emby handlers. Bind web local HLS URLs and stop requests to captured session IDs.
|
||||
4. Enforce real-media authorization on cloud child requests and authenticated ownership on local stop.
|
||||
5. Split read-only cloud capability lookup from explicit cloud enqueue. Update existing cloud tests to use the explicit start operation and test no push during reads.
|
||||
6. Run focused and full Go tests, frontend tests, TypeScript/Vite build, lint diagnostics and review the diff for security, lifecycle and API compatibility. Fix regressions before reporting completion.
|
||||
|
||||
## Verification commands
|
||||
|
||||
- `node --experimental-strip-types --test src/pages/hlsPlayback.test.ts src/pages/directPlaySeek.test.ts src/pages/playerPageModel.test.ts src/pages/playbackQuality.test.ts src/pages/playbackRequestGuard.test.ts src/utils/skipSegments.test.ts`
|
||||
- `go test ./internal/service ./internal/handler ./internal/middleware -count=1`
|
||||
- `go test ./...`
|
||||
- `npm run build` and targeted ESLint for changed frontend files.
|
||||
|
||||
## Out of scope
|
||||
|
||||
New remux/audio-only ffmpeg pipelines, speculative browser codec policy changes, player replacement, broad UI refactors, performance improvement claims without measurements, and production deployment.
|
||||
|
||||
## Verification outcome
|
||||
|
||||
- Six frontend playback regression scripts pass; targeted ESLint and the TypeScript/Vite production build pass. The existing bundle-size warning remains.
|
||||
- Playback/stream/transcode/progress tests in service, handler and middleware pass, including scoped output reads, stale generation rejection, scoped stops, device-to-segment propagation, read-only capability queries and concurrent cloud start deduplication.
|
||||
- Whole-repository tests pass with only `TestQueueLibraryRootScanImportsNewLibraryMedia` and `TestSQLiteRestoreStagesWithoutChangingLiveDatabase` excluded. Both were run independently and reproduce Windows file-lock / SQLite errors in unmodified scan/database code. They are not silently disabled in source.
|
||||
- FFmpeg is not installed in this environment, so the existing real-transcode lifecycle test skips. Actual video, upstream 115 and browser end-to-end playback remain unverified.
|
||||
- Clients with no playback-session or device identifier retain per-user legacy isolation; new web requests always carry an independent session ID.
|
||||
- No commit, push or deployment was performed.
|
||||
@@ -45,14 +45,13 @@ func setDefaults(v *viper.Viper) {
|
||||
v.SetDefault("logging.max_backups", 10)
|
||||
|
||||
v.SetDefault("cache.cache_dir", "./cache")
|
||||
v.SetDefault("cache.images_max_size_mb", 2000)
|
||||
v.SetDefault("cache.images_max_size_mb", 2048)
|
||||
v.SetDefault("cache.images_originals_max_size_mb", 256)
|
||||
v.SetDefault("cache.images_originals_ttl_hours", 168)
|
||||
v.SetDefault("cache.memory_max_size_mb", DefaultCacheMemoryMaxSizeMB)
|
||||
v.SetDefault("cache.redis_url", "")
|
||||
v.SetDefault("cache.redis_prefix", "mebox")
|
||||
v.SetDefault("cache.media_ttl_seconds", 90)
|
||||
v.SetDefault("cache.emby_latest_ttl_seconds", 300)
|
||||
v.SetDefault("cache.emby_latest_ttl_seconds", 900)
|
||||
v.SetDefault("cache.reader_content_max_size_mb", 1024)
|
||||
v.SetDefault("cache.reader_content_ttl_hours", 168)
|
||||
v.SetDefault("cache.reader_files_ttl_hours", 720)
|
||||
|
||||
@@ -50,9 +50,6 @@ func (c *Config) normalize() error {
|
||||
if c.Cache.ImagesOriginalsMaxSizeMB < 0 {
|
||||
c.Cache.ImagesOriginalsMaxSizeMB = 0
|
||||
}
|
||||
if c.Cache.ImagesOriginalsTTLHours < 0 {
|
||||
c.Cache.ImagesOriginalsTTLHours = 0
|
||||
}
|
||||
if c.Cache.MemoryMaxSizeMB <= 0 {
|
||||
c.Cache.MemoryMaxSizeMB = DefaultCacheMemoryMaxSizeMB
|
||||
}
|
||||
|
||||
@@ -120,14 +120,16 @@ type LoggingConfig struct {
|
||||
|
||||
// CacheConfig 控制磁盘转写缓存和进程内热缓存。
|
||||
type CacheConfig struct {
|
||||
CacheDir string `mapstructure:"cache_dir"`
|
||||
ImagesMaxSizeMB int `mapstructure:"images_max_size_mb"`
|
||||
CacheDir string `mapstructure:"cache_dir"`
|
||||
// ImagesMaxSizeMB 是整个磁盘缓存的总容量上限(MB),不是 images 子目录
|
||||
// 的配额。参与统计的池为 hls、images、library-covers;超限时按该顺序淘汰
|
||||
// 最旧文件。reader-audio 不在其中:它按 reader.audioMediaMaxBytes 的固定
|
||||
// 配额(当前 1GiB)在写入时独立淘汰,不参与这里的总量。
|
||||
// 0 表示不限制。
|
||||
ImagesMaxSizeMB int `mapstructure:"images_max_size_mb"`
|
||||
// ImagesOriginalsMaxSizeMB 限制 cache/images 下原图子配额(MB)。原图只是
|
||||
// 生成缩略图的原料,丢了可以重新回源;0 表示不单独限制(只受总量约束)。
|
||||
ImagesOriginalsMaxSizeMB int `mapstructure:"images_originals_max_size_mb"`
|
||||
// ImagesOriginalsTTLHours 是原图的保留时长(小时),超过即删除;0 表示
|
||||
// 不按时间淘汰。缩短它能让原图缓存保持在一个滚动窗口内。
|
||||
ImagesOriginalsTTLHours int `mapstructure:"images_originals_ttl_hours"`
|
||||
// MemoryMaxSizeMB 限制进程内 L1 缓存总字节数,JSON/对象缓存共用该预算。
|
||||
MemoryMaxSizeMB int `mapstructure:"memory_max_size_mb"`
|
||||
RedisURL string `mapstructure:"redis_url"`
|
||||
@@ -136,7 +138,8 @@ type CacheConfig struct {
|
||||
// EmbyLatestTTLSeconds 是 Emby「最新添加」(Items/Latest) 的缓存时长。
|
||||
// 客户端刷新首页时会并发请求全部媒体库的 Latest(生产环境观察到 73 个
|
||||
// 并发),缓存过短会让这批请求同时穿透并各自重建 payload,在低配主机
|
||||
// 上造成秒级延迟。默认 300 秒,新入库内容最迟 5 分钟后出现在最新列表。
|
||||
// 上造成秒级延迟。默认 900 秒;过期后仍会先返回陈旧副本并在后台重建,
|
||||
// 新入库内容最迟 15 分钟后出现在最新列表(扫描完成会主动失效缓存)。
|
||||
EmbyLatestTTLSeconds int `mapstructure:"emby_latest_ttl_seconds"`
|
||||
// ReaderContentMaxSizeMB 阅读正文持久缓存的总容量(MB),0 表示不限制。
|
||||
ReaderContentMaxSizeMB int `mapstructure:"reader_content_max_size_mb"`
|
||||
|
||||
@@ -376,6 +376,11 @@ func ensurePerformanceIndexes(db *gorm.DB) error {
|
||||
`DROP INDEX IF EXISTS idx_media_library_release_active`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_library_created_active ON media(library_id, created_at DESC) WHERE deleted_at IS NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_library_episode_active ON media(library_id, season_num, episode_num, created_at DESC) WHERE deleted_at IS NULL`,
|
||||
// 虚拟剧集分组索引:series_id 为空的剧集行要按
|
||||
// (season_num, episode_num, episode_fraction, created_at) 排序后做内存
|
||||
// 分组。没有它 SQLite 只能全表扫描再建临时 B-tree,而 Emby 的
|
||||
// 卡片/海报查询每次只关心其中一部剧(生产环境实测单次 1.3s)。
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_virtual_series_group ON media(season_num, episode_num, episode_fraction, created_at) WHERE deleted_at IS NULL AND (series_id IS NULL OR series_id = '')`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_library_root_active ON media(library_id, library_root_id) WHERE deleted_at IS NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_series_active ON media(series_id, season_num, episode_num) WHERE deleted_at IS NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_media_library_series_active ON media(library_id, series_id, season_num, episode_num) WHERE deleted_at IS NULL`,
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
var sqliteRestoreMu sync.Mutex
|
||||
|
||||
func sqliteRestorePath(cfg *config.Config) (string, error) {
|
||||
if cfg == nil || strings.TrimSpace(cfg.Database.DBPath) == "" || cfg.Database.DBPath == ":memory:" || strings.HasPrefix(cfg.Database.DBPath, "file:") {
|
||||
return "", errors.New("恢复需要配置有效的 SQLite 数据库文件路径")
|
||||
}
|
||||
return filepath.Abs(cfg.Database.DBPath)
|
||||
}
|
||||
|
||||
func sqliteRestoreDSN(path, mode string) string {
|
||||
u := url.URL{Scheme: "file", Path: filepath.ToSlash(path)}
|
||||
q := url.Values{"mode": {mode}, "_pragma": {"busy_timeout(5000)"}}
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func validateSQLiteRestore(ctx context.Context, path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return errors.New("备份必须是常规文件")
|
||||
}
|
||||
db, err := sql.Open("sqlite", sqliteRestoreDSN(path, "ro"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
var result string
|
||||
if err := db.QueryRowContext(ctx, "PRAGMA quick_check").Scan(&result); err != nil {
|
||||
return fmt.Errorf("SQLite 备份校验失败: %w", err)
|
||||
}
|
||||
if result != "ok" {
|
||||
return fmt.Errorf("SQLite 备份校验失败: %s", result)
|
||||
}
|
||||
var tables int
|
||||
if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN ('users', 'media', 'settings')").Scan(&tables); err != nil {
|
||||
return err
|
||||
}
|
||||
if tables == 0 {
|
||||
return errors.New("备份不包含 MeBox 数据表")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StageSQLiteRestore leaves the live database and its WAL untouched. The
|
||||
// validated copy is applied on the next startup, before any database is opened.
|
||||
func StageSQLiteRestore(ctx context.Context, cfg *config.Config, source string) error {
|
||||
sqliteRestoreMu.Lock()
|
||||
defer sqliteRestoreMu.Unlock()
|
||||
dbPath, err := sqliteRestorePath(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pending := dbPath + ".restore-pending"
|
||||
if _, err := os.Lstat(pending); err == nil {
|
||||
return errors.New("已有等待恢复的备份,请重启 MeBox 后再操作")
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, source); err != nil {
|
||||
return err
|
||||
}
|
||||
in, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.CreateTemp(filepath.Dir(dbPath), ".mebox-restore-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := out.Name()
|
||||
defer os.Remove(tmp)
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, pending)
|
||||
}
|
||||
|
||||
// ApplyPendingSQLiteRestore must run before database.Open and before workers or
|
||||
// HTTP listeners start. The previous database is retained alongside the file.
|
||||
func ApplyPendingSQLiteRestore(ctx context.Context, cfg *config.Config, log *zap.Logger) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
dialect := normalizeDatabaseType(cfg.Database.Type)
|
||||
if dialect == "auto" {
|
||||
dialect = effectiveAutoDatabaseType(cfg)
|
||||
}
|
||||
if dialect != "sqlite" {
|
||||
return nil
|
||||
}
|
||||
sqliteRestoreMu.Lock()
|
||||
defer sqliteRestoreMu.Unlock()
|
||||
dbPath, err := sqliteRestorePath(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pending := dbPath + ".restore-pending"
|
||||
if _, err := os.Lstat(pending); errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSQLiteRestore(ctx, pending); err != nil {
|
||||
return err
|
||||
}
|
||||
previous := ""
|
||||
if _, err := os.Lstat(dbPath); err == nil {
|
||||
// A clean checkpoint prevents an old WAL from being replayed into the
|
||||
// replacement database. A busy checkpoint leaves all files untouched.
|
||||
live, err := sql.Open("sqlite", sqliteRestoreDSN(dbPath, "rw"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var busy, pages, copied int
|
||||
err = live.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &pages, &copied)
|
||||
closeErr := live.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if busy != 0 {
|
||||
return errors.New("SQLite 数据库仍被占用,恢复已保留等待下次启动")
|
||||
}
|
||||
old, err := os.CreateTemp(filepath.Dir(dbPath), filepath.Base(dbPath)+".before-restore-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
previous = old.Name()
|
||||
if err := old.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(previous); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(dbPath, previous); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(pending, dbPath); err != nil {
|
||||
if previous != "" {
|
||||
if rollbackErr := os.Rename(previous, dbPath); rollbackErr != nil {
|
||||
return fmt.Errorf("恢复失败: %w;回退失败: %v,原数据库保留于 %s", err, rollbackErr, previous)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
if log != nil {
|
||||
log.Warn("SQLite backup restored before startup", zap.String("previous_database", previous))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
func TestSQLiteRestoreStagesWithoutChangingLiveDatabase(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.Database.Type = "sqlite"
|
||||
cfg.Database.DBPath = filepath.Join(root, "live.db")
|
||||
live, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "rwc"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer live.Close()
|
||||
for _, statement := range []string{
|
||||
"PRAGMA journal_mode=WAL", "CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT)",
|
||||
"INSERT INTO settings VALUES ('state', 'backup')",
|
||||
} {
|
||||
if _, err := live.Exec(statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
source := filepath.Join(root, "backup.db")
|
||||
if _, err := live.Exec("VACUUM INTO ?", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := live.Exec("UPDATE settings SET value = 'live'"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value string
|
||||
if err := live.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "live" {
|
||||
t.Fatalf("staging changed live database: value=%q err=%v", value, err)
|
||||
}
|
||||
if _, err := live.Exec("UPDATE settings SET value = 'after-stage'"); err != nil {
|
||||
t.Fatalf("database could not write after staging: %v", err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
|
||||
t.Fatal("second staging should not replace the pending restore")
|
||||
}
|
||||
if err := os.Remove(source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := live.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restored, err := sql.Open("sqlite", sqliteRestoreDSN(cfg.Database.DBPath, "ro"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer restored.Close()
|
||||
if err := restored.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "backup" {
|
||||
t.Fatalf("restore failed: value=%q err=%v", value, err)
|
||||
}
|
||||
previous, err := filepath.Glob(cfg.Database.DBPath + ".before-restore-*")
|
||||
if err != nil || len(previous) != 1 {
|
||||
t.Fatalf("previous database not preserved: files=%v err=%v", previous, err)
|
||||
}
|
||||
old, err := sql.Open("sqlite", sqliteRestoreDSN(previous[0], "ro"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer old.Close()
|
||||
if err := old.QueryRow("SELECT value FROM settings").Scan(&value); err != nil || value != "after-stage" {
|
||||
t.Fatalf("old database lost latest writes: value=%q err=%v", value, err)
|
||||
}
|
||||
if err := ApplyPendingSQLiteRestore(t.Context(), cfg, nil); err != nil {
|
||||
t.Fatalf("repeat startup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteRestoreRejectsCorruptAndSymlinkBackups(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.Database.DBPath = filepath.Join(root, "live.db")
|
||||
source := filepath.Join(root, "corrupt.db")
|
||||
if err := os.WriteFile(source, []byte("not a SQLite database"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, source); err == nil {
|
||||
t.Fatal("corrupt backup accepted")
|
||||
}
|
||||
link := filepath.Join(root, "link.db")
|
||||
if err := os.Symlink(source, link); err == nil {
|
||||
if err := StageSQLiteRestore(t.Context(), cfg, link); err == nil {
|
||||
t.Fatal("symlink backup accepted")
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(cfg.Database.DBPath + ".restore-pending"); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed validation left pending data: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -87,9 +87,9 @@ func updateSettingHandler(svc *service.Container) gin.HandlerFunc {
|
||||
if req.Key == "transcode.hw_enabled" || req.Key == "transcode.hw_accel" || req.Key == "transcoder.hardware_accel" || req.Key == "transcoder.encoder" {
|
||||
svc.Transcoder.StopAll()
|
||||
}
|
||||
if req.Key == "cache.images_max_size_mb" || req.Key == "cache.images_originals_max_size_mb" || req.Key == "cache.images_originals_ttl_hours" {
|
||||
if req.Key == "cache.images_max_size_mb" || req.Key == "cache.images_originals_max_size_mb" {
|
||||
if svc.Scheduler != nil {
|
||||
_ = svc.Scheduler.RunNowAsync(c.Request.Context(), "image_cache_cleanup")
|
||||
_ = svc.Scheduler.RunNowAsync(c.Request.Context(), "cache_cleanup")
|
||||
}
|
||||
}
|
||||
if req.Key == "cache.memory_max_size_mb" && svc.Cache != nil {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func backgroundTasksHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if svc == nil || svc.Tasks == nil {
|
||||
c.JSON(http.StatusOK, service.TaskSnapshot{Active: []service.BackgroundTask{}, Recent: []service.BackgroundTask{}})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, svc.Tasks.Snapshot())
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
@@ -41,12 +42,28 @@ func deleteBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func downloadBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
filename := c.Query("filename")
|
||||
path, err := svc.Backup.ResolvePath(filename)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.FileAttachment(path, filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
func restoreBackupHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.Backup.Restore(c.Request.Context(), c.Query("filename")); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"message": "restored — restart the server to apply"})
|
||||
message := "数据库已恢复,请重启 MeBox 以刷新运行时配置"
|
||||
if svc.Repo.DB.Dialector.Name() == "sqlite" {
|
||||
message = "备份已校验并暂存,请重启 MeBox 后应用恢复"
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"message": message, "restart_required": true})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,10 +124,32 @@ func cloud115HLSSessionHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "missing media_id"})
|
||||
return
|
||||
}
|
||||
if !enforceScopedPlaybackToken(c, mediaID) {
|
||||
if c.Request.Context().Err() != nil {
|
||||
c.AbortWithStatus(499)
|
||||
return
|
||||
}
|
||||
err := svc.Cloud115.HLSProxy().ServeChild(
|
||||
actualMediaID, err := svc.Cloud115.HLSProxy().SessionMediaID(c.Param("session"), c.Param("key"))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"code": "hls_session_not_found", "error": "hls session not found"})
|
||||
return
|
||||
}
|
||||
if mediaID != actualMediaID {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "playback session media mismatch"})
|
||||
return
|
||||
}
|
||||
if !enforceScopedPlaybackToken(c, actualMediaID) {
|
||||
return
|
||||
}
|
||||
if svc.Repo == nil || svc.Repo.Media == nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "media not found"})
|
||||
return
|
||||
}
|
||||
m, findErr := svc.Repo.Media.FindByID(c.Request.Context(), actualMediaID)
|
||||
if findErr != nil || m == nil || !mediaVisibleForRequest(c, svc, m) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "media not found"})
|
||||
return
|
||||
}
|
||||
err = svc.Cloud115.HLSProxy().ServeChild(
|
||||
c.Request.Context(),
|
||||
c.Writer,
|
||||
c.Request,
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
// 云下载(115 离线下载)HTTP 接口。
|
||||
//
|
||||
// 全部挂在 /api/admin 下(管理员专用):首期不开放普通用户,否则单个用户
|
||||
// 可以把账号的 115 云下载配额打空(见 docs/cloud-download-plan.md 第 9 节)。
|
||||
package handler
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// cloudDownloadUploadHardLimit 是 .torrent 上传的硬上限(字节)。
|
||||
// 真实上限由设置项 cloud_download.torrent_max_mb 控制,这里只防内存打爆。
|
||||
const cloudDownloadUploadHardLimit = 256 << 20
|
||||
|
||||
func registerAdminCloudDownloadRoutes(admin *gin.RouterGroup, svc *service.Container) {
|
||||
admin.GET("/cloud-downloads", listCloudDownloadsHandler(svc))
|
||||
admin.POST("/cloud-downloads", createCloudDownloadsHandler(svc))
|
||||
admin.POST("/cloud-downloads/torrent", uploadCloudDownloadTorrentHandler(svc))
|
||||
admin.GET("/cloud-downloads/quota", cloudDownloadQuotaHandler(svc))
|
||||
admin.POST("/cloud-downloads/:id/retry", retryCloudDownloadHandler(svc))
|
||||
admin.POST("/cloud-downloads/:id/retry-pipeline", retryCloudDownloadPipelineHandler(svc))
|
||||
admin.POST("/cloud-downloads/:id/cancel", cancelCloudDownloadHandler(svc))
|
||||
admin.DELETE("/cloud-downloads/:id", deleteCloudDownloadHandler(svc))
|
||||
admin.POST("/cloud-downloads/clear-finished", clearFinishedCloudDownloadsHandler(svc))
|
||||
}
|
||||
|
||||
func listCloudDownloadsHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.Query("page"))
|
||||
pageSize, _ := strconv.Atoi(c.Query("page_size"))
|
||||
res, err := svc.CloudDownload.List(c.Request.Context(), repository.CloudDownloadQuery{
|
||||
Status: c.Query("status"),
|
||||
SyncPathID: c.Query("sync_path_id"),
|
||||
AccountID: c.Query("account_id"),
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, res)
|
||||
}
|
||||
}
|
||||
|
||||
func createCloudDownloadsHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
var req service.CloudDownloadSubmit
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
// 逐条结果里已经包含失败原因,因此部分失败仍返回 200,前端按 items 展示。
|
||||
res, err := svc.CloudDownload.Submit(c.Request.Context(), req)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, res)
|
||||
}
|
||||
}
|
||||
|
||||
func uploadCloudDownloadTorrentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, cloudDownloadUploadHardLimit)
|
||||
fileHeader, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "缺少种子文件(form 字段名 file):" + err.Error()})
|
||||
return
|
||||
}
|
||||
f, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "打开种子文件失败:" + err.Error()})
|
||||
return
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "读取种子文件失败:" + err.Error()})
|
||||
return
|
||||
}
|
||||
res, err := svc.CloudDownload.SubmitTorrent(c.Request.Context(), service.CloudDownloadTorrentRequest{
|
||||
AccountID: c.PostForm("account_id"),
|
||||
SyncPathID: c.PostForm("sync_path_id"),
|
||||
FileName: fileHeader.Filename,
|
||||
Data: data,
|
||||
})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, res)
|
||||
}
|
||||
}
|
||||
|
||||
func cloudDownloadQuotaHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
quota, err := svc.CloudDownload.Quota(c.Request.Context(), c.Query("account_id"))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, quota)
|
||||
}
|
||||
}
|
||||
|
||||
func retryCloudDownloadHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.CloudDownload.Retry(c.Request.Context(), c.Param("id")); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
}
|
||||
|
||||
func retryCloudDownloadPipelineHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.CloudDownload.RetryPipeline(c.Request.Context(), c.Param("id")); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
}
|
||||
|
||||
func cancelCloudDownloadHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.CloudDownload.Cancel(c.Request.Context(), c.Param("id")); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
}
|
||||
|
||||
func deleteCloudDownloadHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if err := svc.CloudDownload.Delete(c.Request.Context(), c.Param("id")); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
}
|
||||
}
|
||||
|
||||
func clearFinishedCloudDownloadsHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
n, err := svc.CloudDownload.ClearFinished(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"deleted": n})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// 客户端刷新首页时会用 IncludeItemTypes=Movie,Series,Video 一次取「最新」行。
|
||||
// 电影库必须返回电影,而不是因为请求里带了 Series 就整行判空。
|
||||
func TestEmbyItemsMixedTypesKeepsMoviesInMovieLibrary(t *testing.T) {
|
||||
router, svc, _ := newEmbyDiscoveryEnv(t)
|
||||
if err := svc.Repo.DB.AutoMigrate(&model.Series{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
libID := seedEmbyLibrary(t, svc, "movie")
|
||||
if err := svc.Repo.DB.WithContext(context.Background()).Create(&model.Media{
|
||||
Base: model.Base{ID: "movie-1"},
|
||||
LibraryID: libID,
|
||||
Title: "Movie One",
|
||||
Path: "/media/movie/one.mp4",
|
||||
Year: 2024,
|
||||
ReleaseDate: "2024-01-01",
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create movie: %v", err)
|
||||
}
|
||||
|
||||
path := "/emby/Items?ParentId=" + libID +
|
||||
"&IncludeItemTypes=Movie,Series,Video&Recursive=true" +
|
||||
"&SortBy=DateCreated,SortName&SortOrder=Descending&Limit=12&StartIndex=0"
|
||||
w := embyGet(t, router, path, signedTestToken(t, "test-secret"))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status: %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
items := decodeItemsEnvelope(t, w.Body.Bytes())
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected the movie to be returned, got %d items: %s", len(items), w.Body.String())
|
||||
}
|
||||
if items[0]["Id"] != "movie-1" || items[0]["Type"] != "Movie" {
|
||||
t.Fatalf("unexpected item payload: %#v", items[0])
|
||||
}
|
||||
}
|
||||
|
||||
// 同一组混合类型在剧集库上仍要折叠成剧集卡片,修复不能把这条路径弄回单集。
|
||||
func TestEmbyItemsMixedTypesKeepsSeriesCardsInTvLibrary(t *testing.T) {
|
||||
router, svc, _ := newEmbyDiscoveryEnv(t)
|
||||
if err := svc.Repo.DB.AutoMigrate(&model.Series{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
libID := seedEmbyLibrary(t, svc, "tv")
|
||||
for _, ep := range []struct {
|
||||
id string
|
||||
num int
|
||||
path string
|
||||
}{
|
||||
{id: "ep-1", num: 1, path: "/media/tv/show/S01E01.mkv"},
|
||||
{id: "ep-2", num: 2, path: "/media/tv/show/S01E02.mkv"},
|
||||
} {
|
||||
if err := svc.Repo.DB.WithContext(context.Background()).Create(&model.Media{
|
||||
Base: model.Base{ID: ep.id},
|
||||
LibraryID: libID,
|
||||
SeriesID: "series-1",
|
||||
Title: "Show",
|
||||
Path: ep.path,
|
||||
SeasonNum: 1,
|
||||
EpisodeNum: ep.num,
|
||||
Year: 2024,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("create episode: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
path := "/emby/Items?ParentId=" + libID +
|
||||
"&IncludeItemTypes=Movie,Series,Video&Recursive=true" +
|
||||
"&SortBy=DateCreated,SortName&SortOrder=Descending&Limit=12&StartIndex=0"
|
||||
w := embyGet(t, router, path, signedTestToken(t, "test-secret"))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status: %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
items := decodeItemsEnvelope(t, w.Body.Bytes())
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected one series card, got %d items: %s", len(items), w.Body.String())
|
||||
}
|
||||
if items[0]["Type"] != "Series" {
|
||||
t.Fatalf("expected Series card, got %#v", items[0])
|
||||
}
|
||||
}
|
||||
|
||||
// Emby 的 SpecialFeatures 响应是 BaseItemDto[],客户端按 List 强转;
|
||||
// 返回 {"Items":[]} 会直接抛 type cast 异常。
|
||||
func TestEmbySpecialFeaturesReturnsBareArray(t *testing.T) {
|
||||
router, _, _ := newEmbyDiscoveryEnv(t)
|
||||
w := embyGet(t, router, "/emby/Items/movie-1/SpecialFeatures", signedTestToken(t, "test-secret"))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status: %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if trimmed := bytes.TrimSpace(w.Body.Bytes()); len(trimmed) == 0 || trimmed[0] != '[' {
|
||||
t.Fatalf("SpecialFeatures must be a JSON array, got %s", w.Body.String())
|
||||
}
|
||||
var items []any
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &items); err != nil {
|
||||
t.Fatalf("decode array: %v body=%s", err, w.Body.String())
|
||||
}
|
||||
if len(items) != 0 {
|
||||
t.Fatalf("expected empty array, got %d entries", len(items))
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,25 @@ func embyServeImage(c *gin.Context, svc *service.Container, id, imageType, tag s
|
||||
embyServePlaceholderImage(c)
|
||||
return
|
||||
}
|
||||
// 媒体库主图:按 Emby 的做法,用库内几张成员海报拼一张封面(folder
|
||||
// image)。没有自定义封面时才生成;自定义封面继续走下面的代理路径。
|
||||
if svc.ImageProxy != nil && strings.EqualFold(strings.TrimSpace(imageType), "Primary") {
|
||||
if sources, lib := service.LibraryCoverSourcesForLibrary(ctx, svc.Repo, id); lib != nil &&
|
||||
strings.TrimSpace(lib.CoverURL) == "" && len(sources) > 0 {
|
||||
width, height := libraryCoverRequestSize(c)
|
||||
if data, err := svc.ImageProxy.LibraryCoverImage(ctx, id, sources, width, height); err == nil && len(data) > 0 {
|
||||
c.Header("Cache-Control", "public, max-age=300, stale-while-revalidate=86400")
|
||||
c.Header("Content-Type", "image/jpeg")
|
||||
c.Header("Content-Length", strconv.Itoa(len(data)))
|
||||
if c.Request.Method == http.MethodHead {
|
||||
c.Status(http.StatusOK)
|
||||
return
|
||||
}
|
||||
c.Data(http.StatusOK, "image/jpeg", data)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// PersonImageURL handles TMDb/remote people first and falls back to regular
|
||||
// media/library artwork, so the official /Items/{personId}/Images route
|
||||
// works for synthetic person IDs as well as normal item IDs.
|
||||
|
||||
@@ -86,6 +86,12 @@ func embySegmentItems(m *model.Media, rows []model.MediaSegment, want map[string
|
||||
}
|
||||
endMs := row.EndMs
|
||||
if endMs <= 0 {
|
||||
// 只有片尾类允许「延续到片尾」。片头/回顾的结束点缺失意味着这段区间
|
||||
// 本身不可信(数据源用 0 表示「这段不存在」),补成片长会让客户端把
|
||||
// 整集标成片头,所以直接丢弃。
|
||||
if row.Kind != model.SegmentKindCredits && row.Kind != model.SegmentKindPreview {
|
||||
continue
|
||||
}
|
||||
if durationMs <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -418,7 +418,11 @@ func embyVideoHLSPlaylistHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"))
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"), scope)
|
||||
if errors.Is(err, service.ErrTranscodeDisabled) {
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "transcode disabled"})
|
||||
return
|
||||
@@ -444,7 +448,11 @@ func embyVideoHLSSegmentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if err := svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg")); err != nil {
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg"), scope); err != nil {
|
||||
c.Status(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,9 +212,11 @@ func registerEmbyAuthenticatedItemRoutes(auth *gin.RouterGroup, svc *service.Con
|
||||
auth.GET("/Items/:id/Similar", embySimilarHandler(svc))
|
||||
auth.GET("/Items/:id/ThumbnailSet", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/Items/:id/ThemeMedia", embyThemeMediaHandler(svc))
|
||||
auth.GET("/Users/:userId/Items/:id/SpecialFeatures", embyEmptyItemsHandler(svc))
|
||||
// SpecialFeatures 在 Emby 官方 schema 里是 BaseItemDto[](裸数组),
|
||||
// 返回 {"Items":[]} 会让按 List 强转的客户端抛 type cast 异常。
|
||||
auth.GET("/Users/:userId/Items/:id/SpecialFeatures", embyEmptyArrayHandler(svc))
|
||||
auth.GET("/Users/:userId/Items/:id/Intros", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/Items/:id/SpecialFeatures", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/Items/:id/SpecialFeatures", embyEmptyArrayHandler(svc))
|
||||
auth.GET("/Items/:id/Intros", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/api/danmu/:id/raw", embyDanmuRawHandler(svc))
|
||||
}
|
||||
|
||||
@@ -55,9 +55,9 @@ func registerLowercaseEmbyItemRoutes(auth *gin.RouterGroup, svc *service.Contain
|
||||
auth.GET("/items/:id/similar", embySimilarHandler(svc))
|
||||
auth.GET("/items/:id/thumbnailset", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/items/:id/thememedia", embyThemeMediaHandler(svc))
|
||||
auth.GET("/users/:userId/items/:id/specialfeatures", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/users/:userId/items/:id/specialfeatures", embyEmptyArrayHandler(svc))
|
||||
auth.GET("/users/:userId/items/:id/intros", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/items/:id/specialfeatures", embyEmptyItemsHandler(svc))
|
||||
auth.GET("/items/:id/specialfeatures", embyEmptyArrayHandler(svc))
|
||||
auth.GET("/items/:id/intros", embyEmptyItemsHandler(svc))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func TestStopHLSChecksMediaVisibility(t *testing.T) {
|
||||
router, svc, secret := newPlaybackScopeTestRouter(t)
|
||||
svc.Transcoder = service.NewTranscoderService(svc.Cfg, zap.NewNop(), svc.Repo, nil)
|
||||
router.DELETE("/api/hls/:id", middleware.AuthRequired(secret), stopTranscodeHandler(svc))
|
||||
req := httptest.NewRequest(http.MethodDelete, "/api/hls/nonexistent?session_id=player-a", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+signedTestToken(t, secret))
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("stop of a nonexistent/invisible media: status=%d body=%s, want 404", rec.Code, rec.Body.String())
|
||||
}
|
||||
profile := &model.PlayProfile{UserID: "user-1", Name: "restricted", IsDefault: true, AllowedLibraryIDs: `["other-library"]`}
|
||||
if err := svc.Repo.DB.Create(profile).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodDelete, "/api/hls/media-1?session_id=player-a", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+signedTestToken(t, secret))
|
||||
rec = httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("invisible existing media can be stopped: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// libraryCoverHandler 输出媒体库封面图(Emby 风格的成员海报拼图)。
|
||||
//
|
||||
// 之前 Web 端是拿两张成员海报在前端用 CSS 拼格子,Emby 客户端则直接拿一张
|
||||
// 评分最高的海报,结果同一个库在不同客户端长得完全不一样。现在服务端生成
|
||||
// 一张 16:9 的拼图(Emby 的 folder image 做法),Emby 客户端、Web 和 App
|
||||
// 都指向这一份,外观统一。
|
||||
func libraryCoverHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
if id == "" || svc == nil || svc.ImageProxy == nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "cover unavailable"})
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(c.Request.Context(), 25*time.Second)
|
||||
defer cancel()
|
||||
|
||||
sources, lib := service.LibraryCoverSourcesForLibrary(ctx, svc.Repo, id)
|
||||
if lib == nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "library not found"})
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(lib.CoverURL) != "" {
|
||||
// 管理员自定义封面优先,保持既有行为(代理该地址)。
|
||||
if err := svc.ImageProxy.Serve(ctx, c.Writer, c.Request, lib.CoverURL); err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "custom cover unavailable"})
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(sources) == 0 {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "no library items with artwork"})
|
||||
return
|
||||
}
|
||||
width, height := libraryCoverRequestSize(c)
|
||||
data, err := svc.ImageProxy.LibraryCoverImage(ctx, id, sources, width, height)
|
||||
if err != nil || len(data) == 0 {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "cover generation failed"})
|
||||
return
|
||||
}
|
||||
writeLibraryCoverImage(c, data, http.StatusOK)
|
||||
}
|
||||
}
|
||||
|
||||
// libraryCoverRequestSize 解析请求的封面尺寸:Web 用 w/h,Emby 用
|
||||
// maxWidth/maxHeight,两种都兼容。服务端统一输出 16:9 画布(与该库在 Emby
|
||||
// 里上报的 PrimaryImageAspectRatio 一致),客户端传的尺寸只当作「不超过」的
|
||||
// 边界使用。
|
||||
func libraryCoverRequestSize(c *gin.Context) (int, int) {
|
||||
return firstPositiveQueryIntParam(c, "maxWidth", "maxwidth", "width", "w"),
|
||||
firstPositiveQueryIntParam(c, "maxHeight", "maxheight", "height", "h")
|
||||
}
|
||||
|
||||
func firstPositiveQueryIntParam(c *gin.Context, names ...string) int {
|
||||
for _, name := range names {
|
||||
raw := strings.TrimSpace(c.Query(name))
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
if n, err := strconv.Atoi(raw); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
if f, err := strconv.ParseFloat(raw, 64); err == nil && f > 0 {
|
||||
return int(f)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func writeLibraryCoverImage(c *gin.Context, data []byte, status int) {
|
||||
sum := sha256.Sum256(data)
|
||||
etag := `"` + hex.EncodeToString(sum[:8]) + `"`
|
||||
c.Header("ETag", etag)
|
||||
c.Header("Cache-Control", "public, max-age=300, stale-while-revalidate=86400")
|
||||
if match := strings.TrimSpace(c.GetHeader("If-None-Match")); match != "" && match == etag {
|
||||
c.Status(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
c.Header("Content-Type", "image/jpeg")
|
||||
c.Header("Content-Length", strconv.Itoa(len(data)))
|
||||
if c.Request.Method == http.MethodHead {
|
||||
c.Status(status)
|
||||
return
|
||||
}
|
||||
c.Data(status, "image/jpeg", data)
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func coverTestToken(t *testing.T, secret, userID string) string {
|
||||
t.Helper()
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, middleware.Claims{
|
||||
UserID: userID,
|
||||
Role: "admin",
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)),
|
||||
IssuedAt: jwt.NewNumericDate(time.Now()),
|
||||
Subject: userID,
|
||||
},
|
||||
})
|
||||
signed, err := token.SignedString([]byte(secret))
|
||||
if err != nil {
|
||||
t.Fatalf("sign token: %v", err)
|
||||
}
|
||||
return signed
|
||||
}
|
||||
|
||||
// TestLibraryCoverRouteServesImageThroughFullRouter 走完整的 handler.Register
|
||||
// 路由表请求封面:既验证路由挂载正确,也验证鉴权链(AuthRequired +
|
||||
// activeUserRequired)不会把它挡成 404。
|
||||
func TestLibraryCoverRouteServesImageThroughFullRouter(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
const secret = "test-secret"
|
||||
|
||||
poster := coverTestJPEG(t, color.RGBA{R: 200, G: 120, B: 60, A: 255})
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(poster)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
parsed, _ := url.Parse(upstream.URL)
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.User{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
user := model.User{Base: model.Base{ID: "user-1"}, Username: "viewer", IsActive: true}
|
||||
if err := db.Create(&user).Error; err != nil {
|
||||
t.Fatalf("create user: %v", err)
|
||||
}
|
||||
lib := model.Library{Name: "电影", Path: "/media/movies", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatalf("create library: %v", err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
row := model.Media{
|
||||
Base: model.Base{ID: "m" + string(rune('a'+i))},
|
||||
LibraryID: lib.ID,
|
||||
Title: "Movie " + string(rune('A'+i)),
|
||||
Path: "/media/movies/m" + string(rune('a'+i)) + ".mkv",
|
||||
PosterURL: upstream.URL + "/p" + string(rune('1'+i)) + ".jpg",
|
||||
}
|
||||
if err := db.Create(&row).Error; err != nil {
|
||||
t.Fatalf("create media: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
cfg := &config.Config{
|
||||
Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")},
|
||||
Secrets: config.SecretsConfig{JWTSecret: secret},
|
||||
}
|
||||
proxy := service.NewImageProxy(cfg, zap.NewNop())
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string { return []string{parsed.Host} })
|
||||
svc := &service.Container{Repo: repos, ImageProxy: proxy}
|
||||
|
||||
engine := gin.New()
|
||||
Register(engine, cfg, zap.NewNop(), svc)
|
||||
|
||||
token := coverTestToken(t, secret, user.ID)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/libraries/"+lib.ID+"/cover?w=480&h=270&api_key="+token, nil)
|
||||
w := httptest.NewRecorder()
|
||||
engine.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if ct := w.Header().Get("Content-Type"); ct != "image/jpeg" {
|
||||
t.Fatalf("content type = %q, want image/jpeg", ct)
|
||||
}
|
||||
img, err := jpeg.Decode(bytes.NewReader(w.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("decode cover: %v", err)
|
||||
}
|
||||
if img.Bounds().Dx() != 480 || img.Bounds().Dy() != 270 {
|
||||
t.Fatalf("cover size = %v, want 480x270", img.Bounds())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func coverTestJPEG(t *testing.T, c color.RGBA) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, 160, 240))
|
||||
for y := 0; y < 240; y++ {
|
||||
for x := 0; x < 160; x++ {
|
||||
img.Set(x, y, c)
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil {
|
||||
t.Fatalf("encode jpeg: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// newLibraryCoverTestRouter 构造一个带真实图片代理的路由:成员海报来自
|
||||
// 本地 httptest 服务,走的是与生产一致的 ImageProxy 抓取路径。
|
||||
func newLibraryCoverTestRouter(t *testing.T) (*gin.Engine, *repository.Container, *httptest.Server) {
|
||||
t.Helper()
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
poster := coverTestJPEG(t, color.RGBA{R: 200, G: 120, B: 60, A: 255})
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(poster)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
|
||||
parsed, err := url.Parse(upstream.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse upstream: %v", err)
|
||||
}
|
||||
cfg := &config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}
|
||||
proxy := service.NewImageProxy(cfg, zap.NewNop())
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string { return []string{parsed.Host} })
|
||||
|
||||
svc := &service.Container{Repo: repos, ImageProxy: proxy}
|
||||
router := gin.New()
|
||||
router.GET("/libraries/:id/cover", libraryCoverHandler(svc))
|
||||
router.HEAD("/libraries/:id/cover", libraryCoverHandler(svc))
|
||||
return router, repos, upstream
|
||||
}
|
||||
|
||||
func seedCoverLibrary(t *testing.T, repos *repository.Container, posterBase string, coverURL string) model.Library {
|
||||
t.Helper()
|
||||
lib := model.Library{Name: "电影", Path: "/media/movies", Type: "movie", Enabled: true, CoverURL: coverURL}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatalf("create library: %v", err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
row := model.Media{
|
||||
Base: model.Base{ID: "m" + string(rune('a'+i))},
|
||||
LibraryID: lib.ID,
|
||||
Title: "Movie " + string(rune('A'+i)),
|
||||
Path: "/media/movies/m" + string(rune('a'+i)) + ".mkv",
|
||||
PosterURL: posterBase + "/p" + string(rune('1'+i)) + ".jpg",
|
||||
}
|
||||
if err := repos.DB.Create(&row).Error; err != nil {
|
||||
t.Fatalf("create media: %v", err)
|
||||
}
|
||||
}
|
||||
return lib
|
||||
}
|
||||
|
||||
// TestLibraryCoverHandlerServesGeneratedCollage 覆盖 Web/App 的封面接口:
|
||||
// 返回的是服务端拼好的 16:9 图片,而不是单张成员海报。
|
||||
func TestLibraryCoverHandlerServesGeneratedCollage(t *testing.T) {
|
||||
router, repos, upstream := newLibraryCoverTestRouter(t)
|
||||
lib := seedCoverLibrary(t, repos, upstream.URL, "")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/libraries/"+lib.ID+"/cover?w=480&h=270", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if ct := w.Header().Get("Content-Type"); ct != "image/jpeg" {
|
||||
t.Fatalf("content type = %q, want image/jpeg", ct)
|
||||
}
|
||||
img, err := jpeg.Decode(bytes.NewReader(w.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("decode cover: %v", err)
|
||||
}
|
||||
if img.Bounds().Dx() != 480 || img.Bounds().Dy() != 270 {
|
||||
t.Fatalf("cover size = %v, want 480x270", img.Bounds())
|
||||
}
|
||||
if etag := w.Header().Get("ETag"); etag == "" {
|
||||
t.Fatal("expected an ETag so clients can revalidate")
|
||||
}
|
||||
|
||||
// 条件请求应拿到 304。
|
||||
req = httptest.NewRequest(http.MethodGet, "/libraries/"+lib.ID+"/cover?w=480&h=270", nil)
|
||||
req.Header.Set("If-None-Match", w.Header().Get("ETag"))
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotModified {
|
||||
t.Fatalf("conditional request status = %d, want 304", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCoverHandlerUnknownLibraryReturns404(t *testing.T) {
|
||||
router, _, _ := newLibraryCoverTestRouter(t)
|
||||
req := httptest.NewRequest(http.MethodGet, "/libraries/missing/cover", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want 404", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLibraryCoverHandlerServesCustomCover 管理员填了 cover_url 时必须继续
|
||||
// 用自定义封面,不能在不知不觉中换成拼图。
|
||||
func TestLibraryCoverHandlerServesCustomCover(t *testing.T) {
|
||||
router, repos, upstream := newLibraryCoverTestRouter(t)
|
||||
lib := seedCoverLibrary(t, repos, upstream.URL, upstream.URL+"/custom.jpg")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/libraries/"+lib.ID+"/cover", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if _, err := jpeg.Decode(bytes.NewReader(w.Body.Bytes())); err != nil {
|
||||
t.Fatalf("custom cover should be an image: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmbyLibraryPrimaryUsesGeneratedCollage 保证 Emby 客户端请求库主图时
|
||||
// 拿到的也是拼图(与 Web/App 同一份生成逻辑)。
|
||||
func TestEmbyLibraryPrimaryUsesGeneratedCollage(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
poster := coverTestJPEG(t, color.RGBA{G: 180, R: 90, B: 60, A: 255})
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(poster)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
parsed, _ := url.Parse(upstream.URL)
|
||||
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "剧集", Path: "/media/tv", Type: "tv", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatalf("create library: %v", err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
row := model.Media{
|
||||
Base: model.Base{ID: "e" + string(rune('a'+i))},
|
||||
LibraryID: lib.ID,
|
||||
Title: "Show " + string(rune('A'+i)),
|
||||
Path: "/media/tv/show" + string(rune('a'+i)) + "/S01E01.mkv",
|
||||
SeasonNum: 1, EpisodeNum: 1,
|
||||
PosterURL: upstream.URL + "/p" + string(rune('1'+i)) + ".jpg",
|
||||
}
|
||||
if err := repos.DB.Create(&row).Error; err != nil {
|
||||
t.Fatalf("create media: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
cfg := &config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}
|
||||
proxy := service.NewImageProxy(cfg, zap.NewNop())
|
||||
proxy.SetAllowedRemoteHostsProvider(func() []string { return []string{parsed.Host} })
|
||||
svc := &service.Container{
|
||||
Repo: repos,
|
||||
Emby: service.NewEmbyService(cfg, zap.NewNop(), repos),
|
||||
ImageProxy: proxy,
|
||||
}
|
||||
router := gin.New()
|
||||
registerEmbyRoutes(router, "test-secret", svc)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/Items/"+lib.ID+"/Images/Primary?maxWidth=400&maxHeight=300", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
img, err := jpeg.Decode(bytes.NewReader(w.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("decode emby library cover: %v", err)
|
||||
}
|
||||
// 请求的是 400×300 的盒子,拼图按 16:9 装进去 → 400×225。
|
||||
if img.Bounds().Dx() != 400 || img.Bounds().Dy() != 225 {
|
||||
t.Fatalf("cover size = %v, want 400x225 (16:9 inside the requested box)", img.Bounds())
|
||||
}
|
||||
}
|
||||
+282
-115
@@ -4,8 +4,11 @@ package handler
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -38,8 +41,32 @@ type webLibraryPayload struct {
|
||||
RemoteSource string `json:"remote_source,omitempty"`
|
||||
Total int64 `json:"total,omitempty"`
|
||||
Cards []service.SeriesCardView `json:"cards,omitempty"`
|
||||
// CoverImageURL 是客户端可直接使用的「有效封面」:管理员自定义的
|
||||
// cover_url 优先,否则指向服务端生成的成员海报拼图接口。Web / App 都
|
||||
// 读这一个字段,避免各自用卡片在前端拼格子、长得还不一样。
|
||||
CoverImageURL string `json:"cover_image_url,omitempty"`
|
||||
}
|
||||
|
||||
// effectiveLibraryCoverURL 计算库封面的有效地址。远程挂载库的封面由上游
|
||||
// Emby 决定,这里返回空串,调用方继续用卡片拼图兜底。
|
||||
func effectiveLibraryCoverURL(lib model.Library, isRemote bool) string {
|
||||
if isRemote {
|
||||
return ""
|
||||
}
|
||||
if raw := strings.TrimSpace(lib.CoverURL); raw != "" {
|
||||
return raw
|
||||
}
|
||||
if strings.TrimSpace(lib.ID) == "" {
|
||||
return ""
|
||||
}
|
||||
return "/api/libraries/" + url.PathEscape(lib.ID) + "/cover"
|
||||
}
|
||||
|
||||
// remoteLibraryPreviewBudget 限制 /api/libraries?with_preview=1 里远程 Emby
|
||||
// 库预览(总条数 + 最新卡片)的总耗时。这些预览只用于首页卡片,值得为它
|
||||
// 排队等待的量很小;此前它跟随请求上下文,上游超时会让整页等 15 秒以上。
|
||||
var remoteLibraryPreviewBudget = 3 * time.Second
|
||||
|
||||
// remoteLibraryItemTypes 远程库内容拉取时按 CollectionType 过滤直属条目,
|
||||
// 避免电影库里的合集文件夹(Folder) 漏出为电影卡片。
|
||||
func remoteLibraryItemTypes(collectionType string) string {
|
||||
@@ -117,9 +144,10 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
for _, p := range previews {
|
||||
out = append(out, webLibraryPayload{
|
||||
Library: p.Library,
|
||||
Total: p.Total,
|
||||
Cards: service.NewSeriesCardViews(p.Cards),
|
||||
Library: p.Library,
|
||||
Total: p.Total,
|
||||
Cards: service.NewSeriesCardViews(p.Cards),
|
||||
CoverImageURL: effectiveLibraryCoverURL(p.Library, false),
|
||||
})
|
||||
}
|
||||
} else {
|
||||
@@ -138,7 +166,7 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
if counts != nil {
|
||||
total = counts[l.ID]
|
||||
}
|
||||
out = append(out, webLibraryPayload{Library: l, Total: total})
|
||||
out = append(out, webLibraryPayload{Library: l, Total: total, CoverImageURL: effectiveLibraryCoverURL(l, false)})
|
||||
}
|
||||
}
|
||||
// 精确指定目标库时,如果目标全是本地库,就不必枚举远程挂载。
|
||||
@@ -176,6 +204,12 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
remotePayloads[i] = webLibraryPayload{Library: v.Library, IsRemoteEmby: true, RemoteSource: v.AccountName}
|
||||
}
|
||||
if withPreview && len(allowedViews) > 0 {
|
||||
// 远程库预览只是首页卡片上的「最新几条」,绝不能拖住整个
|
||||
// 库列表:上游 Emby 一旦超时,每个远程库都要等满
|
||||
// embyRemoteHTTPTimeout(15s),串起来就是 17–30 秒。
|
||||
// 给整段预览一个总预算,超时就只返回库名,卡片留给下一次刷新。
|
||||
previewCtx, cancelPreview := context.WithTimeout(ctx, remoteLibraryPreviewBudget)
|
||||
defer cancelPreview()
|
||||
const maxRemotePreviewWorkers = 6
|
||||
sem := make(chan struct{}, maxRemotePreviewWorkers)
|
||||
var wg sync.WaitGroup
|
||||
@@ -187,11 +221,11 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
defer func() { <-sem }()
|
||||
case <-ctx.Done():
|
||||
case <-previewCtx.Done():
|
||||
return
|
||||
}
|
||||
helper.Run(svc.Log, "media.remotePreview", func() {
|
||||
acct := svc.EmbyRemote.AccountByID(ctx, v.AccountID)
|
||||
acct := svc.EmbyRemote.AccountByID(previewCtx, v.AccountID)
|
||||
if acct == nil {
|
||||
return
|
||||
}
|
||||
@@ -203,10 +237,10 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
|
||||
Name: v.Library.Name,
|
||||
}
|
||||
itemTypes := remoteLibraryItemTypes(v.CollectionType)
|
||||
if _, total, err := svc.EmbyRemote.RemoteLibraryMedia(ctx, tmpMount, acct, v.RemoteID, itemTypes, 0, 1); err == nil {
|
||||
if _, total, err := svc.EmbyRemote.RemoteLibraryMedia(previewCtx, tmpMount, acct, v.RemoteID, itemTypes, 0, 1); err == nil {
|
||||
remotePayloads[i].Total = total
|
||||
}
|
||||
if cards, err := svc.EmbyRemote.RemoteLatestCards(ctx, tmpMount, acct, v.RemoteID, limit); err == nil {
|
||||
if cards, err := svc.EmbyRemote.RemoteLatestCards(previewCtx, tmpMount, acct, v.RemoteID, limit); err == nil {
|
||||
remotePayloads[i].Cards = service.NewSeriesCardViews(cards)
|
||||
}
|
||||
})
|
||||
@@ -259,9 +293,9 @@ func getLibraryHandler(svc *service.Container) gin.HandlerFunc {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, webLibraryPayload{Library: libs[0]})
|
||||
c.JSON(http.StatusOK, webLibraryPayload{Library: libs[0], CoverImageURL: effectiveLibraryCoverURL(libs[0], false)})
|
||||
} else {
|
||||
c.JSON(http.StatusOK, webLibraryPayload{Library: *lib})
|
||||
c.JSON(http.StatusOK, webLibraryPayload{Library: *lib, CoverImageURL: effectiveLibraryCoverURL(*lib, false)})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -654,6 +688,9 @@ func paginateSlice[T any](items []T, page, size int) []T {
|
||||
if len(items) == 0 {
|
||||
return []T{}
|
||||
}
|
||||
if page-1 > len(items)/size {
|
||||
return []T{}
|
||||
}
|
||||
start := (page - 1) * size
|
||||
if start >= len(items) {
|
||||
return []T{}
|
||||
@@ -668,127 +705,257 @@ func paginateSlice[T any](items []T, page, size int) []T {
|
||||
func searchMediaHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
q := c.Query("q")
|
||||
query := strings.TrimSpace(c.Query("q"))
|
||||
visibility := mediaVisibilityForRequest(c, svc)
|
||||
groupVersions := c.DefaultQuery("group_versions", "1") != "0"
|
||||
|
||||
fetchRemote := func(limit int) []model.Media {
|
||||
if svc.EmbyRemote == nil || strings.TrimSpace(q) == "" {
|
||||
return nil
|
||||
}
|
||||
remoteItems, _ := svc.EmbyRemote.RemoteSearchMedia(ctx, q, limit, visibility)
|
||||
return remoteItems
|
||||
filter := service.MediaSearchFilter{
|
||||
LibraryID: strings.TrimSpace(c.Query("library_id")),
|
||||
YearMin: boundedSearchYear(c.Query("year_min")),
|
||||
YearMax: boundedSearchYear(c.Query("year_max")),
|
||||
}
|
||||
|
||||
if c.DefaultQuery("group_series", "0") != "0" {
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
for _, genre := range c.QueryArray("genre") {
|
||||
if genre = strings.TrimSpace(genre); genre != "" && len(filter.Genres) < 8 {
|
||||
filter.Genres = append(filter.Genres, genre)
|
||||
}
|
||||
remoteItems := fetchRemote(50000)
|
||||
all := service.GroupMediaSeriesItems(append(localItems, remoteItems...))
|
||||
|
||||
if c.Query("page") != "" || c.Query("page_size") != "" {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
paged := paginateSlice(all, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(all),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": all})
|
||||
return
|
||||
}
|
||||
|
||||
if c.Query("page") != "" || c.Query("page_size") != "" {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if !groupVersions {
|
||||
localItems, _, err := svc.Media.SearchMediaVisiblePage(ctx, q, 1, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(size * 2)
|
||||
all := append(localItems, remoteItems...)
|
||||
paged := paginateSlice(all, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(all),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(size * 2)
|
||||
all := append(localItems, remoteItems...)
|
||||
grouped := service.GroupMediaVersions(all)
|
||||
paged := service.PaginateMediaItems(grouped, page, size)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paged,
|
||||
"total": len(grouped),
|
||||
"page": page,
|
||||
"page_size": size,
|
||||
})
|
||||
return
|
||||
if filter.YearMin > 0 && filter.YearMax > 0 && filter.YearMin > filter.YearMax {
|
||||
filter.YearMin, filter.YearMax = filter.YearMax, filter.YearMin
|
||||
}
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if size < 1 {
|
||||
size = 50
|
||||
} else if size > 2000 {
|
||||
size = 2000
|
||||
}
|
||||
|
||||
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||
if limit <= 0 {
|
||||
if limit < 1 {
|
||||
limit = 50
|
||||
} else if limit > 50000 {
|
||||
limit = 50000
|
||||
}
|
||||
if !groupVersions {
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, limit, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(limit)
|
||||
all := append(localItems, remoteItems...)
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
if all == nil {
|
||||
all = []model.Media{}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": all})
|
||||
return
|
||||
}
|
||||
|
||||
localItems, err := svc.Media.SearchMediaVisible(ctx, q, 50000, visibility)
|
||||
paginated := c.Query("page") != "" || c.Query("page_size") != ""
|
||||
sortBy := normalizeMediaSearchSort(c.Query("sort"))
|
||||
// 远程搜索与本地查询并行发起:本地结果不再排在远程超时之后才返回。
|
||||
remoteCh := startRemoteMediaSearch(ctx, svc, query, visibility, filter)
|
||||
// Sorting and grouping must operate on the same candidate set before
|
||||
// either a page size or the caller's output limit is applied.
|
||||
items, err := svc.Media.SearchMediaVisibleFiltered(ctx, query, 50000, visibility, filter)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
remoteItems := fetchRemote(limit)
|
||||
all := append(localItems, remoteItems...)
|
||||
grouped := service.GroupMediaVersions(all)
|
||||
items := service.FirstMediaItems(grouped, limit)
|
||||
if items == nil {
|
||||
items = []service.MediaItem{}
|
||||
if remoteCh != nil {
|
||||
for _, item := range <-remoteCh {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items})
|
||||
if c.DefaultQuery("group_series", "0") != "0" {
|
||||
items = service.GroupMediaSeriesItems(items)
|
||||
sortSearchMedia(items, sortBy)
|
||||
writeMediaSearchResults(c, items, page, size, limit, paginated)
|
||||
return
|
||||
}
|
||||
if c.DefaultQuery("group_versions", "1") != "0" {
|
||||
grouped := service.GroupMediaVersions(items)
|
||||
if sortBy != "" {
|
||||
sort.SliceStable(grouped, func(i, j int) bool {
|
||||
return mediaSearchSortLess(grouped[i].Media, grouped[j].Media, sortBy)
|
||||
})
|
||||
}
|
||||
writeMediaSearchResults(c, grouped, page, size, limit, paginated)
|
||||
return
|
||||
}
|
||||
sortSearchMedia(items, sortBy)
|
||||
writeMediaSearchResults(c, items, page, size, limit, paginated)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
// remoteSearchItemLimit 是每个远程 Emby 挂载点返回的条目上限。此前这里传
|
||||
// 50000(等于让上游做一次全库查询),上游稍慢就会一路跑到 5 秒超时,把每次
|
||||
// 搜索都拖成秒级;搜索场景只需要少量候选,多出来的结果也会被分页丢掉。
|
||||
remoteSearchItemLimit = 100
|
||||
// remoteSearchBudget 限制远程结果最多给一次请求增加多少延迟。
|
||||
remoteSearchBudget = 2 * time.Second
|
||||
// remoteSearchCacheTTL 让同一关键词的翻页与重复搜索复用远程结果,避免每翻
|
||||
// 一页都重新打一遍远程挂载点。
|
||||
remoteSearchCacheTTL = 30 * time.Second
|
||||
// remoteSearchFailureCacheTTL 用于上游超时/失败时的短缓存:部分结果只保留
|
||||
// 几秒,避免反复吃满 remoteSearchBudget,又不至于长期压住后续重试。
|
||||
remoteSearchFailureCacheTTL = 5 * time.Second
|
||||
)
|
||||
|
||||
// startRemoteMediaSearch 并发发起远程搜索,返回一个只投递一次结果的通道。
|
||||
// 不满足远程搜索条件时返回 nil。通道带缓冲,调用方即使提前返回也不会泄漏
|
||||
// goroutine;请求结束时请求上下文被取消,远程调用会随之退出。
|
||||
func startRemoteMediaSearch(
|
||||
ctx context.Context,
|
||||
svc *service.Container,
|
||||
query string,
|
||||
visibility service.MediaVisibility,
|
||||
filter service.MediaSearchFilter,
|
||||
) <-chan []model.Media {
|
||||
if svc == nil || svc.EmbyRemote == nil || query == "" {
|
||||
return nil
|
||||
}
|
||||
if filter.LibraryID != "" && !service.IsEmbyRemoteID(filter.LibraryID) {
|
||||
return nil
|
||||
}
|
||||
results := make(chan []model.Media, 1)
|
||||
go func() {
|
||||
results <- remoteMediaSearchItems(ctx, svc, query, visibility, filter)
|
||||
}()
|
||||
return results
|
||||
}
|
||||
|
||||
func remoteMediaSearchItems(
|
||||
ctx context.Context,
|
||||
svc *service.Container,
|
||||
query string,
|
||||
visibility service.MediaVisibility,
|
||||
filter service.MediaSearchFilter,
|
||||
) []model.Media {
|
||||
cacheKey := remoteSearchCacheKey(query, visibility)
|
||||
if svc.Cache != nil {
|
||||
if cached, ok := svc.Cache.GetObject(cacheKey); ok {
|
||||
if rows, ok := cached.([]model.Media); ok {
|
||||
return filterRemoteSearchItems(rows, filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
searchCtx, cancel := context.WithTimeout(ctx, remoteSearchBudget)
|
||||
defer cancel()
|
||||
remote, _ := svc.EmbyRemote.RemoteSearchMedia(searchCtx, query, remoteSearchItemLimit, visibility)
|
||||
if svc.Cache != nil {
|
||||
// 空结果也要缓存:否则查不到片名时每次都会重新等到 remoteSearchBudget。
|
||||
if remote == nil {
|
||||
remote = []model.Media{}
|
||||
}
|
||||
ttl := remoteSearchCacheTTL
|
||||
if searchCtx.Err() != nil {
|
||||
ttl = remoteSearchFailureCacheTTL
|
||||
}
|
||||
svc.Cache.SetObject(cacheKey, remote, ttl)
|
||||
}
|
||||
return filterRemoteSearchItems(remote, filter)
|
||||
}
|
||||
|
||||
func filterRemoteSearchItems(remote []model.Media, filter service.MediaSearchFilter) []model.Media {
|
||||
if len(remote) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]model.Media, 0, len(remote))
|
||||
for _, item := range remote {
|
||||
if remoteSearchMatchesFilter(item, filter) {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// remoteSearchCacheKey 按可见范围隔离远程搜索结果,避免把某个用户可见挂载
|
||||
// 点的结果复用给可见范围不同的用户。
|
||||
func remoteSearchCacheKey(query string, visibility service.MediaVisibility) string {
|
||||
allowed := append([]string(nil), visibility.AllowedLibraryIDs...)
|
||||
hidden := append([]string(nil), visibility.HiddenLibraryIDs...)
|
||||
sort.Strings(allowed)
|
||||
sort.Strings(hidden)
|
||||
return fmt.Sprintf("media:obj:remote-search:%t|%s|%s|%s",
|
||||
visibility.IncludeNSFW, strings.Join(allowed, ","), strings.Join(hidden, ","), query)
|
||||
}
|
||||
|
||||
func writeMediaSearchResults[T any](c *gin.Context, items []T, page, size, limit int, paginated bool) {
|
||||
if items == nil {
|
||||
items = []T{}
|
||||
}
|
||||
if paginated {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"items": paginateSlice(items, page, size),
|
||||
"total": len(items), "page": page, "page_size": size,
|
||||
})
|
||||
return
|
||||
}
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items})
|
||||
}
|
||||
|
||||
func remoteSearchMatchesFilter(item model.Media, filter service.MediaSearchFilter) bool {
|
||||
if filter.LibraryID != "" && item.LibraryID != filter.LibraryID && item.DisplayLibraryID != filter.LibraryID {
|
||||
return false
|
||||
}
|
||||
if filter.YearMin > 0 && item.Year < filter.YearMin || filter.YearMax > 0 && item.Year > filter.YearMax {
|
||||
return false
|
||||
}
|
||||
if len(filter.Genres) == 0 {
|
||||
return true
|
||||
}
|
||||
normalize := func(value string) string {
|
||||
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), " ", ""))
|
||||
}
|
||||
for _, genre := range strings.Split(strings.ReplaceAll(item.Genres, ",", ","), ",") {
|
||||
for _, wanted := range filter.Genres {
|
||||
if normalize(genre) == normalize(wanted) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeMediaSearchSort(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
switch value {
|
||||
case "title_asc", "year_desc", "rating_desc":
|
||||
return value
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func sortSearchMedia(items []model.Media, sortBy string) {
|
||||
if sortBy == "" {
|
||||
return
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
return mediaSearchSortLess(items[i], items[j], sortBy)
|
||||
})
|
||||
}
|
||||
|
||||
func mediaSearchSortLess(a, b model.Media, sortBy string) bool {
|
||||
switch sortBy {
|
||||
case "title_asc":
|
||||
left, right := strings.ToLower(strings.TrimSpace(a.Title)), strings.ToLower(strings.TrimSpace(b.Title))
|
||||
if left != right {
|
||||
return left < right
|
||||
}
|
||||
case "year_desc":
|
||||
if a.Year != b.Year {
|
||||
return a.Year > b.Year
|
||||
}
|
||||
case "rating_desc":
|
||||
if a.Rating != b.Rating {
|
||||
return a.Rating > b.Rating
|
||||
}
|
||||
}
|
||||
if !a.CreatedAt.Equal(b.CreatedAt) {
|
||||
return a.CreatedAt.After(b.CreatedAt)
|
||||
}
|
||||
return a.ID < b.ID
|
||||
}
|
||||
|
||||
func boundedSearchYear(value string) int {
|
||||
year, err := strconv.Atoi(strings.TrimSpace(value))
|
||||
if err != nil || year < 1870 || year > 2200 {
|
||||
return 0
|
||||
}
|
||||
return year
|
||||
}
|
||||
|
||||
func streamHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// TestListLibrariesRemotePreviewIsBounded 锁定远程库预览的总预算:
|
||||
// 上游 Emby 卡住时,/api/libraries?with_preview=1 必须按时返回远程库条目
|
||||
// (只是没有卡片),而不是跟着上游一起等十几秒。线上实测这个接口因此出现过
|
||||
// 17.4 秒、30 秒(前端放弃)的响应,设置页也跟着整页空转。
|
||||
func TestListLibrariesRemotePreviewIsBounded(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
oldBudget := remoteLibraryPreviewBudget
|
||||
remoteLibraryPreviewBudget = 200 * time.Millisecond
|
||||
t.Cleanup(func() { remoteLibraryPreviewBudget = oldBudget })
|
||||
|
||||
// Views / 用户列表正常返回,Preview(Items 查询)永远不返回:模拟上游半挂。
|
||||
release := make(chan struct{})
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
path := r.URL.Path
|
||||
switch {
|
||||
case strings.HasSuffix(path, "/Views"):
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Items": []map[string]any{{
|
||||
"Id": "view-1", "Name": "远程动画", "CollectionType": "tvshows",
|
||||
}},
|
||||
})
|
||||
return
|
||||
case strings.HasSuffix(path, "/Users"):
|
||||
// api_key 账号要先解析出远程用户 Id,否则请求路径会落到兜底 "0"。
|
||||
_ = json.NewEncoder(w).Encode([]map[string]any{{"Id": "uid-1", "Name": "admin"}})
|
||||
return
|
||||
case r.Method == http.MethodPost:
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"AccessToken": "fake-token",
|
||||
"User": map[string]any{"Id": "uid-1"},
|
||||
})
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-release:
|
||||
case <-r.Context().Done():
|
||||
case <-time.After(30 * time.Second):
|
||||
}
|
||||
w.WriteHeader(http.StatusGatewayTimeout)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
defer close(release)
|
||||
|
||||
// 唯一库名 + cache=shared:预览协程会并发读库(账号解析)。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.StrmAccount{}, &model.EmbyMount{}, &model.Setting{}, &model.User{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
if err := db.Create(&model.User{Base: model.Base{ID: "admin-1"}, Username: "admin", Role: "admin"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawConfig, _ := json.Marshal(map[string]string{"url": upstream.URL, "token": "fake-token"})
|
||||
acct := model.StrmAccount{
|
||||
Base: model.Base{ID: "acct-1"}, Name: "远程Emby", Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig), Enabled: true,
|
||||
}
|
||||
if err := repos.StrmAccount.Create(t.Context(), &acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mount := model.EmbyMount{
|
||||
Base: model.Base{ID: "mount-1"}, AccountID: acct.ID, RemoteViewID: "view-1",
|
||||
RemoteViewName: "远程动画", CollectionType: "tvshows", Enabled: true,
|
||||
}
|
||||
if err := repos.EmbyMount.Create(t.Context(), &mount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := &service.Container{
|
||||
Repo: repos,
|
||||
Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos),
|
||||
EmbyRemote: service.NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, service.NewCryptoService("", zap.NewNop())),
|
||||
Cache: service.NewRuntimeCacheService(&config.Config{}, zap.NewNop()),
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Set(middleware.CtxUserID, "admin-1")
|
||||
c.Set(middleware.CtxUserRole, "admin")
|
||||
c.Request = httptest.NewRequest(http.MethodGet, "/api/libraries?with_preview=1&include_hidden=1", nil)
|
||||
|
||||
start := time.Now()
|
||||
listLibrariesHandler(svc)(c)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if elapsed > 2*time.Second {
|
||||
t.Fatalf("request took %v with a hung upstream; the preview budget must bound it", elapsed)
|
||||
}
|
||||
var payload []webLibraryPayload
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("decode response: %v (%s)", err, w.Body.String())
|
||||
}
|
||||
var remote *webLibraryPayload
|
||||
for i := range payload {
|
||||
if payload[i].IsRemoteEmby {
|
||||
remote = &payload[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if remote == nil {
|
||||
t.Fatalf("remote library missing from the bounded response: %#v", payload)
|
||||
}
|
||||
if len(remote.Cards) != 0 || remote.Total != 0 {
|
||||
t.Fatalf("expected the preview to be skipped on timeout, got total=%d cards=%d", remote.Total, len(remote.Cards))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func mediaQualityHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "50"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
} else if page > 1000000 {
|
||||
page = 1000000
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 50
|
||||
} else if pageSize > 100 {
|
||||
pageSize = 100
|
||||
}
|
||||
issue := c.DefaultQuery("issue", "unmatched")
|
||||
if issue != "unmatched" && issue != "poster" && issue != "missing" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "未知检查分类"})
|
||||
return
|
||||
}
|
||||
items, total, err := svc.Media.QualityIssues(c.Request.Context(), issue, page, pageSize)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": items, "total": total, "page": page, "page_size": pageSize})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
func TestSearchMediaSortLimitFiltersAndDeepPagination(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sqlDB, _ := db.DB()
|
||||
defer sqlDB.Close()
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.User{}, &model.Setting{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
lib := model.Library{Name: "Review", Path: "/review", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := make([]model.Media, 2001)
|
||||
for i := range rows {
|
||||
rows[i] = model.Media{
|
||||
Base: model.Base{ID: fmt.Sprintf("review-%04d", i)}, LibraryID: lib.ID,
|
||||
Title: fmt.Sprintf("Review %04d", i), Path: fmt.Sprintf("/review/%04d.mp4", i), Year: 2020, Genres: "Comedy",
|
||||
}
|
||||
}
|
||||
rows[2000].Year = 2025
|
||||
rows[2000].Genres = "Drama"
|
||||
if err := db.CreateInBatches(rows, 100).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := &service.Container{Repo: repos, Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos)}
|
||||
cases := []struct {
|
||||
name, query, expectedID string
|
||||
count, total, page, size int
|
||||
}{
|
||||
{name: "sorted output limit", query: "sort=year_desc&limit=1", expectedID: "review-2000", count: 1},
|
||||
{name: "page past old 2000 cap", query: "sort=title_asc&page=2001&page_size=1", expectedID: "review-2000", count: 1, total: 2001, page: 2001, size: 1},
|
||||
{name: "genre filtering", query: "genre=Drama&library_id=" + lib.ID, expectedID: "review-2000", count: 1},
|
||||
{name: "normalized page", query: "page=0&page_size=0", count: 50, total: 2001, page: 1, size: 50},
|
||||
{name: "bounded page size", query: "page=1&page_size=99999", count: 2000, total: 2001, page: 1, size: 2000},
|
||||
{name: "huge page does not overflow", query: "page=" + strconv.Itoa(int(^uint(0)>>1)) + "&page_size=2000", count: 0, total: 2001, page: int(^uint(0) >> 1), size: 2000},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(w)
|
||||
ctx.Set(middleware.CtxUserID, "admin")
|
||||
ctx.Set(middleware.CtxUserRole, "admin")
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/media?q=Review&group_versions=0&"+tc.query, nil)
|
||||
searchMediaHandler(svc)(ctx)
|
||||
var response struct {
|
||||
Items []model.Media `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
Size int `json:"page_size"`
|
||||
}
|
||||
if w.Code != http.StatusOK || json.Unmarshal(w.Body.Bytes(), &response) != nil {
|
||||
t.Fatalf("search failed: status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if len(response.Items) != tc.count || response.Total != tc.total || response.Page != tc.page || response.Size != tc.size {
|
||||
t.Fatalf("response count=%d total=%d page=%d size=%d, want %#v", len(response.Items), response.Total, response.Page, response.Size, tc)
|
||||
}
|
||||
if tc.expectedID != "" && response.Items[0].ID != tc.expectedID {
|
||||
t.Fatalf("first ID=%q want %q", response.Items[0].ID, tc.expectedID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/middleware"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
)
|
||||
|
||||
// TestSearchMediaRemoteSearchBoundedAndCached 防止远程搜索再次退化:
|
||||
// 1. 每个挂载点的 Limit 必须是 remoteSearchItemLimit,而不是历史全库查询的 50000;
|
||||
// 2. 同一关键词的重复搜索应命中缓存,不再打上游;
|
||||
// 3. 指定本地媒体库时不应枚举远程挂载点。
|
||||
func TestSearchMediaRemoteSearchBoundedAndCached(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
var mu sync.Mutex
|
||||
searchCalls := 0
|
||||
searchLimits := make([]string, 0, 4)
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if term := r.URL.Query().Get("SearchTerm"); term != "" {
|
||||
mu.Lock()
|
||||
searchCalls++
|
||||
searchLimits = append(searchLimits, r.URL.Query().Get("Limit"))
|
||||
mu.Unlock()
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"TotalRecordCount": 1,
|
||||
"Items": []map[string]any{{
|
||||
"Id": "remote-1",
|
||||
"Name": "远方剧集",
|
||||
"Type": "Series",
|
||||
"ProductionYear": 2024,
|
||||
}},
|
||||
})
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
|
||||
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.StrmAccount{}, &model.EmbyMount{}, &model.Setting{}, &model.User{}, &model.PlayProfile{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
if err := repos.DB.Create(&model.User{Base: model.Base{ID: "user-1"}, Username: "admin", Role: "admin"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lib := model.Library{Name: "本地电影", Path: "/media/movies", Type: "movie", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.Media{
|
||||
Base: model.Base{ID: "local-1"}, LibraryID: lib.ID, Title: "流浪地球", Path: "/media/movies/流浪地球.mkv",
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawConfig, _ := json.Marshal(map[string]string{"url": upstream.URL, "token": "fake-token"})
|
||||
acct := model.StrmAccount{
|
||||
Base: model.Base{ID: "acct"}, Name: "远程Emby", Provider: model.StrmProviderEmbyRemote,
|
||||
Config: string(rawConfig), Enabled: true,
|
||||
}
|
||||
if err := repos.StrmAccount.Create(t.Context(), &acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mount := model.EmbyMount{
|
||||
Base: model.Base{ID: "mount-1"}, AccountID: acct.ID, RemoteViewID: "view-1",
|
||||
RemoteViewName: "远程剧集", CollectionType: "tvshows", Enabled: true,
|
||||
}
|
||||
if err := repos.EmbyMount.Create(t.Context(), &mount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc := &service.Container{
|
||||
Repo: repos,
|
||||
Media: service.NewMediaService(&config.Config{}, zap.NewNop(), repos),
|
||||
EmbyRemote: service.NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, service.NewCryptoService("", zap.NewNop())),
|
||||
Cache: service.NewRuntimeCacheService(&config.Config{}, zap.NewNop()),
|
||||
}
|
||||
|
||||
search := func(params string) []service.MediaItem {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Set(middleware.CtxUserID, "user-1")
|
||||
c.Set(middleware.CtxUserRole, "admin")
|
||||
c.Request = httptest.NewRequest(http.MethodGet, "/api/media?"+params, nil)
|
||||
searchMediaHandler(svc)(c)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("search status=%d, body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var response struct {
|
||||
Items []service.MediaItem `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return response.Items
|
||||
}
|
||||
callCount := func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return searchCalls
|
||||
}
|
||||
|
||||
if items := search("q=远方&page=1&page_size=48"); len(items) != 1 || items[0].Title != "远方剧集" {
|
||||
t.Fatalf("expected remote result, got %#v", items)
|
||||
}
|
||||
if calls := callCount(); calls != 1 {
|
||||
t.Fatalf("expected exactly one upstream search, got %d", calls)
|
||||
}
|
||||
mu.Lock()
|
||||
limit := searchLimits[0]
|
||||
mu.Unlock()
|
||||
if limit != "100" {
|
||||
t.Fatalf("remote search Limit=%q, want bounded 100 (50000 means a full-library query)", limit)
|
||||
}
|
||||
|
||||
// 同一关键词再来一次(例如刷新/重复搜索)应复用缓存,不再请求上游。
|
||||
if items := search("q=远方&page=1&page_size=48"); len(items) != 1 || items[0].Title != "远方剧集" {
|
||||
t.Fatalf("cached search lost the remote result: %#v", items)
|
||||
}
|
||||
if calls := callCount(); calls != 1 {
|
||||
t.Fatalf("expected cached remote search to skip upstream, got %d calls", calls)
|
||||
}
|
||||
|
||||
// 指定本地媒体库时不应枚举远程挂载点。
|
||||
if items := search("q=流浪&page=1&page_size=48&library_id=" + lib.ID); len(items) != 1 || items[0].ID != "local-1" {
|
||||
t.Fatalf("expected local-only result, got %#v", items)
|
||||
}
|
||||
if calls := callCount(); calls != 1 {
|
||||
t.Fatalf("local library filter must not hit upstream, got %d calls", calls)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -27,13 +29,17 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
|
||||
"TotalRecordCount": 1,
|
||||
"Items": []map[string]any{{
|
||||
"Id": "same-series", "Name": "龙珠改", "Type": "Series",
|
||||
"ProviderIds": map[string]string{"Tmdb": "61709"},
|
||||
"ProviderIds": map[string]string{"Tmdb": "61709"},
|
||||
"ProductionYear": 2024, "Genres": []string{"Science Fiction"},
|
||||
}},
|
||||
})
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
|
||||
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -108,4 +114,9 @@ func TestSearchMediaSeriesKeepsRemoteLibrariesSeparate(t *testing.T) {
|
||||
}
|
||||
seen[item.DisplayLibraryID] = true
|
||||
}
|
||||
remoteLibraryID := service.EncodeEmbyRemoteID("mount-a", "view-a")
|
||||
items, total := request("page=1&page_size=8&library_id=" + url.QueryEscape(remoteLibraryID) + "&year_min=2020&genre=Science+Fiction")
|
||||
if total != 1 || len(items) != 1 || items[0].DisplayLibraryID != remoteLibraryID {
|
||||
t.Fatalf("remote library/year/genre filters lost results: total=%d items=%#v", total, items)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -559,7 +559,10 @@ func TestSearchMediaHandlerIncludesEmbyRemote(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
// 唯一库名 + cache=shared:远程搜索与本地查询会并发读库,
|
||||
// 而 :memory: 下每个连接各自一个空库,并发连接会读到「no such table」。
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
// Package handler — 片头/片尾片段接口。
|
||||
//
|
||||
// GET /playback/:id/segments
|
||||
// GET /playback/:id/segments
|
||||
// PUT /playback/:id/segments {kind, position_ms, clear}
|
||||
//
|
||||
// 单独开一个接口而不是塞进 /media/:id/playback,有两个原因:一是抓取外部数据
|
||||
// 可能要几秒,不能拖慢决定能否起播的那个请求;二是片段与播放来源(本地 / 云盘 /
|
||||
// 远程 Emby)无关,独立出来对所有媒体一致。
|
||||
//
|
||||
// 写入接口只处理「手动标记」:用户播放时把当前位置标成片头结束 / 片尾开始。
|
||||
// 外部数据永远由提供方刷新,不接受客户端写入。
|
||||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -42,6 +47,72 @@ func playbackSegmentsHandler(svc *service.Container) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// playbackSegmentsMarkRequest 是手动标记的入参。
|
||||
type playbackSegmentsMarkRequest struct {
|
||||
// Kind 只接受 intro / credits。
|
||||
Kind string `json:"kind" binding:"required"`
|
||||
// PositionMs 是当前播放位置:标片头时表示片头结束点,标片尾时表示片尾开始点。
|
||||
PositionMs int64 `json:"position_ms"`
|
||||
// Clear 为 true 时删除该类型的标记。
|
||||
Clear bool `json:"clear"`
|
||||
}
|
||||
|
||||
// playbackSegmentsMarkHandler 写入或清除一条手动标记,返回更新后的完整片段列表。
|
||||
//
|
||||
// 返回整份列表而不是只回 204:播放器需要立刻反映合并后的结果(手动标记优先级
|
||||
// 最高),避免自己再发一次 GET 造成竞态。
|
||||
func playbackSegmentsMarkHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if svc.Segments == nil {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "segments unavailable"})
|
||||
return
|
||||
}
|
||||
m, err := findMediaForPlaybackEndpoint(c, svc, c.Param("id"))
|
||||
if err != nil || m == nil || !mediaVisibleForRequest(c, svc, m) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "media not found"})
|
||||
return
|
||||
}
|
||||
// 远程 Emby 挂载是上游库的投影,本地标记没有可落地的地方。
|
||||
if service.IsEmbyRemoteID(m.ID) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "remote media cannot be marked"})
|
||||
return
|
||||
}
|
||||
var req playbackSegmentsMarkRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"})
|
||||
return
|
||||
}
|
||||
err = svc.Segments.SetManualMark(c.Request.Context(), m, service.ManualMarkRequest{
|
||||
Kind: req.Kind,
|
||||
PositionMs: req.PositionMs,
|
||||
Clear: req.Clear,
|
||||
})
|
||||
if errors.Is(err, service.ErrInvalidManualMark) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid skip mark"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
if svc.Log != nil {
|
||||
svc.Log.Warn("save manual skip mark failed",
|
||||
zap.String("media_id", m.ID), zap.Error(err))
|
||||
}
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "save failed"})
|
||||
return
|
||||
}
|
||||
rows, listErr := svc.Segments.ListForPlayback(c.Request.Context(), m)
|
||||
if listErr != nil {
|
||||
if svc.Log != nil {
|
||||
svc.Log.Debug("list media segments after mark failed",
|
||||
zap.String("media_id", m.ID), zap.Error(listErr))
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"segments": service.ToSegmentViews(rows),
|
||||
"auto_skip": resolveAutoSkipFlag(c, svc),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// resolveAutoSkipFlag reads the「自动跳过片头」switch off whichever profile is
|
||||
// currently in effect. It reuses selectedPlayProfile so the server agrees with
|
||||
// the UI about which profile is active (explicit header first, then the user's
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
"github.com/truewhile/MeBox/internal/service/reader"
|
||||
)
|
||||
|
||||
// 音轨「区间直通」的行为契约。
|
||||
//
|
||||
// 背景(真实故障):播放器(media3)取一条 m4a 会开两次请求——先整段打开嗅探
|
||||
// 容器,再带 Range 从目标位置重开。旧实现只允许「无 Range + 无分片」的请求
|
||||
// 边下边发,于是带 Range 的那次(真正的起播请求)必须把整章下完再按区间下发。
|
||||
// 线上日志:每章固定 200(探针,被播放器掐断)+ 206(整章下完才吐第一个字节),
|
||||
// 42 章里「媒体已装载 → 实际起播」平均 4.79 秒,最坏 35 秒。
|
||||
//
|
||||
// 现在的契约:
|
||||
// - 带 Range 的冷缓存请求 → 上游一到字节就往下发,不等整章;
|
||||
// - 分片已有的前缀从本地发,上游只续下剩余字节;
|
||||
// - 整章落满时把分片提升为正式缓存,之后走本地;
|
||||
// - 上游不认 Range 时也要给出正确的 206(跳过前缀),且不写带洞的分片;
|
||||
// - 拿不到写锁时降级成纯透传,绝不排队等前一个下载。
|
||||
|
||||
// rangeUpstream 支持 Range 的上游:/content 给出音轨地址,/media 回音频本体。
|
||||
type rangeUpstream struct {
|
||||
*httptest.Server
|
||||
payload []byte
|
||||
honorRange bool
|
||||
// firstChunk > 0 且 release 非 nil 时:先写这么多字节再等放行,
|
||||
// 用来证明「客户端不必等整章下完」。
|
||||
firstChunk int
|
||||
release chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
ranges []string
|
||||
hits int
|
||||
}
|
||||
|
||||
func newRangeUpstream(t *testing.T, payload []byte, honorRange bool) *rangeUpstream {
|
||||
t.Helper()
|
||||
u := &rangeUpstream{payload: payload, honorRange: honorRange}
|
||||
u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasPrefix(r.URL.Path, "/book/1"):
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`<html><body><h1>区间之书</h1></body></html>`))
|
||||
case r.URL.Path == "/content":
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
_, _ = w.Write([]byte(fmt.Sprintf(`{"data":{"url":"%s/media"}}`, u.URL)))
|
||||
case r.URL.Path == "/media":
|
||||
u.serveMedia(w, r)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(u.Close)
|
||||
return u
|
||||
}
|
||||
|
||||
func (u *rangeUpstream) serveMedia(w http.ResponseWriter, r *http.Request) {
|
||||
u.mu.Lock()
|
||||
u.hits++
|
||||
u.ranges = append(u.ranges, r.Header.Get("Range"))
|
||||
u.mu.Unlock()
|
||||
|
||||
total := len(u.payload)
|
||||
start, end := 0, total-1
|
||||
partial := false
|
||||
if u.honorRange {
|
||||
if s, e, ok := parseReaderAudioRange(r.Header.Get("Range")); ok {
|
||||
start = int(s)
|
||||
if e >= 0 && int(e) < end {
|
||||
end = int(e)
|
||||
}
|
||||
partial = true
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
if start >= total {
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
body := u.payload[start : end+1]
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
if partial {
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, total))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
if u.release != nil && u.firstChunk > 0 && len(body) > u.firstChunk {
|
||||
_, _ = w.Write(body[:u.firstChunk])
|
||||
if flusher, ok := w.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
select {
|
||||
case <-u.release:
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
body = body[u.firstChunk:]
|
||||
}
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
func (u *rangeUpstream) rangeHeaders() []string {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
return append([]string(nil), u.ranges...)
|
||||
}
|
||||
|
||||
func (u *rangeUpstream) requestCount() int {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
return u.hits
|
||||
}
|
||||
|
||||
// prepareRangeBook 建一本「书源直连上游」的音频书,并取回该音轨的签名代理路径。
|
||||
func prepareRangeBook(t *testing.T, upstream string) (*service.Container, *model.ReaderBook, string) {
|
||||
t.Helper()
|
||||
svc := newReaderHandlerContainer(t)
|
||||
svc.Cfg.Cache.CacheDir = t.TempDir()
|
||||
srcJSON := fmt.Sprintf(`{
|
||||
"bookSourceName": "区间直通测试",
|
||||
"bookSourceType": 1,
|
||||
"bookSourceUrl": %q,
|
||||
"ruleContent": {"content": "$.data.url"}
|
||||
}`, upstream)
|
||||
if _, err := svc.Reader.ImportSources(t.Context(), "u1", srcJSON); err != nil {
|
||||
t.Fatalf("导入书源失败: %v", err)
|
||||
}
|
||||
srcs, err := svc.Reader.ListSources(t.Context(), "u1")
|
||||
if err != nil || len(srcs) == 0 {
|
||||
t.Fatalf("读不到书源: %v", err)
|
||||
}
|
||||
book, err := svc.Reader.AddBook(t.Context(), "u1", reader.SearchOrigin{
|
||||
SourceID: srcs[0].ID,
|
||||
Origin: upstream,
|
||||
OriginName: "区间直通测试",
|
||||
OriginType: 1,
|
||||
BookURL: upstream + "/book/1",
|
||||
}, "区间之书", "某作者", "")
|
||||
if err != nil {
|
||||
t.Fatalf("加入书架失败: %v", err)
|
||||
}
|
||||
if err := svc.Reader.SaveChapters(t.Context(), book.ID, []reader.ChapterInput{
|
||||
{Index: 0, Title: "第一章", URL: upstream + "/content"},
|
||||
}); err != nil {
|
||||
t.Fatalf("保存目录失败: %v", err)
|
||||
}
|
||||
out, err := svc.Reader.GetContentForBook(t.Context(), "u1", book.ID, 0, reader.ContentOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("取章节失败: %v", err)
|
||||
}
|
||||
if len(out.Tracks) != 1 {
|
||||
t.Fatalf("音轨数 = %d,期望 1", len(out.Tracks))
|
||||
}
|
||||
return svc, book, out.Tracks[0]
|
||||
}
|
||||
|
||||
func audioPayload(n int) []byte {
|
||||
payload := make([]byte, n)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i % 251)
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
// waitCached 等待分片被提升为正式缓存(提升发生在响应收尾,与客户端读完成之间有微小竞态)。
|
||||
func waitCached(t *testing.T, svc *service.Container, bookID, mediaURL string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if svc.Reader.HasCachedRemoteAudio(bookID, mediaURL) {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("整章下完后应提升为正式缓存")
|
||||
}
|
||||
|
||||
// TestReaderAudioRangeStreamsWithoutWaitingForWholeChapter 冷缓存 + Range:
|
||||
// 上游只吐了第一段,客户端就必须已经拿到字节。
|
||||
func TestReaderAudioRangeStreamsWithoutWaitingForWholeChapter(t *testing.T) {
|
||||
payload := audioPayload(512 << 10)
|
||||
upstream := newRangeUpstream(t, payload, true)
|
||||
upstream.firstChunk = 64 << 10
|
||||
upstream.release = make(chan struct{})
|
||||
|
||||
svc, book, track := prepareRangeBook(t, upstream.URL)
|
||||
mediaURL := upstream.URL + "/media"
|
||||
srv := httptest.NewServer(mediaRouterForTest(svc))
|
||||
defer srv.Close()
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, srv.URL+track, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("构造请求失败: %v", err)
|
||||
}
|
||||
req.Header.Set("Range", "bytes=0-")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("请求失败: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
t.Fatalf("状态码 = %d,期望 206", resp.StatusCode)
|
||||
}
|
||||
if got := resp.Header.Get("Content-Range"); got != fmt.Sprintf("bytes 0-%d/%d", len(payload)-1, len(payload)) {
|
||||
t.Fatalf("Content-Range = %q", got)
|
||||
}
|
||||
if got := resp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
|
||||
t.Fatalf("Content-Length = %q,期望 %d", got, len(payload))
|
||||
}
|
||||
|
||||
// 上游此时还被 release 卡着:能在超时内读到第一段就说明没有等整章。
|
||||
head := make([]byte, upstream.firstChunk)
|
||||
readDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, readErr := io.ReadFull(resp.Body, head)
|
||||
readDone <- readErr
|
||||
}()
|
||||
select {
|
||||
case readErr := <-readDone:
|
||||
if readErr != nil {
|
||||
t.Fatalf("读首段失败: %v", readErr)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("上游还没给完就卡住了:客户端没能在整章下完前拿到第一个字节")
|
||||
}
|
||||
if !bytes.Equal(head, payload[:upstream.firstChunk]) {
|
||||
t.Fatalf("首段字节不一致")
|
||||
}
|
||||
|
||||
close(upstream.release)
|
||||
rest, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("读剩余字节失败: %v", err)
|
||||
}
|
||||
if got := append(head, rest...); !bytes.Equal(got, payload) {
|
||||
t.Fatalf("整体字节不一致: got %d bytes", len(got))
|
||||
}
|
||||
if got := upstream.rangeHeaders(); len(got) != 1 || got[0] != "bytes=0-" {
|
||||
t.Fatalf("上游收到的 Range = %v,期望且仅一次 bytes=0-", got)
|
||||
}
|
||||
waitCached(t, svc, book.ID, mediaURL)
|
||||
}
|
||||
|
||||
// TestReaderAudioRangeResumesFromPartialFragment 分片里已有前缀时,前缀从本地发,
|
||||
// 上游只续下剩余字节;整章落满后提升为正式缓存。
|
||||
func TestReaderAudioRangeResumesFromPartialFragment(t *testing.T) {
|
||||
payload := audioPayload(256 << 10)
|
||||
upstream := newRangeUpstream(t, payload, true)
|
||||
svc, book, track := prepareRangeBook(t, upstream.URL)
|
||||
mediaURL := upstream.URL + "/media"
|
||||
|
||||
// 模拟「播放器嗅探完就断开」留下的分片:前 96KB 已经在盘上。
|
||||
const partLen = 96 << 10
|
||||
cachePath := svc.Reader.RemoteAudioCachePath(book.ID, mediaURL)
|
||||
if cachePath == "" {
|
||||
t.Fatalf("缓存路径为空")
|
||||
}
|
||||
if mkErr := os.MkdirAll(cachePath[:strings.LastIndex(cachePath, string(os.PathSeparator))], 0o750); mkErr != nil {
|
||||
t.Fatalf("建缓存目录失败: %v", mkErr)
|
||||
}
|
||||
if writeErr := os.WriteFile(cachePath+".part", payload[:partLen], 0o640); writeErr != nil {
|
||||
t.Fatalf("写分片失败: %v", writeErr)
|
||||
}
|
||||
|
||||
code, body := getReaderMediaWithRange(t, mediaRouterForTest(svc), track, "bytes=0-")
|
||||
if code != http.StatusPartialContent {
|
||||
t.Fatalf("状态码 = %d,期望 206 body=%.120s", code, body)
|
||||
}
|
||||
if !bytes.Equal([]byte(body), payload) {
|
||||
t.Fatalf("拼接后的字节不一致: got %d bytes want %d", len(body), len(payload))
|
||||
}
|
||||
headers := upstream.rangeHeaders()
|
||||
if len(headers) != 1 || headers[0] != fmt.Sprintf("bytes=%d-", partLen) {
|
||||
t.Fatalf("上游收到的 Range = %v,期望从分片长度续下 bytes=%d-", headers, partLen)
|
||||
}
|
||||
waitCached(t, svc, book.ID, mediaURL)
|
||||
|
||||
// 提升之后同一首歌不再回源。
|
||||
before := upstream.requestCount()
|
||||
code, body = getReaderMedia(t, mediaRouterForTest(svc), track)
|
||||
if code != http.StatusOK || !bytes.Equal([]byte(body), payload) {
|
||||
t.Fatalf("命中缓存失败 status=%d len=%d", code, len(body))
|
||||
}
|
||||
if after := upstream.requestCount(); after != before {
|
||||
t.Fatalf("命中缓存后不应再回源,多了 %d 次", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderAudioRangeHandlesUpstreamThatIgnoresRange 上游不认 Range(回 200 整段)时,
|
||||
// 仍要按客户端要的起点给出正确的 206,并且不写带洞的分片。
|
||||
func TestReaderAudioRangeHandlesUpstreamThatIgnoresRange(t *testing.T) {
|
||||
payload := audioPayload(300 << 10)
|
||||
upstream := newRangeUpstream(t, payload, false)
|
||||
svc, book, track := prepareRangeBook(t, upstream.URL)
|
||||
mediaURL := upstream.URL + "/media"
|
||||
|
||||
const start = 1000
|
||||
code, body := getReaderMediaWithRange(t, mediaRouterForTest(svc), track, fmt.Sprintf("bytes=%d-", start))
|
||||
if code != http.StatusPartialContent {
|
||||
t.Fatalf("状态码 = %d,期望 206 body=%.120s", code, body)
|
||||
}
|
||||
if !bytes.Equal([]byte(body), payload[start:]) {
|
||||
t.Fatalf("区间字节不一致: got %d want %d", len(body), len(payload)-start)
|
||||
}
|
||||
// 起点不在分片内 → 不能落盘(否则分片开头会是空洞)。
|
||||
if svc.Reader.HasPartialRemoteAudio(book.ID, mediaURL) {
|
||||
t.Fatalf("起点靠后的区间请求不应生成分片")
|
||||
}
|
||||
if svc.Reader.HasCachedRemoteAudio(book.ID, mediaURL) {
|
||||
t.Fatalf("只下了一段不应生成完整缓存")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderAudioRangeFallsBackToPassthroughWhenWriterBusy 拿不到写锁时不排队:
|
||||
// 仍然立刻把区间发出去,只是这一轮不落盘。
|
||||
func TestReaderAudioRangeFallsBackToPassthroughWhenWriterBusy(t *testing.T) {
|
||||
payload := audioPayload(128 << 10)
|
||||
upstream := newRangeUpstream(t, payload, true)
|
||||
svc, book, track := prepareRangeBook(t, upstream.URL)
|
||||
mediaURL := upstream.URL + "/media"
|
||||
|
||||
cacheKey := svc.Reader.RemoteAudioCachePath(book.ID, mediaURL)
|
||||
if cacheKey == "" {
|
||||
t.Fatalf("缓存路径为空")
|
||||
}
|
||||
unlock := readerAudioStreamLocks.Lock(cacheKey)
|
||||
defer unlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
var code int
|
||||
var body string
|
||||
go func() {
|
||||
code, body = getReaderMediaWithRange(t, mediaRouterForTest(svc), track, "bytes=0-")
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("写锁被占用时不该让请求排队等待")
|
||||
}
|
||||
if code != http.StatusPartialContent {
|
||||
t.Fatalf("状态码 = %d,期望 206 body=%.120s", code, body)
|
||||
}
|
||||
if !bytes.Equal([]byte(body), payload) {
|
||||
t.Fatalf("区间字节不一致: got %d want %d", len(body), len(payload))
|
||||
}
|
||||
if svc.Reader.HasCachedRemoteAudio(book.ID, mediaURL) {
|
||||
t.Fatalf("降级透传不应写缓存")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderAudioRangeCases(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
start int64
|
||||
end int64
|
||||
want bool
|
||||
}{
|
||||
{"bytes=0-", 0, -1, true},
|
||||
{"bytes=100-199", 100, 199, true},
|
||||
{"Bytes=1024-", 1024, -1, true},
|
||||
{" bytes=5-5 ", 5, 5, true},
|
||||
{"bytes=-500", 0, 0, false},
|
||||
{"bytes=0-99,200-299", 0, 0, false},
|
||||
{"items=0-", 0, 0, false},
|
||||
{"bytes=abc-", 0, 0, false},
|
||||
{"bytes=200-100", 0, 0, false},
|
||||
{"", 0, 0, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
start, end, ok := parseReaderAudioRange(c.in)
|
||||
if ok != c.want || start != c.start || end != c.end {
|
||||
t.Errorf("parseReaderAudioRange(%q) = (%d,%d,%v),期望 (%d,%d,%v)",
|
||||
c.in, start, end, ok, c.start, c.end, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getReaderMediaWithRange 发一次带 Range 头的媒体代理请求。
|
||||
func getReaderMediaWithRange(t *testing.T, router *gin.Engine, signedPath, rangeHeader string) (int, string) {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, signedPath, nil)
|
||||
req.Header.Set("Range", rangeHeader)
|
||||
router.ServeHTTP(w, req)
|
||||
return w.Code, w.Body.String()
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/service/reader"
|
||||
)
|
||||
|
||||
func TestReaderContentOptions(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
for _, tc := range []struct {
|
||||
query string
|
||||
want reader.ContentOptions
|
||||
}{
|
||||
{"", reader.ContentOptions{}},
|
||||
{"?media=direct", reader.ContentOptions{MediaDirect: true}},
|
||||
{"?media=proxy", reader.ContentOptions{ForceProxy: true}},
|
||||
{"?audio=native", reader.ContentOptions{NativeAudio: true}},
|
||||
{"?audio=native&media=direct", reader.ContentOptions{NativeAudio: true, MediaDirect: true}},
|
||||
{"?audio=native&media=proxy", reader.ContentOptions{NativeAudio: true, ForceProxy: true}},
|
||||
{"?audio=%20NaTiVe%20&media=%20DiReCt%20", reader.ContentOptions{NativeAudio: true, MediaDirect: true}},
|
||||
{"?audio=native&media=unknown", reader.ContentOptions{NativeAudio: true}},
|
||||
{"?audio=browser&media=proxy", reader.ContentOptions{ForceProxy: true}},
|
||||
{"?audio=unknown&media=direct", reader.ContentOptions{MediaDirect: true}},
|
||||
} {
|
||||
t.Run(tc.query, func(t *testing.T) {
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodGet, "/api/reader/books/book/content"+tc.query, nil)
|
||||
if got := readerContentOptions(c); got != tc.want {
|
||||
t.Fatalf("options = %+v, want %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,764 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service"
|
||||
"github.com/truewhile/MeBox/internal/service/reader"
|
||||
)
|
||||
|
||||
// 媒体代理在「上游签名票被拒」时的行为契约。
|
||||
//
|
||||
// 背景(真实故障):聆阅·懒人听书源的音频直链是第三方中转现签的腾讯 CDN 票,
|
||||
// 票被 CDN 判掉时上游回的是**空的 401**(无 Content-Type、无 body)。旧行为把它
|
||||
// 原样透传给播放器,播放器重试的还是同一个 URL,于是 App 日志里连续 4~8 个
|
||||
// ERROR_CODE_IO_BAD_HTTP_STATUS (2004),表现为「这本书怎么都播不了」。
|
||||
//
|
||||
// 现在的契约:
|
||||
// - 能换票 → 用新票重拉,直接返回媒体本体(第一个请求就被治好);
|
||||
// - 换不到 → 502 + 明确文案,而不是把 401 丢给播放器;
|
||||
// - 非音轨地址(漫画图片防盗链等)→ 维持原样透传,不改变既有行为。
|
||||
|
||||
func TestReaderMediaPayloadRejected(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
head string
|
||||
contentType string
|
||||
want bool
|
||||
}{
|
||||
{"HTML 错误页", "<!DOCTYPE html><html><body>403</body></html>", "text/html; charset=utf-8", true},
|
||||
{"JSON 报错", `{"code":403,"message":"expired"}`, "application/json", true},
|
||||
{"数组形态报错", `[{"error":"bad ticket"}]`, "application/json; charset=utf-8", true},
|
||||
{"纯文本报错", "ticket expired", "text/plain", false},
|
||||
{"正常 m4a", "\x00\x00\x00\x20ftypmp42", "audio/mp4", false},
|
||||
{"正常 mp3", "ID3\x04\x00", "audio/mpeg", false},
|
||||
{"正常图片", "\xff\xd8\xff\xe0", "image/jpeg", false},
|
||||
{"播放列表", "#EXTM3U\n#EXTINF:10,", "application/vnd.apple.mpegurl", false},
|
||||
{"空体", "", "text/html", false},
|
||||
{"HTML 类型但二进制体", "\x00\x00\x00\x20ftypmp42", "text/html", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := readerMediaPayloadRejected([]byte(c.head), c.contentType)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: readerMediaPayloadRejected(%q, %q) = %v,期望 %v",
|
||||
c.name, c.head, c.contentType, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderMediaProxyHealsStaleTicket 票被上游拒绝时代理自己换票重拉,
|
||||
// 第一个请求就返回媒体本体(不必等播放器重试)。
|
||||
func TestReaderMediaProxyHealsStaleTicket(t *testing.T) {
|
||||
svc, srv, book := prepareMediaTicketBook(t)
|
||||
router := mediaRouterForTest(svc)
|
||||
|
||||
signed := primeTicketBook(t, svc, book.ID)
|
||||
// 让中转再签一张新票:章节缓存里那张就此变成「旧票」。
|
||||
srv.rotate()
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("换票重试后 status=%d body=%s", code, body)
|
||||
}
|
||||
if !strings.Contains(body, "fake-audio-bytes") {
|
||||
t.Fatalf("响应体不是媒体: %q", body)
|
||||
}
|
||||
if got := srv.contentHits(); got < 2 {
|
||||
t.Fatalf("应当回源重取章节,实际请求 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderMediaProxyFailsClearlyWhenRefreshImpossible 换不到票时给 502 + 明确文案,
|
||||
// 而不是把 401 透传给播放器(那样播放器只会对着必然失败的地址反复重试)。
|
||||
func TestReaderMediaProxyFailsClearlyWhenRefreshImpossible(t *testing.T) {
|
||||
svc, srv, book := prepareMediaTicketBook(t)
|
||||
router := mediaRouterForTest(svc)
|
||||
|
||||
signed := primeTicketBook(t, svc, book.ID)
|
||||
srv.rotate()
|
||||
srv.stopIssuing()
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusBadGateway {
|
||||
t.Fatalf("换票失败应当返回 502,实际 %d body=%s", code, body)
|
||||
}
|
||||
if !strings.Contains(body, "签名失效") {
|
||||
t.Fatalf("错误文案应说明是音源签名失效: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderMediaProxyStopsResendingRejectedTicket 上游成批拒绝时(连新签的票
|
||||
// 也是空 401),不能在播放器那一次请求里把同一张被拒的票打上十几遍:连续两轮
|
||||
// 换票都是同一个状态码就立刻失败,把明确文案交出去。
|
||||
func TestReaderMediaProxyStopsResendingRejectedTicket(t *testing.T) {
|
||||
svc, srv, book := prepareMediaTicketBook(t)
|
||||
router := mediaRouterForTest(svc)
|
||||
|
||||
signed := primeTicketBook(t, svc, book.ID)
|
||||
srv.denyAlways()
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusBadGateway {
|
||||
t.Fatalf("上游持续拒绝应返回 502,实际 %d body=%s", code, body)
|
||||
}
|
||||
if !strings.Contains(body, "401") {
|
||||
t.Fatalf("错误文案应带上上游状态码: %s", body)
|
||||
}
|
||||
// 每轮 fetchReaderMedia = 首拉(401) + 换票后重拉(401),两轮即收手。
|
||||
if got := srv.mediaRequests(); got > 4 {
|
||||
t.Fatalf("无进展的拒绝应当两轮就收手,实际上游请求 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderAudioCacheResumesTruncatedUpstream 上游中途 RST 后用 Range 续传:
|
||||
// 播放器最终拿到的仍是完整的一章,而不是从头重下。
|
||||
func TestReaderAudioCacheResumesTruncatedUpstream(t *testing.T) {
|
||||
const bodyBytes = 300 << 10
|
||||
payload := bytes.Repeat([]byte("m4a!"), bodyBytes/4)
|
||||
var ranges []string
|
||||
var mu sync.Mutex
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
ranges = append(ranges, r.Header.Get("Range"))
|
||||
count := len(ranges)
|
||||
mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
start := 0
|
||||
if count > 1 {
|
||||
// 第二次起按 Range 续传。
|
||||
start = bodyBytes / 2
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, bodyBytes-1, bodyBytes))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
} else {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(bodyBytes))
|
||||
}
|
||||
chunk := payload[start:]
|
||||
if count == 1 {
|
||||
// 第一次只写一半就断开。
|
||||
chunk = payload[:bodyBytes/2]
|
||||
}
|
||||
for written := 0; written < len(chunk); written += 8 << 10 {
|
||||
end := written + 8<<10
|
||||
if end > len(chunk) {
|
||||
end = len(chunk)
|
||||
}
|
||||
if _, err := w.Write(chunk[written:end]); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
if count == 1 {
|
||||
if hj, ok := w.(http.Hijacker); ok {
|
||||
if conn, _, err := hj.Hijack(); err == nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
svc, _, book := prepareMediaTicketBookWithUpstream(t, srv.URL)
|
||||
router := mediaRouterForTest(svc)
|
||||
signed := svc.Reader.ProxyURL(book.ID, srv.URL+"/media")
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("续传后应当成功,实际 status=%d body=%.120s", code, body)
|
||||
}
|
||||
if len(body) != bodyBytes {
|
||||
t.Fatalf("交付字节 = %d,期望 %d", len(body), bodyBytes)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(ranges) < 2 || ranges[0] != "" {
|
||||
t.Fatalf("请求序列 = %q,期望先整段再续传", ranges)
|
||||
}
|
||||
if !strings.HasPrefix(ranges[1], "bytes=") {
|
||||
t.Fatalf("第二轮应当带 Range,实际 %q", ranges[1])
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderMediaProxyPassesThroughNonTrackError 漫画图片这类不可换票的地址,
|
||||
// 上游 4xx 仍按原状态透传(既有行为不变)。
|
||||
func TestReaderMediaProxyPassesThroughNonTrackError(t *testing.T) {
|
||||
svc, srv, book := prepareMediaTicketBook(t)
|
||||
router := mediaRouterForTest(svc)
|
||||
|
||||
// 书源域名下的图片地址:不在任何音轨缓存里。
|
||||
imgURL := srv.URL + "/img/forbidden.jpg"
|
||||
signed := svc.Reader.ProxyURL(book.ID, imgURL)
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code == http.StatusBadGateway || code == http.StatusUnauthorized {
|
||||
t.Fatalf("非音轨地址不该走换票分支,实际 %d body=%s", code, body)
|
||||
}
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("图片 403 应原样透传,实际 %d body=%s", code, body)
|
||||
}
|
||||
if got := srv.contentHits(); got != 0 {
|
||||
t.Fatalf("图片失败不该触发回源取章节,实际 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderMediaProxyRejectsNonMediaPayload 上游用 200 回一段 HTML/JSON 错误页时,
|
||||
// 代理直接判失败,不把网页喂给播放器(否则报「格式不支持」)。
|
||||
func TestReaderMediaProxyRejectsNonMediaPayload(t *testing.T) {
|
||||
svc, srv, book := prepareMediaTicketBook(t)
|
||||
router := mediaRouterForTest(svc)
|
||||
|
||||
signed := primeTicketBook(t, svc, book.ID)
|
||||
srv.serveErrorPage()
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusBadGateway {
|
||||
t.Fatalf("非媒体内容应当返回 502,实际 %d body=%.160s", code, body)
|
||||
}
|
||||
if !strings.Contains(body, "非媒体内容") {
|
||||
t.Fatalf("错误文案应说明上游返回了非媒体内容: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── 上游抖动:连接被 reset 时的重试与截断可见化 ─────────────────────────
|
||||
|
||||
// TestReaderMediaProbeRetriesConnectionReset 上游在响应体中途 RST 连接时,
|
||||
// 代理要退避重试,而不是第一次读失败就回 502。
|
||||
//
|
||||
// 真实故障:腾讯 CDN 的跨境边缘在并发/长传输时会直接 RST,同一章重试一次
|
||||
// 往往就好。以前这里读 body 失败即判死,把一次可恢复的抖动变成「这本书播不了」。
|
||||
func TestReaderMediaProbeRetriesConnectionReset(t *testing.T) {
|
||||
var attempts int32
|
||||
const bodyBytes = 200 << 10
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
n := atomic.AddInt32(&attempts, 1)
|
||||
if n == 1 {
|
||||
// 第一次:声明长度,写一小段后直接断开连接(模拟 RST/提前 EOF)。
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(bodyBytes))
|
||||
_, _ = w.Write([]byte("\x00\x00\x00\x20ftypmp42"))
|
||||
w.(http.Flusher).Flush()
|
||||
if hj, ok := w.(http.Hijacker); ok {
|
||||
if conn, _, err := hj.Hijack(); err == nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(bodyBytes))
|
||||
payload := make([]byte, 8<<10)
|
||||
for written := 0; written < bodyBytes; written += len(payload) {
|
||||
if _, err := w.Write(payload); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
svc, _, book := prepareMediaTicketBookWithUpstream(t, srv.URL)
|
||||
router := mediaRouterForTest(svc)
|
||||
signed := svc.Reader.ProxyURL(book.ID, srv.URL+"/media")
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("连接被 reset 后应当重试成功,实际 status=%d body=%.200s", code, body)
|
||||
}
|
||||
if int32(len(body)) != bodyBytes {
|
||||
t.Fatalf("交付字节 = %d,期望 %d", len(body), bodyBytes)
|
||||
}
|
||||
if got := atomic.LoadInt32(&attempts); got < 2 {
|
||||
t.Fatalf("应当重试过,实际上游请求 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderMediaAbortsWhenUpstreamTruncates 非音轨的流式透传路径:上游中途
|
||||
// 断开且重试都失败时,必须让客户端看到「传输不完整」,不能返回一个长度正确
|
||||
// 却内容缺失的响应。
|
||||
//
|
||||
// 以前这里是 `_, _ = io.Copy(...)`,错误被丢掉,客户端拿到一个正常结束的
|
||||
// 半截文件 —— 播放器只能报「格式不支持」(3003),根因被彻底掩盖。
|
||||
//
|
||||
// 音轨走的是整段落盘缓存(见 TestReaderAudioCacheRejectsTruncatedUpstream):
|
||||
// 拉不完整就重试,全部失败回 502,绝不会把半截音频交给播放器。
|
||||
func TestReaderMediaAbortsWhenUpstreamTruncates(t *testing.T) {
|
||||
const declared = 400 << 10
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(declared))
|
||||
// 先写够探测用的前缀,再断开:探测能过,透传阶段才断。
|
||||
payload := make([]byte, 8<<10)
|
||||
for written := 0; written < 128<<10; written += len(payload) {
|
||||
if _, err := w.Write(payload); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
if hj, ok := w.(http.Hijacker); ok {
|
||||
if conn, _, err := hj.Hijack(); err == nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
svc, _, book := prepareMediaTicketBookWithUpstream(t, srv.URL)
|
||||
router := mediaRouterForTest(svc)
|
||||
// .mp4 不是音频后缀:走流式透传分支(音轨会被整段落盘缓存)。
|
||||
signed := svc.Reader.ProxyURL(book.ID, srv.URL+"/media.mp4")
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, signed, nil)
|
||||
// panic(http.ErrAbortHandler) 由 net/http 捕获;httptest 下会向上冒泡,
|
||||
// 这里断言它确实发生了(即「没有静默地把半截文件当成功」)。
|
||||
defer func() {
|
||||
recovered := recover()
|
||||
if recovered == nil {
|
||||
t.Fatalf("上游截断时必须中止响应(panic(http.ErrAbortHandler)),实际正常返回了 %d 字节",
|
||||
w.Body.Len())
|
||||
}
|
||||
if recovered != http.ErrAbortHandler {
|
||||
t.Fatalf("中止响应时应使用 http.ErrAbortHandler,实际 %v", recovered)
|
||||
}
|
||||
}()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Body.Len() >= declared {
|
||||
t.Fatalf("不应交付完整长度的响应体(上游只写了 128KB)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderAudioCacheServesFromDisk 音轨首次拉取后落盘;后续请求即使带着一张
|
||||
// 新签的票(query 全变了)也不再回源——这是让播放不受跨境链路抖动影响的关键。
|
||||
func TestReaderAudioCacheServesFromDisk(t *testing.T) {
|
||||
svc, srv, book := prepareMediaTicketBook(t)
|
||||
router := mediaRouterForTest(svc)
|
||||
signed := primeTicketBook(t, svc, book.ID)
|
||||
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusOK || !strings.Contains(body, "fake-audio-bytes") {
|
||||
t.Fatalf("首次拉取失败 status=%d body=%q", code, body)
|
||||
}
|
||||
if got := srv.mediaRequests(); got != 1 {
|
||||
t.Fatalf("首次拉取应对上游请求 1 次,实际 %d", got)
|
||||
}
|
||||
|
||||
// 同一段音频的另一张票:URL 的 query 不同,缓存键必须一致。
|
||||
rotated := svc.Reader.ProxyURL(book.ID, srv.URL+"/media?t=ticket-rotated")
|
||||
code, body = getReaderMedia(t, router, rotated)
|
||||
if code != http.StatusOK || !strings.Contains(body, "fake-audio-bytes") {
|
||||
t.Fatalf("命中缓存失败 status=%d body=%q", code, body)
|
||||
}
|
||||
if got := srv.mediaRequests(); got != 1 {
|
||||
t.Fatalf("命中缓存后不该再回源,实际上游请求 %d 次", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReaderAudioCacheAbortsOnTruncatedUpstream 首播直通模式下上游持续截断的契约:
|
||||
//
|
||||
// 冷启动走边下边发(见 handleStreamingReaderAudio),响应头和前缀字节在探测
|
||||
// 通过后就已发出,因此中途截断不可能再改写成 502 —— 必须中止连接让客户端
|
||||
// 判定「传输不完整」,同时保留 .part 分片。
|
||||
//
|
||||
// 关键不变量:**绝不能留下可命中的完整缓存**(否则后续会把截断音频当真完整
|
||||
// 文件下发,播放器表现为「时长正确但没有声音」)。下一次请求带着分片走续传
|
||||
// 路径,那时才可能给出 502 + 明确文案。
|
||||
func TestReaderAudioCacheAbortsOnTruncatedUpstream(t *testing.T) {
|
||||
const declared = 400 << 10
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(declared))
|
||||
payload := make([]byte, 8<<10)
|
||||
for written := 0; written < 128<<10; written += len(payload) {
|
||||
if _, err := w.Write(payload); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
if hj, ok := w.(http.Hijacker); ok {
|
||||
if conn, _, err := hj.Hijack(); err == nil {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
svc, _, book := prepareMediaTicketBookWithUpstream(t, srv.URL)
|
||||
router := mediaRouterForTest(svc)
|
||||
// 无后缀地址 + 音频书籍 → 走音轨落盘缓存。
|
||||
signed := svc.Reader.ProxyURL(book.ID, srv.URL+"/media")
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, signed, nil)
|
||||
// panic(http.ErrAbortHandler) 由 net/http 捕获;httptest 下会向上冒泡,
|
||||
// 这里断言它确实发生了(即「没有静默地把半截音频当成功」)。
|
||||
func() {
|
||||
defer func() {
|
||||
recovered := recover()
|
||||
if recovered == nil {
|
||||
t.Fatalf("上游截断时必须中止响应,实际正常返回了 %d 字节", w.Body.Len())
|
||||
}
|
||||
if recovered != http.ErrAbortHandler {
|
||||
t.Fatalf("中止响应时应使用 http.ErrAbortHandler,实际 %v", recovered)
|
||||
}
|
||||
}()
|
||||
router.ServeHTTP(w, req)
|
||||
}()
|
||||
|
||||
if w.Body.Len() >= declared {
|
||||
t.Fatalf("不应交付完整长度的响应体(上游只写了 128KB)")
|
||||
}
|
||||
if _, _, ok := svc.Reader.OpenCachedRemoteAudio(book.ID, srv.URL+"/media"); ok {
|
||||
t.Fatalf("中断的下载不能留下可命中的完整缓存")
|
||||
}
|
||||
if !svc.Reader.HasPartialRemoteAudio(book.ID, srv.URL+"/media") {
|
||||
t.Fatalf("中断后应保留 .part 分片供续传")
|
||||
}
|
||||
|
||||
// 二次请求带分片走续传路径;上游依旧忽略 Range 并截断 → 这时才给 502。
|
||||
code, body := getReaderMedia(t, router, signed)
|
||||
if code != http.StatusBadGateway {
|
||||
t.Fatalf("续传路径持续失败应返回 502,实际 %d body=%.120s", code, body)
|
||||
}
|
||||
if strings.Contains(body, "ftyp") {
|
||||
t.Fatalf("不能把半截音频交给客户端: %q", body)
|
||||
}
|
||||
if _, _, ok := svc.Reader.OpenCachedRemoteAudio(book.ID, srv.URL+"/media"); ok {
|
||||
t.Fatalf("失败的续传仍不应留下完整缓存")
|
||||
}
|
||||
}
|
||||
|
||||
func mediaRouterForTest(svc *service.Container) *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
r := gin.New()
|
||||
// /api/reader/media 挂在公共 api 组(签名鉴权,不需要 JWT),单独注册即可。
|
||||
r.GET("/api/reader/media", readerMediaProxyHandler(svc))
|
||||
return r
|
||||
}
|
||||
|
||||
func getReaderMedia(t *testing.T, router *gin.Engine, signedPath string) (int, string) {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, signedPath, nil)
|
||||
router.ServeHTTP(w, req)
|
||||
return w.Code, w.Body.String()
|
||||
}
|
||||
|
||||
// primeTicketBook 先读一次章节(签出第一张票并落缓存),返回该音轨的签名代理路径。
|
||||
func primeTicketBook(t *testing.T, svc *service.Container, bookID string) string {
|
||||
t.Helper()
|
||||
out, err := svc.Reader.GetContentForBook(t.Context(), "u1", bookID, 0, reader.ContentOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("首次取章节失败: %v", err)
|
||||
}
|
||||
if len(out.Tracks) != 1 {
|
||||
t.Fatalf("音轨数 = %d,期望 1", len(out.Tracks))
|
||||
}
|
||||
return out.Tracks[0]
|
||||
}
|
||||
|
||||
// ─── 模拟「中转现签 CDN 直链」的上游 ───────────────────────────────────────
|
||||
|
||||
// mediaTicketServer:
|
||||
// - /book/1 详情页
|
||||
// - /content 章节正文,每次请求签一张新票
|
||||
// - /media 媒体本体,只认最新签出的票;旧票回**空的 401**(与腾讯 CDN 一致)
|
||||
// - /img/* 图片资源,固定 403(模拟防盗链)
|
||||
type mediaTicketServer struct {
|
||||
*httptest.Server
|
||||
mu sync.Mutex
|
||||
issued []string
|
||||
hits int
|
||||
mediaHits int
|
||||
noToken bool
|
||||
errorPage bool
|
||||
alwaysDeny bool
|
||||
}
|
||||
|
||||
func newMediaTicketServer(t *testing.T) *mediaTicketServer {
|
||||
t.Helper()
|
||||
s := &mediaTicketServer{}
|
||||
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasPrefix(r.URL.Path, "/book/1"):
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`<html><body><h1>票源之书</h1></body></html>`))
|
||||
case r.URL.Path == "/content":
|
||||
s.mu.Lock()
|
||||
s.hits++
|
||||
n := s.hits
|
||||
noToken := s.noToken
|
||||
s.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
if noToken {
|
||||
_, _ = w.Write([]byte(`{"data":{"url":""}}`))
|
||||
return
|
||||
}
|
||||
token := s.newTokenLocked(fmt.Sprintf("ticket-%d", n))
|
||||
_, _ = w.Write([]byte(fmt.Sprintf(`{"data":{"url":"%s/media?t=%s"}}`, s.URL, token)))
|
||||
case r.URL.Path == "/media":
|
||||
s.mu.Lock()
|
||||
s.mediaHits++
|
||||
deny := s.alwaysDeny
|
||||
s.mu.Unlock()
|
||||
if deny {
|
||||
// 与腾讯 CDN 一致:连新签的票也直接空 401(成批拒绝的那几分钟)。
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
if s.errorPageNow() {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`<html><body>ticket expired</body></html>`))
|
||||
return
|
||||
}
|
||||
token := r.URL.Query().Get("t")
|
||||
s.mu.Lock()
|
||||
valid := len(s.issued) > 0 && s.issued[len(s.issued)-1] == token
|
||||
s.mu.Unlock()
|
||||
if !valid {
|
||||
// 与上游一致:空 body、无 Content-Type。
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
_, _ = w.Write([]byte("\x00\x00\x00\x20ftypmp42fake-audio-bytes"))
|
||||
case strings.HasPrefix(r.URL.Path, "/img/"):
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
// newTokenLocked 签一张新票(调用方需持有或随后获取锁;这里内部加锁)。
|
||||
func (s *mediaTicketServer) newTokenLocked(token string) string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.issued = append(s.issued, token)
|
||||
return token
|
||||
}
|
||||
|
||||
// rotate 模拟「中转又签了一张新票」:章节缓存里那张就此失效。
|
||||
func (s *mediaTicketServer) rotate() {
|
||||
s.mu.Lock()
|
||||
s.issued = append(s.issued, fmt.Sprintf("rotated-%d", len(s.issued)))
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// stopIssuing 让书源此后给不出播放地址(模拟中转本身挂了)。
|
||||
func (s *mediaTicketServer) stopIssuing() {
|
||||
s.mu.Lock()
|
||||
s.noToken = true
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// serveErrorPage 让媒体端点改用 200 + HTML 错误页(直链失效的另一种形态)。
|
||||
func (s *mediaTicketServer) serveErrorPage() {
|
||||
s.mu.Lock()
|
||||
s.errorPage = true
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// denyAlways 让媒体端点连新签的票也一律空 401(模拟上游成批拒绝的那几分钟)。
|
||||
func (s *mediaTicketServer) denyAlways() {
|
||||
s.mu.Lock()
|
||||
s.alwaysDeny = true
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *mediaTicketServer) errorPageNow() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.errorPage
|
||||
}
|
||||
|
||||
func (s *mediaTicketServer) contentHits() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.hits
|
||||
}
|
||||
|
||||
func (s *mediaTicketServer) mediaRequests() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.mediaHits
|
||||
}
|
||||
|
||||
// prepareMediaTicketBookWithUpstream 用指定的上游地址建一本书(上游由调用方提供)。
|
||||
func prepareMediaTicketBookWithUpstream(t *testing.T, upstream string) (*service.Container, *mediaTicketServer, *model.ReaderBook) {
|
||||
t.Helper()
|
||||
svc := newReaderHandlerContainer(t)
|
||||
svc.Cfg.Cache.CacheDir = t.TempDir()
|
||||
// 占位 ticket server:只用来满足清理钩子,不参与请求。
|
||||
srv := newMediaTicketServer(t)
|
||||
srcJSON := fmt.Sprintf(`{
|
||||
"bookSourceName": "上游直连测试",
|
||||
"bookSourceType": 1,
|
||||
"bookSourceUrl": %q,
|
||||
"ruleContent": {"content": "$.data.url"}
|
||||
}`, upstream)
|
||||
if _, err := svc.Reader.ImportSources(t.Context(), "u1", srcJSON); err != nil {
|
||||
t.Fatalf("导入书源失败: %v", err)
|
||||
}
|
||||
srcs, err := svc.Reader.ListSources(t.Context(), "u1")
|
||||
if err != nil || len(srcs) == 0 {
|
||||
t.Fatalf("读不到书源: %v", err)
|
||||
}
|
||||
book, err := svc.Reader.AddBook(t.Context(), "u1", reader.SearchOrigin{
|
||||
SourceID: srcs[0].ID,
|
||||
Origin: upstream,
|
||||
OriginName: "上游直连测试",
|
||||
OriginType: 1,
|
||||
BookURL: upstream + "/book/1",
|
||||
}, "上游直连之书", "某作者", "")
|
||||
if err != nil {
|
||||
t.Fatalf("加入书架失败: %v", err)
|
||||
}
|
||||
return svc, srv, book
|
||||
}
|
||||
|
||||
func prepareMediaTicketBook(t *testing.T) (*service.Container, *mediaTicketServer, *model.ReaderBook) {
|
||||
t.Helper()
|
||||
svc := newReaderHandlerContainer(t)
|
||||
// 章节缓存必须落在磁盘上:换票要凭落盘载荷找回「这个地址属于哪一章」。
|
||||
// newReaderHandlerContainer 只给空 cfg,这里补一个临时缓存目录。
|
||||
svc.Cfg.Cache.CacheDir = t.TempDir()
|
||||
srv := newMediaTicketServer(t)
|
||||
srcJSON := fmt.Sprintf(`{
|
||||
"bookSourceName": "票源测试",
|
||||
"bookSourceType": 1,
|
||||
"bookSourceUrl": %q,
|
||||
"ruleContent": {"content": "$.data.url"}
|
||||
}`, srv.URL)
|
||||
if _, err := svc.Reader.ImportSources(t.Context(), "u1", srcJSON); err != nil {
|
||||
t.Fatalf("导入书源失败: %v", err)
|
||||
}
|
||||
srcs, err := svc.Reader.ListSources(t.Context(), "u1")
|
||||
if err != nil || len(srcs) == 0 {
|
||||
t.Fatalf("读不到书源: %v", err)
|
||||
}
|
||||
book, err := svc.Reader.AddBook(t.Context(), "u1", reader.SearchOrigin{
|
||||
SourceID: srcs[0].ID,
|
||||
Origin: srv.URL,
|
||||
OriginName: "票源测试",
|
||||
OriginType: 1,
|
||||
BookURL: srv.URL + "/book/1",
|
||||
}, "票源之书", "某作者", "")
|
||||
if err != nil {
|
||||
t.Fatalf("加入书架失败: %v", err)
|
||||
}
|
||||
if err := svc.Reader.SaveChapters(t.Context(), book.ID, []reader.ChapterInput{
|
||||
{Index: 0, Title: "第一章", URL: srv.URL + "/content"},
|
||||
}); err != nil {
|
||||
t.Fatalf("保存目录失败: %v", err)
|
||||
}
|
||||
return svc, srv, book
|
||||
}
|
||||
|
||||
// TestReaderAudioStreamsFirstByteBeforeDownloadCompletes 首播 TTFB 回归:
|
||||
// 冷启动必须边下边发,而不是等整章下完再一次性下发。
|
||||
//
|
||||
// 上游在第一块之后卡住不放:直通实现此时应已把那块转给客户端,所以读端能
|
||||
// 拿到字节;旧实现(整段下载后才 ServeContent)在 release 关闭前一个字节都
|
||||
// 不会发出,读端会一直阻塞。
|
||||
func TestReaderAudioStreamsFirstByteBeforeDownloadCompletes(t *testing.T) {
|
||||
// 总长必须大于探测窗口(readerMediaProbeBytes=64KB):首播要先读完这一小段
|
||||
// 才能判断上游给的是错误页还是媒体,这之后才可能外发。卡点在探测结束之后,
|
||||
// 才能证明「整段下完才发」和「边下边发」的区别。
|
||||
const chunks = 16
|
||||
const chunkBytes = 16 << 10
|
||||
const stallAfter = 4 // 写满探测窗口(4×16KB=64KB)后卡住
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "audio/mp4")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(chunks*chunkBytes))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
for i := 0; i < chunks; i++ {
|
||||
if _, err := w.Write(bytes.Repeat([]byte("m4a!"), chunkBytes/4)); err != nil {
|
||||
return
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
if i+1 == stallAfter {
|
||||
// 探测窗口刚写满就卡住,直到测试确认客户端已经收到字节。
|
||||
<-release
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
svc, _, book := prepareMediaTicketBookWithUpstream(t, upstream.URL)
|
||||
signed := svc.Reader.ProxyURL(book.ID, upstream.URL+"/media")
|
||||
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
router.GET("/api/reader/media", readerMediaProxyHandler(svc))
|
||||
proxy := httptest.NewServer(router)
|
||||
defer proxy.Close()
|
||||
|
||||
resp, err := http.Get(proxy.URL + signed)
|
||||
if err != nil {
|
||||
t.Fatalf("请求媒体代理失败: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("直通应返回 200,实际 %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// 上游仍卡在 release 上:此时能读到第一批字节,才说明是边下边发。
|
||||
first := make([]byte, 4)
|
||||
readDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, readErr := io.ReadFull(resp.Body, first)
|
||||
readDone <- readErr
|
||||
}()
|
||||
select {
|
||||
case readErr := <-readDone:
|
||||
if readErr != nil {
|
||||
t.Fatalf("读取首批字节失败: %v", readErr)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
unblock()
|
||||
t.Fatalf("首播没有边下边发:上游卡在中途(仍有 %d 字节未发)时客户端读不到任何字节",
|
||||
(chunks-stallAfter)*chunkBytes)
|
||||
}
|
||||
if string(first) != "m4a!" {
|
||||
t.Fatalf("首批字节内容不对: %q", first)
|
||||
}
|
||||
|
||||
// 放行上游,把剩余字节读完并确认缓存落盘完整。
|
||||
unblock()
|
||||
rest, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
t.Fatalf("读取剩余字节失败: %v", readErr)
|
||||
}
|
||||
if total := 4 + len(rest); total != chunks*chunkBytes {
|
||||
t.Fatalf("交付字节 = %d,期望 %d", total, chunks*chunkBytes)
|
||||
}
|
||||
// 客户端读完最后一个字节时,服务端可能还在做 truncate + rename:
|
||||
// 落盘完成是 handler 的收尾动作,给一个有限窗口等它。
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !svc.Reader.HasCachedRemoteAudio(book.ID, upstream.URL+"/media") {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("直通完成后应命中缓存")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
+1052
-26
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,8 @@ func registerAdminRoutes(api *gin.RouterGroup, cfg *config.Config, svc *service.
|
||||
admin := api.Group("/admin")
|
||||
admin.Use(middleware.AuthRequired(cfg.Secrets.JWTSecret), middleware.AdminRequired())
|
||||
registerAdminUserRoutes(admin, svc)
|
||||
admin.GET("/tasks", backgroundTasksHandler(svc))
|
||||
admin.GET("/quality", mediaQualityHandler(svc))
|
||||
registerAdminPermissionRoutes(admin, svc)
|
||||
registerAdminSystemRoutes(admin, svc)
|
||||
registerAdminBackupRoutes(admin, svc)
|
||||
@@ -20,6 +22,7 @@ func registerAdminRoutes(api *gin.RouterGroup, cfg *config.Config, svc *service.
|
||||
registerAdminAPIConfigRoutes(admin, svc)
|
||||
registerAdminRecognitionWordRoutes(admin, svc)
|
||||
registerAdminStrmRoutes(admin, svc)
|
||||
registerAdminCloudDownloadRoutes(admin, svc)
|
||||
registerAdminScraperRoutes(admin, svc)
|
||||
registerAdminDatabaseRoutes(admin, svc)
|
||||
|
||||
@@ -143,6 +146,7 @@ func registerAdminSystemRoutes(admin *gin.RouterGroup, svc *service.Container) {
|
||||
func registerAdminBackupRoutes(admin *gin.RouterGroup, svc *service.Container) {
|
||||
admin.GET("/backups", listBackupsHandler(svc))
|
||||
admin.POST("/backups", createBackupHandler(svc))
|
||||
admin.GET("/backups/download", downloadBackupHandler(svc))
|
||||
admin.DELETE("/backups", deleteBackupHandler(svc))
|
||||
admin.POST("/backups/restore", restoreBackupHandler(svc))
|
||||
}
|
||||
|
||||
@@ -36,6 +36,9 @@ func registerAuthedLibraryRoutes(authed *gin.RouterGroup, svc *service.Container
|
||||
authed.GET("/libraries", listLibrariesHandler(svc))
|
||||
authed.POST("/libraries", middleware.AdminRequired(), createLibraryHandler(svc))
|
||||
authed.GET("/libraries/:id", getLibraryHandler(svc))
|
||||
// 服务端生成的媒体库封面(成员海报拼图),Web / App / Emby 共用同一份。
|
||||
authed.GET("/libraries/:id/cover", libraryCoverHandler(svc))
|
||||
authed.HEAD("/libraries/:id/cover", libraryCoverHandler(svc))
|
||||
authed.PATCH("/libraries/:id", middleware.AdminRequired(), updateLibraryHandler(svc))
|
||||
authed.PUT("/libraries/reorder", middleware.AdminRequired(), reorderLibrariesHandler(svc))
|
||||
authed.DELETE("/libraries/:id", middleware.AdminRequired(), deleteLibraryHandler(svc))
|
||||
|
||||
@@ -74,6 +74,7 @@ func registerAuthedPlaybackExtraRoutes(authed *gin.RouterGroup, svc *service.Con
|
||||
authed.GET("/playback/:id/info", playbackInfoHandler(svc))
|
||||
authed.GET("/playback/:id/resume", playbackResumeHandler(svc))
|
||||
authed.GET("/playback/:id/segments", playbackSegmentsHandler(svc))
|
||||
authed.PUT("/playback/:id/segments", playbackSegmentsMarkHandler(svc))
|
||||
authed.POST("/playback/:id/progress", playbackProgressHandler(svc))
|
||||
authed.GET("/playback/:id/external-players", externalPlayersHandler(svc))
|
||||
authed.GET("/playback/:id/external-url", externalURLHandler(svc))
|
||||
|
||||
@@ -27,7 +27,15 @@ func hlsPlaylistHandler(svc *service.Container) gin.HandlerFunc {
|
||||
if !enforceScopedPlaybackToken(c, m.ID) {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"))
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSPlaylist(c.Writer, c.Request, c.Param("id"), scope)
|
||||
if errors.Is(err, service.ErrHLSGenerationSuperseded) {
|
||||
c.JSON(http.StatusConflict, gin.H{"code": "hls_generation_superseded", "error": err.Error()})
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrMediaNotFound) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
|
||||
return
|
||||
@@ -62,7 +70,11 @@ func hlsSegmentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
if !enforceScopedPlaybackToken(c, m.ID) {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg"))
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
err = svc.Stream.ServeHLSSegment(c.Writer, c.Request, c.Param("id"), c.Param("seg"), scope)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -72,11 +84,39 @@ func hlsSegmentHandler(svc *service.Container) gin.HandlerFunc {
|
||||
|
||||
func stopTranscodeHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
svc.Transcoder.StopJob(c.Param("id"))
|
||||
m, err := findMediaForPlaybackEndpoint(c, svc, c.Param("id"))
|
||||
if err != nil || m == nil || !mediaVisibleForRequest(c, svc, m) {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "media not found"})
|
||||
return
|
||||
}
|
||||
if !enforceScopedPlaybackToken(c, m.ID) {
|
||||
return
|
||||
}
|
||||
scope, ok := localHLSPlaybackScope(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if svc.Transcoder != nil {
|
||||
svc.Transcoder.StopPlayback(m.ID, scope)
|
||||
}
|
||||
c.Status(http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
|
||||
func localHLSPlaybackScope(c *gin.Context) (service.HLSPlaybackScope, bool) {
|
||||
userID := currentUserID(c)
|
||||
if userID == "" {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "missing playback owner"})
|
||||
return service.HLSPlaybackScope{}, false
|
||||
}
|
||||
scope, err := service.HLSPlaybackScopeForRequest(c.Request, userID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return service.HLSPlaybackScope{}, false
|
||||
}
|
||||
return scope, true
|
||||
}
|
||||
|
||||
func imageProxyHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
raw := c.Query("url")
|
||||
|
||||
@@ -27,21 +27,45 @@ import (
|
||||
|
||||
// historyListHandler returns the caller's history rows joined with the
|
||||
// matching media in a single response.
|
||||
//
|
||||
// include_media=0 只返回历史行本身。客户端判断「继续播放」指向哪一集时只
|
||||
// 需要 media_id/position_ms;带上整条 media 会让 300 条的响应从 ~40KB 涨到
|
||||
// ~420KB。更重要的是服务端不再 hydrate:不查整条 media、也不为挂载条目
|
||||
// 逐条请求上游 Emby(见 PlaybackService.RecentHistoryWithoutMedia)。
|
||||
func historyListHandler(svc *service.Container) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
uid, _ := c.Get(middleware.CtxUserID)
|
||||
userID := toString(uid)
|
||||
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 50
|
||||
}
|
||||
items, err := svc.Playback.RecentHistory(c.Request.Context(), toString(uid), limit)
|
||||
includeMedia := true
|
||||
if raw := strings.TrimSpace(c.Query("include_media")); raw != "" {
|
||||
if parsed, err := strconv.ParseBool(raw); err == nil {
|
||||
includeMedia = parsed
|
||||
}
|
||||
}
|
||||
ctx := c.Request.Context()
|
||||
visibility := mediaVisibilityForRequest(c, svc)
|
||||
if !includeMedia {
|
||||
// 可见性由服务层用窄列查询判定,handler 不再需要 media。
|
||||
items, err := svc.Playback.RecentHistoryWithoutMedia(ctx, userID, limit, visibility)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, items)
|
||||
return
|
||||
}
|
||||
items, err := svc.Playback.RecentHistory(ctx, userID, limit)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
visibility := mediaVisibilityForRequest(c, svc)
|
||||
filtered := make([]service.HistoryItem, 0, len(items))
|
||||
for _, item := range items {
|
||||
// 可见性按 media 判断:媒体已删除/远端取不到时仍然保留这条历史。
|
||||
if item.Media == nil || visibility.Allows(item.Media) {
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
|
||||
@@ -103,3 +103,149 @@ func TestHistoryContinueRemovesMissingMedia(t *testing.T) {
|
||||
t.Fatalf("stale history rows remaining = %d, want 0", staleCount)
|
||||
}
|
||||
}
|
||||
|
||||
// include_media=0 只是不把 media 写进响应,行数和可见性判断都不变。
|
||||
func TestHistoryListCanOmitEmbeddedMedia(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.PlaybackHistory{}, &model.Media{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
media := model.Media{
|
||||
Base: model.Base{ID: "media-1"},
|
||||
Title: "影片",
|
||||
Path: "/media/one.mkv",
|
||||
}
|
||||
if err := db.Create(&media).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.PlaybackHistory{
|
||||
Base: model.Base{ID: "history-1"},
|
||||
UserID: "user-1",
|
||||
MediaID: media.ID,
|
||||
PositionMs: 30_000,
|
||||
DurationMs: 120_000,
|
||||
WatchedAt: time.Now(),
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
repos := repository.New(db)
|
||||
svc := &service.Container{
|
||||
Log: zap.NewNop(),
|
||||
Repo: repos,
|
||||
Playback: service.NewPlaybackService(zap.NewNop(), repos),
|
||||
}
|
||||
router := gin.New()
|
||||
router.Use(func(c *gin.Context) {
|
||||
c.Set(middleware.CtxUserID, "user-1")
|
||||
c.Next()
|
||||
})
|
||||
router.GET("/watch-history", historyListHandler(svc))
|
||||
|
||||
type listRow struct {
|
||||
MediaID string `json:"media_id"`
|
||||
Media *model.Media `json:"media"`
|
||||
}
|
||||
|
||||
full := httptest.NewRecorder()
|
||||
router.ServeHTTP(full, httptest.NewRequest(http.MethodGet, "/watch-history?limit=10", nil))
|
||||
var fullRows []listRow
|
||||
if err := json.Unmarshal(full.Body.Bytes(), &fullRows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fullRows) != 1 || fullRows[0].Media == nil || fullRows[0].Media.ID != media.ID {
|
||||
t.Fatalf("default response should embed media: %s", full.Body.String())
|
||||
}
|
||||
|
||||
lean := httptest.NewRecorder()
|
||||
router.ServeHTTP(lean, httptest.NewRequest(http.MethodGet, "/watch-history?limit=10&include_media=0", nil))
|
||||
var leanRows []listRow
|
||||
if err := json.Unmarshal(lean.Body.Bytes(), &leanRows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(leanRows) != 1 || leanRows[0].MediaID != media.ID {
|
||||
t.Fatalf("lean response should keep the row: %s", lean.Body.String())
|
||||
}
|
||||
if leanRows[0].Media != nil {
|
||||
t.Fatalf("include_media=0 must not embed media: %s", lean.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestHistoryListIncludeMediaFalseSkipsHydration 保证 include_media=0 时服务端
|
||||
// 真的不 hydrate media:响应里只有历史行(客户端按 media_id 匹配续播目标)。
|
||||
func TestHistoryListIncludeMediaFalseSkipsHydration(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.PlaybackHistory{}, &model.Media{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&model.Media{
|
||||
Base: model.Base{ID: "media-1"},
|
||||
Title: "影片",
|
||||
Path: "/media/1.mkv",
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
for i, mediaID := range []string{"media-1", "media-deleted"} {
|
||||
if err := db.Create(&model.PlaybackHistory{
|
||||
Base: model.Base{ID: "history-" + mediaID},
|
||||
UserID: "user-1",
|
||||
MediaID: mediaID,
|
||||
PositionMs: 30_000,
|
||||
DurationMs: 120_000,
|
||||
WatchedAt: now.Add(-time.Duration(i) * time.Minute),
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
repos := repository.New(db)
|
||||
svc := &service.Container{
|
||||
Log: zap.NewNop(),
|
||||
Repo: repos,
|
||||
Playback: service.NewPlaybackService(zap.NewNop(), repos),
|
||||
}
|
||||
router := gin.New()
|
||||
router.Use(func(c *gin.Context) {
|
||||
c.Set(middleware.CtxUserID, "user-1")
|
||||
c.Next()
|
||||
})
|
||||
router.GET("/watch-history", historyListHandler(svc))
|
||||
|
||||
for _, path := range []string{
|
||||
"/watch-history?limit=10&include_media=0",
|
||||
"/watch-history?limit=10&include_media=false",
|
||||
} {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s status = %d, body = %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
var rows []map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
|
||||
t.Fatalf("%s decode: %v", path, err)
|
||||
}
|
||||
// 媒体已删除的历史也保留:客户端只按 media_id 判断有没有记录。
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("%s rows = %d, want 2 (%s)", path, len(rows), rec.Body.String())
|
||||
}
|
||||
for _, row := range rows {
|
||||
if _, ok := row["media"]; ok {
|
||||
t.Fatalf("%s should not embed media: %v", path, row)
|
||||
}
|
||||
if row["media_id"] == "" || row["media_id"] == nil {
|
||||
t.Fatalf("%s missing media_id: %v", path, row)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+43
-4
@@ -7,6 +7,7 @@ import (
|
||||
"compress/flate"
|
||||
"compress/gzip"
|
||||
"compress/zlib"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -88,10 +89,51 @@ func StripAcceptEncoding(h http.Header) {
|
||||
//
|
||||
// When useProxy is false, the client is created without proxy plumbing so
|
||||
// the request goes out direct, matching the user's checkbox intent.
|
||||
//
|
||||
// 注意 http.Client.Timeout 是「整个请求」的上限,包含读完响应体。书源的
|
||||
// 搜索/目录/正文都是小响应,30 秒足够;但同一个 client 也被用来流转发
|
||||
// 音频/视频(见 NewMediaHTTPClient),那里必须换一个没有整体超时的 client。
|
||||
func NewSiteHTTPClient(timeoutSeconds int, useProxy bool) *http.Client {
|
||||
if timeoutSeconds <= 0 {
|
||||
timeoutSeconds = 15
|
||||
}
|
||||
return &http.Client{
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
Transport: newSiteTransport(useProxy),
|
||||
}
|
||||
}
|
||||
|
||||
// NewMediaHTTPClient 拉取并转发媒体流(音频/视频/大图)的客户端。
|
||||
//
|
||||
// 为什么不能用 NewSiteHTTPClient:http.Client.Timeout 会覆盖「读完 body」这个
|
||||
// 阶段。一个 4MB 的 m4a 在跨境链路上很容易超过 30 秒,一旦超时,net/http 会
|
||||
// 直接掐断响应体,而调用方的 io.Copy 只是正常返回 —— 表现为**静默截断**
|
||||
// (播放器拿到半首歌,或判「格式不支持」),或者上游慢到 Do 本身报错 → 502。
|
||||
//
|
||||
// 为什么强制 HTTP/1.1:实测腾讯 CDN(vb.stream.tencentmusic.com)的 h2 边缘在
|
||||
// 长连接大文件上不按预期回 WINDOW_UPDATE,Go 的 http2 Transport 初始连接窗口
|
||||
// 是 1MiB,于是下载**稳定卡在 1MiB/2MiB/3MiB 整数倍**处断开;同一 URL 用
|
||||
// HTTP/1.1(wget、curl --http1.1)从头到尾都完整。因此媒体通路固定走 h1.1:
|
||||
// 音视频本来就没有「多路复用省请求」的收益,反而是流控 bug 的受害者。
|
||||
func NewMediaHTTPClient(useProxy bool) *http.Client {
|
||||
tr := newSiteTransport(useProxy)
|
||||
// 等响应头最多 90 秒:跨境 CDN 首包偶发要几十秒,但绝不能无上限挂着。
|
||||
tr.ResponseHeaderTimeout = 90 * time.Second
|
||||
// 媒体是长连接大流量,连接池开大一些,避免并发取图/拖动进度时互相排队。
|
||||
tr.MaxIdleConns = 32
|
||||
tr.MaxIdleConnsPerHost = 16
|
||||
// 关闭 HTTP/2:见上面的流控说明。
|
||||
tr.ForceAttemptHTTP2 = false
|
||||
tr.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
|
||||
// 不复用连接:实测腾讯 CDN 的边缘在「同一条 keep-alive 连接上连续取多个
|
||||
// 大对象」时会直接 RST 第二个请求的响应体(容器内 wget/curl 每次新建连接
|
||||
// 就完全正常)。音视频请求本来就稀疏,为省一次 TLS 握手去冒截断风险不划算。
|
||||
tr.DisableKeepAlives = true
|
||||
return &http.Client{Transport: tr}
|
||||
}
|
||||
|
||||
// newSiteTransport 书源请求共用的 Transport(媒体与规则请求配置一致的部分)。
|
||||
func newSiteTransport(useProxy bool) *http.Transport {
|
||||
tr := &http.Transport{
|
||||
MaxIdleConns: 16,
|
||||
MaxIdleConnsPerHost: 4,
|
||||
@@ -104,10 +146,7 @@ func NewSiteHTTPClient(timeoutSeconds int, useProxy bool) *http.Client {
|
||||
return ProxyFromEnvironmentOrSystem(r)
|
||||
}
|
||||
}
|
||||
return &http.Client{
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
Transport: tr,
|
||||
}
|
||||
return tr
|
||||
}
|
||||
|
||||
// HTTPHeaderPresets returns a map of realistic browser HTTP headers.
|
||||
|
||||
@@ -17,12 +17,24 @@ var gzipExcludedExtensions = []string{
|
||||
|
||||
// apiGzipExcludedPrefixes 大文件流式传输(Range 语义)与 WS/SSE 长连接
|
||||
// 不参与 gzip:压缩会破坏 Range / 逐块推送语义。
|
||||
//
|
||||
// 阅读媒体代理(/api/reader/media、/api/reader/local/audio、
|
||||
// /api/reader/audio/transcode)同样必须排除,原因有两条:
|
||||
// - 它们透传上游的 206 Partial Content。gin-contrib/gzip 在压缩时会删掉
|
||||
// Content-Length,却把 Content-Range 原样留下,客户端看到的是「声明
|
||||
// N 字节、实际收到 M 字节的 gzip 流」,自己发 Accept-Encoding 的播放器
|
||||
// (ExoPlayer 不透明处理)直接判 ERROR_CODE_PARSING_CONTAINER_UNSUPPORTED。
|
||||
// - 音视频本来就是压缩格式,再 gzip 一遍只烧 CPU(3.5MB 的 m4a 压完还是
|
||||
// 3.5MB 量级),对带宽没有收益。
|
||||
var apiGzipExcludedPrefixes = []string{
|
||||
"/api/stream/",
|
||||
"/api/hls/",
|
||||
"/api/img",
|
||||
"/api/subtitles/",
|
||||
"/api/strm/play/",
|
||||
"/api/reader/media",
|
||||
"/api/reader/local/audio",
|
||||
"/api/reader/audio/transcode",
|
||||
"/api/ws",
|
||||
"/api/events",
|
||||
}
|
||||
|
||||
@@ -70,6 +70,52 @@ func TestGzipAPIExcludesStreamPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestGzipAPIExcludesReaderMedia 阅读媒体代理必须排除在 gzip 之外。
|
||||
//
|
||||
// 它会透传 CDN 的 206 Partial Content。gin-contrib/gzip 压缩时会删掉
|
||||
// Content-Length 却保留 Content-Range,客户端看到「声明 N 字节、实收 M 字节」,
|
||||
// 自己发 Accept-Encoding 的播放器(ExoPlayer)直接判
|
||||
// ERROR_CODE_PARSING_CONTAINER_UNSUPPORTED,表现为「音频格式不支持」。
|
||||
func TestGzipAPIExcludesReaderMedia(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
api := router.Group("/api")
|
||||
api.Use(GzipAPI())
|
||||
// 用 206 + Content-Range 模拟真实的音频分段响应。
|
||||
segment := func(c *gin.Context) {
|
||||
c.Header("Content-Range", "bytes 0-99/100")
|
||||
c.Data(http.StatusPartialContent, "audio/mp4", []byte(strings.Repeat("x", 4096)))
|
||||
}
|
||||
for _, path := range []string{
|
||||
"/reader/media",
|
||||
"/reader/local/audio",
|
||||
"/reader/audio/transcode",
|
||||
} {
|
||||
api.GET(path, segment)
|
||||
}
|
||||
|
||||
for _, path := range []string{
|
||||
"/reader/media",
|
||||
"/reader/local/audio",
|
||||
"/reader/audio/transcode",
|
||||
} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api"+path+"?b=book&u=x&s=y", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if got := w.Header().Get("Content-Encoding"); got == "gzip" {
|
||||
t.Errorf("%s 不该被 gzip(Range 语义会被破坏)", path)
|
||||
}
|
||||
if got := w.Header().Get("Content-Range"); got != "bytes 0-99/100" {
|
||||
t.Errorf("%s 的 Content-Range 被破坏: %q", path, got)
|
||||
}
|
||||
if w.Body.Len() != 4096 {
|
||||
t.Errorf("%s 响应体长度 = %d,期望 4096", path, w.Body.Len())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGzipStaticCompressesAssets(t *testing.T) {
|
||||
router := newGzipTestRouter()
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// BackgroundTaskRecord persists the operator-facing lifecycle of scan,
|
||||
// organize, and other long-running application tasks.
|
||||
type BackgroundTaskRecord struct {
|
||||
ID string `gorm:"primaryKey;size:36" json:"id"`
|
||||
Kind string `gorm:"index;size:32;not null" json:"kind"`
|
||||
Name string `gorm:"size:255;not null" json:"name"`
|
||||
Status string `gorm:"index;size:24;not null" json:"status"`
|
||||
Stage string `gorm:"size:64" json:"stage"`
|
||||
SourcePath string `gorm:"size:2048" json:"source_path"`
|
||||
DestPath string `gorm:"size:2048" json:"dest_path"`
|
||||
Message string `gorm:"type:text" json:"message"`
|
||||
Error string `gorm:"type:text" json:"error"`
|
||||
Details string `gorm:"type:text" json:"details"`
|
||||
Metrics string `gorm:"type:text" json:"metrics"`
|
||||
Items string `gorm:"type:text" json:"items"`
|
||||
StartedAt time.Time `gorm:"index" json:"started_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// 115 云下载(离线下载)任务数据模型。
|
||||
//
|
||||
// 一条 CloudDownloadTask 表示用户提交的一个离线下载来源(磁力 / BT 种子 /
|
||||
// ed2k / HTTP),并记录它在「115 下载 → STRM 同步 → 媒体库扫描 → 刮削」
|
||||
// 链路上的位置。下载状态(Status)与后处理阶段(Stage)刻意分成两组字段:
|
||||
// STRM 同步或刮削失败不能把 115 侧的「已下载成功」改写掉,用户重做后处理时
|
||||
// 也不需要重新下载。
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// 云下载任务状态(115 离线下载侧)。
|
||||
const (
|
||||
CloudDownloadQueued = "queued" // 本地已建行,尚未提交 115
|
||||
CloudDownloadSubmitted = "submitted" // 115 已受理
|
||||
CloudDownloadDownloading = "downloading" // 115 正在下载(status 0/1)
|
||||
CloudDownloadSucceeded = "succeeded" // 115 下载完成(status 2)
|
||||
CloudDownloadFailed = "failed" // 提交被拒 / status -1 / 超时
|
||||
CloudDownloadCanceled = "canceled" // 用户取消
|
||||
)
|
||||
|
||||
// 云下载后处理阶段(STRM 同步 → 扫描 → 刮削)。
|
||||
const (
|
||||
CloudDownloadStagePending = "pending" // 尚未开始(下载未成功)
|
||||
CloudDownloadStageWaitingSync = "waiting_sync" // 已成功,等待去抖窗口或等待目录空闲
|
||||
CloudDownloadStageSyncing = "syncing" // STRM 增量同步中
|
||||
CloudDownloadStageScanning = "scanning" // 媒体库扫描中
|
||||
CloudDownloadStageScraping = "scraping" // 刮削已入队
|
||||
CloudDownloadStageDone = "done" // 全链路完成
|
||||
CloudDownloadStageNA = "na" // 无法继续(同步目录/媒体库不存在),不算失败
|
||||
CloudDownloadStageFailed = "failed" // 阶段失败,可用「重做后处理」重试
|
||||
)
|
||||
|
||||
// 云下载来源类型。
|
||||
const (
|
||||
CloudDownloadSourceMagnet = "magnet"
|
||||
CloudDownloadSourceTorrent = "torrent"
|
||||
CloudDownloadSourceED2K = "ed2k"
|
||||
CloudDownloadSourceHTTP = "http"
|
||||
CloudDownloadSourceFTP = "ftp"
|
||||
)
|
||||
|
||||
// CloudDownloadTask 是一条云下载任务。
|
||||
type CloudDownloadTask struct {
|
||||
Base
|
||||
|
||||
AccountID string `gorm:"size:36;index" json:"account_id"` // 115 网盘账号(model.StrmAccount)
|
||||
SyncPathID string `gorm:"size:36;index" json:"sync_path_id"` // 绑定的 STRM 同步目录
|
||||
LibraryID string `gorm:"size:36;index" json:"library_id"` // 由本地目录反查得到,可为空
|
||||
Provider string `gorm:"size:32" json:"provider"` // 固定 cloud115
|
||||
|
||||
SourceType string `gorm:"size:16;index" json:"source_type"` // CloudDownloadSource*
|
||||
SourceURL string `gorm:"type:text" json:"source_url"` // 原始链接;.torrent 时存上传的文件名
|
||||
// SourceKey 是轮询匹配用的归一化键:BT 类任务存大写 info_hash,链接类任务
|
||||
// 存 trim 后的 url。提交时即写入,避免每轮重新计算。
|
||||
SourceKey string `gorm:"size:512;index" json:"source_key"`
|
||||
InfoHash string `gorm:"size:64;index" json:"info_hash,omitempty"`
|
||||
|
||||
TorrentName string `gorm:"size:512" json:"torrent_name,omitempty"`
|
||||
TorrentSha1 string `gorm:"size:64" json:"torrent_sha1,omitempty"` // .torrent 提交路径用
|
||||
PickCode string `gorm:"size:128" json:"pick_code,omitempty"` // .torrent 提交路径用
|
||||
|
||||
WpPathID string `gorm:"size:64" json:"wp_path_id"` // 115 目标目录 ID(= syncPath.RemotePath)
|
||||
SavePath string `gorm:"size:512" json:"save_path,omitempty"` // 相对 wp_path_id 的子路径(预留)
|
||||
|
||||
Status string `gorm:"size:16;index" json:"status"` // CloudDownload*
|
||||
Percent int `json:"percent"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Error string `gorm:"size:1024" json:"error"`
|
||||
|
||||
Stage string `gorm:"size:16;index" json:"stage"` // CloudDownloadStage*
|
||||
StageError string `gorm:"size:1024" json:"stage_error"`
|
||||
StrmRecordID string `gorm:"size:36" json:"strm_record_id,omitempty"`
|
||||
ScanAdded int `json:"scan_added"`
|
||||
ScrapeQueued int `json:"scrape_queued"`
|
||||
|
||||
RetryCount int `json:"retry_count"`
|
||||
LastPollAt *time.Time `json:"last_poll_at,omitempty"`
|
||||
NextPollAt *time.Time `gorm:"index" json:"next_poll_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// Library 表示一个逻辑媒体库。Path 保留为兼容字段,指向第一条 LibraryRoot。
|
||||
type Library struct {
|
||||
Base
|
||||
@@ -26,12 +28,15 @@ type LibraryRoot struct {
|
||||
// Media 是单个可播放项。剧集链接到 SeriesID;电影 SeriesID == ""。
|
||||
type Media struct {
|
||||
Base
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
AvailabilityState string `gorm:"index;size:16;not null;default:available" json:"availability_state"`
|
||||
MissingSince *time.Time `json:"missing_since,omitempty"`
|
||||
LastSeenAt *time.Time `json:"last_seen_at,omitempty"`
|
||||
LibraryID string `gorm:"index;size:36;index:idx_media_library_release,priority:1" json:"library_id"`
|
||||
LibraryRootID string `gorm:"index;size:36" json:"library_root_id,omitempty"`
|
||||
SeriesID string `gorm:"index;size:128" json:"series_id,omitempty"`
|
||||
Title string `gorm:"size:255;not null" json:"title"`
|
||||
OriginalName string `gorm:"size:255" json:"original_name,omitempty"`
|
||||
EpisodeTitle string `gorm:"size:255" json:"episode_title,omitempty"`
|
||||
// Path 的唯一性由 database 包按方言显式建索引,这里刻意不用 uniqueIndex:
|
||||
// PostgreSQL 的 btree 单条索引项上限 2704 字节,而 varchar(1024) 在中文路径
|
||||
// 下可达 3072 字节,直接对整列建唯一索引会让长路径的扫描插入直接失败
|
||||
|
||||
@@ -60,7 +60,9 @@ func AllModels() []interface{} {
|
||||
&StrmDownloadTask{},
|
||||
&StrmUploadTask{},
|
||||
&StrmDirCache{},
|
||||
&CloudDownloadTask{},
|
||||
&ScrapeTask{},
|
||||
&BackgroundTaskRecord{},
|
||||
&EmbyMount{},
|
||||
&ReaderBookSource{},
|
||||
&ReaderSourceState{},
|
||||
|
||||
@@ -38,6 +38,12 @@ type ReaderBookSource struct {
|
||||
// NeedsBrowser 是否依赖 WebView/无头浏览器(webView 选项或 webjs 规则)。
|
||||
// 服务端没有浏览器,这类源注定不可用,列表接口按需计算让前端能提前提示。
|
||||
NeedsBrowser bool `gorm:"-" json:"needs_browser"`
|
||||
// MediaDirect 允许客户端直连上游取实体媒体(音频/漫画图片),服务端只负责
|
||||
// 解析地址与下发所需请求头。
|
||||
//
|
||||
// 只在客户端显式请求直连时生效(见 ContentOptions):网页端不声明该能力,
|
||||
// 行为与开启此字段前完全一致。
|
||||
MediaDirect bool `gorm:"default:false" json:"media_direct"`
|
||||
}
|
||||
|
||||
// ReaderSourceState 书源会话状态:对应 legado 中按书源 key 存储的
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// CloudDownloadRepository persists model.CloudDownloadTask(115 云下载任务)。
|
||||
type CloudDownloadRepository struct{ db *gorm.DB }
|
||||
|
||||
// CloudDownloadQuery 是任务列表的过滤条件。
|
||||
type CloudDownloadQuery struct {
|
||||
Status string // 精确匹配;空表示全部
|
||||
SyncPathID string
|
||||
AccountID string
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
// Create 插入一条云下载任务。
|
||||
func (r *CloudDownloadRepository) Create(ctx context.Context, t *model.CloudDownloadTask) error {
|
||||
return withSQLiteBusyRetry(ctx, func() error {
|
||||
return r.db.WithContext(ctx).Create(t).Error
|
||||
})
|
||||
}
|
||||
|
||||
// Update 更新任务的可变字段(不含主键与创建时间)。
|
||||
func (r *CloudDownloadRepository) Update(ctx context.Context, t *model.CloudDownloadTask) error {
|
||||
return r.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"account_id": t.AccountID,
|
||||
"sync_path_id": t.SyncPathID,
|
||||
"library_id": t.LibraryID,
|
||||
"provider": t.Provider,
|
||||
"source_type": t.SourceType,
|
||||
"source_url": t.SourceURL,
|
||||
"source_key": t.SourceKey,
|
||||
"info_hash": t.InfoHash,
|
||||
"torrent_name": t.TorrentName,
|
||||
"torrent_sha1": t.TorrentSha1,
|
||||
"pick_code": t.PickCode,
|
||||
"wp_path_id": t.WpPathID,
|
||||
"save_path": t.SavePath,
|
||||
"status": t.Status,
|
||||
"percent": t.Percent,
|
||||
"size_bytes": t.SizeBytes,
|
||||
"error": t.Error,
|
||||
"stage": t.Stage,
|
||||
"stage_error": t.StageError,
|
||||
"strm_record_id": t.StrmRecordID,
|
||||
"scan_added": t.ScanAdded,
|
||||
"scrape_queued": t.ScrapeQueued,
|
||||
"retry_count": t.RetryCount,
|
||||
"last_poll_at": t.LastPollAt,
|
||||
"next_poll_at": t.NextPollAt,
|
||||
"completed_at": t.CompletedAt,
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateFields 局部更新:调用方只写自己关心的列,避免覆盖并发写入的其它阶段字段
|
||||
// (轮询线程写 Status,后处理线程写 Stage,两者必须互不覆盖)。
|
||||
func (r *CloudDownloadRepository) UpdateFields(ctx context.Context, id string, fields map[string]any) error {
|
||||
if id == "" || len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
fields["updated_at"] = time.Now()
|
||||
return withSQLiteBusyRetry(ctx, func() error {
|
||||
return r.db.WithContext(ctx).Model(&model.CloudDownloadTask{}).Where("id = ?", id).Updates(fields).Error
|
||||
})
|
||||
}
|
||||
|
||||
// FindByID 按主键查询,未找到返回 (nil, nil)。
|
||||
func (r *CloudDownloadRepository) FindByID(ctx context.Context, id string) (*model.CloudDownloadTask, error) {
|
||||
var t model.CloudDownloadTask
|
||||
err := r.db.WithContext(ctx).Where("id = ?", id).First(&t).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// List 分页查询任务列表,同时返回总数。
|
||||
func (r *CloudDownloadRepository) List(ctx context.Context, q CloudDownloadQuery) ([]model.CloudDownloadTask, int64, error) {
|
||||
db := r.db.WithContext(ctx).Model(&model.CloudDownloadTask{})
|
||||
if q.Status != "" {
|
||||
db = db.Where("status = ?", q.Status)
|
||||
}
|
||||
if q.SyncPathID != "" {
|
||||
db = db.Where("sync_path_id = ?", q.SyncPathID)
|
||||
}
|
||||
if q.AccountID != "" {
|
||||
db = db.Where("account_id = ?", q.AccountID)
|
||||
}
|
||||
var total int64
|
||||
if err := db.Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
page := q.Page
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
size := q.PageSize
|
||||
if size <= 0 {
|
||||
size = 20
|
||||
}
|
||||
if size > 200 {
|
||||
size = 200
|
||||
}
|
||||
var rows []model.CloudDownloadTask
|
||||
err := db.Order("created_at desc").
|
||||
Offset((page - 1) * size).
|
||||
Limit(size).
|
||||
Find(&rows).Error
|
||||
return rows, total, err
|
||||
}
|
||||
|
||||
// ListByStatuses 取指定状态的任务(轮询用)。
|
||||
func (r *CloudDownloadRepository) ListByStatuses(ctx context.Context, statuses []string) ([]model.CloudDownloadTask, error) {
|
||||
if len(statuses) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var rows []model.CloudDownloadTask
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("status IN ?", statuses).
|
||||
Order("created_at asc").
|
||||
Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// ListBySyncPathAndStage 取某个同步目录下处于指定阶段的任务(后处理推进用)。
|
||||
func (r *CloudDownloadRepository) ListBySyncPathAndStage(ctx context.Context, syncPathID, stage string) ([]model.CloudDownloadTask, error) {
|
||||
var rows []model.CloudDownloadTask
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("sync_path_id = ? AND stage = ?", syncPathID, stage).
|
||||
Order("created_at asc").
|
||||
Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// ListPipelineCandidates 取「下载已成功、后处理尚未完成」的任务。
|
||||
func (r *CloudDownloadRepository) ListPipelineCandidates(ctx context.Context) ([]model.CloudDownloadTask, error) {
|
||||
var rows []model.CloudDownloadTask
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("status = ? AND stage IN ?", model.CloudDownloadSucceeded,
|
||||
[]string{model.CloudDownloadStagePending, model.CloudDownloadStageWaitingSync}).
|
||||
Order("created_at asc").
|
||||
Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// ResetStaleStages 把上次进程退出时停在中间阶段的任务重置为待推进。
|
||||
// STRM 同步/扫描/刮削都是幂等操作,重启后重新推进是安全的。
|
||||
func (r *CloudDownloadRepository) ResetStaleStages(ctx context.Context) (int64, error) {
|
||||
var affected int64
|
||||
err := withSQLiteBusyRetry(ctx, func() error {
|
||||
res := r.db.WithContext(ctx).Model(&model.CloudDownloadTask{}).
|
||||
Where("stage IN ?", []string{
|
||||
model.CloudDownloadStageSyncing,
|
||||
model.CloudDownloadStageScanning,
|
||||
model.CloudDownloadStageScraping,
|
||||
}).
|
||||
Updates(map[string]any{
|
||||
"stage": model.CloudDownloadStagePending,
|
||||
"stage_error": "",
|
||||
"updated_at": time.Now(),
|
||||
})
|
||||
affected = res.RowsAffected
|
||||
return res.Error
|
||||
})
|
||||
return affected, err
|
||||
}
|
||||
|
||||
// FindActiveBySourceKeys 返回给定匹配键中仍处于活跃状态的任务键集合。
|
||||
// 用于提交时去重:同一个磁力重复提交只会浪费 115 配额。
|
||||
func (r *CloudDownloadRepository) FindActiveBySourceKeys(ctx context.Context, keys []string) (map[string]bool, error) {
|
||||
out := make(map[string]bool, len(keys))
|
||||
if len(keys) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var found []string
|
||||
err := r.db.WithContext(ctx).Model(&model.CloudDownloadTask{}).
|
||||
Where("source_key IN ?", keys).
|
||||
Where("status IN ?", []string{
|
||||
model.CloudDownloadQueued,
|
||||
model.CloudDownloadSubmitted,
|
||||
model.CloudDownloadDownloading,
|
||||
}).
|
||||
Pluck("source_key", &found).Error
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, k := range found {
|
||||
out[k] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Delete 硬删除一条任务记录。
|
||||
func (r *CloudDownloadRepository) Delete(ctx context.Context, id string) error {
|
||||
return withSQLiteBusyRetry(ctx, func() error {
|
||||
return r.db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.CloudDownloadTask{}).Error
|
||||
})
|
||||
}
|
||||
|
||||
// ClearFinished 删除所有「无后续价值」的终态任务(失败/已取消)。
|
||||
// 成功的任务保留:它们承载「本地已生成 strm / 已刮削」的证据,供用户回查。
|
||||
func (r *CloudDownloadRepository) ClearFinished(ctx context.Context) (int64, error) {
|
||||
var affected int64
|
||||
err := withSQLiteBusyRetry(ctx, func() error {
|
||||
res := r.db.WithContext(ctx).Unscoped().
|
||||
Where("status IN ?", []string{
|
||||
model.CloudDownloadFailed,
|
||||
model.CloudDownloadCanceled,
|
||||
}).
|
||||
Delete(&model.CloudDownloadTask{})
|
||||
affected = res.RowsAffected
|
||||
return res.Error
|
||||
})
|
||||
return affected, err
|
||||
}
|
||||
|
||||
// CountBySyncPath 统计某同步目录下仍在活跃的任务数(前端展示用)。
|
||||
func (r *CloudDownloadRepository) CountBySyncPath(ctx context.Context, syncPathID string) (int64, error) {
|
||||
var n int64
|
||||
err := r.db.WithContext(ctx).Model(&model.CloudDownloadTask{}).
|
||||
Where("sync_path_id = ?", syncPathID).
|
||||
Where("status IN ?", []string{
|
||||
model.CloudDownloadQueued,
|
||||
model.CloudDownloadSubmitted,
|
||||
model.CloudDownloadDownloading,
|
||||
}).
|
||||
Count(&n).Error
|
||||
return n, err
|
||||
}
|
||||
@@ -123,6 +123,20 @@ func (r *LibraryRepository) ListRoots(ctx context.Context, libraryID string) ([]
|
||||
return roots, err
|
||||
}
|
||||
|
||||
// ListEnabledRoots 返回全部已启用媒体库的全部启用路径。
|
||||
// 用于「本地目录 → 媒体库」的反查(云下载完成后只扫相关库,而不是全库重扫)。
|
||||
func (r *LibraryRepository) ListEnabledRoots(ctx context.Context) ([]model.LibraryRoot, error) {
|
||||
if !r.hasLibraryRootsTable() {
|
||||
return nil, nil
|
||||
}
|
||||
var roots []model.LibraryRoot
|
||||
err := r.db.WithContext(ctx).
|
||||
Where("enabled = ?", true).
|
||||
Order("sort_order asc, created_at asc").
|
||||
Find(&roots).Error
|
||||
return roots, err
|
||||
}
|
||||
|
||||
func (r *LibraryRepository) FindRootByID(ctx context.Context, libraryID, rootID string) (*model.LibraryRoot, error) {
|
||||
if !r.hasLibraryRootsTable() {
|
||||
return nil, nil
|
||||
|
||||
@@ -237,10 +237,16 @@ func (r *MediaRepository) ListSeasonSiblings(ctx context.Context, m *model.Media
|
||||
} else {
|
||||
libraryID := strings.TrimSpace(m.LibraryID)
|
||||
title := strings.TrimSpace(m.Title)
|
||||
if libraryID == "" || title == "" || m.TMDbID <= 0 {
|
||||
if libraryID == "" || title == "" {
|
||||
return nil, nil
|
||||
}
|
||||
query = query.Where("library_id = ? AND title = ? AND tm_db_id = ?", libraryID, title, m.TMDbID)
|
||||
// 没有 series_id 也没有 TMDb id 时,退到「同库 + 同标题」:手动标记的按季
|
||||
// 传播是用户显式操作,同库同名已经足够可靠;而在缺 id 时直接放弃会让这个
|
||||
// 功能在未刮削的库上完全不生效。
|
||||
query = query.Where("library_id = ? AND title = ?", libraryID, title)
|
||||
if m.TMDbID > 0 {
|
||||
query = query.Where("tm_db_id = ?", m.TMDbID)
|
||||
}
|
||||
}
|
||||
var rows []model.Media
|
||||
if err := query.Find(&rows).Error; err != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
@@ -163,6 +164,12 @@ func (r *MediaRepository) upsertWithDB(ctx context.Context, db *gorm.DB, m *mode
|
||||
existing.Path = m.Path
|
||||
existing.DeletedAt = gorm.DeletedAt{}
|
||||
}
|
||||
if _, ok := updates["availability_state"]; ok {
|
||||
existing.AvailabilityState = "available"
|
||||
existing.MissingSince = nil
|
||||
now := updates["last_seen_at"].(time.Time)
|
||||
existing.LastSeenAt = &now
|
||||
}
|
||||
*m = existing
|
||||
return existing.ID, nil
|
||||
}
|
||||
@@ -216,6 +223,10 @@ func (r *MediaRepository) findOrCreateMediaByPath(ctx context.Context, db *gorm.
|
||||
if m.ScrapeStatus == "" {
|
||||
m.ScrapeStatus = "pending"
|
||||
}
|
||||
now := time.Now()
|
||||
m.AvailabilityState = "available"
|
||||
m.MissingSince = nil
|
||||
m.LastSeenAt = &now
|
||||
// PostgreSQL 唯一键异常会中止整个事务,之后重查也会失败。
|
||||
// DO NOTHING 保持事务可用,并兼容 path 的表达式唯一索引。
|
||||
result := db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(m)
|
||||
@@ -265,6 +276,11 @@ func (r *MediaRepository) findMediaByAlias(ctx context.Context, db *gorm.DB, cur
|
||||
|
||||
func mediaUpsertUpdates(existing, incoming model.Media, reclaimLibrary bool) map[string]any {
|
||||
updates := map[string]any{}
|
||||
if existing.AvailabilityState != "available" || existing.MissingSince != nil {
|
||||
updates["availability_state"] = "available"
|
||||
updates["missing_since"] = nil
|
||||
updates["last_seen_at"] = time.Now()
|
||||
}
|
||||
addMediaFileScanUpdates(updates, existing, incoming)
|
||||
addMediaTitleUpdates(updates, existing, incoming)
|
||||
addMediaExternalIDUpdates(updates, existing, incoming)
|
||||
|
||||
@@ -56,6 +56,23 @@ func (r *MediaSegmentRepository) ReplaceForMedia(ctx context.Context, mediaID, s
|
||||
})
|
||||
}
|
||||
|
||||
// ReplaceForMediaSourceKind swaps the segments of one (media, source, kind)
|
||||
// triple in a single transaction. Used by manual marking, where the user sets
|
||||
// one boundary at a time and must not disturb the other kinds.
|
||||
func (r *MediaSegmentRepository) ReplaceForMediaSourceKind(ctx context.Context, mediaID, source, kind string, rows []model.MediaSegment) error {
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Unscoped().
|
||||
Where("media_id = ? AND source = ? AND kind = ?", mediaID, source, kind).
|
||||
Delete(&model.MediaSegment{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
return tx.Create(&rows).Error
|
||||
})
|
||||
}
|
||||
|
||||
// GetFetch returns the fetch ledger row for (media, source), or (nil, nil).
|
||||
func (r *MediaSegmentRepository) GetFetch(ctx context.Context, mediaID, source string) (*model.MediaSegmentFetch, error) {
|
||||
var row model.MediaSegmentFetch
|
||||
|
||||
@@ -35,6 +35,7 @@ type Container struct {
|
||||
StrmDownload *StrmDownloadTaskRepository
|
||||
StrmUpload *StrmUploadTaskRepository
|
||||
StrmDirCache *StrmDirCacheRepository
|
||||
CloudDownload *CloudDownloadRepository
|
||||
ScrapeTask *ScrapeTaskRepository
|
||||
EmbyMount *EmbyMountRepository
|
||||
Reader *ReaderRepository
|
||||
@@ -69,6 +70,7 @@ func New(db *gorm.DB) *Container {
|
||||
StrmDownload: &StrmDownloadTaskRepository{db: db},
|
||||
StrmUpload: &StrmUploadTaskRepository{db: db},
|
||||
StrmDirCache: &StrmDirCacheRepository{db: db},
|
||||
CloudDownload: &CloudDownloadRepository{db: db},
|
||||
ScrapeTask: &ScrapeTaskRepository{db: db},
|
||||
EmbyMount: &EmbyMountRepository{db: db},
|
||||
Reader: &ReaderRepository{db: db},
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
_ "image/png" // register PNG decoder
|
||||
"strings"
|
||||
|
||||
"github.com/metatube-community/metatube-sdk-go/common/number"
|
||||
"github.com/metatube-community/metatube-sdk-go/detector"
|
||||
_ "golang.org/x/image/webp" // register WebP decoder for AVE covers
|
||||
)
|
||||
@@ -46,32 +47,50 @@ func IsAdultArtworkURL(raw string) bool {
|
||||
strings.Contains(lower, "pics.dmm.co.jp")
|
||||
}
|
||||
|
||||
// CropAdultCoverPoster checks if the image data is a wide full-jacket DVD cover (width > height * 1.15).
|
||||
// If so, it uses MetaTube's Pigo face detector to position a 2:3 poster crop.
|
||||
// When no face is detected, it falls back to the conventional right-side front cover.
|
||||
// CropAdultCoverPoster checks if the image data is a wide full-jacket DVD cover (width > height * 1.15)
|
||||
// and crops the conventional right-side front cover out of it. It is the code-agnostic entry point;
|
||||
// prefer CropAdultCoverPosterForCode when the movie number is known.
|
||||
// If the image is already portrait or cannot be decoded, it safely returns the original data.
|
||||
func CropAdultCoverPoster(data []byte) ([]byte, string, error) {
|
||||
return CropAdultCoverPosterForCode(data, "")
|
||||
}
|
||||
|
||||
// CropAdultCoverPosterForCode crops a 2:3 poster out of a wide full-jacket DVD cover.
|
||||
//
|
||||
// Layout matters: a regular DVD jacket is [封底 | 书脊 | 封面] unfolded right-to-left, so the
|
||||
// front cover is always the right-most panel — exactly what upstream MetaTube does with its
|
||||
// default position 1.0. Face detection is only meaningful for "special" numbers
|
||||
// (FC2 / HEYZO / 素人 / 无码) whose cover is a snapshot whose composition varies
|
||||
// (see metatube-sdk-go/common/number.RequiresFaceDetection).
|
||||
//
|
||||
// Running the detector on a regular jacket is actively harmful: the back cover of those jackets is a
|
||||
// montage of sample stills that is packed with faces, so the dominant face cluster lands on the montage
|
||||
// (or right on the spine), and a crop centred there straddles the seam giving "half back cover + half
|
||||
// front cover" posters.
|
||||
func CropAdultCoverPosterForCode(data []byte, code string) ([]byte, string, error) {
|
||||
if len(data) == 0 {
|
||||
return data, "", nil
|
||||
}
|
||||
src, format, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return data, format, err
|
||||
return data, imageMIMEForFormat(format), err
|
||||
}
|
||||
bounds := src.Bounds()
|
||||
width := bounds.Dx()
|
||||
height := bounds.Dy()
|
||||
if width <= 0 || height <= 0 {
|
||||
return data, format, nil
|
||||
return data, imageMIMEForFormat(format), nil
|
||||
}
|
||||
// If the image is already portrait or square, keep it as-is.
|
||||
if float64(width) <= float64(height)*1.15 {
|
||||
return data, format, nil
|
||||
return data, imageMIMEForFormat(format), nil
|
||||
}
|
||||
|
||||
position := 1.0
|
||||
if detected, ok := findPrimaryFaceAxisRatio(src, adultPosterRatio, true); ok {
|
||||
position = detected
|
||||
if adultCoverNeedsFaceDetection(code) {
|
||||
if detected, ok := findPrimaryFaceAxisRatio(src, adultPosterRatio, true); ok {
|
||||
position = snapAdultCoverPosition(detected)
|
||||
}
|
||||
}
|
||||
cropped := cropImageAtPosition(src, adultPosterRatio, position)
|
||||
|
||||
@@ -82,6 +101,48 @@ func CropAdultCoverPoster(data []byte) ([]byte, string, error) {
|
||||
return buf.Bytes(), "image/jpeg", nil
|
||||
}
|
||||
|
||||
// imageMIMEForFormat maps the format name returned by image.Decode ("jpeg" / "png" / "webp" …) to a MIME
|
||||
// type. The sidecar extension is derived from the MIME type, so handing back the bare format name made
|
||||
// every already-portrait poster fall through to the legacy ".img" extension.
|
||||
func imageMIMEForFormat(format string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(format)) {
|
||||
case "jpeg", "jpg":
|
||||
return "image/jpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "webp":
|
||||
return "image/webp"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
case "avif":
|
||||
return "image/avif"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// adultCoverNeedsFaceDetection reports whether the cover composition has to be probed with the face
|
||||
// detector, mirroring upstream MetaTube: only "special" numbers (FC2 / HEYZO / 素人 / 无码) qualify,
|
||||
// regular DVD numbers are always cropped at the right-most panel.
|
||||
func adultCoverNeedsFaceDetection(code string) bool {
|
||||
code = strings.TrimSpace(code)
|
||||
if code == "" {
|
||||
return false
|
||||
}
|
||||
return number.RequiresFaceDetection(code)
|
||||
}
|
||||
|
||||
// snapAdultCoverPosition quantises a detected face axis onto a whole panel. A wide jacket only has two
|
||||
// legal framings — the left panel or the right panel — so the crop window must never be centred on the
|
||||
// detected axis itself: when that axis sits on/near the spine the result is a poster showing the right
|
||||
// half of the back cover plus the left sliver of the front cover.
|
||||
func snapAdultCoverPosition(position float64) float64 {
|
||||
if position >= 0.5 {
|
||||
return 1.0 // right panel: flush with the right edge
|
||||
}
|
||||
return 0.0 // left panel: flush with the left edge
|
||||
}
|
||||
|
||||
func cropImageAtPosition(src image.Image, ratio, position float64) image.Image {
|
||||
bounds := src.Bounds()
|
||||
width := bounds.Dx()
|
||||
|
||||
@@ -80,7 +80,7 @@ func TestCropAdultCoverPosterDecodesWebP(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCropAdultCoverPosterCentersDetectedFace(t *testing.T) {
|
||||
func TestCropAdultCoverPosterUsesDetectedPanelForSpecialNumber(t *testing.T) {
|
||||
originalDetector := findPrimaryFaceAxisRatio
|
||||
findPrimaryFaceAxisRatio = func(image.Image, float64, bool) (float64, bool) {
|
||||
return 0.25, true
|
||||
@@ -90,7 +90,8 @@ func TestCropAdultCoverPosterCentersDetectedFace(t *testing.T) {
|
||||
}()
|
||||
|
||||
origBytes := createTestImage(900, 600, color.RGBA{R: 255, A: 255}, color.RGBA{B: 255, A: 255})
|
||||
croppedBytes, _, err := CropAdultCoverPoster(origBytes)
|
||||
// 素人/FC2 这类番号构图不固定,允许人脸检测决定取哪一侧面板。
|
||||
croppedBytes, _, err := CropAdultCoverPosterForCode(origBytes, "FC2-PPV-123456")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -100,7 +101,65 @@ func TestCropAdultCoverPosterCentersDetectedFace(t *testing.T) {
|
||||
}
|
||||
r, _, b, _ := croppedImg.At(croppedImg.Bounds().Dx()/2, croppedImg.Bounds().Dy()/2).RGBA()
|
||||
if r <= b {
|
||||
t.Fatalf("face-positioned crop did not follow detected left-side axis: r=%d b=%d", r, b)
|
||||
t.Fatalf("special-number crop did not follow detected left-side axis: r=%d b=%d", r, b)
|
||||
}
|
||||
}
|
||||
|
||||
// 常规 DVD 番号的封套是 [封底 | 书脊 | 封面] 横图,封底那半张是样本剧照拼贴、充满人脸。
|
||||
// 做人脸检测会让主簇落在拼贴或书脊上,裁出「半张封底 + 半张封面」。
|
||||
// 这里模拟检测器给出跨缝的 0.42,要求常规番号完全忽略检测结果,只裁最右侧封面面板。
|
||||
func TestCropAdultCoverPosterIgnoresFaceDetectionForStandardNumber(t *testing.T) {
|
||||
originalDetector := findPrimaryFaceAxisRatio
|
||||
findPrimaryFaceAxisRatio = func(image.Image, float64, bool) (float64, bool) {
|
||||
return 0.42, true
|
||||
}
|
||||
defer func() {
|
||||
findPrimaryFaceAxisRatio = originalDetector
|
||||
}()
|
||||
|
||||
origBytes := createTestImage(900, 600, color.RGBA{R: 255, A: 255}, color.RGBA{B: 255, A: 255})
|
||||
croppedBytes, _, err := CropAdultCoverPosterForCode(origBytes, "SSNI-677")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
croppedImg, _, err := image.Decode(bytes.NewReader(croppedBytes))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, _, b, _ := croppedImg.At(croppedImg.Bounds().Dx()/2, croppedImg.Bounds().Dy()/2).RGBA()
|
||||
if b <= r {
|
||||
t.Fatalf("standard-number crop should be the right-most panel (blue), got r=%d b=%d", r, b)
|
||||
}
|
||||
}
|
||||
|
||||
// 即使需要人脸定位,裁切框也必须吸附到整块面板,绝不能跨在书脊上。
|
||||
func TestCropAdultCoverPosterSnapsToPanelForSpecialNumber(t *testing.T) {
|
||||
originalDetector := findPrimaryFaceAxisRatio
|
||||
findPrimaryFaceAxisRatio = func(image.Image, float64, bool) (float64, bool) {
|
||||
return 0.42, true // 介于两个面板之间:落在书脊上
|
||||
}
|
||||
defer func() {
|
||||
findPrimaryFaceAxisRatio = originalDetector
|
||||
}()
|
||||
|
||||
origBytes := createTestImage(900, 600, color.RGBA{R: 255, A: 255}, color.RGBA{B: 255, A: 255})
|
||||
croppedBytes, _, err := CropAdultCoverPosterForCode(origBytes, "HEYZO-1234")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
croppedImg, _, err := image.Decode(bytes.NewReader(croppedBytes))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bounds := croppedImg.Bounds()
|
||||
// 左侧面板 = 纯红;任何偏蓝像素都意味着裁切框越过了中线。
|
||||
for y := bounds.Min.Y; y < bounds.Max.Y; y += 8 {
|
||||
for x := bounds.Min.X; x < bounds.Max.X; x += 8 {
|
||||
r, _, b, _ := croppedImg.At(x, y).RGBA()
|
||||
if b > r {
|
||||
t.Fatalf("crop straddles the panel seam at (%d,%d): r=%d b=%d", x, y, r, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,10 +167,14 @@ func TestCropAdultCoverPosterKeepsPortrait(t *testing.T) {
|
||||
// Create an already vertical portrait image (500x700)
|
||||
origBytes := createTestImage(500, 700, color.RGBA{R: 255, A: 255}, color.RGBA{B: 255, A: 255})
|
||||
|
||||
croppedBytes, _, err := CropAdultCoverPoster(origBytes)
|
||||
croppedBytes, ctype, err := CropAdultCoverPoster(origBytes)
|
||||
if err != nil {
|
||||
t.Fatalf("CropAdultCoverPoster error: %v", err)
|
||||
}
|
||||
// 未裁剪时必须回传真正的 MIME(image/jpeg),否则侧车文件扩展名会退化成 .img。
|
||||
if ctype != "image/jpeg" {
|
||||
t.Fatalf("portrait ctype = %q, want image/jpeg", ctype)
|
||||
}
|
||||
|
||||
croppedImg, _, err := image.Decode(bytes.NewReader(croppedBytes))
|
||||
if err != nil {
|
||||
|
||||
@@ -96,7 +96,8 @@ func TestAdultProviderRouting(t *testing.T) {
|
||||
if len(candidates) != 1 {
|
||||
t.Fatalf("expected 1 candidate, got %d", len(candidates))
|
||||
}
|
||||
wantPoster := mtServer.URL + "/v1/images/primary/javdb/999?auto=true&pos=1&quality=90&ratio=-1&url=https%3A%2F%2Fexample.com%2Fposter.jpg"
|
||||
// SSIS-001 是常规番号:MetaTube 代理不得开启人脸定位,固定取最右侧封面面板。
|
||||
wantPoster := mtServer.URL + "/v1/images/primary/javdb/999?auto=false&pos=1&quality=90&ratio=-1&url=https%3A%2F%2Fexample.com%2Fposter.jpg"
|
||||
wantBackdrop := mtServer.URL + "/v1/images/backdrop/javdb/999?quality=90"
|
||||
if candidates[0].PosterURL != wantPoster || candidates[0].BackdropURL != wantBackdrop {
|
||||
t.Fatalf("candidate artwork was not enriched: %#v", candidates[0])
|
||||
@@ -170,8 +171,8 @@ func TestBuiltinAdultScrapeUsesMetaTubeFaceAwareArtwork(t *testing.T) {
|
||||
if match == nil {
|
||||
t.Fatal("expected built-in match")
|
||||
}
|
||||
wantPoster := metaTubeServer.URL + "/v1/images/primary/AVE/999?auto=true&pos=1&quality=90&ratio=-1&url=https%3A%2F%2Fexample.com%2Fwide-cover.jpg"
|
||||
wantPoster := metaTubeServer.URL + "/v1/images/primary/AVE/999?auto=false&pos=1&quality=90&ratio=-1&url=https%3A%2F%2Fexample.com%2Fwide-cover.jpg"
|
||||
if match.PosterURL != wantPoster {
|
||||
t.Fatalf("built-in poster = %q, want face-aware URL %q", match.PosterURL, wantPoster)
|
||||
t.Fatalf("built-in poster = %q, want MetaTube-processed URL %q", match.PosterURL, wantPoster)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,14 @@ func (s *ApiConfigService) TestConnection(ctx context.Context, provider string)
|
||||
switch provider {
|
||||
case "tmdb":
|
||||
return s.testTMDb(cfg)
|
||||
case "bangumi":
|
||||
return s.testBangumi(ctx, cfg)
|
||||
case "thetvdb":
|
||||
return s.testTheTVDB(ctx, cfg)
|
||||
case "fanart":
|
||||
return s.testFanart(ctx, cfg)
|
||||
case "douban":
|
||||
return s.testDouban(ctx, cfg)
|
||||
case "openai":
|
||||
return s.testOpenAI(cfg)
|
||||
case "deepseek":
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// Bangumi's public metadata endpoints work without a token. If one was saved,
|
||||
// also check /v0/me: public endpoints alone may accept an expired token.
|
||||
func (s *ApiConfigService) testBangumi(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://api.bgm.tv", "/v0/subjects/253", http.MethodGet, nil)
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
if token := strings.TrimSpace(cfg.APIKey); token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
identityReq := req.Clone(ctx)
|
||||
identityURL := *req.URL
|
||||
identityURL.Path = strings.TrimSuffix(identityURL.Path, "/subjects/253") + "/me"
|
||||
identityReq.URL = &identityURL
|
||||
var identity struct {
|
||||
ID int `json:"id"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(identityReq, "Bangumi", &identity); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if identity.ID <= 0 {
|
||||
return "error", errors.New("Bangumi returned an invalid user response")
|
||||
}
|
||||
}
|
||||
var subject struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "Bangumi", &subject); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if subject.ID != 253 || strings.TrimSpace(subject.Name) == "" {
|
||||
return "error", errors.New("Bangumi returned an invalid subject response")
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func (s *ApiConfigService) testTheTVDB(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
key := strings.TrimSpace(cfg.APIKey)
|
||||
if key == "" {
|
||||
return "error", errors.New("TheTVDB API key is required")
|
||||
}
|
||||
body, err := json.Marshal(map[string]string{"apikey": key})
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://api4.thetvdb.com/v4", "/login", http.MethodPost, strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
var response struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "TheTVDB", &response); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if response.Status != "success" || strings.TrimSpace(response.Data.Token) == "" {
|
||||
return "error", errors.New("TheTVDB login did not return a valid token")
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func (s *ApiConfigService) testFanart(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
key := strings.TrimSpace(cfg.APIKey)
|
||||
if key == "" {
|
||||
return "error", errors.New("Fanart API key is required")
|
||||
}
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://webservice.fanart.tv/v3", "/movies/550", http.MethodGet, nil)
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
query := req.URL.Query()
|
||||
query.Set("api_key", key)
|
||||
req.URL.RawQuery = query.Encode()
|
||||
var response struct {
|
||||
TMDbID json.Number `json:"tmdb_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "Fanart", &response); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if response.TMDbID.String() != "550" || strings.TrimSpace(response.Name) == "" {
|
||||
return "error", errors.New("Fanart returned an invalid movie response")
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func (s *ApiConfigService) testDouban(ctx context.Context, cfg *model.APIConfig) (string, error) {
|
||||
req, err := newMetadataProbeRequest(ctx, cfg, "https://movie.douban.com", "/j/subject_suggest", http.MethodGet, nil)
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
query := req.URL.Query()
|
||||
query.Set("q", "Inception")
|
||||
req.URL.RawQuery = query.Encode()
|
||||
req.Header.Set("User-Agent", userAgents[0])
|
||||
req.Header.Set("Referer", "https://movie.douban.com/")
|
||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
||||
if cookie := strings.TrimSpace(cfg.APIKey); cookie != "" {
|
||||
req.Header.Set("Cookie", cookie)
|
||||
}
|
||||
var suggestions []struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
if result, err := testMetadataProbeJSON(req, "Douban", &suggestions); err != nil {
|
||||
return result, err
|
||||
}
|
||||
// An empty array is a valid search result; null or an error object is not.
|
||||
if suggestions == nil {
|
||||
return "error", errors.New("Douban returned an invalid search response (login or verification may be required)")
|
||||
}
|
||||
for _, suggestion := range suggestions {
|
||||
if suggestion.ID == "" || strings.TrimSpace(suggestion.Title) == "" {
|
||||
return "error", errors.New("Douban returned an invalid search suggestion")
|
||||
}
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
|
||||
func newMetadataProbeRequest(ctx context.Context, cfg *model.APIConfig, defaultBase, path, method string, body io.Reader) (*http.Request, error) {
|
||||
base := strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/")
|
||||
if base == "" {
|
||||
base = defaultBase
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, base+path, body)
|
||||
if err != nil {
|
||||
// The saved base URL may contain credentials. Never echo it to the UI.
|
||||
return nil, fmt.Errorf("%s: invalid base URL", cfg.Provider)
|
||||
}
|
||||
req.Header.Set("User-Agent", "MeBox/0.1 (https://github.com/truewhile/MeBox)")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// Decode a bounded, complete JSON document rather than accepting any HTTP 200
|
||||
// (reverse proxies and anti-bot/login pages often return 200 as well).
|
||||
func testMetadataProbeJSON(req *http.Request, provider string, out any) (string, error) {
|
||||
client := NewExternalHTTPClient(10 * time.Second)
|
||||
defer client.CloseIdleConnections()
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
// url.Error includes the full request URL, including Fanart's API key.
|
||||
var urlErr *url.Error
|
||||
if errors.As(err, &urlErr) {
|
||||
err = urlErr.Err
|
||||
}
|
||||
return "error", fmt.Errorf("%s connection failed: %w", provider, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
return "invalid", fmt.Errorf("%s authentication failed (HTTP 401): check configured credentials", provider)
|
||||
}
|
||||
if resp.StatusCode == http.StatusForbidden {
|
||||
return "error", fmt.Errorf("%s access denied (HTTP 403): check credentials, permissions or anti-bot verification", provider)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "error", fmt.Errorf("%s API returned status %d", provider, resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, (1<<20)+1))
|
||||
if err != nil {
|
||||
return "error", fmt.Errorf("%s response read failed: %w", provider, err)
|
||||
}
|
||||
if len(body) > 1<<20 {
|
||||
return "error", fmt.Errorf("%s response exceeds 1 MiB", provider)
|
||||
}
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
// Do not include the response body: it may echo credentials.
|
||||
return "error", fmt.Errorf("%s returned invalid JSON (login or verification page may have intercepted the request)", provider)
|
||||
}
|
||||
return "success", nil
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
type metadataProbeFixture struct {
|
||||
provider string
|
||||
path string
|
||||
key string
|
||||
body string
|
||||
}
|
||||
|
||||
var metadataProbeFixtures = []metadataProbeFixture{
|
||||
{"bangumi", "/v0/subjects/253", "", `{"id":253,"name":"カウボーイビバップ","type":2}`},
|
||||
{"thetvdb", "/login", "tvdb-test-secret", `{"status":"success","data":{"token":"test-jwt"}}`},
|
||||
{"fanart", "/movies/550", "fanart+test&secret", `{"name":"Fight Club","tmdb_id":"550","imdb_id":"tt0137523","movieposter":[]}`},
|
||||
{"douban", "/j/subject_suggest", "bid=test-cookie", `[{"id":"3541415","title":"盗梦空间","year":"2010","type":"movie"}]`},
|
||||
}
|
||||
|
||||
func newMetadataConnectionTestService(t *testing.T) *ApiConfigService {
|
||||
t.Helper()
|
||||
log := zap.NewNop()
|
||||
return NewApiConfigService(&config.Config{}, log, repository.New(newServiceTestDB(t)), NewCryptoService("metadata-test-secret", log))
|
||||
}
|
||||
|
||||
func saveMetadataProbeConfig(t *testing.T, svc *ApiConfigService, provider, baseURL, key string) {
|
||||
t.Helper()
|
||||
if _, err := svc.Upsert(t.Context(), provider, key, baseURL, "", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Removing a dispatch branch, ignoring the saved URL/key, or using the wrong
|
||||
// endpoint must fail this integration test (real database and HTTP server).
|
||||
func TestMetadataConnectionUsesSavedConfig(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, fixture := range metadataProbeFixtures {
|
||||
t.Run(fixture.provider, func(t *testing.T) {
|
||||
calls := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
if r.URL.Path != "/proxy"+fixture.path {
|
||||
t.Errorf("request path = %q", r.URL.Path)
|
||||
}
|
||||
if r.Header.Get("User-Agent") == "" {
|
||||
t.Error("missing User-Agent")
|
||||
}
|
||||
if fixture.provider == "thetvdb" {
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if r.Method != http.MethodPost || body["apikey"] != fixture.key || r.Header.Get("Content-Type") != "application/json" {
|
||||
t.Error("TheTVDB login must POST the saved, decrypted API key as JSON")
|
||||
}
|
||||
} else if r.Method != http.MethodGet {
|
||||
t.Errorf("method = %s, want GET", r.Method)
|
||||
}
|
||||
if fixture.provider == "fanart" && r.URL.Query().Get("api_key") != fixture.key {
|
||||
t.Error("Fanart API key was not escaped/passed correctly")
|
||||
}
|
||||
if fixture.provider == "douban" {
|
||||
if r.Header.Get("Cookie") != fixture.key || r.Header.Get("Referer") != "https://movie.douban.com/" || r.URL.Query().Get("q") == "" {
|
||||
t.Error("Douban search must carry the configured cookie, referer and query")
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(fixture.body))
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, fixture.provider, " "+server.URL+"/proxy/ ", fixture.key)
|
||||
result, err := svc.TestConnection(t.Context(), fixture.provider)
|
||||
if err != nil || result != "success" {
|
||||
t.Fatalf("TestConnection = %q, %v, want success", result, err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("HTTP calls = %d, want 1", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionRejectsBadResponses(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, fixture := range metadataProbeFixtures {
|
||||
for _, response := range []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
result string
|
||||
}{
|
||||
{"unauthorized", 401, `{"error":"denied"}`, "invalid"},
|
||||
{"forbidden", 403, `<html>Access denied</html>`, "error"},
|
||||
{"rate limited", 429, `{}`, "error"},
|
||||
{"upstream error", 503, `{}`, "error"},
|
||||
{"challenge page", 200, `<html>Please log in or complete verification</html>`, "error"},
|
||||
{"empty body", 200, ``, "error"},
|
||||
{"wrong schema", 200, `{"error":"intercepted"}`, "error"},
|
||||
{"null", 200, `null`, "error"},
|
||||
{"trailing garbage", 200, fixture.body + `<html>blocked</html>`, "error"},
|
||||
} {
|
||||
t.Run(fixture.provider+"/"+response.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(response.status)
|
||||
_, _ = w.Write([]byte(response.body))
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, fixture.provider, server.URL, fixture.key)
|
||||
result, err := svc.TestConnection(t.Context(), fixture.provider)
|
||||
if err == nil || result != response.result {
|
||||
t.Fatalf("TestConnection = %q, %v, want %q and error", result, err, response.result)
|
||||
}
|
||||
if fixture.key != "" && strings.Contains(err.Error(), fixture.key) {
|
||||
t.Error("error exposes configured credentials")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionRequiresAPIKeys(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, provider := range []string{"thetvdb", "fanart"} {
|
||||
t.Run(provider, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("must reject missing key before making a request")
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, provider, server.URL, " ")
|
||||
result, err := svc.TestConnection(t.Context(), provider)
|
||||
if result != "error" || err == nil || !strings.Contains(err.Error(), "API key") {
|
||||
t.Fatalf("TestConnection = %q, %v, want missing API key error", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionBangumiValidatesConfiguredToken(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, valid := range []bool{true, false} {
|
||||
name := "valid"
|
||||
if !valid {
|
||||
name = "invalid"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
meCalls, subjectCalls := 0, 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer bangumi-test-token" {
|
||||
t.Error("missing saved Bangumi bearer token")
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/v0/me":
|
||||
meCalls++
|
||||
if !valid {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"id":42,"username":"test-user"}`))
|
||||
case "/v0/subjects/253":
|
||||
subjectCalls++
|
||||
_, _ = w.Write([]byte(`{"id":253,"name":"カウボーイビバップ"}`))
|
||||
default:
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, "bangumi", server.URL, "bangumi-test-token")
|
||||
result, err := svc.TestConnection(t.Context(), "bangumi")
|
||||
if valid && (result != "success" || err != nil || subjectCalls != 1) {
|
||||
t.Fatalf("valid token: result=%q err=%v subjectCalls=%d", result, err, subjectCalls)
|
||||
}
|
||||
if !valid && (result != "invalid" || err == nil || subjectCalls != 0) {
|
||||
t.Fatalf("invalid token: result=%q err=%v subjectCalls=%d", result, err, subjectCalls)
|
||||
}
|
||||
if meCalls != 1 {
|
||||
t.Errorf("identity requests = %d, want 1", meCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionDoubanDoesNotRequireCookie(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Cookie") != "" {
|
||||
t.Error("unexpected Cookie for public search")
|
||||
}
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, "douban", server.URL, "")
|
||||
result, err := svc.TestConnection(t.Context(), "douban")
|
||||
if result != "success" || err != nil {
|
||||
t.Fatalf("public search with empty results = %q, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionHonorsContext(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
for _, fixture := range metadataProbeFixtures {
|
||||
t.Run(fixture.provider, func(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
close(started)
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
saveMetadataProbeConfig(t, svc, fixture.provider, server.URL, fixture.key)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-started:
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
result, err := svc.TestConnection(ctx, fixture.provider)
|
||||
if result != "error" || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled request = %q, %v, want context.Canceled", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataConnectionTransportErrorsHideKeys(t *testing.T) {
|
||||
svc := newMetadataConnectionTestService(t)
|
||||
server := httptest.NewServer(http.NotFoundHandler())
|
||||
server.Close()
|
||||
key := "sensitive-fanart-key"
|
||||
saveMetadataProbeConfig(t, svc, "fanart", server.URL, key)
|
||||
result, err := svc.TestConnection(t.Context(), "fanart")
|
||||
if result != "error" || err == nil || strings.Contains(err.Error(), key) {
|
||||
t.Fatalf("transport failure = %q, %v, want error without credentials", result, err)
|
||||
}
|
||||
}
|
||||
+81
-37
@@ -5,7 +5,7 @@
|
||||
//
|
||||
// 两种方言的备份形态不同:
|
||||
// - SQLite:用 "VACUUM INTO"(3.27.0 起可用)产出一个一致的库文件,
|
||||
// 扩展名 .db;恢复即「替换库文件 + 重启」。
|
||||
// 扩展名 .db;恢复先暂存,在下次启动打开数据库前替换文件。
|
||||
// - PostgreSQL:没有单文件语义,用逻辑备份(见 backup_postgres.go),
|
||||
// 扩展名 .json.gz;恢复是在单个事务内重建全部表。
|
||||
package service
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/database"
|
||||
)
|
||||
|
||||
// BackupService manages database backups.
|
||||
@@ -44,6 +45,7 @@ type BackupInfo struct {
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Automatic bool `json:"automatic"`
|
||||
}
|
||||
|
||||
func (b *BackupService) backupDir() string {
|
||||
@@ -69,6 +71,23 @@ func (b *BackupService) backupFilePath(filename string) (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// ResolvePath returns the absolute path of a listed backup after applying the
|
||||
// same filename and directory checks used by restore and delete operations.
|
||||
func (b *BackupService) ResolvePath(filename string) (string, error) {
|
||||
path, err := b.backupFilePath(filename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return "", errors.New("backup is not a regular file")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// isPostgres reports whether the service is backed by PostgreSQL.
|
||||
//
|
||||
// PostgreSQL 不支持 SQLite 的 "VACUUM INTO",按文件替换恢复也会写错目标,
|
||||
@@ -80,11 +99,23 @@ func (b *BackupService) isPostgres() bool {
|
||||
// Create produces a new hot backup. SQLite uses "VACUUM INTO"; PostgreSQL uses
|
||||
// a logical dump.
|
||||
func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, false)
|
||||
}
|
||||
|
||||
func (b *BackupService) CreateAutomatic(ctx context.Context) (*BackupInfo, error) {
|
||||
return b.create(ctx, true)
|
||||
}
|
||||
|
||||
func (b *BackupService) create(ctx context.Context, automatic bool) (*BackupInfo, error) {
|
||||
dir := b.backupDir()
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := time.Now().Format("20060102_150405")
|
||||
now := time.Now()
|
||||
ts := fmt.Sprintf("%s_%09d", now.Format("20060102_150405"), now.Nanosecond())
|
||||
if automatic {
|
||||
ts = "auto_" + ts
|
||||
}
|
||||
|
||||
var (
|
||||
dst string
|
||||
@@ -108,15 +139,24 @@ func (b *BackupService) Create(ctx context.Context) (*BackupInfo, error) {
|
||||
Path: dst,
|
||||
Size: stat.Size(),
|
||||
CreatedAt: stat.ModTime(),
|
||||
Automatic: automatic,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *BackupService) createSQLiteBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), fmt.Sprintf("mebox_%s.db", stamp))
|
||||
// VACUUM INTO creates a clean, non-WAL copy atomically.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", dst).Error; err != nil {
|
||||
tmp := dst + ".tmp"
|
||||
defer os.Remove(tmp)
|
||||
// Publish only completed backups, with credentials protected by file mode.
|
||||
if err := b.db.WithContext(ctx).Exec("VACUUM INTO ?", tmp).Error; err != nil {
|
||||
return "", fmt.Errorf("VACUUM INTO failed: %w", err)
|
||||
}
|
||||
if err := os.Chmod(tmp, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -132,7 +172,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
}
|
||||
out := make([]BackupInfo, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !isBackupFilename(e.Name()) {
|
||||
if !e.Type().IsRegular() || !isBackupFilename(e.Name()) {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
@@ -144,6 +184,7 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
Path: filepath.Join(dir, e.Name()),
|
||||
Size: info.Size(),
|
||||
CreatedAt: info.ModTime(),
|
||||
Automatic: strings.HasPrefix(e.Name(), "mebox_auto_"),
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
@@ -152,6 +193,36 @@ func (b *BackupService) List() ([]BackupInfo, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Prune keeps maxCount automatic backups for the current database dialect.
|
||||
// Manual backups and backups from another dialect are never pruned.
|
||||
// A value below one disables pruning so an invalid setting cannot erase all
|
||||
// recoverable copies.
|
||||
func (b *BackupService) Prune(maxCount int) error {
|
||||
if maxCount < 1 {
|
||||
return nil
|
||||
}
|
||||
items, err := b.List()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible := items[:0]
|
||||
for _, item := range items {
|
||||
if item.Automatic && strings.HasSuffix(item.Filename, postgresBackupSuffix) == b.isPostgres() {
|
||||
eligible = append(eligible, item)
|
||||
}
|
||||
}
|
||||
items = eligible
|
||||
if maxCount >= len(items) {
|
||||
return nil
|
||||
}
|
||||
for _, item := range items[maxCount:] {
|
||||
if err := b.Delete(item.Filename); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a single backup file.
|
||||
func (b *BackupService) Delete(filename string) error {
|
||||
path, err := b.backupFilePath(filename)
|
||||
@@ -161,12 +232,10 @@ func (b *BackupService) Delete(filename string) error {
|
||||
return os.Remove(path)
|
||||
}
|
||||
|
||||
// Restore replaces the live database with the given backup.
|
||||
// WARNING: this is destructive. SQLite restores by swapping the database file
|
||||
// (the caller should shut down the server afterwards); PostgreSQL restores by
|
||||
// rebuilding every table inside a single transaction.
|
||||
// Restore stages SQLite backups for the next startup. PostgreSQL restores
|
||||
// rebuild every table inside a single transaction.
|
||||
func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
src, err := b.backupFilePath(filename)
|
||||
src, err := b.ResolvePath(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -182,35 +251,10 @@ func (b *BackupService) Restore(ctx context.Context, filename string) error {
|
||||
if !strings.HasSuffix(filename, ".db") {
|
||||
return errors.New("该备份是 PostgreSQL 逻辑备份,当前运行在 SQLite 上,无法恢复")
|
||||
}
|
||||
return b.restoreSQLiteBackup(filename, src)
|
||||
}
|
||||
|
||||
func (b *BackupService) restoreSQLiteBackup(filename, src string) error {
|
||||
dbPath, err := filepath.Abs(b.cfg.Database.DBPath)
|
||||
if err != nil || dbPath == "" {
|
||||
return errors.New("invalid database path")
|
||||
}
|
||||
// Strategy: rename live → live.old, copy backup → live, delete old.
|
||||
old := dbPath + ".before_restore"
|
||||
if err := os.Rename(dbPath, old); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
if err := database.StageSQLiteRestore(ctx, b.cfg, src); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(src) // #nosec G304 -- src is constrained by backupFilePath to the configured backups directory.
|
||||
if err != nil {
|
||||
// Revert
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dbPath, data, 0o600); err != nil { // #nosec G703 -- database path is trusted process config and normalized before restore.
|
||||
_ = os.Rename(old, dbPath)
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(old)
|
||||
// Also remove WAL/SHM so SQLite opens the fresh file cleanly.
|
||||
_ = os.Remove(dbPath + "-wal")
|
||||
_ = os.Remove(dbPath + "-shm")
|
||||
b.log.Warn("database restored from backup — restart the server",
|
||||
zap.String("backup", filename))
|
||||
b.log.Warn("SQLite backup staged; restart to apply", zap.String("backup", filename))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -87,7 +88,8 @@ func postgresBackupModelSchemas(db *gorm.DB) (map[string]*schema.Schema, map[str
|
||||
// createPostgresBackup 把全库导出成一个 gzip 压缩的 JSON Lines 文件。
|
||||
func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string) (string, error) {
|
||||
dst := filepath.Join(b.backupDir(), "mebox_"+stamp+postgresBackupSuffix)
|
||||
file, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
tmp := dst + ".tmp"
|
||||
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -96,7 +98,7 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
_ = file.Close()
|
||||
if !complete {
|
||||
// 半截备份文件比没有备份更危险:留着会让人以为有可用的还原点。
|
||||
_ = os.Remove(dst)
|
||||
_ = os.Remove(tmp)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -109,17 +111,30 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("写入备份头失败: %w", err)
|
||||
}
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(b.db, m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !b.db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := b.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return "", err
|
||||
var options []*sql.TxOptions
|
||||
if b.isPostgres() {
|
||||
options = append(options, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||
}
|
||||
// Every table must come from the same snapshot, including while background
|
||||
// workers are inserting related rows during an automatic backup.
|
||||
if err := b.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
snapshot := *b
|
||||
snapshot.db = tx
|
||||
for _, m := range model.AllModels() {
|
||||
table, sch, err := parseModelSchema(tx, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !tx.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
if err := snapshot.dumpPostgresTable(ctx, m, sch, table, enc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, options...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return "", fmt.Errorf("压缩备份失败: %w", err)
|
||||
@@ -127,6 +142,12 @@ func (b *BackupService) createPostgresBackup(ctx context.Context, stamp string)
|
||||
if err := file.Sync(); err != nil {
|
||||
return "", fmt.Errorf("写入备份失败: %w", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return "", err
|
||||
}
|
||||
complete = true
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBackupRetentionPreservesManualAndOtherDialectBackups(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
names := []string{"mebox_auto_new.db", "mebox_auto_old.db", "mebox_manual.db", "mebox_auto_postgres.json.gz"}
|
||||
for i, name := range names {
|
||||
path := filepath.Join(svc.backupDir(), name)
|
||||
if err := os.WriteFile(path, []byte("fixture"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
when := time.Now().Add(-time.Duration(i) * time.Hour)
|
||||
if err := os.Chtimes(path, when, when); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := svc.Prune(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range names {
|
||||
_, err := os.Stat(filepath.Join(svc.backupDir(), name))
|
||||
if name == "mebox_auto_old.db" {
|
||||
if !os.IsNotExist(err) {
|
||||
t.Fatalf("old automatic backup was retained: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("backup %s should have been preserved: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupCreateUsesUniquePrivateCompletedFiles(t *testing.T) {
|
||||
svc, _ := newBackupTestService(t)
|
||||
manual, err := svc.Create(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
automatic, err := svc.CreateAutomatic(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manual.Filename == automatic.Filename || manual.Automatic || !automatic.Automatic {
|
||||
t.Fatalf("invalid backup identities: manual=%#v automatic=%#v", manual, automatic)
|
||||
}
|
||||
items, err := svc.List()
|
||||
if err != nil || len(items) != 2 {
|
||||
t.Fatalf("completed backup list: items=%#v err=%v", items, err)
|
||||
}
|
||||
for _, item := range items {
|
||||
info, err := os.Stat(item.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("backup permissions are not private: %s err=%v", item.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -74,22 +74,20 @@ type ImageCacheScope struct {
|
||||
DropUnknownResizeVariants bool
|
||||
}
|
||||
|
||||
// ImageCachePool 描述一个图片缓存池:范围 + 自身配额 + 保留时长。
|
||||
// ImageCachePool 描述一个图片缓存池:范围 + 自身配额。
|
||||
//
|
||||
// 分池的意义在于两类缓存的“可再生成本”完全不同:原图只是生成缩略图的原料,
|
||||
// 丢了可以重新回源;派生缩略图是客户端热路径真正读取的成品,重新生成代价高。
|
||||
// 因此原图配小配额、短保留,总量超限时也优先淘汰原图。
|
||||
// 因此原图配小配额,总量超限时也优先淘汰原图。
|
||||
//
|
||||
// 淘汰只看占用(配额头寸),不按文件年龄删除:缩略图生成成本高,只要磁盘还
|
||||
// 放得下就该留着,让再次浏览直接命中缓存。
|
||||
type ImageCachePool struct {
|
||||
Name string
|
||||
Scopes []ImageCacheScope
|
||||
MaxBytes int64 // 0 = 不单独限制
|
||||
MaxAge time.Duration // 0 = 不按时间淘汰
|
||||
MaxBytes int64 // 0 = 不单独限制
|
||||
}
|
||||
|
||||
// imageFailMarkerMaxAge 是失败标记文件的保留时长:标记只用于观测/重试,
|
||||
// 过期即视为噪音清掉(不会因为标记残留而阻止重新抓取)。
|
||||
const imageFailMarkerMaxAge = 24 * time.Hour
|
||||
|
||||
// tempImageCacheFile 报告文件名是否是下载/缩放过程中的临时文件。它们正在被
|
||||
// 并发写入,清理时必须跳过。
|
||||
func tempImageCacheFile(name string) bool {
|
||||
@@ -182,12 +180,14 @@ func removeImageCacheEntry(entry imageCacheFileEntry, result *PruneImageCacheRes
|
||||
_ = os.Remove(entry.path + ".fail")
|
||||
}
|
||||
|
||||
// PruneImageCachePools 按池清理图片缓存:
|
||||
// 1. 删除超过池 MaxAge 的文件(原图按保留时长淘汰);
|
||||
// 2. 池自身超过 MaxBytes 时按 mtime 淘汰到 80%(留水位,避免连续写入即触发);
|
||||
// 3. 全部池合计超过 totalBytes 时,仍按“先原图、后派生”的顺序淘汰最旧文件,
|
||||
// PruneImageCachePools 按池清理图片缓存,只做占用淘汰:
|
||||
// 1. 池自身超过 MaxBytes 时按 mtime 淘汰到 80%(留水位,避免连续写入即触发);
|
||||
// 2. 全部池合计超过 totalBytes 时,仍按“先原图、后派生”的顺序淘汰最旧文件,
|
||||
// 使总量上限始终是硬保证,同时让客户端热路径的缩略图活得更久。
|
||||
//
|
||||
// 不按文件年龄删除:过期即删会让一张几天前看过的海报重新解码(2 核机器上
|
||||
// 每张约 0.2 秒),代价远高于占着那几十 KB 磁盘。
|
||||
//
|
||||
// 传入的池顺序即总量超限时的淘汰优先级(排在前面的先被淘汰)。
|
||||
func PruneImageCachePools(pools []ImageCachePool, totalBytes int64) (PruneImageCacheResult, error) {
|
||||
var result PruneImageCacheResult
|
||||
@@ -207,16 +207,18 @@ func PruneImageCachePools(pools []ImageCachePool, totalBytes int64) (PruneImageC
|
||||
}
|
||||
result.RemainingBytes = result.TotalBytesBefore
|
||||
|
||||
// 过期的失败标记直接清掉:它们不计容量,只用于观测与重试。
|
||||
cutoffMarkers := time.Now().Add(-imageFailMarkerMaxAge)
|
||||
// 失败标记不计入容量:只清掉对应缓存文件已经不存在的孤儿标记(它们只是
|
||||
// 观测/重试用的噪音,留着没有任何作用)。
|
||||
for _, marker := range failMarkers {
|
||||
if marker.modTime.Before(cutoffMarkers) {
|
||||
_ = os.Remove(marker.path)
|
||||
base := strings.TrimSuffix(marker.path, ".fail")
|
||||
if _, err := os.Stat(base); err == nil {
|
||||
continue
|
||||
}
|
||||
_ = os.Remove(marker.path)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
deleted := make(map[string]bool, len(entries))
|
||||
|
||||
deleteEntry := func(entry imageCacheFileEntry) {
|
||||
if deleted[entry.path] {
|
||||
return
|
||||
@@ -225,20 +227,7 @@ func PruneImageCachePools(pools []ImageCachePool, totalBytes int64) (PruneImageC
|
||||
removeImageCacheEntry(entry, &result)
|
||||
}
|
||||
|
||||
// 1. 按池保留时长淘汰(原图池)。
|
||||
for i := range pools {
|
||||
if pools[i].MaxAge <= 0 {
|
||||
continue
|
||||
}
|
||||
cutoff := now.Add(-pools[i].MaxAge)
|
||||
for _, entry := range entries {
|
||||
if entry.pool == i && entry.modTime.Before(cutoff) {
|
||||
deleteEntry(entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 池自身配额。
|
||||
// 1. 池自身配额。
|
||||
for i := range pools {
|
||||
maxBytes := pools[i].MaxBytes
|
||||
if maxBytes <= 0 {
|
||||
@@ -263,7 +252,7 @@ func PruneImageCachePools(pools []ImageCachePool, totalBytes int64) (PruneImageC
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 总量硬上限:跨池按淘汰优先级(池顺序)再按 mtime 淘汰。
|
||||
// 2. 总量硬上限:跨池按淘汰优先级(池顺序)再按 mtime 淘汰。
|
||||
if totalBytes > 0 && result.RemainingBytes > totalBytes {
|
||||
target := totalBytes * 80 / 100
|
||||
for _, entry := range evictionOrder(entries, deleted) {
|
||||
@@ -357,11 +346,10 @@ func PruneImageCache(imagesDir string, maxSizeBytes int64) (PruneImageCacheResul
|
||||
// 第一个池是原图:历史版本平铺在 images/ 下的旧缓存也归入此池,升级后会被
|
||||
// 逐步淘汰;第二个池是派生成品(本地缩放结果 + 挂载 Emby 按尺寸返回的成品)。
|
||||
// 池顺序即总量超限时的淘汰优先级。
|
||||
func ImageCachePools(imagesDir string, originalsMaxBytes int64, originalsMaxAge time.Duration) []ImageCachePool {
|
||||
func ImageCachePools(imagesDir string, originalsMaxBytes int64) []ImageCachePool {
|
||||
originals := ImageCachePool{
|
||||
Name: "originals",
|
||||
MaxBytes: originalsMaxBytes,
|
||||
MaxAge: originalsMaxAge,
|
||||
Scopes: []ImageCacheScope{
|
||||
{Root: imagesDir, Recursive: false}, // 旧版平铺布局
|
||||
{Root: filepath.Join(imagesDir, imageOriginalCacheSubdir), Recursive: true},
|
||||
@@ -376,3 +364,199 @@ func ImageCachePools(imagesDir string, originalsMaxBytes int64, originalsMaxAge
|
||||
}
|
||||
return []ImageCachePool{originals, derived}
|
||||
}
|
||||
|
||||
// CacheCleanupResult 是全缓存容量淘汰的统计。
|
||||
type CacheCleanupResult struct {
|
||||
TotalFilesBefore int
|
||||
TotalBytesBefore int64
|
||||
DeletedFiles int
|
||||
FreedBytes int64
|
||||
RemainingBytes int64
|
||||
// FreedByPool 记录每个池释放的字节数,方便日志定位到底是哪类缓存占满了。
|
||||
FreedByPool map[string]int64
|
||||
}
|
||||
|
||||
// cacheEvictionEntry 是全局容量淘汰里的一个候选文件。
|
||||
type cacheEvictionEntry struct {
|
||||
path string
|
||||
pool string
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// cacheEvictionPoolOrder 是全局容量清理的固定优先级。
|
||||
// 顺序即“先清谁”:超限时前面的池会被整池清空,才会轮到后面的池。
|
||||
// reader-audio 不在其中:它由自身固定配额(当前 1GiB,见 reader 包的
|
||||
// audioMediaMaxBytes)在写入时独立淘汰。
|
||||
var cacheEvictionPoolOrder = []string{
|
||||
"hls",
|
||||
"images",
|
||||
"library-covers",
|
||||
}
|
||||
|
||||
// cacheCleanupKeepPercent 是触发清理后回落到的水位(占上限的百分比)。
|
||||
// 留出余量,避免刚清完又被下一次写入立刻顶到上限。
|
||||
const cacheCleanupKeepPercent = 80
|
||||
|
||||
// tempCacheFile 判断文件名是否是缓存写入中的临时产物。这些文件正在被并发
|
||||
// 写入,容量统计和清理都应跳过。
|
||||
func tempCacheFile(name string) bool {
|
||||
if tempImageCacheFile(name) {
|
||||
return true
|
||||
}
|
||||
if strings.HasSuffix(name, ".part") {
|
||||
return true
|
||||
}
|
||||
// 各缓存目录的原子写入临时文件前缀:.content- / .cover- / .audio- / .js-cache-。
|
||||
return strings.HasPrefix(name, ".content-") ||
|
||||
strings.HasPrefix(name, ".cover-") ||
|
||||
strings.HasPrefix(name, ".audio-") ||
|
||||
strings.HasPrefix(name, ".js-cache-")
|
||||
}
|
||||
|
||||
// collectCachePoolFiles 递归收集一个普通缓存池里的候选文件,按 mtime 从旧到新。
|
||||
func collectCachePoolFiles(root, pool string) []cacheEvictionEntry {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return nil
|
||||
}
|
||||
var out []cacheEvictionEntry
|
||||
_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info == nil || info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
name := info.Name()
|
||||
if tempCacheFile(name) || strings.HasSuffix(name, ".fail") {
|
||||
return nil
|
||||
}
|
||||
out = append(out, cacheEvictionEntry{
|
||||
path: path,
|
||||
pool: pool,
|
||||
size: info.Size(),
|
||||
modTime: info.ModTime(),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].modTime.Before(out[j].modTime) })
|
||||
return out
|
||||
}
|
||||
|
||||
// collectImageCacheEntries 收集图片缓存候选文件。返回顺序沿用图片池的优先级:
|
||||
// 先原图、后派生成品,池内按 mtime 从旧到新。
|
||||
func collectImageCacheEntries(imagesDir string) []cacheEvictionEntry {
|
||||
if strings.TrimSpace(imagesDir) == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(imagesDir); err != nil {
|
||||
return nil
|
||||
}
|
||||
pools := ImageCachePools(imagesDir, 0)
|
||||
var entries []imageCacheFileEntry
|
||||
var failMarkers []imageCacheFileEntry
|
||||
for i, pool := range pools {
|
||||
collectImageCachePoolFiles(pool, i, &entries, &failMarkers)
|
||||
}
|
||||
ordered := evictionOrder(entries, map[string]bool{})
|
||||
out := make([]cacheEvictionEntry, 0, len(ordered))
|
||||
for _, e := range ordered {
|
||||
out = append(out, cacheEvictionEntry{
|
||||
path: e.path,
|
||||
pool: "images",
|
||||
size: e.size,
|
||||
modTime: e.modTime,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// removeEmptyDirsUnder 删除 root 下淘汰后留下的空子目录(不含 root 本身)。
|
||||
func removeEmptyDirsUnder(root string) {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return
|
||||
}
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
return
|
||||
}
|
||||
dirs := []string{}
|
||||
_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info == nil {
|
||||
return nil
|
||||
}
|
||||
if info.IsDir() && path != root {
|
||||
dirs = append(dirs, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for i := len(dirs) - 1; i >= 0; i-- {
|
||||
_ = os.Remove(dirs[i])
|
||||
}
|
||||
}
|
||||
|
||||
// PruneCache 按全局容量池清理磁盘缓存。
|
||||
//
|
||||
// 总量口径是 hls、images、library-covers 三个池之和;reader-audio 及正文缓存、
|
||||
// 书源文件缓存都不参与这里的总量计算——reader-audio 由自身固定配额(当前
|
||||
// 1GiB)在写入时淘汰,正文缓存和书源文件缓存另有自己的 TTL / 配额。
|
||||
// 超过 totalBytes 时严格按 hls → images → library-covers 的顺序淘汰最旧文件,
|
||||
// 直到回落到上限的 80%。
|
||||
// originalsMaxBytes 是 images 原图池的独立子配额,在总量淘汰之后单独执行
|
||||
// (即使总量没超也会把原图压回配额)。
|
||||
func PruneCache(cacheDir string, totalBytes, originalsMaxBytes int64) (CacheCleanupResult, error) {
|
||||
result := CacheCleanupResult{FreedByPool: map[string]int64{}}
|
||||
if strings.TrimSpace(cacheDir) == "" {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
imagesDir := filepath.Join(cacheDir, "images")
|
||||
ordered := make([]cacheEvictionEntry, 0, 1024)
|
||||
for _, name := range cacheEvictionPoolOrder {
|
||||
if name == "images" {
|
||||
ordered = append(ordered, collectImageCacheEntries(imagesDir)...)
|
||||
continue
|
||||
}
|
||||
ordered = append(ordered, collectCachePoolFiles(filepath.Join(cacheDir, name), name)...)
|
||||
}
|
||||
|
||||
result.TotalFilesBefore = len(ordered)
|
||||
for _, entry := range ordered {
|
||||
result.TotalBytesBefore += entry.size
|
||||
}
|
||||
result.RemainingBytes = result.TotalBytesBefore
|
||||
|
||||
// 1. 全局总量:严格按池优先级淘汰,直到回落到水位。
|
||||
if totalBytes > 0 && result.RemainingBytes > totalBytes {
|
||||
target := totalBytes * cacheCleanupKeepPercent / 100
|
||||
for _, entry := range ordered {
|
||||
if result.RemainingBytes <= target {
|
||||
break
|
||||
}
|
||||
if err := os.Remove(entry.path); err != nil {
|
||||
continue
|
||||
}
|
||||
result.DeletedFiles++
|
||||
result.FreedBytes += entry.size
|
||||
result.RemainingBytes -= entry.size
|
||||
result.FreedByPool[entry.pool] += entry.size
|
||||
_ = os.Remove(entry.path + ".fail")
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 图片原图子配额独立于总量:即使总量没超,也要把原图压回自己的配额,
|
||||
// 同时清掉旧命名变体和孤儿失败标记。
|
||||
if _, err := os.Stat(imagesDir); err == nil {
|
||||
pools := ImageCachePools(imagesDir, originalsMaxBytes)
|
||||
if sub, err := PruneImageCachePools(pools, 0); err == nil {
|
||||
result.DeletedFiles += sub.DeletedFiles
|
||||
result.FreedBytes += sub.FreedBytes
|
||||
result.RemainingBytes -= sub.FreedBytes
|
||||
result.FreedByPool["images"] += sub.FreedBytes
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range cacheEvictionPoolOrder {
|
||||
removeEmptyDirsUnder(filepath.Join(cacheDir, name))
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// writePoolFile ?????????????????????????
|
||||
// writePoolFile ??????????????? mtime ???????
|
||||
func writePoolFile(t *testing.T, path string, size int, modTime time.Time) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||
@@ -39,42 +39,38 @@ func mustExist(t *testing.T, path string) {
|
||||
}
|
||||
}
|
||||
|
||||
// currentVariantPath ????????????????.<??x<??q<??>.img??
|
||||
// ??????????????????????????
|
||||
// currentVariantPath ???????????????????
|
||||
// <64 ????>.<?>x<?>q<??>.img
|
||||
func currentVariantPath(dir, keyChar string, width, height, quality int) string {
|
||||
key := strings.Repeat(keyChar, sha256HexLength)
|
||||
return filepath.Join(dir, fmt.Sprintf("%s.%dx%dq%d.img", key, width, height, quality))
|
||||
}
|
||||
|
||||
// TestPruneImageCachePoolsOriginalTTL ????????????????????
|
||||
// ?? TTL ??????????????????????
|
||||
func TestPruneImageCachePoolsOriginalTTL(t *testing.T) {
|
||||
// TestPruneImageCachePoolsKeepsOldFilesUnderQuota ????????????
|
||||
// ???????????????????????? / TTL ????
|
||||
func TestPruneImageCachePoolsKeepsOldFilesUnderQuota(t *testing.T) {
|
||||
imagesDir := t.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
stale := filepath.Join(imagesDir, imageOriginalCacheSubdir, "stale-original")
|
||||
fresh := filepath.Join(imagesDir, imageOriginalCacheSubdir, "fresh-original")
|
||||
thumb := currentVariantPath(filepath.Join(imagesDir, imageResizeCacheSubdir), "b", 480, 600, 80)
|
||||
writePoolFile(t, stale, 400, now.Add(-8*24*time.Hour))
|
||||
writePoolFile(t, fresh, 400, now.Add(-1*time.Hour))
|
||||
// ??????????TTL ????
|
||||
writePoolFile(t, thumb, 400, now.Add(-8*24*time.Hour))
|
||||
oldOriginal := filepath.Join(imagesDir, imageOriginalCacheSubdir, "old-original")
|
||||
oldThumb := currentVariantPath(filepath.Join(imagesDir, imageResizeCacheSubdir), "b", 480, 600, 80)
|
||||
writePoolFile(t, oldOriginal, 400, now.Add(-90*24*time.Hour))
|
||||
writePoolFile(t, oldThumb, 400, now.Add(-90*24*time.Hour))
|
||||
|
||||
pools := ImageCachePools(imagesDir, 0, 7*24*time.Hour)
|
||||
pools := ImageCachePools(imagesDir, 0)
|
||||
res, err := PruneImageCachePools(pools, 100*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
}
|
||||
if res.DeletedFiles != 1 {
|
||||
t.Fatalf("deleted files = %d, want 1", res.DeletedFiles)
|
||||
if res.DeletedFiles != 0 {
|
||||
t.Fatalf("deleted files = %d, want 0 (age must not trigger deletion)", res.DeletedFiles)
|
||||
}
|
||||
mustNotExist(t, stale)
|
||||
mustExist(t, fresh)
|
||||
mustExist(t, thumb)
|
||||
mustExist(t, oldOriginal)
|
||||
mustExist(t, oldThumb)
|
||||
}
|
||||
|
||||
// TestPruneImageCachePoolsEvictsOriginalsBeforeDerived ????????????
|
||||
// ????????????????????
|
||||
// ????????????????
|
||||
func TestPruneImageCachePoolsEvictsOriginalsBeforeDerived(t *testing.T) {
|
||||
imagesDir := t.TempDir()
|
||||
now := time.Now()
|
||||
@@ -82,13 +78,13 @@ func TestPruneImageCachePoolsEvictsOriginalsBeforeDerived(t *testing.T) {
|
||||
original := filepath.Join(imagesDir, imageOriginalCacheSubdir, "original")
|
||||
rendition := filepath.Join(imagesDir, imageRenditionCacheSubdir, "rendition")
|
||||
thumb := currentVariantPath(filepath.Join(imagesDir, imageResizeCacheSubdir), "c", 480, 600, 80)
|
||||
// ?????????????????
|
||||
// ?????????????????????????
|
||||
writePoolFile(t, original, 1024*1024, now.Add(-72*time.Hour))
|
||||
writePoolFile(t, rendition, 1024*1024, now.Add(-2*time.Hour))
|
||||
writePoolFile(t, thumb, 1024*1024, now.Add(-1*time.Hour))
|
||||
|
||||
pools := ImageCachePools(imagesDir, 0, 0)
|
||||
// ?? 3MB????2.4MB -> ????? 1.92MB??
|
||||
pools := ImageCachePools(imagesDir, 0)
|
||||
// ?? 3MB??? 2.4MB -> ??? 1.92MB ???
|
||||
res, err := PruneImageCachePools(pools, 2*1024*1024+400*1024)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
@@ -104,8 +100,8 @@ func TestPruneImageCachePoolsEvictsOriginalsBeforeDerived(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPruneImageCachePoolsOriginalQuotaKeepsDerived ????????????
|
||||
// ??????????????
|
||||
// TestPruneImageCachePoolsOriginalQuotaKeepsDerived ???????????
|
||||
// ????????????
|
||||
func TestPruneImageCachePoolsOriginalQuotaKeepsDerived(t *testing.T) {
|
||||
imagesDir := t.TempDir()
|
||||
now := time.Now()
|
||||
@@ -117,8 +113,8 @@ func TestPruneImageCachePoolsOriginalQuotaKeepsDerived(t *testing.T) {
|
||||
writePoolFile(t, newOriginal, 800, now.Add(-1*time.Hour))
|
||||
writePoolFile(t, thumb, 800, now.Add(-2*time.Hour))
|
||||
|
||||
// ???? 1000 ??????????1600 -> ????800????? 1 ????
|
||||
pools := ImageCachePools(imagesDir, 1000, 0)
|
||||
// ???? 1000??? 1600 ?? -> ??? 800??? 1 ???????
|
||||
pools := ImageCachePools(imagesDir, 1000)
|
||||
res, err := PruneImageCachePools(pools, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
@@ -131,24 +127,26 @@ func TestPruneImageCachePoolsOriginalQuotaKeepsDerived(t *testing.T) {
|
||||
mustExist(t, thumb)
|
||||
}
|
||||
|
||||
// TestPruneImageCachePoolsCleansFailMarkers ????????????????
|
||||
// ????????????????
|
||||
// TestPruneImageCachePoolsCleansFailMarkers ???????????????
|
||||
// ???????????????????????????????????
|
||||
func TestPruneImageCachePoolsCleansFailMarkers(t *testing.T) {
|
||||
imagesDir := t.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
entry := filepath.Join(imagesDir, imageOriginalCacheSubdir, "entry")
|
||||
entryMarker := entry + ".fail"
|
||||
kept := filepath.Join(imagesDir, imageOriginalCacheSubdir, "kept")
|
||||
keptMarker := kept + ".fail"
|
||||
orphanMarker := filepath.Join(imagesDir, imageOriginalCacheSubdir, "orphan.fail")
|
||||
freshMarker := filepath.Join(imagesDir, imageOriginalCacheSubdir, "fresh.fail")
|
||||
|
||||
writePoolFile(t, entry, 900, now.Add(-10*time.Hour))
|
||||
writePoolFile(t, entryMarker, 35, now.Add(-10*time.Hour))
|
||||
writePoolFile(t, entry, 900, now.Add(-2*time.Hour))
|
||||
writePoolFile(t, entryMarker, 35, now.Add(-2*time.Hour))
|
||||
writePoolFile(t, kept, 300, now.Add(-1*time.Hour))
|
||||
writePoolFile(t, keptMarker, 35, now.Add(-1*time.Hour))
|
||||
writePoolFile(t, orphanMarker, 35, now.Add(-48*time.Hour))
|
||||
writePoolFile(t, freshMarker, 35, now.Add(-1*time.Minute))
|
||||
|
||||
// ???? 500????900 ?????????????????
|
||||
pools := ImageCachePools(imagesDir, 500, 0)
|
||||
// ???? 500??? 900+300=1200 ???? mtime ????? entry ? 400?
|
||||
pools := ImageCachePools(imagesDir, 500)
|
||||
res, err := PruneImageCachePools(pools, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
@@ -158,26 +156,27 @@ func TestPruneImageCachePoolsCleansFailMarkers(t *testing.T) {
|
||||
}
|
||||
mustNotExist(t, entry)
|
||||
mustNotExist(t, entryMarker)
|
||||
mustExist(t, kept)
|
||||
mustExist(t, keptMarker)
|
||||
mustNotExist(t, orphanMarker)
|
||||
mustExist(t, freshMarker)
|
||||
}
|
||||
|
||||
// TestPruneImageCachePoolsSkipsTemporaryFiles ????????????????
|
||||
// TestPruneImageCachePoolsSkipsTemporaryFiles ??????????????
|
||||
// ?????????????????
|
||||
func TestPruneImageCachePoolsSkipsTemporaryFiles(t *testing.T) {
|
||||
imagesDir := t.TempDir()
|
||||
tmp := filepath.Join(imagesDir, imageOriginalCacheSubdir, "img-1234.tmp")
|
||||
writePoolFile(t, tmp, 4096, time.Now().Add(-48*time.Hour))
|
||||
|
||||
pools := ImageCachePools(imagesDir, 1, 0)
|
||||
pools := ImageCachePools(imagesDir, 1)
|
||||
if _, err := PruneImageCachePools(pools, 1); err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
}
|
||||
mustExist(t, tmp)
|
||||
}
|
||||
|
||||
// TestPruneImageCachePoolsDropsLegacyResizeVariants ????????.img????
|
||||
// ?????????????????????????????????????
|
||||
// TestPruneImageCachePoolsDropsLegacyResizeVariants ?????? .img ??
|
||||
// ???????????????????????????
|
||||
func TestPruneImageCachePoolsDropsLegacyResizeVariants(t *testing.T) {
|
||||
imagesDir := t.TempDir()
|
||||
resized := filepath.Join(imagesDir, imageResizeCacheSubdir)
|
||||
@@ -188,7 +187,7 @@ func TestPruneImageCachePoolsDropsLegacyResizeVariants(t *testing.T) {
|
||||
writePoolFile(t, current, 400, time.Now())
|
||||
writePoolFile(t, other, 400, time.Now())
|
||||
|
||||
pools := ImageCachePools(imagesDir, 0, 0)
|
||||
pools := ImageCachePools(imagesDir, 0)
|
||||
res, err := PruneImageCachePools(pools, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
@@ -201,8 +200,8 @@ func TestPruneImageCachePoolsDropsLegacyResizeVariants(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPruneImageCachePoolsCountsLegacyFlatLayout ??????images/ ????
|
||||
// ?????????????????
|
||||
// TestPruneImageCachePoolsCountsLegacyFlatLayout ???????images/ ???
|
||||
// ???????????????????????
|
||||
func TestPruneImageCachePoolsCountsLegacyFlatLayout(t *testing.T) {
|
||||
imagesDir := t.TempDir()
|
||||
legacy := filepath.Join(imagesDir, "legacy-flat-cache")
|
||||
@@ -210,7 +209,8 @@ func TestPruneImageCachePoolsCountsLegacyFlatLayout(t *testing.T) {
|
||||
writePoolFile(t, legacy, 900, time.Now().Add(-30*24*time.Hour))
|
||||
writePoolFile(t, thumb, 900, time.Now().Add(-30*24*time.Hour))
|
||||
|
||||
pools := ImageCachePools(imagesDir, 0, 24*time.Hour)
|
||||
// ???? 500??????????????????????????
|
||||
pools := ImageCachePools(imagesDir, 500)
|
||||
res, err := PruneImageCachePools(pools, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
|
||||
@@ -134,7 +134,7 @@ func TestPruneImageCache_ZeroOrNegativeLimit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerJobCleanImageCache(t *testing.T) {
|
||||
func TestSchedulerJobCleanCache(t *testing.T) {
|
||||
cacheRoot := t.TempDir()
|
||||
imagesDir := filepath.Join(cacheRoot, "images")
|
||||
if err := os.MkdirAll(imagesDir, 0o750); err != nil {
|
||||
@@ -148,15 +148,117 @@ func TestSchedulerJobCleanImageCache(t *testing.T) {
|
||||
|
||||
scheduler := NewSchedulerService(zap.NewNop(), nil, nil, nil, nil, nil, cacheRoot)
|
||||
// Set limit to 1MB; our file is 2MB -> should be pruned
|
||||
scheduler.SetImageCachePolicyProvider(func() ImageCachePolicy {
|
||||
return ImageCachePolicy{TotalBytes: 1 * 1024 * 1024}
|
||||
scheduler.SetCacheCleanupPolicyProvider(func() CacheCleanupPolicy {
|
||||
return CacheCleanupPolicy{TotalBytes: 1 * 1024 * 1024}
|
||||
})
|
||||
|
||||
if err := scheduler.jobCleanImageCache(context.Background()); err != nil {
|
||||
t.Fatalf("jobCleanImageCache failed: %v", err)
|
||||
if err := scheduler.jobCleanCache(context.Background()); err != nil {
|
||||
t.Fatalf("jobCleanCache failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(f); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected file to be pruned, got err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPruneCacheEvictionOrder 验证总量超限时严格按
|
||||
// hls → images → library-covers 的顺序淘汰,且 reader-audio 不参与。
|
||||
func TestPruneCacheEvictionOrder(t *testing.T) {
|
||||
cacheRoot := t.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
audio := filepath.Join(cacheRoot, "reader-audio", "old.mp3")
|
||||
hls := filepath.Join(cacheRoot, "hls", "job", "seg.ts")
|
||||
image := filepath.Join(cacheRoot, "images", "originals", "poster.jpg")
|
||||
cover := filepath.Join(cacheRoot, "library-covers", "cover.jpg")
|
||||
for _, p := range []string{audio, hls, image, cover} {
|
||||
writePoolFile(t, p, 1024*1024, now.Add(-time.Hour))
|
||||
}
|
||||
|
||||
// 计入总量的只有 hls+images+cover = 3MB,上限 2MB -> 回落到 1.6MB:
|
||||
// 按顺序删 hls、images 两个文件即可;reader-audio 与封面都应保留。
|
||||
res, err := PruneCache(cacheRoot, 2*1024*1024, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
}
|
||||
if res.DeletedFiles != 2 {
|
||||
t.Fatalf("deleted files = %d, want 2", res.DeletedFiles)
|
||||
}
|
||||
mustExist(t, audio)
|
||||
mustNotExist(t, hls)
|
||||
mustNotExist(t, image)
|
||||
mustExist(t, cover)
|
||||
if _, ok := res.FreedByPool["reader-audio"]; ok {
|
||||
t.Fatalf("reader-audio must not be part of the global pool, got %+v", res.FreedByPool)
|
||||
}
|
||||
if res.FreedByPool["hls"] != 1024*1024 || res.FreedByPool["images"] != 1024*1024 {
|
||||
t.Fatalf("unexpected per-pool stats: %+v", res.FreedByPool)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPruneCacheLeavesReaderAudioAlone reader-audio 有独立的 1GB 写入时淘汰,
|
||||
// 不参与总容量统计:即使它单独就超过全局上限,全局任务也不应删它。
|
||||
func TestPruneCacheLeavesReaderAudioAlone(t *testing.T) {
|
||||
cacheRoot := t.TempDir()
|
||||
audio := filepath.Join(cacheRoot, "reader-audio", "book", "chapter.mp3")
|
||||
writePoolFile(t, audio, 4*1024*1024, time.Now().Add(-time.Hour))
|
||||
|
||||
res, err := PruneCache(cacheRoot, 1*1024*1024, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
}
|
||||
if res.DeletedFiles != 0 {
|
||||
t.Fatalf("deleted files = %d, want 0", res.DeletedFiles)
|
||||
}
|
||||
if res.TotalBytesBefore != 0 {
|
||||
t.Fatalf("reader-audio should not count toward total, got %d bytes", res.TotalBytesBefore)
|
||||
}
|
||||
mustExist(t, audio)
|
||||
}
|
||||
|
||||
// TestPruneCacheAppliesOriginalsSubQuota 验证总量没超时,图片原图子配额仍然生效。
|
||||
func TestPruneCacheAppliesOriginalsSubQuota(t *testing.T) {
|
||||
cacheRoot := t.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
oldOriginal := filepath.Join(cacheRoot, "images", "originals", "old.jpg")
|
||||
newOriginal := filepath.Join(cacheRoot, "images", "originals", "new.jpg")
|
||||
derived := currentVariantPath(filepath.Join(cacheRoot, "images", "resize"), "d", 480, 600, 80)
|
||||
writePoolFile(t, oldOriginal, 800, now.Add(-2*time.Hour))
|
||||
writePoolFile(t, newOriginal, 800, now.Add(-time.Hour))
|
||||
writePoolFile(t, derived, 800, now.Add(-time.Hour))
|
||||
|
||||
// 总量上限很大(不触发全局淘汰),原图子配额 1000B -> 删到 800B。
|
||||
res, err := PruneCache(cacheRoot, 100*1024*1024, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
}
|
||||
if res.DeletedFiles != 1 {
|
||||
t.Fatalf("deleted files = %d, want 1", res.DeletedFiles)
|
||||
}
|
||||
mustNotExist(t, oldOriginal)
|
||||
mustExist(t, newOriginal)
|
||||
mustExist(t, derived)
|
||||
}
|
||||
|
||||
// TestPruneCacheOldestFirstWithinPool 验证同一池内先删最旧的文件。
|
||||
func TestPruneCacheOldestFirstWithinPool(t *testing.T) {
|
||||
cacheRoot := t.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
oldHLS := filepath.Join(cacheRoot, "hls", "job-a", "seg1.ts")
|
||||
newHLS := filepath.Join(cacheRoot, "hls", "job-b", "seg1.ts")
|
||||
writePoolFile(t, oldHLS, 1024*1024, now.Add(-2*time.Hour))
|
||||
writePoolFile(t, newHLS, 1024*1024, now.Add(-time.Minute))
|
||||
|
||||
// 上限 1.25MB -> 回落水位 1MB:删掉最旧的 1MB 后正好达标。
|
||||
res, err := PruneCache(cacheRoot, 1280*1024, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("prune failed: %v", err)
|
||||
}
|
||||
if res.DeletedFiles != 1 {
|
||||
t.Fatalf("deleted files = %d, want 1", res.DeletedFiles)
|
||||
}
|
||||
mustNotExist(t, oldHLS)
|
||||
mustExist(t, newHLS)
|
||||
}
|
||||
|
||||
@@ -8,8 +8,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -130,14 +132,24 @@ type RespBase struct {
|
||||
|
||||
// doJSON 执行 HTTP 请求并解析为统一响应;带 AccessToken(access=true 时)。
|
||||
func (c *OpenClient) doJSON(ctx context.Context, method, rawURL string, form map[string]string, access bool, retries int, uas ...string) (*RespBase, error) {
|
||||
executor := c.executor
|
||||
if executor == nil {
|
||||
executor = GetGlobalExecutor()
|
||||
}
|
||||
ua := ""
|
||||
if len(uas) > 0 {
|
||||
ua = uas[0]
|
||||
}
|
||||
return c.doJSONWithBuilder(ctx, func() (*http.Request, error) {
|
||||
return c.buildRequestWithUA(ctx, method, rawURL, form, access, ua)
|
||||
}, access, retries)
|
||||
}
|
||||
|
||||
// doJSONWithBuilder 与 doJSON 共用同一套限流、重试、令牌刷新与错误封装逻辑,
|
||||
// 差别只在请求构造方式:urlencoded(doJSON)、multipart(doAuthMultipart)。
|
||||
// 云下载(offline)系列接口要求 multipart/form-data,若各写一套请求循环,
|
||||
// 限流熔断与 token 轮转的处理迟早会漂移。
|
||||
func (c *OpenClient) doJSONWithBuilder(ctx context.Context, build func() (*http.Request, error), access bool, retries int) (*RespBase, error) {
|
||||
executor := c.executor
|
||||
if executor == nil {
|
||||
executor = GetGlobalExecutor()
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= retries; attempt++ {
|
||||
@@ -146,7 +158,7 @@ func (c *OpenClient) doJSON(ctx context.Context, method, rawURL string, form map
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := c.buildRequestWithUA(ctx, method, rawURL, form, access, ua)
|
||||
req, err := build()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -293,6 +305,45 @@ func (c *OpenClient) buildRequestWithUA(ctx context.Context, method, rawURL stri
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// buildMultipartRequest 构造 multipart/form-data 请求。
|
||||
// 115 云下载(offline)系列写接口要求 multipart/form-data,与其它接口的
|
||||
// urlencoded 表单不同;这里只支持纯文本字段(不含文件),种子文件走上传接口。
|
||||
func (c *OpenClient) buildMultipartRequest(ctx context.Context, method, rawURL string, fields map[string]string) (*http.Request, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
// 固定字段顺序,便于抓包比对与排障(map 遍历顺序随机)。
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if err := w.WriteField(k, fields[k]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, strings.ToUpper(method), rawURL, &buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", DefaultUA)
|
||||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||||
if accessToken := c.currentAccessToken(); accessToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+accessToken)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// doAuthMultipart 以 multipart/form-data 发起需要 access_token 的请求。
|
||||
func (c *OpenClient) doAuthMultipart(ctx context.Context, method, rawURL string, fields map[string]string, retries int) (*RespBase, error) {
|
||||
return c.doJSONWithBuilder(ctx, func() (*http.Request, error) {
|
||||
return c.buildMultipartRequest(ctx, method, rawURL, fields)
|
||||
}, true, retries)
|
||||
}
|
||||
|
||||
// doAuthJSON 带 AccessToken 的业务请求。
|
||||
func (c *OpenClient) doAuthJSON(ctx context.Context, method, rawURL string, form map[string]string, retries int) (*RespBase, error) {
|
||||
return c.doJSON(ctx, method, rawURL, form, true, retries)
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
// 115 开放平台「云下载」(离线下载)API 封装。
|
||||
//
|
||||
// 文档(仓库内离线副本):115doc/115开放平台/API列表/云下载/
|
||||
//
|
||||
// 能力边界(已核对官方文档,勿凭客户端印象假设):
|
||||
// - add_task_urls 支持 HTTP(S)/FTP/磁力/电驴,一次最多 115 条,无「选择文件」参数;
|
||||
// - 选择文件只能走 BT 路径:torrent(解析已上传种子)→ add_task_bt(wanted 传索引);
|
||||
// - torrent 的入参是「种子文件在 115 的 SHA1 + 提取码」,不接受 magnet。
|
||||
//
|
||||
// 本文件只做协议封装:不碰数据库、不做编排、不判断业务状态,方便单测。
|
||||
package cloud115
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// 云下载接口路径。
|
||||
const (
|
||||
OfflinePathAddTaskURLs = "/open/offline/add_task_urls"
|
||||
OfflinePathTorrent = "/open/offline/torrent"
|
||||
OfflinePathAddTaskBT = "/open/offline/add_task_bt"
|
||||
OfflinePathTaskList = "/open/offline/get_task_list"
|
||||
OfflinePathQuotaInfo = "/open/offline/get_quota_info"
|
||||
OfflinePathDelTask = "/open/offline/del_task"
|
||||
OfflinePathClearTask = "/open/offline/clear_task"
|
||||
)
|
||||
|
||||
// OfflineMaxURLsPerBatch 是 add_task_urls 单批链接上限(官方文档:1000011 超 115 个)。
|
||||
const OfflineMaxURLsPerBatch = 115
|
||||
|
||||
// 云下载业务错误码。
|
||||
const (
|
||||
OfflineCodeOK = 0
|
||||
OfflineCodeQuotaUsedUp = 1000012 // 云下载配额已用完
|
||||
OfflineCodeStorageFull = 91006 // 存储空间不足
|
||||
OfflineCodeTorrentMissing = 20018 // 文件不存在或已删除(种子无效)
|
||||
OfflineCodeTooManyURLs = 1000011 // 链接数量超过 115 个
|
||||
OfflineCodeParamError = 990002 // 参数错误
|
||||
OfflineCodeOperationFailed = 980004 // 操作失败,可重试
|
||||
)
|
||||
|
||||
// 云下载任务状态(get_task_list → data.tasks[].status)。
|
||||
const (
|
||||
OfflineTaskStatusFailed = -1 // 下载失败
|
||||
OfflineTaskStatusAllocating = 0 // 分配中
|
||||
OfflineTaskStatusDownloading = 1 // 下载中
|
||||
OfflineTaskStatusDone = 2 // 下载成功
|
||||
)
|
||||
|
||||
// 清空任务的类型(clear_task → flag)。
|
||||
const (
|
||||
OfflineClearDone = 0
|
||||
OfflineClearAll = 1
|
||||
OfflineClearFailed = 2
|
||||
OfflineClearRunning = 3
|
||||
OfflineClearDoneWithFile = 4
|
||||
OfflineClearAllWithFile = 5
|
||||
)
|
||||
|
||||
// OfflineTaskURLResult 是 add_task_urls 返回的逐条结果。
|
||||
type OfflineTaskURLResult struct {
|
||||
State bool `json:"state"`
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// OfflineTorrentFile 是种子解析结果中的单个文件。
|
||||
type OfflineTorrentFile struct {
|
||||
Size int64 `json:"size"`
|
||||
Path string `json:"path"`
|
||||
Wanted int `json:"wanted"` // 115 认为该文件是否默认选中
|
||||
}
|
||||
|
||||
// OfflineTorrentInfo 是 /offline/torrent 的解析结果。
|
||||
type OfflineTorrentInfo struct {
|
||||
FileSize int64 `json:"file_size"`
|
||||
Name string `json:"torrent_name"`
|
||||
FileCount int `json:"file_count"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Files []OfflineTorrentFile `json:"torrent_filelist"`
|
||||
}
|
||||
|
||||
// OfflineTask 是云下载任务列表中的一条任务。
|
||||
type OfflineTask struct {
|
||||
InfoHash string `json:"info_hash"`
|
||||
AddTime int64 `json:"add_time"`
|
||||
PercentDone int `json:"percentDone"` // 注意:官方字段是驼峰,其余为 snake_case
|
||||
Size int64 `json:"size"`
|
||||
Name string `json:"name"`
|
||||
LastUpdate int64 `json:"last_update"`
|
||||
FileID string `json:"file_id"`
|
||||
DeleteFileID string `json:"delete_file_id"`
|
||||
Status int `json:"status"`
|
||||
URL string `json:"url"`
|
||||
WpPathID string `json:"wp_path_id"`
|
||||
Def2 int `json:"def2"`
|
||||
PlayLong int `json:"play_long"`
|
||||
CanAppeal int `json:"can_appeal"`
|
||||
}
|
||||
|
||||
// OfflineTaskPage 是分页任务列表。
|
||||
type OfflineTaskPage struct {
|
||||
Page int `json:"page"`
|
||||
PageCount int `json:"page_count"`
|
||||
Count int `json:"count"`
|
||||
Tasks []OfflineTask `json:"tasks"`
|
||||
}
|
||||
|
||||
// OfflineQuotaPackage 是某一类配额的使用情况。
|
||||
type OfflineQuotaPackage struct {
|
||||
Surplus int `json:"surplus"`
|
||||
Used int `json:"used"`
|
||||
Count int `json:"count"`
|
||||
Name string `json:"name"`
|
||||
ExpireInfo []OfflineQuotaExpire `json:"expire_info"`
|
||||
}
|
||||
|
||||
// OfflineQuotaExpire 是配额过期明细。
|
||||
type OfflineQuotaExpire struct {
|
||||
Surplus int `json:"surplus"`
|
||||
ExpireTime int64 `json:"expire_time"`
|
||||
}
|
||||
|
||||
// OfflineQuota 是云下载配额总览。
|
||||
type OfflineQuota struct {
|
||||
Package []OfflineQuotaPackage `json:"package"`
|
||||
Count int `json:"count"`
|
||||
Surplus int `json:"surplus"`
|
||||
Used int `json:"used"`
|
||||
}
|
||||
|
||||
// OfflineBTRequest 是 add_task_bt 的请求参数。
|
||||
type OfflineBTRequest struct {
|
||||
InfoHash string
|
||||
Wanted string // 半角逗号分隔的文件索引;官方文档:「可以为 0,但不能为空」
|
||||
SavePath string // 相对 wp_path_id 的保存路径,可为空
|
||||
TorrentSha1 string
|
||||
PickCode string
|
||||
WpPathID string
|
||||
}
|
||||
|
||||
// AddTaskURLs 批量添加链接任务(HTTP/FTP/磁力/电驴),单批 ≤115 条。
|
||||
// 返回逐条结果:整体请求成功但个别链接失败是常态,调用方必须逐条判断 State。
|
||||
// 注意:data[].info_hash 只在受理成功时才有值,链接类任务可能为空。
|
||||
func (c *OpenClient) AddTaskURLs(ctx context.Context, urls []string, wpPathID string) ([]OfflineTaskURLResult, error) {
|
||||
cleaned := make([]string, 0, len(urls))
|
||||
for _, u := range urls {
|
||||
if u = strings.TrimSpace(u); u != "" {
|
||||
cleaned = append(cleaned, u)
|
||||
}
|
||||
}
|
||||
if len(cleaned) == 0 {
|
||||
return nil, fmt.Errorf("115: 云下载链接为空")
|
||||
}
|
||||
if len(cleaned) > OfflineMaxURLsPerBatch {
|
||||
return nil, fmt.Errorf("115: 单批云下载链接最多 %d 条,当前 %d 条", OfflineMaxURLsPerBatch, len(cleaned))
|
||||
}
|
||||
if wpPathID = strings.TrimSpace(wpPathID); wpPathID == "" {
|
||||
wpPathID = "0"
|
||||
}
|
||||
fields := map[string]string{
|
||||
"urls": strings.Join(cleaned, "\n"),
|
||||
"wp_path_id": wpPathID,
|
||||
}
|
||||
resp, err := c.doAuthMultipart(ctx, "POST", ProAPIBase+OfflinePathAddTaskURLs, fields, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var results []OfflineTaskURLResult
|
||||
if err := unmarshalOfflineData(resp, &results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// ParseTorrent 解析已上传到 115 的 BT 种子文件。
|
||||
// torrentSha1/pickCode 必须来自该账号空间内的种子文件(上传接口返回值)。
|
||||
func (c *OpenClient) ParseTorrent(ctx context.Context, torrentSha1, pickCode string) (*OfflineTorrentInfo, error) {
|
||||
torrentSha1 = strings.TrimSpace(torrentSha1)
|
||||
pickCode = strings.TrimSpace(pickCode)
|
||||
if torrentSha1 == "" || pickCode == "" {
|
||||
return nil, fmt.Errorf("115: 解析种子需要 torrent_sha1 与 pick_code")
|
||||
}
|
||||
fields := map[string]string{
|
||||
"torrent_sha1": torrentSha1,
|
||||
"pick_code": pickCode,
|
||||
}
|
||||
resp, err := c.doAuthMultipart(ctx, "POST", ProAPIBase+OfflinePathTorrent, fields, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info OfflineTorrentInfo
|
||||
if err := unmarshalOfflineData(resp, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &info, nil
|
||||
}
|
||||
|
||||
// AddTaskBT 根据已解析的种子添加云下载 BT 任务。
|
||||
// wanted 传文件索引(半角逗号分隔);官方文档允许传 "0",但语义未在文档中写明,
|
||||
// 因此调用方应优先传解析结果推导出的索引列表(见 service 层)。
|
||||
func (c *OpenClient) AddTaskBT(ctx context.Context, req OfflineBTRequest) error {
|
||||
infoHash := strings.TrimSpace(req.InfoHash)
|
||||
torrentSha1 := strings.TrimSpace(req.TorrentSha1)
|
||||
pickCode := strings.TrimSpace(req.PickCode)
|
||||
if infoHash == "" || torrentSha1 == "" || pickCode == "" {
|
||||
return fmt.Errorf("115: 添加 BT 任务缺少 info_hash/torrent_sha1/pick_code")
|
||||
}
|
||||
wanted := strings.TrimSpace(req.Wanted)
|
||||
if wanted == "" {
|
||||
wanted = "0"
|
||||
}
|
||||
wpPathID := strings.TrimSpace(req.WpPathID)
|
||||
if wpPathID == "" {
|
||||
wpPathID = "0"
|
||||
}
|
||||
fields := map[string]string{
|
||||
"info_hash": infoHash,
|
||||
"wanted": wanted,
|
||||
"save_path": strings.TrimSpace(req.SavePath),
|
||||
"torrent_sha1": torrentSha1,
|
||||
"pick_code": pickCode,
|
||||
"wp_path_id": wpPathID,
|
||||
}
|
||||
if _, err := c.doAuthMultipart(ctx, "POST", ProAPIBase+OfflinePathAddTaskBT, fields, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetOfflineTaskList 分页获取云下载任务列表(page 从 1 开始)。
|
||||
func (c *OpenClient) GetOfflineTaskList(ctx context.Context, page int) (*OfflineTaskPage, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
resp, err := c.doAuthJSON(ctx, "GET", ProAPIBase+OfflinePathTaskList, map[string]string{
|
||||
"page": fmt.Sprint(page),
|
||||
}, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var list OfflineTaskPage
|
||||
if err := unmarshalOfflineData(resp, &list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &list, nil
|
||||
}
|
||||
|
||||
// GetOfflineQuota 获取云下载配额信息。
|
||||
func (c *OpenClient) GetOfflineQuota(ctx context.Context) (*OfflineQuota, error) {
|
||||
resp, err := c.doAuthJSON(ctx, "GET", ProAPIBase+OfflinePathQuotaInfo, nil, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var quota OfflineQuota
|
||||
if err := unmarshalOfflineData(resp, "a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return "a, nil
|
||||
}
|
||||
|
||||
// DeleteOfflineTask 删除指定云下载任务(按 info_hash)。
|
||||
// delSourceFile=true 时同时删除已下载的源文件。
|
||||
func (c *OpenClient) DeleteOfflineTask(ctx context.Context, infoHash string, delSourceFile bool) error {
|
||||
infoHash = strings.TrimSpace(infoHash)
|
||||
if infoHash == "" {
|
||||
return fmt.Errorf("115: 删除云下载任务需要 info_hash")
|
||||
}
|
||||
del := "0"
|
||||
if delSourceFile {
|
||||
del = "1"
|
||||
}
|
||||
fields := map[string]string{
|
||||
"info_hash": infoHash,
|
||||
"del_source_file": del,
|
||||
}
|
||||
_, err := c.doAuthMultipart(ctx, "POST", ProAPIBase+OfflinePathDelTask, fields, 1)
|
||||
return err
|
||||
}
|
||||
|
||||
// ClearOfflineTasks 按类型清空云下载任务。
|
||||
func (c *OpenClient) ClearOfflineTasks(ctx context.Context, flag int) error {
|
||||
fields := map[string]string{"flag": fmt.Sprint(flag)}
|
||||
_, err := c.doAuthMultipart(ctx, "POST", ProAPIBase+OfflinePathClearTask, fields, 1)
|
||||
return err
|
||||
}
|
||||
|
||||
// unmarshalOfflineData 解析 data 字段。115 的 /offline/* 接口在 state=true 时
|
||||
// 仍可能给出空 data(例如 add_task_bt 成功时 data 为空数组),调用方需容忍零值。
|
||||
func unmarshalOfflineData(resp *RespBase, out any) error {
|
||||
if resp == nil || len(resp.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(resp.Data, out); err != nil {
|
||||
return fmt.Errorf("115: 解析云下载响应失败:%w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package cloud115
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// newOfflineTestClient 构造一个指向 mock server 的客户端。
|
||||
// 令牌直接写入内存字段,避免走文件/数据库。
|
||||
func newOfflineTestClient(accessToken string) *OpenClient {
|
||||
return NewOpenClient("test-app", accessToken, "")
|
||||
}
|
||||
|
||||
func TestAddTaskURLsUsesMultipartAndBatches(t *testing.T) {
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != OfflinePathAddTaskURLs {
|
||||
t.Errorf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("unexpected method %s", r.Method)
|
||||
}
|
||||
if ct := r.Header.Get("Content-Type"); !strings.HasPrefix(ct, "multipart/form-data") {
|
||||
t.Errorf("expected multipart content-type, got %q", ct)
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); !strings.HasPrefix(got, "Bearer ") || len(got) <= len("Bearer ") {
|
||||
t.Errorf("unexpected authorization %q", got)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
if got := r.FormValue("wp_path_id"); got != "12345" {
|
||||
t.Errorf("wp_path_id = %q, want 12345", got)
|
||||
}
|
||||
urls := r.FormValue("urls")
|
||||
if want := "magnet:?xt=urn:btih:AAA\ned2k://|file|x|1|H|/"; urls != want {
|
||||
t.Errorf("urls = %q, want %q", urls, want)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"state":true,"code":0,"data":[{"state":true,"code":0,"info_hash":"ABC","url":"magnet:?xt=urn:btih:AAA"}]}`))
|
||||
})
|
||||
|
||||
client := newOfflineTestClient("token-abc")
|
||||
res, err := client.AddTaskURLs(context.Background(), []string{
|
||||
" magnet:?xt=urn:btih:AAA ",
|
||||
"ed2k://|file|x|1|H|/",
|
||||
}, "12345")
|
||||
if err != nil {
|
||||
t.Fatalf("AddTaskURLs: %v", err)
|
||||
}
|
||||
if len(res) != 1 || !res[0].State || res[0].InfoHash != "ABC" {
|
||||
t.Fatalf("unexpected result %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddTaskURLsRejectsOverLimitWithoutRequest(t *testing.T) {
|
||||
called := false
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
called = true
|
||||
})
|
||||
urls := make([]string, OfflineMaxURLsPerBatch+1)
|
||||
for i := range urls {
|
||||
urls[i] = "https://example.com/a"
|
||||
}
|
||||
if _, err := newOfflineTestClient("t").AddTaskURLs(context.Background(), urls, "1"); err == nil {
|
||||
t.Fatal("expected error for over-limit batch")
|
||||
}
|
||||
if called {
|
||||
t.Fatal("must not call 115 when batch exceeds the documented limit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddTaskURLsSurfacesBusinessError(t *testing.T) {
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"state":false,"code":1000012,"message":"云下载配额已用完"}`))
|
||||
})
|
||||
_, err := newOfflineTestClient("t").AddTaskURLs(context.Background(), []string{"magnet:?xt=urn:btih:AAA"}, "1")
|
||||
if err == nil {
|
||||
t.Fatal("expected business error")
|
||||
}
|
||||
apiErr, ok := err.(*OpenAPIError)
|
||||
if !ok {
|
||||
t.Fatalf("expected *OpenAPIError, got %T", err)
|
||||
}
|
||||
if apiErr.Code != OfflineCodeQuotaUsedUp {
|
||||
t.Fatalf("code = %d, want %d", apiErr.Code, OfflineCodeQuotaUsedUp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTorrentDecodesFileList(t *testing.T) {
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != OfflinePathTorrent {
|
||||
t.Errorf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
if r.FormValue("torrent_sha1") != "SHA1" || r.FormValue("pick_code") != "PICK" {
|
||||
t.Errorf("unexpected form: %v", r.MultipartForm.Value)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"state":true,"code":0,"data":{
|
||||
"file_size":1024,"torrent_name":"Demo","file_count":2,"info_hash":"HASH",
|
||||
"torrent_filelist":[{"size":512,"path":"a.mkv","wanted":1},{"size":512,"path":"b.srt","wanted":0}]}}`))
|
||||
})
|
||||
info, err := newOfflineTestClient("t").ParseTorrent(context.Background(), "SHA1", "PICK")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseTorrent: %v", err)
|
||||
}
|
||||
if info.InfoHash != "HASH" || info.Name != "Demo" || len(info.Files) != 2 {
|
||||
t.Fatalf("unexpected info %+v", info)
|
||||
}
|
||||
if info.Files[0].Path != "a.mkv" || info.Files[0].Size != 512 {
|
||||
t.Fatalf("unexpected file %+v", info.Files[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddTaskBTRequiresSeedFields(t *testing.T) {
|
||||
called := false
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) { called = true })
|
||||
client := newOfflineTestClient("t")
|
||||
if err := client.AddTaskBT(context.Background(), OfflineBTRequest{InfoHash: "H"}); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
if called {
|
||||
t.Fatal("must validate locally before calling 115")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOfflineTaskListParsesCamelCasePercentDone(t *testing.T) {
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != OfflinePathTaskList {
|
||||
t.Errorf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
if r.URL.Query().Get("page") != "2" {
|
||||
t.Errorf("page = %q, want 2", r.URL.Query().Get("page"))
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"state":true,"code":0,"data":{"page":2,"page_count":3,"count":30,
|
||||
"tasks":[{"info_hash":"H","percentDone":42,"size":100,"name":"n","status":1,"url":"u","wp_path_id":"9"}]}}`))
|
||||
})
|
||||
page, err := newOfflineTestClient("t").GetOfflineTaskList(context.Background(), 2)
|
||||
if err != nil {
|
||||
t.Fatalf("GetOfflineTaskList: %v", err)
|
||||
}
|
||||
if page.Page != 2 || page.PageCount != 3 || len(page.Tasks) != 1 {
|
||||
t.Fatalf("unexpected page %+v", page)
|
||||
}
|
||||
// 官方字段是驼峰 percentDone:写成 snake_case 会永远 0,这里必须锁死。
|
||||
if page.Tasks[0].PercentDone != 42 {
|
||||
t.Fatalf("percentDone = %d, want 42", page.Tasks[0].PercentDone)
|
||||
}
|
||||
if page.Tasks[0].Status != OfflineTaskStatusDownloading {
|
||||
t.Fatalf("status = %d", page.Tasks[0].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOfflineQuota(t *testing.T) {
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != OfflinePathQuotaInfo {
|
||||
t.Errorf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"state":true,"code":0,"data":{"package":[{"surplus":3,"used":1,"count":4,"name":"p"}],"count":4,"surplus":3,"used":1}}`))
|
||||
})
|
||||
q, err := newOfflineTestClient("t").GetOfflineQuota(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetOfflineQuota: %v", err)
|
||||
}
|
||||
if q.Surplus != 3 || q.Count != 4 || len(q.Package) != 1 {
|
||||
t.Fatalf("unexpected quota %+v", q)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCloud115HLSChildRejectsMismatchedSessionMediaBeforeFetch(t *testing.T) {
|
||||
fetched := false
|
||||
proxy := &Cloud115HLSProxy{sessions: map[string]*cloud115HLSSession{}}
|
||||
proxy.client = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
fetched = true
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"video/mp2t"}}, Body: io.NopCloser(strings.NewReader("private-segment")), Request: req}, nil
|
||||
})}
|
||||
proxy.sessions["session-b"] = &cloud115HLSSession{
|
||||
ID: "session-b", MediaID: "media-b", ExpiresAt: time.Now().Add(time.Hour),
|
||||
entries: map[string]string{"segment": "https://cpats01.115.com/seg.ts"},
|
||||
}
|
||||
for _, query := range []string{"?media_id=media-a", ""} {
|
||||
fetched = false
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/cloud115/hls/session-b/segment"+query, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
err := proxy.ServeChild(context.Background(), rec, req, "session-b", "segment")
|
||||
if err == nil || fetched {
|
||||
t.Errorf("query %q must reject the wrong/missing media before fetch; err=%v fetched=%v", query, err, fetched)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,8 +20,9 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCloud115HLSSessionNotFound = errors.New("115 hls session not found")
|
||||
ErrCloud115HLSUpstreamExpired = errors.New("115 hls upstream expired")
|
||||
ErrCloud115HLSSessionNotFound = errors.New("115 hls session not found")
|
||||
ErrCloud115HLSUpstreamExpired = errors.New("115 hls upstream expired")
|
||||
ErrCloud115HLSSessionMediaMismatch = errors.New("115 hls session media mismatch")
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -131,6 +132,9 @@ func (p *Cloud115HLSProxy) ServeChild(ctx context.Context, w http.ResponseWriter
|
||||
if !ok {
|
||||
return ErrCloud115HLSSessionNotFound
|
||||
}
|
||||
if r == nil || strings.TrimSpace(r.URL.Query().Get("media_id")) != session.MediaID {
|
||||
return ErrCloud115HLSSessionMediaMismatch
|
||||
}
|
||||
resp, err := p.fetchUpstream(ctx, upstream, r.Header.Get("Range"))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -186,6 +190,15 @@ func (p *Cloud115HLSProxy) ServeChild(ctx context.Context, w http.ResponseWriter
|
||||
return err
|
||||
}
|
||||
|
||||
// SessionMediaID resolves the server-owned media identity before handler authorization.
|
||||
func (p *Cloud115HLSProxy) SessionMediaID(sessionID, key string) (string, error) {
|
||||
session, _, ok := p.lookup(sessionID, key)
|
||||
if !ok {
|
||||
return "", ErrCloud115HLSSessionNotFound
|
||||
}
|
||||
return session.MediaID, nil
|
||||
}
|
||||
|
||||
func (p *Cloud115HLSProxy) newSession(mediaID string, definition int) *cloud115HLSSession {
|
||||
now := time.Now()
|
||||
return &cloud115HLSSession{
|
||||
|
||||
@@ -104,11 +104,13 @@ func (s *Cloud115PlaybackService) HLSProxy() *Cloud115HLSProxy {
|
||||
return s.proxy
|
||||
}
|
||||
|
||||
// PlaybackInfo 返回媒体在 direct/115-cloud/local 三种模式下的能力描述。
|
||||
//
|
||||
// requestedDefinition 为空或 0 时默认选择 1080P;该档位不可用时返回
|
||||
// transcoding 状态(并只触发一次加速转码,避免轮询风暴)。
|
||||
// PlaybackInfo only reads direct/115-cloud/local playback capabilities.
|
||||
// Cloud work is queued exclusively by StartTranscode, never by polling.
|
||||
func (s *Cloud115PlaybackService) PlaybackInfo(ctx context.Context, mediaID string, requestedDefinition int) (*PlaybackInfo, error) {
|
||||
return s.playbackInfo(ctx, mediaID, requestedDefinition, false)
|
||||
}
|
||||
|
||||
func (s *Cloud115PlaybackService) playbackInfo(ctx context.Context, mediaID string, requestedDefinition int, startTranscode bool) (*PlaybackInfo, error) {
|
||||
if s == nil || s.repo == nil || s.repo.Media == nil {
|
||||
return nil, ErrMediaNotFound
|
||||
}
|
||||
@@ -179,13 +181,26 @@ func (s *Cloud115PlaybackService) PlaybackInfo(ctx context.Context, mediaID stri
|
||||
return info, nil
|
||||
}
|
||||
|
||||
info.Transcode = s.ensureCloudTranscode(ctx, account.ID, pickCode, selected, client)
|
||||
if startTranscode {
|
||||
info.Transcode = s.ensureCloudTranscode(ctx, account.ID, pickCode, selected, client)
|
||||
} else {
|
||||
info.Transcode = Cloud115TranscodeState{
|
||||
State: "idle", Definition: selected.ID, Message: "该清晰度尚未就绪,可请求 115 云端转码", RetryAfterSec: 5,
|
||||
}
|
||||
s.mu.Lock()
|
||||
if last, ok := s.pushState[account.ID+":"+pickCode]; ok {
|
||||
info.Transcode.State = "transcoding"
|
||||
info.Transcode.Message = last.message
|
||||
info.Transcode.StartedAt = last.at.Unix()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// StartTranscode 显式触发某个清晰度的云端转码,返回最新状态。
|
||||
func (s *Cloud115PlaybackService) StartTranscode(ctx context.Context, mediaID string, definition int) (*PlaybackInfo, error) {
|
||||
return s.PlaybackInfo(ctx, mediaID, definition)
|
||||
return s.playbackInfo(ctx, mediaID, definition, true)
|
||||
}
|
||||
|
||||
// ResolveCloud115URL 返回指定清晰度的 115 云端 m3u8 地址。
|
||||
@@ -258,6 +273,9 @@ func (s *Cloud115PlaybackService) ensureCloudTranscode(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Reserve the in-flight attempt before releasing the lock. Two simultaneous
|
||||
// explicit starts must not both submit a push while the first awaits 115.
|
||||
s.pushState[key] = cloud115PushAttempt{at: now, message: "正在提交 115 云端转码请求…"}
|
||||
s.mu.Unlock()
|
||||
|
||||
err := client.SubmitVideoPush(ctx, pickCode, "vip_push")
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
func TestCloud115PlaybackInfoTriggersTranscodeOnce(t *testing.T) {
|
||||
func TestCloud115PlaybackInfoIsReadOnlyAndStartTriggersTranscodeOnce(t *testing.T) {
|
||||
var playCalls int32
|
||||
var pushCalls int32
|
||||
var ready int32
|
||||
@@ -93,9 +93,16 @@ func TestCloud115PlaybackInfoTriggersTranscodeOnce(t *testing.T) {
|
||||
}
|
||||
svc := NewCloud115PlaybackService(nil, zap.NewNop(), repos, strm)
|
||||
|
||||
info, err := svc.PlaybackInfo(ctx, media.ID, 0)
|
||||
if _, err := svc.PlaybackInfo(ctx, media.ID, 0); err != nil {
|
||||
t.Fatalf("read playback info: %v", err)
|
||||
}
|
||||
if got := atomic.LoadInt32(&pushCalls); got != 0 {
|
||||
t.Fatalf("read-only playback info triggered %d cloud transcode pushes", got)
|
||||
}
|
||||
|
||||
info, err := svc.StartTranscode(ctx, media.ID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("first playback info: %v", err)
|
||||
t.Fatalf("start transcode: %v", err)
|
||||
}
|
||||
if info.Provider != model.StrmProvider115 || info.DefaultQuality != "4" {
|
||||
t.Fatalf("unexpected playback info: %#v", info)
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
func TestCloud115ConcurrentStartsDoNotDuplicateInFlightPush(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
var pushCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if pushCalls.Add(1) == 1 {
|
||||
close(started)
|
||||
}
|
||||
<-release
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"state": true, "code": 0, "data": []any{}})
|
||||
}))
|
||||
defer server.Close()
|
||||
defer releaseOnce.Do(func() { close(release) })
|
||||
oldBase := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = server.URL
|
||||
defer func() { cloud115.ProAPIBase = oldBase }()
|
||||
client := cloud115.NewOpenClient("review-app", "access", "refresh")
|
||||
svc := NewCloud115PlaybackService(nil, zap.NewNop(), nil, nil)
|
||||
quality := PlaybackQuality{ID: "4", Label: "1080P", Source: "cloud"}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
first := make(chan struct{})
|
||||
go func() { defer close(first); svc.ensureCloudTranscode(ctx, "account", "pick", quality, client) }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("first push did not reach the local server")
|
||||
}
|
||||
second := make(chan Cloud115TranscodeState, 1)
|
||||
go func() { second <- svc.ensureCloudTranscode(ctx, "account", "pick", quality, client) }()
|
||||
secondReceived := false
|
||||
select {
|
||||
case state := <-second:
|
||||
secondReceived = true
|
||||
if state.State != "transcoding" {
|
||||
t.Errorf("in-flight request should be reported as transcoding: %#v", state)
|
||||
}
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
t.Error("second start issued another request instead of reusing the in-flight push")
|
||||
}
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
select {
|
||||
case <-first:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("first push did not finish")
|
||||
}
|
||||
// The duplicate path is awaited before restoring the package-wide fake API.
|
||||
if !secondReceived {
|
||||
select {
|
||||
case <-second:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("second start did not finish")
|
||||
}
|
||||
}
|
||||
if got := pushCalls.Load(); got != 1 {
|
||||
t.Fatalf("pushes=%d, want 1", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
// 115 云下载(离线下载)服务:提交链接、轮询 115 任务、驱动后处理管线
|
||||
// (STRM 增量同步 → 媒体库扫描 → 自动刮削)。
|
||||
//
|
||||
// 职责边界:
|
||||
// - 115 协议细节在 internal/service/cloud115/offline.go;
|
||||
// - 本服务只做编排与状态机,不直接写 .strm 文件、不直接解析媒体;
|
||||
// - 115 客户端必须经 StrmService.Open115Client 取得,禁止自建(见该方法注释)。
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/helper"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
// 云下载设置键(存于 Setting 表,cloud_download.* 前缀)。
|
||||
const (
|
||||
CloudDownloadSettingPollInterval = "cloud_download.poll_interval_sec"
|
||||
CloudDownloadSettingSyncDebounce = "cloud_download.sync_debounce_sec"
|
||||
CloudDownloadSettingAutoScrape = "cloud_download.auto_scrape"
|
||||
CloudDownloadSettingTimeoutHours = "cloud_download.task_timeout_hours"
|
||||
CloudDownloadSettingTorrentMaxMB = "cloud_download.torrent_max_mb"
|
||||
CloudDownloadSettingMaxPages = "cloud_download.max_pages"
|
||||
)
|
||||
|
||||
// 默认值与取值范围。
|
||||
const (
|
||||
cloudDownloadDefaultPollInterval = 60
|
||||
cloudDownloadDefaultDebounce = 30
|
||||
cloudDownloadDefaultTimeoutHours = 72
|
||||
cloudDownloadDefaultTorrentMaxMB = 20
|
||||
cloudDownloadDefaultMaxPages = 10
|
||||
|
||||
cloudDownloadMinPollInterval = 30
|
||||
cloudDownloadMaxPollInterval = 600
|
||||
cloudDownloadMaxDebounce = 300
|
||||
|
||||
// cloudDownloadPipelineTick 是后处理推进的扫描周期(去抖窗口以秒计,
|
||||
// 15s 足够把「等窗口结束」的延迟控制在可接受范围)。
|
||||
cloudDownloadPipelineTick = 15 * time.Second
|
||||
|
||||
// cloudDownloadTorrentTempPrefix 是种子临时文件前缀(上传后立即删除)。
|
||||
cloudDownloadTorrentTempPrefix = "mebox-clouddl-"
|
||||
)
|
||||
|
||||
// CloudDownloadService 提供云下载任务的全生命周期管理。
|
||||
type CloudDownloadService struct {
|
||||
log *zap.Logger
|
||||
repo *repository.Container
|
||||
strm *StrmService
|
||||
scanner *ScannerService
|
||||
scraper *ScraperService
|
||||
|
||||
stopOnce sync.Once
|
||||
stopCh chan struct{}
|
||||
baseCtx context.Context
|
||||
|
||||
// 去抖状态:同一同步目录在窗口内的多次完成合并成一次同步。
|
||||
debounceMu sync.Mutex
|
||||
lastTrigger map[string]time.Time
|
||||
}
|
||||
|
||||
// NewCloudDownloadService 构造云下载服务。
|
||||
// strm/scanner/scraper 必须已构造完成(见 service_builder 的装配顺序)。
|
||||
func NewCloudDownloadService(
|
||||
log *zap.Logger,
|
||||
repo *repository.Container,
|
||||
strm *StrmService,
|
||||
scanner *ScannerService,
|
||||
scraper *ScraperService,
|
||||
) *CloudDownloadService {
|
||||
return &CloudDownloadService{
|
||||
log: log,
|
||||
repo: repo,
|
||||
strm: strm,
|
||||
scanner: scanner,
|
||||
scraper: scraper,
|
||||
stopCh: make(chan struct{}),
|
||||
baseCtx: context.Background(),
|
||||
lastTrigger: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
// Start 启动后处理循环,并把上次进程退出时停在中间阶段的任务重置为待推进。
|
||||
func (s *CloudDownloadService) Start(ctx context.Context) {
|
||||
s.baseCtx = ctx
|
||||
if s.repo != nil && s.repo.CloudDownload != nil {
|
||||
if n, err := s.repo.CloudDownload.ResetStaleStages(ctx); err != nil {
|
||||
s.log.Warn("reset stale cloud download stages failed", zap.Error(err))
|
||||
} else if n > 0 {
|
||||
s.log.Info("cloud download stages reset after restart", zap.Int64("count", n))
|
||||
}
|
||||
}
|
||||
helper.Go(s.log, "cloudDownload.pipelineLoop", func() {
|
||||
ticker := time.NewTicker(cloudDownloadPipelineTick)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-s.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.runPipelineOnce(s.baseCtx)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Stop 停止后处理循环。
|
||||
func (s *CloudDownloadService) Stop() {
|
||||
s.stopOnce.Do(func() { close(s.stopCh) })
|
||||
}
|
||||
|
||||
// ─── 设置读取 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *CloudDownloadService) settingInt(key string, fallback, min, max int) int {
|
||||
if s.repo == nil || s.repo.Setting == nil {
|
||||
return fallback
|
||||
}
|
||||
raw, err := s.repo.Setting.Get(context.Background(), key)
|
||||
if err != nil || strings.TrimSpace(raw) == "" {
|
||||
return fallback
|
||||
}
|
||||
v, err := strconv.Atoi(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
if v < min {
|
||||
return min
|
||||
}
|
||||
if v > max {
|
||||
return max
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (s *CloudDownloadService) settingBool(key string, fallback bool) bool {
|
||||
if s.repo == nil || s.repo.Setting == nil {
|
||||
return fallback
|
||||
}
|
||||
raw, err := s.repo.Setting.Get(context.Background(), key)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(raw)) {
|
||||
case "":
|
||||
return fallback
|
||||
case "1", "true", "yes", "on", "enabled":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// pollInterval 轮询 115 任务列表的间隔。
|
||||
func (s *CloudDownloadService) pollInterval() time.Duration {
|
||||
sec := s.settingInt(CloudDownloadSettingPollInterval, cloudDownloadDefaultPollInterval,
|
||||
cloudDownloadMinPollInterval, cloudDownloadMaxPollInterval)
|
||||
return time.Duration(sec) * time.Second
|
||||
}
|
||||
|
||||
// debounceWindow 同一同步目录的完成去抖窗口。
|
||||
func (s *CloudDownloadService) debounceWindow() time.Duration {
|
||||
sec := s.settingInt(CloudDownloadSettingSyncDebounce, cloudDownloadDefaultDebounce, 0, cloudDownloadMaxDebounce)
|
||||
return time.Duration(sec) * time.Second
|
||||
}
|
||||
|
||||
// autoScrapeEnabled 是否在同步完成后自动刮削。
|
||||
func (s *CloudDownloadService) autoScrape() bool {
|
||||
return s.settingBool(CloudDownloadSettingAutoScrape, true)
|
||||
}
|
||||
|
||||
// taskTimeout 下载超时判定。
|
||||
func (s *CloudDownloadService) taskTimeout() time.Duration {
|
||||
hours := s.settingInt(CloudDownloadSettingTimeoutHours, cloudDownloadDefaultTimeoutHours, 1, 24*30)
|
||||
return time.Duration(hours) * time.Hour
|
||||
}
|
||||
|
||||
// torrentMaxBytes 种子上传大小上限。
|
||||
func (s *CloudDownloadService) torrentMaxBytes() int64 {
|
||||
mb := s.settingInt(CloudDownloadSettingTorrentMaxMB, cloudDownloadDefaultTorrentMaxMB, 1, 200)
|
||||
return int64(mb) << 20
|
||||
}
|
||||
|
||||
// maxPollPages 单轮轮询最多翻页数。
|
||||
func (s *CloudDownloadService) maxPollPages() int {
|
||||
return s.settingInt(CloudDownloadSettingMaxPages, cloudDownloadDefaultMaxPages, 1, 100)
|
||||
}
|
||||
|
||||
// ─── 查询与管理 ───────────────────────────────────────────────────────────────
|
||||
|
||||
// CloudDownloadTaskView 是列表返回的任务视图(附带展示用的派生字段)。
|
||||
type CloudDownloadTaskView struct {
|
||||
model.CloudDownloadTask
|
||||
SyncPathName string `json:"sync_path_name,omitempty"`
|
||||
RemoteDisplayPath string `json:"remote_display_path,omitempty"`
|
||||
LocalPath string `json:"local_path,omitempty"`
|
||||
}
|
||||
|
||||
// CloudDownloadList 是分页列表结果。
|
||||
type CloudDownloadList struct {
|
||||
Items []CloudDownloadTaskView `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
}
|
||||
|
||||
// List 分页查询云下载任务,并补充同步目录的展示信息。
|
||||
func (s *CloudDownloadService) List(ctx context.Context, q repository.CloudDownloadQuery) (*CloudDownloadList, error) {
|
||||
rows, total, err := s.repo.CloudDownload.List(ctx, q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := map[string]*model.StrmSyncPath{}
|
||||
items := make([]CloudDownloadTaskView, 0, len(rows))
|
||||
for i := range rows {
|
||||
v := CloudDownloadTaskView{CloudDownloadTask: rows[i]}
|
||||
if p, ok := paths[rows[i].SyncPathID]; ok {
|
||||
if p != nil {
|
||||
v.SyncPathName, v.RemoteDisplayPath, v.LocalPath = p.Name, p.RemoteDisplayPath, p.LocalPath
|
||||
}
|
||||
} else if s.repo.StrmSyncPath != nil {
|
||||
p, err := s.repo.StrmSyncPath.FindByID(ctx, rows[i].SyncPathID)
|
||||
if err == nil {
|
||||
paths[rows[i].SyncPathID] = p
|
||||
if p != nil {
|
||||
v.SyncPathName, v.RemoteDisplayPath, v.LocalPath = p.Name, p.RemoteDisplayPath, p.LocalPath
|
||||
}
|
||||
}
|
||||
}
|
||||
items = append(items, v)
|
||||
}
|
||||
page := q.Page
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
pageSize := q.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
return &CloudDownloadList{Items: items, Total: total, Page: page, PageSize: pageSize}, nil
|
||||
}
|
||||
|
||||
// Get 按 ID 取单条任务。
|
||||
func (s *CloudDownloadService) Get(ctx context.Context, id string) (*model.CloudDownloadTask, error) {
|
||||
t, err := s.repo.CloudDownload.FindByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if t == nil {
|
||||
return nil, errors.New("云下载任务不存在")
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Cancel 取消任务:先尽力删除 115 侧任务,再把本地状态置为已取消。
|
||||
// 115 已成功下载的任务无法取消(文件已入库),只能删除记录。
|
||||
func (s *CloudDownloadService) Cancel(ctx context.Context, id string) error {
|
||||
t, err := s.Get(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isCloudDownloadTerminal(t.Status) {
|
||||
return errors.New("任务已结束,无法取消")
|
||||
}
|
||||
if strings.TrimSpace(t.InfoHash) != "" {
|
||||
client, cerr := s.strm.Open115Client(ctx, t.AccountID)
|
||||
if cerr != nil {
|
||||
s.log.Warn("cloud download cancel: open 115 client failed",
|
||||
zap.String("task_id", t.ID), zap.Error(cerr))
|
||||
} else if derr := client.DeleteOfflineTask(ctx, t.InfoHash, false); derr != nil {
|
||||
// 远端删除失败不阻断本地取消:否则用户永远清不掉一条僵尸记录。
|
||||
s.log.Warn("cloud download cancel: delete remote task failed",
|
||||
zap.String("task_id", t.ID), zap.String("info_hash", t.InfoHash), zap.Error(derr))
|
||||
}
|
||||
}
|
||||
return s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"status": model.CloudDownloadCanceled,
|
||||
"stage": model.CloudDownloadStageNA,
|
||||
"stage_error": "任务已取消",
|
||||
})
|
||||
}
|
||||
|
||||
// Retry 重新提交失败任务:链接类重新走提交接口,BT 类用已保存的种子参数重提。
|
||||
func (s *CloudDownloadService) Retry(ctx context.Context, id string) error {
|
||||
t, err := s.Get(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t.Status != model.CloudDownloadFailed {
|
||||
return errors.New("只有失败的任务可以重试")
|
||||
}
|
||||
if t.SourceType == model.CloudDownloadSourceTorrent {
|
||||
if strings.TrimSpace(t.InfoHash) == "" || strings.TrimSpace(t.TorrentSha1) == "" || strings.TrimSpace(t.PickCode) == "" {
|
||||
return errors.New("该种子任务的提交参数不完整,请重新上传 .torrent 文件")
|
||||
}
|
||||
client, err := s.strm.Open115Client(ctx, t.AccountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.AddTaskBT(ctx, cloud115.OfflineBTRequest{
|
||||
InfoHash: t.InfoHash,
|
||||
Wanted: "0",
|
||||
TorrentSha1: t.TorrentSha1,
|
||||
PickCode: t.PickCode,
|
||||
WpPathID: t.WpPathID,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"status": model.CloudDownloadSubmitted,
|
||||
"error": "",
|
||||
"retry_count": t.RetryCount + 1,
|
||||
})
|
||||
}
|
||||
if strings.TrimSpace(t.SourceURL) == "" {
|
||||
return errors.New("该任务缺少原始链接,无法重试")
|
||||
}
|
||||
// 复用提交链路:新建一条记录会让历史重复,这里直接把原记录复位再提交。
|
||||
if err := s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"status": model.CloudDownloadQueued,
|
||||
"error": "",
|
||||
"percent": 0,
|
||||
"retry_count": t.RetryCount + 1,
|
||||
"stage": model.CloudDownloadStagePending,
|
||||
"stage_error": "",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
res := s.submitLinks(ctx, t.AccountID, t.SyncPathID, t.WpPathID, []cloudDownloadParsedLink{{
|
||||
SourceType: t.SourceType,
|
||||
URL: t.SourceURL,
|
||||
Key: t.SourceKey,
|
||||
}}, t.ID)
|
||||
if len(res.Items) == 1 && !res.Items[0].OK {
|
||||
return errors.New(res.Items[0].Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RetryPipeline 重做失败的后处理阶段(不重新下载)。
|
||||
func (s *CloudDownloadService) RetryPipeline(ctx context.Context, id string) error {
|
||||
t, err := s.Get(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t.Status != model.CloudDownloadSucceeded {
|
||||
return errors.New("只有下载成功的任务可以重做后处理")
|
||||
}
|
||||
if t.Stage != model.CloudDownloadStageFailed && t.Stage != model.CloudDownloadStageNA {
|
||||
return errors.New("该任务的后处理未处于失败状态")
|
||||
}
|
||||
if err := s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"stage": model.CloudDownloadStagePending,
|
||||
"stage_error": "",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
helper.Go(s.log, "cloudDownload.retryPipeline", func() {
|
||||
s.runPipelineOnce(s.baseCtx)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete 删除任务记录。活跃任务必须先取消,避免留下无人追问的 115 任务。
|
||||
func (s *CloudDownloadService) Delete(ctx context.Context, id string) error {
|
||||
t, err := s.Get(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isCloudDownloadTerminal(t.Status) {
|
||||
return errors.New("任务进行中,请先取消再删除")
|
||||
}
|
||||
return s.repo.CloudDownload.Delete(ctx, id)
|
||||
}
|
||||
|
||||
// ClearFinished 清空失败/已取消的任务。
|
||||
func (s *CloudDownloadService) ClearFinished(ctx context.Context) (int64, error) {
|
||||
return s.repo.CloudDownload.ClearFinished(ctx)
|
||||
}
|
||||
|
||||
// Quota 查询当前账号的云下载配额。
|
||||
func (s *CloudDownloadService) Quota(ctx context.Context, accountID string) (*cloud115.OfflineQuota, error) {
|
||||
client, err := s.strm.Open115Client(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client.GetOfflineQuota(ctx)
|
||||
}
|
||||
|
||||
// ─── 工具 ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
// isCloudDownloadTerminal 判断任务是否已到终态。
|
||||
func isCloudDownloadTerminal(status string) bool {
|
||||
switch status {
|
||||
case model.CloudDownloadSucceeded, model.CloudDownloadFailed, model.CloudDownloadCanceled:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// offlineErrorMessage 把 115 错误码翻译成可操作的中文提示。
|
||||
func offlineErrorMessage(code int, message string) string {
|
||||
switch code {
|
||||
case cloud115.OfflineCodeQuotaUsedUp:
|
||||
return "115 云下载配额已用尽,请在 115 端获取更多配额后重试"
|
||||
case cloud115.OfflineCodeStorageFull:
|
||||
return "115 存储空间不足"
|
||||
case cloud115.OfflineCodeTorrentMissing:
|
||||
return "115 找不到该种子文件(可能已被清理),请重新上传 .torrent"
|
||||
case cloud115.OfflineCodeTooManyURLs:
|
||||
return "单批链接超过 115 条上限"
|
||||
case cloud115.OfflineCodeParamError:
|
||||
return "115 拒绝了提交参数:" + message
|
||||
case cloud115.OfflineCodeOperationFailed:
|
||||
return "115 操作失败,请稍后重试"
|
||||
}
|
||||
if strings.TrimSpace(message) != "" {
|
||||
return message
|
||||
}
|
||||
return fmt.Sprintf("115 返回错误码 %d", code)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
// 云下载输入解析:把用户粘贴的多行文本拆成可提交的链接,并生成轮询匹配键。
|
||||
//
|
||||
// 支持的来源(与 115 /open/offline/add_task_urls 能力一致):
|
||||
// - 磁力:magnet:?xt=urn:btih:<hex40 或 base32>
|
||||
// - 电驴:ed2k://|file|...
|
||||
// - HTTP(S) / FTP 直链
|
||||
//
|
||||
// 只做「字符串层面」的识别,绝不主动请求这些 URL(无 SSRF 面)。
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// cloudDownloadParsedLink 是一条解析后的待提交链接。
|
||||
type cloudDownloadParsedLink struct {
|
||||
SourceType string // model.CloudDownloadSource*
|
||||
URL string // 原始(trim 后)链接
|
||||
Key string // 去重与轮询匹配键:BT 类为大写 info_hash,其余为链接本身
|
||||
InfoHash string // 仅 BT 类有值
|
||||
}
|
||||
|
||||
// cloudDownloadSkipped 是一条被跳过的原始输入。
|
||||
type cloudDownloadSkipped struct {
|
||||
Raw string `json:"raw"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// parseCloudDownloadLinks 解析多行输入。
|
||||
// 返回去重后的有效链接与被跳过的行(用于前端逐条反馈,而不是笼统报错)。
|
||||
func parseCloudDownloadLinks(raw []string) ([]cloudDownloadParsedLink, []cloudDownloadSkipped) {
|
||||
out := make([]cloudDownloadParsedLink, 0, len(raw))
|
||||
skipped := make([]cloudDownloadSkipped, 0)
|
||||
seen := make(map[string]bool, len(raw))
|
||||
|
||||
for _, line := range raw {
|
||||
// 允许多行文本整体粘贴:按换行再拆一次。
|
||||
for _, part := range strings.Split(line, "\n") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
link, reason := parseCloudDownloadLink(part)
|
||||
if reason != "" {
|
||||
skipped = append(skipped, cloudDownloadSkipped{Raw: part, Reason: reason})
|
||||
continue
|
||||
}
|
||||
if seen[link.Key] {
|
||||
skipped = append(skipped, cloudDownloadSkipped{Raw: part, Reason: "重复链接(本批已存在)"})
|
||||
continue
|
||||
}
|
||||
seen[link.Key] = true
|
||||
out = append(out, link)
|
||||
}
|
||||
}
|
||||
return out, skipped
|
||||
}
|
||||
|
||||
// parseCloudDownloadLink 识别单条链接;reason 非空表示不支持。
|
||||
func parseCloudDownloadLink(raw string) (cloudDownloadParsedLink, string) {
|
||||
lower := strings.ToLower(raw)
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "magnet:"):
|
||||
hash := magnetInfoHash(raw)
|
||||
if hash == "" {
|
||||
return cloudDownloadParsedLink{}, "磁力链接缺少有效的 btih 哈希"
|
||||
}
|
||||
return cloudDownloadParsedLink{
|
||||
SourceType: "magnet",
|
||||
URL: raw,
|
||||
Key: hash,
|
||||
InfoHash: hash,
|
||||
}, ""
|
||||
case strings.HasPrefix(lower, "ed2k://"):
|
||||
return cloudDownloadParsedLink{
|
||||
SourceType: "ed2k",
|
||||
URL: raw,
|
||||
Key: raw,
|
||||
}, ""
|
||||
case strings.HasPrefix(lower, "http://"), strings.HasPrefix(lower, "https://"):
|
||||
return cloudDownloadParsedLink{
|
||||
SourceType: "http",
|
||||
URL: raw,
|
||||
Key: raw,
|
||||
}, ""
|
||||
case strings.HasPrefix(lower, "ftp://"):
|
||||
return cloudDownloadParsedLink{
|
||||
SourceType: "ftp",
|
||||
URL: raw,
|
||||
Key: raw,
|
||||
}, ""
|
||||
default:
|
||||
return cloudDownloadParsedLink{}, "不支持的链接类型(仅支持磁力 / ed2k / http(s) / ftp)"
|
||||
}
|
||||
}
|
||||
|
||||
// magnetInfoHash 从磁力链接中提取大写化的 btih 哈希。
|
||||
// 兼容 hex(40 字符)与 base32(32 字符)两种写法;无法识别时返回空串。
|
||||
func magnetInfoHash(link string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(link))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, xt := range u.Query()["xt"] {
|
||||
const prefix = "urn:btih:"
|
||||
if len(xt) <= len(prefix) || !strings.EqualFold(xt[:len(prefix)], prefix) {
|
||||
continue
|
||||
}
|
||||
hash := strings.TrimSpace(xt[len(prefix):])
|
||||
// 去掉可能存在的 &dn= 之类残留(少数客户端拼接不规范)。
|
||||
if idx := strings.IndexAny(hash, "&?#"); idx >= 0 {
|
||||
hash = hash[:idx]
|
||||
}
|
||||
if len(hash) != 40 && len(hash) != 32 {
|
||||
continue
|
||||
}
|
||||
return strings.ToUpper(hash)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
// 云下载后处理管线:下载成功 → STRM 增量同步 → 媒体库扫描 → 自动刮削。
|
||||
//
|
||||
// 与 STRM 同步的耦合方式:
|
||||
// - 这里只负责「发起同步」,不知道同步内部怎么走;
|
||||
// - StrmService.finishSync 通过 SetSyncFinishHook 回调 onStrmSyncFinished,
|
||||
// 由后者把 syncing 状态推进到 scanning/done。
|
||||
//
|
||||
// 去抖:bench 上常见一场「一次提交 20 条链接」,如果每条完成都触发一次全树
|
||||
// 同步(walk115Flat 是整树自适应拉取),115 限流与本地 IO 都会被打满。
|
||||
// 因此同一同步目录在 debounce 窗口内只发起一次同步,其余任务停在 waiting_sync。
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
)
|
||||
|
||||
// runPipelineOnce 推进所有待处理任务。由 Start 里的 ticker 周期调用,也可手动触发。
|
||||
func (s *CloudDownloadService) runPipelineOnce(ctx context.Context) {
|
||||
if s.repo == nil || s.repo.CloudDownload == nil {
|
||||
return
|
||||
}
|
||||
candidates, err := s.repo.CloudDownload.ListPipelineCandidates(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("cloud download pipeline: list candidates failed", zap.Error(err))
|
||||
return
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return
|
||||
}
|
||||
groups := map[string][]model.CloudDownloadTask{}
|
||||
for _, t := range candidates {
|
||||
groups[t.SyncPathID] = append(groups[t.SyncPathID], t)
|
||||
}
|
||||
for syncPathID, tasks := range groups {
|
||||
s.advanceSyncPath(ctx, syncPathID, tasks)
|
||||
}
|
||||
}
|
||||
|
||||
// advanceSyncPath 推进一个同步目录下的待处理任务。
|
||||
func (s *CloudDownloadService) advanceSyncPath(ctx context.Context, syncPathID string, tasks []model.CloudDownloadTask) {
|
||||
path, err := s.repo.StrmSyncPath.FindByID(ctx, syncPathID)
|
||||
if err != nil {
|
||||
s.log.Warn("cloud download pipeline: load sync path failed",
|
||||
zap.String("path_id", syncPathID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if path == nil || !path.Enabled || path.Provider != model.StrmProvider115 {
|
||||
s.failStage(ctx, tasks, "同步目录不存在、已禁用或不是 115 网盘目录", model.CloudDownloadStageNA)
|
||||
return
|
||||
}
|
||||
|
||||
// 同步进行中:本轮只标记等待,避免把 StartSync 的「正在同步中」错误打成失败。
|
||||
if s.strm != nil && s.strm.IsSyncRunning(syncPathID) {
|
||||
s.markWaiting(ctx, tasks)
|
||||
return
|
||||
}
|
||||
|
||||
// 去抖窗口内不重复发起同步。
|
||||
if !s.debounceElapsed(syncPathID) {
|
||||
s.markWaiting(ctx, tasks)
|
||||
return
|
||||
}
|
||||
|
||||
if s.strm == nil {
|
||||
s.failStage(ctx, tasks, "STRM 同步服务不可用", model.CloudDownloadStageFailed)
|
||||
return
|
||||
}
|
||||
if err := s.strm.StartSync(ctx, syncPathID); err != nil {
|
||||
// 最常见的竞态:刚判断完不忙,另一个人(或 cron)抢先启动。
|
||||
if strings.Contains(err.Error(), "正在同步中") {
|
||||
s.markWaiting(ctx, tasks)
|
||||
return
|
||||
}
|
||||
s.failStage(ctx, tasks, "触发 STRM 同步失败:"+err.Error(), model.CloudDownloadStageFailed)
|
||||
return
|
||||
}
|
||||
s.noteTrigger(syncPathID)
|
||||
for _, t := range tasks {
|
||||
_ = s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"stage": model.CloudDownloadStageSyncing,
|
||||
"stage_error": "",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// onStrmSyncFinished 是 StrmService 的同步完成回调。
|
||||
// 只处理「因云下载而发起且当前处于 syncing」的任务:手动/Cron 同步也会触发
|
||||
// 这个回调,没有云下载任务时直接返回。
|
||||
func (s *CloudDownloadService) onStrmSyncFinished(ctx context.Context, syncPathID, status string) {
|
||||
if s.repo == nil || s.repo.CloudDownload == nil {
|
||||
return
|
||||
}
|
||||
tasks, err := s.repo.CloudDownload.ListBySyncPathAndStage(ctx, syncPathID, model.CloudDownloadStageSyncing)
|
||||
if err != nil {
|
||||
s.log.Warn("cloud download pipeline: list syncing tasks failed",
|
||||
zap.String("path_id", syncPathID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return
|
||||
}
|
||||
if status != model.StrmSyncRecordDone {
|
||||
for _, t := range tasks {
|
||||
_ = s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"stage": model.CloudDownloadStageFailed,
|
||||
"stage_error": "STRM 同步" + strmStatusLabel(status),
|
||||
})
|
||||
}
|
||||
// 同步失败也要记一次触发时间,给用户一个冷静期,避免立即重试形成惊吓式循环。
|
||||
s.noteTrigger(syncPathID)
|
||||
return
|
||||
}
|
||||
|
||||
path, err := s.repo.StrmSyncPath.FindByID(ctx, syncPathID)
|
||||
if err != nil || path == nil {
|
||||
s.failStage(ctx, tasks, "同步目录已被删除", model.CloudDownloadStageNA)
|
||||
return
|
||||
}
|
||||
root := s.matchLibraryRoot(ctx, path.LocalPath)
|
||||
if root == nil {
|
||||
s.failStage(ctx, tasks, "STRM 输出目录未加入任何媒体库(本地目录:"+path.LocalPath+")", model.CloudDownloadStageNA)
|
||||
return
|
||||
}
|
||||
|
||||
for _, t := range tasks {
|
||||
_ = s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"stage": model.CloudDownloadStageScanning,
|
||||
"library_id": root.LibraryID,
|
||||
})
|
||||
}
|
||||
|
||||
scanAdded, scanErr := s.scanLibrary(ctx, root.LibraryID)
|
||||
if scanErr != nil {
|
||||
s.failStage(ctx, tasks, "扫描媒体库失败:"+scanErr.Error(), model.CloudDownloadStageFailed)
|
||||
return
|
||||
}
|
||||
|
||||
scrapeQueued := 0
|
||||
if s.autoScrape() && s.scraper != nil && s.scraper.AnyEnabled() {
|
||||
// EnqueueLibrary 默认只挑 scrape_status 为空的 pending 行,
|
||||
// 因此全库入队不会重刷已刮好的条目。
|
||||
if n, err := s.scraper.EnqueueLibrary(ctx, root.LibraryID, ScrapeOptions{}); err != nil {
|
||||
s.log.Warn("cloud download pipeline: enqueue scrape failed",
|
||||
zap.String("library_id", root.LibraryID), zap.Error(err))
|
||||
} else {
|
||||
scrapeQueued = n
|
||||
}
|
||||
}
|
||||
|
||||
stageError := ""
|
||||
if s.autoScrape() && (s.scraper == nil || !s.scraper.AnyEnabled()) {
|
||||
stageError = "未配置任何刮削源,已跳过自动刮削"
|
||||
}
|
||||
|
||||
for _, t := range tasks {
|
||||
_ = s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"stage": model.CloudDownloadStageDone,
|
||||
"stage_error": stageError,
|
||||
"library_id": root.LibraryID,
|
||||
"scan_added": scanAdded,
|
||||
"scrape_queued": scrapeQueued,
|
||||
})
|
||||
}
|
||||
s.log.Info("cloud download pipeline finished",
|
||||
zap.String("path_id", syncPathID),
|
||||
zap.String("library_id", root.LibraryID),
|
||||
zap.Int("tasks", len(tasks)),
|
||||
zap.Int("scan_added", scanAdded),
|
||||
zap.Int("scrape_queued", scrapeQueued))
|
||||
}
|
||||
|
||||
// markWaiting 把任务置为等待同步。
|
||||
func (s *CloudDownloadService) markWaiting(ctx context.Context, tasks []model.CloudDownloadTask) {
|
||||
for _, t := range tasks {
|
||||
if t.Stage == model.CloudDownloadStageWaitingSync {
|
||||
continue
|
||||
}
|
||||
_ = s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"stage": model.CloudDownloadStageWaitingSync,
|
||||
"stage_error": "",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// failStage 把任务置为阶段失败/跳过。
|
||||
func (s *CloudDownloadService) failStage(ctx context.Context, tasks []model.CloudDownloadTask, msg, stage string) {
|
||||
for _, t := range tasks {
|
||||
_ = s.repo.CloudDownload.UpdateFields(ctx, t.ID, map[string]any{
|
||||
"stage": stage,
|
||||
"stage_error": msg,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// matchLibraryRoot 按 STRM 本地输出目录反查媒体库根路径。
|
||||
// 命中条件:路径相同,或 localPath 位于 root.Path 之下(同一块盘的子目录也算)。
|
||||
func (s *CloudDownloadService) matchLibraryRoot(ctx context.Context, localPath string) *model.LibraryRoot {
|
||||
localPath = strings.TrimSpace(localPath)
|
||||
if localPath == "" || s.repo.Library == nil {
|
||||
return nil
|
||||
}
|
||||
roots, err := s.repo.Library.ListEnabledRoots(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("cloud download pipeline: list library roots failed", zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
local := filepath.Clean(localPath)
|
||||
for i := range roots {
|
||||
root := roots[i]
|
||||
candidate := filepath.Clean(strings.TrimSpace(root.Path))
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
if sameLibraryPath(candidate, local) {
|
||||
return &root
|
||||
}
|
||||
// 子目录判断也走同一套路径归一化(Windows 盘符映射/容器前缀)。
|
||||
withSep := candidate
|
||||
if !strings.HasSuffix(withSep, string(filepath.Separator)) {
|
||||
withSep += string(filepath.Separator)
|
||||
}
|
||||
if strings.HasPrefix(local, withSep) || strings.HasPrefix(cleanPathForVolumeMapping(local), cleanPathForVolumeMapping(withSep)) {
|
||||
return &root
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanLibrary 扫描媒体库并返回新增条目数。
|
||||
func (s *CloudDownloadService) scanLibrary(ctx context.Context, libraryID string) (int, error) {
|
||||
if s.scanner == nil {
|
||||
return 0, nil
|
||||
}
|
||||
res, err := s.scanner.ScanLibrary(ctx, libraryID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if res == nil {
|
||||
return 0, nil
|
||||
}
|
||||
return res.Added, nil
|
||||
}
|
||||
|
||||
// debounceElapsed 判断距上次触发是否已过去一个去抖窗口。
|
||||
func (s *CloudDownloadService) debounceElapsed(syncPathID string) bool {
|
||||
window := s.debounceWindow()
|
||||
if window <= 0 {
|
||||
return true
|
||||
}
|
||||
s.debounceMu.Lock()
|
||||
defer s.debounceMu.Unlock()
|
||||
last, ok := s.lastTrigger[syncPathID]
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return time.Since(last) >= window
|
||||
}
|
||||
|
||||
// noteTrigger 记录本次触发时间。
|
||||
func (s *CloudDownloadService) noteTrigger(syncPathID string) {
|
||||
s.debounceMu.Lock()
|
||||
s.lastTrigger[syncPathID] = time.Now()
|
||||
s.debounceMu.Unlock()
|
||||
}
|
||||
|
||||
// strmStatusLabel 把同步记录状态翻译成中文。
|
||||
func strmStatusLabel(status string) string {
|
||||
switch status {
|
||||
case model.StrmSyncRecordFailed:
|
||||
return "失败"
|
||||
case model.StrmSyncRecordCanceled:
|
||||
return "被取消"
|
||||
case model.StrmSyncRecordDone:
|
||||
return "完成"
|
||||
default:
|
||||
return status
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
// 云下载轮询:把 115 任务列表的状态映射回本地任务表。
|
||||
//
|
||||
// 设计约束:
|
||||
// - 没有活跃任务时不发起任何 115 请求(避免无意义配额与限流风险);
|
||||
// - 按账号分组,一个账号一轮只翻页一次,供该账号下所有任务复用;
|
||||
// - 只做状态映射,不触发后处理(后处理由 pipeline 负责,两者解耦)。
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
// PollOnce 执行一轮 115 任务状态同步。导出以便测试与手动触发。
|
||||
func (s *CloudDownloadService) PollOnce(ctx context.Context) error {
|
||||
active, err := s.repo.CloudDownload.ListByStatuses(ctx, []string{
|
||||
model.CloudDownloadSubmitted,
|
||||
model.CloudDownloadDownloading,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(active) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 按账号分组:同一账号只拉一次任务列表。
|
||||
byAccount := map[string][]model.CloudDownloadTask{}
|
||||
for _, t := range active {
|
||||
byAccount[t.AccountID] = append(byAccount[t.AccountID], t)
|
||||
}
|
||||
|
||||
for accountID, tasks := range byAccount {
|
||||
s.pollAccount(ctx, accountID, tasks)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pollAccount 拉取单个账号的任务列表并回写本地任务状态。
|
||||
func (s *CloudDownloadService) pollAccount(ctx context.Context, accountID string, tasks []model.CloudDownloadTask) {
|
||||
client, err := s.strm.Open115Client(ctx, accountID)
|
||||
if err != nil {
|
||||
// 账号被禁用/删除时不改任务状态:等用户处理账号,任务保留可重试。
|
||||
s.log.Warn("cloud download poll: open 115 client failed",
|
||||
zap.String("account_id", accountID), zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
byHash := map[string]cloud115.OfflineTask{}
|
||||
byURL := map[string]cloud115.OfflineTask{}
|
||||
pages := s.maxPollPages()
|
||||
for page := 1; page <= pages; page++ {
|
||||
list, err := client.GetOfflineTaskList(ctx, page)
|
||||
if err != nil {
|
||||
s.log.Warn("cloud download poll: list tasks failed",
|
||||
zap.String("account_id", accountID), zap.Int("page", page), zap.Error(err))
|
||||
break
|
||||
}
|
||||
if list == nil {
|
||||
break
|
||||
}
|
||||
for _, t := range list.Tasks {
|
||||
if h := strings.ToUpper(strings.TrimSpace(t.InfoHash)); h != "" {
|
||||
byHash[h] = t
|
||||
}
|
||||
if u := strings.TrimSpace(t.URL); u != "" {
|
||||
byURL[u] = t
|
||||
}
|
||||
}
|
||||
if list.PageCount <= page || len(list.Tasks) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
timeout := s.taskTimeout()
|
||||
for _, t := range tasks {
|
||||
offline := matchOfflineTask(t, byHash, byURL)
|
||||
fields := map[string]any{"last_poll_at": now}
|
||||
|
||||
if offline == nil {
|
||||
// 115 侧暂未出现(受理有延迟)或已不在列表(被清理)。
|
||||
// 只在超时后才判失败,避免把「刚提交」误判成失败。
|
||||
if !t.CreatedAt.IsZero() && now.Sub(t.CreatedAt) > timeout {
|
||||
fields["status"] = model.CloudDownloadFailed
|
||||
fields["error"] = "下载超时:115 侧长时间未完成(任务可能已被清理)"
|
||||
}
|
||||
s.applyPollFields(ctx, t.ID, fields)
|
||||
continue
|
||||
}
|
||||
|
||||
switch offline.Status {
|
||||
case cloud115.OfflineTaskStatusDone:
|
||||
fields["status"] = model.CloudDownloadSucceeded
|
||||
fields["percent"] = 100
|
||||
if offline.Size > 0 {
|
||||
fields["size_bytes"] = offline.Size
|
||||
}
|
||||
if t.CompletedAt == nil {
|
||||
fields["completed_at"] = now
|
||||
}
|
||||
fields["error"] = ""
|
||||
case cloud115.OfflineTaskStatusFailed:
|
||||
fields["status"] = model.CloudDownloadFailed
|
||||
fields["error"] = "115 云下载失败(可能资源失效或无人做种)"
|
||||
default: // 0 分配中 / 1 下载中
|
||||
fields["status"] = model.CloudDownloadDownloading
|
||||
fields["percent"] = offline.PercentDone
|
||||
if offline.Size > 0 {
|
||||
fields["size_bytes"] = offline.Size
|
||||
}
|
||||
}
|
||||
s.applyPollFields(ctx, t.ID, fields)
|
||||
}
|
||||
}
|
||||
|
||||
// applyPollFields 只写轮询关心的列,绝不能整行 Update:
|
||||
// 后处理线程可能在并发写 stage/scan_added 等字段。
|
||||
func (s *CloudDownloadService) applyPollFields(ctx context.Context, taskID string, fields map[string]any) {
|
||||
if err := s.repo.CloudDownload.UpdateFields(ctx, taskID, fields); err != nil {
|
||||
s.log.Warn("cloud download poll: update task failed",
|
||||
zap.String("task_id", taskID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// matchOfflineTask 匹配 115 任务:info_hash 优先,其次原始链接。
|
||||
func matchOfflineTask(t model.CloudDownloadTask, byHash, byURL map[string]cloud115.OfflineTask) *cloud115.OfflineTask {
|
||||
if h := strings.ToUpper(strings.TrimSpace(t.InfoHash)); h != "" {
|
||||
if item, ok := byHash[h]; ok {
|
||||
return &item
|
||||
}
|
||||
}
|
||||
for _, key := range []string{t.SourceKey, t.SourceURL} {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if item, ok := byURL[key]; ok {
|
||||
return &item
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
// 云下载提交:校验目标、配额预检、批量提交链接、.torrent 上传解析后提交。
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
)
|
||||
|
||||
// cloudDownloadSeedDirName 是 115 上用于暂存 .torrent 的目录(两级)。
|
||||
// 官方文档建议把种子放进「云下载/种子文件」,但不是硬性要求;不存在时自动创建。
|
||||
var cloudDownloadSeedDirPath = []string{"云下载", "种子文件"}
|
||||
|
||||
// CloudDownloadSubmit 是链接提交请求。
|
||||
// AccountID 允许为空:始终以同步目录绑定的账号为准,避免前端传错账号导致
|
||||
// 「任务挂在 A 账号、文件落到 B 目录」这种无法自动恢复的错配。
|
||||
type CloudDownloadSubmit struct {
|
||||
AccountID string `json:"account_id"`
|
||||
SyncPathID string `json:"sync_path_id"`
|
||||
URLs []string `json:"urls"`
|
||||
}
|
||||
|
||||
// CloudDownloadTorrentRequest 是 .torrent 提交请求(文件内容由 handler 读入)。
|
||||
type CloudDownloadTorrentRequest struct {
|
||||
AccountID string
|
||||
SyncPathID string
|
||||
FileName string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// CloudDownloadTaskResult 是单条来源的提交结果。
|
||||
type CloudDownloadTaskResult struct {
|
||||
TaskID string `json:"task_id,omitempty"`
|
||||
Source string `json:"source"`
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// CloudDownloadSubmitResult 是批量提交结果:部分成功也返回 200 + 明细。
|
||||
type CloudDownloadSubmitResult struct {
|
||||
Accepted int `json:"accepted"`
|
||||
Rejected int `json:"rejected"`
|
||||
Items []CloudDownloadTaskResult `json:"items"`
|
||||
Skipped []cloudDownloadSkipped `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// Submit 提交一批链接任务。
|
||||
func (s *CloudDownloadService) Submit(ctx context.Context, req CloudDownloadSubmit) (*CloudDownloadSubmitResult, error) {
|
||||
p, err := s.resolveTarget(ctx, req.SyncPathID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
links, skipped := parseCloudDownloadLinks(req.URLs)
|
||||
res := &CloudDownloadSubmitResult{Skipped: skipped}
|
||||
if len(links) == 0 {
|
||||
for _, sk := range skipped {
|
||||
res.Items = append(res.Items, CloudDownloadTaskResult{Source: sk.Raw, Error: sk.Reason})
|
||||
res.Rejected++
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
if err := s.ensureQuota(ctx, p.AccountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
submitRes := s.submitLinks(ctx, p.AccountID, p.ID, p.RemotePath, links, "")
|
||||
submitRes.Skipped = skipped
|
||||
return submitRes, nil
|
||||
}
|
||||
|
||||
// SubmitTorrent 上传 .torrent 到 115、解析后提交为 BT 任务。
|
||||
//
|
||||
// 本期不做「选择部分文件」:115 开放平台的磁力链路没有文件选择能力,
|
||||
// .torrent 链路虽然支持 wanted,但产品上暂不提供勾选 UI(见 docs/cloud-download-plan.md)。
|
||||
// 因此 wanted 传 "0"(官方文档唯一明确允许的取值),语义待真机验证。
|
||||
func (s *CloudDownloadService) SubmitTorrent(ctx context.Context, req CloudDownloadTorrentRequest) (*CloudDownloadSubmitResult, error) {
|
||||
p, err := s.resolveTarget(ctx, req.SyncPathID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(req.Data) == 0 {
|
||||
return nil, errors.New("种子文件为空")
|
||||
}
|
||||
if int64(len(req.Data)) > s.torrentMaxBytes() {
|
||||
return nil, fmt.Errorf("种子文件超过大小上限(%d MB)", s.torrentMaxBytes()>>20)
|
||||
}
|
||||
if err := s.ensureQuota(ctx, p.AccountID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.strm.Open115Client(ctx, p.AccountID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 1. 写入临时文件:115 上传接口只接受本地路径。
|
||||
tmp, err := os.CreateTemp("", cloudDownloadTorrentTempPrefix+"*.torrent")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建种子临时文件失败:%w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer func() { _ = os.Remove(tmpPath) }()
|
||||
if _, err := tmp.Write(req.Data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return nil, fmt.Errorf("写入种子临时文件失败:%w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return nil, fmt.Errorf("关闭种子临时文件失败:%w", err)
|
||||
}
|
||||
|
||||
// 2. 上传到「云下载/种子文件」;建目录失败时回落到同步目录,避免整条链路失败。
|
||||
parentCID, derr := s.ensureSeedDir(ctx, client)
|
||||
if derr != nil {
|
||||
s.log.Warn("cloud download: ensure seed dir failed, fallback to sync path",
|
||||
zap.String("path_id", p.ID), zap.Error(derr))
|
||||
parentCID = p.RemotePath
|
||||
}
|
||||
uploaded, err := client.Upload(ctx, tmpPath, parentCID, "", "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("上传种子到 115 失败:%w", err)
|
||||
}
|
||||
if strings.TrimSpace(uploaded.Sha1) == "" || strings.TrimSpace(uploaded.PickCode) == "" {
|
||||
return nil, errors.New("115 上传种子未返回 sha1/pick_code")
|
||||
}
|
||||
|
||||
// 3. 解析种子(拿 info_hash 与任务名)。
|
||||
info, err := client.ParseTorrent(ctx, uploaded.Sha1, uploaded.PickCode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("解析 BT 种子失败:%w", err)
|
||||
}
|
||||
infoHash := strings.ToUpper(strings.TrimSpace(info.InfoHash))
|
||||
if infoHash == "" {
|
||||
return nil, errors.New("115 解析种子未返回 info_hash")
|
||||
}
|
||||
|
||||
// 4. 提交 BT 任务。
|
||||
if err := client.AddTaskBT(ctx, cloud115.OfflineBTRequest{
|
||||
InfoHash: infoHash,
|
||||
Wanted: "0",
|
||||
TorrentSha1: uploaded.Sha1,
|
||||
PickCode: uploaded.PickCode,
|
||||
WpPathID: p.RemotePath,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("提交 BT 任务失败:%w", err)
|
||||
}
|
||||
|
||||
// 5. 落库。
|
||||
task := &model.CloudDownloadTask{
|
||||
AccountID: p.AccountID,
|
||||
SyncPathID: p.ID,
|
||||
Provider: model.StrmProvider115,
|
||||
SourceType: model.CloudDownloadSourceTorrent,
|
||||
SourceURL: req.FileName,
|
||||
SourceKey: infoHash,
|
||||
InfoHash: infoHash,
|
||||
TorrentName: info.Name,
|
||||
TorrentSha1: uploaded.Sha1,
|
||||
PickCode: uploaded.PickCode,
|
||||
WpPathID: p.RemotePath,
|
||||
Status: model.CloudDownloadSubmitted,
|
||||
SizeBytes: info.FileSize,
|
||||
Stage: model.CloudDownloadStagePending,
|
||||
}
|
||||
if err := s.repo.CloudDownload.Create(ctx, task); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CloudDownloadSubmitResult{
|
||||
Accepted: 1,
|
||||
Items: []CloudDownloadTaskResult{{
|
||||
TaskID: task.ID,
|
||||
Source: req.FileName,
|
||||
OK: true,
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// resolveTarget 校验同步目录并返回它。云下载只支持 115 同步目录,
|
||||
// 且远端必须是有效目录 ID(0 表示网盘根目录,落到根目录的离线文件永远扫不到)。
|
||||
func (s *CloudDownloadService) resolveTarget(ctx context.Context, syncPathID string) (*model.StrmSyncPath, error) {
|
||||
syncPathID = strings.TrimSpace(syncPathID)
|
||||
if syncPathID == "" {
|
||||
return nil, errors.New("请选择目标 STRM 同步目录")
|
||||
}
|
||||
p, err := s.repo.StrmSyncPath.FindByID(ctx, syncPathID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p == nil {
|
||||
return nil, errors.New("STRM 同步目录不存在")
|
||||
}
|
||||
if !p.Enabled {
|
||||
return nil, errors.New("STRM 同步目录已禁用")
|
||||
}
|
||||
if p.Provider != model.StrmProvider115 {
|
||||
return nil, errors.New("只有 115 网盘同步目录支持云下载")
|
||||
}
|
||||
if strings.TrimSpace(p.AccountID) == "" {
|
||||
return nil, errors.New("该同步目录未关联 115 账号")
|
||||
}
|
||||
wp := strings.TrimSpace(p.RemotePath)
|
||||
if wp == "" || wp == "0" {
|
||||
return nil, errors.New("同步目录的远端目录无效(不能是网盘根目录)")
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// ensureQuota 提交前的配额闸门。配额接口失败不阻断提交:
|
||||
// 部分应用可能未开放该接口权限,真正的失败由 add_task_* 的错误码兜底。
|
||||
func (s *CloudDownloadService) ensureQuota(ctx context.Context, accountID string) error {
|
||||
client, err := s.strm.Open115Client(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
quota, err := client.GetOfflineQuota(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("cloud download: quota probe failed, continue submit",
|
||||
zap.String("account_id", accountID), zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
if quota != nil && quota.Surplus <= 0 {
|
||||
return errors.New("115 云下载配额已用尽,请在 115 端获取更多配额后重试")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// submitLinks 分批提交链接,并逐条落库。
|
||||
// existingTaskID 非空时表示「重试单条历史任务」,此时更新原记录而不是新建。
|
||||
func (s *CloudDownloadService) submitLinks(
|
||||
ctx context.Context,
|
||||
accountID, syncPathID, wpPathID string,
|
||||
links []cloudDownloadParsedLink,
|
||||
existingTaskID string,
|
||||
) *CloudDownloadSubmitResult {
|
||||
res := &CloudDownloadSubmitResult{}
|
||||
active := map[string]bool{}
|
||||
|
||||
if existingTaskID == "" {
|
||||
keys := make([]string, 0, len(links))
|
||||
for _, l := range links {
|
||||
keys = append(keys, l.Key)
|
||||
}
|
||||
if dedup, err := s.repo.CloudDownload.FindActiveBySourceKeys(ctx, keys); err == nil {
|
||||
active = dedup
|
||||
}
|
||||
filtered := make([]cloudDownloadParsedLink, 0, len(links))
|
||||
for _, l := range links {
|
||||
if active[l.Key] {
|
||||
res.Items = append(res.Items, CloudDownloadTaskResult{
|
||||
Source: l.URL,
|
||||
Error: "该链接已有进行中的任务",
|
||||
})
|
||||
res.Rejected++
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, l)
|
||||
}
|
||||
links = filtered
|
||||
}
|
||||
|
||||
if len(links) == 0 {
|
||||
return res
|
||||
}
|
||||
|
||||
client, err := s.strm.Open115Client(ctx, accountID)
|
||||
if err != nil {
|
||||
for _, l := range links {
|
||||
res.Items = append(res.Items, CloudDownloadTaskResult{Source: l.URL, Error: err.Error()})
|
||||
res.Rejected++
|
||||
s.persistSubmitFailure(ctx, accountID, syncPathID, wpPathID, l, err.Error(), existingTaskID)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
for start := 0; start < len(links); start += cloud115.OfflineMaxURLsPerBatch {
|
||||
end := start + cloud115.OfflineMaxURLsPerBatch
|
||||
if end > len(links) {
|
||||
end = len(links)
|
||||
}
|
||||
batch := links[start:end]
|
||||
urls := make([]string, 0, len(batch))
|
||||
for _, l := range batch {
|
||||
urls = append(urls, l.URL)
|
||||
}
|
||||
results, reqErr := client.AddTaskURLs(ctx, urls, wpPathID)
|
||||
if reqErr != nil {
|
||||
msg := reqErr.Error()
|
||||
for _, l := range batch {
|
||||
res.Items = append(res.Items, CloudDownloadTaskResult{Source: l.URL, Error: msg})
|
||||
res.Rejected++
|
||||
s.persistSubmitFailure(ctx, accountID, syncPathID, wpPathID, l, msg, existingTaskID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// 115 不保证结果顺序与入参一致:先按 url 建索引,缺失项再按顺序兜底。
|
||||
byURL := make(map[string]cloud115.OfflineTaskURLResult, len(results))
|
||||
for _, r := range results {
|
||||
if u := strings.TrimSpace(r.URL); u != "" {
|
||||
byURL[u] = r
|
||||
}
|
||||
}
|
||||
for i, l := range batch {
|
||||
r, ok := byURL[l.URL]
|
||||
if !ok && i < len(results) {
|
||||
r = results[i]
|
||||
}
|
||||
if !r.State {
|
||||
msg := offlineErrorMessage(r.Code, r.Message)
|
||||
res.Items = append(res.Items, CloudDownloadTaskResult{Source: l.URL, Error: msg})
|
||||
res.Rejected++
|
||||
s.persistSubmitFailure(ctx, accountID, syncPathID, wpPathID, l, msg, existingTaskID)
|
||||
continue
|
||||
}
|
||||
infoHash := strings.ToUpper(strings.TrimSpace(r.InfoHash))
|
||||
if infoHash == "" {
|
||||
infoHash = l.InfoHash
|
||||
}
|
||||
key := l.Key
|
||||
if infoHash != "" {
|
||||
key = infoHash
|
||||
}
|
||||
taskID, perr := s.persistSubmitSuccess(ctx, accountID, syncPathID, wpPathID, l, key, infoHash, existingTaskID)
|
||||
if perr != nil {
|
||||
res.Items = append(res.Items, CloudDownloadTaskResult{Source: l.URL, Error: "115 已受理,但本地记录失败:" + perr.Error()})
|
||||
res.Rejected++
|
||||
continue
|
||||
}
|
||||
res.Items = append(res.Items, CloudDownloadTaskResult{TaskID: taskID, Source: l.URL, OK: true})
|
||||
res.Accepted++
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// persistSubmitSuccess 写入一条已受理的任务,返回任务 ID。
|
||||
func (s *CloudDownloadService) persistSubmitSuccess(
|
||||
ctx context.Context, accountID, syncPathID, wpPathID string,
|
||||
link cloudDownloadParsedLink, key, infoHash, existingTaskID string,
|
||||
) (string, error) {
|
||||
if existingTaskID != "" {
|
||||
return existingTaskID, s.repo.CloudDownload.UpdateFields(ctx, existingTaskID, map[string]any{
|
||||
"status": model.CloudDownloadSubmitted,
|
||||
"error": "",
|
||||
"percent": 0,
|
||||
"info_hash": infoHash,
|
||||
"source_key": key,
|
||||
"last_poll_at": nil,
|
||||
"stage": model.CloudDownloadStagePending,
|
||||
"stage_error": "",
|
||||
})
|
||||
}
|
||||
task := &model.CloudDownloadTask{
|
||||
AccountID: accountID,
|
||||
SyncPathID: syncPathID,
|
||||
Provider: model.StrmProvider115,
|
||||
SourceType: link.SourceType,
|
||||
SourceURL: link.URL,
|
||||
SourceKey: key,
|
||||
InfoHash: infoHash,
|
||||
WpPathID: wpPathID,
|
||||
Status: model.CloudDownloadSubmitted,
|
||||
Stage: model.CloudDownloadStagePending,
|
||||
}
|
||||
if err := s.repo.CloudDownload.Create(ctx, task); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return task.ID, nil
|
||||
}
|
||||
|
||||
// persistSubmitFailure 记录一条提交失败的任务,供用户在列表里看到原因并重试。
|
||||
func (s *CloudDownloadService) persistSubmitFailure(
|
||||
ctx context.Context, accountID, syncPathID, wpPathID string,
|
||||
link cloudDownloadParsedLink, msg, existingTaskID string,
|
||||
) {
|
||||
if existingTaskID != "" {
|
||||
_ = s.repo.CloudDownload.UpdateFields(ctx, existingTaskID, map[string]any{
|
||||
"status": model.CloudDownloadFailed,
|
||||
"error": msg,
|
||||
})
|
||||
return
|
||||
}
|
||||
task := &model.CloudDownloadTask{
|
||||
AccountID: accountID,
|
||||
SyncPathID: syncPathID,
|
||||
Provider: model.StrmProvider115,
|
||||
SourceType: link.SourceType,
|
||||
SourceURL: link.URL,
|
||||
SourceKey: link.Key,
|
||||
InfoHash: link.InfoHash,
|
||||
WpPathID: wpPathID,
|
||||
Status: model.CloudDownloadFailed,
|
||||
Error: msg,
|
||||
Stage: model.CloudDownloadStagePending,
|
||||
}
|
||||
if err := s.repo.CloudDownload.Create(ctx, task); err != nil {
|
||||
s.log.Warn("cloud download: persist failed task failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// ensureSeedDir 返回 115 上「云下载/种子文件」目录的 cid,不存在则创建。
|
||||
func (s *CloudDownloadService) ensureSeedDir(ctx context.Context, client *cloud115.OpenClient) (string, error) {
|
||||
parent := "0"
|
||||
for _, name := range cloudDownloadSeedDirPath {
|
||||
cid, err := s.findOrCreateDir(ctx, client, parent, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parent = cid
|
||||
}
|
||||
return parent, nil
|
||||
}
|
||||
|
||||
// findOrCreateDir 在 parentCID 下按名称查找子目录,未命中则创建。
|
||||
// 先查后建的原因:115 的 folder/add 对已存在同名目录的行为文档未定义,
|
||||
// 直接创建可能得到重复目录或错误。
|
||||
func (s *CloudDownloadService) findOrCreateDir(ctx context.Context, client *cloud115.OpenClient, parentCID, name string) (string, error) {
|
||||
const pageSize = 1150
|
||||
offset := 0
|
||||
for {
|
||||
files, _, err := client.GetFsList(ctx, parentCID, offset, pageSize)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, f := range files {
|
||||
if f.Category == cloud115.TypeDir && strings.EqualFold(strings.TrimSpace(f.FileName), name) {
|
||||
return f.FileId, nil
|
||||
}
|
||||
}
|
||||
if len(files) < pageSize {
|
||||
break
|
||||
}
|
||||
offset += len(files)
|
||||
}
|
||||
return client.MkDir(ctx, parentCID, name)
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
func TestParseCloudDownloadLinks(t *testing.T) {
|
||||
links, skipped := parseCloudDownloadLinks([]string{
|
||||
" magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567&dn=x ",
|
||||
"ed2k://|file|a.mkv|100|0123456789ABCDEF0123456789ABCDEF|/",
|
||||
"https://example.com/a.mkv",
|
||||
"ftp://example.com/b.mkv",
|
||||
"magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567", // 与第一条重复
|
||||
"not-a-link",
|
||||
"",
|
||||
"magnet:?xt=urn:btih:tooshort", // 非法哈希
|
||||
})
|
||||
if len(links) != 4 {
|
||||
t.Fatalf("links = %d, want 4: %+v", len(links), links)
|
||||
}
|
||||
if links[0].SourceType != model.CloudDownloadSourceMagnet {
|
||||
t.Errorf("first source type = %q", links[0].SourceType)
|
||||
}
|
||||
if links[0].Key != "0123456789ABCDEF0123456789ABCDEF01234567" {
|
||||
t.Errorf("magnet key = %q, want uppercase info hash", links[0].Key)
|
||||
}
|
||||
if links[1].SourceType != model.CloudDownloadSourceED2K {
|
||||
t.Errorf("second source type = %q", links[1].SourceType)
|
||||
}
|
||||
if links[2].SourceType != model.CloudDownloadSourceHTTP {
|
||||
t.Errorf("third source type = %q", links[2].SourceType)
|
||||
}
|
||||
if links[3].SourceType != model.CloudDownloadSourceFTP {
|
||||
t.Errorf("fourth source type = %q", links[3].SourceType)
|
||||
}
|
||||
// 空行不算错误,只有重复链接/不支持的类型/非法哈希会被记为跳过。
|
||||
if len(skipped) != 3 {
|
||||
t.Fatalf("skipped = %d, want 3: %+v", len(skipped), skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCloudDownloadLinksAcceptsMultilinePaste(t *testing.T) {
|
||||
links, _ := parseCloudDownloadLinks([]string{
|
||||
"magnet:?xt=urn:btih:0123456789abcdef0123456789abcdef01234567\nhttps://example.com/a.mkv\n",
|
||||
})
|
||||
if len(links) != 2 {
|
||||
t.Fatalf("links = %d, want 2", len(links))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMagnetInfoHashSupportsBase32(t *testing.T) {
|
||||
got := magnetInfoHash("magnet:?xt=urn:btih:ABCDEFGHIJKLMNOPQRSTUVWXYZ234567")
|
||||
if got != "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" {
|
||||
t.Fatalf("base32 hash = %q", got)
|
||||
}
|
||||
if got := magnetInfoHash("magnet:?dn=x"); got != "" {
|
||||
t.Fatalf("missing btih should yield empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflineErrorMessage(t *testing.T) {
|
||||
cases := []struct {
|
||||
code int
|
||||
want string
|
||||
}{
|
||||
{1000012, "115 云下载配额已用尽,请在 115 端获取更多配额后重试"},
|
||||
{91006, "115 存储空间不足"},
|
||||
{20018, "115 找不到该种子文件(可能已被清理),请重新上传 .torrent"},
|
||||
{1000011, "单批链接超过 115 条上限"},
|
||||
{990002, "115 拒绝了提交参数:bad"},
|
||||
{980004, "115 操作失败,请稍后重试"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := offlineErrorMessage(c.code, "bad"); got != c.want {
|
||||
t.Errorf("code %d => %q, want %q", c.code, got, c.want)
|
||||
}
|
||||
}
|
||||
if got := offlineErrorMessage(1234, "原始信息"); got != "原始信息" {
|
||||
t.Errorf("unknown code should keep 115 message, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudDownloadDebounce(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
repos := repository.New(db)
|
||||
svc := NewCloudDownloadService(zap.NewNop(), repos, nil, nil, nil)
|
||||
|
||||
if !svc.debounceElapsed("p1") {
|
||||
t.Fatal("first trigger must be allowed")
|
||||
}
|
||||
svc.noteTrigger("p1")
|
||||
if svc.debounceElapsed("p1") {
|
||||
t.Fatal("second trigger inside window must be suppressed")
|
||||
}
|
||||
// 另一个同步目录互不影响。
|
||||
if !svc.debounceElapsed("p2") {
|
||||
t.Fatal("debounce must be per sync path")
|
||||
}
|
||||
// 窗口置 0 表示不合并。
|
||||
if err := repos.Setting.Set(context.Background(), CloudDownloadSettingSyncDebounce, "0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !svc.debounceElapsed("p1") {
|
||||
t.Fatal("zero window should disable debounce")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchLibraryRoot(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
repos := repository.New(db)
|
||||
ctx := context.Background()
|
||||
|
||||
lib := &model.Library{Name: "电影", Path: "/media/movies", Type: "movie", Enabled: true}
|
||||
root := &model.LibraryRoot{LibraryID: "tmp", Path: "/media/movies", Enabled: true}
|
||||
if err := repos.Library.CreateWithRoots(ctx, lib, []model.LibraryRoot{*root}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc := NewCloudDownloadService(zap.NewNop(), repos, nil, nil, nil)
|
||||
if got := svc.matchLibraryRoot(ctx, "/media/movies"); got == nil || got.LibraryID != lib.ID {
|
||||
t.Fatalf("exact path should match library, got %+v", got)
|
||||
}
|
||||
if got := svc.matchLibraryRoot(ctx, "/media/movies/2024"); got == nil || got.LibraryID != lib.ID {
|
||||
t.Fatalf("sub directory should match library, got %+v", got)
|
||||
}
|
||||
if got := svc.matchLibraryRoot(ctx, "/media/tv"); got != nil {
|
||||
t.Fatalf("unrelated path must not match, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCloudDownloadTerminal(t *testing.T) {
|
||||
terminal := []string{model.CloudDownloadSucceeded, model.CloudDownloadFailed, model.CloudDownloadCanceled}
|
||||
for _, s := range terminal {
|
||||
if !isCloudDownloadTerminal(s) {
|
||||
t.Errorf("%s should be terminal", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{model.CloudDownloadQueued, model.CloudDownloadSubmitted, model.CloudDownloadDownloading} {
|
||||
if isCloudDownloadTerminal(s) {
|
||||
t.Errorf("%s should not be terminal", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTargetRejectsInvalidSyncPath(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
repos := repository.New(db)
|
||||
ctx := context.Background()
|
||||
svc := NewCloudDownloadService(zap.NewNop(), repos, nil, nil, nil)
|
||||
|
||||
if _, err := svc.resolveTarget(ctx, ""); err == nil {
|
||||
t.Fatal("empty sync path should fail")
|
||||
}
|
||||
if _, err := svc.resolveTarget(ctx, "missing"); err == nil {
|
||||
t.Fatal("unknown sync path should fail")
|
||||
}
|
||||
|
||||
// 远端目录为 0(网盘根目录):离线文件永远扫不到,必须拒绝。
|
||||
bad := &model.StrmSyncPath{
|
||||
Name: "根目录", AccountID: "acc", Provider: model.StrmProvider115,
|
||||
RemotePath: "0", LocalPath: "/media/root", Enabled: true,
|
||||
}
|
||||
if err := repos.StrmSyncPath.Create(ctx, bad); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := svc.resolveTarget(ctx, bad.ID); err == nil {
|
||||
t.Fatal("remote root path should be rejected")
|
||||
}
|
||||
|
||||
// 非 115 同步目录不支持云下载。
|
||||
cd2 := &model.StrmSyncPath{
|
||||
Name: "CD2", AccountID: "acc", Provider: model.StrmProviderCloudDrive,
|
||||
RemotePath: "/movies", LocalPath: "/media/cd2", Enabled: true,
|
||||
}
|
||||
if err := repos.StrmSyncPath.Create(ctx, cd2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := svc.resolveTarget(ctx, cd2.ID); err == nil {
|
||||
t.Fatal("non-115 provider should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudDownloadSettingsDefaults(t *testing.T) {
|
||||
db := newServiceTestDB(t)
|
||||
svc := NewCloudDownloadService(zap.NewNop(), repository.New(db), nil, nil, nil)
|
||||
|
||||
if got := svc.pollInterval(); got != 60*time.Second {
|
||||
t.Errorf("default poll interval = %v", got)
|
||||
}
|
||||
if got := svc.debounceWindow(); got != 30*time.Second {
|
||||
t.Errorf("default debounce = %v", got)
|
||||
}
|
||||
if !svc.autoScrape() {
|
||||
t.Error("auto scrape should default to true")
|
||||
}
|
||||
if got := svc.taskTimeout(); got != 72*time.Hour {
|
||||
t.Errorf("default timeout = %v", got)
|
||||
}
|
||||
if got := svc.torrentMaxBytes(); got != 20<<20 {
|
||||
t.Errorf("default torrent max = %d", got)
|
||||
}
|
||||
if got := svc.maxPollPages(); got != 10 {
|
||||
t.Errorf("default max pages = %d", got)
|
||||
}
|
||||
|
||||
// 越界值必须被夹紧,避免把轮询间隔配成 1 秒打爆 115 限流。
|
||||
if err := repository.New(db).Setting.Set(context.Background(), CloudDownloadSettingPollInterval, "1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := svc.pollInterval(); got != 30*time.Second {
|
||||
t.Errorf("clamped poll interval = %v, want 30s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
// resetEmbyInvalThrottle 清掉全局失效节流,让测试能断言「这一次调用到底有
|
||||
// 没有真的清缓存」。测试结束后恢复原值,避免影响同包其他用例。
|
||||
func resetEmbyInvalThrottle(t *testing.T) {
|
||||
t.Helper()
|
||||
embyInvalMu.Lock()
|
||||
previous := embyInvalLast
|
||||
embyInvalLast = time.Time{}
|
||||
embyInvalMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
embyInvalMu.Lock()
|
||||
embyInvalLast = previous
|
||||
embyInvalMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func newCachedTestEmbyService(t *testing.T) *EmbyService {
|
||||
t.Helper()
|
||||
svc := newTestEmbyService(t)
|
||||
cache := NewRuntimeCacheService(&config.Config{}, zap.NewNop())
|
||||
t.Cleanup(func() { _ = cache.Close() })
|
||||
return svc.SetRuntimeCache(cache)
|
||||
}
|
||||
|
||||
// TestRecordProgressKeepsEmbyPayloadCache 覆盖首页加载慢的核心回归点:
|
||||
// 播放期间客户端每 5-10 秒上报一次进度,如果每次都清空 media:emby:*,
|
||||
// 客户端刷新首页时 73 个媒体库的 Latest 会全部回源重建(生产环境实测
|
||||
// 首批 8-25s)。普通进度上报必须保留载荷缓存;只有「已看完」翻转才失效。
|
||||
func TestRecordProgressKeepsEmbyPayloadCache(t *testing.T) {
|
||||
svc := newCachedTestEmbyService(t)
|
||||
const mediaID = "ep-1"
|
||||
media := model.Media{
|
||||
Base: model.Base{ID: mediaID},
|
||||
Title: "第 1 集",
|
||||
Path: `/media/tv/Show/Season 01/S01E01.mkv`,
|
||||
DurationSec: 100,
|
||||
}
|
||||
if err := svc.repo.DB.Create(&media).Error; err != nil {
|
||||
t.Fatalf("create media: %v", err)
|
||||
}
|
||||
|
||||
key := svc.embyLatestCacheKey("user-1", "", 16)
|
||||
svc.cache.SetJSONWithStale(t.Context(), key, embyLatestCacheValue{
|
||||
Items: []map[string]any{{"Id": "cached-item"}},
|
||||
}, time.Minute, time.Hour)
|
||||
|
||||
resetEmbyInvalThrottle(t)
|
||||
// 60s / 100s:播放中途
|
||||
if err := svc.RecordProgressWithSession(t.Context(), "user-1", mediaID, 600_000_000, 1_000_000_000, "session-1"); err != nil {
|
||||
t.Fatalf("record progress: %v", err)
|
||||
}
|
||||
if _, stale, ok := svc.cachedLatestItemsWithStale(t.Context(), key); !ok || stale {
|
||||
t.Fatal("普通进度上报不应让 Emby 列表载荷缓存失效")
|
||||
}
|
||||
|
||||
resetEmbyInvalThrottle(t)
|
||||
// 98s / 100s:进入片尾 30s 内的「已看完」区间
|
||||
if err := svc.RecordProgressWithSession(t.Context(), "user-1", mediaID, 980_000_000, 1_000_000_000, "session-2"); err != nil {
|
||||
t.Fatalf("record completed progress: %v", err)
|
||||
}
|
||||
if _, _, ok := svc.cachedLatestItemsWithStale(t.Context(), key); ok {
|
||||
t.Fatal("看完状态翻转后应让 Emby 列表载荷缓存失效")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFindSeriesGroupCachesSiblingVirtualSeries 覆盖 N+1 回归点:一次全表
|
||||
// 扫描算出了整库的剧集分组,若只缓存命中的那一部,同一页的 16 张卡片和
|
||||
// 16 张海报会各自重扫一遍 media 表。批量记忆后同批分组都应可命中。
|
||||
func TestFindSeriesGroupCachesSiblingVirtualSeries(t *testing.T) {
|
||||
svc := newTestEmbyService(t)
|
||||
lib := model.Library{Name: "剧集", Path: `/media/tv`, Type: "tv", Enabled: true}
|
||||
if err := svc.repo.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatalf("create library: %v", err)
|
||||
}
|
||||
|
||||
shows := []string{"Alpha", "Bravo", "Charlie"}
|
||||
rows := make([]model.Media, 0, len(shows)*2)
|
||||
for _, show := range shows {
|
||||
for ep := 1; ep <= 2; ep++ {
|
||||
rows = append(rows, model.Media{
|
||||
Base: model.Base{ID: show + "-" + strconv.Itoa(ep)},
|
||||
LibraryID: lib.ID,
|
||||
Title: show,
|
||||
Path: fmt.Sprintf("/media/tv/%s/Season 01/S01E%02d.mkv", show, ep),
|
||||
SeasonNum: 1,
|
||||
EpisodeNum: ep,
|
||||
})
|
||||
}
|
||||
}
|
||||
for i := range rows {
|
||||
if err := svc.repo.DB.Create(&rows[i]).Error; err != nil {
|
||||
t.Fatalf("create media: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
seriesIDs := map[string]string{}
|
||||
for i := range rows {
|
||||
if rows[i].EpisodeNum == 1 {
|
||||
seriesIDs[rows[i].Title] = svc.seriesIDForMedia(t.Context(), &rows[i])
|
||||
}
|
||||
}
|
||||
target, ok := seriesIDs["Alpha"]
|
||||
if !ok {
|
||||
t.Fatal("missing fixture series")
|
||||
}
|
||||
if _, found, err := svc.findSeriesGroup(t.Context(), target, ""); err != nil || !found {
|
||||
t.Fatalf("findSeriesGroup(%s) = (%v, %v), want found", target, found, err)
|
||||
}
|
||||
|
||||
for show, id := range seriesIDs {
|
||||
if show == "Alpha" {
|
||||
continue
|
||||
}
|
||||
if _, cached := svc.cachedSeriesGroup(id); !cached {
|
||||
t.Fatalf("同一次扫描算出的 %s 应一并进入缓存,否则每张卡片/海报都会重扫整库", show)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLatestPayloadKeepsStaleCopyForIdleClients 保证「最新添加」载荷在新鲜期
|
||||
// 之后仍有可用的陈旧副本:客户端空闲很久后再打开首页时,先拿陈旧结果并由
|
||||
// 后台重建,而不是让全部媒体库同时回源。
|
||||
func TestLatestPayloadKeepsStaleCopyForIdleClients(t *testing.T) {
|
||||
svc := newCachedTestEmbyService(t)
|
||||
key := svc.embyLatestCacheKey("user-1", "", 16)
|
||||
svc.cache.SetJSONWithStale(t.Context(), key, embyLatestCacheValue{
|
||||
Items: []map[string]any{{"Id": "stale-item"}},
|
||||
}, time.Nanosecond, embyLatestStaleWindow)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
|
||||
items, stale, ok := svc.cachedLatestItemsWithStale(t.Context(), key)
|
||||
if !ok || !stale {
|
||||
t.Fatalf("expected a stale hit, got ok=%v stale=%v", ok, stale)
|
||||
}
|
||||
if len(items) != 1 || items[0]["Id"] != "stale-item" {
|
||||
t.Fatalf("unexpected stale payload: %#v", items)
|
||||
}
|
||||
}
|
||||
|
||||
// mediaScanCounter 统计真正落到 media 表上的查询次数,用来断言「整库数据
|
||||
// 只扫一次」:GORM 的 Trace 回调里能拿到最终 SQL。
|
||||
type mediaScanCounter struct {
|
||||
logger.Interface
|
||||
mu sync.Mutex
|
||||
scans int
|
||||
}
|
||||
|
||||
func (c *mediaScanCounter) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
|
||||
sql, _ := fc()
|
||||
if strings.Contains(sql, "FROM `media`") {
|
||||
c.mu.Lock()
|
||||
c.scans++
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *mediaScanCounter) Scans() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.scans
|
||||
}
|
||||
|
||||
// TestVirtualSeriesLookupsShareOneScan 覆盖冷启动风暴的回归点:客户端刷新
|
||||
// 首页时 73 个库的卡片和海报各自要一个虚拟 series 分组。修好之后,第一次
|
||||
// 扫描把整库分组都写进缓存,同一页剩下的查询都不该再落 SQL;随后一个不存在
|
||||
// 的虚拟 ID 也不该反复触发整库重扫。
|
||||
func TestVirtualSeriesLookupsShareOneScan(t *testing.T) {
|
||||
counter := &mediaScanCounter{Interface: logger.Discard}
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: counter})
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
}
|
||||
if err := db.AutoMigrate(model.AllModels()...); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
svc := NewEmbyService(&config.Config{}, zap.NewNop(), repository.New(db))
|
||||
|
||||
lib := model.Library{Name: "剧集", Path: `/media/tv`, Type: "tv", Enabled: true}
|
||||
if err := svc.repo.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatalf("create library: %v", err)
|
||||
}
|
||||
shows := []string{"Alpha", "Bravo", "Charlie"}
|
||||
rows := make([]model.Media, 0, len(shows)*2)
|
||||
for _, show := range shows {
|
||||
for ep := 1; ep <= 2; ep++ {
|
||||
rows = append(rows, model.Media{
|
||||
Base: model.Base{ID: show + "-" + strconv.Itoa(ep)},
|
||||
LibraryID: lib.ID,
|
||||
Title: show,
|
||||
Path: fmt.Sprintf("/media/tv/%s/Season 01/S01E%02d.mkv", show, ep),
|
||||
SeasonNum: 1,
|
||||
EpisodeNum: ep,
|
||||
})
|
||||
}
|
||||
}
|
||||
for i := range rows {
|
||||
if err := svc.repo.DB.Create(&rows[i]).Error; err != nil {
|
||||
t.Fatalf("create media: %v", err)
|
||||
}
|
||||
}
|
||||
seriesIDs := map[string]string{}
|
||||
for i := range rows {
|
||||
if rows[i].EpisodeNum == 1 {
|
||||
seriesIDs[rows[i].Title] = svc.seriesIDForMedia(t.Context(), &rows[i])
|
||||
}
|
||||
}
|
||||
|
||||
if _, found, err := svc.findSeriesGroup(t.Context(), seriesIDs["Alpha"], ""); err != nil || !found {
|
||||
t.Fatalf("findSeriesGroup(Alpha) = (%v, %v), want found", found, err)
|
||||
}
|
||||
if got := counter.Scans(); got != 1 {
|
||||
t.Fatalf("首次查询应恰好扫描 media 一次,实际 %d 次", got)
|
||||
}
|
||||
|
||||
for _, show := range []string{"Bravo", "Charlie"} {
|
||||
if _, found, err := svc.findSeriesGroup(t.Context(), seriesIDs[show], ""); err != nil || !found {
|
||||
t.Fatalf("findSeriesGroup(%s) = (%v, %v), want found", show, found, err)
|
||||
}
|
||||
}
|
||||
if got := counter.Scans(); got != 1 {
|
||||
t.Fatalf("同批分组应命中缓存,不应再扫 media,实际累计 %d 次", got)
|
||||
}
|
||||
|
||||
if _, found, err := svc.findSeriesGroup(t.Context(), embyVirtualSeriesPrefix+"missing", ""); err != nil || found {
|
||||
t.Fatalf("未知虚拟 ID 应为 not found,实际 found=%v err=%v", found, err)
|
||||
}
|
||||
if got := counter.Scans(); got != 1 {
|
||||
t.Fatalf("刚扫过整库后,未知 ID 不应重扫 media,实际累计 %d 次", got)
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,17 @@ type EmbyService struct {
|
||||
virtualSeries map[string]embySeriesCacheEntry
|
||||
virtualSeasons map[string]embySeasonCacheEntry
|
||||
virtualArtwork map[string]embyArtworkCacheEntry
|
||||
// virtualGroups 把「整库虚拟剧集分组」的扫描折叠成一次。
|
||||
//
|
||||
// 客户端刷新首页时会同时请求 73 个库的 Latest,每个库的卡片和海报都要
|
||||
// 各自的虚拟 series 分组。不同剧的分组缓存键不同,singleflight 键也不同,
|
||||
// 于是第一波请求会并发重扫整张 media 表(生产环境实测冷启动 14-17s)。
|
||||
// 按用户折叠成一次扫描后,同一波请求共享结果并整批进缓存。
|
||||
virtualGroups singleflight.Group
|
||||
// virtualScanAt 记录每个用户最近一次整库扫描的时间,用于避免为同一个
|
||||
// 不存在的 id 反复重扫(必须按用户区分:可见性不同,别人的快照不能当
|
||||
// 自己的答案)。由 virtualMu 保护。
|
||||
virtualScanAt map[string]time.Time
|
||||
|
||||
visibilityMu sync.RWMutex
|
||||
visibilityCache map[string]embyVisibilityCacheEntry
|
||||
@@ -186,6 +197,15 @@ const (
|
||||
embyVirtualSeriesCap = 8000
|
||||
embyVirtualSeasonCap = 16000
|
||||
embyVirtualArtworkCap = 24000
|
||||
// embySeriesGroupBulkCacheEpisodes 限制一次批量记忆的剧集行数。
|
||||
//
|
||||
// 批量记忆把「一次扫描服务整页卡片 + 整页海报」的收益拿满,但同时让整库
|
||||
// 剧集常驻内存。超过该量级时退回只缓存命中的那一部,避免大库吃光内存。
|
||||
embySeriesGroupBulkCacheEpisodes = 8000
|
||||
// embyVirtualScanGrace 是「刚做过整库虚拟剧集扫描」的保护窗口。窗口内再
|
||||
// 请求一个不在缓存里的虚拟 series ID,直接按不存在处理,避免为同一个
|
||||
// 未知 id 反复重扫整库。
|
||||
embyVirtualScanGrace = 10 * time.Second
|
||||
// Clients that already cached the 1x1 placeholder treat a stable tag as
|
||||
// immutable. Virtual ids are the ones that served that placeholder, so
|
||||
// only those tags get a suffix that forces a refetch.
|
||||
@@ -325,8 +345,24 @@ func (e *EmbyService) Items(ctx context.Context, p ItemsParams) (map[string]any,
|
||||
}
|
||||
}
|
||||
|
||||
// 客户端刷新首页时常用 Movie,Series,Video 这种混合类型一次取「最新」行。
|
||||
// 只有剧集库才该折叠成剧集卡片:seriesItemsForLibrary 只查
|
||||
// (season_num>0 OR episode_num>0),电影库走到这里会让整行返回 0 条。
|
||||
if containsItemType(p.IncludeItemTypes, "Series") && !containsItemType(p.IncludeItemTypes, "Episode") {
|
||||
return e.seriesItemsForLibrary(ctx, p.ParentID, p)
|
||||
episodic := true
|
||||
if p.ParentID != "" {
|
||||
if value, err := e.libraryIsEpisodic(ctx, p.ParentID); err == nil {
|
||||
episodic = value
|
||||
}
|
||||
}
|
||||
if episodic {
|
||||
return e.seriesItemsForLibrary(ctx, p.ParentID, p)
|
||||
}
|
||||
// 非剧集库:只请求 Series 时确实没有可返回的条目;混合类型则落回
|
||||
// mediaItems,由它按 Movie 过滤,而不是整行判空。
|
||||
if !containsItemType(p.IncludeItemTypes, "Movie") && !containsItemType(p.IncludeItemTypes, "Video") {
|
||||
return emptyItemsEnvelope(p.StartIndex), nil
|
||||
}
|
||||
}
|
||||
|
||||
// 电影库的「常规浏览」(未指定 IncludeItemTypes): 电影库里偶尔混入按
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type embyItemsCacheValue struct {
|
||||
@@ -77,7 +78,15 @@ func (e *EmbyService) embySimilarCacheKey(mediaID, userID string, limit int) str
|
||||
}
|
||||
|
||||
// defaultEmbyLatestCacheTTLSeconds 是 Emby「最新添加」缓存的兜底时长。
|
||||
const defaultEmbyLatestCacheTTLSeconds = 300
|
||||
const defaultEmbyLatestCacheTTLSeconds = 900
|
||||
|
||||
// embyLatestStaleWindow 是「最新添加」载荷在进程内保留的陈旧副本时长。
|
||||
//
|
||||
// 客户端空闲很久之后再打开首页时,新鲜期早已过去:生产环境观察到这时
|
||||
// 73 个媒体库的 Latest 同时回源,首批请求要 8-25s 才回来,客户端等不及
|
||||
// 还会重试,进一步放大压力。保留陈旧副本后,这类请求立刻拿到上次的结果,
|
||||
// 由后台异步重建(见 refreshLatestItemsAsync),用户不再等 SQL。
|
||||
const embyLatestStaleWindow = 24 * time.Hour
|
||||
|
||||
// embyLatestCacheTTLSeconds 返回「最新添加」列表的缓存时长。它刻意比通用
|
||||
// 媒体缓存更长:客户端刷新首页时会同时请求全部媒体库的 Latest(生产环境
|
||||
|
||||
@@ -153,7 +153,7 @@ func (e *EmbyService) loadLatestItemsCached(ctx context.Context, userID, parentI
|
||||
}
|
||||
if e.cache != nil {
|
||||
freshTTL := time.Duration(e.embyLatestCacheTTLSeconds()) * time.Second
|
||||
e.cache.SetJSONWithStale(loadCtx, cacheKey, value, freshTTL, freshTTL+30*time.Minute)
|
||||
e.cache.SetJSONWithStale(loadCtx, cacheKey, value, freshTTL, freshTTL+embyLatestStaleWindow)
|
||||
}
|
||||
e.rememberArtworkRefs(value.Artwork)
|
||||
return value, nil
|
||||
|
||||
+130
-40
@@ -41,47 +41,65 @@ func (e *EmbyService) findSeriesGroup(ctx context.Context, id, userID string) (e
|
||||
return embySeriesGroup{}, false, nil
|
||||
}
|
||||
if strings.HasPrefix(id, embyVirtualSeriesPrefix) {
|
||||
if group, ok := e.cachedSeriesGroup(id); ok {
|
||||
return group, true, nil
|
||||
}
|
||||
return e.findVirtualSeriesGroup(ctx, id, userID)
|
||||
}
|
||||
var rows []model.Media
|
||||
q := e.repo.DB.WithContext(ctx).Model(&model.Media{}).Where("season_num > 0 OR episode_num > 0")
|
||||
q = e.applyUserMediaVisibility(ctx, q, userID)
|
||||
if !strings.HasPrefix(id, embyVirtualSeriesPrefix) {
|
||||
q = q.Where("series_id = ?", id)
|
||||
} else {
|
||||
// 虚拟 series ID 只可能来自 series_id 为空的媒体:
|
||||
// 有 series_id 时分组 key 就是 series_id 本身(UUID,不带虚拟前缀)。
|
||||
q = q.Where("series_id IS NULL OR series_id = ''")
|
||||
}
|
||||
q = q.Where("series_id = ?", id)
|
||||
if err := q.Order("media.season_num asc, media.episode_num asc, media.episode_fraction asc, media.created_at asc").Limit(embySeriesGroupingLimit).Find(&rows).Error; err != nil {
|
||||
return embySeriesGroup{}, false, err
|
||||
}
|
||||
for _, group := range e.seriesGroupsFromMedia(ctx, rows) {
|
||||
groups := e.seriesGroupsFromMedia(ctx, rows)
|
||||
bulkCached := e.rememberScannedSeriesGroups(groups)
|
||||
for _, group := range groups {
|
||||
if group.ID == id {
|
||||
e.rememberSeriesGroup(group)
|
||||
if !bulkCached {
|
||||
e.rememberSeriesGroup(group)
|
||||
}
|
||||
return group, true, nil
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(id, embyVirtualSeriesPrefix) {
|
||||
if series, err := e.repo.Series.FindByID(ctx, id); err != nil {
|
||||
return embySeriesGroup{}, false, err
|
||||
} else if series != nil {
|
||||
return embySeriesGroup{
|
||||
ID: series.ID,
|
||||
LibraryID: series.LibraryID,
|
||||
Name: series.Title,
|
||||
PosterURL: series.PosterURL,
|
||||
BackdropURL: series.BackdropURL,
|
||||
Overview: series.Overview,
|
||||
Rating: series.Rating,
|
||||
Year: series.Year,
|
||||
TMDbID: series.TMDbID,
|
||||
BangumiID: series.BangumiID,
|
||||
CreatedAt: series.CreatedAt,
|
||||
DateLastMediaAdded: series.CreatedAt,
|
||||
}, true, nil
|
||||
if series, err := e.repo.Series.FindByID(ctx, id); err != nil {
|
||||
return embySeriesGroup{}, false, err
|
||||
} else if series != nil {
|
||||
return embySeriesGroup{
|
||||
ID: series.ID,
|
||||
LibraryID: series.LibraryID,
|
||||
Name: series.Title,
|
||||
PosterURL: series.PosterURL,
|
||||
BackdropURL: series.BackdropURL,
|
||||
Overview: series.Overview,
|
||||
Rating: series.Rating,
|
||||
Year: series.Year,
|
||||
TMDbID: series.TMDbID,
|
||||
BangumiID: series.BangumiID,
|
||||
CreatedAt: series.CreatedAt,
|
||||
DateLastMediaAdded: series.CreatedAt,
|
||||
}, true, nil
|
||||
}
|
||||
return embySeriesGroup{}, false, nil
|
||||
}
|
||||
|
||||
// findVirtualSeriesGroup 处理 series_id 为空的剧集分组(虚拟 series ID)。
|
||||
//
|
||||
// 虚拟 ID 是 hash(库名 + 剧名),无法反解,只能整库算出分组后按 ID 取用。
|
||||
// 这里走 loadVirtualSeriesGroups,把同一用户的并发请求折叠成一次扫描:
|
||||
// 首页一次刷新会同时要 73 个库的卡片和海报,各扫一遍在 2 核主机上要 14-17s。
|
||||
func (e *EmbyService) findVirtualSeriesGroup(ctx context.Context, id, userID string) (embySeriesGroup, bool, error) {
|
||||
if group, ok := e.cachedSeriesGroup(id); ok {
|
||||
return group, true, nil
|
||||
}
|
||||
if e.virtualScanRecently(userID) {
|
||||
return embySeriesGroup{}, false, nil
|
||||
}
|
||||
groups, err := e.loadVirtualSeriesGroups(ctx, userID)
|
||||
if err != nil {
|
||||
return embySeriesGroup{}, false, err
|
||||
}
|
||||
for _, group := range groups {
|
||||
if group.ID == id {
|
||||
return group, true, nil
|
||||
}
|
||||
}
|
||||
return embySeriesGroup{}, false, nil
|
||||
@@ -103,20 +121,14 @@ func (e *EmbyService) findSeasonGroup(ctx context.Context, id, userID string) (e
|
||||
return season, true, nil
|
||||
}
|
||||
// 回退:未刮削(series_id 为空,虚拟 key 由库名+名称派生)的媒体只能全量分组。
|
||||
var rows []model.Media
|
||||
q := e.repo.DB.WithContext(ctx).Model(&model.Media{}).
|
||||
Where("(series_id IS NULL OR series_id = '') AND (season_num > 0 OR episode_num > 0)")
|
||||
q = e.applyUserMediaVisibility(ctx, q, userID)
|
||||
if err := q.
|
||||
Order("media.season_num asc, media.episode_num asc, media.episode_fraction asc, media.created_at asc").
|
||||
Limit(embySeriesGroupingLimit).
|
||||
Find(&rows).Error; err != nil {
|
||||
// 复用共享扫描:首页并发刷新时整库只扫一次。
|
||||
groups, err := e.loadVirtualSeriesGroups(ctx, userID)
|
||||
if err != nil {
|
||||
return embySeasonGroup{}, false, err
|
||||
}
|
||||
for _, series := range e.seriesGroupsFromMedia(ctx, rows) {
|
||||
for _, series := range groups {
|
||||
for _, season := range e.seasonsForSeries(series) {
|
||||
if season.ID == id {
|
||||
e.rememberSeriesGroup(series)
|
||||
return season, true, nil
|
||||
}
|
||||
}
|
||||
@@ -186,6 +198,84 @@ func (e *EmbyService) seasonGroupForSeries(ctx context.Context, id, seriesID, us
|
||||
return embySeasonGroup{}, false, nil
|
||||
}
|
||||
|
||||
// rememberScannedSeriesGroups 批量记住一次扫描得到的全部分组,返回是否真的
|
||||
// 批量写入(超过剧集预算时不做,调用方需自行缓存命中的那一部)。
|
||||
//
|
||||
// 只缓存命中的一部时,同一部剧的多张卡片/海报仍要各扫一次;整批写入后,
|
||||
// 同一次扫描算出来的其他剧也直接命中,于是整页 16 张卡片 + 16 张海报共用
|
||||
// 一次全表扫描。
|
||||
func (e *EmbyService) rememberScannedSeriesGroups(groups []embySeriesGroup) bool {
|
||||
if e == nil || len(groups) == 0 {
|
||||
return false
|
||||
}
|
||||
episodes := 0
|
||||
for i := range groups {
|
||||
episodes += len(groups[i].Episodes)
|
||||
if episodes > embySeriesGroupBulkCacheEpisodes {
|
||||
return false
|
||||
}
|
||||
}
|
||||
e.rememberSeriesGroups(groups)
|
||||
return true
|
||||
}
|
||||
|
||||
// loadVirtualSeriesGroups 返回 series_id 为空的整库剧集分组。
|
||||
//
|
||||
// 同一用户的并发调用共享同一次扫描:客户端刷新首页时 73 个库的 Latest 同时
|
||||
// 进来,每个库的卡片和海报又各自要自己的虚拟 series 分组。分组缓存键各不
|
||||
// 相同,singleflight 也就不会合并,于是第一波请求把整张 media 表并发重算
|
||||
// 了 73 次(生产环境实测 14-17s)。这里按用户折叠成一次,结果整批进缓存,
|
||||
// 调用方拿到切片后按 ID 取用即可。
|
||||
func (e *EmbyService) loadVirtualSeriesGroups(ctx context.Context, userID string) ([]embySeriesGroup, error) {
|
||||
if e == nil || e.repo == nil {
|
||||
return nil, nil
|
||||
}
|
||||
v, err, _ := e.virtualGroups.Do("virtual:"+userID, func() (any, error) {
|
||||
// 与 loadLatestItemsCached 一致:一个客户端断开不应取消正在为其他
|
||||
// 客户端填充的共享扫描。
|
||||
loadCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
defer cancel()
|
||||
var rows []model.Media
|
||||
q := e.repo.DB.WithContext(loadCtx).Model(&model.Media{}).
|
||||
Where("(series_id IS NULL OR series_id = '') AND (season_num > 0 OR episode_num > 0)")
|
||||
q = e.applyUserMediaVisibility(loadCtx, q, userID)
|
||||
if err := q.
|
||||
Order("media.season_num asc, media.episode_num asc, media.episode_fraction asc, media.created_at asc").
|
||||
Limit(embySeriesGroupingLimit).
|
||||
Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groups := e.seriesGroupsFromMedia(loadCtx, rows)
|
||||
e.rememberSeriesGroups(groups)
|
||||
e.markVirtualScan(userID)
|
||||
return groups, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groups, _ := v.([]embySeriesGroup)
|
||||
return groups, nil
|
||||
}
|
||||
|
||||
// markVirtualScan 记录该用户最近一次整库扫描的时间。
|
||||
func (e *EmbyService) markVirtualScan(userID string) {
|
||||
e.virtualMu.Lock()
|
||||
if e.virtualScanAt == nil {
|
||||
e.virtualScanAt = make(map[string]time.Time, 4)
|
||||
}
|
||||
e.virtualScanAt[userID] = time.Now()
|
||||
e.virtualMu.Unlock()
|
||||
}
|
||||
|
||||
// virtualScanRecently 报告该用户是否刚做过一次整库扫描。窗口内仍不在缓存里
|
||||
// 的目标 ID 按不存在处理,避免为同一个未知 ID 反复重扫整库。
|
||||
func (e *EmbyService) virtualScanRecently(userID string) bool {
|
||||
e.virtualMu.RLock()
|
||||
scannedAt := e.virtualScanAt[userID]
|
||||
e.virtualMu.RUnlock()
|
||||
return !scannedAt.IsZero() && time.Since(scannedAt) < embyVirtualScanGrace
|
||||
}
|
||||
|
||||
func (e *EmbyService) seriesGroupsFromMedia(ctx context.Context, rows []model.Media) []embySeriesGroup {
|
||||
byID := map[string]*embySeriesGroup{}
|
||||
order := []string{}
|
||||
|
||||
@@ -26,19 +26,45 @@ func (e *EmbyService) rememberSeriesGroup(group embySeriesGroup) {
|
||||
if e == nil || strings.TrimSpace(group.ID) == "" {
|
||||
return
|
||||
}
|
||||
e.rememberSeriesGroups([]embySeriesGroup{group})
|
||||
}
|
||||
|
||||
// rememberSeriesGroups 批量记住一组剧集分组(含派生季与封面)。
|
||||
//
|
||||
// 一次全表扫描会同时算出整库的剧集分组,而调用方往往只关心其中一部。若只
|
||||
// 逐个 rememberSeriesGroup,同一次首页刷新里 16 张卡片 + 16 张海报都要各自
|
||||
// 命中自己的虚拟 series ID,未命中就重扫整张 media 表(生产环境实测单次
|
||||
// 1.3s,73 个库并发时把连接池排满,首页要 8-25s)。批量写入后冷缓存只付
|
||||
// 一次扫描,其余全部是 O(1) 命中。
|
||||
func (e *EmbyService) rememberSeriesGroups(groups []embySeriesGroup) {
|
||||
if e == nil || len(groups) == 0 {
|
||||
return
|
||||
}
|
||||
expiresAt := time.Now().Add(embyVirtualCacheTTL)
|
||||
e.virtualMu.Lock()
|
||||
defer e.virtualMu.Unlock()
|
||||
if e.virtualSeries == nil {
|
||||
e.virtualSeries = make(map[string]embySeriesCacheEntry)
|
||||
e.virtualSeries = make(map[string]embySeriesCacheEntry, len(groups))
|
||||
}
|
||||
if e.virtualSeasons == nil {
|
||||
e.virtualSeasons = make(map[string]embySeasonCacheEntry)
|
||||
e.virtualSeasons = make(map[string]embySeasonCacheEntry, len(groups))
|
||||
}
|
||||
if e.virtualArtwork == nil {
|
||||
e.virtualArtwork = make(map[string]embyArtworkCacheEntry)
|
||||
e.virtualArtwork = make(map[string]embyArtworkCacheEntry, len(groups)*2)
|
||||
}
|
||||
for _, group := range groups {
|
||||
e.rememberSeriesGroupLocked(group, expiresAt)
|
||||
}
|
||||
// 写入后再裁剪:逐条裁剪会让整批写入退化成 O(n²),且中途裁剪可能把
|
||||
// 同一批里刚写进去的分组又淘汰掉。
|
||||
e.trimVirtualCachesLocked(time.Now())
|
||||
}
|
||||
|
||||
// rememberSeriesGroupLocked 写入单个分组,调用方必须已持有 virtualMu。
|
||||
func (e *EmbyService) rememberSeriesGroupLocked(group embySeriesGroup, expiresAt time.Time) {
|
||||
if strings.TrimSpace(group.ID) == "" {
|
||||
return
|
||||
}
|
||||
e.virtualSeries[group.ID] = embySeriesCacheEntry{group: group, expiresAt: expiresAt}
|
||||
e.virtualArtwork[group.ID] = embyArtworkCacheEntry{primary: group.PosterURL, backdrop: group.BackdropURL, expiresAt: expiresAt}
|
||||
e.virtualArtwork[group.ID+"-bd"] = embyArtworkCacheEntry{primary: group.PosterURL, backdrop: group.BackdropURL, expiresAt: expiresAt}
|
||||
|
||||
@@ -2,6 +2,9 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -278,6 +281,20 @@ func isSupportedEmbyCollectionType(t string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// libraryCoverTag 生成媒体库拼图封面的图片版本号。
|
||||
//
|
||||
// 第三方客户端把库主图按 (URL, ImageTag) 缓存,tag 不变就不会重新请求。
|
||||
// 用成员海报列表的指纹做 tag,库内容变化或生成规则调整时客户端自然会刷新。
|
||||
func libraryCoverTag(libraryID string, sources []string) string {
|
||||
h := sha256.New()
|
||||
_, _ = io.WriteString(h, libraryID)
|
||||
for _, src := range sources {
|
||||
_, _ = io.WriteString(h, src)
|
||||
_, _ = h.Write([]byte{0})
|
||||
}
|
||||
return libraryID + "-" + hex.EncodeToString(h.Sum(nil))[:8]
|
||||
}
|
||||
|
||||
func (e *EmbyService) libraryAsView(ctx context.Context, l *model.Library) map[string]any {
|
||||
collectionType := "movies"
|
||||
switch l.Type {
|
||||
@@ -292,9 +309,17 @@ func (e *EmbyService) libraryAsView(ctx context.Context, l *model.Library) map[s
|
||||
}
|
||||
// 库封面:与剧集/电影一样,只有在能解析出封面时才广告 ImageTags.Primary,
|
||||
// 否则客户端会认为该库无主图而不去请求 /Images/Primary。
|
||||
//
|
||||
// 拼图封面的 tag 带上「成员海报」的指纹:库内容变化或生成规则调整(例如
|
||||
// 从 2×2 改成竖切)时 tag 会变,客户端才会丢缓存重新拉图;固定用库 ID
|
||||
// 会让第三方客户端永远显示第一次缓存下来的封面。
|
||||
imageTags := map[string]string{}
|
||||
if e.LibraryHasCover(ctx, l.ID) {
|
||||
imageTags["Primary"] = l.ID
|
||||
if sources, lib := LibraryCoverSourcesForLibrary(ctx, e.repo, l.ID); lib != nil {
|
||||
if len(sources) > 0 {
|
||||
imageTags["Primary"] = libraryCoverTag(l.ID, sources)
|
||||
} else if strings.TrimSpace(lib.CoverURL) != "" {
|
||||
imageTags["Primary"] = l.ID
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"Id": l.ID,
|
||||
|
||||
@@ -136,7 +136,12 @@ func (e *EmbyService) RecordProgressWithSession(ctx context.Context, userID, med
|
||||
SessionID: playSessionID,
|
||||
SessionStartedAtMs: embyPlaySessionStartedAtMs(playSessionID),
|
||||
})
|
||||
if err == nil {
|
||||
// 普通进度上报不清列表载荷缓存。播放期间客户端每 5-10 秒上报一次,
|
||||
// 每次清空 media:emby:* 会让「最新添加」整批 payload(客户端刷新首页时
|
||||
// 一次并发 73 个库)反复回源 SQL,生产环境实测因此出现 8-25s 的加载。
|
||||
// 续播位置由 /Items/Resume 与条目详情实时查库,列表里的进度条允许滞后;
|
||||
// 只有「已看完」这类状态翻转才让缓存失效(position 到达片尾 30s 内)。
|
||||
if err == nil && playbackProgressCompleted(pos, dur) {
|
||||
e.invalidateEmbyItemsCache(ctx)
|
||||
}
|
||||
return err
|
||||
@@ -287,10 +292,11 @@ func applyMeBoxUserData(raw any, history *model.PlaybackHistory, favorite bool)
|
||||
return userData
|
||||
}
|
||||
|
||||
// embyInvalMu 节流全量缓存失效:播放期间客户端每 5-10s 上报一次进度,
|
||||
// 每次都 SCAN+DEL 全部 media:emby:* 缓存会把缓存命中率持续打穿(其他
|
||||
// 客户端每次翻页都回源 SQL)。条目载荷的 UserData 在请求时动态合并,
|
||||
// 进度类变更做 30s 节流即可,不影响正确性观感。
|
||||
// embyInvalMu 节流全量缓存失效:收藏/标记已看这类显式操作会改变列表里的
|
||||
// UserData,必须让 payload 失效。客户端连续点选时会短时间触发多次,节流到
|
||||
// 30s 一次即可,避免每次都 SCAN+DEL 全部 media:emby:* 缓存。
|
||||
// 播放进度上报不走这里(见 RecordProgressWithSession):续播位置实时查库,
|
||||
// 列表里的进度条允许滞后,不值得为它把整批首页载荷打回源 SQL。
|
||||
var (
|
||||
embyInvalMu sync.Mutex
|
||||
embyInvalLast time.Time
|
||||
|
||||
@@ -144,3 +144,43 @@ func (c *Container) NormalizePollutedEpisodeMetadata(ctx context.Context) (int,
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
// segmentRangeCleanupSettingKey marks the one-time removal of impossible
|
||||
// skip ranges written by the older end_ms: 0 interpretation.
|
||||
const segmentRangeCleanupSettingKey = "segment.range_cleanup_v1_done"
|
||||
|
||||
// NormalizeImpossibleSegments 删除历史误写进来的「不可能区间」。
|
||||
//
|
||||
// 旧解析把 TheIntroDB 的 end_ms: 0 当成「延续到片尾」,于是
|
||||
// intro:[{start_ms:null,end_ms:0}] 被落成 intro (0, 0),客户端补齐后覆盖整集;
|
||||
// 同理 recap (0,0) 也会让整集变成回顾。这类行只会让播放器一开播就弹跳过按钮,
|
||||
// 没有任何保留价值,直接硬删。
|
||||
//
|
||||
// 幂等:完成后写 setting 标记。只处理片头/回顾——片尾类允许开放结束点。
|
||||
func (c *Container) NormalizeImpossibleSegments(ctx context.Context) (int, error) {
|
||||
if c == nil || c.Repo == nil || c.Repo.DB == nil {
|
||||
return 0, nil
|
||||
}
|
||||
if c.Repo.Setting != nil {
|
||||
if v, err := c.Repo.Setting.Get(ctx, segmentRangeCleanupSettingKey); err == nil && strings.TrimSpace(v) == "true" {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
res := c.Repo.DB.WithContext(ctx).Unscoped().
|
||||
Where("kind IN ?", []string{model.SegmentKindIntro, model.SegmentKindRecap}).
|
||||
Where("end_ms <= start_ms").
|
||||
Delete(&model.MediaSegment{})
|
||||
if res.Error != nil {
|
||||
return 0, res.Error
|
||||
}
|
||||
cleaned := int(res.RowsAffected)
|
||||
if c.Repo.Setting != nil {
|
||||
if err := c.Repo.Setting.Set(ctx, segmentRangeCleanupSettingKey, "true"); err != nil && c.Log != nil {
|
||||
c.Log.Warn("mark segment range cleanup done failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
if cleaned > 0 && c.Log != nil {
|
||||
c.Log.Info("impossible skip segments removed", zap.Int("rows", cleaned))
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var ErrInvalidHLSPlaybackScope = errors.New("invalid HLS playback scope")
|
||||
var ErrHLSGenerationSuperseded = errors.New("HLS playback generation superseded")
|
||||
|
||||
// HLSPlaybackScope separates a media's independent users and player instances.
|
||||
// UserID must come from server authentication, never from a request parameter.
|
||||
type HLSPlaybackScope struct {
|
||||
UserID string
|
||||
SessionID string
|
||||
}
|
||||
|
||||
func HLSPlaybackScopeForRequest(r *http.Request, userID string) (HLSPlaybackScope, error) {
|
||||
scope := HLSPlaybackScope{UserID: strings.TrimSpace(userID)}
|
||||
if scope.UserID == "" || r == nil {
|
||||
return scope, ErrInvalidHLSPlaybackScope
|
||||
}
|
||||
for _, key := range []string{"session_id", "PlaySessionId", "playSessionId", "DeviceId", "deviceId"} {
|
||||
if value := strings.TrimSpace(r.URL.Query().Get(key)); value != "" {
|
||||
scope.SessionID = value
|
||||
break
|
||||
}
|
||||
}
|
||||
if scope.SessionID == "" {
|
||||
if device := strings.TrimSpace(r.Header.Get("X-Emby-Device-Id")); device != "" {
|
||||
scope.SessionID = device
|
||||
}
|
||||
}
|
||||
if len(scope.SessionID) > 256 || strings.IndexFunc(scope.SessionID, unicode.IsControl) >= 0 {
|
||||
return scope, ErrInvalidHLSPlaybackScope
|
||||
}
|
||||
// Clients without a session/device identifier remain compatible but only
|
||||
// share a legacy task with the same authenticated user, never other users.
|
||||
return scope, nil
|
||||
}
|
||||
|
||||
func (scope HLSPlaybackScope) jobKey(mediaID string) string {
|
||||
if scope.UserID == "" && scope.SessionID == "" {
|
||||
return mediaID // Existing internal callers/tests retain their legacy API.
|
||||
}
|
||||
sum := sha256.Sum256([]byte(scope.UserID + "\x00" + mediaID + "\x00" + scope.SessionID))
|
||||
return "player-" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func hlsPlaybackJobKey(mediaID string, scopes []HLSPlaybackScope) string {
|
||||
if len(scopes) == 0 {
|
||||
return mediaID
|
||||
}
|
||||
return scopes[0].jobKey(mediaID)
|
||||
}
|
||||
|
||||
func (job *hlsJob) key() string {
|
||||
if job.jobKey != "" {
|
||||
return job.jobKey
|
||||
}
|
||||
return job.mediaID
|
||||
}
|
||||
|
||||
func (t *TranscoderService) jobPlaylistPath(job *hlsJob) string {
|
||||
dir := job.outputDir
|
||||
if dir == "" {
|
||||
dir = t.HLSDir(job.key())
|
||||
}
|
||||
return filepath.Join(dir, "index.m3u8")
|
||||
}
|
||||
|
||||
// hlsOutputDir captures one immutable run directory. Even if a new run starts
|
||||
// after this check, the request cannot accidentally open its replacement files.
|
||||
func (t *TranscoderService) hlsOutputDir(key string, generation int64) (string, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
job := t.jobs[key]
|
||||
if job == nil {
|
||||
return "", ErrMediaNotFound
|
||||
}
|
||||
if job.seekGen != generation {
|
||||
return "", ErrHLSGenerationSuperseded
|
||||
}
|
||||
t.touchJobLocked(key)
|
||||
if job.outputDir != "" {
|
||||
return job.outputDir, nil
|
||||
}
|
||||
return t.HLSDir(key), nil
|
||||
}
|
||||
|
||||
func (t *TranscoderService) StopPlayback(mediaID string, scope HLSPlaybackScope) {
|
||||
key := scope.jobKey(mediaID)
|
||||
unlock := t.lockJobStart(key)
|
||||
defer unlock()
|
||||
t.mu.Lock()
|
||||
job := t.detachJobLocked(key)
|
||||
t.mu.Unlock()
|
||||
waitJobExit(job, 12*time.Second)
|
||||
// Keep cleanup under the same gate as start: a delayed stop must not wipe
|
||||
// an output directory created by a concurrent replacement request.
|
||||
_ = os.RemoveAll(t.HLSDir(key))
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
)
|
||||
|
||||
func cachedScopedHLS(t *testing.T, scopes ...HLSPlaybackScope) (*StreamService, *TranscoderService, map[string]string) {
|
||||
t.Helper()
|
||||
cfg := &config.Config{}
|
||||
cfg.Cache.CacheDir = t.TempDir()
|
||||
cfg.Transcoder.Enabled = true
|
||||
db := newServiceTestDB(t)
|
||||
if err := db.Create(&model.Media{Base: model.Base{ID: "movie"}, Path: "unused-cached-source", DurationSec: 60}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder := NewTranscoderService(cfg, zap.NewNop(), repository.New(db), nil)
|
||||
paths := map[string]string{}
|
||||
for _, scope := range scopes {
|
||||
key := scope.jobKey("movie")
|
||||
dir := filepath.Join(transcoder.HLSDir(key), "run-cached")
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
playlist := filepath.Join(dir, "index.m3u8")
|
||||
if err := os.WriteFile(playlist, []byte("#EXTM3U\n#EXTINF:4,\nseg_00000.ts\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "seg_00000.ts"), []byte(scope.UserID+"/"+scope.SessionID), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder.jobs[key] = &hlsJob{mediaID: "movie", jobKey: key, outputDir: dir, subtitleStream: -1, startedAt: time.Now(), cancel: func() {}}
|
||||
paths[key] = playlist
|
||||
}
|
||||
return NewStreamService(cfg, zap.NewNop(), repository.New(db), transcoder), transcoder, paths
|
||||
}
|
||||
|
||||
func TestHLSPlaybackEnsureReusesOnlyMatchingOwnerAndSession(t *testing.T) {
|
||||
a := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-b"}
|
||||
_, transcoder, paths := cachedScopedHLS(t, a, b)
|
||||
for _, scope := range []HLSPlaybackScope{a, b} {
|
||||
playlist, err := transcoder.EnsureJobFromSubtitleQuality(t.Context(), "movie", 0, 0, -1, "", scope)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if playlist != paths[scope.jobKey("movie")] {
|
||||
t.Fatalf("scope %#v reused another session's playlist %q", scope, playlist)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackStopRemovesOnlyItsSessionOutput(t *testing.T) {
|
||||
a := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-b"}
|
||||
_, transcoder, paths := cachedScopedHLS(t, a, b)
|
||||
transcoder.StopPlayback("movie", a)
|
||||
if _, err := os.Stat(paths[a.jobKey("movie")]); !os.IsNotExist(err) {
|
||||
t.Fatalf("stopped session output must be cleaned up, err=%v", err)
|
||||
}
|
||||
if _, err := os.Stat(paths[b.jobKey("movie")]); err != nil {
|
||||
t.Fatalf("another player's output was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackIdleCleanupRemovesExpiredSessionOutput(t *testing.T) {
|
||||
a := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "viewer", SessionID: "tab-b"}
|
||||
_, transcoder, paths := cachedScopedHLS(t, a, b)
|
||||
job := transcoder.jobs[a.jobKey("movie")]
|
||||
job.lastAccess = time.Now().Add(-2 * time.Minute)
|
||||
if !transcoder.stopIdleJob(job, time.Minute) {
|
||||
t.Fatal("expired session was not stopped")
|
||||
}
|
||||
if _, err := os.Stat(paths[a.jobKey("movie")]); !os.IsNotExist(err) {
|
||||
t.Fatalf("expired output was retained: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(paths[b.jobKey("movie")]); err != nil {
|
||||
t.Fatalf("active session output was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackPropagatesHeaderDeviceIdentityToSegments(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/hls/movie/index.m3u8?token=test", nil)
|
||||
req.Header.Set("X-Emby-Device-Id", "legacy-device")
|
||||
scope, err := HLSPlaybackScopeForRequest(req, "viewer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc, _, _ := cachedScopedHLS(t, scope)
|
||||
rec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSPlaylist(rec, req, "movie", scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var segment string
|
||||
for _, line := range strings.Split(rec.Body.String(), "\n") {
|
||||
if strings.HasPrefix(line, "seg_") {
|
||||
segment = line
|
||||
break
|
||||
}
|
||||
}
|
||||
u, err := url.Parse(segment)
|
||||
if err != nil || u.Query().Get("session_id") == "" {
|
||||
t.Fatalf("playlist must carry the header-only device scope to child URLs: %q err=%v", segment, err)
|
||||
}
|
||||
child := httptest.NewRequest(http.MethodGet, "/api/hls/movie/"+segment, nil)
|
||||
childScope, err := HLSPlaybackScopeForRequest(child, "viewer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if childScope.jobKey("movie") != scope.jobKey("movie") {
|
||||
t.Fatal("child request changed its owner's device scope")
|
||||
}
|
||||
childRec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSSegment(childRec, child, "movie", "seg_00000.ts", childScope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if childRec.Body.String() != "viewer/"+scope.SessionID {
|
||||
t.Fatalf("wrong scoped segment: %s", childRec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/config"
|
||||
)
|
||||
|
||||
func TestHLSPlaybackScopesIsolateSegmentReadsAndStop(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
cfg.Cache.CacheDir = t.TempDir()
|
||||
transcoder := NewTranscoderService(cfg, zap.NewNop(), nil, nil)
|
||||
svc := NewStreamService(cfg, zap.NewNop(), nil, transcoder)
|
||||
a := HLSPlaybackScope{UserID: "user-a", SessionID: "tab-a"}
|
||||
b := HLSPlaybackScope{UserID: "user-b", SessionID: "tab-a"}
|
||||
c := HLSPlaybackScope{UserID: "user-a", SessionID: "tab-c"}
|
||||
for _, scope := range []HLSPlaybackScope{a, b, c} {
|
||||
key := scope.jobKey("movie")
|
||||
dir := filepath.Join(cfg.Cache.CacheDir, key, "run-current")
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "seg_00000.ts"), []byte(scope.UserID+"/"+scope.SessionID), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder.jobs[key] = &hlsJob{mediaID: "movie", jobKey: key, outputDir: dir, seekGen: 100, cancel: func() {}}
|
||||
}
|
||||
for _, scope := range []HLSPlaybackScope{a, b, c} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/hls/movie/seg_00000.ts?_seek=100", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSSegment(rec, req, "movie", "seg_00000.ts", scope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := rec.Body.String(), scope.UserID+"/"+scope.SessionID; got != want {
|
||||
t.Fatalf("segment=%q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
transcoder.StopPlayback("movie", a)
|
||||
if transcoder.jobs[a.jobKey("movie")] != nil {
|
||||
t.Fatal("stopped scope is still active")
|
||||
}
|
||||
if transcoder.jobs[b.jobKey("movie")] == nil || transcoder.jobs[c.jobKey("movie")] == nil {
|
||||
t.Fatal("stopping one player affected another user/tab")
|
||||
}
|
||||
// A delayed stop from a previous local-playback session must not affect a replacement.
|
||||
transcoder.StopPlayback("movie", a)
|
||||
if transcoder.jobs[c.jobKey("movie")] == nil {
|
||||
t.Fatal("old stop cancelled a new session")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackRejectsStaleSegmentGeneration(t *testing.T) {
|
||||
cfg := &config.Config{}
|
||||
cfg.Cache.CacheDir = t.TempDir()
|
||||
transcoder := NewTranscoderService(cfg, zap.NewNop(), nil, nil)
|
||||
svc := NewStreamService(cfg, zap.NewNop(), nil, transcoder)
|
||||
scope := HLSPlaybackScope{UserID: "viewer", SessionID: "tab"}
|
||||
key := scope.jobKey("movie")
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "seg_00000.ts"), []byte("new-generation"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcoder.jobs[key] = &hlsJob{mediaID: "movie", jobKey: key, outputDir: dir, seekGen: 200, cancel: func() {}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/hls/movie/seg_00000.ts?_seek=100", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
if err := svc.ServeHLSSegment(rec, req, "movie", "seg_00000.ts", scope); err == nil {
|
||||
t.Fatal("stale segment request read a replacement generation")
|
||||
}
|
||||
if rec.Body.Len() != 0 {
|
||||
t.Fatal("stale request exposed replacement bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSPlaybackScopeRequiresServerOwner(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/?session_id=tab&user_id=attacker-controlled", nil)
|
||||
if _, err := HLSPlaybackScopeForRequest(req, ""); err == nil {
|
||||
t.Fatal("missing authenticated owner was accepted")
|
||||
}
|
||||
a, err := HLSPlaybackScopeForRequest(req, "real-owner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.UserID != "real-owner" {
|
||||
t.Fatal("client parameter supplied the owner")
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodGet, "/?PlaySessionId=emby-session", nil)
|
||||
b, err := HLSPlaybackScopeForRequest(req, "real-owner")
|
||||
if err != nil || b.SessionID == "" {
|
||||
t.Fatalf("Emby session must stay scoped: %#v %v", b, err)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,10 @@ type ImageProxy struct {
|
||||
resizeSemMu sync.Mutex
|
||||
resizeSem chan struct{}
|
||||
|
||||
// resizeQueueWait 是一次请求最多为一格解码槽排队的时长。为 0 时用
|
||||
// defaultResizeQueueWait;测试会把它调小以免真的等待两秒。
|
||||
resizeQueueWait time.Duration
|
||||
|
||||
// libraryRootsFn returns the configured media library roots so that
|
||||
// sidecar poster/artwork files stored alongside media (under arbitrary
|
||||
// per-library paths) are allowed by isAllowedLocalPath. It is provided
|
||||
@@ -86,6 +90,12 @@ const (
|
||||
// imageRenditionCacheSubdir 存放挂载 Emby 直接按尺寸产出的成品图:它们
|
||||
// 已经是客户端要的最终尺寸,和派生缩略图一样属于长期保留的热路径缓存。
|
||||
imageRenditionCacheSubdir = "renditions"
|
||||
|
||||
// defaultResizeQueueWait 是请求为一格解码槽排队的最长时间。一张 3MP
|
||||
// 海报的解码+缩放约 0.15s,2 核机器两格槽位足以在几秒内消化一整屏
|
||||
// 海报;等待远好于「槽位忙就回退多兆字节原图」(那样每张图在外部网络
|
||||
// 下要十几秒)。超时后仍回退原图,保证请求不会被无限期挂住。
|
||||
defaultResizeQueueWait = 3 * time.Second
|
||||
)
|
||||
|
||||
// NewImageProxy is the constructor.
|
||||
@@ -189,21 +199,19 @@ func imageResizeConcurrency() int {
|
||||
return n
|
||||
}
|
||||
|
||||
// Prune removes oldest cached images until disk usage is within the configured limit.
|
||||
// 原图池按独立配额与保留时长优先淘汰,客户端热路径读取的成品图最后淘汰。
|
||||
// Prune 只执行图片原图子配额与旧缓存清理;整个磁盘缓存的总量淘汰由调度器的
|
||||
// cache_cleanup 任务(PruneCache)负责。
|
||||
func (p *ImageProxy) Prune() (PruneImageCacheResult, error) {
|
||||
if p.cfg == nil || p.cfg.Cache.ImagesMaxSizeMB <= 0 {
|
||||
if p.cfg == nil {
|
||||
return PruneImageCacheResult{}, nil
|
||||
}
|
||||
var originalsMaxBytes int64
|
||||
if p.cfg.Cache.ImagesOriginalsMaxSizeMB > 0 {
|
||||
originalsMaxBytes = int64(p.cfg.Cache.ImagesOriginalsMaxSizeMB) * 1024 * 1024
|
||||
}
|
||||
var originalsMaxAge time.Duration
|
||||
if p.cfg.Cache.ImagesOriginalsTTLHours > 0 {
|
||||
originalsMaxAge = time.Duration(p.cfg.Cache.ImagesOriginalsTTLHours) * time.Hour
|
||||
if originalsMaxBytes <= 0 {
|
||||
return PruneImageCacheResult{}, nil
|
||||
}
|
||||
totalBytes := int64(p.cfg.Cache.ImagesMaxSizeMB) * 1024 * 1024
|
||||
pools := ImageCachePools(p.cacheDir, originalsMaxBytes, originalsMaxAge)
|
||||
return PruneImageCachePools(pools, totalBytes)
|
||||
pools := ImageCachePools(p.cacheDir, originalsMaxBytes)
|
||||
return PruneImageCachePools(pools, 0)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@@ -138,8 +139,8 @@ func fitSize(srcW, srcH, maxW, maxH int) (int, int) {
|
||||
|
||||
var errImageResizeTooLarge = errors.New("image source exceeds resize pixel budget")
|
||||
|
||||
// resizeImageData 按选项缩放图片并重新编码。返回 unchanged=true 表示原图
|
||||
// 本身已满足目标尺寸,调用方应直接输出原始字节(不重复编码、不损失质量)。
|
||||
// resizeImageData 按选项缩放图片并重新编码。返回 unchanged=true 表示源图
|
||||
// 已经是可直接下发的紧凑字节,调用方应直接输出原始字节(不重复编码)。
|
||||
func resizeImageData(data []byte, o imageResizeOptions) (out []byte, ctype string, unchanged bool, err error) {
|
||||
if len(data) == 0 || !o.active() {
|
||||
return data, detectContentType(data), true, nil
|
||||
@@ -151,12 +152,21 @@ func resizeImageData(data []byte, o imageResizeOptions) (out []byte, ctype strin
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return nil, "", false, errors.New("invalid image dimensions")
|
||||
}
|
||||
// 先用 DecodeConfig 判断是否需要解码整图:图已够小就零成本返回原字节。
|
||||
// 先用 DecodeConfig 判断是否需要缩放。尺寸已在目标框内时仍要重编码:
|
||||
// 刮削来源的海报常是 1000×1500、1MB 以上的 PNG,直接原样下发会让一处
|
||||
// 海报墙就从「几十 KB 缩略图」变成「几十 MB 原图」——外部网络下每张
|
||||
// 都要十几秒。只有本来就是紧凑 Web 格式(JPEG/WebP)且体积很小的源图
|
||||
// 才值得零成本直出。
|
||||
dstW, dstH := fitSize(cfg.Width, cfg.Height, o.MaxWidth, o.MaxHeight)
|
||||
if dstW == cfg.Width && dstH == cfg.Height {
|
||||
unchangedSize := dstW == cfg.Width && dstH == cfg.Height
|
||||
if unchangedSize && isCompactWebFormat(format) && len(data) <= compactImageSkipBytes {
|
||||
return data, detectContentType(data), true, nil
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > imageResizeMaxSourcePixels {
|
||||
if unchangedSize {
|
||||
// 尺寸已满足但像素量超出解码预算:宁可直出原图,也不冒 OOM 风险。
|
||||
return data, detectContentType(data), true, nil
|
||||
}
|
||||
return nil, "", false, errImageResizeTooLarge
|
||||
}
|
||||
|
||||
@@ -165,7 +175,12 @@ func resizeImageData(data []byte, o imageResizeOptions) (out []byte, ctype strin
|
||||
return nil, "", false, err
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dstW, dstH))
|
||||
resizeScaled(dst, src, cfg.Width, cfg.Height)
|
||||
if unchangedSize {
|
||||
// 不缩放时省掉一次全图重采样,只做格式转换。
|
||||
draw.Draw(dst, dst.Bounds(), src, src.Bounds().Min, draw.Src)
|
||||
} else {
|
||||
resizeScaled(dst, src, cfg.Width, cfg.Height)
|
||||
}
|
||||
|
||||
// 只有可能带透明的源格式才需要逐像素确认,避免 JPEG 的无谓遍历。
|
||||
// 写实海报的 PNG 通常比等价 JPEG 大一个数量级,因此在确认不含透明
|
||||
@@ -231,9 +246,12 @@ func isOpaqueImage(img *image.RGBA) bool {
|
||||
// resizeSourceKey 生成源图身份键:只覆盖源文件身份(路径 + 大小 + mtime),不含
|
||||
// 目标尺寸。同一源图的所有尺寸档位共用这个前缀,才能互相列举与派生(小图直接
|
||||
// 从已缓存的大图缩小,而不是重新解码原图)。
|
||||
//
|
||||
// 前缀里的 v3 是缓存格式版本:v2 时期「尺寸已在目标框内」的 PNG 会原样直出、
|
||||
// 不写缓存,而现在同样条件会重编码成 JPEG,旧缓存与新结果语义不同,必须换键。
|
||||
func (o imageResizeOptions) resizeSourceKey(sourceID string, stat os.FileInfo) string {
|
||||
h := sha256.New()
|
||||
_, _ = fmt.Fprintf(h, "v2|%s", sourceID)
|
||||
_, _ = fmt.Fprintf(h, "v3|%s", sourceID)
|
||||
if stat != nil {
|
||||
_, _ = fmt.Fprintf(h, "|%d|%d", stat.Size(), stat.ModTime().UnixNano())
|
||||
}
|
||||
@@ -356,13 +374,53 @@ func (p *ImageProxy) tryAcquireResizeSlot() (func(), bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// acquireResizeSlotFor 对中等体积的图只尝试一次槽位。忙则放弃缩放,
|
||||
// 避免背景图墙在 2 核机器上排成数秒。超大原图仍等待,以免直出多兆字节。
|
||||
func (p *ImageProxy) resizeQueueTimeout() time.Duration {
|
||||
if p != nil && p.resizeQueueWait > 0 {
|
||||
return p.resizeQueueWait
|
||||
}
|
||||
return defaultResizeQueueWait
|
||||
}
|
||||
|
||||
// acquireResizeSlotFor 对紧凑小图只尝试一次槽位,其余情况最多排队
|
||||
// resizeQueueTimeout。
|
||||
//
|
||||
// 从前这里对 1.5MB 以下的图一律「槽位忙就跳过缩放」,结果是并发加载海报墙
|
||||
// 时几乎每张图都回退成 1MB 上下的 PNG 原图:一页要传几十 MB,外部网络下单张
|
||||
// 图就要二十多秒。缩略图本身只有几十 KB,排队几百毫秒明显更划算;只有真正
|
||||
// 大到会拖慢页面的原图才值得在超时后直出。
|
||||
func (p *ImageProxy) acquireResizeSlotFor(ctx context.Context, size int64) (func(), bool) {
|
||||
if size > 0 && size <= resizeQueueBypassBytes {
|
||||
if size > 0 && size <= compactImageSkipBytes {
|
||||
return p.tryAcquireResizeSlot()
|
||||
}
|
||||
return p.acquireResizeSlot(ctx)
|
||||
return p.acquireResizeSlotWithin(ctx, p.resizeQueueTimeout())
|
||||
}
|
||||
|
||||
// acquireResizeSlotWithin 在 timeout 内等待一格解码槽;超时或请求取消返回
|
||||
// false,调用方回退原图。
|
||||
func (p *ImageProxy) acquireResizeSlotWithin(ctx context.Context, timeout time.Duration) (func(), bool) {
|
||||
if p == nil {
|
||||
return func() {}, true
|
||||
}
|
||||
if timeout <= 0 {
|
||||
return p.tryAcquireResizeSlot()
|
||||
}
|
||||
p.resizeSemMu.Lock()
|
||||
if p.resizeSem == nil {
|
||||
p.resizeSem = make(chan struct{}, imageResizeConcurrency())
|
||||
}
|
||||
sem := p.resizeSem
|
||||
p.resizeSemMu.Unlock()
|
||||
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
return func() { <-sem }, true
|
||||
case <-timer.C:
|
||||
return nil, false
|
||||
case <-ctx.Done():
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// serveResizedFromFile 从 srcPath 读取图片,按选项缩放后写出,并把结果缓存
|
||||
@@ -472,10 +530,6 @@ func (p *ImageProxy) ensureResizeCache(ctx context.Context, srcPath string, o im
|
||||
// ponytail: 200KB 覆盖这台机器上的典型海报(平均约 100KB);更大的图仍尝试缩放。
|
||||
const compactImageSkipBytes = 200 * 1024
|
||||
|
||||
// resizeQueueBypassBytes 是“槽位忙就放弃缩放”的上限。超过它的原图继续排队,
|
||||
// 以免把未压缩的剧照直接发给客户端。
|
||||
const resizeQueueBypassBytes = 1536 * 1024
|
||||
|
||||
// serveCompactOriginal 在源文件已经很小且是浏览器可直接显示的 JPEG/WebP 时
|
||||
// 跳过解码。返回 false 表示仍应走缩放路径。
|
||||
func serveCompactOriginal(w http.ResponseWriter, r *http.Request, srcPath string, stat os.FileInfo) bool {
|
||||
@@ -504,6 +558,17 @@ func isCompactWebImage(header []byte) bool {
|
||||
string(header[8:12]) == "WEBP"
|
||||
}
|
||||
|
||||
// isCompactWebFormat 判断 image.DecodeConfig 报告的格式是否已经适合直接下发
|
||||
// 给浏览器。PNG/BMP/GIF 即使尺寸很小也可能有几百 KB,需要重编码成 JPEG。
|
||||
func isCompactWebFormat(format string) bool {
|
||||
switch strings.ToLower(format) {
|
||||
case "jpeg", "jpg", "webp":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// writeResizeCache 原子写入缩放结果;失败只记日志,不影响本次响应。
|
||||
// 与 writeImageCache 一样不再持有全局锁:临时文件名唯一、rename 原子,
|
||||
// 持锁只会把缩略图的写盘和原图的写盘串成一条队。
|
||||
|
||||
@@ -123,17 +123,58 @@ func TestResizeImageDataScalesDownAndReencodesAsJPEG(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResizeImageDataLeavesImagesWithinBoundsUntouched(t *testing.T) {
|
||||
// 尺寸已经在目标框内的 PNG 也必须转成 JPEG。刮削来的海报常是 1000×1500、
|
||||
// 1MB 级的 PNG;此前这种情况原样直出,一处海报墙就要下载几十 MB,外部网络下
|
||||
// 每张图要十几秒(线上实测单张 1.3MB / 20 多秒)。
|
||||
func TestResizeImageDataReencodesPNGWithinBounds(t *testing.T) {
|
||||
data := encodeTestPNG(t, 200, 300, 255)
|
||||
out, _, unchanged, err := resizeImageData(data, imageResizeOptions{MaxWidth: 400, MaxHeight: 600})
|
||||
out, ctype, unchanged, err := resizeImageData(data, imageResizeOptions{MaxWidth: 400, MaxHeight: 600, Quality: 80})
|
||||
if err != nil {
|
||||
t.Fatalf("resize: %v", err)
|
||||
}
|
||||
if unchanged {
|
||||
t.Fatal("PNG within the target box must still be re-encoded as a compact web format")
|
||||
}
|
||||
if ctype != "image/jpeg" {
|
||||
t.Fatalf("content type = %q, want image/jpeg", ctype)
|
||||
}
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(out))
|
||||
if err != nil {
|
||||
t.Fatalf("decode re-encoded image: %v", err)
|
||||
}
|
||||
if format != "jpeg" {
|
||||
t.Fatalf("format = %q, want jpeg", format)
|
||||
}
|
||||
if cfg.Width != 200 || cfg.Height != 300 {
|
||||
t.Fatalf("re-encoded to %dx%d, want the original 200x300 (no upscaling)", cfg.Width, cfg.Height)
|
||||
}
|
||||
}
|
||||
|
||||
// 已经是紧凑 Web 格式且体积很小的源图仍然零成本直出。
|
||||
func TestResizeImageDataLeavesSmallCompactSourceUntouched(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
img := image.NewRGBA(image.Rect(0, 0, 200, 300))
|
||||
for y := 0; y < 300; y++ {
|
||||
for x := 0; x < 200; x++ {
|
||||
img.Set(x, y, color.RGBA{R: uint8(x % 16 * 16), G: uint8(y % 16 * 16), B: 200, A: 255})
|
||||
}
|
||||
}
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 80}); err != nil {
|
||||
t.Fatalf("encode test jpeg: %v", err)
|
||||
}
|
||||
data := buf.Bytes()
|
||||
if len(data) > compactImageSkipBytes {
|
||||
t.Fatalf("test jpeg is %d bytes, expected a compact source", len(data))
|
||||
}
|
||||
out, ctype, unchanged, err := resizeImageData(data, imageResizeOptions{MaxWidth: 400, MaxHeight: 600})
|
||||
if err != nil {
|
||||
t.Fatalf("resize: %v", err)
|
||||
}
|
||||
if !unchanged {
|
||||
t.Fatal("expected an image already within bounds to be returned unchanged")
|
||||
t.Fatal("a compact JPEG already within bounds should be served unchanged")
|
||||
}
|
||||
if !bytes.Equal(out, data) {
|
||||
t.Fatal("expected the original bytes to be returned verbatim")
|
||||
if !bytes.Equal(out, data) || ctype != "image/jpeg" {
|
||||
t.Fatalf("expected the original jpeg bytes back, got %d bytes of %q", len(out), ctype)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,7 +343,9 @@ func TestServeResizedFromFileSkipsDecodeForCompactJPEG(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeResizedFromFileDoesNotQueueWhenResizeBusy(t *testing.T) {
|
||||
// 排队上限:槽位一直不空时,请求最多等 resizeQueueWait,然后回退原图,
|
||||
// 不会被无限期挂住。这条约束保护的是请求本身,而不是「不等就直出原图」。
|
||||
func TestServeResizedFromFileFallsBackWhenResizeQueueStaysBusy(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := dir + string(os.PathSeparator) + "backdrop.jpg"
|
||||
data := make([]byte, compactImageSkipBytes+1024)
|
||||
@@ -310,7 +353,7 @@ func TestServeResizedFromFileDoesNotQueueWhenResizeBusy(t *testing.T) {
|
||||
if err := os.WriteFile(src, data, 0o644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
proxy := &ImageProxy{cacheDir: dir + string(os.PathSeparator) + "cache"}
|
||||
proxy := &ImageProxy{cacheDir: dir + string(os.PathSeparator) + "cache", resizeQueueWait: 50 * time.Millisecond}
|
||||
var releases []func()
|
||||
for i := 0; i < imageResizeConcurrency(); i++ {
|
||||
release, ok := proxy.acquireResizeSlot(context.Background())
|
||||
@@ -327,10 +370,77 @@ func TestServeResizedFromFileDoesNotQueueWhenResizeBusy(t *testing.T) {
|
||||
|
||||
start := time.Now()
|
||||
handled := proxy.serveResizedFromFile(httptest.NewRecorder(), httptest.NewRequest("GET", "/x?maxWidth=480", nil), src, imageResizeOptions{MaxWidth: 480})
|
||||
if time.Since(start) > 200*time.Millisecond {
|
||||
t.Fatal("resize slot was busy but the request still waited in the queue")
|
||||
if elapsed := time.Since(start); elapsed > 500*time.Millisecond {
|
||||
t.Fatalf("request waited %v with every slot held; the queue wait must be bounded", elapsed)
|
||||
}
|
||||
if handled {
|
||||
t.Fatal("expected a busy resize slot to fall back so the caller can serve the original")
|
||||
t.Fatal("expected a permanently busy queue to fall back so the caller can serve the original")
|
||||
}
|
||||
}
|
||||
|
||||
// 槽位只是短暂忙碌时必须排队等待并产出缩略图,而不是直出多兆字节原图。
|
||||
// 这是线上「海报墙每张图 20 多秒」的根因:并发加载时槽位全忙,旧逻辑把
|
||||
// 1MB 级 PNG 原样下发。
|
||||
// encodeNoisyPNG 生成难以压缩的 PNG:规则色块会被压到 1KB 级别,测不出
|
||||
// 「大体积 PNG 被原样下发」这条回归。
|
||||
func encodeNoisyPNG(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
seed := uint32(0x9e3779b9)
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
seed = seed*1664525 + 1013904223
|
||||
img.Set(x, y, color.RGBA{R: uint8(seed >> 24), G: uint8(seed >> 16), B: uint8(seed >> 8), A: 255})
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatalf("encode noisy png: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestServeResizedFromFileWaitsForSlotInsteadOfServingOriginal(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := dir + string(os.PathSeparator) + "poster.png"
|
||||
original := encodeNoisyPNG(t, 378, 540)
|
||||
if len(original) <= compactImageSkipBytes {
|
||||
t.Fatalf("test poster is %d bytes, want a source above the compact threshold", len(original))
|
||||
}
|
||||
if err := os.WriteFile(src, original, 0o644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
proxy := &ImageProxy{cacheDir: dir + string(os.PathSeparator) + "cache"}
|
||||
var releases []func()
|
||||
for i := 0; i < imageResizeConcurrency(); i++ {
|
||||
release, ok := proxy.acquireResizeSlot(context.Background())
|
||||
if !ok {
|
||||
t.Fatal("expected a resize slot")
|
||||
}
|
||||
releases = append(releases, release)
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
releases[0]()
|
||||
}()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
if !proxy.serveResizedFromFile(rec, httptest.NewRequest("GET", "/x?maxWidth=480&maxHeight=600", nil), src, imageResizeOptions{MaxWidth: 480, MaxHeight: 600}) {
|
||||
t.Fatal("expected the request to wait for the slot and serve a thumbnail")
|
||||
}
|
||||
body := rec.Body.Bytes()
|
||||
if bytes.Equal(body, original) {
|
||||
t.Fatal("served the multi-hundred-KB PNG original instead of a re-encoded thumbnail")
|
||||
}
|
||||
if rec.Header().Get("Content-Type") != "image/jpeg" {
|
||||
t.Fatalf("content type = %q, want image/jpeg", rec.Header().Get("Content-Type"))
|
||||
}
|
||||
// 同一请求第二次必须直接命中缩放缓存。
|
||||
rec2 := httptest.NewRecorder()
|
||||
if !proxy.serveResizedFromFile(rec2, httptest.NewRequest("GET", "/x?maxWidth=480&maxHeight=600", nil), src, imageResizeOptions{MaxWidth: 480, MaxHeight: 600}) {
|
||||
t.Fatal("expected the cached thumbnail to be reused")
|
||||
}
|
||||
if !bytes.Equal(rec2.Body.Bytes(), body) {
|
||||
t.Fatal("expected the cached thumbnail bytes to be identical")
|
||||
}
|
||||
}
|
||||
|
||||
+31
-17
@@ -41,15 +41,6 @@ const (
|
||||
introDBMaxRetryDelay = 3 * time.Second
|
||||
)
|
||||
|
||||
// IntroDBSpan is one resolved skip range, still in provider terms.
|
||||
// EndMs == 0 means "runs to the end of the media" (TheIntroDB returns
|
||||
// end_ms: null for end credits); the caller resolves it against the duration.
|
||||
type IntroDBSpan struct {
|
||||
Kind string
|
||||
StartMs int64
|
||||
EndMs int64
|
||||
}
|
||||
|
||||
// IntroDBService queries TheIntroDB for one media item.
|
||||
type IntroDBService struct {
|
||||
log *zap.Logger
|
||||
@@ -113,7 +104,7 @@ type introDBResponse struct {
|
||||
// 429/503 会重试一次(社区库在短时间连发下确实会限流)。重试前会先确认调用方
|
||||
// 的 deadline 还够用;预算不够就直接返回错误,让调用方保留自己的缓存,
|
||||
// 把「拿不到片段」维持在「少一个跳过按钮」的量级。
|
||||
func (s *IntroDBService) Fetch(ctx context.Context, tmdbID, season, episode int) ([]IntroDBSpan, error) {
|
||||
func (s *IntroDBService) Fetch(ctx context.Context, tmdbID, season, episode int) ([]SegmentSpan, error) {
|
||||
if s == nil || s.client == nil {
|
||||
return nil, errors.New("introdb service nil")
|
||||
}
|
||||
@@ -135,9 +126,20 @@ func (s *IntroDBService) Fetch(ctx context.Context, tmdbID, season, episode int)
|
||||
}
|
||||
}
|
||||
|
||||
// Source implements SegmentProvider.
|
||||
func (s *IntroDBService) Source() string { return IntroDBSource }
|
||||
|
||||
// FetchSegments implements SegmentProvider. TheIntroDB 以 TMDb id 寻址。
|
||||
func (s *IntroDBService) FetchSegments(ctx context.Context, q SegmentQuery) ([]SegmentSpan, error) {
|
||||
if q.IsMovie {
|
||||
return s.Fetch(ctx, q.TMDbID, 0, 0)
|
||||
}
|
||||
return s.Fetch(ctx, q.TMDbID, q.Season, q.Episode)
|
||||
}
|
||||
|
||||
// introDBFetchAttempt 是一次请求的结果:数据或错误,外加「值不值得重试」。
|
||||
type introDBFetchAttempt struct {
|
||||
spans []IntroDBSpan
|
||||
spans []SegmentSpan
|
||||
err error
|
||||
retryable bool
|
||||
retryAfter time.Duration
|
||||
@@ -246,7 +248,7 @@ func (s *IntroDBService) mediaURL(tmdbID, season, episode int) string {
|
||||
// parseIntroDBResponse flattens the per-type arrays into spans, preserving the
|
||||
// intro -> recap -> credits -> preview order so the player sees the earliest
|
||||
// range first.
|
||||
func parseIntroDBResponse(body []byte) ([]IntroDBSpan, error) {
|
||||
func parseIntroDBResponse(body []byte) ([]SegmentSpan, error) {
|
||||
var raw introDBResponse
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return nil, fmt.Errorf("parse introdb json: %w", err)
|
||||
@@ -260,7 +262,7 @@ func parseIntroDBResponse(body []byte) ([]IntroDBSpan, error) {
|
||||
{"credits", raw.Credits},
|
||||
{"preview", raw.Preview},
|
||||
}
|
||||
spans := make([]IntroDBSpan, 0, len(raw.Intro)+len(raw.Credits))
|
||||
spans := make([]SegmentSpan, 0, len(raw.Intro)+len(raw.Credits))
|
||||
for _, group := range groups {
|
||||
for _, r := range group.ranges {
|
||||
var start int64
|
||||
@@ -274,12 +276,24 @@ func parseIntroDBResponse(body []byte) ([]IntroDBSpan, error) {
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
// end == 0 表示「延续到片尾」,是合法值;其余情况 end 必须大于 start,
|
||||
// 否则这段区间没有任何可跳过的内容,直接丢弃避免在播放器里出现空按钮。
|
||||
if end != 0 && end <= start {
|
||||
// 官方规范(theintrodb.org/openapi.yaml,SegmentTimestamp)区分了两种
|
||||
// 「没有数值」的写法,含义完全不同,必须按指针区分而不能都归零:
|
||||
// end_ms: null —— Credits/Preview 专用,「一直到片尾」,合法,
|
||||
// 落成 0 作为哨兵值由客户端结合片长补齐;
|
||||
// end_ms: 0 —— 「no segment」,即这一段不存在。
|
||||
// 早期实现把两者都当成「到片尾」,于是 intro:[{start_ms:null,end_ms:0}]
|
||||
// 被解释成 [0, 片长],整集都落在「片头」区间里:一开播就弹跳过按钮,
|
||||
// 开了自动跳过更是直接跳到片尾。实测约三分之一的剧集会返回这种占位。
|
||||
if r.EndMs != nil && end <= start {
|
||||
continue
|
||||
}
|
||||
spans = append(spans, IntroDBSpan{Kind: group.kind, StartMs: start, EndMs: end})
|
||||
// 只有片尾类区间才允许「开放结束点」(end_ms: null 落成 0)。片头/回顾
|
||||
// 如果出现 null 结束点,补齐后的区间会覆盖到片尾,等于把整集判成片头,
|
||||
// 所以宁可不显示也不猜。
|
||||
if r.EndMs == nil && group.kind != "credits" && group.kind != "preview" {
|
||||
continue
|
||||
}
|
||||
spans = append(spans, SegmentSpan{Kind: group.kind, StartMs: start, EndMs: end})
|
||||
}
|
||||
}
|
||||
return spans, nil
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
// Package service — IntroDB (https://introdb.app) segment client.
|
||||
//
|
||||
// 与 TheIntroDB 是两家不同的社区库:IntroDB 只接受 IMDb id 定位媒体,
|
||||
// 用 is_movie 区分电影与剧集,响应把每种区间直接给成 {start_ms,end_ms} 对象
|
||||
// (而不是按类型分数组),片尾字段名是 outro 而不是 credits,
|
||||
// 另外还有 post_credits(片尾彩蛋,映射为预告/额外片尾)。
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// IntroDBAppBaseURL 是公开只读端点。POST /submit 需要 API key,这里不做写入。
|
||||
const IntroDBAppBaseURL = "https://api.introdb.app"
|
||||
|
||||
// IntroDBAppSource 写入 media_segments.source,与 TheIntroDB 的 "theintrodb" 区分。
|
||||
const IntroDBAppSource = "introdb"
|
||||
|
||||
const (
|
||||
introDBAppTimeout = 8 * time.Second
|
||||
introDBAppMaxBodySize = 1 << 20
|
||||
)
|
||||
|
||||
// introDBAppSegment 是 IntroDB 的单个区间对象。null 表示该类型没有数据。
|
||||
type introDBAppSegment struct {
|
||||
StartMs *int64 `json:"start_ms"`
|
||||
EndMs *int64 `json:"end_ms"`
|
||||
}
|
||||
|
||||
// introDBAppResponse 对应 GET /segments。
|
||||
type introDBAppResponse struct {
|
||||
IMDbID string `json:"imdb_id"`
|
||||
MediaType string `json:"media_type"`
|
||||
IsMovie bool `json:"is_movie"`
|
||||
Season int `json:"season"`
|
||||
Episode int `json:"episode"`
|
||||
Intro *introDBAppSegment `json:"intro"`
|
||||
Recap *introDBAppSegment `json:"recap"`
|
||||
Outro *introDBAppSegment `json:"outro"`
|
||||
PostCredits *introDBAppSegment `json:"post_credits"`
|
||||
}
|
||||
|
||||
// IntroDBAppService 查询 IntroDB 的片段数据。
|
||||
type IntroDBAppService struct {
|
||||
log *zap.Logger
|
||||
client *http.Client
|
||||
baseURL string
|
||||
}
|
||||
|
||||
// NewIntroDBAppService is the constructor. The client honours environment and
|
||||
// OS proxy settings so it behaves like the other third-party API clients.
|
||||
func NewIntroDBAppService(log *zap.Logger) *IntroDBAppService {
|
||||
return &IntroDBAppService{
|
||||
log: log,
|
||||
client: NewExternalHTTPClient(introDBAppTimeout),
|
||||
baseURL: IntroDBAppBaseURL,
|
||||
}
|
||||
}
|
||||
|
||||
// SetBaseURL overrides the API root (tests, mirrors).
|
||||
func (s *IntroDBAppService) SetBaseURL(base string) *IntroDBAppService {
|
||||
if s != nil && strings.TrimSpace(base) != "" {
|
||||
s.baseURL = strings.TrimRight(strings.TrimSpace(base), "/")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Source implements SegmentProvider.
|
||||
func (s *IntroDBAppService) Source() string { return IntroDBAppSource }
|
||||
|
||||
// FetchSegments implements SegmentProvider.
|
||||
//
|
||||
// 没有 IMDb id 时直接返回空:IntroDB 不接受 TMDb id,用别的 id 拼请求只会拿到
|
||||
// 400,那会被误记成「查询失败」从而保留旧缓存、永远不再刷新。
|
||||
func (s *IntroDBAppService) FetchSegments(ctx context.Context, q SegmentQuery) ([]SegmentSpan, error) {
|
||||
if s == nil || s.client == nil || strings.TrimSpace(q.IMDbID) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
endpoint := s.segmentsURL(q)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
// 404 = 库里没有这部片,是正常的「无数据」,不是错误。
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("introdb: unexpected status %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, introDBAppMaxBodySize))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseIntroDBAppResponse(body)
|
||||
}
|
||||
|
||||
func (s *IntroDBAppService) segmentsURL(q SegmentQuery) string {
|
||||
values := url.Values{}
|
||||
values.Set("imdb_id", strings.TrimSpace(q.IMDbID))
|
||||
if q.IsMovie {
|
||||
values.Set("is_movie", "true")
|
||||
} else if q.Season > 0 && q.Episode > 0 {
|
||||
values.Set("season", strconv.Itoa(q.Season))
|
||||
values.Set("episode", strconv.Itoa(q.Episode))
|
||||
}
|
||||
return s.baseURL + "/segments?" + values.Encode()
|
||||
}
|
||||
|
||||
// parseIntroDBAppResponse 把 IntroDB 的四个区间对象摊平成 spans。
|
||||
//
|
||||
// 顺序固定为 intro -> recap -> outro -> post_credits,与 TheIntroDB 的
|
||||
// intro/recap/credits/preview 对齐,后处理逻辑因此可以对来源无感。
|
||||
func parseIntroDBAppResponse(body []byte) ([]SegmentSpan, error) {
|
||||
var raw introDBAppResponse
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return nil, fmt.Errorf("parse introdb json: %w", err)
|
||||
}
|
||||
groups := []struct {
|
||||
kind string
|
||||
seg *introDBAppSegment
|
||||
}{
|
||||
{"intro", raw.Intro},
|
||||
{"recap", raw.Recap},
|
||||
{"credits", raw.Outro},
|
||||
{"preview", raw.PostCredits},
|
||||
}
|
||||
spans := make([]SegmentSpan, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
if group.seg == nil || group.seg.StartMs == nil || group.seg.EndMs == nil {
|
||||
continue
|
||||
}
|
||||
start, end := *group.seg.StartMs, *group.seg.EndMs
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
if end <= start {
|
||||
continue
|
||||
}
|
||||
spans = append(spans, SegmentSpan{Kind: group.kind, StartMs: start, EndMs: end})
|
||||
}
|
||||
return spans, nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user