mirror of
https://github.com/truewhile/MeBox.git
synced 2026-10-13 00:26:37 +08:00
perf(strm): 优化 OpenList 元数据下载速度 (#15)
* Add favourite buttons to library list and series detail pages Co-authored-by: truewhile <truewhile@users.noreply.github.com> * Fix favourites list for remote Emby mounted media Co-authored-by: truewhile <truewhile@users.noreply.github.com> * Route favourites to library series detail when appropriate Co-authored-by: truewhile <truewhile@users.noreply.github.com> * Fix favourites link to library series for remote anime paths Co-authored-by: truewhile <truewhile@users.noreply.github.com> * Fix remote Emby playback history and continue watching - Preserve duration_ms when web player reports duration=0 - Add GET /playback/:id/resume for reliable per-item resume - Switch home continue watching to /watch-history/continue API - Set library_id on remote media for play-profile visibility - Keep continue-watching rows when remote metadata hydration fails Co-authored-by: truewhile <truewhile@users.noreply.github.com> * Fix mobile sort dropdown overflowing off-screen Align dropdown to button left edge on small screens and cap width to viewport so sort options stay fully visible on mobile. Co-authored-by: truewhile <truewhile@users.noreply.github.com> * Remove stale Windows binary backup files Co-authored-by: truewhile <truewhile@users.noreply.github.com> * Fix STRM account edit form not echoing saved config Return config_preview from account API for server/url/username and secret flags. Merge partial config updates so empty password/token fields do not wipe stored credentials. Co-authored-by: truewhile <truewhile@users.noreply.github.com> * perf(strm): speed up OpenList metadata downloads with provider-aware concurrency - Split download semaphore: 115 stays capped at 3 for WAF safety; OpenList/CloudDrive2 WebDAV metadata uses strm.download_threads (default raised to 6, max 16) - Scope 115 WAF cooldown to 115 tasks only so OpenList downloads are not paused - Requeue claimed tasks back to pending when slot acquisition fails or 115 is cooling - Add tests for separate semaphores and requeue behavior Co-authored-by: truewhile <truewhile@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: truewhile <truewhile@users.noreply.github.com>
This commit is contained in:
truewhile
@@ -68,7 +68,7 @@ var StrmSettingDefs = map[string]struct {
|
||||
StrmSettingUploadMeta: {Default: "false", Label: "上传元数据", Kind: "bool", Help: "同步时把本地元数据上传到远端（需网盘支持写入）"},
|
||||
StrmSettingDeleteDir: {Default: "false", Label: "清理空目录", Kind: "bool", Help: "清理远端已删除的多余 .strm/元数据后，删除空目录"},
|
||||
Strm115RelayKeySetting: {Default: "", Label: "115 中继授权共享密钥", Kind: "text", Help: "QMediaSync/MQFamily 中继授权的共享 AES 密钥（OAUTH_RELAY_ENCRYPTION_KEY）；不配置则中继授权不可用"},
|
||||
StrmSettingDownloadThreads: {Default: "3", Label: "下载队列线程数", Kind: "number", Help: "元数据下载并发数"},
|
||||
StrmSettingDownloadThreads: {Default: "6", Label: "下载队列线程数", Kind: "number", Help: "OpenList/CloudDrive2 元数据下载并发数（115 独立限速为 3）"},
|
||||
StrmSettingUploadThreads: {Default: "2", Label: "上传队列线程数", Kind: "number", Help: "元数据上传并发数"},
|
||||
}
|
||||
|
||||
@@ -91,33 +91,51 @@ type StrmService struct {
|
||||
oauthSessions map[string]*strm115AuthSession
|
||||
wafUntil time.Time // 115 风控/限流熔断截止时间（由 mu 保护）
|
||||
|
||||
downloadSem chan struct{} // 全局下载并发信号量：限制整个进程同时进行「换直链+下载」的并发数
|
||||
downloadSem115 chan struct{} // 115 换直链+下载并发上限（风控兜底）
|
||||
downloadSemDAV chan struct{} // WebDAV/OpenList/CloudDrive2 元数据下载并发上限
|
||||
downloadSemOnce sync.Once
|
||||
}
|
||||
|
||||
// strmWAFCooldown 检测到 115 风控/限流后下载队列的全局冷却时长。
|
||||
// strmWAFCooldown 检测到 115 风控/限流后 115 下载任务的全局冷却时长。
|
||||
const strmWAFCooldown = 3 * time.Minute
|
||||
|
||||
// strmDownloadSemCap 全局同时进行「换直链+下载」的并发上限。
|
||||
// strm115DownloadSemCap 115 同时进行「换直链+下载」的并发上限。
|
||||
//
|
||||
// 115 对换直链接口（/open/ufile/downurl）风控极严：过去把全局 QPS 提到 8 或让多
|
||||
// worker 高并发换链，会瞬时撞上 WAF 返回 405 阻断页并触发 180 秒冷却，反而更慢。
|
||||
// 因此用信号量把整个进程同时换直链的并发数压到 3，与令牌桶限速共同兜底：
|
||||
// 宁可下载稍慢，也绝不触发风控。下载本身走 CDN 不限速。
|
||||
const strmDownloadSemCap = 3
|
||||
// 因此把 115 换直链并发压到 3，与令牌桶限速共同兜底；OpenList/CloudDrive2 元数据
|
||||
// 走 WebDAV，不受此限制，并发由 strm.download_threads 控制。
|
||||
const strm115DownloadSemCap = 3
|
||||
|
||||
// ensureDownloadSem 惰性初始化全局共享的下载并发信号量。
|
||||
func (s *StrmService) ensureDownloadSem() {
|
||||
// initDownloadSems 按设置初始化 115 与 WebDAV 两套下载并发信号量。
|
||||
func (s *StrmService) initDownloadSems(davCap int) {
|
||||
s.downloadSemOnce.Do(func() {
|
||||
s.downloadSem = make(chan struct{}, strmDownloadSemCap)
|
||||
if davCap < 1 {
|
||||
davCap = 1
|
||||
}
|
||||
if davCap > 16 {
|
||||
davCap = 16
|
||||
}
|
||||
s.downloadSem115 = make(chan struct{}, strm115DownloadSemCap)
|
||||
s.downloadSemDAV = make(chan struct{}, davCap)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *StrmService) downloadSemForProvider(provider string) chan struct{} {
|
||||
if provider == model.StrmProvider115 {
|
||||
return s.downloadSem115
|
||||
}
|
||||
return s.downloadSemDAV
|
||||
}
|
||||
|
||||
// acquireDownloadSlot 获取一个下载并发槽位（等待/取消安全）。
|
||||
func (s *StrmService) acquireDownloadSlot(ctx context.Context) bool {
|
||||
s.ensureDownloadSem()
|
||||
func (s *StrmService) acquireDownloadSlot(ctx context.Context, provider string) bool {
|
||||
sem := s.downloadSemForProvider(provider)
|
||||
if sem == nil {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case s.downloadSem <- struct{}{}:
|
||||
case sem <- struct{}{}:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
@@ -125,11 +143,12 @@ func (s *StrmService) acquireDownloadSlot(ctx context.Context) bool {
|
||||
}
|
||||
|
||||
// releaseDownloadSlot 释放一个下载并发槽位。
|
||||
func (s *StrmService) releaseDownloadSlot() {
|
||||
if s.downloadSem == nil {
|
||||
func (s *StrmService) releaseDownloadSlot(provider string) {
|
||||
sem := s.downloadSemForProvider(provider)
|
||||
if sem == nil {
|
||||
return
|
||||
}
|
||||
<-s.downloadSem
|
||||
<-sem
|
||||
}
|
||||
|
||||
// NewStrmService constructs the STRM service.
|
||||
@@ -151,13 +170,14 @@ func NewStrmService(cfg *config.Config, log *zap.Logger, repos *repository.Conta
|
||||
func (s *StrmService) Start(ctx context.Context) {
|
||||
s.sync115RelayKey(ctx)
|
||||
s.recoverInterruptedSyncs(ctx)
|
||||
downloadThreads := s.strmIntSetting(ctx, StrmSettingDownloadThreads, 3)
|
||||
downloadThreads := s.strmIntSetting(ctx, StrmSettingDownloadThreads, 6)
|
||||
if downloadThreads < 1 {
|
||||
downloadThreads = 1
|
||||
}
|
||||
if downloadThreads > 8 {
|
||||
downloadThreads = 8
|
||||
if downloadThreads > 16 {
|
||||
downloadThreads = 16
|
||||
}
|
||||
s.initDownloadSems(downloadThreads)
|
||||
uploadThreads := s.strmIntSetting(ctx, StrmSettingUploadThreads, 2)
|
||||
if uploadThreads < 1 {
|
||||
uploadThreads = 1
|
||||
@@ -237,9 +257,9 @@ func (s *StrmService) strmAccountConfig(acct *model.StrmAccount) (map[string]str
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// mergeEmbyRemoteConfig 对远程 Emby 账号配置做合并式更新：config 中出现的键
|
||||
// 覆盖写入（敏感键按明文加密），未出现的键保留原密文；显式空字符串=清除。
|
||||
func (s *StrmService) mergeEmbyRemoteConfig(existing string, config map[string]string) (string, error) {
|
||||
// mergeStrmAccountConfig 对账号配置做合并式更新：config 中出现的键覆盖写入（敏感键按明文
|
||||
// 加密），未出现的键保留原密文；显式空字符串=清除。
|
||||
func (s *StrmService) mergeStrmAccountConfig(existing string, config map[string]string) (string, error) {
|
||||
out := map[string]string{}
|
||||
if strings.TrimSpace(existing) != "" {
|
||||
if err := json.Unmarshal([]byte(existing), &out); err != nil {
|
||||
@@ -264,6 +284,44 @@ func (s *StrmService) mergeEmbyRemoteConfig(existing string, config map[string]s
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// StrmAccountConfigPreview 返回可安全回显给前端的非敏感配置字段。
|
||||
type StrmAccountConfigPreview struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
Server string `json:"server,omitempty"`
|
||||
Username string `json:"username,omitempty"`
|
||||
HasPassword bool `json:"has_password,omitempty"`
|
||||
HasToken bool `json:"has_token,omitempty"`
|
||||
HasAPIKey bool `json:"has_api_key,omitempty"`
|
||||
}
|
||||
|
||||
// StrmAccountConfigPreviewOf 从账号配置提取可回显字段（不含密码/令牌明文）。
|
||||
func (s *StrmService) StrmAccountConfigPreviewOf(acct *model.StrmAccount) StrmAccountConfigPreview {
|
||||
var out StrmAccountConfigPreview
|
||||
if acct == nil || strings.TrimSpace(acct.Config) == "" {
|
||||
return out
|
||||
}
|
||||
raw := map[string]string{}
|
||||
if err := json.Unmarshal([]byte(acct.Config), &raw); err != nil {
|
||||
return out
|
||||
}
|
||||
out.URL = strings.TrimSpace(raw["url"])
|
||||
out.Server = strings.TrimSpace(raw["server"])
|
||||
out.Username = strings.TrimSpace(raw["username"])
|
||||
for _, k := range []string{"password", "token", "api_key"} {
|
||||
if v, ok := raw[k]; ok && strings.TrimSpace(v) != "" {
|
||||
switch k {
|
||||
case "password":
|
||||
out.HasPassword = true
|
||||
case "token":
|
||||
out.HasToken = true
|
||||
case "api_key":
|
||||
out.HasAPIKey = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HasStrmAccountCredential 报告账号是否已配置核心凭据（用于前端展示）。
|
||||
func HasStrmAccountCredential(acct *model.StrmAccount) bool {
|
||||
switch acct.Provider {
|
||||
@@ -320,22 +378,13 @@ func (s *StrmService) UpdateStrmAccount(ctx context.Context, id, name string, en
|
||||
if enabled != nil {
|
||||
acct.Enabled = *enabled
|
||||
}
|
||||
if len(config) > 0 {
|
||||
var enc string
|
||||
var err error
|
||||
if acct.Provider == model.StrmProviderEmbyRemote {
|
||||
// 远程 Emby 账号：合并式更新。config 中出现的键覆盖（敏感键按明文
|
||||
// 加密写入），未出现的键保留原密文——避免编辑「代理开关」时把已
|
||||
// 保存的地址与凭据清空。
|
||||
enc, err = s.mergeEmbyRemoteConfig(acct.Config, config)
|
||||
} else {
|
||||
enc, err = s.strmAccountConfigJSON(config, true)
|
||||
if len(config) > 0 {
|
||||
enc, err := s.mergeStrmAccountConfig(acct.Config, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
acct.Config = enc
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
acct.Config = enc
|
||||
}
|
||||
if err := s.repo.StrmAccount.Update(ctx, acct); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user