mirror of
https://github.com/truewhile/MeBox.git
synced 2026-09-28 11:16:37 +08:00
fix: limit ffprobe concurrency
This commit is contained in:
+15
-10
@@ -64,16 +64,20 @@ type TranscoderConfig struct {
|
|||||||
|
|
||||||
// AppConfig 保存运行时应用参数。
|
// AppConfig 保存运行时应用参数。
|
||||||
type AppConfig struct {
|
type AppConfig struct {
|
||||||
Port int `mapstructure:"port"`
|
Port int `mapstructure:"port"`
|
||||||
Debug bool `mapstructure:"debug"`
|
Debug bool `mapstructure:"debug"`
|
||||||
Env string `mapstructure:"env"`
|
Env string `mapstructure:"env"`
|
||||||
DataDir string `mapstructure:"data_dir"`
|
DataDir string `mapstructure:"data_dir"`
|
||||||
WebDir string `mapstructure:"web_dir"`
|
WebDir string `mapstructure:"web_dir"`
|
||||||
FFmpegPath string `mapstructure:"ffmpeg_path"`
|
FFmpegPath string `mapstructure:"ffmpeg_path"`
|
||||||
FFprobePath string `mapstructure:"ffprobe_path"`
|
FFprobePath string `mapstructure:"ffprobe_path"`
|
||||||
VAAPIDevice string `mapstructure:"vaapi_device"`
|
// FFprobeMaxConcurrent limits concurrent ffprobe/ffmpeg metadata probes.
|
||||||
CORSOrigins []string `mapstructure:"cors_origins"`
|
// NAS devices can become unresponsive when a scan starts many probe
|
||||||
ServerURL string `mapstructure:"server_url"`
|
// processes at once, so the default is deliberately conservative.
|
||||||
|
FFprobeMaxConcurrent int `mapstructure:"ffprobe_max_concurrent"`
|
||||||
|
VAAPIDevice string `mapstructure:"vaapi_device"`
|
||||||
|
CORSOrigins []string `mapstructure:"cors_origins"`
|
||||||
|
ServerURL string `mapstructure:"server_url"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DatabaseConfig 配置 GORM + SQLite。
|
// DatabaseConfig 配置 GORM + SQLite。
|
||||||
@@ -213,6 +217,7 @@ func setDefaults(v *viper.Viper) {
|
|||||||
v.SetDefault("app.web_dir", "./web/dist")
|
v.SetDefault("app.web_dir", "./web/dist")
|
||||||
v.SetDefault("app.ffmpeg_path", "ffmpeg")
|
v.SetDefault("app.ffmpeg_path", "ffmpeg")
|
||||||
v.SetDefault("app.ffprobe_path", "ffprobe")
|
v.SetDefault("app.ffprobe_path", "ffprobe")
|
||||||
|
v.SetDefault("app.ffprobe_max_concurrent", 1)
|
||||||
v.SetDefault("app.vaapi_device", "/dev/dri/renderD128")
|
v.SetDefault("app.vaapi_device", "/dev/dri/renderD128")
|
||||||
v.SetDefault("app.cors_origins", []string{})
|
v.SetDefault("app.cors_origins", []string{})
|
||||||
v.SetDefault("app.server_url", "")
|
v.SetDefault("app.server_url", "")
|
||||||
|
|||||||
@@ -260,6 +260,9 @@ func updateSettingHandler(svc *service.Container) gin.HandlerFunc {
|
|||||||
_ = svc.Repo.DB.WithContext(c.Request.Context()).Model(&model.User{}).Where("hide_adult = ?", false).Update("hide_adult", true).Error
|
_ = svc.Repo.DB.WithContext(c.Request.Context()).Model(&model.User{}).Where("hide_adult = ?", false).Update("hide_adult", true).Error
|
||||||
}
|
}
|
||||||
service.ApplyRuntimeSetting(svc.Cfg, req.Key, req.Value)
|
service.ApplyRuntimeSetting(svc.Cfg, req.Key, req.Value)
|
||||||
|
if svc.FFprobe != nil && (req.Key == "ffprobe.max_concurrent" || req.Key == "app.ffprobe_max_concurrent") {
|
||||||
|
svc.FFprobe.SetMaxConcurrent(svc.Cfg.App.FFprobeMaxConcurrent)
|
||||||
|
}
|
||||||
if req.Key == "transcode.enabled" && !svc.Cfg.Transcoder.Enabled {
|
if req.Key == "transcode.enabled" && !svc.Cfg.Transcoder.Enabled {
|
||||||
svc.Transcoder.StopAll()
|
svc.Transcoder.StopAll()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ func schemaHandler(_ *service.Container) gin.HandlerFunc {
|
|||||||
{"key": "transcode.idle_timeout_seconds", "type": "number", "label": "转码空闲停止秒数"},
|
{"key": "transcode.idle_timeout_seconds", "type": "number", "label": "转码空闲停止秒数"},
|
||||||
{"key": "ffmpeg.path", "type": "text", "label": "FFmpeg 路径"},
|
{"key": "ffmpeg.path", "type": "text", "label": "FFmpeg 路径"},
|
||||||
{"key": "ffprobe.path", "type": "text", "label": "FFprobe 路径"},
|
{"key": "ffprobe.path", "type": "text", "label": "FFprobe 路径"},
|
||||||
|
{"key": "ffprobe.max_concurrent", "type": "number", "label": "FFprobe 最大并发"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
@@ -25,13 +26,35 @@ import (
|
|||||||
|
|
||||||
// FFprobeService wraps the external ffprobe binary.
|
// FFprobeService wraps the external ffprobe binary.
|
||||||
type FFprobeService struct {
|
type FFprobeService struct {
|
||||||
cfg *config.Config
|
cfg *config.Config
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
|
mu sync.RWMutex
|
||||||
|
limiter chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFFprobeService is the constructor.
|
// NewFFprobeService is the constructor.
|
||||||
func NewFFprobeService(cfg *config.Config, log *zap.Logger) *FFprobeService {
|
func NewFFprobeService(cfg *config.Config, log *zap.Logger) *FFprobeService {
|
||||||
return &FFprobeService{cfg: cfg, log: log}
|
maxConcurrent := normalizeFFprobeMaxConcurrent(cfg.App.FFprobeMaxConcurrent)
|
||||||
|
return &FFprobeService{cfg: cfg, log: log, limiter: make(chan struct{}, maxConcurrent)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeFFprobeMaxConcurrent(n int) int {
|
||||||
|
if n <= 0 {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if n > 8 {
|
||||||
|
return 8
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FFprobeService) SetMaxConcurrent(n int) {
|
||||||
|
if f == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.limiter = make(chan struct{}, normalizeFFprobeMaxConcurrent(n))
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProbeResult is the subset of ffprobe output consumed by the scanner.
|
// ProbeResult is the subset of ffprobe output consumed by the scanner.
|
||||||
@@ -50,6 +73,11 @@ func (f *FFprobeService) Probe(ctx context.Context, path string) (*ProbeResult,
|
|||||||
if f == nil {
|
if f == nil {
|
||||||
return nil, errors.New("ffprobe service nil")
|
return nil, errors.New("ffprobe service nil")
|
||||||
}
|
}
|
||||||
|
token, err := f.acquire(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer f.release(token)
|
||||||
if bin, err := resolveLocalExecutable(f.cfg.App.FFprobePath, "ffprobe"); err == nil {
|
if bin, err := resolveLocalExecutable(f.cfg.App.FFprobePath, "ffprobe"); err == nil {
|
||||||
f.cfg.App.FFprobePath = bin
|
f.cfg.App.FFprobePath = bin
|
||||||
probeCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
probeCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
@@ -73,6 +101,31 @@ func (f *FFprobeService) Probe(ctx context.Context, path string) (*ProbeResult,
|
|||||||
return f.probeWithFFmpeg(ctx, path)
|
return f.probeWithFFmpeg(ctx, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *FFprobeService) acquire(ctx context.Context) (chan struct{}, error) {
|
||||||
|
f.mu.RLock()
|
||||||
|
limiter := f.limiter
|
||||||
|
f.mu.RUnlock()
|
||||||
|
if limiter == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case limiter <- struct{}{}:
|
||||||
|
return limiter, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *FFprobeService) release(limiter chan struct{}) {
|
||||||
|
if limiter == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-limiter:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (f *FFprobeService) probeWithFFmpeg(ctx context.Context, path string) (*ProbeResult, error) {
|
func (f *FFprobeService) probeWithFFmpeg(ctx context.Context, path string) (*ProbeResult, error) {
|
||||||
bin, err := resolveLocalExecutable(f.cfg.App.FFmpegPath, "ffmpeg")
|
bin, err := resolveLocalExecutable(f.cfg.App.FFmpegPath, "ffmpeg")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,24 +1,88 @@
|
|||||||
package service
|
package service
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
func TestParseFFmpegProbeText(t *testing.T) {
|
"go.uber.org/zap"
|
||||||
text := `Input #0, matroska,webm, from 'show.mkv':
|
|
||||||
Duration: 00:23:42.11, start: 0.000000, bitrate: 5132 kb/s
|
|
||||||
Stream #0:0: Video: h264 (Main), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 23.98 fps
|
|
||||||
Stream #0:1(jpn): Audio: eac3, 48000 Hz, stereo, fltp, 128 kb/s (default)`
|
|
||||||
|
|
||||||
got := parseFFmpegProbeText(text)
|
"github.com/ShukeBta/MediaStationGo/internal/config"
|
||||||
if got.Container != "matroska,webm" {
|
)
|
||||||
t.Fatalf("container = %q", got.Container)
|
|
||||||
}
|
func TestFFprobeServiceDefaultsToSingleConcurrentProbe(t *testing.T) {
|
||||||
if got.DurationSec != 1422 {
|
svc := NewFFprobeService(&config.Config{}, zap.NewNop())
|
||||||
t.Fatalf("duration = %d", got.DurationSec)
|
if got := cap(svc.limiter); got != 1 {
|
||||||
}
|
t.Fatalf("limiter capacity = %d, want 1", got)
|
||||||
if got.VideoCodec != "h264" || got.Width != 1920 || got.Height != 1080 {
|
}
|
||||||
t.Fatalf("video = %#v", got)
|
}
|
||||||
}
|
|
||||||
if got.AudioCodec != "eac3" {
|
func TestFFprobeServiceClampsConfiguredConcurrency(t *testing.T) {
|
||||||
t.Fatalf("audio = %q", got.AudioCodec)
|
cfg := &config.Config{}
|
||||||
|
cfg.App.FFprobeMaxConcurrent = 3
|
||||||
|
svc := NewFFprobeService(cfg, zap.NewNop())
|
||||||
|
if got := cap(svc.limiter); got != 3 {
|
||||||
|
t.Fatalf("limiter capacity = %d, want 3", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.App.FFprobeMaxConcurrent = 99
|
||||||
|
svc = NewFFprobeService(cfg, zap.NewNop())
|
||||||
|
if got := cap(svc.limiter); got != 8 {
|
||||||
|
t.Fatalf("limiter capacity = %d, want max clamp 8", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeAcquireHonorsContextWhenLimitReached(t *testing.T) {
|
||||||
|
svc := &FFprobeService{limiter: make(chan struct{}, 1)}
|
||||||
|
token, err := svc.acquire(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first acquire: %v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := svc.acquire(ctx); err == nil {
|
||||||
|
t.Fatal("second acquire should block until context deadline")
|
||||||
|
}
|
||||||
|
svc.release(token)
|
||||||
|
token, err = svc.acquire(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("acquire after release: %v", err)
|
||||||
|
}
|
||||||
|
svc.release(token)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFFprobeSetMaxConcurrentHotSwapsLimiter(t *testing.T) {
|
||||||
|
svc := NewFFprobeService(&config.Config{}, zap.NewNop())
|
||||||
|
firstToken, err := svc.acquire(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first acquire: %v", err)
|
||||||
|
}
|
||||||
|
svc.SetMaxConcurrent(2)
|
||||||
|
secondToken, err := svc.acquire(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("acquire after resize: %v", err)
|
||||||
|
}
|
||||||
|
thirdToken, err := svc.acquire(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second acquire after resize: %v", err)
|
||||||
|
}
|
||||||
|
svc.release(firstToken)
|
||||||
|
svc.release(secondToken)
|
||||||
|
svc.release(thirdToken)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRuntimeSettingFFprobeMaxConcurrent(t *testing.T) {
|
||||||
|
cfg := &config.Config{}
|
||||||
|
ApplyRuntimeSetting(cfg, "ffprobe.max_concurrent", "4")
|
||||||
|
if cfg.App.FFprobeMaxConcurrent != 4 {
|
||||||
|
t.Fatalf("FFprobeMaxConcurrent = %d, want 4", cfg.App.FFprobeMaxConcurrent)
|
||||||
|
}
|
||||||
|
ApplyRuntimeSetting(cfg, "ffprobe.max_concurrent", "99")
|
||||||
|
if cfg.App.FFprobeMaxConcurrent != 8 {
|
||||||
|
t.Fatalf("FFprobeMaxConcurrent = %d, want clamp 8", cfg.App.FFprobeMaxConcurrent)
|
||||||
|
}
|
||||||
|
ApplyRuntimeSetting(cfg, "ffprobe.max_concurrent", "0")
|
||||||
|
if cfg.App.FFprobeMaxConcurrent != 1 {
|
||||||
|
t.Fatalf("FFprobeMaxConcurrent = %d, want clamp 1", cfg.App.FFprobeMaxConcurrent)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,6 +37,16 @@ func ApplyRuntimeSetting(cfg *config.Config, key, value string) {
|
|||||||
cfg.App.FFmpegPath = value
|
cfg.App.FFmpegPath = value
|
||||||
case "ffprobe.path", "app.ffprobe_path":
|
case "ffprobe.path", "app.ffprobe_path":
|
||||||
cfg.App.FFprobePath = value
|
cfg.App.FFprobePath = value
|
||||||
|
case "ffprobe.max_concurrent", "app.ffprobe_max_concurrent":
|
||||||
|
if n, err := strconv.Atoi(value); err == nil {
|
||||||
|
if n < 1 {
|
||||||
|
n = 1
|
||||||
|
}
|
||||||
|
if n > 8 {
|
||||||
|
n = 8
|
||||||
|
}
|
||||||
|
cfg.App.FFprobeMaxConcurrent = n
|
||||||
|
}
|
||||||
case "transcode.enabled", "transcoder.enabled":
|
case "transcode.enabled", "transcoder.enabled":
|
||||||
cfg.Transcoder.Enabled = parseBoolSetting(value, true)
|
cfg.Transcoder.Enabled = parseBoolSetting(value, true)
|
||||||
case "transcode.hw_enabled", "transcoder.hardware_accel":
|
case "transcode.hw_enabled", "transcoder.hardware_accel":
|
||||||
|
|||||||
@@ -121,6 +121,13 @@ const GROUPS: SettingGroup[] = [
|
|||||||
type: 'text',
|
type: 'text',
|
||||||
placeholder: 'ffprobe',
|
placeholder: 'ffprobe',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: 'ffprobe.max_concurrent',
|
||||||
|
label: 'FFprobe 最大并发',
|
||||||
|
type: 'number',
|
||||||
|
hint: 'NAS 建议 1;用于扫描、整理洗版和手动探测,避免同时启动多个 ffprobe 进程',
|
||||||
|
defaultValue: '1',
|
||||||
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user