mirror of
https://github.com/truewhile/MeBox.git
synced 2026-09-29 11:36:36 +08:00
fix organizer diagnostics and site integrations
This commit is contained in:
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/ShukeBta/MediaStationGo/internal/config"
|
||||
)
|
||||
|
||||
func TestProductionLoggerSplitsWarnAndErrorAndDropsInfo(t *testing.T) {
|
||||
func TestProductionLoggerWritesConfiguredInfoToAppLogAndSplitsWarnError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.App.DataDir = dir
|
||||
@@ -26,11 +26,15 @@ func TestProductionLoggerSplitsWarnAndErrorAndDropsInfo(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log.Info("info should be dropped")
|
||||
log.Info("info should be stored")
|
||||
log.Warn("warning only", zap.String("kind", "warn"))
|
||||
log.Error("error only", zap.String("kind", "error"))
|
||||
_ = log.Sync()
|
||||
|
||||
appBytes, err := os.ReadFile(filepath.Join(dir, "logs", "app.log"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
warnBytes, err := os.ReadFile(filepath.Join(dir, "logs", "warn.log"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -39,10 +43,16 @@ func TestProductionLoggerSplitsWarnAndErrorAndDropsInfo(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appLog := string(appBytes)
|
||||
warnLog := string(warnBytes)
|
||||
errorLog := string(errorBytes)
|
||||
if strings.Contains(warnLog, "info should be dropped") || strings.Contains(errorLog, "info should be dropped") {
|
||||
t.Fatal("info log should not be written in production")
|
||||
if !strings.Contains(appLog, "info should be stored") ||
|
||||
!strings.Contains(appLog, "warning only") ||
|
||||
!strings.Contains(appLog, "error only") {
|
||||
t.Fatalf("app log should contain all enabled levels: %s", appLog)
|
||||
}
|
||||
if strings.Contains(warnLog, "info should be stored") || strings.Contains(errorLog, "info should be stored") {
|
||||
t.Fatal("split warn/error logs should not contain info")
|
||||
}
|
||||
if !strings.Contains(warnLog, "warning only") || strings.Contains(warnLog, "error only") {
|
||||
t.Fatalf("warn log not isolated: %s", warnLog)
|
||||
@@ -52,6 +62,35 @@ func TestProductionLoggerSplitsWarnAndErrorAndDropsInfo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionLoggerDefaultsToWarnInAppLog(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfg := &config.Config{}
|
||||
cfg.App.DataDir = dir
|
||||
cfg.Logging.Format = "json"
|
||||
cfg.Logging.OutputPath = filepath.Join(dir, "logs")
|
||||
cfg.Logging.EnableRotation = true
|
||||
|
||||
log, err := newLogger(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log.Info("info should stay quiet by default")
|
||||
log.Warn("warning should be stored")
|
||||
_ = log.Sync()
|
||||
|
||||
appBytes, err := os.ReadFile(filepath.Join(dir, "logs", "app.log"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appLog := string(appBytes)
|
||||
if strings.Contains(appLog, "info should stay quiet by default") {
|
||||
t.Fatalf("default logger should not store info: %s", appLog)
|
||||
}
|
||||
if !strings.Contains(appLog, "warning should be stored") {
|
||||
t.Fatalf("default logger should store warn: %s", appLog)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRotatingFileWriterCapsFileSize(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "app.log")
|
||||
writer, err := newRotatingFileWriter(path, config.LoggingConfig{
|
||||
|
||||
+15
-14
@@ -329,15 +329,12 @@ func isFrontendLibraryRoute(path string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// newLogger 根据 cfg.Logging 构建 Zap。此前 logging.level / logging.format
|
||||
// 配置完全没有生效(固定 NewProduction),用户无法在生产环境降低日志量;
|
||||
// 配合每请求一条 INFO 访问日志,几小时即可产生几十 MB 日志,在 Docker
|
||||
// json-file 驱动下持续消耗磁盘 IO。
|
||||
// newLogger 根据 cfg.Logging 构建 Zap。
|
||||
func newLogger(cfg *config.Config) (*zap.Logger, error) {
|
||||
if cfg.App.Debug {
|
||||
return zap.NewDevelopment()
|
||||
}
|
||||
level := productionLogLevel(cfg.Logging.Level)
|
||||
level := configuredLogLevel(cfg.Logging.Level)
|
||||
encoderCfg := zap.NewProductionEncoderConfig()
|
||||
encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||
var encoder zapcore.Encoder
|
||||
@@ -349,7 +346,14 @@ func newLogger(cfg *config.Config) (*zap.Logger, error) {
|
||||
cores := []zapcore.Core{
|
||||
zapcore.NewCore(encoder, zapcore.Lock(os.Stdout), level),
|
||||
}
|
||||
warnPath, errorPath := logFilePaths(cfg)
|
||||
appPath, warnPath, errorPath := logFilePaths(cfg)
|
||||
if appPath != "" {
|
||||
appWriter, err := newRotatingFileWriter(appPath, cfg.Logging)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cores = append(cores, zapcore.NewCore(encoder, appWriter, level))
|
||||
}
|
||||
if warnPath != "" {
|
||||
warnWriter, err := newRotatingFileWriter(warnPath, cfg.Logging)
|
||||
if err != nil {
|
||||
@@ -371,7 +375,7 @@ func newLogger(cfg *config.Config) (*zap.Logger, error) {
|
||||
return zap.New(zapcore.NewTee(cores...), zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel), zap.ErrorOutput(zapcore.Lock(os.Stderr))), nil
|
||||
}
|
||||
|
||||
func productionLogLevel(raw string) zapcore.Level {
|
||||
func configuredLogLevel(raw string) zapcore.Level {
|
||||
level := zapcore.WarnLevel
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw != "" {
|
||||
@@ -380,25 +384,22 @@ func productionLogLevel(raw string) zapcore.Level {
|
||||
level = parsed
|
||||
}
|
||||
}
|
||||
if level < zapcore.WarnLevel {
|
||||
return zapcore.WarnLevel
|
||||
}
|
||||
return level
|
||||
}
|
||||
|
||||
func logFilePaths(cfg *config.Config) (string, string) {
|
||||
func logFilePaths(cfg *config.Config) (string, string, string) {
|
||||
out := strings.TrimSpace(cfg.Logging.OutputPath)
|
||||
if strings.EqualFold(out, "stdout") || strings.EqualFold(out, "stderr") {
|
||||
return "", ""
|
||||
return "", "", ""
|
||||
}
|
||||
if out == "" {
|
||||
out = filepath.Join(cfg.App.DataDir, "logs")
|
||||
}
|
||||
if ext := filepath.Ext(out); ext != "" {
|
||||
base := strings.TrimSuffix(out, ext)
|
||||
return base + ".warn" + ext, base + ".error" + ext
|
||||
return out, base + ".warn" + ext, base + ".error" + ext
|
||||
}
|
||||
return filepath.Join(out, "warn.log"), filepath.Join(out, "error.log")
|
||||
return filepath.Join(out, "app.log"), filepath.Join(out, "warn.log"), filepath.Join(out, "error.log")
|
||||
}
|
||||
|
||||
// getLocalIP returns the first non-loopback IPv4 address of the machine.
|
||||
|
||||
+2
-2
@@ -49,9 +49,9 @@ secrets:
|
||||
fanart_tv_api_key: ""
|
||||
|
||||
logging:
|
||||
level: warn # production clamps file/stdout logs to warn or above
|
||||
level: warn # warn by default; set info/debug temporarily for detailed diagnosis
|
||||
format: console # console | json
|
||||
output_path: "" # empty = {data_dir}/logs; use stdout/stderr to disable file logs
|
||||
output_path: "" # empty = {data_dir}/logs/app.log + warn.log + error.log; stdout/stderr disables file logs
|
||||
enable_rotation: true
|
||||
max_size_mb: 20
|
||||
max_age_days: 30
|
||||
|
||||
@@ -77,6 +77,11 @@ services:
|
||||
MEDIASTATION_APP_PORT: 8080
|
||||
MEDIASTATION_APP_WEB_DIR: /app/web/dist
|
||||
MEDIASTATION_APP_DATA_DIR: /data
|
||||
# 需要排查订阅/站点搜索/下载器问题时,临时把 level 改成 info 或 debug。
|
||||
# 详细应用日志会保存在 ./data/logs/app.log,warn/error 也会拆分保存。
|
||||
MEDIASTATION_LOGGING_LEVEL: warn
|
||||
MEDIASTATION_LOGGING_FORMAT: console
|
||||
MEDIASTATION_LOGGING_OUTPUT_PATH: /data/logs
|
||||
|
||||
# 轻量模式默认只使用 PostgreSQL,适合大多数 NAS。
|
||||
# 旧版 ./data/mediastation.db 存在时,首次启动会自动迁移到 PostgreSQL。
|
||||
|
||||
+20
-1
@@ -321,7 +321,11 @@ func ApplySiteAuthHeaders(req *http.Request, site *model.Site) {
|
||||
}
|
||||
case "api_key":
|
||||
if site.APIKey != "" {
|
||||
req.Header.Set("x-api-key", site.APIKey)
|
||||
if isYemaPTSite(site) {
|
||||
req.Header.Set("Authorization", site.APIKey)
|
||||
} else {
|
||||
req.Header.Set("x-api-key", site.APIKey)
|
||||
}
|
||||
}
|
||||
case "auth_header":
|
||||
if site.AuthHeader != "" {
|
||||
@@ -335,6 +339,21 @@ func ApplySiteAuthHeaders(req *http.Request, site *model.Site) {
|
||||
}
|
||||
}
|
||||
|
||||
func isYemaPTSite(site *model.Site) bool {
|
||||
if site == nil {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(site.Type), "yemapt") {
|
||||
return true
|
||||
}
|
||||
u, err := url.Parse(strings.TrimSpace(site.URL))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org")
|
||||
}
|
||||
|
||||
// GetPageSource fetches a page with browser-like headers.
|
||||
// Returns (pageSource, cookies, error).
|
||||
func GetPageSource(url string, site *model.Site, timeout int, log *zap.Logger) (string, string, error) {
|
||||
|
||||
+20
-20
@@ -8,41 +8,41 @@ import (
|
||||
// Site PT 站点配置。
|
||||
type Site struct {
|
||||
Base
|
||||
Name string `gorm:"size:128;not null" json:"name"`
|
||||
Type string `gorm:"size:32;not null" json:"type"` // nexusphp / gazelle / unit3d / mteam / discuz / custom_rss
|
||||
URL string `gorm:"size:512;not null" json:"url"`
|
||||
AuthType string `gorm:"size:32;not null" json:"auth_type"` // cookie / api_key / auth_header
|
||||
Cookie string `gorm:"type:text" json:"-"` // AES 加密
|
||||
APIKey string `gorm:"type:text" json:"-"` // AES 加密
|
||||
AuthHeader string `gorm:"type:text" json:"-"` // AES 加密
|
||||
Name string `gorm:"size:128;not null" json:"name"`
|
||||
Type string `gorm:"size:32;not null" json:"type"` // nexusphp / gazelle / unit3d / mteam / yemapt / discuz / custom_rss
|
||||
URL string `gorm:"size:512;not null" json:"url"`
|
||||
AuthType string `gorm:"size:32;not null" json:"auth_type"` // cookie / api_key / auth_header
|
||||
Cookie string `gorm:"type:text" json:"-"` // AES 加密
|
||||
APIKey string `gorm:"type:text" json:"-"` // AES 加密
|
||||
AuthHeader string `gorm:"type:text" json:"-"` // AES 加密
|
||||
|
||||
// ── 高级设置 ──────────────────────────────────────────────────────────
|
||||
UserAgent string `gorm:"size:500" json:"user_agent"` // 自定义 User-Agent
|
||||
RSSURL string `gorm:"size:1000" json:"rss_url"` // RSS 订阅地址
|
||||
Timeout int `gorm:"default:15" json:"timeout"` // 请求超时(秒), 0=不限制
|
||||
Priority int `gorm:"default:50" json:"priority"` // 优先级, 越小越优先
|
||||
UseProxy bool `gorm:"default:false" json:"use_proxy"` // 是否使用代理
|
||||
RateLimit bool `gorm:"default:false" json:"rate_limit"` // 是否限制访问频率
|
||||
BrowserEmulation bool `gorm:"default:false" json:"browser_emulation"` // 浏览器仿真(防爬)
|
||||
UserAgent string `gorm:"size:500" json:"user_agent"` // 自定义 User-Agent
|
||||
RSSURL string `gorm:"size:1000" json:"rss_url"` // RSS 订阅地址
|
||||
Timeout int `gorm:"default:15" json:"timeout"` // 请求超时(秒), 0=不限制
|
||||
Priority int `gorm:"default:50" json:"priority"` // 优先级, 越小越优先
|
||||
UseProxy bool `gorm:"default:false" json:"use_proxy"` // 是否使用代理
|
||||
RateLimit bool `gorm:"default:false" json:"rate_limit"` // 是否限制访问频率
|
||||
BrowserEmulation bool `gorm:"default:false" json:"browser_emulation"` // 浏览器仿真(防爬)
|
||||
|
||||
// ── 状态与统计 ────────────────────────────────────────────────────────
|
||||
LoginStatus string `gorm:"size:20;default:unknown" json:"login_status"` // unknown / ok / fail
|
||||
UploadBytes int64 `gorm:"default:0" json:"upload_bytes"` // 上传字节统计
|
||||
DownloadBytes int64 `gorm:"default:0" json:"download_bytes"` // 下载字节统计
|
||||
LoginStatus string `gorm:"size:20;default:unknown" json:"login_status"` // unknown / ok / fail
|
||||
UploadBytes int64 `gorm:"default:0" json:"upload_bytes"` // 上传字节统计
|
||||
DownloadBytes int64 `gorm:"default:0" json:"download_bytes"` // 下载字节统计
|
||||
|
||||
// ── 关联下载器 ────────────────────────────────────────────────────────
|
||||
Downloader string `gorm:"size:50" json:"downloader"` // qbittorrent / transmission / aria2
|
||||
Downloader string `gorm:"size:50" json:"downloader"` // qbittorrent / transmission / aria2
|
||||
|
||||
Enabled bool `gorm:"default:true" json:"enabled"`
|
||||
IsDefault bool `gorm:"default:false" json:"is_default"`
|
||||
Extra string `gorm:"type:text" json:"-"` // JSON 扩展配置, AES 加密
|
||||
Extra string `gorm:"type:text" json:"-"` // JSON 扩展配置, AES 加密
|
||||
LastError string `gorm:"size:1024" json:"last_error"`
|
||||
LastCheckAt *time.Time `json:"last_check_at"`
|
||||
}
|
||||
|
||||
// SiteType 返回支持的站点类型列表。
|
||||
func SiteTypes() []string {
|
||||
return []string{"nexusphp", "gazelle", "unit3d", "mteam", "discuz", "custom_rss"}
|
||||
return []string{"nexusphp", "gazelle", "unit3d", "mteam", "yemapt", "discuz", "custom_rss"}
|
||||
}
|
||||
|
||||
// AuthTypes 返回支持的认证方式列表。
|
||||
|
||||
@@ -66,6 +66,7 @@ func (d *DiscoverService) TMDbSection(ctx context.Context, key string) ([]Extern
|
||||
Source: "tmdb",
|
||||
MediaType: mediaType,
|
||||
Title: item.Title,
|
||||
OriginalName: item.OriginalName,
|
||||
Overview: item.Overview,
|
||||
PosterURL: item.PosterURL,
|
||||
BackdropURL: item.BackdropURL,
|
||||
@@ -106,15 +107,17 @@ func (d *DiscoverService) Fetch(ctx context.Context, path string) ([]Match, erro
|
||||
u := base + path + "?" + q.Encode()
|
||||
|
||||
type result struct {
|
||||
ID int `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Name string `json:"name"`
|
||||
Overview string `json:"overview"`
|
||||
PosterPath string `json:"poster_path"`
|
||||
BackdropPath string `json:"backdrop_path"`
|
||||
ReleaseDate string `json:"release_date"`
|
||||
FirstAirDate string `json:"first_air_date"`
|
||||
VoteAverage float32 `json:"vote_average"`
|
||||
ID int `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Name string `json:"name"`
|
||||
OriginalTitle string `json:"original_title"`
|
||||
OriginalName string `json:"original_name"`
|
||||
Overview string `json:"overview"`
|
||||
PosterPath string `json:"poster_path"`
|
||||
BackdropPath string `json:"backdrop_path"`
|
||||
ReleaseDate string `json:"release_date"`
|
||||
FirstAirDate string `json:"first_air_date"`
|
||||
VoteAverage float32 `json:"vote_average"`
|
||||
}
|
||||
type page struct {
|
||||
Results []result `json:"results"`
|
||||
@@ -143,10 +146,11 @@ func (d *DiscoverService) Fetch(ctx context.Context, path string) ([]Match, erro
|
||||
title = r.Name
|
||||
}
|
||||
m := Match{
|
||||
TMDbID: r.ID,
|
||||
Title: title,
|
||||
Overview: r.Overview,
|
||||
Rating: r.VoteAverage,
|
||||
TMDbID: r.ID,
|
||||
Title: title,
|
||||
OriginalName: firstNonEmpty(r.OriginalTitle, r.OriginalName),
|
||||
Overview: r.Overview,
|
||||
Rating: r.VoteAverage,
|
||||
}
|
||||
if r.PosterPath != "" {
|
||||
m.PosterURL = d.tmdb.imgCDN + "/w500" + r.PosterPath
|
||||
@@ -309,6 +313,7 @@ func (b *BangumiProvider) Calendar(ctx context.Context) ([]ExternalMediaResult,
|
||||
Source: "bangumi",
|
||||
MediaType: "anime",
|
||||
Title: title,
|
||||
OriginalName: item.Name,
|
||||
Overview: item.Summary,
|
||||
PosterURL: poster,
|
||||
Year: year,
|
||||
|
||||
@@ -3,11 +3,13 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
"github.com/ShukeBta/MediaStationGo/internal/repository"
|
||||
)
|
||||
|
||||
@@ -93,11 +95,23 @@ func (p *OrganizePipelineService) Run(ctx context.Context, req OrganizePipelineR
|
||||
res, path, err := p.runOrganize(ctx, req, opts)
|
||||
if err != nil {
|
||||
p.finishTask(task, err, "organize", p.failureMessage(req), nil)
|
||||
p.recordProblem(ctx, req, nil, err)
|
||||
return nil, err
|
||||
}
|
||||
response.Path = path
|
||||
response.Result = res
|
||||
|
||||
if fatalErr := organizeFatalResultError(res); fatalErr != nil {
|
||||
p.logOrganizeProblem(req, res, fatalErr)
|
||||
p.finishTask(task, fatalErr, "organize", p.failureMessage(req), res)
|
||||
p.recordProblem(ctx, req, res, fatalErr)
|
||||
return nil, fatalErr
|
||||
}
|
||||
if res != nil && len(res.Errors) > 0 {
|
||||
p.logOrganizeProblem(req, res, nil)
|
||||
p.recordProblem(ctx, req, res, nil)
|
||||
}
|
||||
|
||||
if task != nil {
|
||||
task.Update(TaskUpdate{
|
||||
Stage: "organize",
|
||||
@@ -154,6 +168,122 @@ func (p *OrganizePipelineService) Run(ctx context.Context, req OrganizePipelineR
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func organizeFatalResultError(res *OrganizeResult) error {
|
||||
if res == nil || len(res.Errors) == 0 {
|
||||
return nil
|
||||
}
|
||||
if res.Organized > 0 || res.Replaced > 0 {
|
||||
return nil
|
||||
}
|
||||
samples := organizeErrorSamples(res.Errors, 3)
|
||||
detail := strings.Join(samples, "; ")
|
||||
if detail == "" {
|
||||
detail = "unknown error"
|
||||
}
|
||||
return fmt.Errorf("organize failed: %d error(s), organized=0 replaced=0 skipped=%d: %s", len(res.Errors), res.Skipped, detail)
|
||||
}
|
||||
|
||||
func organizeErrorSamples(errors []string, limit int) []string {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, limit)
|
||||
for _, line := range errors {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *OrganizePipelineService) logOrganizeProblem(req OrganizePipelineRequest, res *OrganizeResult, err error) {
|
||||
if p == nil || p.log == nil || res == nil {
|
||||
return
|
||||
}
|
||||
fields := []zap.Field{
|
||||
zap.String("trigger", string(req.Trigger)),
|
||||
zap.String("scope", string(req.Scope)),
|
||||
zap.String("source", res.SourcePath),
|
||||
zap.String("dest", res.DestPath),
|
||||
zap.Int("organized", res.Organized),
|
||||
zap.Int("replaced", res.Replaced),
|
||||
zap.Int("skipped", res.Skipped),
|
||||
zap.Int("errors", len(res.Errors)),
|
||||
zap.Strings("error_samples", organizeErrorSamples(res.Errors, 5)),
|
||||
}
|
||||
if err != nil {
|
||||
fields = append(fields, zap.Error(err))
|
||||
}
|
||||
p.log.Warn("organize pipeline completed with errors", fields...)
|
||||
}
|
||||
|
||||
func (p *OrganizePipelineService) recordProblem(ctx context.Context, req OrganizePipelineRequest, res *OrganizeResult, err error) {
|
||||
if p == nil || p.repo == nil || p.repo.Log == nil {
|
||||
return
|
||||
}
|
||||
action := "organize.warning"
|
||||
if err != nil {
|
||||
action = "organize.failed"
|
||||
}
|
||||
target := strings.TrimSpace(req.SourcePath)
|
||||
detail := organizeAuditDetail(req, res, err)
|
||||
if res != nil {
|
||||
if strings.TrimSpace(res.SourcePath) != "" {
|
||||
target = res.SourcePath
|
||||
}
|
||||
} else if target == "" {
|
||||
target = firstNonEmpty(req.MediaID, req.LibraryID, req.DestPath)
|
||||
}
|
||||
row := &model.AccessLog{
|
||||
Action: action,
|
||||
Target: truncateForAccessLog(target, 255),
|
||||
Detail: truncateForAccessLog(detail, 4000),
|
||||
}
|
||||
if writeErr := p.repo.Log.Create(ctx, row); writeErr != nil && p.log != nil {
|
||||
p.log.Debug("organize audit log write failed", zap.Error(writeErr))
|
||||
}
|
||||
}
|
||||
|
||||
func organizeAuditDetail(req OrganizePipelineRequest, res *OrganizeResult, err error) string {
|
||||
parts := []string{
|
||||
"trigger=" + string(req.Trigger),
|
||||
"scope=" + string(req.Scope),
|
||||
}
|
||||
if res != nil {
|
||||
parts = append(parts,
|
||||
fmt.Sprintf("source=%s", res.SourcePath),
|
||||
fmt.Sprintf("dest=%s", res.DestPath),
|
||||
fmt.Sprintf("organized=%d", res.Organized),
|
||||
fmt.Sprintf("replaced=%d", res.Replaced),
|
||||
fmt.Sprintf("skipped=%d", res.Skipped),
|
||||
fmt.Sprintf("errors=%d", len(res.Errors)),
|
||||
)
|
||||
if samples := organizeErrorSamples(res.Errors, 5); len(samples) > 0 {
|
||||
parts = append(parts, "error_samples="+strings.Join(samples, " | "))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
parts = append(parts, "error="+err.Error())
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
func truncateForAccessLog(value string, limit int) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if limit <= 0 || len(value) <= limit {
|
||||
return value
|
||||
}
|
||||
if limit <= 3 {
|
||||
return value[:limit]
|
||||
}
|
||||
return value[:limit-3] + "..."
|
||||
}
|
||||
|
||||
func (p *OrganizePipelineService) runOrganize(ctx context.Context, req OrganizePipelineRequest, opts OrganizeOptions) (*OrganizeResult, string, error) {
|
||||
switch req.Scope {
|
||||
case OrganizeScopeMedia:
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func TestOrganizeScanRootUsesActualOrganizedTarget(t *testing.T) {
|
||||
@@ -35,3 +39,49 @@ func TestOrganizeScanRootUsesCommonAffectedCategoryRoot(t *testing.T) {
|
||||
t.Fatalf("organizeScanRoot() = %q, want %q", got, root)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrganizePipelineFailsWhenEveryOrganizeItemErrors(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "downloads")
|
||||
dest := filepath.Join(root, "media")
|
||||
writeOrgFile(t, filepath.Join(src, "Dune 2021 1080p.mkv"), "movie")
|
||||
|
||||
origLinkFile := linkFile
|
||||
linkFile = func(_, _ string) error {
|
||||
return errors.New("cross-device link")
|
||||
}
|
||||
t.Cleanup(func() { linkFile = origLinkFile })
|
||||
|
||||
repos := newOrganizerTestRepo(t)
|
||||
organizer := NewOrganizerService(nil, zap.NewNop(), repos)
|
||||
tasks := NewTaskTrackerService(zap.NewNop(), nil)
|
||||
pipeline := NewOrganizePipelineService(zap.NewNop(), repos, organizer, nil, tasks)
|
||||
|
||||
_, err := pipeline.Run(t.Context(), OrganizePipelineRequest{
|
||||
Scope: OrganizeScopeDirectory,
|
||||
Trigger: OrganizeTriggerScheduled,
|
||||
SourcePath: src,
|
||||
DestPath: dest,
|
||||
TransferMode: string(TransferHardlink),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected organize pipeline to fail when every item errors")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "organize failed") || !strings.Contains(err.Error(), "hardlink failed") {
|
||||
t.Fatalf("error = %q, want organize failure with hardlink detail", err.Error())
|
||||
}
|
||||
snap := tasks.Snapshot()
|
||||
if len(snap.Recent) != 1 {
|
||||
t.Fatalf("recent tasks = %d, want 1", len(snap.Recent))
|
||||
}
|
||||
task := snap.Recent[0]
|
||||
if task.Status != TaskStatusFailed {
|
||||
t.Fatalf("task status = %q, want failed", task.Status)
|
||||
}
|
||||
if !strings.Contains(task.Error, "hardlink failed") {
|
||||
t.Fatalf("task error = %q, want hardlink detail", task.Error)
|
||||
}
|
||||
if len(task.Details) == 0 || !strings.Contains(task.Details[0], "cross-device link") {
|
||||
t.Fatalf("task details = %#v, want transfer error detail", task.Details)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -638,6 +638,9 @@ func (o *OrganizerService) isSmartClassifyEnabled(ctx context.Context) bool {
|
||||
}
|
||||
}
|
||||
// Fallback to config.yaml
|
||||
if o == nil || o.cfg == nil {
|
||||
return false
|
||||
}
|
||||
return o.cfg.Organizer.SmartClassify
|
||||
}
|
||||
|
||||
|
||||
@@ -147,30 +147,14 @@ func (o *OrganizerService) OrganizeDirectory(ctx context.Context, opts OrganizeO
|
||||
if skipped, reason := shouldSkipOrganizeSourceVideo(source, filepath.Dir(source)); skipped {
|
||||
res.Skipped++
|
||||
res.Items = append(res.Items, OrganizePreviewItem{Source: source, Action: "skip", Reason: reason})
|
||||
o.log.Info("organize file finished",
|
||||
zap.String("source", source),
|
||||
zap.String("dest", dest),
|
||||
zap.String("mode", string(mode)),
|
||||
zap.Int("organized", res.Organized),
|
||||
zap.Int("replaced", res.Replaced),
|
||||
zap.Int("skipped", res.Skipped),
|
||||
zap.Any("skip_reasons", OrganizeSkipReasonCounts(res)),
|
||||
)
|
||||
o.logOrganizeDirectoryResult("organize file finished", res, mode)
|
||||
return res, nil
|
||||
}
|
||||
if err := o.organizeSourceFile(ctx, source, filepath.Dir(source), dest, mode, opts.MediaType, opts.MediaCategory, opts.DryRun, opts.AllowReplaceExisting, metadataCache, res); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("%s: %s", filepath.Base(source), err.Error()))
|
||||
res.Items = append(res.Items, OrganizePreviewItem{Source: source, Action: "error", Reason: err.Error()})
|
||||
}
|
||||
o.log.Info("organize file finished",
|
||||
zap.String("source", source),
|
||||
zap.String("dest", dest),
|
||||
zap.String("mode", string(mode)),
|
||||
zap.Int("organized", res.Organized),
|
||||
zap.Int("replaced", res.Replaced),
|
||||
zap.Int("skipped", res.Skipped),
|
||||
zap.Any("skip_reasons", OrganizeSkipReasonCounts(res)),
|
||||
)
|
||||
o.logOrganizeDirectoryResult("organize file finished", res, mode)
|
||||
return res, nil
|
||||
}
|
||||
walkErr := walk(source, func(path string, wi walkInfo) error {
|
||||
@@ -195,16 +179,30 @@ func (o *OrganizerService) OrganizeDirectory(ctx context.Context, opts OrganizeO
|
||||
if walkErr != nil {
|
||||
return res, walkErr
|
||||
}
|
||||
o.log.Info("organize directory finished",
|
||||
zap.String("source", source),
|
||||
zap.String("dest", dest),
|
||||
o.logOrganizeDirectoryResult("organize directory finished", res, mode)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (o *OrganizerService) logOrganizeDirectoryResult(message string, res *OrganizeResult, mode TransferMode) {
|
||||
if o == nil || o.log == nil || res == nil {
|
||||
return
|
||||
}
|
||||
fields := []zap.Field{
|
||||
zap.String("source", res.SourcePath),
|
||||
zap.String("dest", res.DestPath),
|
||||
zap.String("mode", string(mode)),
|
||||
zap.Int("organized", res.Organized),
|
||||
zap.Int("replaced", res.Replaced),
|
||||
zap.Int("skipped", res.Skipped),
|
||||
zap.Int("errors", len(res.Errors)),
|
||||
zap.Any("skip_reasons", OrganizeSkipReasonCounts(res)),
|
||||
)
|
||||
return res, nil
|
||||
}
|
||||
if len(res.Errors) > 0 {
|
||||
fields = append(fields, zap.Strings("error_samples", organizeErrorSamples(res.Errors, 5)))
|
||||
o.log.Warn(message, fields...)
|
||||
return
|
||||
}
|
||||
o.log.Info(message, fields...)
|
||||
}
|
||||
|
||||
func ensureOrganizeDestinationWritable(dest string) error {
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
// SchedulerService runs five recurring background jobs that keep the
|
||||
// library up-to-date without operator intervention:
|
||||
//
|
||||
// library_scan every 60 min — re-scan every enabled library so
|
||||
// newly-copied files are picked up.
|
||||
// library_scan every 24 h — optional full re-scan for local libraries;
|
||||
// filesystem watchers handle normal changes.
|
||||
// subscription_pull every 30 min — re-poll RSS feeds (in addition to
|
||||
// the existing SubscriptionService
|
||||
// internal timer).
|
||||
@@ -87,6 +87,7 @@ const (
|
||||
cloudAutoSyncEnabledKey = "cloud.auto_sync_enabled"
|
||||
cloudSyncIntervalSecondsKey = "cloud.sync_interval_seconds"
|
||||
cloudLastAutoSyncDateKey = "cloud.last_auto_sync_date"
|
||||
localLastPeriodicScanDateKey = "scan.last_periodic_date"
|
||||
cloudAutoSyncWindowStartHour = 23
|
||||
cloudAutoSyncWindowEndHour = 5
|
||||
cloudAutoSyncCompletedDateForm = "2006-01-02"
|
||||
@@ -122,7 +123,7 @@ func (s *SchedulerService) Start(ctx context.Context) {
|
||||
s.jobs = []*scheduledJob{
|
||||
{
|
||||
name: "library_scan",
|
||||
interval: 60 * time.Minute,
|
||||
interval: 24 * time.Hour,
|
||||
run: s.jobScanLibraries,
|
||||
},
|
||||
{
|
||||
@@ -337,7 +338,9 @@ func (s *SchedulerService) runReserved(ctx context.Context, j *scheduledJob) err
|
||||
// 仅当用户在设置中显式开启 scan.periodic_enabled 时才执行整库重扫,
|
||||
// 避免对硬盘的高频反复读取造成损伤(用户明确要求)。
|
||||
func (s *SchedulerService) jobScanLibraries(ctx context.Context) error {
|
||||
if !s.periodicScanEnabled(ctx) {
|
||||
manual, _ := ctx.Value(schedulerManualRunKey{}).(bool)
|
||||
now := s.currentTime()
|
||||
if !manual && !s.periodicScanDue(ctx, now) {
|
||||
return nil
|
||||
}
|
||||
libs, err := s.repo.Library.List(ctx)
|
||||
@@ -360,6 +363,9 @@ func (s *SchedulerService) jobScanLibraries(ctx context.Context) error {
|
||||
zap.String("library", l.ID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
if !manual {
|
||||
_ = s.markPeriodicScanCompleted(ctx, now)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -567,6 +573,27 @@ func (s *SchedulerService) periodicScanEnabled(ctx context.Context) bool {
|
||||
return parseBoolSetting(v, false)
|
||||
}
|
||||
|
||||
func (s *SchedulerService) periodicScanDue(ctx context.Context, now time.Time) bool {
|
||||
if !s.periodicScanEnabled(ctx) {
|
||||
return false
|
||||
}
|
||||
if s.repo == nil || s.repo.Setting == nil {
|
||||
return true
|
||||
}
|
||||
last, err := s.repo.Setting.Get(ctx, localLastPeriodicScanDateKey)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
return strings.TrimSpace(last) != now.In(time.Local).Format(cloudAutoSyncCompletedDateForm)
|
||||
}
|
||||
|
||||
func (s *SchedulerService) markPeriodicScanCompleted(ctx context.Context, now time.Time) error {
|
||||
if s.repo == nil || s.repo.Setting == nil {
|
||||
return nil
|
||||
}
|
||||
return s.repo.Setting.Set(ctx, localLastPeriodicScanDateKey, now.In(time.Local).Format(cloudAutoSyncCompletedDateForm))
|
||||
}
|
||||
|
||||
// jobOrganizeSource periodically organizes the configured staging/download
|
||||
// source directory into the configured media destination. It is intentionally
|
||||
// opt-in: manual file management remains available, but background disk walking
|
||||
|
||||
@@ -429,6 +429,103 @@ func TestSchedulerRunNowCloudSyncBypassesNightlyWindow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerPeriodicLocalScanRunsAtMostOncePerDay(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
libraryPath := filepath.Join(root, "library")
|
||||
writeOrgFile(t, filepath.Join(libraryPath, "Daily.Show.S01E01.mkv"), "episode 1")
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.Setting{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
if err := repos.Setting.Set(t.Context(), "scan.periodic_enabled", "true"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lib := model.Library{Name: "本地剧集", Path: libraryPath, Type: "tv", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log := zap.NewNop()
|
||||
scanner := NewScannerService(&config.Config{}, log, repos, NewHub(log), nil, nil)
|
||||
scheduler := NewSchedulerService(log, repos, scanner, nil, nil, nil, NewHub(log), "")
|
||||
scheduler.now = func() time.Time {
|
||||
return time.Date(2026, 6, 20, 10, 0, 0, 0, time.Local)
|
||||
}
|
||||
|
||||
if err := scheduler.jobScanLibraries(t.Context()); err != nil {
|
||||
t.Fatalf("first periodic local scan: %v", err)
|
||||
}
|
||||
if got := countMedia(t, repos); got != 1 {
|
||||
t.Fatalf("media count after first scan = %d, want 1", got)
|
||||
}
|
||||
|
||||
writeOrgFile(t, filepath.Join(libraryPath, "Daily.Show.S01E02.mkv"), "episode 2")
|
||||
if err := scheduler.jobScanLibraries(t.Context()); err != nil {
|
||||
t.Fatalf("same-day periodic local scan: %v", err)
|
||||
}
|
||||
if got := countMedia(t, repos); got != 1 {
|
||||
t.Fatalf("same-day periodic scan should not import new file, media count = %d", got)
|
||||
}
|
||||
|
||||
scheduler.now = func() time.Time {
|
||||
return time.Date(2026, 6, 21, 10, 0, 0, 0, time.Local)
|
||||
}
|
||||
if err := scheduler.jobScanLibraries(t.Context()); err != nil {
|
||||
t.Fatalf("next-day periodic local scan: %v", err)
|
||||
}
|
||||
if got := countMedia(t, repos); got != 2 {
|
||||
t.Fatalf("media count after next-day scan = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerManualLocalScanBypassesDailyPeriodicLimit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
libraryPath := filepath.Join(root, "library")
|
||||
writeOrgFile(t, filepath.Join(libraryPath, "Manual.Show.S01E01.mkv"), "episode 1")
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Library{}, &model.Media{}, &model.Setting{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
if err := repos.Setting.Set(t.Context(), "scan.periodic_enabled", "true"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lib := model.Library{Name: "手动本地剧集", Path: libraryPath, Type: "tv", Enabled: true}
|
||||
if err := repos.Library.Create(t.Context(), &lib); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log := zap.NewNop()
|
||||
scanner := NewScannerService(&config.Config{}, log, repos, NewHub(log), nil, nil)
|
||||
scheduler := NewSchedulerService(log, repos, scanner, nil, nil, nil, NewHub(log), "")
|
||||
scheduler.now = func() time.Time {
|
||||
return time.Date(2026, 6, 20, 10, 0, 0, 0, time.Local)
|
||||
}
|
||||
scheduler.jobs = []*scheduledJob{{
|
||||
name: "library_scan",
|
||||
interval: 24 * time.Hour,
|
||||
run: scheduler.jobScanLibraries,
|
||||
}}
|
||||
|
||||
if err := scheduler.jobScanLibraries(t.Context()); err != nil {
|
||||
t.Fatalf("first periodic local scan: %v", err)
|
||||
}
|
||||
writeOrgFile(t, filepath.Join(libraryPath, "Manual.Show.S01E02.mkv"), "episode 2")
|
||||
if err := scheduler.RunNow(t.Context(), "library_scan"); err != nil {
|
||||
t.Fatalf("manual local scan: %v", err)
|
||||
}
|
||||
if got := countMedia(t, repos); got != 2 {
|
||||
t.Fatalf("manual scan should bypass daily periodic limit, media count = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerCloudSyncDisabledByDefault(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -33,6 +33,7 @@ type SiteService struct {
|
||||
log *zap.Logger
|
||||
repo *repository.Container
|
||||
flareSolverrURL string
|
||||
apiRateLimiter siteAPIRateLimiter
|
||||
}
|
||||
|
||||
// ResolveDownloadURL converts tracker-specific search result URLs into a URL
|
||||
@@ -70,15 +71,34 @@ func (s *SiteService) ResolveDownloadURL(ctx context.Context, raw string) string
|
||||
defer cancel()
|
||||
resolved, err := adapter.GetDownloadURL(resolveCtx, cfg, id)
|
||||
if err != nil || resolved == "" {
|
||||
s.log.Warn("resolve PT download URL failed",
|
||||
zap.String("site", matched.Name),
|
||||
zap.String("raw", raw),
|
||||
zap.Error(err))
|
||||
if s.log != nil {
|
||||
s.log.Warn("resolve PT download URL failed",
|
||||
zap.String("site", matched.Name),
|
||||
zap.String("raw", redactSensitiveDownloadURL(raw)),
|
||||
zap.Error(err))
|
||||
}
|
||||
return raw
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func redactSensitiveDownloadURL(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(strings.ToLower(raw), "magnet:") {
|
||||
return "magnet:?xt=***"
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return "[redacted-download-url]"
|
||||
}
|
||||
u.RawQuery = ""
|
||||
u.Fragment = ""
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (s *SiteService) FetchTorrentFile(ctx context.Context, raw string) ([]byte, string, error) {
|
||||
matched := s.matchSiteForURL(ctx, raw)
|
||||
if matched == nil {
|
||||
@@ -175,7 +195,12 @@ func torrentFilename(rawURL, disposition string) string {
|
||||
|
||||
// NewSiteService is the constructor.
|
||||
func NewSiteService(log *zap.Logger, repo *repository.Container, flareSolverrURL string) *SiteService {
|
||||
return &SiteService{log: log, repo: repo, flareSolverrURL: flareSolverrURL}
|
||||
return &SiteService{
|
||||
log: log,
|
||||
repo: repo,
|
||||
flareSolverrURL: flareSolverrURL,
|
||||
apiRateLimiter: newPersistentSiteAPIRateLimiter(repo),
|
||||
}
|
||||
}
|
||||
|
||||
// Create persists a new site.
|
||||
@@ -308,7 +333,7 @@ func (s *SiteService) TestConnection(ctx context.Context, id string) (bool, stri
|
||||
}).Error
|
||||
return true, "连接成功", nil
|
||||
} else {
|
||||
if site.Type == "mteam" {
|
||||
if site.Type == "mteam" || site.Type == "yemapt" || isYemaPTURL(site.URL) {
|
||||
s.log.Warn("site adapter authenticate failed",
|
||||
zap.String("site", site.Name),
|
||||
zap.String("type", site.Type),
|
||||
@@ -364,6 +389,7 @@ type SearchResult struct {
|
||||
SiteName string `json:"site_name"`
|
||||
SiteID string `json:"site_id"`
|
||||
Title string `json:"title"`
|
||||
Subtitle string `json:"subtitle,omitempty"`
|
||||
TorrentURL string `json:"torrent_url"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
Category string `json:"category,omitempty"`
|
||||
@@ -386,15 +412,18 @@ func (s *SiteService) Search(ctx context.Context, keyword string) ([]SearchResul
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
results []SearchResult
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
enabledCount int
|
||||
failedCount int
|
||||
results []SearchResult
|
||||
)
|
||||
|
||||
for i := range sites {
|
||||
if !sites[i].Enabled {
|
||||
continue
|
||||
}
|
||||
enabledCount++
|
||||
wg.Add(1)
|
||||
go func(site model.Site) {
|
||||
defer wg.Done()
|
||||
@@ -416,10 +445,15 @@ func (s *SiteService) Search(ctx context.Context, keyword string) ([]SearchResul
|
||||
|
||||
result, err := adapter.Search(ctxWithTimeout, cfg, keyword, 1)
|
||||
if err != nil {
|
||||
mu.Lock()
|
||||
failedCount++
|
||||
mu.Unlock()
|
||||
s.log.Warn("site search failed",
|
||||
zap.String("site", site.Name),
|
||||
zap.String("type", site.Type),
|
||||
zap.String("url", site.URL),
|
||||
zap.String("keyword", keyword),
|
||||
zap.Duration("timeout", timeout),
|
||||
zap.Error(err))
|
||||
return
|
||||
}
|
||||
@@ -436,6 +470,7 @@ func (s *SiteService) Search(ctx context.Context, keyword string) ([]SearchResul
|
||||
SiteName: site.Name,
|
||||
SiteID: site.ID,
|
||||
Title: item.Title,
|
||||
Subtitle: item.Subtitle,
|
||||
TorrentURL: item.DetailURL,
|
||||
DownloadURL: item.DownloadURL,
|
||||
Category: item.Category,
|
||||
@@ -459,6 +494,13 @@ func (s *SiteService) Search(ctx context.Context, keyword string) ([]SearchResul
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].Seeders > results[j].Seeders
|
||||
})
|
||||
if s.log != nil {
|
||||
s.log.Info("site search completed",
|
||||
zap.String("keyword", keyword),
|
||||
zap.Int("enabled_sites", enabledCount),
|
||||
zap.Int("failed_sites", failedCount),
|
||||
zap.Int("results_count", len(results)))
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
@@ -486,6 +528,7 @@ func (svc *SiteService) siteModelToConfig(s *model.Site) SiteConfig {
|
||||
}
|
||||
|
||||
return SiteConfig{
|
||||
SiteID: s.ID,
|
||||
Name: s.Name,
|
||||
Type: s.Type,
|
||||
URL: s.URL,
|
||||
@@ -498,5 +541,7 @@ func (svc *SiteService) siteModelToConfig(s *model.Site) SiteConfig {
|
||||
Extra: extra,
|
||||
FlareSolverrURL: flareSolverrURL,
|
||||
UseProxy: s.UseProxy,
|
||||
RateLimit: s.RateLimit,
|
||||
rateLimiter: svc.apiRateLimiter,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
// SiteConfig 站点配置(从 model.Site 解密后的纯文本)。
|
||||
type SiteConfig struct {
|
||||
SiteID string
|
||||
Name string
|
||||
Type string
|
||||
URL string
|
||||
@@ -29,6 +30,8 @@ type SiteConfig struct {
|
||||
Extra map[string]string // JSON 扩展配置
|
||||
FlareSolverrURL string // FlareSolverr 服务地址(用于浏览器模拟绕过 Cloudflare/WAF)
|
||||
UseProxy bool // 通过 HTTP(S)_PROXY 环境变量出站
|
||||
RateLimit bool
|
||||
rateLimiter siteAPIRateLimiter
|
||||
}
|
||||
|
||||
// SiteSearchResult 站点搜索结果(按站点分组的批量搜索结果)。
|
||||
@@ -119,11 +122,13 @@ func buildRequest(ctx context.Context, method, rawURL string, cfg SiteConfig, bo
|
||||
req.Header.Set("Cookie", cfg.Cookie)
|
||||
}
|
||||
case "api_key":
|
||||
// 与旧版参考实现的 ApplySiteAuthHeaders 对齐:
|
||||
// M-Team / UNIT3D 等开放 API 的 PT 站点都使用 `x-api-key` 头部,
|
||||
// 不要再为 mteam 单独走 Authorization: Bearer,否则服务端会 401。
|
||||
if cfg.APIKey != "" {
|
||||
req.Header.Set("x-api-key", cfg.APIKey)
|
||||
if isYemaPTConfig(cfg) {
|
||||
req.Header.Set("Authorization", cfg.APIKey)
|
||||
} else {
|
||||
// M-Team / UNIT3D 等开放 API 的 PT 站点使用 `x-api-key`。
|
||||
req.Header.Set("x-api-key", cfg.APIKey)
|
||||
}
|
||||
}
|
||||
case "auth_header":
|
||||
if cfg.AuthHeader != "" {
|
||||
@@ -278,6 +283,8 @@ func GetAdapterForType(siteType string) SiteAdapter {
|
||||
return NewUNIT3DAdapter()
|
||||
case "mteam":
|
||||
return NewMTeamAdapter()
|
||||
case "yemapt":
|
||||
return NewYemaPTAdapter()
|
||||
case "discuz":
|
||||
return NewDiscuzAdapter()
|
||||
case "custom_rss":
|
||||
@@ -289,5 +296,11 @@ func GetAdapterForType(siteType string) SiteAdapter {
|
||||
|
||||
// NewSiteAdapter 根据站点模型创建对应的适配器。
|
||||
func NewSiteAdapter(site *model.Site) SiteAdapter {
|
||||
if site == nil {
|
||||
return nil
|
||||
}
|
||||
if isYemaPTURL(site.URL) {
|
||||
return NewYemaPTAdapter()
|
||||
}
|
||||
return GetAdapterForType(site.Type)
|
||||
}
|
||||
|
||||
@@ -33,6 +33,9 @@ func (a *MTeamAdapter) Authenticate(ctx context.Context, cfg SiteConfig) error {
|
||||
// 与旧版参考实现对齐:
|
||||
// 用 camelCase 参数(pageNumber / pageSize),同时接受 code 为字符串 "0"
|
||||
// 或数值 0;兼容 M-Team v3 API 不同版本的返回。
|
||||
if err := reserveMTeamAPIQuota(ctx, cfg, mteamAPIEndpointSearch); err != nil {
|
||||
return fmt.Errorf("authenticate: %w", err)
|
||||
}
|
||||
u := cfg.URL + "/api/torrent/search"
|
||||
payload := `{"pageNumber":1,"pageSize":1,"mode":"all"}`
|
||||
data, status, err := doRequestJSON(ctx, a.client, "POST", u, cfg, []byte(payload))
|
||||
@@ -78,6 +81,9 @@ func (a *MTeamAdapter) Search(ctx context.Context, cfg SiteConfig, keyword strin
|
||||
}
|
||||
body, _ := json.Marshal(payload)
|
||||
|
||||
if err := reserveMTeamAPIQuota(ctx, cfg, mteamAPIEndpointSearch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u := cfg.URL + "/api/torrent/search"
|
||||
data, status, err := doRequestJSON(ctx, a.client, "POST", u, cfg, body)
|
||||
if err != nil {
|
||||
@@ -104,6 +110,9 @@ func (a *MTeamAdapter) Browse(ctx context.Context, cfg SiteConfig, category stri
|
||||
}
|
||||
body, _ := json.Marshal(payload)
|
||||
|
||||
if err := reserveMTeamAPIQuota(ctx, cfg, mteamAPIEndpointSearch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u := cfg.URL + "/api/torrent/search"
|
||||
data, status, err := doRequestJSON(ctx, a.client, "POST", u, cfg, body)
|
||||
if err != nil {
|
||||
@@ -117,6 +126,9 @@ func (a *MTeamAdapter) Browse(ctx context.Context, cfg SiteConfig, category stri
|
||||
}
|
||||
|
||||
func (a *MTeamAdapter) GetDetail(ctx context.Context, cfg SiteConfig, id string) (*TorrentDetail, error) {
|
||||
if err := reserveMTeamAPIQuota(ctx, cfg, mteamAPIEndpointDetail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u := cfg.URL + "/api/torrent/detail?id=" + url.QueryEscape(id)
|
||||
data, status, err := doRequestJSON(ctx, a.client, "POST", u, cfg, nil)
|
||||
if err != nil {
|
||||
@@ -184,6 +196,9 @@ func (a *MTeamAdapter) GetDetail(ctx context.Context, cfg SiteConfig, id string)
|
||||
// 拿到的 sign URL 可被任何下载客户端无认证地直接 GET。这是旧版参考实现
|
||||
// _download_torrent_file 方法的子集。
|
||||
func (a *MTeamAdapter) GetDownloadURL(ctx context.Context, cfg SiteConfig, id string) (string, error) {
|
||||
if err := reserveMTeamAPIQuota(ctx, cfg, mteamAPIEndpointDownload); err != nil {
|
||||
return "", err
|
||||
}
|
||||
u := cfg.URL + "/api/torrent/genDlToken?id=" + id
|
||||
// genDlToken 是 POST 但参数走 query string;body 留空。
|
||||
data, status, err := doRequestJSON(ctx, a.client, "POST", u, cfg, []byte("{}"))
|
||||
|
||||
@@ -3,11 +3,19 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/model"
|
||||
"github.com/ShukeBta/MediaStationGo/internal/repository"
|
||||
)
|
||||
|
||||
func TestMTeamAuthenticateRequiresAPIKey(t *testing.T) {
|
||||
@@ -75,3 +83,204 @@ func TestMTeamAuthenticateReportsAPIMessage(t *testing.T) {
|
||||
t.Fatalf("Authenticate error = %v, want key invalid message", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYemaPTAuthenticateUsesAuthorizationHeader(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotAuth string
|
||||
var gotXAPIKey string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
gotXAPIKey = r.Header.Get("x-api-key")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":true,"showType":0,"data":{"id":10,"name":"tester"}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
adapter := NewYemaPTAdapter()
|
||||
err := adapter.Authenticate(context.Background(), SiteConfig{
|
||||
Type: "yemapt",
|
||||
URL: server.URL,
|
||||
AuthType: "api_key",
|
||||
APIKey: "auth-123",
|
||||
Timeout: 5 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Authenticate returned error: %v", err)
|
||||
}
|
||||
if gotPath != "/openApi/user/fetchBasicInfo.json" {
|
||||
t.Fatalf("path = %q, want /openApi/user/fetchBasicInfo.json", gotPath)
|
||||
}
|
||||
if gotAuth != "auth-123" {
|
||||
t.Fatalf("Authorization = %q, want auth-123", gotAuth)
|
||||
}
|
||||
if gotXAPIKey != "" {
|
||||
t.Fatalf("x-api-key = %q, want empty", gotXAPIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYemaPTAuthenticateReportsAPIMessage(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":false,"errorCode":403,"errorMessage":"need api auth"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
adapter := NewYemaPTAdapter()
|
||||
err := adapter.Authenticate(context.Background(), SiteConfig{
|
||||
Type: "yemapt",
|
||||
URL: server.URL,
|
||||
AuthType: "api_key",
|
||||
APIKey: "bad-auth",
|
||||
Timeout: 5 * time.Second,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "need api auth") {
|
||||
t.Fatalf("Authenticate error = %v, want need api auth", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSiteAdapterDetectsYemaPTURL(t *testing.T) {
|
||||
adapter := NewSiteAdapter(&model.Site{
|
||||
Type: "nexusphp",
|
||||
URL: "https://www.yemapt.org",
|
||||
})
|
||||
if _, ok := adapter.(*YemaPTAdapter); !ok {
|
||||
t.Fatalf("adapter = %T, want *YemaPTAdapter", adapter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRequestAPIKeyHeaderBySite(t *testing.T) {
|
||||
yemaReq, err := buildRequest(context.Background(), http.MethodGet, "https://www.yemapt.org/openApi/user/fetchBasicInfo.json", SiteConfig{
|
||||
Type: "yemapt",
|
||||
URL: "https://www.yemapt.org",
|
||||
AuthType: "api_key",
|
||||
APIKey: "yema-auth",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := yemaReq.Header.Get("Authorization"); got != "yema-auth" {
|
||||
t.Fatalf("YemaPT Authorization = %q, want yema-auth", got)
|
||||
}
|
||||
if got := yemaReq.Header.Get("x-api-key"); got != "" {
|
||||
t.Fatalf("YemaPT x-api-key = %q, want empty", got)
|
||||
}
|
||||
|
||||
mteamReq, err := buildRequest(context.Background(), http.MethodGet, "https://api.m-team.cc/api/torrent/search", SiteConfig{
|
||||
Type: "mteam",
|
||||
URL: "https://api.m-team.cc",
|
||||
AuthType: "api_key",
|
||||
APIKey: "mteam-auth",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := mteamReq.Header.Get("x-api-key"); got != "mteam-auth" {
|
||||
t.Fatalf("M-Team x-api-key = %q, want mteam-auth", got)
|
||||
}
|
||||
if got := mteamReq.Header.Get("Authorization"); got != "" {
|
||||
t.Fatalf("M-Team Authorization = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMTeamPublishedAPIRateLimits(t *testing.T) {
|
||||
search := mteamAPIRateLimits(mteamAPIEndpointSearch)
|
||||
if len(search) != 1 || search[0].Limit != 1000 || search[0].Window != 24*time.Hour {
|
||||
t.Fatalf("search limits = %#v, want 1000/24h", search)
|
||||
}
|
||||
detail := mteamAPIRateLimits(mteamAPIEndpointDetail)
|
||||
if len(detail) != 1 || detail[0].Limit != 100 || detail[0].Window != time.Hour {
|
||||
t.Fatalf("detail limits = %#v, want 100/1h", detail)
|
||||
}
|
||||
download := mteamAPIRateLimits(mteamAPIEndpointDownload)
|
||||
if len(download) != 2 ||
|
||||
download[0].Limit != 100 || download[0].Window != time.Hour ||
|
||||
download[1].Limit != 1000 || download[1].Window != 24*time.Hour {
|
||||
t.Fatalf("download limits = %#v, want 100/1h and 1000/24h", download)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentSiteAPIRateLimiterPersistsSlidingWindow(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Setting{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
now := time.Date(2026, 6, 20, 12, 0, 0, 0, time.UTC)
|
||||
limiter := newPersistentSiteAPIRateLimiter(repos)
|
||||
limiter.now = func() time.Time { return now }
|
||||
limit := siteAPIRateLimit{Bucket: "test_1h", Limit: 2, Window: time.Hour}
|
||||
|
||||
if err := limiter.Allow(t.Context(), "mteam:test", limit); err != nil {
|
||||
t.Fatalf("first allow: %v", err)
|
||||
}
|
||||
if err := limiter.Allow(t.Context(), "mteam:test", limit); err != nil {
|
||||
t.Fatalf("second allow: %v", err)
|
||||
}
|
||||
err = limiter.Allow(t.Context(), "mteam:test", limit)
|
||||
var limited *siteAPIRateLimitError
|
||||
if !errors.As(err, &limited) {
|
||||
t.Fatalf("third allow error = %v, want siteAPIRateLimitError", err)
|
||||
}
|
||||
if limited.RetryAfter != time.Hour {
|
||||
t.Fatalf("retry_after = %v, want 1h", limited.RetryAfter)
|
||||
}
|
||||
|
||||
restarted := newPersistentSiteAPIRateLimiter(repos)
|
||||
restarted.now = func() time.Time { return now.Add(30 * time.Minute) }
|
||||
if err := restarted.Allow(t.Context(), "mteam:test", limit); !errors.As(err, &limited) {
|
||||
t.Fatalf("restarted allow error = %v, want persisted limit", err)
|
||||
}
|
||||
|
||||
restarted.now = func() time.Time { return now.Add(time.Hour + time.Second) }
|
||||
if err := restarted.Allow(t.Context(), "mteam:test", limit); err != nil {
|
||||
t.Fatalf("allow after window: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMTeamRateLimitStopsRequestBeforeHTTP(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"code":"0","message":"SUCCESS","data":{"total":"0","data":[]}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
adapter := NewMTeamAdapter()
|
||||
limiter := &staticSiteAPIRateLimiter{err: &siteAPIRateLimitError{
|
||||
Bucket: "torrent_search_24h",
|
||||
Limit: 1000,
|
||||
Window: 24 * time.Hour,
|
||||
RetryAfter: time.Hour,
|
||||
}}
|
||||
_, err := adapter.Search(t.Context(), SiteConfig{
|
||||
URL: server.URL,
|
||||
AuthType: "api_key",
|
||||
APIKey: "token-123",
|
||||
Timeout: 5 * time.Second,
|
||||
rateLimiter: limiter,
|
||||
}, "show", 1)
|
||||
if err == nil || !strings.Contains(err.Error(), "rate limit") {
|
||||
t.Fatalf("Search error = %v, want rate limit", err)
|
||||
}
|
||||
if got := requests.Load(); got != 0 {
|
||||
t.Fatalf("HTTP requests = %d, want 0", got)
|
||||
}
|
||||
if limiter.calls != 1 {
|
||||
t.Fatalf("limiter calls = %d, want 1", limiter.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type staticSiteAPIRateLimiter struct {
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (l *staticSiteAPIRateLimiter) Allow(context.Context, string, ...siteAPIRateLimit) error {
|
||||
l.calls++
|
||||
return l.err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
// Package service — YemaPT site adapter.
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type YemaPTAdapter struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewYemaPTAdapter() *YemaPTAdapter {
|
||||
return &YemaPTAdapter{
|
||||
client: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
func (a *YemaPTAdapter) Authenticate(ctx context.Context, cfg SiteConfig) error {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return errors.New("YemaPT 需要填写个人详情页创建的第三方对接专用 API Auth Key")
|
||||
}
|
||||
u := strings.TrimRight(cfg.URL, "/") + "/openApi/user/fetchBasicInfo.json"
|
||||
data, status, err := doRequestJSON(ctx, a.client, http.MethodGet, u, cfg, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("yemapt authenticate: %w", err)
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
return fmt.Errorf("yemapt authenticate failed: status %d", status)
|
||||
}
|
||||
var resp yemaPTAPIResponse
|
||||
if err := json.Unmarshal(data, &resp); err != nil {
|
||||
return fmt.Errorf("yemapt authenticate parse: %w", err)
|
||||
}
|
||||
if resp.Success {
|
||||
return nil
|
||||
}
|
||||
if resp.ErrorMessage != "" {
|
||||
return fmt.Errorf("yemapt authenticate failed: %s", resp.ErrorMessage)
|
||||
}
|
||||
if resp.ErrorCode != 0 {
|
||||
return fmt.Errorf("yemapt authenticate failed: errorCode=%d", resp.ErrorCode)
|
||||
}
|
||||
return errors.New("yemapt authenticate failed")
|
||||
}
|
||||
|
||||
func (a *YemaPTAdapter) Search(ctx context.Context, cfg SiteConfig, keyword string, page int) (*SiteSearchResult, error) {
|
||||
return nil, errYemaPTTorrentOpenAPIUnsupported()
|
||||
}
|
||||
|
||||
func (a *YemaPTAdapter) Browse(ctx context.Context, cfg SiteConfig, category string, page int) (*SiteSearchResult, error) {
|
||||
return nil, errYemaPTTorrentOpenAPIUnsupported()
|
||||
}
|
||||
|
||||
func (a *YemaPTAdapter) GetDetail(ctx context.Context, cfg SiteConfig, id string) (*TorrentDetail, error) {
|
||||
return nil, errYemaPTTorrentOpenAPIUnsupported()
|
||||
}
|
||||
|
||||
func (a *YemaPTAdapter) GetDownloadURL(ctx context.Context, cfg SiteConfig, id string) (string, error) {
|
||||
return "", errYemaPTTorrentOpenAPIUnsupported()
|
||||
}
|
||||
|
||||
type yemaPTAPIResponse struct {
|
||||
Success bool `json:"success"`
|
||||
ShowType int `json:"showType"`
|
||||
ErrorCode int `json:"errorCode"`
|
||||
ErrorMessage string `json:"errorMessage"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
|
||||
func errYemaPTTorrentOpenAPIUnsupported() error {
|
||||
return errors.New("YemaPT 当前公开 OpenAPI 未提供种子搜索/详情/下载接口")
|
||||
}
|
||||
|
||||
func isYemaPTConfig(cfg SiteConfig) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(cfg.Type), "yemapt") || isYemaPTURL(cfg.URL)
|
||||
}
|
||||
|
||||
func isYemaPTURL(raw string) bool {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
return host == "yemapt.org" || strings.HasSuffix(host, ".yemapt.org")
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ShukeBta/MediaStationGo/internal/repository"
|
||||
)
|
||||
|
||||
const (
|
||||
mteamAPIEndpointSearch = "torrent_search"
|
||||
mteamAPIEndpointDetail = "torrent_detail"
|
||||
mteamAPIEndpointDownload = "torrent_download"
|
||||
)
|
||||
|
||||
type siteAPIRateLimit struct {
|
||||
Bucket string
|
||||
Limit int
|
||||
Window time.Duration
|
||||
}
|
||||
|
||||
type siteAPIRateLimiter interface {
|
||||
Allow(ctx context.Context, siteKey string, limits ...siteAPIRateLimit) error
|
||||
}
|
||||
|
||||
type siteAPIRateLimitError struct {
|
||||
SiteKey string
|
||||
Bucket string
|
||||
Limit int
|
||||
Window time.Duration
|
||||
RetryAfter time.Duration
|
||||
}
|
||||
|
||||
func (e *siteAPIRateLimitError) Error() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("M-Team API rate limit reached for %s: %d requests per %s, retry after %s",
|
||||
e.Bucket, e.Limit, formatRateLimitDuration(e.Window), formatRateLimitDuration(e.RetryAfter))
|
||||
}
|
||||
|
||||
type persistentSiteAPIRateLimiter struct {
|
||||
repo *repository.Container
|
||||
fallback *memorySiteAPIRateLimiter
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newPersistentSiteAPIRateLimiter(repo *repository.Container) *persistentSiteAPIRateLimiter {
|
||||
return &persistentSiteAPIRateLimiter{
|
||||
repo: repo,
|
||||
fallback: newMemorySiteAPIRateLimiter(time.Now),
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func (l *persistentSiteAPIRateLimiter) Allow(ctx context.Context, siteKey string, limits ...siteAPIRateLimit) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if l == nil || l.repo == nil || l.repo.Setting == nil {
|
||||
if l != nil && l.fallback != nil {
|
||||
return l.fallback.Allow(ctx, siteKey, limits...)
|
||||
}
|
||||
return defaultMemorySiteAPIRateLimiter.Allow(ctx, siteKey, limits...)
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
if l.now != nil {
|
||||
now = l.now()
|
||||
}
|
||||
records := make([]siteAPIRateLimitRecord, 0, len(limits))
|
||||
for _, limit := range normalizeSiteAPIRateLimits(limits) {
|
||||
key := siteAPIRateLimitSettingKey(siteKey, limit.Bucket)
|
||||
raw, err := l.repo.Setting.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
timestamps := pruneSiteAPIRateTimestamps(parseSiteAPIRateTimestamps(raw), now, limit.Window)
|
||||
if err := checkSiteAPIRateLimit(siteKey, limit, timestamps, now); err != nil {
|
||||
return err
|
||||
}
|
||||
records = append(records, siteAPIRateLimitRecord{key: key, timestamps: timestamps})
|
||||
}
|
||||
nowUnix := now.Unix()
|
||||
for _, record := range records {
|
||||
next := append(record.timestamps, nowUnix)
|
||||
if err := l.repo.Setting.Set(ctx, record.key, encodeSiteAPIRateTimestamps(next)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type memorySiteAPIRateLimiter struct {
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
buckets map[string][]int64
|
||||
}
|
||||
|
||||
var defaultMemorySiteAPIRateLimiter = newMemorySiteAPIRateLimiter(time.Now)
|
||||
|
||||
func newMemorySiteAPIRateLimiter(now func() time.Time) *memorySiteAPIRateLimiter {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
return &memorySiteAPIRateLimiter{now: now, buckets: map[string][]int64{}}
|
||||
}
|
||||
|
||||
func (l *memorySiteAPIRateLimiter) Allow(ctx context.Context, siteKey string, limits ...siteAPIRateLimit) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
now := l.now()
|
||||
records := make([]siteAPIRateLimitRecord, 0, len(limits))
|
||||
for _, limit := range normalizeSiteAPIRateLimits(limits) {
|
||||
key := siteAPIRateLimitSettingKey(siteKey, limit.Bucket)
|
||||
timestamps := pruneSiteAPIRateTimestamps(l.buckets[key], now, limit.Window)
|
||||
if err := checkSiteAPIRateLimit(siteKey, limit, timestamps, now); err != nil {
|
||||
return err
|
||||
}
|
||||
records = append(records, siteAPIRateLimitRecord{key: key, timestamps: timestamps})
|
||||
}
|
||||
nowUnix := now.Unix()
|
||||
for _, record := range records {
|
||||
l.buckets[record.key] = append(record.timestamps, nowUnix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type siteAPIRateLimitRecord struct {
|
||||
key string
|
||||
timestamps []int64
|
||||
}
|
||||
|
||||
func reserveMTeamAPIQuota(ctx context.Context, cfg SiteConfig, endpoint string) error {
|
||||
limits := mteamAPIRateLimits(endpoint)
|
||||
if len(limits) == 0 {
|
||||
return nil
|
||||
}
|
||||
// M-Team's published API quotas are upstream hard limits, so protect them
|
||||
// regardless of the generic per-site RateLimit toggle.
|
||||
limiter := cfg.rateLimiter
|
||||
if limiter == nil {
|
||||
limiter = defaultMemorySiteAPIRateLimiter
|
||||
}
|
||||
return limiter.Allow(ctx, mteamAPIRateSiteKey(cfg), limits...)
|
||||
}
|
||||
|
||||
func mteamAPIRateLimits(endpoint string) []siteAPIRateLimit {
|
||||
switch endpoint {
|
||||
case mteamAPIEndpointSearch:
|
||||
return []siteAPIRateLimit{{Bucket: "torrent_search_24h", Limit: 1000, Window: 24 * time.Hour}}
|
||||
case mteamAPIEndpointDetail:
|
||||
return []siteAPIRateLimit{{Bucket: "torrent_detail_1h", Limit: 100, Window: time.Hour}}
|
||||
case mteamAPIEndpointDownload:
|
||||
return []siteAPIRateLimit{
|
||||
{Bucket: "torrent_download_1h", Limit: 100, Window: time.Hour},
|
||||
{Bucket: "torrent_download_24h", Limit: 1000, Window: 24 * time.Hour},
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func mteamAPIRateSiteKey(cfg SiteConfig) string {
|
||||
base := strings.TrimRight(strings.ToLower(strings.TrimSpace(cfg.URL)), "/")
|
||||
if base == "" {
|
||||
base = "mteam"
|
||||
}
|
||||
if apiKey := strings.TrimSpace(cfg.APIKey); apiKey != "" {
|
||||
sum := sha1.Sum([]byte(apiKey))
|
||||
return base + "|api:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
if siteID := strings.TrimSpace(cfg.SiteID); siteID != "" {
|
||||
return base + "|site:" + siteID
|
||||
}
|
||||
if name := strings.TrimSpace(cfg.Name); name != "" {
|
||||
return base + "|name:" + strings.ToLower(name)
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func normalizeSiteAPIRateLimits(limits []siteAPIRateLimit) []siteAPIRateLimit {
|
||||
out := make([]siteAPIRateLimit, 0, len(limits))
|
||||
for _, limit := range limits {
|
||||
limit.Bucket = strings.TrimSpace(limit.Bucket)
|
||||
if limit.Bucket == "" || limit.Limit <= 0 || limit.Window <= 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, limit)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func checkSiteAPIRateLimit(siteKey string, limit siteAPIRateLimit, timestamps []int64, now time.Time) error {
|
||||
if len(timestamps) < limit.Limit {
|
||||
return nil
|
||||
}
|
||||
oldest := time.Unix(timestamps[0], 0)
|
||||
retryAfter := oldest.Add(limit.Window).Sub(now)
|
||||
if retryAfter < time.Second {
|
||||
retryAfter = time.Second
|
||||
}
|
||||
return &siteAPIRateLimitError{
|
||||
SiteKey: siteKey,
|
||||
Bucket: limit.Bucket,
|
||||
Limit: limit.Limit,
|
||||
Window: limit.Window,
|
||||
RetryAfter: retryAfter,
|
||||
}
|
||||
}
|
||||
|
||||
func siteAPIRateLimitSettingKey(siteKey, bucket string) string {
|
||||
sum := sha1.Sum([]byte(siteKey))
|
||||
return "site.api_rate." + hex.EncodeToString(sum[:])[:20] + "." + bucket
|
||||
}
|
||||
|
||||
func parseSiteAPIRateTimestamps(raw string) []int64 {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
var out []int64
|
||||
if err := json.Unmarshal([]byte(raw), &out); err != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func encodeSiteAPIRateTimestamps(values []int64) string {
|
||||
data, _ := json.Marshal(values)
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func pruneSiteAPIRateTimestamps(values []int64, now time.Time, window time.Duration) []int64 {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
cutoff := now.Add(-window).Unix()
|
||||
out := make([]int64, 0, len(values))
|
||||
for _, value := range values {
|
||||
if value > cutoff && value <= now.Add(time.Minute).Unix() {
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
|
||||
return out
|
||||
}
|
||||
|
||||
func formatRateLimitDuration(value time.Duration) string {
|
||||
if value < 0 {
|
||||
value = 0
|
||||
}
|
||||
value = value.Round(time.Second)
|
||||
if value%time.Hour == 0 && value >= time.Hour {
|
||||
return fmt.Sprintf("%dh", int(value/time.Hour))
|
||||
}
|
||||
if value%time.Minute == 0 && value >= time.Minute {
|
||||
return fmt.Sprintf("%dm", int(value/time.Minute))
|
||||
}
|
||||
return value.String()
|
||||
}
|
||||
@@ -2,6 +2,9 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
@@ -52,3 +55,74 @@ func TestSiteUpdateKeepsSecretsWhenPatchIsBlank(t *testing.T) {
|
||||
t.Fatalf("URL = %q, want trimmed URL", got.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYemaPTTestConnectionDoesNotFallbackAfterAuthFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"success":false,"errorCode":403,"errorMessage":"need api auth"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.Site{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
svc := NewSiteService(zap.NewNop(), repos, "")
|
||||
site := &model.Site{
|
||||
Name: "YemaPT",
|
||||
Type: "yemapt",
|
||||
URL: server.URL,
|
||||
AuthType: "api_key",
|
||||
APIKey: "bad-auth",
|
||||
Enabled: true,
|
||||
}
|
||||
if err := svc.Create(context.Background(), site); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ok, msg, err := svc.TestConnection(context.Background(), site.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatal("TestConnection succeeded after YemaPT auth failure")
|
||||
}
|
||||
if !strings.Contains(msg, "need api auth") {
|
||||
t.Fatalf("message = %q, want need api auth", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactSensitiveDownloadURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "query secrets",
|
||||
raw: "https://pt.example/download.php?id=123&passkey=secret#frag",
|
||||
want: "https://pt.example/download.php",
|
||||
},
|
||||
{
|
||||
name: "magnet",
|
||||
raw: "magnet:?xt=urn:btih:abc&dn=movie",
|
||||
want: "magnet:?xt=***",
|
||||
},
|
||||
{
|
||||
name: "invalid",
|
||||
raw: "not a url",
|
||||
want: "[redacted-download-url]",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := redactSensitiveDownloadURL(tt.raw); got != tt.want {
|
||||
t.Fatalf("redactSensitiveDownloadURL() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -335,18 +335,55 @@ func (s *SubscriptionService) runOne(ctx context.Context, sub *model.Subscriptio
|
||||
|
||||
func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subscription) (int, error) {
|
||||
if s.site == nil {
|
||||
if s.log != nil {
|
||||
s.log.Warn("site-search subscription service unavailable", subscriptionSiteSearchLogFields(sub, "")...)
|
||||
}
|
||||
return 0, errors.New("site search service unavailable")
|
||||
}
|
||||
keyword := siteSearchKeyword(sub)
|
||||
keywords := siteSearchKeywords(sub)
|
||||
keyword := ""
|
||||
if len(keywords) > 0 {
|
||||
keyword = keywords[0]
|
||||
}
|
||||
if keyword == "" {
|
||||
if s.log != nil {
|
||||
s.log.Warn("site-search subscription keyword missing", subscriptionSiteSearchLogFields(sub, "")...)
|
||||
}
|
||||
return 0, errors.New("site-search subscription keyword required")
|
||||
}
|
||||
if s.log != nil {
|
||||
s.log.Info("site-search subscription run started", subscriptionSiteSearchLogFields(sub, keyword)...)
|
||||
}
|
||||
|
||||
results, err := s.site.Search(ctx, keyword)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
var (
|
||||
results []SearchResult
|
||||
lastSearchErr error
|
||||
searchErrors int
|
||||
)
|
||||
for _, searchKeyword := range keywords {
|
||||
found, err := s.site.Search(ctx, searchKeyword)
|
||||
if err != nil {
|
||||
lastSearchErr = err
|
||||
searchErrors++
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, searchKeyword)
|
||||
fields = append(fields, zap.Error(err))
|
||||
s.log.Warn("site-search subscription search failed", fields...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
results = append(results, found...)
|
||||
}
|
||||
results = dedupeSiteSearchResults(results)
|
||||
if len(results) == 0 && lastSearchErr != nil && searchErrors == len(keywords) {
|
||||
return 0, lastSearchErr
|
||||
}
|
||||
if len(results) == 0 {
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, keyword)
|
||||
fields = append(fields, zap.Int("results_count", 0))
|
||||
s.log.Info("site-search subscription no results", fields...)
|
||||
}
|
||||
now := time.Now()
|
||||
_ = s.repo.DB.Model(sub).Updates(map[string]any{"last_run_at": &now}).Error
|
||||
return 0, nil
|
||||
@@ -365,25 +402,64 @@ func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subs
|
||||
SubscriptionLocalAvailability(ctx, s.repo, sub),
|
||||
s.pendingDownloadAvailability(ctx, sub),
|
||||
)
|
||||
candidates := selectSiteSearchCandidates(results, sub, seenSet, availability)
|
||||
candidates, selectionStats := selectSiteSearchCandidatesWithStats(results, sub, seenSet, availability)
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, keyword)
|
||||
fields = appendSiteSearchSelectionLogFields(fields, selectionStats)
|
||||
fields = appendAvailabilityLogFields(fields, availability)
|
||||
s.log.Info("site-search subscription selection summary", fields...)
|
||||
}
|
||||
var lastEnqueueErr error
|
||||
queued := 0
|
||||
var resources []string
|
||||
for _, candidate := range candidates {
|
||||
item := candidate.Item
|
||||
mediaType, mediaCategory := s.classifySubscriptionItem(ctx, sub, item.Title, item.Category)
|
||||
matchText := subscriptionSearchResultText(item)
|
||||
mediaType, mediaCategory := s.classifySubscriptionItem(ctx, sub, matchText, item.Category)
|
||||
if s.shouldSkipExistingTorrent(ctx, mediaType, candidate) {
|
||||
addAvailabilityTitle(item.Title, availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)), &availability)
|
||||
addSiteSearchCandidateAvailability(candidate, &availability)
|
||||
seen = append(seen, candidate.GUID)
|
||||
seenSet[candidate.GUID] = struct{}{}
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, keyword)
|
||||
fields = append(fields,
|
||||
zap.String("reason", "existing_torrent"),
|
||||
zap.String("title", item.Title),
|
||||
zap.String("subtitle", item.Subtitle),
|
||||
zap.String("site", firstNonEmpty(item.SiteName, item.SiteID)),
|
||||
zap.String("site_category", item.Category),
|
||||
zap.Int("season", candidate.Season),
|
||||
zap.Int("episode", candidate.Episode),
|
||||
zap.Bool("pack", candidate.Pack),
|
||||
zap.String("media_type", mediaType),
|
||||
)
|
||||
s.log.Info("site-search subscription candidate skipped", fields...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
realURL := s.site.ResolveDownloadURL(ctx, candidate.Download)
|
||||
savePath := s.resolveSubscriptionSavePath(ctx, sub, mediaType, mediaCategory)
|
||||
if s.downloadPathHasCandidate(ctx, sub, candidate.Item.Title, savePath) {
|
||||
addAvailabilityTitle(item.Title, availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)), &availability)
|
||||
if s.downloadPathHasCandidate(ctx, sub, matchText, savePath) {
|
||||
addSiteSearchCandidateAvailability(candidate, &availability)
|
||||
seen = append(seen, candidate.GUID)
|
||||
seenSet[candidate.GUID] = struct{}{}
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, keyword)
|
||||
fields = append(fields,
|
||||
zap.String("reason", "download_path_has_candidate"),
|
||||
zap.String("title", item.Title),
|
||||
zap.String("subtitle", item.Subtitle),
|
||||
zap.String("site", firstNonEmpty(item.SiteName, item.SiteID)),
|
||||
zap.String("site_category", item.Category),
|
||||
zap.Int("season", candidate.Season),
|
||||
zap.Int("episode", candidate.Episode),
|
||||
zap.Bool("pack", candidate.Pack),
|
||||
zap.String("media_type", mediaType),
|
||||
zap.String("media_category", mediaCategory),
|
||||
zap.String("save_path", savePath),
|
||||
)
|
||||
s.log.Info("site-search subscription candidate skipped", fields...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := s.downloads.AddDownloadWithMeta(ctx, sub.UserID, realURL, savePath, DownloadTaskMeta{
|
||||
@@ -398,15 +474,36 @@ func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subs
|
||||
AllowExistingLibrary: sub.WashEnabled,
|
||||
}); err != nil {
|
||||
if IsDownloadDedupError(err) {
|
||||
addAvailabilityTitle(item.Title, availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)), &availability)
|
||||
addSiteSearchCandidateAvailability(candidate, &availability)
|
||||
seen = append(seen, candidate.GUID)
|
||||
seenSet[candidate.GUID] = struct{}{}
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, keyword)
|
||||
fields = append(fields,
|
||||
zap.String("reason", "download_dedup"),
|
||||
zap.String("title", item.Title),
|
||||
zap.String("subtitle", item.Subtitle),
|
||||
zap.String("site", firstNonEmpty(item.SiteName, item.SiteID)),
|
||||
zap.String("site_category", item.Category),
|
||||
zap.Int("season", candidate.Season),
|
||||
zap.Int("episode", candidate.Episode),
|
||||
zap.Bool("pack", candidate.Pack),
|
||||
zap.String("media_type", mediaType),
|
||||
zap.String("media_category", mediaCategory),
|
||||
zap.String("save_path", savePath),
|
||||
)
|
||||
s.log.Info("site-search subscription candidate skipped", fields...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
lastEnqueueErr = err
|
||||
s.log.Warn("site-search subscription enqueue failed",
|
||||
zap.String("subscription_id", sub.ID),
|
||||
zap.String("subscription", sub.Name),
|
||||
zap.String("keyword", keyword),
|
||||
zap.String("title", item.Title),
|
||||
zap.String("subtitle", item.Subtitle),
|
||||
zap.String("site", firstNonEmpty(item.SiteName, item.SiteID)),
|
||||
zap.String("site_category", item.Category),
|
||||
zap.String("media_type", mediaType),
|
||||
zap.String("media_category", mediaCategory),
|
||||
@@ -415,10 +512,27 @@ func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subs
|
||||
continue
|
||||
}
|
||||
queued++
|
||||
addAvailabilityTitle(item.Title, availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)), &availability)
|
||||
addSiteSearchCandidateAvailability(candidate, &availability)
|
||||
resources = append(resources, item.Title)
|
||||
seen = append(seen, candidate.GUID)
|
||||
seenSet[candidate.GUID] = struct{}{}
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, keyword)
|
||||
fields = append(fields,
|
||||
zap.String("title", item.Title),
|
||||
zap.String("subtitle", item.Subtitle),
|
||||
zap.String("site", firstNonEmpty(item.SiteName, item.SiteID)),
|
||||
zap.String("site_category", item.Category),
|
||||
zap.Int("season", candidate.Season),
|
||||
zap.Int("episode", candidate.Episode),
|
||||
zap.Bool("pack", candidate.Pack),
|
||||
zap.Int("score", candidate.Score),
|
||||
zap.String("media_type", mediaType),
|
||||
zap.String("media_category", mediaCategory),
|
||||
zap.String("save_path", savePath),
|
||||
)
|
||||
s.log.Info("site-search subscription candidate queued", fields...)
|
||||
}
|
||||
}
|
||||
availability = s.finalizePendingAvailability(sub, availability)
|
||||
if len(seen) > 200 {
|
||||
@@ -442,9 +556,83 @@ func (s *SubscriptionService) runSiteSearch(ctx context.Context, sub *model.Subs
|
||||
if lastEnqueueErr != nil {
|
||||
return 0, fmt.Errorf("找到 PT 资源但加入下载器失败: %w", lastEnqueueErr)
|
||||
}
|
||||
if s.log != nil {
|
||||
fields := subscriptionSiteSearchLogFields(sub, keyword)
|
||||
fields = appendSiteSearchSelectionLogFields(fields, selectionStats)
|
||||
fields = appendAvailabilityLogFields(fields, availability)
|
||||
fields = append(fields, zap.Int("queued", queued))
|
||||
s.log.Info("site-search subscription no candidate queued", fields...)
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func subscriptionSiteSearchLogFields(sub *model.Subscription, keyword string) []zap.Field {
|
||||
fields := []zap.Field{zap.String("keyword", keyword), zap.Strings("search_keywords", siteSearchKeywords(sub))}
|
||||
if sub == nil {
|
||||
return fields
|
||||
}
|
||||
fields = append(fields,
|
||||
zap.String("subscription_id", sub.ID),
|
||||
zap.String("subscription", sub.Name),
|
||||
zap.String("filter", sub.Filter),
|
||||
zap.String("media_type", sub.MediaType),
|
||||
zap.String("media_category", sub.MediaCategory),
|
||||
zap.String("search_mode", sub.SearchMode),
|
||||
zap.String("imdb_id", sub.IMDBID),
|
||||
zap.Bool("wash_enabled", sub.WashEnabled),
|
||||
zap.String("wash_priority", sub.WashPriority),
|
||||
zap.Int("total_episodes", sub.TotalEpisodes),
|
||||
)
|
||||
return fields
|
||||
}
|
||||
|
||||
func appendSiteSearchSelectionLogFields(fields []zap.Field, stats siteSearchSelectionStats) []zap.Field {
|
||||
return append(fields,
|
||||
zap.Int("results_count", stats.Total),
|
||||
zap.Int("query_mismatch_count", stats.QueryMismatch),
|
||||
zap.Int("relaxed_query_match_count", stats.RelaxedQueryMatch),
|
||||
zap.Int("rule_mismatch_count", stats.RuleMismatch),
|
||||
zap.Int("missing_download_count", stats.MissingDownload),
|
||||
zap.Int("seen_count", stats.Seen),
|
||||
zap.Int("prepared_count", stats.Prepared),
|
||||
zap.Int("selected_count", stats.Selected),
|
||||
zap.Bool("local_already_satisfied", stats.LocalAlreadySatisfied),
|
||||
zap.Bool("local_series_pack_present", stats.LocalSeriesPackPresent),
|
||||
zap.Bool("series_complete", stats.SeriesComplete),
|
||||
zap.Int("existing_episode_skipped_count", stats.ExistingEpisodeSkipped),
|
||||
zap.Int("not_missing_episode_skipped_count", stats.NotMissingEpisodeSkipped),
|
||||
zap.Int("no_episode_skipped_count", stats.NoEpisodeSkipped),
|
||||
zap.Bool("pack_fallback_available", stats.PackFallbackAvailable),
|
||||
zap.Bool("pack_fallback_used", stats.PackFallbackUsed),
|
||||
)
|
||||
}
|
||||
|
||||
func appendAvailabilityLogFields(fields []zap.Field, availability LocalAvailability) []zap.Field {
|
||||
missingSample, missingMore := limitedEpisodeSample(availability.MissingEpisodes, 20)
|
||||
return append(fields,
|
||||
zap.Int("local_media_count", availability.LocalMediaCount),
|
||||
zap.Bool("in_library", availability.InLibrary),
|
||||
zap.Bool("has_series_pack", availability.HasSeriesPack),
|
||||
zap.Int("downloaded_episodes", availability.DownloadedEpisodes),
|
||||
zap.Int("availability_total_episodes", availability.TotalEpisodes),
|
||||
zap.Int("missing_episode_count", len(availability.MissingEpisodes)),
|
||||
zap.Ints("missing_episodes", missingSample),
|
||||
zap.Int("missing_episodes_more", missingMore),
|
||||
)
|
||||
}
|
||||
|
||||
func limitedEpisodeSample(values []int, limit int) ([]int, int) {
|
||||
if limit <= 0 || len(values) == 0 {
|
||||
return nil, len(values)
|
||||
}
|
||||
if len(values) <= limit {
|
||||
out := append([]int(nil), values...)
|
||||
return out, 0
|
||||
}
|
||||
out := append([]int(nil), values[:limit]...)
|
||||
return out, len(values) - limit
|
||||
}
|
||||
|
||||
func (s *SubscriptionService) notifySubscriptionHit(sub *model.Subscription, queued int, resources []string) {
|
||||
if s == nil || s.notify == nil || sub == nil || queued <= 0 {
|
||||
return
|
||||
@@ -618,13 +806,14 @@ func inferSearchTotalEpisodes(results []SearchResult, sub *model.Subscription) i
|
||||
}
|
||||
maxEpisode := 0
|
||||
for _, item := range results {
|
||||
if !subscriptionTitleMatchesQuery(sub, item.Title) {
|
||||
matchText := subscriptionSearchResultText(item)
|
||||
if !subscriptionTitleMatchesQuery(sub, matchText) {
|
||||
continue
|
||||
}
|
||||
if !matchesSubscriptionRules(sub, item.Title) {
|
||||
if !matchesSubscriptionRules(sub, matchText) {
|
||||
continue
|
||||
}
|
||||
_, episode := ParseEpisode(item.Title)
|
||||
_, episode := ParseEpisode(matchText)
|
||||
if episode > maxEpisode {
|
||||
maxEpisode = episode
|
||||
}
|
||||
@@ -778,11 +967,15 @@ func subscriptionTitleMatchQueries(sub *model.Subscription) []string {
|
||||
if sub == nil {
|
||||
return nil
|
||||
}
|
||||
return compactUniqueStrings(
|
||||
values := []string{
|
||||
availabilityQuery(subscriptionName(sub), subscriptionFilter(sub)),
|
||||
cleanAvailabilityTitle(subscriptionFilter(sub)),
|
||||
cleanAvailabilityTitle(subscriptionName(sub)),
|
||||
)
|
||||
}
|
||||
for _, alias := range subscriptionFeedAliases(sub) {
|
||||
values = append(values, alias, cleanAvailabilityTitle(alias))
|
||||
}
|
||||
return compactUniqueStrings(values...)
|
||||
}
|
||||
|
||||
func subscriptionEpisodeMetadataQueries(sub *model.Subscription) []string {
|
||||
@@ -879,22 +1072,82 @@ func (s *SubscriptionService) shouldSkipExistingTorrent(ctx context.Context, med
|
||||
return s.downloads.TorrentExistsByName(ctx, candidate.Item.Title)
|
||||
}
|
||||
|
||||
func siteSearchKeyword(sub *model.Subscription) string {
|
||||
func siteSearchKeywords(sub *model.Subscription) []string {
|
||||
if sub == nil {
|
||||
return ""
|
||||
return nil
|
||||
}
|
||||
values := make([]string, 0, 8)
|
||||
if strings.EqualFold(strings.TrimSpace(sub.SearchMode), "imdb") && strings.TrimSpace(sub.IMDBID) != "" {
|
||||
return strings.TrimSpace(sub.IMDBID)
|
||||
values = append(values, strings.TrimSpace(sub.IMDBID))
|
||||
}
|
||||
if u, err := url.Parse(sub.FeedURL); err == nil {
|
||||
if keyword := strings.TrimSpace(u.Query().Get("keyword")); keyword != "" {
|
||||
return keyword
|
||||
values = append(values, keyword)
|
||||
}
|
||||
}
|
||||
if keyword := strings.TrimSpace(sub.Filter); keyword != "" {
|
||||
return keyword
|
||||
if strings.TrimSpace(sub.Filter) != "" {
|
||||
values = append(values, sub.Filter)
|
||||
}
|
||||
return strings.TrimSpace(sub.Name)
|
||||
if len(values) == 0 && strings.TrimSpace(sub.Name) != "" {
|
||||
values = append(values, sub.Name)
|
||||
}
|
||||
values = append(values, subscriptionFeedAliases(sub)...)
|
||||
for _, value := range append([]string(nil), values...) {
|
||||
if cleaned := cleanAvailabilityTitle(value); cleaned != "" {
|
||||
values = append(values, cleaned)
|
||||
}
|
||||
}
|
||||
return compactUniqueStrings(values...)
|
||||
}
|
||||
|
||||
func siteSearchKeyword(sub *model.Subscription) string {
|
||||
keywords := siteSearchKeywords(sub)
|
||||
if len(keywords) == 0 {
|
||||
return ""
|
||||
}
|
||||
return keywords[0]
|
||||
}
|
||||
|
||||
func subscriptionFeedAliases(sub *model.Subscription) []string {
|
||||
if sub == nil {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(sub.FeedURL)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
q := u.Query()
|
||||
values := make([]string, 0, len(q["alias"])+2)
|
||||
values = append(values, q["alias"]...)
|
||||
for _, raw := range q["aliases"] {
|
||||
for _, part := range strings.FieldsFunc(raw, func(r rune) bool {
|
||||
return r == '|' || r == '\n' || r == '\r' || r == '\t'
|
||||
}) {
|
||||
values = append(values, part)
|
||||
}
|
||||
}
|
||||
return compactUniqueStrings(values...)
|
||||
}
|
||||
|
||||
func dedupeSiteSearchResults(results []SearchResult) []SearchResult {
|
||||
if len(results) < 2 {
|
||||
return results
|
||||
}
|
||||
seen := make(map[string]struct{}, len(results))
|
||||
out := make([]SearchResult, 0, len(results))
|
||||
for _, item := range results {
|
||||
download := strings.TrimSpace(item.DownloadURL)
|
||||
if download == "" {
|
||||
download = strings.TrimSpace(item.TorrentURL)
|
||||
}
|
||||
key := stableSiteSearchGUID(item, download)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *SubscriptionService) fetch(ctx context.Context, feedURL string) (*rssFeed, error) {
|
||||
|
||||
@@ -67,7 +67,12 @@ func (s *SubscriptionService) addDownloadTaskAvailability(ctx context.Context, s
|
||||
if !downloadTaskBlocksReadd(row.Status) {
|
||||
continue
|
||||
}
|
||||
if baseSavePath != "" && row.SavePath != "" && !sameOrChildPath(row.SavePath, baseSavePath) && !sameOrChildPath(baseSavePath, row.SavePath) {
|
||||
linkedToSubscription := sub != nil && strings.TrimSpace(row.SubscriptionID) != "" && row.SubscriptionID == sub.ID
|
||||
if !linkedToSubscription && baseSavePath != "" && row.SavePath != "" && !sameOrChildPath(row.SavePath, baseSavePath) && !sameOrChildPath(baseSavePath, row.SavePath) {
|
||||
continue
|
||||
}
|
||||
if linkedToSubscription {
|
||||
addTrustedAvailabilityTitle(row.Title, 0, 0, false, out)
|
||||
continue
|
||||
}
|
||||
addAvailabilityTitle(row.Title, query, out)
|
||||
@@ -105,6 +110,33 @@ func addAvailabilityTitle(title, query string, out *LocalAvailability) {
|
||||
}
|
||||
}
|
||||
|
||||
func addSiteSearchCandidateAvailability(candidate siteSearchCandidate, out *LocalAvailability) {
|
||||
addTrustedAvailabilityTitle(subscriptionSearchResultText(candidate.Item), candidate.Season, candidate.Episode, candidate.Pack, out)
|
||||
}
|
||||
|
||||
func addTrustedAvailabilityTitle(title string, season, episode int, pack bool, out *LocalAvailability) {
|
||||
if out == nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(title) == "" && episode <= 0 && !pack {
|
||||
return
|
||||
}
|
||||
out.LocalMediaCount++
|
||||
if episode <= 0 {
|
||||
season, episode = ParseEpisode(title)
|
||||
}
|
||||
if episode > 0 {
|
||||
if out.ExistingEpisodeKeys == nil {
|
||||
out.ExistingEpisodeKeys = map[string]struct{}{}
|
||||
}
|
||||
out.ExistingEpisodeKeys[episodeKey(season, episode)] = struct{}{}
|
||||
return
|
||||
}
|
||||
if pack || isSeriesPackTitle(title) {
|
||||
out.HasSeriesPack = true
|
||||
}
|
||||
}
|
||||
|
||||
func sameOrChildPath(pathValue, root string) bool {
|
||||
pathValue = filepath.Clean(strings.TrimSpace(pathValue))
|
||||
root = filepath.Clean(strings.TrimSpace(root))
|
||||
|
||||
@@ -27,6 +27,25 @@ type siteSearchCandidate struct {
|
||||
Score int
|
||||
}
|
||||
|
||||
type siteSearchSelectionStats struct {
|
||||
Total int
|
||||
QueryMismatch int
|
||||
RelaxedQueryMatch int
|
||||
RuleMismatch int
|
||||
MissingDownload int
|
||||
Seen int
|
||||
Prepared int
|
||||
Selected int
|
||||
LocalAlreadySatisfied bool
|
||||
LocalSeriesPackPresent bool
|
||||
SeriesComplete bool
|
||||
ExistingEpisodeSkipped int
|
||||
NotMissingEpisodeSkipped int
|
||||
NoEpisodeSkipped int
|
||||
PackFallbackAvailable bool
|
||||
PackFallbackUsed bool
|
||||
}
|
||||
|
||||
// SubscriptionPlanner owns release selection decisions for subscriptions:
|
||||
// rule matching, candidate scoring, and filtering against known availability.
|
||||
type SubscriptionPlanner struct{}
|
||||
@@ -46,13 +65,51 @@ func (SubscriptionPlanner) SelectSiteSearchCandidates(results []SearchResult, su
|
||||
if len(availability) > 0 {
|
||||
local = availability[0]
|
||||
}
|
||||
return selectSiteSearchCandidatesWithAvailability(results, sub, seenSet, local)
|
||||
candidates, _ := selectSiteSearchCandidatesWithStats(results, sub, seenSet, local)
|
||||
return candidates
|
||||
}
|
||||
|
||||
func selectSiteSearchCandidatesWithAvailability(results []SearchResult, sub *model.Subscription, seenSet map[string]struct{}, local LocalAvailability) []siteSearchCandidate {
|
||||
candidates, _ := selectSiteSearchCandidatesWithStats(results, sub, seenSet, local)
|
||||
return candidates
|
||||
}
|
||||
|
||||
func selectSiteSearchCandidatesWithStats(results []SearchResult, sub *model.Subscription, seenSet map[string]struct{}, local LocalAvailability) ([]siteSearchCandidate, siteSearchSelectionStats) {
|
||||
stats := siteSearchSelectionStats{Total: len(results)}
|
||||
if sub == nil {
|
||||
return nil, stats
|
||||
}
|
||||
if seenSet == nil {
|
||||
seenSet = map[string]struct{}{}
|
||||
}
|
||||
candidates := collectSiteSearchCandidates(results, sub, seenSet, false, &stats)
|
||||
if len(candidates) == 0 && shouldRelaxSiteSearchQueryMatch(sub, local) && stats.QueryMismatch > 0 {
|
||||
relaxedStats := siteSearchSelectionStats{Total: len(results)}
|
||||
candidates = collectSiteSearchCandidates(results, sub, seenSet, true, &relaxedStats)
|
||||
stats.RuleMismatch = relaxedStats.RuleMismatch
|
||||
stats.MissingDownload = relaxedStats.MissingDownload
|
||||
stats.Seen = relaxedStats.Seen
|
||||
stats.Prepared = relaxedStats.Prepared
|
||||
stats.RelaxedQueryMatch = relaxedStats.RelaxedQueryMatch
|
||||
}
|
||||
selected := selectPreparedSubscriptionCandidatesWithStats(candidates, sub, local, &stats)
|
||||
return selected, stats
|
||||
}
|
||||
|
||||
func collectSiteSearchCandidates(results []SearchResult, sub *model.Subscription, seenSet map[string]struct{}, allowQueryMismatch bool, stats *siteSearchSelectionStats) []siteSearchCandidate {
|
||||
candidates := make([]siteSearchCandidate, 0, len(results))
|
||||
for _, item := range results {
|
||||
if !matchesSubscriptionRules(sub, item.Title) {
|
||||
matchText := subscriptionSearchResultText(item)
|
||||
if !subscriptionTitleMatchesQuery(sub, matchText) {
|
||||
if allowQueryMismatch {
|
||||
stats.RelaxedQueryMatch++
|
||||
} else {
|
||||
stats.QueryMismatch++
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !matchesSubscriptionRules(sub, matchText) {
|
||||
stats.RuleMismatch++
|
||||
continue
|
||||
}
|
||||
download := strings.TrimSpace(item.DownloadURL)
|
||||
@@ -60,14 +117,17 @@ func selectSiteSearchCandidatesWithAvailability(results []SearchResult, sub *mod
|
||||
download = strings.TrimSpace(item.TorrentURL)
|
||||
}
|
||||
if download == "" {
|
||||
stats.MissingDownload++
|
||||
continue
|
||||
}
|
||||
guid := stableSiteSearchGUID(item, download)
|
||||
if _, ok := seenSet[guid]; ok {
|
||||
stats.Seen++
|
||||
continue
|
||||
}
|
||||
season, episode := ParseEpisode(item.Title)
|
||||
season, episode := ParseEpisode(matchText)
|
||||
score := subscriptionCandidateScore(sub, item)
|
||||
stats.Prepared++
|
||||
candidates = append(candidates, siteSearchCandidate{
|
||||
Item: item,
|
||||
Download: download,
|
||||
@@ -78,7 +138,25 @@ func selectSiteSearchCandidatesWithAvailability(results []SearchResult, sub *mod
|
||||
Score: score,
|
||||
})
|
||||
}
|
||||
return selectPreparedSubscriptionCandidates(candidates, sub, local)
|
||||
return candidates
|
||||
}
|
||||
|
||||
func shouldRelaxSiteSearchQueryMatch(sub *model.Subscription, local LocalAvailability) bool {
|
||||
if sub == nil {
|
||||
return false
|
||||
}
|
||||
mediaType := normalizeMediaType(sub.MediaType, sub.Name+" "+sub.Filter, "")
|
||||
if !isSubscriptionSeriesType(mediaType) {
|
||||
return false
|
||||
}
|
||||
if local.LocalMediaCount == 0 && len(local.ExistingEpisodeKeys) == 0 {
|
||||
return false
|
||||
}
|
||||
return local.TotalEpisodes > 0 || len(local.MissingEpisodes) > 0
|
||||
}
|
||||
|
||||
func subscriptionSearchResultText(item SearchResult) string {
|
||||
return strings.TrimSpace(strings.Join([]string{item.Title, item.Subtitle}, " "))
|
||||
}
|
||||
|
||||
func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, filter *regexp.Regexp, seenSet map[string]struct{}, local LocalAvailability) []siteSearchCandidate {
|
||||
@@ -124,6 +202,10 @@ func selectRSSSubscriptionCandidates(items []rssItem, sub *model.Subscription, f
|
||||
}
|
||||
|
||||
func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub *model.Subscription, local LocalAvailability) []siteSearchCandidate {
|
||||
return selectPreparedSubscriptionCandidatesWithStats(candidates, sub, local, nil)
|
||||
}
|
||||
|
||||
func selectPreparedSubscriptionCandidatesWithStats(candidates []siteSearchCandidate, sub *model.Subscription, local LocalAvailability, stats *siteSearchSelectionStats) []siteSearchCandidate {
|
||||
if len(candidates) > 1 {
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
if candidates[i].Score != candidates[j].Score {
|
||||
@@ -136,24 +218,33 @@ func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub
|
||||
})
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return nil
|
||||
return recordPreparedSelection(nil, stats)
|
||||
}
|
||||
|
||||
mediaType := normalizeMediaType(sub.MediaType, sub.Name+" "+sub.Filter, "")
|
||||
if !isSubscriptionSeriesType(mediaType) {
|
||||
// 非洗版订阅成功下载一次即满足,媒体库/下载中已存在则不再重复下载。
|
||||
if (sub == nil || !sub.WashEnabled) && local.LocalMediaCount > 0 {
|
||||
return nil
|
||||
if stats != nil {
|
||||
stats.LocalAlreadySatisfied = true
|
||||
}
|
||||
return recordPreparedSelection(nil, stats)
|
||||
}
|
||||
return candidates[:1]
|
||||
return recordPreparedSelection(candidates[:1], stats)
|
||||
}
|
||||
|
||||
if local.HasSeriesPack {
|
||||
return nil
|
||||
if stats != nil {
|
||||
stats.LocalSeriesPackPresent = true
|
||||
}
|
||||
return recordPreparedSelection(nil, stats)
|
||||
}
|
||||
if local.LocalMediaCount > 0 {
|
||||
if local.TotalEpisodes > 0 && len(local.MissingEpisodes) == 0 {
|
||||
return nil
|
||||
if stats != nil {
|
||||
stats.SeriesComplete = true
|
||||
}
|
||||
return recordPreparedSelection(nil, stats)
|
||||
}
|
||||
missingSet := missingEpisodeSet(local)
|
||||
onlyMissing := make([]siteSearchCandidate, 0, len(candidates))
|
||||
@@ -164,8 +255,14 @@ func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub
|
||||
// 整季/全集包(无单集号)。剧集完结后站点常只挂全集包,
|
||||
// 这里记下来作兜底:当单集候选不足以补齐缺失集时启用,
|
||||
// 否则"补全缺失集"在站点只有全集包时永远匹配为空。
|
||||
if stats != nil {
|
||||
stats.NoEpisodeSkipped++
|
||||
}
|
||||
if candidate.Pack && packFallback == nil {
|
||||
packFallback = &candidates[i]
|
||||
if stats != nil {
|
||||
stats.PackFallbackAvailable = true
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -174,10 +271,16 @@ func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub
|
||||
season = 1
|
||||
}
|
||||
if _, exists := local.ExistingEpisodeKeys[episodeKey(season, candidate.Episode)]; exists {
|
||||
if stats != nil {
|
||||
stats.ExistingEpisodeSkipped++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if local.TotalEpisodes > 0 {
|
||||
if _, missing := missingSet[candidate.Episode]; !missing {
|
||||
if stats != nil {
|
||||
stats.NotMissingEpisodeSkipped++
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -187,22 +290,32 @@ func selectPreparedSubscriptionCandidates(candidates []siteSearchCandidate, sub
|
||||
if len(selected) == 0 && packFallback != nil {
|
||||
// 没有可用的单集候选,但站点有整季/全集包 → 用包兜底补缺集。
|
||||
// 代价是会重下已有集,但用户主动触发补全时这是可接受的。
|
||||
return []siteSearchCandidate{*packFallback}
|
||||
if stats != nil {
|
||||
stats.PackFallbackUsed = true
|
||||
}
|
||||
return recordPreparedSelection([]siteSearchCandidate{*packFallback}, stats)
|
||||
}
|
||||
return selected
|
||||
return recordPreparedSelection(selected, stats)
|
||||
}
|
||||
|
||||
for _, candidate := range candidates {
|
||||
if candidate.Pack {
|
||||
return []siteSearchCandidate{candidate}
|
||||
return recordPreparedSelection([]siteSearchCandidate{candidate}, stats)
|
||||
}
|
||||
}
|
||||
|
||||
selected := sortedEpisodeCandidates(candidates)
|
||||
if len(selected) == 0 {
|
||||
return candidates[:1]
|
||||
return recordPreparedSelection(candidates[:1], stats)
|
||||
}
|
||||
return selected
|
||||
return recordPreparedSelection(selected, stats)
|
||||
}
|
||||
|
||||
func recordPreparedSelection(candidates []siteSearchCandidate, stats *siteSearchSelectionStats) []siteSearchCandidate {
|
||||
if stats != nil {
|
||||
stats.Selected = len(candidates)
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func stableRSSItemGUID(title, guid, link, enclosureURL string) string {
|
||||
@@ -298,7 +411,7 @@ func matchesSubscriptionRules(sub *model.Subscription, title string) bool {
|
||||
}
|
||||
|
||||
func subscriptionCandidateScore(sub *model.Subscription, item SearchResult) int {
|
||||
title := strings.ToLower(item.Title)
|
||||
title := strings.ToLower(subscriptionSearchResultText(item))
|
||||
score := item.Seeders
|
||||
if sub == nil || !sub.WashEnabled {
|
||||
if item.Free {
|
||||
|
||||
@@ -71,6 +71,49 @@ func TestSelectSiteSearchCandidatesKeepsMovieSingleBest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesRejectsUnrelatedHighSeederResult(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", WashPriority: "seeders"}
|
||||
results := []SearchResult{
|
||||
{Title: "Unrelated Movie 2026 2160p", DownloadURL: "https://pt/download/wrong", Seeders: 999},
|
||||
{Title: "Inception 2010 1080p", DownloadURL: "https://pt/download/right", Seeders: 90},
|
||||
}
|
||||
|
||||
got := selectSiteSearchCandidates(results, sub, map[string]struct{}{})
|
||||
if len(got) != 1 || got[0].Download != "https://pt/download/right" {
|
||||
t.Fatalf("selected %#v, want title-matched result only", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesMatchesTranslatedSubtitle(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "真人快打2 自动订阅", Filter: "真人快打2 2026", MediaType: "movie", WashPriority: "seeders"}
|
||||
results := []SearchResult{
|
||||
{Title: "Unrelated Movie 2026 2160p", DownloadURL: "https://pt/download/wrong", Seeders: 999},
|
||||
{Title: "Mortal Kombat II 2026 1080p WEB-DL", Subtitle: "真人快打2", DownloadURL: "https://pt/download/right", Seeders: 90},
|
||||
}
|
||||
|
||||
got := selectSiteSearchCandidates(results, sub, map[string]struct{}{})
|
||||
if len(got) != 1 || got[0].Download != "https://pt/download/right" {
|
||||
t.Fatalf("selected %#v, want translated subtitle match", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesMatchesFeedAlias(t *testing.T) {
|
||||
sub := &model.Subscription{
|
||||
Name: "真人快打2 自动订阅",
|
||||
FeedURL: "site-search://search?keyword=%E7%9C%9F%E4%BA%BA%E5%BF%AB%E6%89%932%202026&alias=Mortal%20Kombat%20II%202026",
|
||||
Filter: "真人快打2 2026",
|
||||
MediaType: "movie",
|
||||
}
|
||||
results := []SearchResult{
|
||||
{Title: "Mortal Kombat II 2026 1080p WEB-DL", DownloadURL: "https://pt/download/right", Seeders: 90},
|
||||
}
|
||||
|
||||
got := selectSiteSearchCandidates(results, sub, map[string]struct{}{})
|
||||
if len(got) != 1 || got[0].Download != "https://pt/download/right" {
|
||||
t.Fatalf("selected %#v, want alias-matched result", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesDoesNotWashByDefault(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie", WashPriority: "resolution"}
|
||||
results := []SearchResult{
|
||||
@@ -113,6 +156,33 @@ func TestSiteSearchKeywordCanUseIMDB(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiteSearchKeywordsIncludeAliasesAndCleanedKeywords(t *testing.T) {
|
||||
sub := &model.Subscription{
|
||||
Name: "真人快打2 自动订阅",
|
||||
FeedURL: "site-search://search?keyword=%E7%9C%9F%E4%BA%BA%E5%BF%AB%E6%89%932%202026&alias=Mortal%20Kombat%20II%202026",
|
||||
Filter: "真人快打2 2026",
|
||||
}
|
||||
|
||||
got := siteSearchKeywords(sub)
|
||||
for _, want := range []string{"真人快打2 2026", "Mortal Kombat II 2026", "真人快打2", "Mortal Kombat II"} {
|
||||
if !containsString(got, want) {
|
||||
t.Fatalf("keywords = %#v, missing %q", got, want)
|
||||
}
|
||||
}
|
||||
if got[0] != "真人快打2 2026" {
|
||||
t.Fatalf("primary keyword = %q, want feed keyword first", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestStableSiteSearchGUIDIgnoresPrivateTokenChanges(t *testing.T) {
|
||||
item := SearchResult{
|
||||
SiteID: "mteam",
|
||||
@@ -130,6 +200,33 @@ func TestStableSiteSearchGUIDIgnoresPrivateTokenChanges(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesWithStatsExplainsFiltering(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "Stats Show 自动订阅", Filter: "Stats Show", MediaType: "tv"}
|
||||
seenItem := SearchResult{Title: "Stats Show S01E02 1080p", DownloadURL: "https://pt/download/seen", Seeders: 50}
|
||||
seenGUID := stableSiteSearchGUID(seenItem, seenItem.DownloadURL)
|
||||
results := []SearchResult{
|
||||
{Title: "Different Show S01E01 1080p", DownloadURL: "https://pt/download/wrong", Seeders: 90},
|
||||
{Title: "Stats Show S01E01 CAM", DownloadURL: "https://pt/download/cam", Seeders: 80},
|
||||
{Title: "Stats Show S01E02 1080p", Seeders: 70},
|
||||
seenItem,
|
||||
{Title: "Stats Show S01E03 1080p", DownloadURL: "https://pt/download/right", Seeders: 60},
|
||||
}
|
||||
|
||||
got, stats := selectSiteSearchCandidatesWithStats(results, sub, map[string]struct{}{seenGUID: {}}, LocalAvailability{})
|
||||
if len(got) != 1 || got[0].Download != "https://pt/download/right" {
|
||||
t.Fatalf("selected %#v, want only unfiltered candidate", got)
|
||||
}
|
||||
if stats.Total != 5 ||
|
||||
stats.QueryMismatch != 1 ||
|
||||
stats.RuleMismatch != 1 ||
|
||||
stats.MissingDownload != 1 ||
|
||||
stats.Seen != 1 ||
|
||||
stats.Prepared != 1 ||
|
||||
stats.Selected != 1 {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSubscriptionRemovesDownloaderTaskAndSeenState(t *testing.T) {
|
||||
const title = "Delete Subscription Show S01E01 1080p"
|
||||
const hash = "abcdef1234567890abcdef1234567890abcdef12"
|
||||
@@ -306,6 +403,100 @@ func TestSelectSiteSearchCandidatesFullPackUsedAsFallbackWhenLibraryPartiallyExi
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesMissingEpisodeCanMatchSubtitleAlias(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "躲在超市后门抽烟的两人 自动订阅", Filter: "躲在超市后门抽烟的两人", MediaType: "tv", TotalEpisodes: 12}
|
||||
results := []SearchResult{
|
||||
{Title: "Smoking Behind the Supermarket with You S01E01 1080p", Subtitle: "躲在超市后门抽烟的两人", DownloadURL: "https://pt/download/1", Seeders: 100},
|
||||
{Title: "Smoking Behind the Supermarket with You S01E12 1080p", Subtitle: "躲在超市后门抽烟的两人", DownloadURL: "https://pt/download/12", Seeders: 80},
|
||||
}
|
||||
existing := map[string]struct{}{}
|
||||
for episode := 1; episode <= 11; episode++ {
|
||||
existing[episodeKey(1, episode)] = struct{}{}
|
||||
}
|
||||
availability := LocalAvailability{
|
||||
TotalEpisodes: 12,
|
||||
LocalMediaCount: 11,
|
||||
MissingEpisodes: []int{12},
|
||||
ExistingEpisodeKeys: existing,
|
||||
}
|
||||
|
||||
got := selectSiteSearchCandidates(results, sub, map[string]struct{}{}, availability)
|
||||
if len(got) != 1 || got[0].Episode != 12 || got[0].Download != "https://pt/download/12" {
|
||||
t.Fatalf("selected %#v, want subtitle-matched missing episode 12", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesRelaxesQueryForExistingSeriesMissingEpisodes(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "翘楚 S01E06 自动订阅", Filter: "翘楚 S01E06", MediaType: "tv", TotalEpisodes: 24}
|
||||
results := []SearchResult{
|
||||
{Title: "Qiao Chu 2026 S01E06 2160p WEB-DL", DownloadURL: "https://pt/download/6", Seeders: 10},
|
||||
{Title: "Ashes to Crown 2026 S01E21 2160p WEB-DL", DownloadURL: "https://pt/download/21", Seeders: 8},
|
||||
{Title: "Ashes to Crown 2026 S01E99 2160p WEB-DL", DownloadURL: "https://pt/download/99", Seeders: 99},
|
||||
}
|
||||
availability := LocalAvailability{
|
||||
TotalEpisodes: 24,
|
||||
LocalMediaCount: 1,
|
||||
MissingEpisodes: []int{21},
|
||||
ExistingEpisodeKeys: map[string]struct{}{episodeKey(1, 6): {}},
|
||||
}
|
||||
|
||||
got, stats := selectSiteSearchCandidatesWithStats(results, sub, map[string]struct{}{}, availability)
|
||||
if len(got) != 1 || got[0].Episode != 21 || got[0].Download != "https://pt/download/21" {
|
||||
t.Fatalf("selected %#v, want relaxed alias-like missing episode 21 only", got)
|
||||
}
|
||||
if stats.QueryMismatch != 3 || stats.RelaxedQueryMatch != 3 || stats.ExistingEpisodeSkipped != 1 || stats.NotMissingEpisodeSkipped != 1 {
|
||||
t.Fatalf("unexpected relaxed stats: %#v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddSiteSearchCandidateAvailabilityTracksRelaxedAliasCandidate(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "翘楚 S01E06 自动订阅", Filter: "翘楚 S01E06", MediaType: "tv", TotalEpisodes: 24}
|
||||
availability := LocalAvailability{
|
||||
TotalEpisodes: 24,
|
||||
LocalMediaCount: 1,
|
||||
MissingEpisodes: []int{21},
|
||||
ExistingEpisodeKeys: map[string]struct{}{episodeKey(1, 6): {}},
|
||||
MissingEpisodeKeys: map[string]struct{}{episodeKey(1, 21): {}},
|
||||
}
|
||||
candidate := siteSearchCandidate{
|
||||
Item: SearchResult{
|
||||
Title: "Ashes to Crown 2026 S01E21 2160p WEB-DL",
|
||||
DownloadURL: "https://pt/download/21",
|
||||
},
|
||||
Download: "https://pt/download/21",
|
||||
GUID: "site|m-team|ashes-to-crown-21",
|
||||
Season: 1,
|
||||
Episode: 21,
|
||||
}
|
||||
|
||||
addSiteSearchCandidateAvailability(candidate, &availability)
|
||||
availability = NewSubscriptionService(nil, nil, nil, nil, nil, nil).finalizePendingAvailability(sub, availability)
|
||||
|
||||
if _, ok := availability.ExistingEpisodeKeys[episodeKey(1, 21)]; !ok {
|
||||
t.Fatalf("missing relaxed alias candidate E21 key: %#v", availability.ExistingEpisodeKeys)
|
||||
}
|
||||
got := selectSiteSearchCandidates([]SearchResult{candidate.Item}, sub, map[string]struct{}{}, availability)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("selected %#v, want relaxed alias candidate skipped after dedup availability update", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesDoesNotRelaxQueryForMovies(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "玩具总动员 5 自动订阅", Filter: "玩具总动员 5 2026", MediaType: "movie"}
|
||||
results := []SearchResult{
|
||||
{Title: "Toy Story 4 2019 2160p WEB-DL", DownloadURL: "https://pt/download/wrong", Seeders: 100},
|
||||
}
|
||||
availability := LocalAvailability{}
|
||||
|
||||
got, stats := selectSiteSearchCandidatesWithStats(results, sub, map[string]struct{}{}, availability)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("selected %#v, want no relaxed movie match", got)
|
||||
}
|
||||
if stats.QueryMismatch != 1 || stats.RelaxedQueryMatch != 0 {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectSiteSearchCandidatesSingleExistingMovieIsSkippedWhenNotWashing(t *testing.T) {
|
||||
sub := &model.Subscription{Name: "Inception 自动订阅", Filter: "Inception 2010", MediaType: "movie"}
|
||||
results := []SearchResult{
|
||||
@@ -419,6 +610,47 @@ func TestSubscriptionPendingDownloadAvailabilityIncludesQueuedTasks(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubscriptionPendingDownloadAvailabilityIncludesLinkedAliasTask(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.DownloadTask{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repos := repository.New(db)
|
||||
sub := &model.Subscription{
|
||||
Base: model.Base{ID: "sub-qiao-chu"},
|
||||
Name: "翘楚 S01E06 自动订阅",
|
||||
Filter: "翘楚 S01E06",
|
||||
MediaType: "tv",
|
||||
SavePath: "/downloads/tv",
|
||||
TotalEpisodes: 24,
|
||||
}
|
||||
if err := repos.Download.Create(t.Context(), &model.DownloadTask{
|
||||
SubscriptionID: sub.ID,
|
||||
Source: "qbittorrent",
|
||||
URL: "https://pt/download/21",
|
||||
Title: "Ashes to Crown 2026 S01E21 2160p WEB-DL",
|
||||
SavePath: "/downloads/tv",
|
||||
Status: "queued",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
svc := NewSubscriptionService(nil, nil, repos, nil, nil, nil)
|
||||
|
||||
availability := svc.pendingDownloadAvailability(t.Context(), sub)
|
||||
if _, ok := availability.ExistingEpisodeKeys[episodeKey(1, 21)]; !ok {
|
||||
t.Fatalf("missing linked alias E21 key: %#v", availability.ExistingEpisodeKeys)
|
||||
}
|
||||
got := selectSiteSearchCandidates([]SearchResult{
|
||||
{Title: "Ashes to Crown 2026 S01E21 2160p WEB-DL", DownloadURL: "https://pt/download/21", Seeders: 80},
|
||||
}, sub, map[string]struct{}{}, availability)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("selected %#v, want linked alias task to satisfy E21", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubscriptionEnrichProgressIncludesPendingDownloads(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
@@ -735,6 +967,13 @@ func TestInferSubscriptionTotalEpisodesFromSearchAndRSS(t *testing.T) {
|
||||
if got := inferSearchTotalEpisodes(results, sub); got != 12 {
|
||||
t.Fatalf("search inferred total = %d, want 12", got)
|
||||
}
|
||||
subtitleResults := []SearchResult{
|
||||
{Title: "Smoking Behind the Supermarket with You", Subtitle: "躲在超市后门抽烟的两人 S01E12"},
|
||||
}
|
||||
subtitleSub := &model.Subscription{Name: "躲在超市后门抽烟的两人 自动订阅", Filter: "躲在超市后门抽烟的两人", MediaType: "tv"}
|
||||
if got := inferSearchTotalEpisodes(subtitleResults, subtitleSub); got != 12 {
|
||||
t.Fatalf("subtitle search inferred total = %d, want 12", got)
|
||||
}
|
||||
items := []rssItem{
|
||||
{Title: "Some Show S01E02 WEB-DL"},
|
||||
{Title: "Some Show S01E10 WEB-DL"},
|
||||
|
||||
@@ -69,7 +69,7 @@ func transferFile(src, dst string, mode TransferMode) error {
|
||||
// 即使在宿主机上同属一块盘,容器内 os.Link 也会因跨文件系统
|
||||
// (EXDEV) 失败。hardlink 模式必须保持零额外数据占用语义,不能
|
||||
// 自动降级为复制;需要复制时请显式选择 copy。
|
||||
return fmt.Errorf("hardlink failed: %w; source and target must be on the same filesystem, choose copy if you want to duplicate data", err)
|
||||
return fmt.Errorf("hardlink failed: %w; source and target must be on the same filesystem/subvolume from inside the container. If you selected move, disable keep_seeding first because keep_seeding upgrades move to hardlink; choose copy if you want to keep seeding across mounts", err)
|
||||
}
|
||||
return nil
|
||||
case TransferSymlink:
|
||||
|
||||
@@ -14,6 +14,7 @@ export interface ExternalMediaResult {
|
||||
source: string
|
||||
media_type?: string
|
||||
title: string
|
||||
original_name?: string
|
||||
overview?: string
|
||||
poster_url?: string
|
||||
backdrop_url?: string
|
||||
|
||||
@@ -8,6 +8,7 @@ export interface SiteSearchResult {
|
||||
site_name: string
|
||||
site_id: string
|
||||
title: string
|
||||
subtitle?: string
|
||||
torrent_url: string
|
||||
download_url: string
|
||||
category?: string
|
||||
|
||||
@@ -1,6 +1,23 @@
|
||||
import { api } from './client'
|
||||
import type { Subscription } from '../types'
|
||||
|
||||
export function buildSiteSearchFeedURL(keyword: string, source?: string, aliases: string[] = []) {
|
||||
const params = new URLSearchParams()
|
||||
params.set('keyword', keyword)
|
||||
if (source) params.set('source', source)
|
||||
const seen = new Set([keyword.trim().toLowerCase()])
|
||||
aliases
|
||||
.map((alias) => alias.trim())
|
||||
.filter(Boolean)
|
||||
.forEach((alias) => {
|
||||
const key = alias.toLowerCase()
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
params.append('alias', alias)
|
||||
})
|
||||
return `site-search://search?${params.toString()}`
|
||||
}
|
||||
|
||||
export const subscriptionsAPI = {
|
||||
list: () =>
|
||||
api.get<{ items: Subscription[] }>('/subscriptions').then((r) => r.data.items),
|
||||
|
||||
@@ -5,7 +5,7 @@ import toast from 'react-hot-toast'
|
||||
|
||||
import { aiAPI, type ExternalMediaResult, type SearchIntent } from '../api/ai'
|
||||
import { imageURL } from '../api/client'
|
||||
import { subscriptionsAPI } from '../api/subscriptions'
|
||||
import { buildSiteSearchFeedURL, subscriptionsAPI } from '../api/subscriptions'
|
||||
import { MediaCard } from '../components/MediaCard'
|
||||
import type { Media } from '../types'
|
||||
import { groupSeries, seriesCardLink } from '../utils/groupSeries'
|
||||
@@ -238,7 +238,7 @@ export function AIAssistantPage() {
|
||||
onClick={async () => {
|
||||
setSubscribing(key)
|
||||
try {
|
||||
const feed = `site-search://search?keyword=${encodeURIComponent(keyword)}&source=${encodeURIComponent(item.source)}`
|
||||
const feed = buildSiteSearchFeedURL(keyword, item.source, [item.title, item.original_name || ''])
|
||||
const sub = await subscriptionsAPI.create({
|
||||
name: `${item.title} 自动订阅`,
|
||||
feed_url: feed,
|
||||
|
||||
@@ -4,7 +4,7 @@ import { AlertTriangle, Download, Info, Rss, Sparkles, X } from 'lucide-react'
|
||||
|
||||
import { discoverAPI, type DiscoverItem, type DiscoverSection } from '../api/discover'
|
||||
import { imageURL } from '../api/client'
|
||||
import { subscriptionsAPI } from '../api/subscriptions'
|
||||
import { buildSiteSearchFeedURL, subscriptionsAPI } from '../api/subscriptions'
|
||||
|
||||
const defaultSections = [
|
||||
'tmdb_trending_day',
|
||||
@@ -258,7 +258,7 @@ function DiscoverDetailModal({ item, onClose }: { item: DiscoverItem; onClose: (
|
||||
|
||||
const submit = async () => {
|
||||
const finalKeyword = form.keyword.trim() || keyword
|
||||
const feed = `site-search://search?keyword=${encodeURIComponent(finalKeyword)}&source=${encodeURIComponent(source)}`
|
||||
const feed = buildSiteSearchFeedURL(finalKeyword, source, [item.title, item.original_name || ''])
|
||||
setBusy(true)
|
||||
try {
|
||||
const sub = await subscriptionsAPI.create({
|
||||
|
||||
@@ -200,6 +200,8 @@ export function FileManagerPage() {
|
||||
() => libraries.filter((library) => !isCloudLibraryPath(library.path)),
|
||||
[libraries],
|
||||
)
|
||||
const autoMoveKeepsSeeding = autoConfig.transferMode === 'move' && settingOn(autoConfig.keepSeeding)
|
||||
const manualMoveKeepsSeeding = organizeTransferMode === 'move' && settingOn(autoConfig.keepSeeding)
|
||||
|
||||
const refresh = useCallback(() => {
|
||||
setLoading(true)
|
||||
@@ -568,7 +570,7 @@ export function FileManagerPage() {
|
||||
onChange={(event) => changeAutoConfig('transferMode', event.target.value)}
|
||||
>
|
||||
<option value="hardlink">硬链接</option>
|
||||
<option value="move">移动</option>
|
||||
<option value="move">移动(关闭保种才会移动)</option>
|
||||
<option value="copy">复制</option>
|
||||
<option value="symlink">软链接</option>
|
||||
</select>
|
||||
@@ -585,6 +587,13 @@ export function FileManagerPage() {
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{autoMoveKeepsSeeding && (
|
||||
<div className="rounded-xl border border-orange-300 bg-orange-50 px-3 py-2 text-xs text-orange-700">
|
||||
当前同时选择了“移动”和“保种”。为避免 qB 做种源文件被删除,后端会实际使用硬链接;Docker / NAS
|
||||
多挂载或不同子卷下可能报 invalid cross-device link。需要真正移动时请关闭“保种”,需要保种但硬链接失败时请选择“复制”。
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<label className="flex items-center gap-2 rounded-lg border border-gray-200 bg-gray-50 px-2 py-1 text-xs text-ink-100">
|
||||
<input
|
||||
@@ -780,13 +789,19 @@ export function FileManagerPage() {
|
||||
<span className="text-xs text-ink-50">整理方式</span>
|
||||
<select className="input-base w-full" value={organizeTransferMode} onChange={(event) => setOrganizeTransferMode(event.target.value)}>
|
||||
<option value="hardlink">硬链接</option>
|
||||
<option value="move">移动</option>
|
||||
<option value="move">移动(关闭保种才会移动)</option>
|
||||
<option value="copy">复制</option>
|
||||
<option value="symlink">软链接</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{manualMoveKeepsSeeding && (
|
||||
<div className="rounded-xl border border-orange-300 bg-orange-50 px-3 py-2 text-xs text-orange-700">
|
||||
“保种”已开启,选择“移动”时后端会改用硬链接以保留下载源。要执行真正移动,请先在上方自动整理设置里关闭“保种”并保存。
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<label className="flex items-center gap-2 rounded-lg border border-gray-200 bg-gray-50 px-2 py-1 text-xs text-ink-100">
|
||||
<input type="checkbox" checked={scanAfter} onChange={(event) => setScanAfter(event.target.checked)} />
|
||||
|
||||
@@ -6,7 +6,7 @@ import { CheckCircle2, Info, Rss, Sparkles } from 'lucide-react'
|
||||
import { aiAPI, type ExternalMediaResult, type SearchIntent } from '../api/ai'
|
||||
import { imageURL } from '../api/client'
|
||||
import { mediaAPI } from '../api/library'
|
||||
import { subscriptionsAPI } from '../api/subscriptions'
|
||||
import { buildSiteSearchFeedURL, subscriptionsAPI } from '../api/subscriptions'
|
||||
import { MediaCard } from '../components/MediaCard'
|
||||
import type { Media } from '../types'
|
||||
import { groupSeries, seriesCardLink } from '../utils/groupSeries'
|
||||
@@ -219,7 +219,7 @@ export function SearchPage() {
|
||||
const key = `${item.source}:${keyword}`
|
||||
setSubscribing(key)
|
||||
try {
|
||||
const feed = `site-search://search?keyword=${encodeURIComponent(keyword)}&source=${encodeURIComponent(item.source)}`
|
||||
const feed = buildSiteSearchFeedURL(keyword, item.source, [item.title, item.original_name || ''])
|
||||
const sub = await subscriptionsAPI.create({
|
||||
name: `${item.title} 自动订阅`,
|
||||
feed_url: feed,
|
||||
|
||||
@@ -95,6 +95,7 @@ export function SiteFormModal({
|
||||
<option value="gazelle">Gazelle(HDBits等)</option>
|
||||
<option value="unit3d">UNIT3D(BeyondHD等)</option>
|
||||
<option value="mteam">馒头 M-Team(专用API)</option>
|
||||
<option value="yemapt">YemaPT(API Auth)</option>
|
||||
<option value="discuz">Discuz 论坛型</option>
|
||||
<option value="custom_rss">自定义 RSS</option>
|
||||
</select>
|
||||
@@ -156,6 +157,27 @@ export function SiteFormModal({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{form.type === "yemapt" && (
|
||||
<div className="p-3 rounded-xl border border-cyan-500/30 bg-cyan-500/5">
|
||||
<div className="text-sm font-medium text-cyan-400 mb-1">
|
||||
YemaPT 配置指南
|
||||
</div>
|
||||
<div className="text-xs text-ink-50 space-y-1">
|
||||
<div>
|
||||
<b>站点地址:</b>
|
||||
<code className="text-cyan-300">https://www.yemapt.org</code>
|
||||
</div>
|
||||
<div>
|
||||
<b>认证方式:</b>使用个人详情页创建的第三方对接专用 auth
|
||||
</div>
|
||||
<div className="pl-3 text-sand-500">
|
||||
填入下方 API Key;后端会按 Wiki 要求通过 Authorization
|
||||
请求头原样发送。
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<label className="block text-sm text-ink-50 mb-2">认证方式</label>
|
||||
<div className="flex gap-2 mb-3">
|
||||
@@ -229,6 +251,8 @@ export function SiteFormModal({
|
||||
<p className="text-xs text-sand-500 mt-1">
|
||||
{form.type === "mteam"
|
||||
? "馒头:控制台 → 实验室 → 存取令牌;第三方工具通过 x-api-key 请求头访问"
|
||||
: form.type === "yemapt"
|
||||
? "YemaPT:个人详情页 → 第三方对接专用 auth;通过 Authorization 请求头原样访问"
|
||||
: "站点的访问 API Key"}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
+21
-16
@@ -15,6 +15,12 @@ import {
|
||||
import { SiteCard } from "./SiteCard";
|
||||
import { SiteFormModal } from "./SiteFormModal";
|
||||
|
||||
function apiErrorMessage(err: unknown, fallback: string): string {
|
||||
const data = (err as { response?: { data?: { message?: string; error?: string } } })
|
||||
?.response?.data;
|
||||
return data?.message || data?.error || (err instanceof Error ? err.message : fallback);
|
||||
}
|
||||
|
||||
export function SitesPage() {
|
||||
const [sites, setSites] = useState<Site[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
@@ -65,8 +71,10 @@ export function SitesPage() {
|
||||
setEditingId(null);
|
||||
};
|
||||
|
||||
const silentSave = async (): Promise<boolean> => {
|
||||
if (!form.name.trim() || !form.url.trim()) return false;
|
||||
const silentSave = async (): Promise<{ ok: boolean; message?: string }> => {
|
||||
if (!form.name.trim() || !form.url.trim()) {
|
||||
return { ok: false, message: "站点名称和地址不能为空" };
|
||||
}
|
||||
const payload = siteFormToPayload(form, !editingId);
|
||||
try {
|
||||
if (editingId) {
|
||||
@@ -75,22 +83,22 @@ export function SitesPage() {
|
||||
const response = await sitesAPI.create(payload);
|
||||
setEditingId((response.data as Site)?.id ?? null);
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
return { ok: true };
|
||||
} catch (err: unknown) {
|
||||
return { ok: false, message: apiErrorMessage(err, "保存失败") };
|
||||
}
|
||||
};
|
||||
|
||||
const handleSave = async (event: FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault();
|
||||
setSaving(true);
|
||||
const ok = await silentSave();
|
||||
if (ok) {
|
||||
const result = await silentSave();
|
||||
if (result.ok) {
|
||||
toast.success(editingId ? "站点已更新" : "站点已添加");
|
||||
closeModal();
|
||||
await loadSites();
|
||||
} else {
|
||||
toast.error("保存失败");
|
||||
toast.error(result.message || "保存失败");
|
||||
}
|
||||
setSaving(false);
|
||||
};
|
||||
@@ -98,10 +106,10 @@ export function SitesPage() {
|
||||
const handleTest = async (id: string) => {
|
||||
if (editingId === id) {
|
||||
setSaving(true);
|
||||
const ok = await silentSave();
|
||||
const result = await silentSave();
|
||||
setSaving(false);
|
||||
if (!ok) {
|
||||
toast.error("保存失败,无法测试");
|
||||
if (!result.ok) {
|
||||
toast.error(result.message || "保存失败,无法测试");
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -111,10 +119,7 @@ export function SitesPage() {
|
||||
const response = await sitesAPI.test(id);
|
||||
toast.success(response?.message || "连接测试成功");
|
||||
} catch (err: unknown) {
|
||||
const message =
|
||||
(err as { response?: { data?: { message?: string } } })?.response?.data
|
||||
?.message ?? "连接测试失败";
|
||||
toast.error(message);
|
||||
toast.error(apiErrorMessage(err, "连接测试失败"));
|
||||
} finally {
|
||||
setTestingId(null);
|
||||
}
|
||||
@@ -141,7 +146,7 @@ export function SitesPage() {
|
||||
setForm((current) => ({
|
||||
...current,
|
||||
type,
|
||||
auth_type: type === "mteam" ? "api_key" : current.auth_type,
|
||||
auth_type: type === "mteam" || type === "yemapt" ? "api_key" : current.auth_type,
|
||||
}));
|
||||
};
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { Activity } from 'lucide-react'
|
||||
import toast from 'react-hot-toast'
|
||||
import { Activity, Copy } from 'lucide-react'
|
||||
|
||||
import { tasksAPI, type BackgroundTask, type TasksSnapshot } from '../api/tasks'
|
||||
|
||||
@@ -55,16 +56,50 @@ function formatMetrics(metrics?: Record<string, number>): string {
|
||||
.join(' · ')
|
||||
}
|
||||
|
||||
function hasTaskIssues(task: BackgroundTask): boolean {
|
||||
return Boolean(task.metrics?.errors || task.metrics?.scan_errors || task.metrics?.scrape_errors)
|
||||
}
|
||||
|
||||
function statusBadge(task: BackgroundTask) {
|
||||
if (task.status === 'failed') {
|
||||
return <span className="rounded-lg border border-red-400/40 px-1.5 py-0.5 text-xs text-red-500">failed</span>
|
||||
}
|
||||
if (hasTaskIssues(task)) {
|
||||
return <span className="rounded-lg border border-orange-400/40 px-1.5 py-0.5 text-xs text-orange-500">issues</span>
|
||||
}
|
||||
if (task.status === 'completed') {
|
||||
return <span className="rounded-lg border border-emerald-400/40 px-1.5 py-0.5 text-xs text-emerald-500">done</span>
|
||||
}
|
||||
return <span className="rounded-lg border border-yellow-400/40 px-1.5 py-0.5 text-xs text-yellow-500">running</span>
|
||||
}
|
||||
|
||||
function taskCopyText(task: BackgroundTask): string {
|
||||
const lines = [
|
||||
`任务: ${task.name}`,
|
||||
`状态: ${task.status}${hasTaskIssues(task) ? ' (issues)' : ''}`,
|
||||
`阶段: ${task.stage || '-'}`,
|
||||
`来源: ${task.source_path || '-'}`,
|
||||
`目标: ${task.dest_path || '-'}`,
|
||||
`消息: ${task.error || task.message || '-'}`,
|
||||
]
|
||||
const metrics = formatMetrics(task.metrics)
|
||||
if (metrics) lines.push(`指标: ${metrics}`)
|
||||
if (task.details?.length) {
|
||||
lines.push('详情:')
|
||||
lines.push(...task.details)
|
||||
}
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
async function copyTask(task: BackgroundTask) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(taskCopyText(task))
|
||||
toast.success('任务详情已复制')
|
||||
} catch {
|
||||
toast.error('复制失败,请手动选中详情文本')
|
||||
}
|
||||
}
|
||||
|
||||
function BackgroundTaskTable({ tasks, empty }: { tasks: BackgroundTask[]; empty: string }) {
|
||||
if (tasks.length === 0) return <p className="text-sand-500">{empty}</p>
|
||||
return (
|
||||
@@ -90,14 +125,24 @@ function BackgroundTaskTable({ tasks, empty }: { tasks: BackgroundTask[]; empty:
|
||||
<td className="text-ink-100">{task.stage || '-'}</td>
|
||||
<td>{statusBadge(task)}</td>
|
||||
<td className="max-w-md text-ink-100">
|
||||
<div>{task.error || task.message || '-'}</div>
|
||||
<div className="flex items-start gap-2">
|
||||
<div className="min-w-0 flex-1 select-text break-words">{task.error || task.message || '-'}</div>
|
||||
<button
|
||||
type="button"
|
||||
className="rounded-lg border border-gray-200 bg-white p-1 text-sand-500 hover:border-primary-400/40 hover:text-brand-500"
|
||||
title="复制任务详情"
|
||||
onClick={() => void copyTask(task)}
|
||||
>
|
||||
<Copy size={14} />
|
||||
</button>
|
||||
</div>
|
||||
{formatMetrics(task.metrics) && (
|
||||
<div className="mt-1 text-xs text-sand-500">{formatMetrics(task.metrics)}</div>
|
||||
<div className="mt-1 select-text text-xs text-sand-500">{formatMetrics(task.metrics)}</div>
|
||||
)}
|
||||
{task.details && task.details.length > 0 && (
|
||||
<div className="mt-2 max-h-28 overflow-auto rounded-lg border border-gray-200 bg-gray-50 p-2 font-mono text-[11px] text-sand-600">
|
||||
<div className="mt-2 max-h-64 select-text overflow-auto rounded-lg border border-gray-200 bg-gray-50 p-2 font-mono text-[11px] leading-relaxed text-sand-600">
|
||||
{task.details.map((line, index) => (
|
||||
<div key={`${task.id}-detail-${index}`} className="break-all">
|
||||
<div key={`${task.id}-detail-${index}`} className="whitespace-pre-wrap break-words">
|
||||
{line}
|
||||
</div>
|
||||
))}
|
||||
|
||||
@@ -6,6 +6,7 @@ export const SITE_TYPE_LABELS: Record<string, string> = {
|
||||
gazelle: "Gazelle",
|
||||
unit3d: "UNIT3D",
|
||||
mteam: "M-Team",
|
||||
yemapt: "YemaPT",
|
||||
discuz: "Discuz",
|
||||
custom_rss: "自定义 RSS",
|
||||
};
|
||||
@@ -15,6 +16,7 @@ export const SITE_TYPE_ABBR: Record<string, string> = {
|
||||
gazelle: "GZ",
|
||||
unit3d: "U3",
|
||||
mteam: "MT",
|
||||
yemapt: "YM",
|
||||
discuz: "DZ",
|
||||
custom_rss: "RS",
|
||||
};
|
||||
@@ -24,6 +26,7 @@ export const SITE_TYPE_COLORS: Record<string, string> = {
|
||||
gazelle: "bg-purple-500/15 text-purple-400",
|
||||
unit3d: "bg-orange-500/15 text-orange-400",
|
||||
mteam: "bg-green-500/15 text-green-400",
|
||||
yemapt: "bg-cyan-500/15 text-cyan-400",
|
||||
discuz: "bg-yellow-500/15 text-yellow-400",
|
||||
custom_rss: "bg-sand-500/15 text-ink-50",
|
||||
};
|
||||
|
||||
@@ -376,7 +376,7 @@ export interface SchedulerStatus {
|
||||
export interface Site {
|
||||
id: string
|
||||
name: string
|
||||
type: string // nexusphp / gazelle / unit3d / mteam / discuz / custom_rss
|
||||
type: string // nexusphp / gazelle / unit3d / mteam / yemapt / discuz / custom_rss
|
||||
url: string
|
||||
auth_type: string // cookie / api_key / auth_header
|
||||
cookie?: string // decrypted only in detail view
|
||||
|
||||
Reference in New Issue
Block a user