Compare commits

...

13 Commits

Author SHA1 Message Date
truewhile 465e98a792 bug 2026-09-06 19:02:53 +08:00
truewhile 292ae22dcd fix(sync): deduplicate remoteMeta by file ID to prevent deleting valid remote metadata copies 2026-09-06 17:36:54 +08:00
truewhile bc7e5fc79d 内网挂载emby封面无法加载问题处理 2026-09-06 17:35:33 +08:00
truewhile f7fec93d44 fix(upload): eliminate concurrent temp file name collision and support direct local file upload for 115 2026-09-06 17:05:56 +08:00
truewhile b4a1dc38bb 优化同步逻辑
优化同步逻辑
2026-09-06 16:17:42 +08:00
truewhile 3150971f10 feat(sync): implement 115 adaptive hierarchical flat scan to bypass deep paging limit 2026-09-06 16:15:26 +08:00
truewhile a71a18ce82 fix(upload): 删旧元数据失败时不中止任务,继续上传新文件
旧逻辑:DeleteFiles 失败 → uploadTaskFailWithRetry → 任务重试 →
再次 DeleteFiles 失败 → 永远无法上传,形成死循环。

新逻辑:DeleteFiles 失败时记录 warn 日志后继续上传新文件。
旧副本由下次同步的 scanLocalMetaForUpload 检测(新旧两个副本,
命中新版本后把旧版本 cid 收入 pendingDeletes)并通过
cleanupBatchRedundantFiles 异步批量清理。

这修复了小姐姐库在 115 限流环境下每次增量同步都重复上传
大量元数据的问题(Heyzo 等目录文件被上传了 7 次以上)。
2026-09-06 13:42:20 +08:00
truewhile 5a189a44fc Merge pull request #27 from gaodyoffice/fix/strm-sync-metadata
fix: OpenList metadata download 401 - use API instead of WebDAV
2026-09-06 12:30:40 +08:00
truewhile fb84c62e9a fix(sync): fallback to recursive traversal when 115 flat list hits deep-paging limit
115 API's flat list (search under the hood) enforces a hard limit of offset+limit <= 10000.
When syncing huge directories (e.g. >10000 files), this silently truncates results,
causing remote files to appear as missing locally, leading to infinite metadata
re-upload loops and potential wrongful deletion of strm files.

This patch auto-detects if the total file count >= 9500 and dynamically
falls back to the standard recursive concurrent traversal (walkRemote).
2026-09-06 12:27:56 +08:00
Gaodaiyang 355fd06036 fix: OpenList metadata download using API instead of WebDAV
OpenList 同步目录下载元数据(nfo/jpg/png/srt 等)全部失败,错误 http 401。

根因:Resolve() 中非视频文件走 WebDAV 直接下载,用 API token 作为 Authorization。
但 AList WebDAV 端点不接受 API token 认证,需要 Basic Auth。

修复:OpenList 在有 apiBase 时,所有文件都走 API /api/fs/get 获取直链,
不再走 WebDAV。API 失败时非视频文件可回退到 WebDAV。

影响范围:
- 只影响 OpenList 类型的非视频文件下载
- 不影响 115、CloudDrive2 等其他网盘
- 不影响视频播放/strm 生成
2026-09-06 11:41:12 +08:00
truewhile 4173caac5d bug处理 2026-09-06 01:16:30 +08:00
truewhile 2aeedcc182 bug处理 2026-09-06 00:01:40 +08:00
truewhile d96782622d bug处理 2026-09-05 23:02:26 +08:00
27 changed files with 1656 additions and 398 deletions
+25 -8
View File
@@ -170,7 +170,13 @@ func listLibrariesHandler(svc *service.Container) gin.HandlerFunc {
if acct == nil {
return
}
tmpMount := &model.EmbyMount{Base: model.Base{ID: v.MountID}}
tmpMount := &model.EmbyMount{
Base: model.Base{ID: v.MountID},
AccountID: v.AccountID,
RemoteViewID: v.RemoteID,
CollectionType: v.CollectionType,
Name: v.Library.Name,
}
itemTypes := remoteLibraryItemTypes(v.CollectionType)
if _, total, err := svc.EmbyRemote.RemoteLibraryMedia(ctx, tmpMount, acct, v.RemoteID, itemTypes, 0, 1); err == nil {
remotePayloads[i].Total = total
@@ -660,14 +666,25 @@ func streamHandler(svc *service.Container) gin.HandlerFunc {
}
return
}
target, err := svc.EmbyRemote.WebStreamURL(ctx, acct, remoteID)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
target, err := svc.EmbyRemote.WebStreamURL(ctx, acct, remoteID)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
// 现代浏览器在 HTTPS 页面中请求不安全源(HTTP 视频流)会直接报 Mixed Content 拦截导致播放失败。
// 仅当当前前端请求为 HTTPS 且远程直连目标为 HTTP 时,自动降级通过本机反向代理传输流,避免播放被浏览器阻断;
// 其它场景(HTTP 页面访问 HTTP/HTTPS,或 HTTPS 访问 HTTPS)继续 302 直连,最大化节省服务器带宽与流量。
if requestIsHTTPS(c) && strings.HasPrefix(strings.ToLower(target), "http://") {
if err := svc.Emby.ProxyRemoteVideoStream(ctx, c.Writer, c.Request, mountID, remoteID); err != nil {
if !c.Writer.Written() {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
}
}
return
}
setRedirectNoStoreHeaders(c)
c.Redirect(http.StatusFound, target)
return
}
setRedirectNoStoreHeaders(c)
c.Redirect(http.StatusFound, target)
return
}
m, err := svc.Media.GetMedia(ctx, id)
if err != nil || m == nil || !mediaVisibleForRequest(c, svc, m) {
+1
View File
@@ -62,6 +62,7 @@ func registerAdminStrmRoutes(admin *gin.RouterGroup, svc *service.Container) {
admin.DELETE("/strm/accounts/:id", deleteStrmAccountHandler(svc))
admin.POST("/strm/accounts/:id/test", testStrmAccountHandler(svc))
admin.GET("/strm/accounts/:id/list", listStrmRemoteDirHandler(svc))
admin.GET("/strm/accounts/:id/resolve", resolveStrmRemoteDirHandler(svc))
admin.GET("/strm/115/sources", listStrm115SourcesHandler(svc))
admin.POST("/strm/accounts/:id/oauth/start", startStrm115OAuthHandler(svc))
admin.POST("/strm/accounts/:id/oauth/poll", pollStrm115OAuthHandler(svc))
+62 -36
View File
@@ -3,6 +3,7 @@
package handler
import (
"context"
"errors"
"net/http"
"net/url"
@@ -172,6 +173,19 @@ func listStrmRemoteDirHandler(svc *service.Container) gin.HandlerFunc {
}
}
// resolveStrmRemoteDirHandler 按远端目录引用(115 为目录 ID)反查完整展示路径。
func resolveStrmRemoteDirHandler(svc *service.Container) gin.HandlerFunc {
return func(c *gin.Context) {
dir := strings.TrimSpace(c.Query("dir"))
path, err := svc.Strm.ResolveRemoteDirPath(c.Request.Context(), c.Param("id"), dir)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"path": path})
}
}
// ─── 全局设置 ──────────────────────────────────────────────────────────────────
func getStrmSettingsHandler(svc *service.Container) gin.HandlerFunc {
@@ -203,24 +217,25 @@ func updateStrmSettingsHandler(svc *service.Container) gin.HandlerFunc {
// ─── 同步目录 ──────────────────────────────────────────────────────────────────
type strmSyncPathReq struct {
Name string `json:"name"`
AccountID string `json:"account_id"`
Provider string `json:"provider"`
RemotePath string `json:"remote_path"`
LocalPath string `json:"local_path"`
StrmBaseURL string `json:"strm_base_url"`
VideoExt string `json:"video_ext"`
MetaExt string `json:"meta_ext"`
ExcludeName string `json:"exclude_name"`
MinVideoSizeMB int64 `json:"min_video_size_mb"`
AddPath int `json:"add_path"`
DownloadMeta *bool `json:"download_meta"`
UploadMeta *bool `json:"upload_meta"`
DeleteDir *bool `json:"delete_dir"`
Cron string `json:"cron"`
EnableCron *bool `json:"enable_cron"`
SyncMode string `json:"sync_mode"`
Enabled *bool `json:"enabled"`
Name string `json:"name"`
AccountID string `json:"account_id"`
Provider string `json:"provider"`
RemotePath string `json:"remote_path"`
RemoteDisplayPath string `json:"remote_display_path"`
LocalPath string `json:"local_path"`
StrmBaseURL string `json:"strm_base_url"`
VideoExt string `json:"video_ext"`
MetaExt string `json:"meta_ext"`
ExcludeName string `json:"exclude_name"`
MinVideoSizeMB int64 `json:"min_video_size_mb"`
AddPath int `json:"add_path"`
DownloadMeta *bool `json:"download_meta"`
UploadMeta *bool `json:"upload_meta"`
DeleteDir *bool `json:"delete_dir"`
Cron string `json:"cron"`
EnableCron *bool `json:"enable_cron"`
SyncMode string `json:"sync_mode"`
Enabled *bool `json:"enabled"`
}
type strmSyncPathView struct {
@@ -240,6 +255,16 @@ func strmSyncPathViews(svc *service.Container, c *gin.Context, paths []model.Str
view.AccountEnabled = acct.Enabled
}
}
// 历史 115 数据若尚未记录展示路径,尝试反查一次并回写数据库自愈
if p.Provider == model.StrmProvider115 && strings.TrimSpace(p.RemoteDisplayPath) == "" && strings.TrimSpace(p.RemotePath) != "" && p.AccountID != "" {
resolveCtx, cancel := context.WithTimeout(c.Request.Context(), 3*time.Second)
if fullPath, err := svc.Strm.ResolveRemoteDirPath(resolveCtx, p.AccountID, p.RemotePath); err == nil && fullPath != "" {
view.RemoteDisplayPath = fullPath
p.RemoteDisplayPath = fullPath
_ = svc.Repo.StrmSyncPath.Update(context.Background(), &p)
}
cancel()
}
out = append(out, view)
}
return out
@@ -663,24 +688,25 @@ func strmPlayHandler(svc *service.Container) gin.HandlerFunc {
// strmSyncPathFromReq 组装同步目录模型(缺省值交给服务层处理)。
func strmSyncPathFromReq(req strmSyncPathReq) *model.StrmSyncPath {
return &model.StrmSyncPath{
Name: strings.TrimSpace(req.Name),
AccountID: strings.TrimSpace(req.AccountID),
Provider: strings.TrimSpace(req.Provider),
RemotePath: strings.TrimSpace(req.RemotePath),
LocalPath: strings.TrimSpace(req.LocalPath),
StrmBaseURL: strings.TrimSpace(req.StrmBaseURL),
VideoExt: req.VideoExt,
MetaExt: req.MetaExt,
ExcludeName: req.ExcludeName,
MinVideoSizeMB: req.MinVideoSizeMB,
AddPath: req.AddPath,
DownloadMeta: boolValue(req.DownloadMeta, true),
UploadMeta: boolValue(req.UploadMeta, false),
DeleteDir: boolValue(req.DeleteDir, false),
Cron: strings.TrimSpace(req.Cron),
EnableCron: boolValue(req.EnableCron, false),
SyncMode: strings.TrimSpace(req.SyncMode),
Enabled: boolValue(req.Enabled, true),
Name: strings.TrimSpace(req.Name),
AccountID: strings.TrimSpace(req.AccountID),
Provider: strings.TrimSpace(req.Provider),
RemotePath: strings.TrimSpace(req.RemotePath),
RemoteDisplayPath: strings.TrimSpace(req.RemoteDisplayPath),
LocalPath: strings.TrimSpace(req.LocalPath),
StrmBaseURL: strings.TrimSpace(req.StrmBaseURL),
VideoExt: req.VideoExt,
MetaExt: req.MetaExt,
ExcludeName: req.ExcludeName,
MinVideoSizeMB: req.MinVideoSizeMB,
AddPath: req.AddPath,
DownloadMeta: boolValue(req.DownloadMeta, true),
UploadMeta: boolValue(req.UploadMeta, false),
DeleteDir: boolValue(req.DeleteDir, false),
Cron: strings.TrimSpace(req.Cron),
EnableCron: boolValue(req.EnableCron, false),
SyncMode: strings.TrimSpace(req.SyncMode),
Enabled: boolValue(req.Enabled, true),
}
}
+5 -1
View File
@@ -36,7 +36,11 @@ type StrmSyncPath struct {
AccountID string `gorm:"size:36;index" json:"account_id"` // StrmAccount.ID;local 为空
Provider string `gorm:"size:32" json:"provider"` // StrmProvider*(冗余,便于列表展示)
RemotePath string `gorm:"size:1024" json:"remote_path"` // 远端目录:115=目录ID,OpenList/CD2=路径,local=源目录
LocalPath string `gorm:"size:1024" json:"local_path"` // STRM/元数据本地输出目录
// RemoteDisplayPath 是远端目录的完整展示路径(如 /电影/剧集)。115 的
// RemotePath 是目录 ID,用户无法辨认,浏览选择或按 ID 反查时把人类可读
// 路径存到这里;路径型网盘(CD2/OpenList)与 local 留空(RemotePath 即路径)。
RemoteDisplayPath string `gorm:"size:1024" json:"remote_display_path"`
LocalPath string `gorm:"size:1024" json:"local_path"` // STRM/元数据本地输出目录
// STRM 链接配置(空值继承全局 strm.* 设置)
StrmBaseURL string `gorm:"size:512" json:"strm_base_url"` // 覆盖 strm.base_url
VideoExt string `gorm:"size:512" json:"video_ext"` // 逗号分隔,覆盖 strm.video_ext
+23 -22
View File
@@ -95,28 +95,29 @@ func (r *StrmSyncPathRepository) List(ctx context.Context) ([]model.StrmSyncPath
func (r *StrmSyncPathRepository) Update(ctx context.Context, p *model.StrmSyncPath) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Model(&model.StrmSyncPath{}).Where("id = ?", p.ID).Updates(map[string]any{
"name": p.Name,
"account_id": p.AccountID,
"provider": p.Provider,
"remote_path": p.RemotePath,
"local_path": p.LocalPath,
"strm_base_url": p.StrmBaseURL,
"video_ext": p.VideoExt,
"meta_ext": p.MetaExt,
"exclude_name": p.ExcludeName,
"min_video_size_mb": p.MinVideoSizeMB,
"add_path": p.AddPath,
"download_meta": p.DownloadMeta,
"upload_meta": p.UploadMeta,
"delete_dir": p.DeleteDir,
"cron": p.Cron,
"enable_cron": p.EnableCron,
"sync_mode": p.SyncMode,
"enabled": p.Enabled,
"last_sync_at": p.LastSyncAt,
"last_sync_status": p.LastSyncStatus,
"last_sync_message": p.LastSyncMessage,
"updated_at": time.Now(),
"name": p.Name,
"account_id": p.AccountID,
"provider": p.Provider,
"remote_path": p.RemotePath,
"remote_display_path": p.RemoteDisplayPath,
"local_path": p.LocalPath,
"strm_base_url": p.StrmBaseURL,
"video_ext": p.VideoExt,
"meta_ext": p.MetaExt,
"exclude_name": p.ExcludeName,
"min_video_size_mb": p.MinVideoSizeMB,
"add_path": p.AddPath,
"download_meta": p.DownloadMeta,
"upload_meta": p.UploadMeta,
"delete_dir": p.DeleteDir,
"cron": p.Cron,
"enable_cron": p.EnableCron,
"sync_mode": p.SyncMode,
"enabled": p.Enabled,
"last_sync_at": p.LastSyncAt,
"last_sync_status": p.LastSyncStatus,
"last_sync_message": p.LastSyncMessage,
"updated_at": time.Now(),
}).Error
})
}
+7 -7
View File
@@ -87,15 +87,15 @@ func (p *cloudDrive2Provider) Resolve(ctx context.Context, fileRef string) (*Dir
if ref == "/" {
return nil, fmt.Errorf("%s: file reference required", p.name)
}
if p.typ == TypeOpenList && isCloudVideoPlaybackCandidate(ref) {
if p.apiBase == nil {
return nil, fmt.Errorf("%s: pure 302 playback requires an OpenList API server address; configure server/api_url so /api/fs/get can return raw_url", p.name)
}
if p.typ == TypeOpenList && p.apiBase != nil {
link, err := p.resolveOpenListAPIDirect(ctx, ref)
if err != nil {
return nil, fmt.Errorf("%s: pure 302 playback requires OpenList raw_url for %s: %w", p.name, ref, err)
if err == nil {
return link, nil
}
// API 获取直链失败:非视频文件(元数据)回退到 WebDAV;视频文件报错
if isCloudVideoPlaybackCandidate(ref) {
return nil, fmt.Errorf("%s: resolve download URL for %s via API failed: %w", p.name, ref, err)
}
return link, nil
}
if p.typ == TypeCloudDrive2 && isCloudVideoPlaybackCandidate(ref) {
link, err := p.resolveCloudDAVRedirectDirect(ctx, ref)
+3 -2
View File
@@ -61,9 +61,10 @@ func TestOpenListWebDAVListAndResolve(t *testing.T) {
if len(entries) != 1 || entries[0].ID != "/Cloud/Movie.mkv" || entries[0].Size != 1024 {
t.Fatalf("entries = %#v", entries)
}
// Video file: API fails → error (no WebDAV fallback for video)
_, err = p.Resolve(context.Background(), entries[0].ID)
if err == nil || !strings.Contains(err.Error(), "pure 302 playback requires OpenList raw_url") {
t.Fatalf("openlist video resolve should require raw_url instead of WebDAV proxy fallback, err=%v", err)
if err == nil || !strings.Contains(err.Error(), "resolve download URL") || !strings.Contains(err.Error(), "via API failed") {
t.Fatalf("openlist video resolve should error on API failure, err=%v", err)
}
}
@@ -195,10 +195,43 @@ func TestOpenListResolveDoesNotFallbackToWebDAVWhenAPIRawURLFails(t *testing.T)
t.Fatal(err)
}
_, err = p.Resolve(context.Background(), "/Cloud/Movie.mkv")
if err == nil || !strings.Contains(err.Error(), "pure 302 playback requires OpenList raw_url") {
t.Fatalf("resolve error = %v, want raw_url requirement", err)
if err == nil || !strings.Contains(err.Error(), "resolve download URL") || !strings.Contains(err.Error(), "via API failed") {
t.Fatalf("resolve error = %v, want API resolve failure", err)
}
if davSeen {
t.Fatal("openlist video resolve fell back to WebDAV after raw_url failure")
}
}
func TestOpenListResolveMetadataUsesAPIInsteadOfWebDAV(t *testing.T) {
var gotPath, gotAuth string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
gotAuth = r.Header.Get("Authorization")
if r.Method != http.MethodPost || r.URL.Path != "/api/fs/get" {
t.Fatalf("unexpected request %s %s; metadata should use API, not WebDAV", r.Method, r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":200,"data":{"raw_url":"https://cdn.example.test/poster.jpg?sign=1"}}`))
}))
defer srv.Close()
p, err := New(TypeOpenList, map[string]any{"server": srv.URL, "token": "alist-token"}, srv.Client())
if err != nil {
t.Fatal(err)
}
// .nfo metadata file should use API, not WebDAV
link, err := p.Resolve(context.Background(), "/Cloud/Movie/Movie.nfo")
if err != nil {
t.Fatalf("resolve: %v", err)
}
if gotPath != "/api/fs/get" {
t.Fatalf("api path = %q, want /api/fs/get (metadata should not use WebDAV)", gotPath)
}
if gotAuth != "alist-token" {
t.Fatalf("Authorization = %q, want token", gotAuth)
}
if link.URL != "https://cdn.example.test/poster.jpg?sign=1" {
t.Fatalf("url = %q", link.URL)
}
}
+26 -20
View File
@@ -123,35 +123,41 @@ func (p *openAPI115Provider) ResolveBatch(ctx context.Context, fileRefs []string
// OpenClient 暴露底层客户端(token 刷新用)。
func (p *openAPI115Provider) OpenClient() *cloud115.OpenClient { return p.c }
// PutLocalFile 直接上传本地文件,避免通过 io.Reader 复制临时文件产生的磁盘开销与并发重命名碰撞。
func (p *openAPI115Provider) PutLocalFile(ctx context.Context, parentCID, localPath string) error {
_, err := p.c.Upload(ctx, localPath, parentCID, "", "")
return err
}
// PutFileNamed 把本地元数据上传到 115 指定父目录(parentCID 为父目录 cid)。
// io.Reader 无法携带文件名,因此走独立的 named 上传接口。将内容落为临时文件后
// 重命名为目标文件名,再交给 115 上传(/open/upload/init 的 file_name 取真实文件名)。
// 为防止多并发上传线程在同一临时目录下发生同名文件(如 poster.jpg)碰撞覆盖与误删,
// 为每个上传任务分配专属临时子目录。
func (p *openAPI115Provider) PutFileNamed(ctx context.Context, parentCID, fileName string, r io.Reader) error {
tmp, err := os.CreateTemp("", "mebox-upload-*")
tmpDir, err := os.MkdirTemp("", "mebox-upload-*")
if err != nil {
return fmt.Errorf("115: 创建临时目录失败:%w", err)
}
defer func() {
_ = os.RemoveAll(tmpDir)
}()
safeName := filepath.Base(fileName)
if safeName == "" || safeName == "." {
safeName = "file"
}
tmpPath := filepath.Join(tmpDir, safeName)
dst, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil {
return fmt.Errorf("115: 创建临时文件失败:%w", err)
}
tmpPath := tmp.Name()
defer func() {
_ = tmp.Close()
_ = os.Remove(tmpPath)
}()
if _, err := io.Copy(tmp, r); err != nil {
if _, err := io.Copy(dst, r); err != nil {
_ = dst.Close()
return fmt.Errorf("115: 写入临时文件失败:%w", err)
}
if err := tmp.Close(); err != nil {
if err := dst.Close(); err != nil {
return fmt.Errorf("115: 关闭临时文件失败:%w", err)
}
// 重命名为目标文件名,保证上传到 115 后保留原始文件名。
// 重命名失败必须 fail fast:静默用随机临时名上传会导致 115 上的文件名
// 变成 mebox-upload-xxx,破坏元数据文件名契约。
if fileName != "" && fileName != filepath.Base(tmpPath) {
namedPath := filepath.Join(filepath.Dir(tmpPath), fileName)
if err := os.Rename(tmpPath, namedPath); err != nil {
return fmt.Errorf("115: 重命名临时文件为 %s 失败:%w", fileName, err)
}
tmpPath = namedPath
}
_, err = p.c.Upload(ctx, tmpPath, parentCID, "", "")
if err != nil {
return err
+32
View File
@@ -139,6 +139,37 @@ func (r *EmbyRemoteService) ListAccounts(ctx context.Context) ([]model.StrmAccou
return out, nil
}
// ConfiguredRemoteHosts 返回所有已配置的远程 Emby 线路的主机名/IP(去重、不含端口)。
func (r *EmbyRemoteService) ConfiguredRemoteHosts(ctx context.Context) []string {
if r == nil || r.repo == nil || r.repo.StrmAccount == nil {
return nil
}
accounts, err := r.ListAccounts(ctx)
if err != nil || len(accounts) == 0 {
return nil
}
seen := make(map[string]bool)
var hosts []string
for _, acct := range accounts {
lines, _, err := r.LinesOf(&acct)
if err != nil {
continue
}
for _, line := range lines {
u, err := url.Parse(line.URL)
if err != nil || u.Hostname() == "" {
continue
}
h := strings.ToLower(u.Hostname())
if !seen[h] {
seen[h] = true
hosts = append(hosts, h)
}
}
}
return hosts
}
// AccountByID 按 ID 查找远程 Emby 挂载账号(不存在或类型不符返回 nil)。
func (r *EmbyRemoteService) AccountByID(ctx context.Context, id string) *model.StrmAccount {
if strings.TrimSpace(id) == "" {
@@ -789,6 +820,7 @@ func (r *EmbyRemoteService) RemoteLatest(ctx context.Context, mount *model.EmbyM
if parentID != "" {
q.Set("ParentId", parentID)
}
q.Set("Fields", "Overview,Genres,ProviderIds,Path,SeriesPrimaryImage,DateCreated,DateLastMediaAdded,PremiereDate,ProductionYear,CommunityRating,CriticRating")
path := "/Users/" + url.PathEscape(r.remoteUserID(cfg)) + "/Items/Latest"
var out []map[string]any
if err := r.doGet(ctx, acct, cfg, path, q, &out); err != nil {
+70 -16
View File
@@ -180,21 +180,31 @@ func (r *EmbyRemoteService) MapRemoteItemToMedia(ctx context.Context, mount *mod
if _, rid, ok := DecodeEmbyRemoteID(seriesID); ok {
seriesID = rid
}
rating := remoteItemFloat(item, "CommunityRating")
if rating == 0 {
rating = remoteItemFloat(item, "CriticRating")
}
media := model.Media{
Base: model.Base{ID: EncodeEmbyRemoteID(encodeScope, remoteID)},
Title: remoteItemString(item, "Name"),
OriginalName: remoteItemString(item, "OriginalTitle"),
Overview: remoteItemString(item, "Overview"),
Year: remoteItemInt(item, "ProductionYear"),
Rating: float32(rating),
Path: remoteItemString(item, "Path"),
Genres: remoteItemGenres(item),
ScrapeStatus: "done",
}
rating := remoteItemFloat(item, "CommunityRating")
if rating == 0 {
rating = remoteItemFloat(item, "CriticRating")
}
year := remoteItemInt(item, "ProductionYear")
if year == 0 {
year = remoteItemInt(item, "Year")
}
if year == 0 {
year = remoteItemInt(item, "SeriesProductionYear")
}
if year == 0 {
year = remoteItemInt(item, "SeriesYear")
}
media := model.Media{
Base: model.Base{ID: EncodeEmbyRemoteID(encodeScope, remoteID)},
Title: remoteItemString(item, "Name"),
OriginalName: remoteItemString(item, "OriginalTitle"),
Overview: remoteItemString(item, "Overview"),
Year: year,
Rating: float32(rating),
Path: remoteItemString(item, "Path"),
Genres: remoteItemGenres(item),
ScrapeStatus: "done",
}
if date, ok := parseEmbyRemoteDate(remoteItemString(item, "DateCreated")); ok {
media.CreatedAt = date
media.UpdatedAt = date
@@ -561,6 +571,46 @@ func (r *EmbyRemoteService) RemoteLatestCards(ctx context.Context, mount *model.
if r.cache != nil && r.cache.GetJSON(ctx, cacheKey, &cached) {
return cached, nil
}
// 剧集类媒体库:直接拉取最新入库/更新的 Series 剧集本身(按上次添加集日期倒序)。
// 避免 Emby /Items/Latest 默认返回无年份/无系列海报的单集(Episode)。
if mount != nil && (mount.CollectionType == "tvshows" || mount.CollectionType == "tv") {
q := url.Values{}
q.Set("ParentId", remoteViewID)
q.Set("IncludeItemTypes", "Series")
q.Set("Recursive", "false")
q.Set("SortBy", "DateLastContentAdded")
q.Set("SortOrder", "Descending")
q.Set("Limit", strconv.Itoa(limit))
q.Set("Fields", "Overview,Genres,ProviderIds,Path,RecursiveItemCount,SeriesPrimaryImage,DateCreated,DateLastMediaAdded,PremiereDate,ProductionYear,CommunityRating,CriticRating")
var body struct {
Items []map[string]any `json:"Items"`
}
if err := r.doGet(ctx, acct, cfg, "/Users/"+url.PathEscape(r.remoteUserID(cfg))+"/Items", q, &body); err == nil && len(body.Items) > 0 {
cards := make([]SeriesCard, 0, len(body.Items))
for _, it := range body.Items {
RewriteEmbyRemoteIDs(it, mount.ID)
m := r.MapRemoteItemToMedia(ctx, mount, acct, cfg, it)
count := remoteItemInt(it, "RecursiveItemCount")
if count == 0 {
count = remoteItemInt(it, "ChildCount")
}
if count == 0 {
count = 1
}
var lastAdded *time.Time
if date, ok := parseEmbyRemoteDate(remoteItemString(it, "DateLastMediaAdded")); ok {
lastAdded = &date
}
cards = append(cards, SeriesCard{Key: m.ID, Rep: m, LinkMedia: m, Count: count, LastAddedAt: lastAdded})
}
if r.cache != nil {
r.cache.SetJSON(ctx, cacheKey, cards, r.remoteMediaCacheTTL())
}
return cards, nil
}
}
items, err := r.RemoteLatest(ctx, mount, acct, remoteViewID, limit)
if err != nil {
return nil, err
@@ -576,7 +626,11 @@ func (r *EmbyRemoteService) RemoteLatestCards(ctx context.Context, mount *model.
t := m.CreatedAt
lastAdded = &t
}
cards = append(cards, SeriesCard{Key: m.ID, Rep: m, LinkMedia: m, Count: 0, LastAddedAt: lastAdded})
count := remoteItemInt(it, "RecursiveItemCount")
if count == 0 {
count = remoteItemInt(it, "ChildCount")
}
cards = append(cards, SeriesCard{Key: m.ID, Rep: m, LinkMedia: m, Count: count, LastAddedAt: lastAdded})
}
if r.cache != nil {
r.cache.SetJSON(ctx, cacheKey, cards, r.remoteMediaCacheTTL())
+127
View File
@@ -400,3 +400,130 @@ func TestRemoteSearchMedia(t *testing.T) {
t.Fatalf("expected 0 items due to HiddenLibraryIDs, got %d", len(hiddenFiltered))
}
}
func TestRemoteLatestCardsTvShowsYearAndPoster(t *testing.T) {
var requestedFields string
var requestedIncludeItemTypes string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestedFields = r.URL.Query().Get("Fields")
requestedIncludeItemTypes = r.URL.Query().Get("IncludeItemTypes")
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"TotalRecordCount": 1,
"Items": []map[string]any{
{
"Id": "series-100",
"Name": "炒翻天",
"Type": "Series",
"ProductionYear": 2024,
"ImageTags": map[string]any{
"Primary": "tag123",
},
"RecursiveItemCount": 12,
},
},
})
}))
defer server.Close()
db := newServiceTestDB(t, &model.StrmAccount{}, &model.EmbyMount{})
repos := repository.New(db)
svc := NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, NewCryptoService("", zap.NewNop()))
rawConfig, _ := json.Marshal(map[string]string{
"url": server.URL,
"token": "fake-token",
})
acct := &model.StrmAccount{
Base: model.Base{ID: "acct-tv"},
Name: "tv-emby",
Provider: model.StrmProviderEmbyRemote,
Config: string(rawConfig),
Enabled: true,
}
_ = repos.StrmAccount.Create(t.Context(), acct)
mount := &model.EmbyMount{
Base: model.Base{ID: "mount-tv"},
AccountID: acct.ID,
RemoteViewID: "view-tv",
RemoteViewName: "新番连载",
CollectionType: "tvshows",
Enabled: true,
}
_ = repos.EmbyMount.Create(t.Context(), mount)
cards, err := svc.RemoteLatestCards(t.Context(), mount, acct, "view-tv", 10)
if err != nil {
t.Fatalf("RemoteLatestCards failed: %v", err)
}
if len(cards) != 1 {
t.Fatalf("expected 1 card, got %d", len(cards))
}
if cards[0].Rep.Year != 2024 {
t.Fatalf("expected Year 2024, got %d", cards[0].Rep.Year)
}
if cards[0].Count != 12 {
t.Fatalf("expected Count 12, got %d", cards[0].Count)
}
if cards[0].Rep.PosterURL == "" {
t.Fatalf("expected PosterURL not empty")
}
if requestedIncludeItemTypes != "Series" {
t.Fatalf("expected IncludeItemTypes=Series, got %q", requestedIncludeItemTypes)
}
if !strings.Contains(requestedFields, "ProductionYear") {
t.Fatalf("expected Fields to contain ProductionYear, got %q", requestedFields)
}
}
func TestRemoteLatestFields(t *testing.T) {
var requestedFields string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestedFields = r.URL.Query().Get("Fields")
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode([]map[string]any{
{
"Id": "movie-100",
"Name": "测试电影",
"Type": "Movie",
"ProductionYear": 2023,
},
})
}))
defer server.Close()
db := newServiceTestDB(t, &model.StrmAccount{}, &model.EmbyMount{})
repos := repository.New(db)
svc := NewEmbyRemoteService(&config.Config{}, zap.NewNop(), repos, NewCryptoService("", zap.NewNop()))
rawConfig, _ := json.Marshal(map[string]string{
"url": server.URL,
"token": "fake-token",
})
acct := &model.StrmAccount{
Base: model.Base{ID: "acct-movie"},
Name: "movie-emby",
Provider: model.StrmProviderEmbyRemote,
Config: string(rawConfig),
Enabled: true,
}
_ = repos.StrmAccount.Create(t.Context(), acct)
mount := &model.EmbyMount{
Base: model.Base{ID: "mount-movie"},
AccountID: acct.ID,
RemoteViewID: "view-movie",
CollectionType: "movies",
Enabled: true,
}
items, err := svc.RemoteLatest(t.Context(), mount, acct, "view-movie", 10)
if err != nil {
t.Fatalf("RemoteLatest failed: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
if !strings.Contains(requestedFields, "ProductionYear") {
t.Fatalf("expected Fields to contain ProductionYear, got %q", requestedFields)
}
}
+62 -6
View File
@@ -17,6 +17,7 @@ import (
"net"
"net/http"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
@@ -42,6 +43,13 @@ type ImageProxy struct {
libRootsMu sync.Mutex
libRootsCache []string
libRootsAt time.Time
// allowedRemoteHostsFn returns hostnames or IPs of explicitly configured
// upstream services (e.g. remote Emby mounts) that should bypass SSRF private IP checks.
allowedRemoteHostsFn func() []string
allowedHostsMu sync.Mutex
allowedHostsCache map[string]bool
allowedHostsAt time.Time
}
const (
@@ -51,6 +59,12 @@ const (
// NewImageProxy is the constructor.
func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
proxy := &ImageProxy{
cfg: cfg,
log: log,
cacheDir: filepath.Join(cfg.Cache.CacheDir, "images"),
}
// Honor HTTP(S)_PROXY env vars so deployments behind GFW can pull
// from image.tmdb.org via their HTTP proxy without extra config. On
// Windows we also honor the current user's system proxy settings.
@@ -63,6 +77,7 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
// 仅 URL 解析层的 isPrivateHost 可被十进制/十六进制 IP、解析到
// 私网的域名与 DNS rebinding 绕过;在拨号层对最终连接 IP 做二次
// 校验(含重定向后的每条连接)堵住该旁路。
// 用户明确配置的远程挂载源(如内网 Emby)豁免该私网限制。
dialer := &net.Dialer{
Timeout: 15 * time.Second,
Control: func(_, address string, _ syscall.RawConn) error {
@@ -70,6 +85,9 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
if err != nil {
return err
}
if proxy.isAllowedRemoteHost(host) {
return nil
}
ip := net.ParseIP(host)
if ip == nil {
return errors.New("image proxy: refusing non-IP dial target")
@@ -82,12 +100,9 @@ func NewImageProxy(cfg *config.Config, log *zap.Logger) *ImageProxy {
}
transport.DialContext = dialer.DialContext
}
return &ImageProxy{
cfg: cfg,
log: log,
cacheDir: filepath.Join(cfg.Cache.CacheDir, "images"),
client: &http.Client{Timeout: 30 * time.Second, Transport: transport},
}
proxy.client = &http.Client{Timeout: 30 * time.Second, Transport: transport}
return proxy
}
// proxyConfiguredForImageFetch 探测环境变量或系统代理是否会影响图片抓取。
@@ -125,6 +140,47 @@ func (p *ImageProxy) libraryRoots() []string {
return p.libRootsCache
}
// SetAllowedRemoteHostsProvider injects a callback that returns hostnames or IPs
// of explicitly configured remote services (e.g. remote Emby mounts). Requests to
// these hosts bypass SSRF private-IP restrictions.
func (p *ImageProxy) SetAllowedRemoteHostsProvider(fn func() []string) {
p.allowedRemoteHostsFn = fn
}
func (p *ImageProxy) isAllowedRemoteHost(host string) bool {
if p == nil || p.allowedRemoteHostsFn == nil {
return false
}
host = strings.ToLower(strings.TrimSpace(host))
if host == "" {
return false
}
// Strip port if present
if h, _, err := net.SplitHostPort(host); err == nil {
host = strings.ToLower(strings.TrimSpace(h))
}
p.allowedHostsMu.Lock()
defer p.allowedHostsMu.Unlock()
if p.allowedHostsCache == nil || time.Since(p.allowedHostsAt) >= 30*time.Second {
rawList := p.allowedRemoteHostsFn()
cache := make(map[string]bool, len(rawList))
for _, item := range rawList {
item = strings.ToLower(strings.TrimSpace(item))
if item == "" {
continue
}
if h, _, err := net.SplitHostPort(item); err == nil {
item = strings.ToLower(strings.TrimSpace(h))
}
cache[item] = true
}
p.allowedHostsCache = cache
p.allowedHostsAt = time.Now()
}
return p.allowedHostsCache[host]
}
// Prune removes oldest cached images until disk usage is within the configured limit.
func (p *ImageProxy) Prune() (PruneImageCacheResult, error) {
if p.cfg == nil || p.cfg.Cache.ImagesMaxSizeMB <= 0 {
+1 -1
View File
@@ -22,7 +22,7 @@ func (p *ImageProxy) validateURL(raw string) (*url.URL, error) {
if scheme != "http" && scheme != "https" {
return nil, errors.New("unsupported scheme")
}
if isPrivateHost(u.Hostname()) {
if !p.isAllowedRemoteHost(u.Hostname()) && isPrivateHost(u.Hostname()) {
return nil, errors.New("requests to private/internal hosts are not allowed")
}
return u, nil
+12 -8
View File
@@ -51,7 +51,7 @@ func (p *ImageProxy) fetchRemoteImageOnce(ctx context.Context, raw, host string,
p.log.Warn("imageproxy: build request failed", zap.String("url", raw), zap.Error(err))
return nil, "", "", errImageProxyRequestSetup
}
applyRemoteImageHeaders(req, host)
applyRemoteImageHeaders(req, host, raw)
resp, err := candidate.client.Do(req)
if err != nil {
@@ -79,7 +79,7 @@ func (p *ImageProxy) fetchRemoteImageOnce(ctx context.Context, raw, host string,
return data, ctype, resp.Header.Get("Content-Length"), nil
}
func applyRemoteImageHeaders(req *http.Request, host string) {
func applyRemoteImageHeaders(req *http.Request, host, raw string) {
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36")
req.Header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,ja;q=0.8,en;q=0.7")
@@ -88,7 +88,7 @@ func applyRemoteImageHeaders(req *http.Request, host string) {
if cookie := remoteImageCookie(host); cookie != "" {
req.Header.Set("Cookie", cookie)
}
if referer := remoteImageReferer(host); referer != "" {
if referer := remoteImageReferer(host, raw); referer != "" {
req.Header.Set("Referer", referer)
}
}
@@ -105,7 +105,7 @@ func remoteImageCookie(host string) string {
}
}
func remoteImageReferer(host string) string {
func remoteImageReferer(host, raw string) string {
h := strings.ToLower(strings.TrimSpace(host))
switch {
case strings.Contains(h, "doubanio.com"):
@@ -125,7 +125,11 @@ func remoteImageReferer(host string) string {
case strings.Contains(h, "fc2.com"):
return "https://adult.contents.fc2.com/"
case h != "":
return "https://" + h + "/"
scheme := "https"
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(raw)), "http://") {
scheme = "http"
}
return scheme + "://" + h + "/"
default:
return ""
}
@@ -158,9 +162,9 @@ func fetchRemoteImageWithCurl(ctx context.Context, raw, host string) ([]byte, st
"--header", "Cache-Control: no-cache",
"--header", "Pragma: no-cache",
}
if referer := remoteImageReferer(host); referer != "" {
args = append(args, "--referer", referer)
}
if referer := remoteImageReferer(host, raw); referer != "" {
args = append(args, "--referer", referer)
}
if cookie := remoteImageCookie(host); cookie != "" {
args = append(args, "--cookie", cookie)
}
+9 -4
View File
@@ -317,9 +317,14 @@ func TestRemoteImageRefererForAdultHosts(t *testing.T) {
{"example.com", "https://example.com/"},
{"", ""},
}
for _, tt := range tests {
if got := remoteImageReferer(tt.host); got != tt.want {
t.Errorf("remoteImageReferer(%q) = %q, want %q", tt.host, got, tt.want)
for _, tt := range tests {
if got := remoteImageReferer(tt.host, "https://"+tt.host+"/img.jpg"); got != tt.want {
t.Errorf("remoteImageReferer(%q) = %q, want %q", tt.host, got, tt.want)
}
}
// Also verify HTTP protocol preservation for generic hosts
if got := remoteImageReferer("192.168.1.100", "http://192.168.1.100:8096/image"); got != "http://192.168.1.100/" {
t.Errorf("remoteImageReferer for HTTP host = %q, want http://192.168.1.100/", got)
}
}
}
+26 -3
View File
@@ -117,9 +117,32 @@ func TestIsPrivateHost(t *testing.T) {
// Hostnames must NOT be blocked even though GFW DNS poisoning may resolve
// them to private/loopback IPs — blocking them broke legitimate posters.
allowed := []string{"image.tmdb.org", "lain.bgm.tv", "example.com", "8.8.8.8"}
for _, h := range allowed {
if isPrivateHost(h) {
t.Errorf("isPrivateHost(%q) = true, want false", h)
for _, h := range allowed {
if isPrivateHost(h) {
t.Errorf("isPrivateHost(%q) = true, want false", h)
}
}
}
func TestImageProxyAllowedRemoteHostBypassesPrivateCheck(t *testing.T) {
proxy := NewImageProxy(&config.Config{Cache: config.CacheConfig{CacheDir: filepath.Join(t.TempDir(), "cache")}}, zap.NewNop())
rawURL := "http://192.168.1.100:8096/emby/Items/123/Images/Primary"
// Before setting allowed remote hosts, private host is rejected by validateURL
if _, err := proxy.validateURL(rawURL); err == nil {
t.Fatal("expected validateURL to reject private IP before whitelist")
}
// After configuring whitelist with the Emby host
proxy.SetAllowedRemoteHostsProvider(func() []string {
return []string{"192.168.1.100:8096"}
})
u, err := proxy.validateURL(rawURL)
if err != nil {
t.Fatalf("expected validateURL to allow whitelisted host, got: %v", err)
}
if u.Hostname() != "192.168.1.100" {
t.Fatalf("hostname = %s, want 192.168.1.100", u.Hostname())
}
}
+5
View File
@@ -177,6 +177,11 @@ func (b *serviceContainerBuilder) initIdentityServices() {
func (b *serviceContainerBuilder) initImageProxy() {
b.c.ImageProxy = NewImageProxy(b.cfg, b.log)
b.c.ImageProxy.SetLibraryRootsProvider(b.libraryRoots)
if b.c.EmbyRemote != nil {
b.c.ImageProxy.SetAllowedRemoteHostsProvider(func() []string {
return b.c.EmbyRemote.ConfiguredRemoteHosts(context.Background())
})
}
b.c.Scan.SetImageProxy(b.c.ImageProxy)
b.c.Scraper.SetImageProxy(b.c.ImageProxy)
}
+35 -17
View File
@@ -366,33 +366,51 @@ func (s *StrmService) processUpload115(ctx context.Context, task *model.StrmUplo
finish(model.StrmTaskFailed, "该网盘不支持元数据上传")
return
}
// 以本地为准:网盘端已有同名但内容不同的旧元数据时,先删除旧文件再上传。
// 115 的上传接口不保证同名覆盖,直接上传可能产生同名重复文件;删除失败则
// 任务重试(旧文件 ID 失效的场景会在下次同步后自动修复)。
// 以本地为准:网盘端已有同名但内容不同的旧元数据时,先尝试批量删除所有旧副本再上传。
// 115 的上传接口不保证同名覆盖,直接上传可能产生同名重复文件。
// 删除失败时不中止任务——继续上传新文件,旧副本交由下次同步的 cleanupBatchRedundantFiles
// 按目录批量清理(下次同步会看到新旧两个版本,命中新版本后把旧版本 cid 收入 pendingDeletes
// 异步删除)。这样避免了「删旧失败 → 任务重试 → 再次删旧失败 → 永远无法上传」的死循环。
if task.RemoteRef != "" {
open115, ok := provider.(cloud.OpenAPI115Provider)
if !ok {
finish(model.StrmTaskFailed, "该网盘不支持删除远端旧元数据")
return
}
if err := open115.OpenClient().DeleteFiles(ctx, task.RemotePath, task.RemoteRef); err != nil {
s.uploadTaskFailWithRetry(task, "删除网盘旧元数据失败:"+err.Error())
refs := strings.Split(task.RemoteRef, ",")
if err := open115.OpenClient().DeleteFiles(ctx, task.RemotePath, refs...); err != nil {
s.log.Warn("删除网盘旧元数据失败,跳过删除继续上传新文件",
zap.String("task_id", task.ID),
zap.String("local_path", task.LocalPath),
zap.Error(err))
// 不 return:继续上传新文件,旧副本由下次同步清理
}
}
// 优先使用直接本地文件上传接口,零拷贝且彻底根除并发临时文件同名碰撞
if localUploader, ok := provider.(interface {
PutLocalFile(ctx context.Context, parentCID, localPath string) error
}); ok {
if err := localUploader.PutLocalFile(ctx, task.RemotePath, task.LocalPath); err != nil {
s.uploadTaskFailWithRetry(task, "上传失败:"+err.Error())
return
}
finish(model.StrmTaskDone, "")
return
}
f, err := os.Open(task.LocalPath)
if err != nil {
s.uploadTaskFailWithRetry(task, "打开本地文件失败:"+err.Error())
return
}
if err := named.PutFileNamed(ctx, task.RemotePath, task.FileName, f); err != nil {
_ = f.Close()
s.uploadTaskFailWithRetry(task, "上传失败:"+err.Error())
return
}
}
f, err := os.Open(task.LocalPath)
if err != nil {
s.uploadTaskFailWithRetry(task, "打开本地文件失败:"+err.Error())
return
}
if err := named.PutFileNamed(ctx, task.RemotePath, task.FileName, f); err != nil {
_ = f.Close()
s.uploadTaskFailWithRetry(task, "上传失败:"+err.Error())
return
finish(model.StrmTaskDone, "")
}
_ = f.Close()
finish(model.StrmTaskDone, "")
}
// downloadTaskFailWithRetry 下载失败任务按退避重试,超过上限标记 failed。
func (s *StrmService) downloadTaskFailWithRetry(task *model.StrmDownloadTask, message string) {
+520 -144
View File
@@ -6,10 +6,10 @@ import (
"context"
"errors"
"fmt"
"reflect"
"net/url"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
@@ -25,6 +25,16 @@ import (
"github.com/truewhile/MeBox/internal/service/cloud115"
)
var errFallbackToWalkRemote = errors.New("fallback to walk remote")
// remoteMetaItem 记录远端存在的单个元数据文件副本信息(大小、文件ID、内容SHA1、修改时间)。
type remoteMetaItem struct {
ID string
Size int64
Sha1 string
MTime int64
}
// strmSyncState 是一次同步执行的上下文。
type strmSyncState struct {
s *StrmService
@@ -37,13 +47,11 @@ type strmSyncState struct {
syncType string
mu sync.Mutex
processed int // 已处理文件计数(用于定期落库进度)
lastProgressFlush time.Time // 上次进度落库时间
seenVideo map[string]bool // "v:"+去掉扩展名的相对路径 → 远端存在该视频
seenMeta map[string]bool // "m:"+相对路径 → 远端存在该元数据
remoteMeta map[string]int64 // 远端元数据大小(上传比对用)
remoteMetaRef map[string]string // "m:"+相对路径 → 远端元数据文件引用(115 文件 ID,覆盖上传前删除旧文件用)
remoteMetaSha1 map[string]string // "m:"+相对路径 → 远端元数据内容 SHA1(115 列表返回;上传/下载精确比对用,其他网盘为空)
processed int // 已处理文件计数(用于定期落库进度)
lastProgressFlush time.Time // 上次进度落库时间
seenVideo map[string]bool // "v:"+去掉扩展名的相对路径 → 远端存在该视频
seenMeta map[string]bool // "m:"+相对路径 → 远端存在该元数据
remoteMeta map[string][]remoteMetaItem // "m:"+相对路径 → 远端元数据副本列表(多副本聚合,支持择优比对与冗余清理)
seenMetaTarget map[string]cloud.FileEntry
seenVideoTarget map[string]cloud.FileEntry
activeDownloadPaths map[string]bool // 本地已在排队/进行的下载任务路径(内存去重)
@@ -184,6 +192,66 @@ func (s *StrmService) ListRemoteDir(ctx context.Context, accountID, dir string)
return provider.List(ctx, dir)
}
// ResolveRemoteDirPath 解析远端目录的完整展示路径。115 的目录以 ID 存储,
// 用户无法辨认,这里按 ID 反查 115 返回的祖先链拼出人类可读路径;路径型
// 网盘(CD2/OpenList)与本地目录的 remote_path 本身就是路径,原样返回。
func (s *StrmService) ResolveRemoteDirPath(ctx context.Context, accountID, dir string) (string, error) {
acct, err := s.repo.StrmAccount.FindByID(ctx, accountID)
if err != nil || acct == nil {
return "", errNotFoundOr(err, "网盘账号不存在")
}
dir = strings.TrimSpace(dir)
if dir == "" {
return "", nil
}
if acct.Provider != model.StrmProvider115 {
return dir, nil
}
if dir == "0" {
return "", nil
}
provider, err := s.providerFor(ctx, acct)
if err != nil {
return "", err
}
oc, ok := provider.(interface{ OpenClient() *cloud115.OpenClient })
if !ok {
return "", fmt.Errorf("115: 客户端初始化失败")
}
detail, err := oc.OpenClient().GetFsDetailByCid(ctx, dir)
if err != nil {
return "", fmt.Errorf("115: 解析目录路径失败:%w", err)
}
return cloud115FullPath(detail), nil
}
// cloud115FullPath 把 115 目录详情的祖先链拼成以 / 开头的完整路径。
func cloud115FullPath(detail *cloud115.RemoteFileDetail) string {
if detail == nil {
return ""
}
segments := make([]string, 0, len(detail.Paths)+1)
hasSelf := false
for _, p := range detail.Paths {
if p.FileId == "0" || p.FileId == "" {
continue
}
if p.FileId == detail.FileId {
hasSelf = true
}
if name := strings.TrimSpace(p.Name); name != "" {
segments = append(segments, name)
}
}
if !hasSelf && strings.TrimSpace(detail.FileName) != "" {
segments = append(segments, strings.TrimSpace(detail.FileName))
}
if len(segments) == 0 {
return ""
}
return "/" + strings.Join(segments, "/")
}
// runSync 执行同步主体;结束时更新记录与目录状态。
func (s *StrmService) runSync(ctx context.Context, p *model.StrmSyncPath, rec *model.StrmSyncRecord, cancel context.CancelFunc) {
defer s.clearRunning(p.ID, cancel)
@@ -203,9 +271,7 @@ func (s *StrmService) runSync(ctx context.Context, p *model.StrmSyncPath, rec *m
syncType: rec.SyncType,
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
@@ -288,8 +354,17 @@ func (st *strmSyncState) run() error {
if st.provider != nil {
if open115, ok := st.provider.(cloud.OpenAPI115Provider); ok && st.p.Provider == model.StrmProvider115 {
if err := st.walk115Flat(open115.OpenClient()); err != nil {
return err
err := st.walk115Flat(open115.OpenClient())
if err != nil {
if errors.Is(err, errFallbackToWalkRemote) {
st.s.log.Warn("115: 扁平列表文件数超限,自动降级为传统并发递归同步",
zap.String("path_id", st.p.ID))
if errWalk := st.walkRemote(); errWalk != nil {
return errWalk
}
} else {
return err
}
}
} else {
if err := st.walkRemote(); err != nil {
@@ -439,28 +514,31 @@ func (st *strmSyncState) walkRemote() error {
if task.rel != "" {
rel = task.rel + "/" + cleanName
}
if entry.IsDir {
push(dirTask{id: entry.ID, rel: rel})
} else {
st.processRemoteFile(entry, rel)
if entry.IsDir {
st.dirCache.Store(entry.ID, rel)
st.deferDirCacheSave(entry.ID, rel)
push(dirTask{id: entry.ID, rel: rel})
} else {
st.processRemoteFile(entry, rel)
}
}
walkMu.Lock()
pending--
if pending == 0 {
walkCond.Broadcast()
}
walkMu.Unlock()
}
walkMu.Lock()
pending--
if pending == 0 {
walkCond.Broadcast()
}
walkMu.Unlock()
}); err != nil {
cancel()
}
}); err != nil {
cancel()
}
}()
}
wg.Wait()
if firstErr != nil {
return firstErr
}
}()
}
wg.Wait()
st.flushDirCacheSave()
if firstErr != nil {
return firstErr
}
return ctx.Err()
}
@@ -599,86 +677,16 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
if pathCounts[item.Path] > 1 {
continue
}
st.dirCache.Store(item.DirID, cleanDirRel(item.Path))
st.dirCache.Store(item.DirID, cleanDirRel(item.Path))
}
}
}
}
// 2. 探测文件总数
const pageSize = 1150
firstBatch, totalCount, err := open115.GetFsListFlat(ctx, rootCID, 0, pageSize)
if err != nil {
return fmt.Errorf("115: 获取文件列表失败:%w", err)
}
st.updateSyncMessage(fmt.Sprintf("正在拉取远端文件列表 (共 %d 个文件)...", totalCount))
allFiles := make([]cloud115.RemoteFile, 0, totalCount)
allFiles = append(allFiles, firstBatch...)
// 3. 并发分页拉取剩余文件
if totalCount > int64(len(firstBatch)) {
totalPages := int((totalCount + pageSize - 1) / pageSize)
type pageTask struct {
offset int
// 2. 自适应分治拉取文件列表(单目录超 9500 时自动对子目录并发分治扁平化)
allFiles, err := st.fetch115FilesAdaptive(ctx, open115, rootCID)
if err != nil {
return err
}
pageTasks := make([]pageTask, 0, totalPages-1)
for page := 1; page < totalPages; page++ {
pageTasks = append(pageTasks, pageTask{offset: page * pageSize})
}
var (
filesMu sync.Mutex
wg sync.WaitGroup
taskCh = make(chan pageTask, len(pageTasks))
errMu sync.Mutex
fetchErr error
)
for _, t := range pageTasks {
taskCh <- t
}
close(taskCh)
workers := 8
if len(pageTasks) < workers {
workers = len(pageTasks)
}
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
// 分页拉取 panic 时取消整个同步;正常退出不取消。
if err := helper.Recover(st.s.log, "strm.sync.walk115.page", func() error {
for t := range taskCh {
if ctx.Err() != nil {
return nil
}
files, _, err := open115.GetFsListFlat(ctx, rootCID, t.offset, pageSize)
if err != nil {
errMu.Lock()
if fetchErr == nil {
fetchErr = err
}
errMu.Unlock()
return nil
}
filesMu.Lock()
allFiles = append(allFiles, files...)
filesMu.Unlock()
}
return nil
}); err != nil {
cancel()
}
}()
}
wg.Wait()
if fetchErr != nil {
return fmt.Errorf("115: 分页拉取失败:%w", fetchErr)
}
}
if ctx.Err() != nil {
return ctx.Err()
@@ -819,9 +827,9 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
}
fileCh := make(chan strmFileTask)
var (
procWg sync.WaitGroup
procWg sync.WaitGroup
procErrMu sync.Mutex
procErr error
procErr error
)
for i := 0; i < strmProcessWorkers; i++ {
procWg.Add(1)
@@ -894,6 +902,289 @@ feed:
return ctx.Err()
}
const (
flat115PageSize = 1150
flat115Threshold = 9500
max115AdaptiveDepth = 10
)
type adaptive115Task struct {
cid string
rel string
depth int
}
// fetch115FlatSubtree 扁平拉取单个文件数在安全阈值内的子树全部文件。
func fetch115FlatSubtree(ctx context.Context, open115 *cloud115.OpenClient, cid string, firstBatch []cloud115.RemoteFile, totalCount int64, log *zap.Logger) ([]cloud115.RemoteFile, error) {
allFiles := make([]cloud115.RemoteFile, 0, totalCount)
allFiles = append(allFiles, firstBatch...)
if totalCount <= int64(len(firstBatch)) {
return allFiles, nil
}
totalPages := int((totalCount + flat115PageSize - 1) / flat115PageSize)
type pageTask struct {
offset int
}
pageTasks := make([]pageTask, 0, totalPages-1)
for page := 1; page < totalPages; page++ {
pageTasks = append(pageTasks, pageTask{offset: page * flat115PageSize})
}
var (
filesMu sync.Mutex
wg sync.WaitGroup
taskCh = make(chan pageTask, len(pageTasks))
errMu sync.Mutex
fetchErr error
)
for _, t := range pageTasks {
taskCh <- t
}
close(taskCh)
workers := 8
if len(pageTasks) < workers {
workers = len(pageTasks)
}
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if err := helper.Recover(log, "strm.sync.walk115.page", func() error {
for t := range taskCh {
if ctx.Err() != nil {
return nil
}
files, _, err := open115.GetFsListFlat(ctx, cid, t.offset, flat115PageSize)
if err != nil {
errMu.Lock()
if fetchErr == nil {
fetchErr = err
}
errMu.Unlock()
return nil
}
filesMu.Lock()
allFiles = append(allFiles, files...)
filesMu.Unlock()
}
return nil
}); err != nil {
errMu.Lock()
if fetchErr == nil {
fetchErr = err
}
errMu.Unlock()
}
}()
}
wg.Wait()
if fetchErr != nil {
return nil, fmt.Errorf("115: 分页拉取失败:%w", fetchErr)
}
return allFiles, nil
}
// list115DirDirect 列出指定目录下的直接子项(单层 cur=1&show_dir=1)。
func list115DirDirect(ctx context.Context, open115 *cloud115.OpenClient, cid string) ([]cloud115.RemoteFile, error) {
var out []cloud115.RemoteFile
for offset := 0; ; offset += flat115PageSize {
files, _, err := open115.GetFsList(ctx, cid, offset, flat115PageSize)
if err != nil {
return nil, err
}
out = append(out, files...)
if len(files) < flat115PageSize {
break
}
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
}
return out, nil
}
// fetch115FilesAdaptive 采用自适应分治策略抓取 115 目录树下的全部文件:
// 115 开放平台扁平搜索对 offset+limit 有 10000 的最大深度限制。
// - 若子树文件总数 < 9500,直接使用全速扁平分页批量拉取;
// - 若子树文件总数 >= 9500(大库或超大分类目录),自动分治:仅单层列出该目录的直属子项(cur=1),
// 直属纯文件直接收集,直属子目录则派发为独立的子树任务继续递归探测与拉取;
// - 若超大单目录下无子目录或层级过深(>10层),安全回退到 errFallbackToWalkRemote。
func (st *strmSyncState) fetch115FilesAdaptive(ctx context.Context, open115 *cloud115.OpenClient, rootCID string) ([]cloud115.RemoteFile, error) {
var (
allFiles []cloud115.RemoteFile
filesMu sync.Mutex
walkMu sync.Mutex
walkCond = sync.NewCond(&walkMu)
work []adaptive115Task
pending int
errMu sync.Mutex
firstErr error
)
push := func(t adaptive115Task) {
walkMu.Lock()
work = append(work, t)
pending++
walkCond.Signal()
walkMu.Unlock()
}
go func() {
<-ctx.Done()
walkMu.Lock()
walkCond.Broadcast()
walkMu.Unlock()
}()
push(adaptive115Task{cid: rootCID, rel: "", depth: 0})
workers := 8
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if err := helper.Recover(st.s.log, "strm.sync.walk115.adaptive", func() error {
for {
walkMu.Lock()
for len(work) == 0 {
if ctx.Err() != nil || pending == 0 {
walkMu.Unlock()
return nil
}
walkCond.Wait()
}
task := work[0]
work = work[1:]
walkMu.Unlock()
if ctx.Err() != nil {
walkMu.Lock()
pending--
walkCond.Broadcast()
walkMu.Unlock()
return nil
}
firstBatch, totalCount, err := open115.GetFsListFlat(ctx, task.cid, 0, flat115PageSize)
if err != nil {
errMu.Lock()
if firstErr == nil {
firstErr = fmt.Errorf("115: 获取文件列表失败(cid=%s):%w", task.cid, err)
}
errMu.Unlock()
walkMu.Lock()
pending--
walkCond.Broadcast()
walkMu.Unlock()
return nil
}
if totalCount < flat115Threshold {
// 安全深度内:直接扁平拉取该子树全部文件
files, err := fetch115FlatSubtree(ctx, open115, task.cid, firstBatch, totalCount, st.s.log)
if err != nil {
errMu.Lock()
if firstErr == nil {
firstErr = err
}
errMu.Unlock()
walkMu.Lock()
pending--
walkCond.Broadcast()
walkMu.Unlock()
return nil
}
filesMu.Lock()
allFiles = append(allFiles, files...)
currentCount := len(allFiles)
filesMu.Unlock()
st.updateSyncMessage(fmt.Sprintf("正在拉取远端文件列表 (已获取 %d 个文件)...", currentCount))
} else {
// 子树过大(>=9500):分治展开该目录直接子项
if task.depth >= max115AdaptiveDepth {
// 深度超限兜底:单目录嵌套超 10 层仍超 9500,回退为传统递归
errMu.Lock()
if firstErr == nil {
firstErr = errFallbackToWalkRemote
}
errMu.Unlock()
walkMu.Lock()
pending--
walkCond.Broadcast()
walkMu.Unlock()
return nil
}
st.s.log.Info("115: 目录文件数超限,自动分治展开子目录并发扁平拉取",
zap.String("cid", task.cid),
zap.String("rel", task.rel),
zap.Int64("total_count", totalCount),
zap.Int("depth", task.depth))
directEntries, err := list115DirDirect(ctx, open115, task.cid)
if err != nil {
errMu.Lock()
if firstErr == nil {
firstErr = fmt.Errorf("115: 列出单层目录失败(cid=%s):%w", task.cid, err)
}
errMu.Unlock()
walkMu.Lock()
pending--
walkCond.Broadcast()
walkMu.Unlock()
return nil
}
for _, f := range directEntries {
if f.Category == cloud115.TypeDir {
cleanName := cleanEntryName(f.FileName, true)
subRel := cleanName
if task.rel != "" {
subRel = task.rel + "/" + cleanName
}
st.dirCache.Store(f.FileId, subRel)
st.deferDirCacheSave(f.FileId, subRel)
push(adaptive115Task{cid: f.FileId, rel: subRel, depth: task.depth + 1})
} else {
filesMu.Lock()
allFiles = append(allFiles, f)
filesMu.Unlock()
}
}
}
walkMu.Lock()
pending--
if pending == 0 {
walkCond.Broadcast()
}
walkMu.Unlock()
}
}); err != nil {
errMu.Lock()
if firstErr == nil {
firstErr = err
}
errMu.Unlock()
}
}()
}
wg.Wait()
if firstErr != nil {
return nil, firstErr
}
return allFiles, nil
}
// handleVideo 生成/更新 .strm 文件。
func (st *strmSyncState) handleVideo(entry cloud.FileEntry, rel, ext string) {
relSansExt := rel[:len(rel)-len(ext)]
@@ -1049,25 +1340,29 @@ func (st *strmSyncState) localSha1Matches(path, remoteSha1 string) bool {
return strings.EqualFold(local, remoteSha1)
}
// recordRemoteMeta 记录远端存在的元数据索引、文件大小、文件引用及内容 SHA1。
// recordRemoteMeta 记录远端存在的元数据索引、文件大小、文件引用及内容 SHA1(多副本聚合追加)。
// 对同一文件 ID 严格去重,避免当 download_meta 与 upload_meta 同时开启时因重复记录引发误判与自杀式删除。
func (st *strmSyncState) recordRemoteMeta(entry cloud.FileEntry, rel string) {
st.mu.Lock()
defer st.mu.Unlock()
if st.remoteMeta == nil {
st.remoteMeta = map[string]int64{}
st.remoteMeta = map[string][]remoteMetaItem{}
}
if st.remoteMetaRef == nil {
st.remoteMetaRef = map[string]string{}
key := "m:" + rel
st.seenMeta[key] = true
if entry.ID != "" {
for _, existing := range st.remoteMeta[key] {
if existing.ID == entry.ID {
return
}
}
}
if st.remoteMetaSha1 == nil {
st.remoteMetaSha1 = map[string]string{}
}
st.seenMeta["m:"+rel] = true
st.remoteMeta["m:"+rel] = entry.Size
st.remoteMetaRef["m:"+rel] = entry.ID
if sha := usableSha1(entry.Sha1); sha != "" {
st.remoteMetaSha1["m:"+rel] = sha
}
st.mu.Unlock()
st.remoteMeta[key] = append(st.remoteMeta[key], remoteMetaItem{
ID: entry.ID,
Size: entry.Size,
Sha1: usableSha1(entry.Sha1),
MTime: entry.MTime,
})
}
func (st *strmSyncState) flushPendingDownloads() {
@@ -1289,8 +1584,12 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
st.activeUploadPaths = map[string]bool{}
}
}
var pendingDeletes map[string][]string
if st.p.Provider == model.StrmProvider115 {
pendingDeletes = map[string][]string{}
}
localRoot := filepath.Clean(st.p.LocalPath)
return filepath.WalkDir(localRoot, func(path string, d os.DirEntry, err error) error {
err := filepath.WalkDir(localRoot, func(path string, d os.DirEntry, err error) error {
if err != nil {
return nil
}
@@ -1319,22 +1618,48 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
return nil
}
st.mu.Lock()
remoteSize, exists := st.remoteMeta["m:"+rel]
remoteRef := st.remoteMetaRef["m:"+rel]
remoteSha1 := st.remoteMetaSha1["m:"+rel]
entries, exists := st.remoteMeta["m:"+rel]
st.mu.Unlock()
if exists && remoteSize == info.Size() {
// 网盘端同名同大小:候选同一文件。115 提供远端 SHA1 时做内容级比对,
// 识别"同大小不同内容"(如 nfo 改一个字符长度不变)避免漏传;
// 无哈希(其他网盘/列表未返回)视为同一文件,保持大小比对兜底。
if remoteSha1 == "" || st.localSha1Matches(path, remoteSha1) {
return nil
if exists && len(entries) > 0 {
// 择优比对:只要远端存在任一副本的大小匹配且 SHA1 匹配(或无 SHA1),即判定远端已有最新副本,无需上传
matchedIdx := -1
for i, it := range entries {
if it.Size == info.Size() {
if it.Sha1 == "" || st.localSha1Matches(path, it.Sha1) {
matchedIdx = i
break
}
}
}
if matchedIdx >= 0 {
// 远端已存在完全一致的副本,跳过上传!
// 若远端还存在其他同名脏副本(副本总数 > 1),在 115 下收集待删除 ID,稍后按目录批量删除。
// 严禁将已命中的最新副本 ID (matchedID) 放入待删列表,杜绝误杀唯一有效副本。
if len(entries) > 1 && st.p.Provider == model.StrmProvider115 {
parentCID := st.uploadRemoteTarget(rel)
if parentCID != "" {
matchedID := entries[matchedIdx].ID
for i, it := range entries {
if i != matchedIdx && it.ID != "" && it.ID != matchedID {
pendingDeletes[parentCID] = append(pendingDeletes[parentCID], it.ID)
}
}
}
}
return nil
}
}
// 入队上传(以本地为准):网盘端不存在;或同名但大小不同(必然内容不同);
// 或同名同大小但 SHA1 不同(精确比对发现的同大小不同内容)。
// 115 的上传接口不保证同名覆盖,任务携带远端旧文件 ID(RemoteRef),
// 由上传端先删旧文件再上传;WebDAV/OpenList 的 PutFile 本身即覆盖上传。
remoteTarget := st.uploadRemoteTarget(rel)
if st.p.Provider == model.StrmProvider115 && remoteTarget == "" {
// 115 远端不存在对应父目录(如孤儿子目录),禁止降级到根目录上传以防错位死循环
return nil
}
// 入队上传(以本地为准):网盘端不存在;或所有远端副本均内容不同。
// 115 的上传接口不保证同名覆盖,任务携带远端所有旧副本 ID(RemoteRef,以逗号连接),
// 由上传端一次性批量删除所有旧文件后再上传,彻底根除同名文件堆积。
st.mu.Lock()
if st.activeUploadPaths != nil && st.activeUploadPaths[path] {
st.mu.Unlock()
@@ -1351,12 +1676,18 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
Provider: st.p.Provider,
FileName: filepath.Base(rel),
LocalPath: path,
RemotePath: st.uploadRemoteTarget(rel),
RemotePath: remoteTarget,
Size: info.Size(),
Status: model.StrmTaskPending,
}
if exists && st.p.Provider == model.StrmProvider115 {
task.RemoteRef = remoteRef
if exists && len(entries) > 0 && st.p.Provider == model.StrmProvider115 {
var oldIDs []string
for _, it := range entries {
if it.ID != "" {
oldIDs = append(oldIDs, it.ID)
}
}
task.RemoteRef = strings.Join(oldIDs, ",")
}
st.mu.Lock()
st.pendingUploads = append(st.pendingUploads, task)
@@ -1369,6 +1700,51 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
}
return nil
})
if err != nil {
return err
}
// 批量清理 115 冗余旧元数据副本(按父目录聚合,一次请求批量删除该目录下全部冗余文件,彻底避免并发触发限流器超时)
if len(pendingDeletes) > 0 {
st.cleanupBatchRedundantFiles(pendingDeletes)
}
return nil
}
// cleanupBatchRedundantFiles 异步按父目录批量清理 115 远端冗余旧元数据副本。
func (st *strmSyncState) cleanupBatchRedundantFiles(deletesByParent map[string][]string) {
if len(deletesByParent) == 0 || st.provider == nil {
return
}
open115, ok := st.provider.(cloud.OpenAPI115Provider)
if !ok {
return
}
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
totalPruned := 0
for parentCID, fileIDs := range deletesByParent {
if ctx.Err() != nil {
break
}
if len(fileIDs) == 0 {
continue
}
if err := open115.OpenClient().DeleteFiles(ctx, parentCID, fileIDs...); err != nil {
st.s.log.Warn("批量清理 115 冗余旧元数据副本失败",
zap.String("parent_cid", parentCID),
zap.Int("count", len(fileIDs)),
zap.Error(err))
} else {
totalPruned += len(fileIDs)
}
time.Sleep(100 * time.Millisecond)
}
if totalPruned > 0 {
st.s.log.Info("已完成批量清理 115 冗余旧元数据副本", zap.Int("total_pruned", totalPruned))
}
}()
}
// remoteUploadPath 远端元数据目标路径 = 同步目录远端根 + 相对路径。
@@ -1398,9 +1774,9 @@ func (st *strmSyncState) uploadRemoteTarget(rel string) string {
if cid, ok := st.dirPathToID[dir]; ok && cid != "" {
return cid
}
// 父目录未在缓存中(父目录可能本次未扫描到),降级为用户配置的同步根 cid,
// 由上传端尽力处理(可能失败记日志,不影响下载)。
return st.p.RemotePath
// 子目录在 115 远端不存在(本地孤儿子目录或远端已删除该分类文件夹),
// 返回空串,禁止降级回退到根目录上传以防污染根目录与错位死循环。
return ""
}
return st.remoteUploadPath(rel)
}
+342 -50
View File
@@ -368,19 +368,18 @@ func TestScanLocalMetaForUpload(t *testing.T) {
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
}
// 模拟远端已存在 poster.jpg(与本地同一文件)和 tvshow.nfo(与本地不同)
st.remoteMeta["m:动漫/poster.jpg"] = int64(len("poster-data"))
st.remoteMeta["m:动漫/tvshow.nfo"] = 999
st.remoteMeta["m:动漫/poster.jpg"] = []remoteMetaItem{{ID: "f1", Size: int64(len("poster-data"))}}
st.remoteMeta["m:动漫/tvshow.nfo"] = []remoteMetaItem{{ID: "f2", Size: 999}}
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
@@ -442,17 +441,15 @@ func TestScanLocalMetaForUpload115CarriesRemoteRef(t *testing.T) {
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
}
st.remoteMeta["m:movie.nfo"] = 1
st.remoteMetaRef["m:movie.nfo"] = "file-42"
st.remoteMeta["m:movie.nfo"] = []remoteMetaItem{{ID: "file-42", Size: 1}}
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
@@ -522,7 +519,7 @@ func TestPruneLocalKeepsLocalMeta(t *testing.T) {
syncType: model.StrmSyncTypeFull,
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMeta: map[string][]remoteMetaItem{},
}
// 本次远端扫描既没有看到视频,也没有看到任何元数据
if err := st.pruneLocal(); err != nil {
@@ -569,8 +566,7 @@ func TestHandleMetaKeepsLocalWhenUploadEnabled(t *testing.T) {
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
}
@@ -630,22 +626,21 @@ func TestScanLocalMetaForUploadSha1Identity(t *testing.T) {
UploadMeta: true,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
}
// 115 返回大写 SHA1,本地计算为小写:同时验证大小写不敏感比对
st.remoteMeta["m:same.nfo"] = int64(len("same-content"))
st.remoteMetaSha1["m:same.nfo"] = strings.ToUpper(sameSha)
st.remoteMeta["m:diff.nfo"] = int64(len("diff-content"))
st.remoteMetaSha1["m:diff.nfo"] = strings.ToUpper(otherSha)
st.remoteMetaRef["m:diff.nfo"] = "old-diff-1"
st.remoteMeta["m:same.nfo"] = []remoteMetaItem{
{ID: "same-1", Size: int64(len("same-content")), Sha1: strings.ToUpper(sameSha)},
}
st.remoteMeta["m:diff.nfo"] = []remoteMetaItem{
{ID: "old-diff-1", Size: int64(len("diff-content")), Sha1: strings.ToUpper(otherSha)},
}
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
@@ -703,9 +698,7 @@ func TestHandleMetaSha1Identity(t *testing.T) {
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: false, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
}
@@ -802,7 +795,7 @@ func TestWalkRemoteConcurrent(t *testing.T) {
rec: &model.StrmSyncRecord{},
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMeta: map[string][]remoteMetaItem{},
}
if err := st.walkRemote(); err != nil {
t.Fatalf("walkRemote failed: %v", err)
@@ -911,7 +904,7 @@ func TestStrmDuplicateFileConflictResolution(t *testing.T) {
cfg: &strmPathConfig{DownloadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
@@ -950,7 +943,7 @@ func TestStrmDuplicateFileConflictResolution(t *testing.T) {
cfg: &strmPathConfig{DownloadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
@@ -1019,7 +1012,7 @@ func TestWalk115FlatAbortsOnDirResolveFailure(t *testing.T) {
dirCache: sync.Map{},
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMeta: map[string][]remoteMetaItem{},
}
err := st.walk115Flat(oc)
if err == nil {
@@ -1088,9 +1081,7 @@ func TestWalk115FlatConcurrentProcessing(t *testing.T) {
dirCache: sync.Map{},
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
@@ -1145,9 +1136,7 @@ func TestWalk115FlatConcurrentProcessing(t *testing.T) {
dirCache: sync.Map{},
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
@@ -1161,3 +1150,306 @@ func TestWalk115FlatConcurrentProcessing(t *testing.T) {
t.Fatalf("incremental sync should skip all, new = %d skipped = %d", st2.rec.NewStrm, st2.rec.Skipped)
}
}
func TestCloud115FullPath(t *testing.T) {
paths := func(ids, names []string) []struct {
FileId string `json:"file_id"`
Name string `json:"file_name"`
} {
out := make([]struct {
FileId string `json:"file_id"`
Name string `json:"file_name"`
}, 0, len(ids))
for i := range ids {
out = append(out, struct {
FileId string `json:"file_id"`
Name string `json:"file_name"`
}{FileId: ids[i], Name: names[i]})
}
return out
}
// 祖先链已包含当前目录自身(115 常规返回)
d := &cloud115.RemoteFileDetail{
FileId: "330",
FileName: "剧集",
Paths: paths([]string{"0", "100", "330"}, []string{"", "电影", "剧集"}),
}
if got := cloud115FullPath(d); got != "/电影/剧集" {
t.Fatalf("full path = %q, want /电影/剧集", got)
}
// 祖先链不含当前目录自身,用 FileName 兜底
d = &cloud115.RemoteFileDetail{
FileId: "330",
FileName: "剧集",
Paths: paths([]string{"0", "100"}, []string{"", "电影"}),
}
if got := cloud115FullPath(d); got != "/电影/剧集" {
t.Fatalf("full path = %q, want /电影/剧集", got)
}
// 根目录:无有效祖先段
d = &cloud115.RemoteFileDetail{FileId: "0"}
if got := cloud115FullPath(d); got != "" {
t.Fatalf("root full path = %q, want empty", got)
}
}
// TestScanLocalMetaForUploadMultipleCopiesBestEffortMatch 验证:
// 当远端同一路径存在多个副本(1个与本地一致的副本 + 1个脏副本)时,
// 能够择优识别出匹配的副本,跳过上传(uploaded = 0),避免盲盒覆盖导致的重复上传。
func TestScanLocalMetaForUploadMultipleCopiesBestEffortMatch(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
nfoPath := filepath.Join(localDir, "test.nfo")
writeFile(t, nfoPath, "correct-content")
correctSha, err := cloud115.FileSHA1(nfoPath)
if err != nil {
t.Fatal(err)
}
p := &model.StrmSyncPath{
Base: model.Base{ID: "multi-copy-match-path"},
Provider: model.StrmProvider115,
RemotePath: "root-cid",
LocalPath: localDir,
UploadMeta: true,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
}
// 模拟远端存在两个同名副本:一个脏副本(较早),一个正确副本(较晚)
st.recordRemoteMeta(cloud.FileEntry{
ID: "stale-id",
Name: "test.nfo",
Size: int64(len("stale-dirty-content")),
Sha1: "STALE_SHA1",
MTime: 1000,
}, "test.nfo")
st.recordRemoteMeta(cloud.FileEntry{
ID: "correct-id",
Name: "test.nfo",
Size: int64(len("correct-content")),
Sha1: correctSha,
MTime: 2000,
}, "test.nfo")
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
}
// 择优匹配:命中 correct-id,不产生上传任务
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 0 {
t.Fatalf("expected 0 upload tasks when a matching copy exists, got %d: %v", len(tasks), taskNames(tasks))
}
}
// TestScanLocalMetaForUploadMultipleCopiesAllStale 验证:
// 当远端同一路径存在多个副本,且所有副本均与本地内容不一致时,
// 上传任务应携带所有旧副本的 ID(逗号分隔),以便上传前批量清理所有旧副本。
func TestScanLocalMetaForUploadMultipleCopiesAllStale(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
nfoPath := filepath.Join(localDir, "test.nfo")
writeFile(t, nfoPath, "brand-new-content")
p := &model.StrmSyncPath{
Base: model.Base{ID: "multi-copy-stale-path"},
Provider: model.StrmProvider115,
RemotePath: "root-cid",
LocalPath: localDir,
UploadMeta: true,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
}
// 模拟远端存在两个不同大小和哈希的旧副本
st.recordRemoteMeta(cloud.FileEntry{
ID: "old-1",
Name: "test.nfo",
Size: 10,
Sha1: "OLD_SHA1",
MTime: 1000,
}, "test.nfo")
st.recordRemoteMeta(cloud.FileEntry{
ID: "old-2",
Name: "test.nfo",
Size: 20,
Sha1: "OLD_SHA2",
MTime: 2000,
}, "test.nfo")
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
}
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 upload task, got %d", len(tasks))
}
if tasks[0].RemoteRef != "old-1,old-2" {
t.Fatalf("expected RemoteRef to be 'old-1,old-2', got %q", tasks[0].RemoteRef)
}
}
// TestRecordRemoteMetaDeduplication 验证当 download_meta 与 upload_meta 同时开启时,
// 同一远端元数据被多次入账不会在 remoteMeta 中生成重复副本,杜绝误判自杀式删除。
func TestRecordRemoteMetaDeduplication(t *testing.T) {
st := &strmSyncState{
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
}
entry := cloud.FileEntry{
ID: "unique-fid-1",
Name: "test.nfo",
Size: 100,
Sha1: "AAAABBBBCCCC",
MTime: 12345,
}
// 连续记录两次同一文件
st.recordRemoteMeta(entry, "dir/test.nfo")
st.recordRemoteMeta(entry, "dir/test.nfo")
items := st.remoteMeta["m:dir/test.nfo"]
if len(items) != 1 {
t.Fatalf("expected 1 item after duplicate record, got %d", len(items))
}
}
// TestWalk115AdaptiveHierarchicalFlatScan 验证自适应分治扁平化扫描:
// 当根目录探测总数 >= 9500 时,系统自动分治展开单层直接子项,对各子目录分别执行扁平拉取,
// 正确合并根目录直属文件与各子目录深层文件,突破 115 开放平台 10000 深度限制。
func TestWalk115AdaptiveHierarchicalFlatScan(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
acct := &model.StrmAccount{Name: "fake115-adaptive", Provider: "cloud115", Config: "{}", Enabled: true}
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
t.Fatal(err)
}
p := &model.StrmSyncPath{
Base: model.Base{ID: "adaptive-path"},
AccountID: acct.ID,
Provider: model.StrmProvider115,
RemotePath: "0",
LocalPath: localDir,
}
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
cid := q.Get("cid")
cur := q.Get("cur")
switch r.URL.Path {
case "/open/ufile/files":
if cid == "0" && cur == "0" {
// 根目录扁平探测:模拟总文件数 12000 超限 (>= 9500)
w.Write([]byte(`{"state":true,"count":12000,"data":[]}`))
return
}
if cid == "0" && cur == "1" {
// 根目录单层列举:返回 1 个直属视频和 2 个子目录
w.Write([]byte(`{"state":true,"count":3,"data":[
{"fid":"100","pid":"0","fc":"1","fn":"root.mkv","pc":"pcr","upt":1700000000,"fs":1024},
{"fid":"1001","pid":"0","fc":"0","fn":"Heyzo","upt":1700000001,"fs":0},
{"fid":"1002","pid":"0","fc":"0","fn":"S1","upt":1700000002,"fs":0}]}`))
return
}
if cid == "1001" {
// 子目录 Heyzo 扁平拉取:文件数安全 (< 9500)
w.Write([]byte(`{"state":true,"count":2,"data":[
{"fid":"201","pid":"1001","fc":"1","fn":"h1.mkv","pc":"pc201","upt":1700000001,"fs":1024},
{"fid":"202","pid":"1001","fc":"1","fn":"h2.mkv","pc":"pc202","upt":1700000002,"fs":2048}]}`))
return
}
if cid == "1002" {
// 子目录 S1 扁平拉取:文件数安全 (< 9500)
w.Write([]byte(`{"state":true,"count":1,"data":[
{"fid":"301","pid":"1002","fc":"1","fn":"s1.mkv","pc":"pc301","upt":1700000003,"fs":4096}]}`))
return
}
t.Errorf("unexpected files query: %s", r.URL.RawQuery)
case "/open/folder/get_info":
fileID := q.Get("file_id")
switch fileID {
case "1001":
w.Write([]byte(`{"state":true,"data":{"file_id":"1001","file_name":"Heyzo","file_category":"0",
"paths":[{"file_id":"0","file_name":"根目录"},{"file_id":"1001","file_name":"Heyzo"}]}}`))
case "1002":
w.Write([]byte(`{"state":true,"data":{"file_id":"1002","file_name":"S1","file_category":"0",
"paths":[{"file_id":"0","file_name":"根目录"},{"file_id":"1002","file_name":"S1"}]}}`))
default:
t.Errorf("unexpected get_info file_id: %s", fileID)
}
default:
t.Errorf("unexpected path %s", r.URL.Path)
}
}))
defer api.Close()
oldPro := cloud115.ProAPIBase
cloud115.ProAPIBase = api.URL
defer func() { cloud115.ProAPIBase = oldPro }()
oc := cloud115.NewOpenClient("app", "at", "rt")
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
provider: cloud.NewOpenAPI115("app", "at", "rt"),
cfg: &strmPathConfig{VideoExt: []string{"mkv"}, MetaExt: []string{"nfo"}, AddPath: 1},
rec: &model.StrmSyncRecord{},
syncType: model.StrmSyncTypeFull,
dirCache: sync.Map{},
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string][]remoteMetaItem{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
if err := st.walk115Flat(oc); err != nil {
t.Fatalf("walk115Flat adaptive failed: %v", err)
}
// 1个根目录视频 + 2个Heyzo视频 + 1个S1视频 = 共4个视频成功生成 .strm
if st.rec.NewStrm != 4 {
t.Fatalf("expected 4 strm created, got %d", st.rec.NewStrm)
}
expectedFiles := []string{
filepath.Join(localDir, "root.strm"),
filepath.Join(localDir, "Heyzo", "h1.strm"),
filepath.Join(localDir, "Heyzo", "h2.strm"),
filepath.Join(localDir, "S1", "s1.strm"),
}
for _, f := range expectedFiles {
if _, err := os.Stat(f); err != nil {
t.Fatalf("expected strm %s to exist, err: %v", f, err)
}
}
}
+6
View File
@@ -80,6 +80,12 @@ export const strmAPI = {
.get<StrmRemoteEntry[]>(`/admin/strm/accounts/${accountId}/list`, { params: { dir } })
.then((r) => r.data),
// 按远端目录引用(115 为目录 ID)反查完整展示路径
resolveRemoteDirPath: (accountId: string, dir: string) =>
api
.get<{ path: string }>(`/admin/strm/accounts/${accountId}/resolve`, { params: { dir } })
.then((r) => r.data.path),
list115Sources: () =>
api.get<Strm115Sources>('/admin/strm/115/sources').then((r) => r.data),
+14 -1
View File
@@ -2,6 +2,7 @@ import { useEffect, useRef, useState, type DragEvent, type MouseEvent, type Reac
import { createPortal } from 'react-dom'
import { Folder, GripVertical, Image, MoreVertical, Plus, Power, PowerOff, RefreshCw, Save, Trash2 } from 'lucide-react'
import { imageURL } from '../api/client'
import { LocalDirBrowserDialog } from '../components/LocalDirBrowserDialog'
import type { Library, LibraryRoot } from '../types'
import type { RootDraft } from './adminLibraryPanelModel'
@@ -169,7 +170,19 @@ function LibraryTableRow({ library, dragging, dragOver, onDragStart, onDragOver,
</td>
<td className="py-2 pr-3 font-medium text-ink-600">
<div className="flex items-center gap-2">
{library.cover_url && <img src={library.cover_url} alt="" loading="lazy" decoding="async" className="h-10 w-8 rounded object-cover" />}
{library.cover_url && (
<img
src={imageURL(library.cover_url, library.updated_at)}
alt=""
loading="lazy"
decoding="async"
referrerPolicy="no-referrer"
className="h-10 w-8 rounded object-cover"
onError={(e) => {
e.currentTarget.style.visibility = 'hidden'
}}
/>
)}
<span>{library.name}</span>
</div>
</td>
+161 -45
View File
@@ -29,7 +29,7 @@ import {
normalizeEmbyRemoteLines,
type EmbyRemoteLine,
} from '../utils/embyRemoteLines'
import { lastPathSegment, syncLocalPathWithRemote } from '../utils/strmPaths'
import { lastPathSegment, remoteTailNameOf, syncLocalPathWithRemote } from '../utils/strmPaths'
// ─── 弹框外壳 ────────────────────────────────────────────────────────────────
@@ -573,22 +573,33 @@ export function StrmSettingsDialog({ onClose }: { onClose: () => void }) {
// ─── 添加/编辑同步目录 ────────────────────────────────────────────────────────
/** 推断已有配置当前实际拼在本地输出目录末尾的尾段(目录名或 115 目录 ID)。 */
function initRemoteTail(existing: StrmSyncPath): string {
const cidTail = lastPathSegment(existing.remote_path)
const nameTail = existing.remote_display_path ? lastPathSegment(existing.remote_display_path) : ''
if (nameTail && lastPathSegment(existing.local_path) === nameTail) return nameTail
return cidTail
}
export function StrmSyncPathDialog({
accounts,
existing,
onClose,
onSaved,
onOpenSettings,
}: {
accounts: StrmAccount[]
existing: StrmSyncPath | null
onClose: () => void
onSaved: () => void
onOpenSettings?: () => void
}) {
const [form, setForm] = useState<StrmSyncPathInput>(() => ({
name: existing?.name ?? '',
provider: existing?.provider ?? 'cloud115',
account_id: existing?.account_id ?? '',
remote_path: existing?.remote_path ?? '',
remote_display_path: existing?.remote_display_path ?? '',
local_path: existing?.local_path ?? '',
strm_base_url: existing?.strm_base_url ?? '',
video_ext: existing?.video_ext ?? '',
@@ -607,22 +618,68 @@ export function StrmSyncPathDialog({
const [saving, setSaving] = useState(false)
const [browsing, setBrowsing] = useState(false)
const [browsingLocal, setBrowsingLocal] = useState<null | 'remote_path' | 'local_path'>(null)
const prevRemoteTailRef = useRef(existing ? lastPathSegment(existing.remote_path) : '')
// 输入框显示的远端目录文本:对于 115 优先显示完整路径,若尚未反查到则显示 ID
const [remoteInputValue, setRemoteInputValue] = useState(
() => existing?.remote_display_path || existing?.remote_path || '',
)
// 反查展示路径的序号守卫:远端目录快速连续变更时丢弃过期响应
const resolveSeqRef = useRef(0)
// prevRemoteTailRef 记录当前拼在本地输出目录末尾、由本弹窗管理的尾段。
// 兼容两类历史数据:新版保存的 local_path 末段是目录名,旧版是目录 ID。
const prevRemoteTailRef = useRef(existing ? initRemoteTail(existing) : '')
const set = <K extends keyof StrmSyncPathInput>(key: K, value: StrmSyncPathInput[K]) =>
setForm((f) => ({ ...f, [key]: value }))
const updateRemotePath = (remotePath: string) => {
const updateRemotePath = (remotePath: string, displayPath?: string) => {
const shown = (displayPath || remotePath).trim()
setRemoteInputValue(shown)
setForm((f) => {
const synced = syncLocalPathWithRemote(f.local_path, remotePath, prevRemoteTailRef.current)
const tail = remoteTailNameOf(remotePath, displayPath)
const synced = syncLocalPathWithRemote(f.local_path, remotePath, prevRemoteTailRef.current, tail)
prevRemoteTailRef.current = synced.remoteTail
return { ...f, remote_path: remotePath, local_path: synced.localPath }
return {
...f,
remote_path: remotePath,
remote_display_path: displayPath ?? '',
local_path: synced.localPath,
}
})
}
// 按 ID 反查完整展示路径。默认仅补展示、不改动已配置的本地输出目录;
// resyncTail 用于手动输入 ID 的新配置:把刚拼上的 ID 尾段替换为目录名。
const resolveDisplayPath = (accountId: string, remotePath: string, resyncTail = false) => {
const seq = ++resolveSeqRef.current
strmAPI
.resolveRemoteDirPath(accountId, remotePath)
.then((fullPath) => {
if (seq !== resolveSeqRef.current) return
if (fullPath) setRemoteInputValue(fullPath)
setForm((f) => {
if (f.remote_path !== remotePath) return f
if (resyncTail) {
const tail = remoteTailNameOf(remotePath, fullPath)
const synced = syncLocalPathWithRemote(f.local_path, remotePath, prevRemoteTailRef.current, tail)
prevRemoteTailRef.current = synced.remoteTail
return { ...f, remote_display_path: fullPath, local_path: synced.localPath }
}
// 旧配置的 local_path 末段若恰好就是目录名,把它认作受管尾段,
// 后续重新选择目录时才能正确替换而不是叠加
const tail = lastPathSegment(fullPath)
if (tail && prevRemoteTailRef.current !== tail && lastPathSegment(f.local_path) === tail) {
prevRemoteTailRef.current = tail
}
return { ...f, remote_display_path: fullPath }
})
})
.catch(() => undefined)
}
const commitLocalPath = (localPath: string) => {
setForm((f) => {
const synced = syncLocalPathWithRemote(localPath, f.remote_path, prevRemoteTailRef.current)
const tail = remoteTailNameOf(f.remote_path, f.remote_display_path)
const synced = syncLocalPathWithRemote(localPath, f.remote_path, prevRemoteTailRef.current, tail)
prevRemoteTailRef.current = synced.remoteTail
return { ...f, local_path: synced.localPath }
})
@@ -630,11 +687,30 @@ export function StrmSyncPathDialog({
const isLocal = form.provider === 'local'
const availableAccounts = accounts.filter((a) => a.provider === form.provider && a.has_credential)
const remoteDisplayPath = isLocal ? '' : form.remote_display_path?.trim() ?? ''
const remoteTail = isLocal ? '' : remoteTailNameOf(form.remote_path, remoteDisplayPath)
// 编辑旧配置时若尚未存展示路径,按 ID 反查补齐并更新输入框展示
useEffect(() => {
if (isLocal || form.provider !== 'cloud115') return
if (!form.account_id || !form.remote_path || form.remote_display_path) return
resolveDisplayPath(form.account_id, form.remote_path)
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
const submit = async (event: FormEvent) => {
event.preventDefault()
setSaving(true)
try {
if (!existing) {
const settings = await strmAPI.getSettings()
if (!settings?.['strm.base_url']?.trim()) {
toast.error('未填写strm地址')
onClose()
onOpenSettings?.()
return
}
}
if (existing) {
await strmAPI.updatePath(existing.id, form)
toast.success('同步目录已更新')
@@ -671,6 +747,9 @@ export function StrmSyncPathDialog({
const provider = e.target.value as StrmProvider
set('provider', provider)
set('account_id', '')
set('remote_path', '')
set('remote_display_path', '')
setRemoteInputValue('')
}}
>
<option value="cloud115">115 网盘</option>
@@ -707,17 +786,39 @@ export function StrmSyncPathDialog({
isLocal
? '扫描该目录下的视频生成 strm'
: form.provider === 'cloud115'
? '115 目录 ID(可通过浏览选择)'
? form.remote_path
? `115 目录 ID:${form.remote_path}(可通过右侧「浏览」选择更换)`
: '可通过右侧「浏览」选择 115 目录,或直接粘贴目录 ID'
: '远端路径(可通过浏览选择)'
}
>
<input
className={inputCls}
value={form.remote_path}
placeholder={isLocal ? 'D:\\movies' : '/'}
onChange={(e) => set('remote_path', e.target.value)}
value={remoteInputValue}
placeholder={isLocal ? 'D:\\movies' : form.provider === 'cloud115' ? '点击右侧「浏览」选择目录' : '/'}
onChange={(e) => setRemoteInputValue(e.target.value)}
onBlur={(e) => {
if (!isLocal) updateRemotePath(e.target.value)
const value = e.target.value.trim()
if (isLocal) {
set('remote_path', value)
return
}
if (form.provider === 'cloud115') {
if (!value) {
updateRemotePath('', '')
return
}
// 若用户直接输入/粘贴纯数字目录 ID,触发反查并自动替换为完整路径
if (/^\d+$/.test(value)) {
updateRemotePath(value, '')
if (form.account_id) resolveDisplayPath(form.account_id, value, true)
return
}
// 若当前展示的是完整路径且未变动,不做处理
if (value === form.remote_display_path) return
} else {
updateRemotePath(value)
}
}}
/>
</Field>
@@ -745,8 +846,8 @@ export function StrmSyncPathDialog({
<Field
label="本地输出目录"
hint={
!isLocal && lastPathSegment(form.remote_path)
? `将自动拼接远端末级目录「${lastPathSegment(form.remote_path)}」`
!isLocal && remoteTail
? `将自动拼接远端末级目录「${remoteTail}」`
: '生成的 .strm 与下载的元数据写到这里的对应目录结构下'
}
>
@@ -774,20 +875,6 @@ export function StrmSyncPathDialog({
<details className="rounded-2xl border border-gray-100 bg-gray-50/50 p-3">
<summary className="cursor-pointer text-sm font-semibold text-ink-600">高级选项</summary>
<div className="mt-3 space-y-3">
<div className="grid gap-3 md:grid-cols-2">
<Field label="STRM 基础地址(覆盖全局)" hint="留空使用全局 STRM 设置">
<input className={inputCls} value={form.strm_base_url ?? ''} placeholder="http://host:port" onChange={(e) => set('strm_base_url', e.target.value)} />
</Field>
<Field label="最小视频大小(MB)" hint="0 表示继承全局设置">
<input
className={inputCls}
type="number"
min={0}
value={form.min_video_size_mb ?? 0}
onChange={(e) => set('min_video_size_mb', Number(e.target.value))}
/>
</Field>
</div>
<div className="grid gap-3 md:grid-cols-3">
<Field label="视频扩展名(覆盖全局)">
<input className={inputCls} value={form.video_ext ?? ''} placeholder="mkv,mp4,avi" onChange={(e) => set('video_ext', e.target.value)} />
@@ -800,6 +887,15 @@ export function StrmSyncPathDialog({
</Field>
</div>
<div className="grid gap-3 md:grid-cols-3">
<Field label="最小视频大小(MB)" hint="0 表示继承全局设置">
<input
className={inputCls}
type="number"
min={0}
value={form.min_video_size_mb ?? 0}
onChange={(e) => set('min_video_size_mb', Number(e.target.value))}
/>
</Field>
<Field label="STRM 链接 path 参数">
<select className={inputCls} value={form.add_path ?? 1} onChange={(e) => set('add_path', Number(e.target.value))}>
<option value={1}>完整远端路径</option>
@@ -813,10 +909,10 @@ export function StrmSyncPathDialog({
<option value="full">全量同步(全量校验)</option>
</select>
</Field>
<Field label="定时同步 Cron" hint="5 段表达式,如 0 */6 * * *">
<input className={inputCls} value={form.cron ?? ''} placeholder="0 */6 * * *" onChange={(e) => set('cron', e.target.value)} />
</Field>
</div>
<Field label="定时同步 Cron" hint="5 段表达式,如 0 */6 * * *">
<input className={inputCls} value={form.cron ?? ''} placeholder="0 */6 * * *" onChange={(e) => set('cron', e.target.value)} />
</Field>
<div className="grid gap-2 md:grid-cols-2">
<ToggleRow label="下载元数据" checked={form.download_meta ?? true} onChange={(v) => set('download_meta', v)} />
<ToggleRow label="上传元数据" checked={form.upload_meta ?? false} onChange={(v) => set('upload_meta', v)} />
@@ -842,8 +938,8 @@ export function StrmSyncPathDialog({
<StrmDirBrowserDialog
accountId={form.account_id}
initialDir={form.remote_path || undefined}
onSelect={(id) => {
updateRemotePath(id)
onSelect={(id, _name, fullPath) => {
updateRemotePath(id, form.provider === 'cloud115' ? fullPath : undefined)
setBrowsing(false)
}}
onClose={() => setBrowsing(false)}
@@ -882,6 +978,12 @@ function ToggleRow({ label, checked, onChange }: { label: string; checked: boole
)
}
/** 把浏览时经过的目录名称链拼成以 / 开头的完整展示路径。 */
function chainFullPath(chain: { id: string; name: string }[]): string {
const names = chain.map((item) => item.name.trim()).filter(Boolean)
return names.length > 0 ? '/' + names.join('/') : ''
}
// ─── 远端目录浏览选择器 ───────────────────────────────────────────────────────
function StrmDirBrowserDialog({
@@ -892,25 +994,28 @@ function StrmDirBrowserDialog({
}: {
accountId: string
initialDir?: string
onSelect: (path: string) => void
onSelect: (id: string, name?: string, fullPath?: string) => void
onClose: () => void
}) {
const [dir, setDir] = useState(initialDir ?? '')
const [crumbs, setCrumbs] = useState<string[]>([])
const [entries, setEntries] = useState<StrmRemoteEntry[]>([])
const [loading, setLoading] = useState(true)
// 已进入目录的名称链:面包屑按名称展示,「选择当前目录」时回传末级名称
const [dirChain, setDirChain] = useState<{ id: string; name: string }[]>([])
const dirChainRef = useRef<{ id: string; name: string }[]>([])
// 递增序号守卫:快速连续进入目录时丢弃过期目录响应
const loadSeqRef = useRef(0)
const load = async (target: string) => {
const load = async (target: string, nextChain?: { id: string; name: string }[]) => {
const seq = ++loadSeqRef.current
setLoading(true)
try {
const list = await strmAPI.listRemoteDir(accountId, target)
if (seq !== loadSeqRef.current) return
if (nextChain) dirChainRef.current = nextChain
setEntries(list)
setDir(target)
setCrumbs(target ? target.split('/').filter(Boolean) : [])
setDirChain([...dirChainRef.current])
} catch (err) {
if (seq !== loadSeqRef.current) return
toast.error(apiErrorMessage(err))
@@ -924,7 +1029,13 @@ function StrmDirBrowserDialog({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [accountId])
const enterDir = (id: string) => load(id).catch(() => undefined)
const enterDir = (entry: StrmRemoteEntry) => {
const chain = [...dirChainRef.current]
const existingIdx = chain.findIndex((item) => item.id === entry.id)
if (existingIdx >= 0) chain.splice(existingIdx + 1)
else chain.push({ id: entry.id, name: entry.name })
load(entry.id, chain).catch(() => undefined)
}
return (
<div
@@ -944,13 +1055,13 @@ function StrmDirBrowserDialog({
</button>
</div>
<div className="flex items-center gap-1 border-b border-gray-100 px-6 py-2.5 text-xs text-sand-500">
<button type="button" className="hover:text-brand-500" onClick={() => enterDir('')}>
<button type="button" className="hover:text-brand-500" onClick={() => load('', [])}>
根目录
</button>
{crumbs.map((crumb, index) => (
<span key={crumb + index} className="flex items-center gap-1">
{(dirChain.length > 0 ? dirChain.map((item) => item.name) : dir.split('/').filter(Boolean)).map((label, index) => (
<span key={label + index} className="flex items-center gap-1">
<ChevronRight size={12} />
<span>{crumb}</span>
<span>{label}</span>
</span>
))}
<span className="ml-2 text-ink-50">{dir || '(根目录 / 0)'}</span>
@@ -971,8 +1082,8 @@ function StrmDirBrowserDialog({
key={entry.id}
type="button"
className="flex w-full items-center gap-3 rounded-xl px-3 py-2 text-left text-sm transition hover:bg-gray-50"
onClick={() => (entry.is_dir ? enterDir(entry.id) : undefined)}
onDoubleClick={() => entry.is_dir && enterDir(entry.id)}
onClick={() => (entry.is_dir ? enterDir(entry) : undefined)}
onDoubleClick={() => entry.is_dir && enterDir(entry)}
>
<Icon size={16} className={entry.is_dir ? 'text-brand-400' : 'text-sand-400'} />
<span className="flex-1 truncate text-ink-600">{entry.name}</span>
@@ -982,7 +1093,7 @@ function StrmDirBrowserDialog({
className="rounded-lg border border-brand-200 bg-brand-50 px-2.5 py-1 text-xs font-semibold text-brand-500 hover:bg-brand-100"
onClick={(e) => {
e.stopPropagation()
onSelect(entry.id)
onSelect(entry.id, entry.name, chainFullPath([...dirChainRef.current, { id: entry.id, name: entry.name }]))
}}
>
选择此目录
@@ -998,7 +1109,12 @@ function StrmDirBrowserDialog({
</div>
<div className="flex items-center justify-between border-t border-gray-100 px-6 py-3">
<span className="text-xs text-sand-500">双击进入目录,点击「选择此目录」使用该目录 ID / 路径</span>
<button type="button" className="neon-button" onClick={() => onSelect(dir)} disabled={loading}>
<button
type="button"
className="neon-button"
onClick={() => onSelect(dir, dirChain[dirChain.length - 1]?.name, chainFullPath(dirChain))}
disabled={loading}
>
选择当前目录
</button>
</div>
+25 -3
View File
@@ -163,6 +163,22 @@ export function StrmManagePage() {
}
}
const openAddPathDialog = async () => {
try {
const settings = await strmAPI.getSettings()
const baseUrl = settings?.['strm.base_url']?.trim()
if (!baseUrl) {
toast.error('未填写strm地址')
setDialog('settings')
return
}
setEditingPath(null)
setDialog('path')
} catch (err) {
toast.error(apiErrorMessage(err))
}
}
return (
<div className="space-y-6">
<header className="flex flex-wrap items-start justify-between gap-4">
@@ -186,7 +202,7 @@ export function StrmManagePage() {
<Settings size={16} />
STRM 设置
</button>
<button type="button" onClick={() => { setEditingPath(null); setDialog('path') }} className="neon-button">
<button type="button" onClick={openAddPathDialog} className="neon-button">
<FolderPlus size={16} />
添加同步目录
</button>
@@ -210,7 +226,7 @@ export function StrmManagePage() {
<SyncPathSection
paths={paths}
actingPath={actingPath}
onAdd={() => { setEditingPath(null); setDialog('path') }}
onAdd={openAddPathDialog}
onEdit={(path) => { setEditingPath(path); setDialog('path') }}
onDelete={deletePath}
onStart={startSync}
@@ -236,6 +252,7 @@ export function StrmManagePage() {
existing={editingPath}
onClose={() => setDialog(null)}
onSaved={() => { setDialog(null); refresh().catch(() => undefined) }}
onOpenSettings={() => setDialog('settings')}
/>
)}
</div>
@@ -442,7 +459,12 @@ function SyncPathSection({
</div>
</div>
<div className="mt-2.5 flex flex-wrap items-center gap-x-2 gap-y-1 font-mono text-xs text-ink-50">
<span className="max-w-[45%] truncate">{path.remote_path || '(根目录)'}</span>
<span
className="max-w-[45%] truncate"
title={path.remote_display_path || path.remote_path || '(根目录)'}
>
{path.remote_display_path || path.remote_path || '(根目录)'}
</span>
<span className="text-sand-500">→</span>
<span className="max-w-[45%] truncate">{path.local_path}</span>
</div>
+2
View File
@@ -59,6 +59,7 @@ export interface StrmSyncPath {
account_id: string
provider: StrmProvider
remote_path: string
remote_display_path?: string
local_path: string
strm_base_url: string
video_ext: string
@@ -86,6 +87,7 @@ export interface StrmSyncPathInput {
account_id?: string
provider: StrmProvider
remote_path: string
remote_display_path?: string
local_path: string
strm_base_url?: string
video_ext?: string
+20 -2
View File
@@ -6,6 +6,23 @@ export function lastPathSegment(path: string): string {
return parts[parts.length - 1] ?? ''
}
/**
* 取远端末级目录的展示/拼接名。115 等以 ID 作为 remote_path 的网盘
* 传入浏览时拿到的目录名 tailName,避免把数字 ID 拼进本地路径。
*/
export function resolveRemoteTailName(remotePath: string, tailName?: string): string {
const name = tailName?.trim()
if (name) return name
return lastPathSegment(remotePath)
}
/** 优先取展示路径(115 浏览/反查得到)的末级目录名,否则取远端路径末段。 */
export function remoteTailNameOf(remotePath: string, displayPath?: string): string {
const trimmed = displayPath?.trim()
if (trimmed) return lastPathSegment(trimmed)
return lastPathSegment(remotePath)
}
function pathSeparator(path: string): '/' | '\\' {
return path.includes('\\') ? '\\' : '/'
}
@@ -30,13 +47,14 @@ function stripRemoteTail(localPath: string, prevRemoteTail: string): string {
return trimmed
}
/** 将远端最后一级目录名拼到本地输出目录末尾;远端变更时替换旧尾段。 */
/** 将远端最后一级目录名拼到本地输出目录末尾;远端变更时替换旧尾段。tailName 为浏览时拿到的目录名。 */
export function syncLocalPathWithRemote(
localPath: string,
remotePath: string,
prevRemoteTail = '',
tailName?: string,
): { localPath: string; remoteTail: string } {
const remoteTail = lastPathSegment(remotePath)
const remoteTail = resolveRemoteTailName(remotePath, tailName)
const base = stripRemoteTail(localPath, prevRemoteTail)
if (!remoteTail) {
return { localPath: base || localPath.trim(), remoteTail: '' }