优化,添加115接口文档

This commit is contained in:
truewhile
2026-09-05 14:32:58 +08:00
parent d00b14df55
commit 0e9fb2c937
57 changed files with 5682 additions and 76 deletions
+1
View File
@@ -124,6 +124,7 @@ type StrmUploadTask struct {
FileName string `gorm:"size:512" json:"file_name"`
LocalPath string `gorm:"size:1024" json:"local_path"` // 本地源文件
RemotePath string `gorm:"size:1024" json:"remote_path"` // 远端目标路径
RemoteRef string `gorm:"size:1024" json:"remote_ref"` // 远端同名旧文件引用(115 文件 ID;上传覆盖前先删除旧文件,WebDAV/OpenList 直接覆盖无需删除)
Size int64 `json:"size"`
Status string `gorm:"size:16;index" json:"status"`
Error string `gorm:"size:1024" json:"error"`
+47
View File
@@ -901,6 +901,53 @@ func (r *StrmDirCacheRepository) Set(ctx context.Context, syncPathID, dirID, pat
})
}
// SetBatch 批量 upsert 目录缓存(dirID → 相对路径)。单个事务内先查出已存在
// 行再分流更新/插入,替代同步流程逐目录单条 Set,避免首次全量同步上万目录时
// 的 SQLite 写锁竞争。同一 dirID 的重复项以 map 语义取最后一次写入。
func (r *StrmDirCacheRepository) SetBatch(ctx context.Context, syncPathID string, paths map[string]string) error {
if len(paths) == 0 {
return nil
}
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
ids := make([]string, 0, len(paths))
for dirID := range paths {
ids = append(ids, dirID)
}
var existing []model.StrmDirCache
if err := tx.Where("sync_path_id = ? AND dir_id IN ?", syncPathID, ids).Find(&existing).Error; err != nil {
return err
}
existingRowID := make(map[string]string, len(existing))
for _, row := range existing {
existingRowID[row.DirID] = row.ID
}
now := time.Now()
var creates []model.StrmDirCache
for dirID, path := range paths {
if rowID, ok := existingRowID[dirID]; ok {
if err := tx.Model(&model.StrmDirCache{}).Where("id = ?", rowID).Updates(map[string]any{
"path": path,
"updated_at": now,
}).Error; err != nil {
return err
}
continue
}
creates = append(creates, model.StrmDirCache{
SyncPathID: syncPathID,
DirID: dirID,
Path: path,
})
}
if len(creates) > 0 {
return tx.CreateInBatches(creates, 100).Error
}
return nil
})
})
}
func (r *StrmDirCacheRepository) DeleteBySyncPathID(ctx context.Context, syncPathID string) error {
return withSQLiteBusyRetry(ctx, func() error {
return r.db.WithContext(ctx).Unscoped().Where("sync_path_id = ?", syncPathID).Delete(&model.StrmDirCache{}).Error
+12
View File
@@ -45,6 +45,9 @@ type FileEntry struct {
MTime int64 `json:"mtime,omitempty"`
// PickCode is 115-specific; other providers use ID directly.
PickCode string `json:"pick_code,omitempty"`
// Sha1 is 115-specific content hash(大写 hex,目录/未完成文件可能为空或占位符)。
// 其他网盘不提供,留空时调用方退回大小比对。用于元数据"是否同一文件"的精确判定。
Sha1 string `json:"sha1,omitempty"`
}
// DirectLink is a resolved playback target.
@@ -72,6 +75,15 @@ type Provider interface {
Resolve(ctx context.Context, fileRef string) (*DirectLink, error)
}
// BatchResolver is implemented by providers that can resolve several file
// references in fewer API calls(115 的 downurl 接口支持逗号分隔多个 pick_code,
// 批量换取可显著降低下载队列的换链请求量)。返回以原始引用为键的直链 map;
// 解析失败的引用不在结果中,err 汇报批量机制本身的失败,调用方应据此对缺失
// 项回退到逐个 Resolve。
type BatchResolver interface {
ResolveBatch(ctx context.Context, fileRefs []string) (map[string]*DirectLink, error)
}
// MutableProvider is implemented by cloud bridges that support safe folder
// management through their official API or standard WebDAV methods.
type MutableProvider interface {
+11 -5
View File
@@ -81,7 +81,12 @@ func Test115OpenAPIListPaginates(t *testing.T) {
t.Fatalf("unexpected path %s", r.URL.Path)
}
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
count := 100
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
if limit <= 0 {
limit = 100
}
// 首页返回满页,之后返回 1 条:驱动按 offset/limit 翻页直到短页
count := limit
if offset > 0 {
count = 1
}
@@ -97,11 +102,12 @@ func Test115OpenAPIListPaginates(t *testing.T) {
if err != nil {
t.Fatalf("list: %v", err)
}
if len(entries) != 101 {
t.Fatalf("entries = %d, want 101", len(entries))
// List 使用文档上限 limit=1150:首页 1150 条 + 短页 1 条
if len(entries) != 1151 {
t.Fatalf("entries = %d, want 1151", len(entries))
}
if entries[100].ID != "100" || entries[100].PickCode != "pick100" {
t.Fatalf("last entry wrong: %#v", entries[100])
if entries[1150].ID != "1150" || entries[1150].PickCode != "pick1150" {
t.Fatalf("last entry wrong: %#v", entries[1150])
}
}
+17 -2
View File
@@ -56,8 +56,9 @@ func (p *openAPI115Provider) Ping(ctx context.Context) error {
}
func (p *openAPI115Provider) List(ctx context.Context, dirID string) ([]FileEntry, error) {
// 115 开放平台列表接口按 offset/limit 分页,这里循环取完整个目录
const pageSize = 100
// 115 开放平台列表接口按 offset/limit 分页,这里循环取完整个目录;
// limit 上限 1150(官方文档《获取文件列表》),取上限减少大目录翻页次数
const pageSize = 1150
var out []FileEntry
for offset := 0; ; offset += pageSize {
files, _, err := p.c.GetFsList(ctx, dirID, offset, pageSize)
@@ -72,6 +73,7 @@ func (p *openAPI115Provider) List(ctx context.Context, dirID string) ([]FileEntr
Size: f.FileSize,
MTime: f.ModifiedAt(),
PickCode: f.PickCode,
Sha1: f.Sha1,
})
}
if len(files) < pageSize {
@@ -105,6 +107,19 @@ func (p *openAPI115Provider) ResolveWithUA(ctx context.Context, fileRef, ua stri
return &DirectLink{URL: url, Proxy: false, Headers: map[string]string{"User-Agent": bound}}, nil
}
// ResolveBatch 批量换取直链(downurl 支持逗号分隔多 pick_code,一次请求覆盖
// 整批下载任务的换链)。返回 pickcode → 直链,未解析成功的引用不在结果中;
// err 非 nil 表示批量过程部分/全部失败,调用方对缺失项回退到逐个 Resolve。
// 下载队列统一使用默认 UA,与单个换取的防盗链绑定语义一致。
func (p *openAPI115Provider) ResolveBatch(ctx context.Context, fileRefs []string) (map[string]*DirectLink, error) {
urls, err := p.c.GetDownloadURLsBatch(ctx, fileRefs, "")
out := make(map[string]*DirectLink, len(urls))
for pc, u := range urls {
out[pc] = &DirectLink{URL: u, Proxy: false, Headers: map[string]string{"User-Agent": cloud115.DefaultUA}}
}
return out, err
}
// OpenClient 暴露底层客户端(token 刷新用)。
func (p *openAPI115Provider) OpenClient() *cloud115.OpenClient { return p.c }
@@ -6,6 +6,7 @@ import (
"net/http/httptest"
"net/url"
"strings"
"sync"
"testing"
"time"
)
@@ -472,3 +473,65 @@ func TestFsListRefreshContinue(t *testing.T) {
t.Fatalf("want 1 file, got %d", len(files))
}
}
// TestGetDownloadURLsBatch 验证批量换链:多个 pick_code 合并为一次逗号分隔
// 请求;响应按文件 ID 为键、以条目内 pick_code 映射回请求侧;已缓存的
// pick_code 不再发起请求;缺失项(空 URL)不出现在结果中。
func TestGetDownloadURLsBatch(t *testing.T) {
var mu sync.Mutex
var requests []string
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/open/ufile/downurl" {
t.Errorf("unexpected path %s", r.URL.Path)
}
pc := r.PostFormValue("pick_code")
mu.Lock()
requests = append(requests, pc)
mu.Unlock()
w.Write([]byte(`{"state":true,"data":{
"111":{"pick_code":"batch-pc-a","url":{"url":"https://cdn/a.mkv"}},
"222":{"pick_code":"batch-pc-b","url":{"url":"https://cdn/b.jpg"}},
"333":{"pick_code":"batch-pc-empty","url":{"url":""}}}}`))
})
t.Cleanup(func() {
ClearDownloadURLCache("batch-pc-a")
ClearDownloadURLCache("batch-pc-b")
ClearDownloadURLCache("batch-pc-empty")
})
c := NewOpenClient("100195125", "at1", "rt1")
urls, err := c.GetDownloadURLsBatch(context.Background(),
[]string{"batch-pc-a", "batch-pc-b", "batch-pc-empty", "", "batch-pc-a"}, "")
if err != nil {
t.Fatalf("batch downurl: %v", err)
}
if urls["batch-pc-a"] != "https://cdn/a.mkv" || urls["batch-pc-b"] != "https://cdn/b.jpg" {
t.Fatalf("bad urls: %#v", urls)
}
if _, ok := urls["batch-pc-empty"]; ok {
t.Fatalf("empty-url entry should be absent: %#v", urls)
}
if _, ok := urls[""]; ok {
t.Fatalf("empty pickcode should be absent: %#v", urls)
}
mu.Lock()
if len(requests) != 1 || requests[0] != "batch-pc-a,batch-pc-b,batch-pc-empty" {
mu.Unlock()
t.Fatalf("unexpected downurl requests: %v", requests)
}
mu.Unlock()
// 第二次调用全部命中缓存:不再发任何请求
urls2, err := c.GetDownloadURLsBatch(context.Background(), []string{"batch-pc-a", "batch-pc-b"}, "")
if err != nil {
t.Fatalf("cached batch downurl: %v", err)
}
if urls2["batch-pc-a"] != "https://cdn/a.mkv" {
t.Fatalf("cached url lost: %#v", urls2)
}
mu.Lock()
defer mu.Unlock()
if len(requests) != 1 {
t.Fatalf("cache hit should not issue requests, got %v", requests)
}
}
+35
View File
@@ -0,0 +1,35 @@
// 115 开放平台删除类 API:元数据覆盖上传前清理远端旧文件。
package cloud115
import (
"context"
"strings"
)
// DeleteFiles 批量删除 115 文件(官方接口 POST /open/ufile/delete)。
// 删除为异步执行,文件移入回收站。parentID 为待删除文件所在父目录 ID
//(可选提示,空串省略)。fileIDs 中的空项自动忽略,全为空时直接返回成功。
func (c *OpenClient) DeleteFiles(ctx context.Context, parentID string, fileIDs ...string) error {
ids := make([]string, 0, len(fileIDs))
for _, id := range fileIDs {
if id = strings.TrimSpace(id); id != "" {
ids = append(ids, id)
}
}
if len(ids) == 0 {
return nil
}
params := map[string]string{"file_ids": strings.Join(ids, ",")}
if parentID = strings.TrimSpace(parentID); parentID != "" {
params["parent_id"] = parentID
}
resp, err := c.doAuthJSON(ctx, "POST", ProAPIBase+"/open/ufile/delete", params, 2)
if err != nil {
return err
}
// doJSON 已把 state=false 转为错误返回,这里兜底防御响应外壳异常
if !resp.State {
return NewOpenAPIResponseError(resp.Code, resp.Errno, resp.Message, resp.Error, "115 删除文件失败")
}
return nil
}
+56
View File
@@ -225,6 +225,62 @@ func (c *OpenClient) GetDownloadURLWithUA(ctx context.Context, pickCode, ua stri
return first.URL.URL, nil
}
// downurlBatchSize 单次批量换取直链的 pick_code 数上限。官方 /open/ufile/downurl
// 支持逗号分隔多个 pick_code,批量可大幅降低元数据下载的换链请求量;大小取
// 保守值,减小单个违规/异常文件导致整批失败的爆炸半径。
const downurlBatchSize = 10
// GetDownloadURLsBatch 批量获取下载直链(pickcode → URL)。先查进程内缓存,
// 仅对未命中的 pick_code 分片发起批量请求;单个分片失败时返回已解析的部分与
// 错误,调用方对缺失项回退到逐个 GetDownloadURLWithUA。UA 语义与单个换取
// 一致:直链绑定换取时的 UA,后续下载必须携带同一 UA。
func (c *OpenClient) GetDownloadURLsBatch(ctx context.Context, pickCodes []string, ua string) (map[string]string, error) {
ua = strings.TrimSpace(ua)
out := make(map[string]string, len(pickCodes))
seen := make(map[string]struct{}, len(pickCodes))
missing := make([]string, 0, len(pickCodes))
for _, pc := range pickCodes {
pc = strings.TrimSpace(pc)
if pc == "" {
continue
}
if _, dup := seen[pc]; dup {
continue
}
seen[pc] = struct{}{}
if cached := GetDownloadURLCache(pc, ua); cached != "" {
out[pc] = cached
continue
}
missing = append(missing, pc)
}
for start := 0; start < len(missing); start += downurlBatchSize {
end := start + downurlBatchSize
if end > len(missing) {
end = len(missing)
}
chunk := missing[start:end]
params := map[string]string{"pick_code": strings.Join(chunk, ",")}
resp, err := c.doAuthJSONWithUA(ctx, "POST", ProAPIBase+"/open/ufile/downurl", params, 1, ua)
if err != nil {
return out, err
}
var data map[string]downloadURLData
if err := json.Unmarshal(resp.Data, &data); err != nil {
return out, fmt.Errorf("115: 解析下载地址失败:%w", err)
}
// 响应以文件 ID 为键,条目内的 pick_code 用于映射回请求侧
for _, item := range data {
if item.PickCode == "" || item.URL.URL == "" {
continue
}
SetDownloadURLCache(item.PickCode, item.URL.URL, ua)
out[item.PickCode] = item.URL.URL
}
}
return out, nil
}
// ─── 授权(设备码扫码) ──────────────────────────────────────────────────────
// QrCodeScanStatus 扫码状态。
+111 -14
View File
@@ -52,16 +52,28 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
sleepContext(ctx, 2*time.Second)
continue
}
var wg sync.WaitGroup
// 处于 WAF 冷却的 115 任务先退回,剩余任务在派发前按账号批量换链:
// downurl 支持逗号分隔多个 pick_code,整批任务一次请求即可完成解析,
// 显著减少全局 QPS 限流下的换链请求量。
runnable := make([]*model.StrmDownloadTask, 0, len(tasks))
for i := range tasks {
task := &tasks[i]
if task.Provider == model.StrmProvider115 && s.wafCooldownLeft() > 0 {
s.requeueDownloadTask(task)
continue
}
runnable = append(runnable, task)
}
if len(runnable) == 0 {
continue
}
resolved := s.batchResolve115Links(ctx, runnable)
var wg sync.WaitGroup
for i := range runnable {
wg.Add(1)
go func(i int) {
defer wg.Done()
task := &tasks[i]
if task.Provider == model.StrmProvider115 && s.wafCooldownLeft() > 0 {
s.requeueDownloadTask(task)
return
}
task := runnable[i]
if !s.acquireDownloadSlot(ctx, task.Provider) {
s.requeueDownloadTask(task)
return
@@ -76,7 +88,7 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
s.downloadTaskFailWithRetry(task, "任务执行异常中断")
}
}()
s.processDownloadTask(ctx, task)
s.processDownloadTask(ctx, task, resolved)
completed = true
})
}(i)
@@ -85,6 +97,69 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
}
}
// dlResolveKey 构造批量换链结果 map 的键(按账号隔离,避免极端情况下不同
// 账号的引用串扰)。
func dlResolveKey(accountID, fileRef string) string {
return accountID + "|" + fileRef
}
// batchResolve115Links 在派发执行前对 115 下载任务做批量换链。官方 downurl
// 接口支持逗号分隔多个 pick_code(文档《获取文件下载地址》),按账号把整批
// 任务的 pickcode 合并换取,减少 QPS 限流下的换链请求量。解析结果写入
// pickcode 直链缓存供任务执行时命中;批量失败只记日志并触发风控冷却判定,
// 未解析成功的任务在执行时回退到逐个 Resolve,不影响任务本身。
func (s *StrmService) batchResolve115Links(ctx context.Context, tasks []*model.StrmDownloadTask) map[string]*cloud.DirectLink {
byAcct := map[string][]string{}
seenRef := map[string]map[string]struct{}{}
for _, task := range tasks {
if task.Provider != model.StrmProvider115 {
continue
}
ref := strings.TrimSpace(task.RemoteRef)
if ref == "" {
continue
}
if seenRef[task.AccountID] == nil {
seenRef[task.AccountID] = map[string]struct{}{}
}
if _, dup := seenRef[task.AccountID][ref]; dup {
continue
}
seenRef[task.AccountID][ref] = struct{}{}
byAcct[task.AccountID] = append(byAcct[task.AccountID], ref)
}
resolved := map[string]*cloud.DirectLink{}
for acctID, refs := range byAcct {
acct, err := s.repo.StrmAccount.FindByID(ctx, acctID)
if err != nil || acct == nil {
continue
}
provider, err := s.providerFor(ctx, acct)
if err != nil {
continue
}
batch, ok := provider.(cloud.BatchResolver)
if !ok {
continue
}
links, err := batch.ResolveBatch(ctx, refs)
if err != nil {
if is115Blocked(err) {
s.triggerWAFCooldown()
}
s.log.Warn("batch resolve 115 download links failed; fall back to per-task resolve",
zap.String("account_id", acctID), zap.Int("refs", len(refs)), zap.Error(err))
}
for ref, link := range links {
if link == nil || link.URL == "" {
continue
}
resolved[dlResolveKey(acctID, ref)] = link
}
}
return resolved
}
// requeueDownloadTask 把已认领但未实际执行的任务退回 pending,避免长期停留在 running。
// 退回时必须设置 NextTryAt(WAF 冷却剩余时间):claim 只过滤 next_try_at
// 已过期的任务,不设会让同一批任务被立刻再认领,形成 claim/requeue
@@ -104,7 +179,9 @@ func (s *StrmService) requeueDownloadTask(task *model.StrmDownloadTask) {
}
}
func (s *StrmService) processDownloadTask(ctx context.Context, task *model.StrmDownloadTask) {
// processDownloadTask 处理单个下载任务:解析直链(优先使用批量换链预取的
// 结果,未命中时逐个 Resolve)→ 下载 → 落盘。
func (s *StrmService) processDownloadTask(ctx context.Context, task *model.StrmDownloadTask, resolved map[string]*cloud.DirectLink) {
cleanPath := sanitizeLocalPath(task.LocalPath)
if cleanPath != "" && cleanPath != task.LocalPath {
task.LocalPath = cleanPath
@@ -137,13 +214,16 @@ func (s *StrmService) processDownloadTask(ctx context.Context, task *model.StrmD
s.downloadTaskFailWithRetry(task, err.Error())
return
}
link, err := provider.Resolve(ctx, task.RemoteRef)
if err != nil {
if is115Blocked(err) {
s.triggerWAFCooldown()
link, ok := resolved[dlResolveKey(task.AccountID, task.RemoteRef)]
if !ok || link == nil || link.URL == "" {
link, err = provider.Resolve(ctx, task.RemoteRef)
if err != nil {
if is115Blocked(err) {
s.triggerWAFCooldown()
}
s.downloadTaskFailWithRetry(task, "解析下载地址失败:"+err.Error())
return
}
s.downloadTaskFailWithRetry(task, "解析下载地址失败:"+err.Error())
return
}
if err := downloadToFile(ctx, link, task.LocalPath, s.http); err != nil {
// 直链失效(403/404/410 等):清掉缓存让下一轮重新换取
@@ -286,6 +366,20 @@ func (s *StrmService) processUpload115(ctx context.Context, task *model.StrmUplo
finish(model.StrmTaskFailed, "该网盘不支持元数据上传")
return
}
// 以本地为准:网盘端已有同名但内容不同的旧元数据时,先删除旧文件再上传。
// 115 的上传接口不保证同名覆盖,直接上传可能产生同名重复文件;删除失败则
// 任务重试(旧文件 ID 失效的场景会在下次同步后自动修复)。
if task.RemoteRef != "" {
open115, ok := provider.(cloud.OpenAPI115Provider)
if !ok {
finish(model.StrmTaskFailed, "该网盘不支持删除远端旧元数据")
return
}
if err := open115.OpenClient().DeleteFiles(ctx, task.RemotePath, task.RemoteRef); err != nil {
s.uploadTaskFailWithRetry(task, "删除网盘旧元数据失败:"+err.Error())
return
}
}
f, err := os.Open(task.LocalPath)
if err != nil {
s.uploadTaskFailWithRetry(task, "打开本地文件失败:"+err.Error())
@@ -765,12 +859,15 @@ func (s *StrmService) wafCooldownLeft() time.Duration {
}
// is115Blocked 判断错误是否来自 115 的风控/限流(WAF 405 拦截页或限流错误码)。
// 覆盖两层文案:HTTP 层(doJSON 的"接口触发频控/安全拦截(HTTP 405)")与
// 业务错误码层(OpenAPIError 的"115 接口错误(406/770004)")。
func is115Blocked(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "115 接口返回 http 405") ||
strings.Contains(msg, "115 接口触发频控/安全拦截") ||
strings.Contains(msg, "访问被阻断") ||
strings.Contains(msg, "request has been blocked") ||
strings.Contains(msg, "115 接口错误(770004") ||
+194
View File
@@ -3,11 +3,18 @@ package service
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/truewhile/MeBox/internal/model"
"github.com/truewhile/MeBox/internal/repository"
"github.com/truewhile/MeBox/internal/service/cloud"
"github.com/truewhile/MeBox/internal/service/cloud115"
"go.uber.org/zap"
)
@@ -18,6 +25,7 @@ func TestIs115Blocked(t *testing.T) {
}{
{errors.New("115 接口返回 HTTP 405:<!doctypehtml>...访问被阻断"), true},
{errors.New("115 接口返回 HTTP 405"), true},
{errors.New("115 接口触发频控/安全拦截(HTTP 405):阿里云 WAF 拦截页"), true},
{errors.New("115 接口错误(770004):访问频率过高"), true},
{errors.New("115 接口错误(406):达到访问上限"), true},
{errors.New("下载失败:http 403"), false},
@@ -164,3 +172,189 @@ func TestRequeueDownloadTask(t *testing.T) {
t.Fatalf("task not requeued: %+v", got)
}
}
// TestProcessUpload115DeletesStaleRemoteMetaFirst 验证 115 覆盖上传语义(以本地为准):
// 任务携带网盘旧文件 ID 时,必须先调用 /open/ufile/delete 删除旧元数据再上传本地文件,
// 避免 115 出现同名重复文件;删除请求应携带 file_ids 与父目录 cid。
func TestProcessUpload115DeletesStaleRemoteMetaFirst(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
localFile := filepath.Join(localDir, "movie.nfo")
if err := os.WriteFile(localFile, []byte("local-nfo-data"), 0o644); err != nil {
t.Fatal(err)
}
acct := &model.StrmAccount{Name: "fake115", Provider: "cloud115", Config: "{}", Enabled: true}
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
t.Fatal(err)
}
var mu sync.Mutex
var calls []string
deleteForm := map[string]string{}
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = r.ParseForm()
mu.Lock()
defer mu.Unlock()
switch r.URL.Path {
case "/open/ufile/delete":
calls = append(calls, "delete")
deleteForm["file_ids"] = r.FormValue("file_ids")
deleteForm["parent_id"] = r.FormValue("parent_id")
w.Write([]byte(`{"state":true,"data":[]}`))
case "/open/upload/init":
calls = append(calls, "upload")
// 返回秒传成功,跳过 OSS 真实上传
w.Write([]byte(`{"state":true,"data":{"status":2,"file_id":"new-1","pick_code":"new-pc-1","callback":null}}`))
default:
t.Errorf("unexpected 115 api path %s", r.URL.Path)
w.Write([]byte(`{"state":false,"message":"unexpected path"}`))
}
}))
defer api.Close()
oldPro := cloud115.ProAPIBase
cloud115.ProAPIBase = api.URL
defer func() { cloud115.ProAPIBase = oldPro }()
task := &model.StrmUploadTask{
Base: model.Base{ID: "up-del-1"},
SyncPathID: "p1",
AccountID: acct.ID,
Provider: model.StrmProvider115,
FileName: "movie.nfo",
LocalPath: localFile,
RemotePath: "777",
RemoteRef: "old-file-1",
Status: model.StrmTaskRunning,
}
svc.processUpload115(context.Background(), task)
if task.Status != model.StrmTaskDone {
t.Fatalf("upload task should succeed, status = %s, error = %s", task.Status, task.Error)
}
mu.Lock()
defer mu.Unlock()
if len(calls) != 2 || calls[0] != "delete" || calls[1] != "upload" {
t.Fatalf("expected delete before upload, got calls = %v", calls)
}
if deleteForm["file_ids"] != "old-file-1" {
t.Fatalf("delete file_ids = %q, want old-file-1", deleteForm["file_ids"])
}
if deleteForm["parent_id"] != "777" {
t.Fatalf("delete parent_id = %q, want 777", deleteForm["parent_id"])
}
}
// TestBatchResolve115Links 验证下载队列的批量换链:同账号多个 115 任务的
// pickcode 合并为一次 downurl 请求(官方接口支持逗号分隔多 pick_code),
// 重复引用去重、非 115 任务不参与、直链携带绑定 UA。
func TestBatchResolve115Links(t *testing.T) {
svc := testStrmService(t)
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
t.Fatal(err)
}
var mu sync.Mutex
var requests []string
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/open/ufile/downurl":
_ = r.ParseForm()
mu.Lock()
requests = append(requests, r.PostFormValue("pick_code"))
mu.Unlock()
w.Write([]byte(`{"state":true,"data":{
"111":{"pick_code":"q-pc-a","url":{"url":"http://cdn/a.mkv"}},
"222":{"pick_code":"q-pc-b","url":{"url":"http://cdn/b.jpg"}}}}`))
default:
t.Errorf("unexpected 115 api path %s", r.URL.Path)
w.Write([]byte(`{"state":false,"message":"unexpected path"}`))
}
}))
defer api.Close()
oldPro := cloud115.ProAPIBase
cloud115.ProAPIBase = api.URL
defer func() { cloud115.ProAPIBase = oldPro }()
defer cloud115.ClearDownloadURLCache("q-pc-a")
defer cloud115.ClearDownloadURLCache("q-pc-b")
tasks := []*model.StrmDownloadTask{
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: "q-pc-a"},
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: "q-pc-a"}, // 重复引用
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: "q-pc-b"},
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProviderOpenList, RemoteRef: "ol-ref"},
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: " "}, // 空引用
}
resolved := svc.batchResolve115Links(context.Background(), tasks)
if got := resolved[dlResolveKey(acct.ID, "q-pc-a")]; got == nil || got.URL != "http://cdn/a.mkv" {
t.Fatalf("missing/bad link for q-pc-a: %+v", resolved)
}
if got := resolved[dlResolveKey(acct.ID, "q-pc-b")]; got == nil || got.URL != "http://cdn/b.jpg" {
t.Fatalf("missing/bad link for q-pc-b: %+v", resolved)
}
if got := resolved[dlResolveKey(acct.ID, "q-pc-a")].Headers["User-Agent"]; got != cloud115.DefaultUA {
t.Fatalf("link UA = %q, want default bound UA", got)
}
if _, ok := resolved[dlResolveKey(acct.ID, "ol-ref")]; ok {
t.Fatal("non-115 task should not be batch resolved")
}
mu.Lock()
defer mu.Unlock()
if len(requests) != 1 || requests[0] != "q-pc-a,q-pc-b" {
t.Fatalf("expected single batched downurl request, got %v", requests)
}
}
// TestProcessDownloadTaskUsesPreResolvedLink 验证任务执行时优先使用批量换链
// 预取的直链:downurl 接口保持失败,若任务仍走逐个换链则必然失败。
func TestProcessDownloadTaskUsesPreResolvedLink(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
t.Fatal(err)
}
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer api.Close()
oldPro := cloud115.ProAPIBase
cloud115.ProAPIBase = api.URL
defer func() { cloud115.ProAPIBase = oldPro }()
contentSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("User-Agent") != cloud115.DefaultUA {
t.Errorf("download UA = %q, want bound default UA", r.Header.Get("User-Agent"))
}
_, _ = w.Write([]byte("nfo-content"))
}))
defer contentSrv.Close()
task := &model.StrmDownloadTask{
SyncPathID: "p1",
AccountID: acct.ID,
Provider: model.StrmProvider115,
FileName: "movie.nfo",
RemoteRef: "q-pc-pre",
LocalPath: filepath.Join(localDir, "movie.nfo"),
Status: model.StrmTaskRunning,
}
if err := svc.repo.StrmDownload.Create(context.Background(), task); err != nil {
t.Fatal(err)
}
resolved := map[string]*cloud.DirectLink{
dlResolveKey(acct.ID, "q-pc-pre"): {URL: contentSrv.URL, Headers: map[string]string{"User-Agent": cloud115.DefaultUA}},
}
svc.processDownloadTask(context.Background(), task, resolved)
if task.Status != model.StrmTaskDone {
t.Fatalf("task should be done via pre-resolved link, status = %s, error = %s", task.Status, task.Error)
}
data, err := os.ReadFile(task.LocalPath)
if err != nil || string(data) != "nfo-content" {
t.Fatalf("downloaded file mismatch: err = %v, data = %q", err, string(data))
}
}
+1 -1
View File
@@ -67,7 +67,7 @@ var StrmSettingDefs = map[string]struct {
StrmSettingMinVideoSizeMB: {Default: "0", Label: "最小视频大小(MB)", Kind: "number", Help: "小于该大小的视频文件不生成 STRM,0 表示不限"},
StrmSettingAddPath: {Default: "1", Label: "STRM 链接 path 参数", Kind: "choice", Choices: []string{"1", "2", "3"}, Help: "1=附带完整远端路径 2=仅文件名 3=不带 path"},
StrmSettingDownloadMeta: {Default: "true", Label: "下载元数据", Kind: "bool", Help: "同步时把远端 nfo/图片/字幕下载到本地输出目录"},
StrmSettingUploadMeta: {Default: "false", Label: "上传元数据", Kind: "bool", Help: "同步时把本地元数据上传到远端(需网盘支持写入)"},
StrmSettingUploadMeta: {Default: "false", Label: "上传元数据", Kind: "bool", Help: "同步时把本地元数据上传到远端;本地与网盘元数据不同时以本地为准覆盖(需网盘支持写入)"},
StrmSettingDeleteDir: {Default: "false", Label: "清理空目录", Kind: "bool", Help: "清理远端已删除的多余 .strm/元数据后,删除空目录"},
Strm115RelayKeySetting: {Default: "", Label: "115 中继授权共享密钥", Kind: "text", Help: "QMediaSync/MQFamily 中继授权的共享 AES 密钥(OAUTH_RELAY_ENCRYPTION_KEY);不配置则中继授权不可用"},
StrmSettingDownloadThreads: {Default: "6", Label: "下载队列线程数", Kind: "number", Help: "OpenList/CloudDrive2 元数据下载并发数(115 独立限速为 3)"},
+189 -35
View File
@@ -41,7 +41,9 @@ type strmSyncState struct {
lastProgressFlush time.Time // 上次进度落库时间
seenVideo map[string]bool // "v:"+去掉扩展名的相对路径 → 远端存在该视频
seenMeta map[string]bool // "m:"+相对路径 → 远端存在该元数据
remoteMeta map[string]int64 // 远端元数据大小(上传比对用)
remoteMeta map[string]int64 // 远端元数据大小(上传比对用)
remoteMetaRef map[string]string // "m:"+相对路径 → 远端元数据文件引用(115 文件 ID,覆盖上传前删除旧文件用)
remoteMetaSha1 map[string]string // "m:"+相对路径 → 远端元数据内容 SHA1(115 列表返回;上传/下载精确比对用,其他网盘为空)
seenMetaTarget map[string]cloud.FileEntry
seenVideoTarget map[string]cloud.FileEntry
activeDownloadPaths map[string]bool // 本地已在排队/进行的下载任务路径(内存去重)
@@ -50,6 +52,7 @@ type strmSyncState struct {
pendingUploads []*model.StrmUploadTask
dirCache sync.Map // dirID (string) -> relativePath (string)
dirPathToID map[string]string // relativePath (string) -> dirID(115 上传父目录寻址用,walk 后构建)
dirCacheDirty map[string]string // 待批量落库的目录缓存(dirID → 相对路径),避免逐目录单条 upsert
scanIncomplete atomic.Bool // 远端目录树/文件列表本次扫描不完整 → 禁止增量 prune 误删本地文件
}
@@ -201,6 +204,8 @@ func (s *StrmService) runSync(ctx context.Context, p *model.StrmSyncPath, rec *m
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
@@ -331,6 +336,11 @@ func (st *strmSyncState) run() error {
// 多个目录列表请求的网络往返彼此重叠,大幅缩短大目录树同步耗时。
const strmScanWorkers = 8
// strmProcessWorkers 115 平铺拉取后本地文件分类处理(生成 strm/元数据入队)
// 的 worker 数。本地磁盘 I/O 是大库同步的尾部瓶颈,输出目录在网络挂载上尤甚;
// processRemoteFile 的共享状态均由 st.mu 保护,可安全并发。
const strmProcessWorkers = 8
// walkRemote 并发广度优先遍历网盘目录树。
// 多个 worker 并行执行 List(受全局 115 令牌桶限流约束),子目录动态
// 入队;任一目录失败则取消其余 worker 并返回错误(与旧串行版语义一致)。
@@ -509,8 +519,8 @@ func (st *strmSyncState) isMetaExt(ext string) bool {
// cleanDirRel 对 115 扁平化拉取的目录相对路径逐段套用目录级文件名清洗,
// 确保与 walkRemote / joinLocalRel(sanitizeRelativePath)使用同一套清洗规则。
// 若不清洗,目录名中的冒号等非法字符会直达 rel,而 seenVideo/seenMeta 的 key
// 与磁盘实际路径不一致,导致 pruneLocal 误删已下载的 strm / 元数据。
// 若不清洗,目录名中的冒号等非法字符会直达 rel,而 seenVideo/remoteMeta 的 key
// 与磁盘实际路径不一致,导致 pruneLocal 误删已下载的 strm、上传误传或重复下载。
// 空 rel(根目录)原样返回。
func cleanDirRel(rel string) string {
if rel == "" {
@@ -530,6 +540,34 @@ func cleanDirRel(rel string) string {
return strings.Join(out, "/")
}
// deferDirCacheSave 暂存一条目录缓存写入,由 flushDirCacheSave 统一批量落库。
// 首次全量同步可能有上万个目录,逐目录单条 upsert 会造成明显的 SQLite 写锁
// 竞争;内存 dirCache(sync.Map)始终即时可用,落库仅服务于下次增量预加载。
func (st *strmSyncState) deferDirCacheSave(dirID, relPath string) {
st.mu.Lock()
if st.dirCacheDirty == nil {
st.dirCacheDirty = map[string]string{}
}
st.dirCacheDirty[dirID] = relPath
st.mu.Unlock()
}
// flushDirCacheSave 把暂存的目录缓存一次性批量落库;失败仅记日志(缓存缺失
// 只影响下次增量的目录解析提速,正确性由"重新向 115 获取"兜底)。
func (st *strmSyncState) flushDirCacheSave() {
st.mu.Lock()
dirty := st.dirCacheDirty
st.dirCacheDirty = nil
st.mu.Unlock()
if len(dirty) == 0 {
return
}
if err := st.s.repo.StrmDirCache.SetBatch(st.ctx, st.p.ID, dirty); err != nil {
st.s.log.Warn("batch save strm dir cache failed",
zap.Error(err), zap.Int("count", len(dirty)), zap.String("path_id", st.p.ID))
}
}
// walk115Flat 使用 115 开放平台扁平化分页批量拉取机制与目录拓扑缓存(参考 QMediaSync)。
// 极大地降低 API 请求次数并支持毫秒级/秒级增量同步。
func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
@@ -721,7 +759,7 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
// 解析相对路径
relPath := cleanDirRel(detail.RelativePath(rootCID))
st.dirCache.Store(pid, relPath)
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, pid, relPath)
st.deferDirCacheSave(pid, relPath)
// 顺便解析并缓存 detail.Paths 中包含的中间各层级目录
for _, ancestor := range detail.Paths {
@@ -742,7 +780,7 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
}
ancestorRel := cleanDirRel(subDetail.RelativePath(rootCID))
st.dirCache.Store(ancestor.FileId, ancestorRel)
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, ancestor.FileId, ancestorRel)
st.deferDirCacheSave(ancestor.FileId, ancestorRel)
}
}
}
@@ -757,6 +795,9 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
}()
}
pwg.Wait()
// 目录解析阶段结束即批量落库已解析的缓存:失败路径也保留部分成果,
// 下次同步可少解析一批目录。
st.flushDirCacheSave()
if firstErr != nil {
// 目录树解析失败会导致 rel 塌缩,若继续处理会让大量本地文件
// 被错误判定为"云端不存在"而重复下载/上传,并可能误删本地文件。
@@ -767,10 +808,57 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
st.updateSyncMessage(fmt.Sprintf("正在生成 STRM 与同步文件 (共 %d 个)...", len(allFiles)))
// 5. 分类处理所有文件
// 5. 分类处理所有文件。本地磁盘 I/O(Stat/读内容比对/写盘)远慢于列表
// 拉取,串行消化是大库同步的尾部瓶颈(输出目录在网络挂载上尤甚);
// processRemoteFile 的共享状态均由 st.mu 保护(walkRemote 已并发调用),
// 这里用有界 worker 池并行处理。rel 构建依赖 dirCache 且需在父目录缺失
// 时整体中止,保留在生产者侧串行完成。
type strmFileTask struct {
file cloud115.RemoteFile
rel string
}
fileCh := make(chan strmFileTask)
var (
procWg sync.WaitGroup
procErrMu sync.Mutex
procErr error
)
for i := 0; i < strmProcessWorkers; i++ {
procWg.Add(1)
go func() {
defer procWg.Done()
if err := helper.Recover(st.s.log, "strm.sync.walk115.process", func() error {
for t := range fileCh {
// 中止(ctx 取消)后排空队列即可,不再产生任何写操作
if ctx.Err() != nil {
continue
}
entry := cloud.FileEntry{
ID: t.file.FileId,
Name: t.file.FileName,
IsDir: false,
Size: t.file.FileSize,
MTime: t.file.Utime,
PickCode: t.file.PickCode,
Sha1: t.file.Sha1,
}
st.processRemoteFile(entry, t.rel)
}
return nil
}); err != nil {
procErrMu.Lock()
if procErr == nil {
procErr = err
}
procErrMu.Unlock()
cancel()
}
}()
}
feed:
for _, f := range allFiles {
if ctx.Err() != nil {
return ctx.Err()
break
}
cleanName := cleanEntryName(f.FileName, false)
var rel string
@@ -783,21 +871,27 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
// 父目录不在目录缓存,无法还原真实相对路径。若继续用塌缩后的
// 根路径处理,该文件会被错误判定,导致重复下载/上传或误删本地文件。
// 目录树不完整时宁可中止本次同步,也不带着损坏的 rel 继续执行。
return fmt.Errorf("115: 文件 %s 的父目录未解析成功,目录树不完整,中止同步以防误删/误传", cleanName)
procErrMu.Lock()
if procErr == nil {
procErr = fmt.Errorf("115: 文件 %s 的父目录未解析成功,目录树不完整,中止同步以防误删/误传", cleanName)
}
procErrMu.Unlock()
cancel()
break
}
}
entry := cloud.FileEntry{
ID: f.FileId,
Name: f.FileName,
IsDir: false,
Size: f.FileSize,
MTime: f.Utime,
PickCode: f.PickCode,
select {
case fileCh <- strmFileTask{file: f, rel: rel}:
case <-ctx.Done():
break feed
}
st.processRemoteFile(entry, rel)
}
return nil
close(fileCh)
procWg.Wait()
if procErr != nil {
return procErr
}
return ctx.Err()
}
// handleVideo 生成/更新 .strm 文件。
@@ -933,11 +1027,46 @@ func (st *strmSyncState) strmPathParam(rel string) string {
}
}
// recordRemoteMeta 记录远端存在的元数据索引及文件大小。
// usableSha1 归一化远端内容哈希:115 对目录/未完成文件可能返回空串或占位符 "-",均视为不可用。
func usableSha1(sha string) string {
sha = strings.TrimSpace(sha)
if sha == "-" {
return ""
}
return sha
}
// localSha1Matches 计算本地文件 SHA1 并与远端哈希做大小写不敏感比对
// (115 列表返回大写 hex,本地计算为小写)。读取/哈希失败按"视为同一文件"
// 处理,避免瞬时读文件错误触发大规模重复上传/下载。
func (st *strmSyncState) localSha1Matches(path, remoteSha1 string) bool {
local, err := cloud115.FileSHA1(path)
if err != nil {
st.s.log.Warn("strm 计算本地元数据 SHA1 失败,按同一文件处理",
zap.String("path", path), zap.Error(err))
return true
}
return strings.EqualFold(local, remoteSha1)
}
// recordRemoteMeta 记录远端存在的元数据索引、文件大小、文件引用及内容 SHA1。
func (st *strmSyncState) recordRemoteMeta(entry cloud.FileEntry, rel string) {
st.mu.Lock()
if st.remoteMeta == nil {
st.remoteMeta = map[string]int64{}
}
if st.remoteMetaRef == nil {
st.remoteMetaRef = map[string]string{}
}
if st.remoteMetaSha1 == nil {
st.remoteMetaSha1 = map[string]string{}
}
st.seenMeta["m:"+rel] = true
st.remoteMeta["m:"+rel] = entry.Size
st.remoteMetaRef["m:"+rel] = entry.ID
if sha := usableSha1(entry.Sha1); sha != "" {
st.remoteMetaSha1["m:"+rel] = sha
}
st.mu.Unlock()
}
@@ -971,7 +1100,8 @@ func (st *strmSyncState) flushPendingUploads() {
}
}
// handleMeta 元数据入下载队列(本地已存在且大小一致则跳过)。
// handleMeta 元数据入下载队列。本地已存在时:开启上传元数据则一律跳过(以本地
// 为准);否则大小与 SHA1(115 提供)均一致视为同一文件跳过,内容不同则下载覆盖。
func (st *strmSyncState) handleMeta(entry cloud.FileEntry, rel, ext string) {
st.recordRemoteMeta(entry, rel)
@@ -993,9 +1123,22 @@ func (st *strmSyncState) handleMeta(entry cloud.FileEntry, rel, ext string) {
st.seenMetaTarget[target] = entry
st.mu.Unlock()
if info, err := os.Stat(target); err == nil && info.Size() == entry.Size {
st.touchProgress()
return
if info, err := os.Stat(target); err == nil {
// 开启上传元数据时以本地为准:本地已存在的元数据不再用远端版本覆盖,
// 与网盘版本的差异交给上传队列把本地文件推回网盘,避免下载/上传
// 两个队列互相覆盖形成回环。
if st.cfg.UploadMeta {
st.touchProgress()
return
}
// 同名同大小:115 提供远端 SHA1 时做内容级比对,网盘更新了同大小
// 元数据也能被下载到本地;无哈希(其他网盘/列表未返回)或哈希一致
// 视为同一文件跳过。
remoteSha := usableSha1(entry.Sha1)
if info.Size() == entry.Size && (remoteSha == "" || st.localSha1Matches(target, remoteSha)) {
st.touchProgress()
return
}
}
st.mu.Lock()
if st.activeDownloadPaths == nil {
@@ -1135,6 +1278,8 @@ func (st *strmSyncState) walkLocalSource() error {
}
// scanLocalMetaForUpload 扫描本地元数据,与远端比对后入上传队列。
// 以本地为准:网盘端不存在、同名不同大小、或同名同大小但 SHA1 不同(115 提供
// 远端哈希时做内容级比对)均入队覆盖上传;同名同大小同内容视为同一文件跳过。
func (st *strmSyncState) scanLocalMetaForUpload() error {
defer st.flushPendingUploads()
if st.activeUploadPaths == nil {
@@ -1174,12 +1319,22 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
return nil
}
st.mu.Lock()
_, exists := st.remoteMeta["m:"+rel]
remoteSize, exists := st.remoteMeta["m:"+rel]
remoteRef := st.remoteMetaRef["m:"+rel]
remoteSha1 := st.remoteMetaSha1["m:"+rel]
st.mu.Unlock()
if exists {
// 网盘端已存在该元数据文件,跳过上传
return nil
if exists && remoteSize == info.Size() {
// 网盘端同名同大小:候选同一文件。115 提供远端 SHA1 时做内容级比对,
// 识别"同大小不同内容"(如 nfo 改一个字符长度不变)避免漏传;
// 无哈希(其他网盘/列表未返回)视为同一文件,保持大小比对兜底。
if remoteSha1 == "" || st.localSha1Matches(path, remoteSha1) {
return nil
}
}
// 入队上传(以本地为准):网盘端不存在;或同名但大小不同(必然内容不同);
// 或同名同大小但 SHA1 不同(精确比对发现的同大小不同内容)。
// 115 的上传接口不保证同名覆盖,任务携带远端旧文件 ID(RemoteRef),
// 由上传端先删旧文件再上传;WebDAV/OpenList 的 PutFile 本身即覆盖上传。
st.mu.Lock()
if st.activeUploadPaths != nil && st.activeUploadPaths[path] {
st.mu.Unlock()
@@ -1200,6 +1355,9 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
Size: info.Size(),
Status: model.StrmTaskPending,
}
if exists && st.p.Provider == model.StrmProvider115 {
task.RemoteRef = remoteRef
}
st.mu.Lock()
st.pendingUploads = append(st.pendingUploads, task)
shouldFlush := len(st.pendingUploads) >= 100
@@ -1260,10 +1418,11 @@ func (st *strmSyncState) taskExists(kind, syncPathID, localPath string) bool {
return count > 0
}
// pruneLocal 清理本地多余 .strm 与元数据(远端已不存在),可选删除空目录。
// pruneLocal 清理本地多余 .strm(远端已不存在的视频),可选删除空目录。
// 元数据文件(nfo/图片/字幕等)一律保留:本地刮削结果不因网盘端缺失而被删除。
func (st *strmSyncState) pruneLocal() error {
// 增量同步保护:本次远端扫描不完整(目录详情解析失败 / 文件父路径降级)时,
// seenVideo/seenMeta 覆盖不全,按"远端不存在"清理会误删刚下载或已存在的本地文件,
// seenVideo 覆盖不全,按"远端不存在"清理会误删刚下载或已存在的本地 .strm,
// 进而触发"下次增量重新下载"的循环。此时跳过清理,仅做进度落库。
if st.syncType == model.StrmSyncTypeIncremental && st.scanIncomplete.Load() {
st.s.log.Warn("strm 增量同步跳过清理:本次远端扫描不完整,prune 已禁用",
@@ -1295,16 +1454,11 @@ func (st *strmSyncState) pruneLocal() error {
rel = filepath.ToSlash(rel)
ext := strings.ToLower(filepath.Ext(rel))
remove := false
switch {
case ext == ".strm":
if ext == ".strm" {
relSansExt := rel[:len(rel)-len(ext)]
st.mu.Lock()
remove = !st.seenVideo["v:"+relSansExt]
st.mu.Unlock()
case st.isMetaExt(ext) && st.cfg.DownloadMeta && !st.cfg.UploadMeta && st.p.Provider != model.StrmProviderLocal:
st.mu.Lock()
remove = !st.seenMeta["m:"+rel]
st.mu.Unlock()
}
if remove {
if err := os.Remove(path); err == nil {
+464 -19
View File
@@ -344,14 +344,19 @@ func TestSanitizePathWithSpecialChars(t *testing.T) {
}
}
// TestScanLocalMetaForUpload 验证元数据上传比对逻辑:网盘已存在跳过,网盘不存在才入队上传。
// TestScanLocalMetaForUpload 验证元数据上传比对逻辑:网盘同名同大小(同一文件)跳过,
// 网盘不存在或同名不同大小(内容不同的元数据文件)以本地为准入队上传。
func TestScanLocalMetaForUpload(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
// 本地有 2 个元数据:poster.jpg 和 fanart.jpg
// 本地有 3 个元数据:
// poster.jpg — 网盘已有同名同大小 → 同一文件,跳过
// fanart.jpg — 网盘没有 → 上传
// tvshow.nfo — 网盘已有同名但大小不同(内容不同的元数据文件)→ 以本地为准覆盖上传
writeFile(t, filepath.Join(localDir, "动漫", "poster.jpg"), "poster-data")
writeFile(t, filepath.Join(localDir, "动漫", "fanart.jpg"), "fanart-data")
writeFile(t, filepath.Join(localDir, "动漫", "tvshow.nfo"), "local-nfo-data")
p := &model.StrmSyncPath{
Base: model.Base{ID: "test-path-upload"},
@@ -363,35 +368,47 @@ func TestScanLocalMetaForUpload(t *testing.T) {
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
}
// 模拟远端已存在 poster.jpg
st.remoteMeta["m:动漫/poster.jpg"] = 1000
// 模拟远端已存在 poster.jpg(与本地同一文件)和 tvshow.nfo(与本地不同)
st.remoteMeta["m:动漫/poster.jpg"] = int64(len("poster-data"))
st.remoteMeta["m:动漫/tvshow.nfo"] = 999
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
}
// 此时应该只有 fanart.jpg 入队上传,poster.jpg 被跳过
// fanart.jpg(网盘缺失)与 tvshow.nfo(网盘版本不同)入队,poster.jpg 跳过
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 upload task (fanart.jpg), got %d", len(tasks))
if len(tasks) != 2 {
t.Fatalf("expected 2 upload tasks (fanart.jpg, tvshow.nfo), got %d", len(tasks))
}
if tasks[0].FileName != "fanart.jpg" {
t.Errorf("expected upload task for fanart.jpg, got %s", tasks[0].FileName)
got := map[string]model.StrmUploadTask{}
for _, task := range tasks {
got[task.FileName] = task
}
if _, ok := got["fanart.jpg"]; !ok {
t.Errorf("expected upload task for fanart.jpg, got %v", taskNames(tasks))
}
if _, ok := got["tvshow.nfo"]; !ok {
t.Errorf("expected upload task for tvshow.nfo (local wins), got %v", taskNames(tasks))
}
if _, ok := got["poster.jpg"]; ok {
t.Errorf("poster.jpg (same size on remote) should be skipped")
}
// 再次扫描:fanart.jpg 已经在队列中,应自动去重,不重复入队
// 再次扫描:已入队的文件自动去重,不重复入队
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("second scanLocalMetaForUpload failed: %v", err)
}
@@ -399,8 +416,313 @@ func TestScanLocalMetaForUpload(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(tasks) != 2 {
t.Fatalf("expected still 2 upload tasks after dedup, got %d", len(tasks))
}
}
// TestScanLocalMetaForUpload115CarriesRemoteRef 验证 115 覆盖上传时任务携带
// 网盘旧文件 ID(供上传前删除旧文件,避免同名重复),网盘无同名文件时不携带。
func TestScanLocalMetaForUpload115CarriesRemoteRef(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
// movie.nfo 网盘已有同名但大小不同(需要删除旧文件后覆盖上传)
writeFile(t, filepath.Join(localDir, "movie.nfo"), "local-nfo-data")
// fresh.nfo 网盘没有(普通上传,不携带 ref)
writeFile(t, filepath.Join(localDir, "fresh.nfo"), "fresh-nfo")
p := &model.StrmSyncPath{
Base: model.Base{ID: "test-path-upload-115"},
AccountID: "acct-1",
Provider: model.StrmProvider115,
RemotePath: "100",
LocalPath: localDir,
UploadMeta: true,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
}
st.remoteMeta["m:movie.nfo"] = 1
st.remoteMetaRef["m:movie.nfo"] = "file-42"
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
}
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 2 {
t.Fatalf("expected 2 upload tasks, got %d", len(tasks))
}
for _, task := range tasks {
switch task.FileName {
case "movie.nfo":
if task.RemoteRef != "file-42" {
t.Errorf("movie.nfo upload task should carry remote ref %q, got %q", "file-42", task.RemoteRef)
}
if task.RemotePath != "100" {
t.Errorf("movie.nfo upload task remote path = %q, want parent cid %q", task.RemotePath, "100")
}
case "fresh.nfo":
if task.RemoteRef != "" {
t.Errorf("fresh.nfo (not on remote) should not carry remote ref, got %q", task.RemoteRef)
}
default:
t.Errorf("unexpected upload task %s", task.FileName)
}
}
}
func taskNames(tasks []model.StrmUploadTask) []string {
names := make([]string, 0, len(tasks))
for _, task := range tasks {
names = append(names, task.FileName)
}
return names
}
// TestPruneLocalKeepsLocalMeta 验证清理规则:远端已删除的视频 .strm 仍会被清理,
// 但本地元数据一律保留(即使开启"下载元数据"且未开启"上传元数据"、网盘端没有
// 该元数据,也不再删除本地刮削好的 nfo/图片/字幕)。
func TestPruneLocalKeepsLocalMeta(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
// 阿凡达.strm(对应视频已被网盘删除 → 应清理)+ 阿凡达.nfo(网盘没有 → 保留)
writeFile(t, filepath.Join(localDir, "电影", "阿凡达.strm"), "http://test.local:8096/x")
writeFile(t, filepath.Join(localDir, "电影", "阿凡达.nfo"), "<local scraped meta/>")
writeFile(t, filepath.Join(localDir, "电影", "poster.jpg"), "local-poster")
p := &model.StrmSyncPath{
Base: model.Base{ID: "prune-meta-path"},
Provider: model.StrmProvider115,
RemotePath: "0",
LocalPath: localDir,
DownloadMeta: true,
UploadMeta: false,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: false, MetaExt: []string{"nfo", "jpg"}},
rec: &model.StrmSyncRecord{},
syncType: model.StrmSyncTypeFull,
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
}
// 本次远端扫描既没有看到视频,也没有看到任何元数据
if err := st.pruneLocal(); err != nil {
t.Fatalf("pruneLocal failed: %v", err)
}
if _, err := os.Stat(filepath.Join(localDir, "电影", "阿凡达.strm")); !os.IsNotExist(err) {
t.Fatalf("orphan .strm should be pruned, stat err = %v", err)
}
if _, err := os.Stat(filepath.Join(localDir, "电影", "阿凡达.nfo")); err != nil {
t.Fatalf("local meta must be kept even when missing on remote: %v", err)
}
if _, err := os.Stat(filepath.Join(localDir, "电影", "poster.jpg")); err != nil {
t.Fatalf("local poster must be kept even when missing on remote: %v", err)
}
if st.rec.Pruned != 1 {
t.Fatalf("expected 1 pruned (strm only), got %d", st.rec.Pruned)
}
}
// TestHandleMetaKeepsLocalWhenUploadEnabled 验证下载侧规则:开启"上传元数据"时
// 以本地为准——本地已存在的元数据(即使与网盘大小不同)不再入下载队列被网盘版本
// 覆盖;本地不存在的元数据仍正常入队下载。
func TestHandleMetaKeepsLocalWhenUploadEnabled(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
// 本地已有 test.nfo(大小 50,与网盘版本大小 100 不同)
writeFile(t, filepath.Join(localDir, "test.nfo"), strings.Repeat("L", 50))
p := &model.StrmSyncPath{
Base: model.Base{ID: "meta-local-wins-path"},
Provider: model.StrmProvider115,
RemotePath: "0",
LocalPath: localDir,
DownloadMeta: true,
UploadMeta: true,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
seenMetaTarget: map[string]cloud.FileEntry{},
}
// 网盘版本与本地不同:本地存在 → 不入下载队列(以本地为准)
entry := cloud.FileEntry{ID: "r1", Name: "test.nfo", Size: 100, PickCode: "pc1"}
st.handleMeta(entry, "test.nfo", ".nfo")
// 网盘独有:本地不存在 → 正常入下载队列
missing := cloud.FileEntry{ID: "r2", Name: "absent.nfo", Size: 200, PickCode: "pc2"}
st.handleMeta(missing, "absent.nfo", ".nfo")
st.flushPendingDownloads()
tasks, _, err := svc.repo.StrmDownload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 1 {
t.Fatalf("expected still 1 upload task after dedup, got %d", len(tasks))
t.Fatalf("expected only 1 download task (absent.nfo), got %d", len(tasks))
}
if tasks[0].FileName != "absent.nfo" {
t.Fatalf("expected download task for absent.nfo, got %s", tasks[0].FileName)
}
if st.rec.NewMeta != 1 {
t.Fatalf("expected NewMeta = 1, got %d", st.rec.NewMeta)
}
}
// TestScanLocalMetaForUploadSha1Identity 验证 115 远端 SHA1 可用时按内容精确比对:
// 同名同大小同内容(大小写不敏感)跳过;同名同大小不同内容以本地为准覆盖上传,
// 且任务携带网盘旧文件 ID。纯大小比对无法识别"同大小不同内容"(如 nfo 改一个字符)。
func TestScanLocalMetaForUploadSha1Identity(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
// same.nfo / diff.nfo 本地与网盘大小均相同:
// same.nfo 网盘内容与本地一致(SHA1 相同)→ 同一文件,跳过;
// diff.nfo 网盘上是另一个同大小文件(SHA1 不同)→ 以本地为准覆盖上传。
writeFile(t, filepath.Join(localDir, "same.nfo"), "same-content")
writeFile(t, filepath.Join(localDir, "diff.nfo"), "diff-content")
otherFile := filepath.Join(localDir, "other.tmp")
writeFile(t, otherFile, "other-content!")
sameSha, err := cloud115.FileSHA1(filepath.Join(localDir, "same.nfo"))
if err != nil {
t.Fatal(err)
}
otherSha, err := cloud115.FileSHA1(otherFile)
if err != nil {
t.Fatal(err)
}
p := &model.StrmSyncPath{
Base: model.Base{ID: "test-path-upload-sha1"},
AccountID: "acct-1",
Provider: model.StrmProvider115,
RemotePath: "100",
LocalPath: localDir,
UploadMeta: true,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
}
// 115 返回大写 SHA1,本地计算为小写:同时验证大小写不敏感比对
st.remoteMeta["m:same.nfo"] = int64(len("same-content"))
st.remoteMetaSha1["m:same.nfo"] = strings.ToUpper(sameSha)
st.remoteMeta["m:diff.nfo"] = int64(len("diff-content"))
st.remoteMetaSha1["m:diff.nfo"] = strings.ToUpper(otherSha)
st.remoteMetaRef["m:diff.nfo"] = "old-diff-1"
if err := st.scanLocalMetaForUpload(); err != nil {
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
}
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 upload task (diff.nfo), got %d: %v", len(tasks), taskNames(tasks))
}
if tasks[0].FileName != "diff.nfo" {
t.Fatalf("expected upload task for diff.nfo, got %s", tasks[0].FileName)
}
if tasks[0].RemoteRef != "old-diff-1" {
t.Fatalf("diff.nfo task should carry remote ref %q, got %q", "old-diff-1", tasks[0].RemoteRef)
}
}
// TestHandleMetaSha1Identity 验证下载侧(未开启上传时镜像网盘元数据):
// 同名同大小同 SHA1 跳过;网盘更新了同大小元数据(SHA1 不同)仍会下载;
// 网盘未返回哈希时退回大小比对。
func TestHandleMetaSha1Identity(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
writeFile(t, filepath.Join(localDir, "same.nfo"), "identical-data") // 与网盘内容一致
writeFile(t, filepath.Join(localDir, "stale.nfo"), "stale-content!!") // 网盘已更新为同大小新内容
writeFile(t, filepath.Join(localDir, "nohash.nfo"), "nohash-content") // 网盘未返回 SHA1
otherFile := filepath.Join(localDir, "other.tmp")
writeFile(t, otherFile, "totally-newdata")
sameSha, err := cloud115.FileSHA1(filepath.Join(localDir, "same.nfo"))
if err != nil {
t.Fatal(err)
}
otherSha, err := cloud115.FileSHA1(otherFile)
if err != nil {
t.Fatal(err)
}
p := &model.StrmSyncPath{
Base: model.Base{ID: "meta-sha1-path"},
Provider: model.StrmProvider115,
RemotePath: "0",
LocalPath: localDir,
DownloadMeta: true,
UploadMeta: false,
}
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: false, MetaExt: []string{"nfo"}},
rec: &model.StrmSyncRecord{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
seenMetaTarget: map[string]cloud.FileEntry{},
}
st.handleMeta(cloud.FileEntry{ID: "r1", Name: "same.nfo", Size: int64(len("identical-data")), Sha1: strings.ToUpper(sameSha), PickCode: "pc1"}, "same.nfo", ".nfo")
st.handleMeta(cloud.FileEntry{ID: "r2", Name: "stale.nfo", Size: int64(len("stale-content!!")), Sha1: strings.ToUpper(otherSha), PickCode: "pc2"}, "stale.nfo", ".nfo")
st.handleMeta(cloud.FileEntry{ID: "r3", Name: "nohash.nfo", Size: int64(len("nohash-content")), PickCode: "pc3"}, "nohash.nfo", ".nfo")
st.flushPendingDownloads()
tasks, _, err := svc.repo.StrmDownload.List(context.Background(), "", 1, 10)
if err != nil {
t.Fatal(err)
}
if len(tasks) != 1 {
t.Fatalf("expected only 1 download task (stale.nfo), got %d", len(tasks))
}
if tasks[0].FileName != "stale.nfo" {
t.Fatalf("expected download task for stale.nfo, got %s", tasks[0].FileName)
}
}
@@ -672,7 +994,7 @@ func TestWalk115FlatAbortsOnDirResolveFailure(t *testing.T) {
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/open/ufile/files":
w.Write([]byte(`{"state":true,"count":1,"data":[{"fid":"100","pid":"999","fc":1,"fn":"movie.mkv","pc":"pc1","upt":1700000000,"fs":1024}]}`))
w.Write([]byte(`{"state":true,"count":1,"data":[{"fid":"100","pid":"999","fc":"1","fn":"movie.mkv","pc":"pc1","upt":1700000000,"fs":1024}]}`))
case "/open/folder/get_info":
w.Write([]byte(`{"state":false,"code":40140123,"message":"access_token 格式错误"}`))
default:
@@ -716,3 +1038,126 @@ func TestWalk115FlatAbortsOnDirResolveFailure(t *testing.T) {
t.Fatalf("expected no .strm written after abort, got %d", strmCount)
}
}
// TestWalk115FlatConcurrentProcessing 验证 115 平铺拉取后文件分类处理走并发
// worker 池:同一父目录下多个视频的 strm 全部生成,且解析出的目录拓扑缓存
// 通过 SetBatch 批量落库,供下次增量同步预加载(省掉重复的 get_info 调用)。
func TestWalk115FlatConcurrentProcessing(t *testing.T) {
svc := testStrmService(t)
localDir := t.TempDir()
acct := &model.StrmAccount{Name: "fake115", Provider: "cloud115", Config: "{}", Enabled: true}
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
t.Fatal(err)
}
p := &model.StrmSyncPath{
Base: model.Base{ID: "flat-path"},
AccountID: acct.ID,
Provider: model.StrmProvider115,
RemotePath: "0",
LocalPath: localDir,
}
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/open/ufile/files":
w.Write([]byte(`{"state":true,"count":3,"data":[
{"fid":"101","pid":"999","fc":"1","fn":"m1.mkv","pc":"pc1","upt":1700000001,"fs":1024},
{"fid":"102","pid":"999","fc":"1","fn":"m2.mkv","pc":"pc2","upt":1700000002,"fs":2048},
{"fid":"103","pid":"999","fc":"1","fn":"m3.mkv","pc":"pc3","upt":1700000003,"fs":4096}]}`))
case "/open/folder/get_info":
w.Write([]byte(`{"state":true,"data":{"file_id":"999","file_name":"Movies","file_category":"0",
"paths":[{"file_id":"0","file_name":"根目录"},{"file_id":"999","file_name":"Movies"}]}}`))
default:
t.Errorf("unexpected path %s", r.URL.Path)
}
}))
defer api.Close()
oldPro := cloud115.ProAPIBase
cloud115.ProAPIBase = api.URL
defer func() { cloud115.ProAPIBase = oldPro }()
oc := cloud115.NewOpenClient("app", "at", "rt")
st := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
provider: cloud.NewOpenAPI115("app", "at", "rt"),
cfg: &strmPathConfig{VideoExt: []string{"mkv"}, MetaExt: []string{"nfo"}, AddPath: 1},
rec: &model.StrmSyncRecord{},
syncType: model.StrmSyncTypeFull,
dirCache: sync.Map{},
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
if err := st.walk115Flat(oc); err != nil {
t.Fatalf("walk115Flat: %v", err)
}
// 3 个视频的 strm 全部生成
if st.rec.NewStrm != 3 {
t.Fatalf("expected 3 strm created, got %d", st.rec.NewStrm)
}
for _, name := range []string{"m1.strm", "m2.strm", "m3.strm"} {
if _, err := os.Stat(filepath.Join(localDir, "Movies", name)); err != nil {
t.Fatalf("strm %s missing: %v", name, err)
}
}
// 目录拓扑缓存已批量落库
rows, err := svc.repo.StrmDirCache.ListBySyncPathID(context.Background(), p.ID)
if err != nil {
t.Fatal(err)
}
if len(rows) != 1 || rows[0].DirID != "999" || rows[0].Path != "Movies" {
t.Fatalf("dir cache rows = %#v, want one row for dir 999 -> Movies", rows)
}
// 增量同步复用目录缓存:不再发起 get_info 调用
var infoCalls int
api.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/open/ufile/files":
w.Write([]byte(`{"state":true,"count":3,"data":[
{"fid":"101","pid":"999","fc":"1","fn":"m1.mkv","pc":"pc1","upt":1700000001,"fs":1024},
{"fid":"102","pid":"999","fc":"1","fn":"m2.mkv","pc":"pc2","upt":1700000002,"fs":2048},
{"fid":"103","pid":"999","fc":"1","fn":"m3.mkv","pc":"pc3","upt":1700000003,"fs":4096}]}`))
case "/open/folder/get_info":
infoCalls++
w.Write([]byte(`{"state":true,"data":{"file_id":"999","file_name":"Movies","file_category":"0","paths":[]}}`))
default:
t.Errorf("unexpected path %s", r.URL.Path)
}
})
st2 := &strmSyncState{
s: svc,
ctx: context.Background(),
p: p,
provider: cloud.NewOpenAPI115("app", "at", "rt"),
cfg: st.cfg,
rec: &model.StrmSyncRecord{},
syncType: model.StrmSyncTypeIncremental,
dirCache: sync.Map{},
seenVideo: map[string]bool{},
seenMeta: map[string]bool{},
remoteMeta: map[string]int64{},
remoteMetaRef: map[string]string{},
remoteMetaSha1: map[string]string{},
seenMetaTarget: map[string]cloud.FileEntry{},
seenVideoTarget: map[string]cloud.FileEntry{},
}
if err := st2.walk115Flat(oc); err != nil {
t.Fatalf("incremental walk115Flat: %v", err)
}
if infoCalls != 0 {
t.Fatalf("incremental sync should reuse dir cache, got %d get_info calls", infoCalls)
}
if st2.rec.NewStrm != 0 || st2.rec.Skipped != 3 {
t.Fatalf("incremental sync should skip all, new = %d skipped = %d", st2.rec.NewStrm, st2.rec.Skipped)
}
}