mirror of
https://github.com/truewhile/MeBox.git
synced 2026-10-01 20:16:36 +08:00
优化,添加115接口文档
This commit is contained in:
@@ -124,6 +124,7 @@ type StrmUploadTask struct {
|
||||
FileName string `gorm:"size:512" json:"file_name"`
|
||||
LocalPath string `gorm:"size:1024" json:"local_path"` // 本地源文件
|
||||
RemotePath string `gorm:"size:1024" json:"remote_path"` // 远端目标路径
|
||||
RemoteRef string `gorm:"size:1024" json:"remote_ref"` // 远端同名旧文件引用(115 文件 ID;上传覆盖前先删除旧文件,WebDAV/OpenList 直接覆盖无需删除)
|
||||
Size int64 `json:"size"`
|
||||
Status string `gorm:"size:16;index" json:"status"`
|
||||
Error string `gorm:"size:1024" json:"error"`
|
||||
|
||||
@@ -901,6 +901,53 @@ func (r *StrmDirCacheRepository) Set(ctx context.Context, syncPathID, dirID, pat
|
||||
})
|
||||
}
|
||||
|
||||
// SetBatch 批量 upsert 目录缓存(dirID → 相对路径)。单个事务内先查出已存在
|
||||
// 行再分流更新/插入,替代同步流程逐目录单条 Set,避免首次全量同步上万目录时
|
||||
// 的 SQLite 写锁竞争。同一 dirID 的重复项以 map 语义取最后一次写入。
|
||||
func (r *StrmDirCacheRepository) SetBatch(ctx context.Context, syncPathID string, paths map[string]string) error {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
return withSQLiteBusyRetry(ctx, func() error {
|
||||
return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
ids := make([]string, 0, len(paths))
|
||||
for dirID := range paths {
|
||||
ids = append(ids, dirID)
|
||||
}
|
||||
var existing []model.StrmDirCache
|
||||
if err := tx.Where("sync_path_id = ? AND dir_id IN ?", syncPathID, ids).Find(&existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
existingRowID := make(map[string]string, len(existing))
|
||||
for _, row := range existing {
|
||||
existingRowID[row.DirID] = row.ID
|
||||
}
|
||||
now := time.Now()
|
||||
var creates []model.StrmDirCache
|
||||
for dirID, path := range paths {
|
||||
if rowID, ok := existingRowID[dirID]; ok {
|
||||
if err := tx.Model(&model.StrmDirCache{}).Where("id = ?", rowID).Updates(map[string]any{
|
||||
"path": path,
|
||||
"updated_at": now,
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
creates = append(creates, model.StrmDirCache{
|
||||
SyncPathID: syncPathID,
|
||||
DirID: dirID,
|
||||
Path: path,
|
||||
})
|
||||
}
|
||||
if len(creates) > 0 {
|
||||
return tx.CreateInBatches(creates, 100).Error
|
||||
}
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (r *StrmDirCacheRepository) DeleteBySyncPathID(ctx context.Context, syncPathID string) error {
|
||||
return withSQLiteBusyRetry(ctx, func() error {
|
||||
return r.db.WithContext(ctx).Unscoped().Where("sync_path_id = ?", syncPathID).Delete(&model.StrmDirCache{}).Error
|
||||
|
||||
@@ -45,6 +45,9 @@ type FileEntry struct {
|
||||
MTime int64 `json:"mtime,omitempty"`
|
||||
// PickCode is 115-specific; other providers use ID directly.
|
||||
PickCode string `json:"pick_code,omitempty"`
|
||||
// Sha1 is 115-specific content hash(大写 hex,目录/未完成文件可能为空或占位符)。
|
||||
// 其他网盘不提供,留空时调用方退回大小比对。用于元数据"是否同一文件"的精确判定。
|
||||
Sha1 string `json:"sha1,omitempty"`
|
||||
}
|
||||
|
||||
// DirectLink is a resolved playback target.
|
||||
@@ -72,6 +75,15 @@ type Provider interface {
|
||||
Resolve(ctx context.Context, fileRef string) (*DirectLink, error)
|
||||
}
|
||||
|
||||
// BatchResolver is implemented by providers that can resolve several file
|
||||
// references in fewer API calls(115 的 downurl 接口支持逗号分隔多个 pick_code,
|
||||
// 批量换取可显著降低下载队列的换链请求量)。返回以原始引用为键的直链 map;
|
||||
// 解析失败的引用不在结果中,err 汇报批量机制本身的失败,调用方应据此对缺失
|
||||
// 项回退到逐个 Resolve。
|
||||
type BatchResolver interface {
|
||||
ResolveBatch(ctx context.Context, fileRefs []string) (map[string]*DirectLink, error)
|
||||
}
|
||||
|
||||
// MutableProvider is implemented by cloud bridges that support safe folder
|
||||
// management through their official API or standard WebDAV methods.
|
||||
type MutableProvider interface {
|
||||
|
||||
@@ -81,7 +81,12 @@ func Test115OpenAPIListPaginates(t *testing.T) {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
|
||||
count := 100
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
// 首页返回满页,之后返回 1 条:驱动按 offset/limit 翻页直到短页
|
||||
count := limit
|
||||
if offset > 0 {
|
||||
count = 1
|
||||
}
|
||||
@@ -97,11 +102,12 @@ func Test115OpenAPIListPaginates(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(entries) != 101 {
|
||||
t.Fatalf("entries = %d, want 101", len(entries))
|
||||
// List 使用文档上限 limit=1150:首页 1150 条 + 短页 1 条
|
||||
if len(entries) != 1151 {
|
||||
t.Fatalf("entries = %d, want 1151", len(entries))
|
||||
}
|
||||
if entries[100].ID != "100" || entries[100].PickCode != "pick100" {
|
||||
t.Fatalf("last entry wrong: %#v", entries[100])
|
||||
if entries[1150].ID != "1150" || entries[1150].PickCode != "pick1150" {
|
||||
t.Fatalf("last entry wrong: %#v", entries[1150])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -56,8 +56,9 @@ func (p *openAPI115Provider) Ping(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (p *openAPI115Provider) List(ctx context.Context, dirID string) ([]FileEntry, error) {
|
||||
// 115 开放平台列表接口按 offset/limit 分页,这里循环取完整个目录
|
||||
const pageSize = 100
|
||||
// 115 开放平台列表接口按 offset/limit 分页,这里循环取完整个目录;
|
||||
// limit 上限 1150(官方文档《获取文件列表》),取上限减少大目录翻页次数
|
||||
const pageSize = 1150
|
||||
var out []FileEntry
|
||||
for offset := 0; ; offset += pageSize {
|
||||
files, _, err := p.c.GetFsList(ctx, dirID, offset, pageSize)
|
||||
@@ -72,6 +73,7 @@ func (p *openAPI115Provider) List(ctx context.Context, dirID string) ([]FileEntr
|
||||
Size: f.FileSize,
|
||||
MTime: f.ModifiedAt(),
|
||||
PickCode: f.PickCode,
|
||||
Sha1: f.Sha1,
|
||||
})
|
||||
}
|
||||
if len(files) < pageSize {
|
||||
@@ -105,6 +107,19 @@ func (p *openAPI115Provider) ResolveWithUA(ctx context.Context, fileRef, ua stri
|
||||
return &DirectLink{URL: url, Proxy: false, Headers: map[string]string{"User-Agent": bound}}, nil
|
||||
}
|
||||
|
||||
// ResolveBatch 批量换取直链(downurl 支持逗号分隔多 pick_code,一次请求覆盖
|
||||
// 整批下载任务的换链)。返回 pickcode → 直链,未解析成功的引用不在结果中;
|
||||
// err 非 nil 表示批量过程部分/全部失败,调用方对缺失项回退到逐个 Resolve。
|
||||
// 下载队列统一使用默认 UA,与单个换取的防盗链绑定语义一致。
|
||||
func (p *openAPI115Provider) ResolveBatch(ctx context.Context, fileRefs []string) (map[string]*DirectLink, error) {
|
||||
urls, err := p.c.GetDownloadURLsBatch(ctx, fileRefs, "")
|
||||
out := make(map[string]*DirectLink, len(urls))
|
||||
for pc, u := range urls {
|
||||
out[pc] = &DirectLink{URL: u, Proxy: false, Headers: map[string]string{"User-Agent": cloud115.DefaultUA}}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// OpenClient 暴露底层客户端(token 刷新用)。
|
||||
func (p *openAPI115Provider) OpenClient() *cloud115.OpenClient { return p.c }
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -472,3 +473,65 @@ func TestFsListRefreshContinue(t *testing.T) {
|
||||
t.Fatalf("want 1 file, got %d", len(files))
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetDownloadURLsBatch 验证批量换链:多个 pick_code 合并为一次逗号分隔
|
||||
// 请求;响应按文件 ID 为键、以条目内 pick_code 映射回请求侧;已缓存的
|
||||
// pick_code 不再发起请求;缺失项(空 URL)不出现在结果中。
|
||||
func TestGetDownloadURLsBatch(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var requests []string
|
||||
mockAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/open/ufile/downurl" {
|
||||
t.Errorf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
pc := r.PostFormValue("pick_code")
|
||||
mu.Lock()
|
||||
requests = append(requests, pc)
|
||||
mu.Unlock()
|
||||
w.Write([]byte(`{"state":true,"data":{
|
||||
"111":{"pick_code":"batch-pc-a","url":{"url":"https://cdn/a.mkv"}},
|
||||
"222":{"pick_code":"batch-pc-b","url":{"url":"https://cdn/b.jpg"}},
|
||||
"333":{"pick_code":"batch-pc-empty","url":{"url":""}}}}`))
|
||||
})
|
||||
t.Cleanup(func() {
|
||||
ClearDownloadURLCache("batch-pc-a")
|
||||
ClearDownloadURLCache("batch-pc-b")
|
||||
ClearDownloadURLCache("batch-pc-empty")
|
||||
})
|
||||
|
||||
c := NewOpenClient("100195125", "at1", "rt1")
|
||||
urls, err := c.GetDownloadURLsBatch(context.Background(),
|
||||
[]string{"batch-pc-a", "batch-pc-b", "batch-pc-empty", "", "batch-pc-a"}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("batch downurl: %v", err)
|
||||
}
|
||||
if urls["batch-pc-a"] != "https://cdn/a.mkv" || urls["batch-pc-b"] != "https://cdn/b.jpg" {
|
||||
t.Fatalf("bad urls: %#v", urls)
|
||||
}
|
||||
if _, ok := urls["batch-pc-empty"]; ok {
|
||||
t.Fatalf("empty-url entry should be absent: %#v", urls)
|
||||
}
|
||||
if _, ok := urls[""]; ok {
|
||||
t.Fatalf("empty pickcode should be absent: %#v", urls)
|
||||
}
|
||||
mu.Lock()
|
||||
if len(requests) != 1 || requests[0] != "batch-pc-a,batch-pc-b,batch-pc-empty" {
|
||||
mu.Unlock()
|
||||
t.Fatalf("unexpected downurl requests: %v", requests)
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// 第二次调用全部命中缓存:不再发任何请求
|
||||
urls2, err := c.GetDownloadURLsBatch(context.Background(), []string{"batch-pc-a", "batch-pc-b"}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("cached batch downurl: %v", err)
|
||||
}
|
||||
if urls2["batch-pc-a"] != "https://cdn/a.mkv" {
|
||||
t.Fatalf("cached url lost: %#v", urls2)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(requests) != 1 {
|
||||
t.Fatalf("cache hit should not issue requests, got %v", requests)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// 115 开放平台删除类 API:元数据覆盖上传前清理远端旧文件。
|
||||
package cloud115
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DeleteFiles 批量删除 115 文件(官方接口 POST /open/ufile/delete)。
|
||||
// 删除为异步执行,文件移入回收站。parentID 为待删除文件所在父目录 ID
|
||||
//(可选提示,空串省略)。fileIDs 中的空项自动忽略,全为空时直接返回成功。
|
||||
func (c *OpenClient) DeleteFiles(ctx context.Context, parentID string, fileIDs ...string) error {
|
||||
ids := make([]string, 0, len(fileIDs))
|
||||
for _, id := range fileIDs {
|
||||
if id = strings.TrimSpace(id); id != "" {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
params := map[string]string{"file_ids": strings.Join(ids, ",")}
|
||||
if parentID = strings.TrimSpace(parentID); parentID != "" {
|
||||
params["parent_id"] = parentID
|
||||
}
|
||||
resp, err := c.doAuthJSON(ctx, "POST", ProAPIBase+"/open/ufile/delete", params, 2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// doJSON 已把 state=false 转为错误返回,这里兜底防御响应外壳异常
|
||||
if !resp.State {
|
||||
return NewOpenAPIResponseError(resp.Code, resp.Errno, resp.Message, resp.Error, "115 删除文件失败")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -225,6 +225,62 @@ func (c *OpenClient) GetDownloadURLWithUA(ctx context.Context, pickCode, ua stri
|
||||
return first.URL.URL, nil
|
||||
}
|
||||
|
||||
// downurlBatchSize 单次批量换取直链的 pick_code 数上限。官方 /open/ufile/downurl
|
||||
// 支持逗号分隔多个 pick_code,批量可大幅降低元数据下载的换链请求量;大小取
|
||||
// 保守值,减小单个违规/异常文件导致整批失败的爆炸半径。
|
||||
const downurlBatchSize = 10
|
||||
|
||||
// GetDownloadURLsBatch 批量获取下载直链(pickcode → URL)。先查进程内缓存,
|
||||
// 仅对未命中的 pick_code 分片发起批量请求;单个分片失败时返回已解析的部分与
|
||||
// 错误,调用方对缺失项回退到逐个 GetDownloadURLWithUA。UA 语义与单个换取
|
||||
// 一致:直链绑定换取时的 UA,后续下载必须携带同一 UA。
|
||||
func (c *OpenClient) GetDownloadURLsBatch(ctx context.Context, pickCodes []string, ua string) (map[string]string, error) {
|
||||
ua = strings.TrimSpace(ua)
|
||||
out := make(map[string]string, len(pickCodes))
|
||||
seen := make(map[string]struct{}, len(pickCodes))
|
||||
missing := make([]string, 0, len(pickCodes))
|
||||
for _, pc := range pickCodes {
|
||||
pc = strings.TrimSpace(pc)
|
||||
if pc == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[pc]; dup {
|
||||
continue
|
||||
}
|
||||
seen[pc] = struct{}{}
|
||||
if cached := GetDownloadURLCache(pc, ua); cached != "" {
|
||||
out[pc] = cached
|
||||
continue
|
||||
}
|
||||
missing = append(missing, pc)
|
||||
}
|
||||
for start := 0; start < len(missing); start += downurlBatchSize {
|
||||
end := start + downurlBatchSize
|
||||
if end > len(missing) {
|
||||
end = len(missing)
|
||||
}
|
||||
chunk := missing[start:end]
|
||||
params := map[string]string{"pick_code": strings.Join(chunk, ",")}
|
||||
resp, err := c.doAuthJSONWithUA(ctx, "POST", ProAPIBase+"/open/ufile/downurl", params, 1, ua)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var data map[string]downloadURLData
|
||||
if err := json.Unmarshal(resp.Data, &data); err != nil {
|
||||
return out, fmt.Errorf("115: 解析下载地址失败:%w", err)
|
||||
}
|
||||
// 响应以文件 ID 为键,条目内的 pick_code 用于映射回请求侧
|
||||
for _, item := range data {
|
||||
if item.PickCode == "" || item.URL.URL == "" {
|
||||
continue
|
||||
}
|
||||
SetDownloadURLCache(item.PickCode, item.URL.URL, ua)
|
||||
out[item.PickCode] = item.URL.URL
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ─── 授权(设备码扫码) ──────────────────────────────────────────────────────
|
||||
|
||||
// QrCodeScanStatus 扫码状态。
|
||||
|
||||
+111
-14
@@ -52,16 +52,28 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
|
||||
sleepContext(ctx, 2*time.Second)
|
||||
continue
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
// 处于 WAF 冷却的 115 任务先退回,剩余任务在派发前按账号批量换链:
|
||||
// downurl 支持逗号分隔多个 pick_code,整批任务一次请求即可完成解析,
|
||||
// 显著减少全局 QPS 限流下的换链请求量。
|
||||
runnable := make([]*model.StrmDownloadTask, 0, len(tasks))
|
||||
for i := range tasks {
|
||||
task := &tasks[i]
|
||||
if task.Provider == model.StrmProvider115 && s.wafCooldownLeft() > 0 {
|
||||
s.requeueDownloadTask(task)
|
||||
continue
|
||||
}
|
||||
runnable = append(runnable, task)
|
||||
}
|
||||
if len(runnable) == 0 {
|
||||
continue
|
||||
}
|
||||
resolved := s.batchResolve115Links(ctx, runnable)
|
||||
var wg sync.WaitGroup
|
||||
for i := range runnable {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
task := &tasks[i]
|
||||
if task.Provider == model.StrmProvider115 && s.wafCooldownLeft() > 0 {
|
||||
s.requeueDownloadTask(task)
|
||||
return
|
||||
}
|
||||
task := runnable[i]
|
||||
if !s.acquireDownloadSlot(ctx, task.Provider) {
|
||||
s.requeueDownloadTask(task)
|
||||
return
|
||||
@@ -76,7 +88,7 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
|
||||
s.downloadTaskFailWithRetry(task, "任务执行异常中断")
|
||||
}
|
||||
}()
|
||||
s.processDownloadTask(ctx, task)
|
||||
s.processDownloadTask(ctx, task, resolved)
|
||||
completed = true
|
||||
})
|
||||
}(i)
|
||||
@@ -85,6 +97,69 @@ func (s *StrmService) downloadWorker(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// dlResolveKey 构造批量换链结果 map 的键(按账号隔离,避免极端情况下不同
|
||||
// 账号的引用串扰)。
|
||||
func dlResolveKey(accountID, fileRef string) string {
|
||||
return accountID + "|" + fileRef
|
||||
}
|
||||
|
||||
// batchResolve115Links 在派发执行前对 115 下载任务做批量换链。官方 downurl
|
||||
// 接口支持逗号分隔多个 pick_code(文档《获取文件下载地址》),按账号把整批
|
||||
// 任务的 pickcode 合并换取,减少 QPS 限流下的换链请求量。解析结果写入
|
||||
// pickcode 直链缓存供任务执行时命中;批量失败只记日志并触发风控冷却判定,
|
||||
// 未解析成功的任务在执行时回退到逐个 Resolve,不影响任务本身。
|
||||
func (s *StrmService) batchResolve115Links(ctx context.Context, tasks []*model.StrmDownloadTask) map[string]*cloud.DirectLink {
|
||||
byAcct := map[string][]string{}
|
||||
seenRef := map[string]map[string]struct{}{}
|
||||
for _, task := range tasks {
|
||||
if task.Provider != model.StrmProvider115 {
|
||||
continue
|
||||
}
|
||||
ref := strings.TrimSpace(task.RemoteRef)
|
||||
if ref == "" {
|
||||
continue
|
||||
}
|
||||
if seenRef[task.AccountID] == nil {
|
||||
seenRef[task.AccountID] = map[string]struct{}{}
|
||||
}
|
||||
if _, dup := seenRef[task.AccountID][ref]; dup {
|
||||
continue
|
||||
}
|
||||
seenRef[task.AccountID][ref] = struct{}{}
|
||||
byAcct[task.AccountID] = append(byAcct[task.AccountID], ref)
|
||||
}
|
||||
resolved := map[string]*cloud.DirectLink{}
|
||||
for acctID, refs := range byAcct {
|
||||
acct, err := s.repo.StrmAccount.FindByID(ctx, acctID)
|
||||
if err != nil || acct == nil {
|
||||
continue
|
||||
}
|
||||
provider, err := s.providerFor(ctx, acct)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
batch, ok := provider.(cloud.BatchResolver)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
links, err := batch.ResolveBatch(ctx, refs)
|
||||
if err != nil {
|
||||
if is115Blocked(err) {
|
||||
s.triggerWAFCooldown()
|
||||
}
|
||||
s.log.Warn("batch resolve 115 download links failed; fall back to per-task resolve",
|
||||
zap.String("account_id", acctID), zap.Int("refs", len(refs)), zap.Error(err))
|
||||
}
|
||||
for ref, link := range links {
|
||||
if link == nil || link.URL == "" {
|
||||
continue
|
||||
}
|
||||
resolved[dlResolveKey(acctID, ref)] = link
|
||||
}
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
// requeueDownloadTask 把已认领但未实际执行的任务退回 pending,避免长期停留在 running。
|
||||
// 退回时必须设置 NextTryAt(WAF 冷却剩余时间):claim 只过滤 next_try_at
|
||||
// 已过期的任务,不设会让同一批任务被立刻再认领,形成 claim/requeue
|
||||
@@ -104,7 +179,9 @@ func (s *StrmService) requeueDownloadTask(task *model.StrmDownloadTask) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StrmService) processDownloadTask(ctx context.Context, task *model.StrmDownloadTask) {
|
||||
// processDownloadTask 处理单个下载任务:解析直链(优先使用批量换链预取的
|
||||
// 结果,未命中时逐个 Resolve)→ 下载 → 落盘。
|
||||
func (s *StrmService) processDownloadTask(ctx context.Context, task *model.StrmDownloadTask, resolved map[string]*cloud.DirectLink) {
|
||||
cleanPath := sanitizeLocalPath(task.LocalPath)
|
||||
if cleanPath != "" && cleanPath != task.LocalPath {
|
||||
task.LocalPath = cleanPath
|
||||
@@ -137,13 +214,16 @@ func (s *StrmService) processDownloadTask(ctx context.Context, task *model.StrmD
|
||||
s.downloadTaskFailWithRetry(task, err.Error())
|
||||
return
|
||||
}
|
||||
link, err := provider.Resolve(ctx, task.RemoteRef)
|
||||
if err != nil {
|
||||
if is115Blocked(err) {
|
||||
s.triggerWAFCooldown()
|
||||
link, ok := resolved[dlResolveKey(task.AccountID, task.RemoteRef)]
|
||||
if !ok || link == nil || link.URL == "" {
|
||||
link, err = provider.Resolve(ctx, task.RemoteRef)
|
||||
if err != nil {
|
||||
if is115Blocked(err) {
|
||||
s.triggerWAFCooldown()
|
||||
}
|
||||
s.downloadTaskFailWithRetry(task, "解析下载地址失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
s.downloadTaskFailWithRetry(task, "解析下载地址失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
if err := downloadToFile(ctx, link, task.LocalPath, s.http); err != nil {
|
||||
// 直链失效(403/404/410 等):清掉缓存让下一轮重新换取
|
||||
@@ -286,6 +366,20 @@ func (s *StrmService) processUpload115(ctx context.Context, task *model.StrmUplo
|
||||
finish(model.StrmTaskFailed, "该网盘不支持元数据上传")
|
||||
return
|
||||
}
|
||||
// 以本地为准:网盘端已有同名但内容不同的旧元数据时,先删除旧文件再上传。
|
||||
// 115 的上传接口不保证同名覆盖,直接上传可能产生同名重复文件;删除失败则
|
||||
// 任务重试(旧文件 ID 失效的场景会在下次同步后自动修复)。
|
||||
if task.RemoteRef != "" {
|
||||
open115, ok := provider.(cloud.OpenAPI115Provider)
|
||||
if !ok {
|
||||
finish(model.StrmTaskFailed, "该网盘不支持删除远端旧元数据")
|
||||
return
|
||||
}
|
||||
if err := open115.OpenClient().DeleteFiles(ctx, task.RemotePath, task.RemoteRef); err != nil {
|
||||
s.uploadTaskFailWithRetry(task, "删除网盘旧元数据失败:"+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
f, err := os.Open(task.LocalPath)
|
||||
if err != nil {
|
||||
s.uploadTaskFailWithRetry(task, "打开本地文件失败:"+err.Error())
|
||||
@@ -765,12 +859,15 @@ func (s *StrmService) wafCooldownLeft() time.Duration {
|
||||
}
|
||||
|
||||
// is115Blocked 判断错误是否来自 115 的风控/限流(WAF 405 拦截页或限流错误码)。
|
||||
// 覆盖两层文案:HTTP 层(doJSON 的"接口触发频控/安全拦截(HTTP 405)")与
|
||||
// 业务错误码层(OpenAPIError 的"115 接口错误(406/770004)")。
|
||||
func is115Blocked(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "115 接口返回 http 405") ||
|
||||
strings.Contains(msg, "115 接口触发频控/安全拦截") ||
|
||||
strings.Contains(msg, "访问被阻断") ||
|
||||
strings.Contains(msg, "request has been blocked") ||
|
||||
strings.Contains(msg, "115 接口错误(770004") ||
|
||||
|
||||
@@ -3,11 +3,18 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/truewhile/MeBox/internal/model"
|
||||
"github.com/truewhile/MeBox/internal/repository"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud"
|
||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
@@ -18,6 +25,7 @@ func TestIs115Blocked(t *testing.T) {
|
||||
}{
|
||||
{errors.New("115 接口返回 HTTP 405:<!doctypehtml>...访问被阻断"), true},
|
||||
{errors.New("115 接口返回 HTTP 405"), true},
|
||||
{errors.New("115 接口触发频控/安全拦截(HTTP 405):阿里云 WAF 拦截页"), true},
|
||||
{errors.New("115 接口错误(770004):访问频率过高"), true},
|
||||
{errors.New("115 接口错误(406):达到访问上限"), true},
|
||||
{errors.New("下载失败:http 403"), false},
|
||||
@@ -164,3 +172,189 @@ func TestRequeueDownloadTask(t *testing.T) {
|
||||
t.Fatalf("task not requeued: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessUpload115DeletesStaleRemoteMetaFirst 验证 115 覆盖上传语义(以本地为准):
|
||||
// 任务携带网盘旧文件 ID 时,必须先调用 /open/ufile/delete 删除旧元数据再上传本地文件,
|
||||
// 避免 115 出现同名重复文件;删除请求应携带 file_ids 与父目录 cid。
|
||||
func TestProcessUpload115DeletesStaleRemoteMetaFirst(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
localFile := filepath.Join(localDir, "movie.nfo")
|
||||
if err := os.WriteFile(localFile, []byte("local-nfo-data"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
acct := &model.StrmAccount{Name: "fake115", Provider: "cloud115", Config: "{}", Enabled: true}
|
||||
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var calls []string
|
||||
deleteForm := map[string]string{}
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch r.URL.Path {
|
||||
case "/open/ufile/delete":
|
||||
calls = append(calls, "delete")
|
||||
deleteForm["file_ids"] = r.FormValue("file_ids")
|
||||
deleteForm["parent_id"] = r.FormValue("parent_id")
|
||||
w.Write([]byte(`{"state":true,"data":[]}`))
|
||||
case "/open/upload/init":
|
||||
calls = append(calls, "upload")
|
||||
// 返回秒传成功,跳过 OSS 真实上传
|
||||
w.Write([]byte(`{"state":true,"data":{"status":2,"file_id":"new-1","pick_code":"new-pc-1","callback":null}}`))
|
||||
default:
|
||||
t.Errorf("unexpected 115 api path %s", r.URL.Path)
|
||||
w.Write([]byte(`{"state":false,"message":"unexpected path"}`))
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
oldPro := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = api.URL
|
||||
defer func() { cloud115.ProAPIBase = oldPro }()
|
||||
|
||||
task := &model.StrmUploadTask{
|
||||
Base: model.Base{ID: "up-del-1"},
|
||||
SyncPathID: "p1",
|
||||
AccountID: acct.ID,
|
||||
Provider: model.StrmProvider115,
|
||||
FileName: "movie.nfo",
|
||||
LocalPath: localFile,
|
||||
RemotePath: "777",
|
||||
RemoteRef: "old-file-1",
|
||||
Status: model.StrmTaskRunning,
|
||||
}
|
||||
svc.processUpload115(context.Background(), task)
|
||||
|
||||
if task.Status != model.StrmTaskDone {
|
||||
t.Fatalf("upload task should succeed, status = %s, error = %s", task.Status, task.Error)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(calls) != 2 || calls[0] != "delete" || calls[1] != "upload" {
|
||||
t.Fatalf("expected delete before upload, got calls = %v", calls)
|
||||
}
|
||||
if deleteForm["file_ids"] != "old-file-1" {
|
||||
t.Fatalf("delete file_ids = %q, want old-file-1", deleteForm["file_ids"])
|
||||
}
|
||||
if deleteForm["parent_id"] != "777" {
|
||||
t.Fatalf("delete parent_id = %q, want 777", deleteForm["parent_id"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestBatchResolve115Links 验证下载队列的批量换链:同账号多个 115 任务的
|
||||
// pickcode 合并为一次 downurl 请求(官方接口支持逗号分隔多 pick_code),
|
||||
// 重复引用去重、非 115 任务不参与、直链携带绑定 UA。
|
||||
func TestBatchResolve115Links(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
|
||||
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var requests []string
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/open/ufile/downurl":
|
||||
_ = r.ParseForm()
|
||||
mu.Lock()
|
||||
requests = append(requests, r.PostFormValue("pick_code"))
|
||||
mu.Unlock()
|
||||
w.Write([]byte(`{"state":true,"data":{
|
||||
"111":{"pick_code":"q-pc-a","url":{"url":"http://cdn/a.mkv"}},
|
||||
"222":{"pick_code":"q-pc-b","url":{"url":"http://cdn/b.jpg"}}}}`))
|
||||
default:
|
||||
t.Errorf("unexpected 115 api path %s", r.URL.Path)
|
||||
w.Write([]byte(`{"state":false,"message":"unexpected path"}`))
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
oldPro := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = api.URL
|
||||
defer func() { cloud115.ProAPIBase = oldPro }()
|
||||
defer cloud115.ClearDownloadURLCache("q-pc-a")
|
||||
defer cloud115.ClearDownloadURLCache("q-pc-b")
|
||||
|
||||
tasks := []*model.StrmDownloadTask{
|
||||
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: "q-pc-a"},
|
||||
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: "q-pc-a"}, // 重复引用
|
||||
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: "q-pc-b"},
|
||||
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProviderOpenList, RemoteRef: "ol-ref"},
|
||||
{SyncPathID: "p1", AccountID: acct.ID, Provider: model.StrmProvider115, RemoteRef: " "}, // 空引用
|
||||
}
|
||||
resolved := svc.batchResolve115Links(context.Background(), tasks)
|
||||
|
||||
if got := resolved[dlResolveKey(acct.ID, "q-pc-a")]; got == nil || got.URL != "http://cdn/a.mkv" {
|
||||
t.Fatalf("missing/bad link for q-pc-a: %+v", resolved)
|
||||
}
|
||||
if got := resolved[dlResolveKey(acct.ID, "q-pc-b")]; got == nil || got.URL != "http://cdn/b.jpg" {
|
||||
t.Fatalf("missing/bad link for q-pc-b: %+v", resolved)
|
||||
}
|
||||
if got := resolved[dlResolveKey(acct.ID, "q-pc-a")].Headers["User-Agent"]; got != cloud115.DefaultUA {
|
||||
t.Fatalf("link UA = %q, want default bound UA", got)
|
||||
}
|
||||
if _, ok := resolved[dlResolveKey(acct.ID, "ol-ref")]; ok {
|
||||
t.Fatal("non-115 task should not be batch resolved")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(requests) != 1 || requests[0] != "q-pc-a,q-pc-b" {
|
||||
t.Fatalf("expected single batched downurl request, got %v", requests)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcessDownloadTaskUsesPreResolvedLink 验证任务执行时优先使用批量换链
|
||||
// 预取的直链:downurl 接口保持失败,若任务仍走逐个换链则必然失败。
|
||||
func TestProcessDownloadTaskUsesPreResolvedLink(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
acct := &model.StrmAccount{Name: "fake115", Provider: model.StrmProvider115, Config: "{}", Enabled: true}
|
||||
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer api.Close()
|
||||
oldPro := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = api.URL
|
||||
defer func() { cloud115.ProAPIBase = oldPro }()
|
||||
|
||||
contentSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("User-Agent") != cloud115.DefaultUA {
|
||||
t.Errorf("download UA = %q, want bound default UA", r.Header.Get("User-Agent"))
|
||||
}
|
||||
_, _ = w.Write([]byte("nfo-content"))
|
||||
}))
|
||||
defer contentSrv.Close()
|
||||
|
||||
task := &model.StrmDownloadTask{
|
||||
SyncPathID: "p1",
|
||||
AccountID: acct.ID,
|
||||
Provider: model.StrmProvider115,
|
||||
FileName: "movie.nfo",
|
||||
RemoteRef: "q-pc-pre",
|
||||
LocalPath: filepath.Join(localDir, "movie.nfo"),
|
||||
Status: model.StrmTaskRunning,
|
||||
}
|
||||
if err := svc.repo.StrmDownload.Create(context.Background(), task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resolved := map[string]*cloud.DirectLink{
|
||||
dlResolveKey(acct.ID, "q-pc-pre"): {URL: contentSrv.URL, Headers: map[string]string{"User-Agent": cloud115.DefaultUA}},
|
||||
}
|
||||
svc.processDownloadTask(context.Background(), task, resolved)
|
||||
|
||||
if task.Status != model.StrmTaskDone {
|
||||
t.Fatalf("task should be done via pre-resolved link, status = %s, error = %s", task.Status, task.Error)
|
||||
}
|
||||
data, err := os.ReadFile(task.LocalPath)
|
||||
if err != nil || string(data) != "nfo-content" {
|
||||
t.Fatalf("downloaded file mismatch: err = %v, data = %q", err, string(data))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,7 +67,7 @@ var StrmSettingDefs = map[string]struct {
|
||||
StrmSettingMinVideoSizeMB: {Default: "0", Label: "最小视频大小(MB)", Kind: "number", Help: "小于该大小的视频文件不生成 STRM,0 表示不限"},
|
||||
StrmSettingAddPath: {Default: "1", Label: "STRM 链接 path 参数", Kind: "choice", Choices: []string{"1", "2", "3"}, Help: "1=附带完整远端路径 2=仅文件名 3=不带 path"},
|
||||
StrmSettingDownloadMeta: {Default: "true", Label: "下载元数据", Kind: "bool", Help: "同步时把远端 nfo/图片/字幕下载到本地输出目录"},
|
||||
StrmSettingUploadMeta: {Default: "false", Label: "上传元数据", Kind: "bool", Help: "同步时把本地元数据上传到远端(需网盘支持写入)"},
|
||||
StrmSettingUploadMeta: {Default: "false", Label: "上传元数据", Kind: "bool", Help: "同步时把本地元数据上传到远端;本地与网盘元数据不同时以本地为准覆盖(需网盘支持写入)"},
|
||||
StrmSettingDeleteDir: {Default: "false", Label: "清理空目录", Kind: "bool", Help: "清理远端已删除的多余 .strm/元数据后,删除空目录"},
|
||||
Strm115RelayKeySetting: {Default: "", Label: "115 中继授权共享密钥", Kind: "text", Help: "QMediaSync/MQFamily 中继授权的共享 AES 密钥(OAUTH_RELAY_ENCRYPTION_KEY);不配置则中继授权不可用"},
|
||||
StrmSettingDownloadThreads: {Default: "6", Label: "下载队列线程数", Kind: "number", Help: "OpenList/CloudDrive2 元数据下载并发数(115 独立限速为 3)"},
|
||||
|
||||
+189
-35
@@ -41,7 +41,9 @@ type strmSyncState struct {
|
||||
lastProgressFlush time.Time // 上次进度落库时间
|
||||
seenVideo map[string]bool // "v:"+去掉扩展名的相对路径 → 远端存在该视频
|
||||
seenMeta map[string]bool // "m:"+相对路径 → 远端存在该元数据
|
||||
remoteMeta map[string]int64 // 远端元数据大小(上传比对用)
|
||||
remoteMeta map[string]int64 // 远端元数据大小(上传比对用)
|
||||
remoteMetaRef map[string]string // "m:"+相对路径 → 远端元数据文件引用(115 文件 ID,覆盖上传前删除旧文件用)
|
||||
remoteMetaSha1 map[string]string // "m:"+相对路径 → 远端元数据内容 SHA1(115 列表返回;上传/下载精确比对用,其他网盘为空)
|
||||
seenMetaTarget map[string]cloud.FileEntry
|
||||
seenVideoTarget map[string]cloud.FileEntry
|
||||
activeDownloadPaths map[string]bool // 本地已在排队/进行的下载任务路径(内存去重)
|
||||
@@ -50,6 +52,7 @@ type strmSyncState struct {
|
||||
pendingUploads []*model.StrmUploadTask
|
||||
dirCache sync.Map // dirID (string) -> relativePath (string)
|
||||
dirPathToID map[string]string // relativePath (string) -> dirID(115 上传父目录寻址用,walk 后构建)
|
||||
dirCacheDirty map[string]string // 待批量落库的目录缓存(dirID → 相对路径),避免逐目录单条 upsert
|
||||
|
||||
scanIncomplete atomic.Bool // 远端目录树/文件列表本次扫描不完整 → 禁止增量 prune 误删本地文件
|
||||
}
|
||||
@@ -201,6 +204,8 @@ func (s *StrmService) runSync(ctx context.Context, p *model.StrmSyncPath, rec *m
|
||||
seenVideo: map[string]bool{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
remoteMetaSha1: map[string]string{},
|
||||
seenMetaTarget: map[string]cloud.FileEntry{},
|
||||
seenVideoTarget: map[string]cloud.FileEntry{},
|
||||
}
|
||||
@@ -331,6 +336,11 @@ func (st *strmSyncState) run() error {
|
||||
// 多个目录列表请求的网络往返彼此重叠,大幅缩短大目录树同步耗时。
|
||||
const strmScanWorkers = 8
|
||||
|
||||
// strmProcessWorkers 115 平铺拉取后本地文件分类处理(生成 strm/元数据入队)
|
||||
// 的 worker 数。本地磁盘 I/O 是大库同步的尾部瓶颈,输出目录在网络挂载上尤甚;
|
||||
// processRemoteFile 的共享状态均由 st.mu 保护,可安全并发。
|
||||
const strmProcessWorkers = 8
|
||||
|
||||
// walkRemote 并发广度优先遍历网盘目录树。
|
||||
// 多个 worker 并行执行 List(受全局 115 令牌桶限流约束),子目录动态
|
||||
// 入队;任一目录失败则取消其余 worker 并返回错误(与旧串行版语义一致)。
|
||||
@@ -509,8 +519,8 @@ func (st *strmSyncState) isMetaExt(ext string) bool {
|
||||
|
||||
// cleanDirRel 对 115 扁平化拉取的目录相对路径逐段套用目录级文件名清洗,
|
||||
// 确保与 walkRemote / joinLocalRel(sanitizeRelativePath)使用同一套清洗规则。
|
||||
// 若不清洗,目录名中的冒号等非法字符会直达 rel,而 seenVideo/seenMeta 的 key
|
||||
// 与磁盘实际路径不一致,导致 pruneLocal 误删已下载的 strm / 元数据。
|
||||
// 若不清洗,目录名中的冒号等非法字符会直达 rel,而 seenVideo/remoteMeta 的 key
|
||||
// 与磁盘实际路径不一致,导致 pruneLocal 误删已下载的 strm、上传误传或重复下载。
|
||||
// 空 rel(根目录)原样返回。
|
||||
func cleanDirRel(rel string) string {
|
||||
if rel == "" {
|
||||
@@ -530,6 +540,34 @@ func cleanDirRel(rel string) string {
|
||||
return strings.Join(out, "/")
|
||||
}
|
||||
|
||||
// deferDirCacheSave 暂存一条目录缓存写入,由 flushDirCacheSave 统一批量落库。
|
||||
// 首次全量同步可能有上万个目录,逐目录单条 upsert 会造成明显的 SQLite 写锁
|
||||
// 竞争;内存 dirCache(sync.Map)始终即时可用,落库仅服务于下次增量预加载。
|
||||
func (st *strmSyncState) deferDirCacheSave(dirID, relPath string) {
|
||||
st.mu.Lock()
|
||||
if st.dirCacheDirty == nil {
|
||||
st.dirCacheDirty = map[string]string{}
|
||||
}
|
||||
st.dirCacheDirty[dirID] = relPath
|
||||
st.mu.Unlock()
|
||||
}
|
||||
|
||||
// flushDirCacheSave 把暂存的目录缓存一次性批量落库;失败仅记日志(缓存缺失
|
||||
// 只影响下次增量的目录解析提速,正确性由"重新向 115 获取"兜底)。
|
||||
func (st *strmSyncState) flushDirCacheSave() {
|
||||
st.mu.Lock()
|
||||
dirty := st.dirCacheDirty
|
||||
st.dirCacheDirty = nil
|
||||
st.mu.Unlock()
|
||||
if len(dirty) == 0 {
|
||||
return
|
||||
}
|
||||
if err := st.s.repo.StrmDirCache.SetBatch(st.ctx, st.p.ID, dirty); err != nil {
|
||||
st.s.log.Warn("batch save strm dir cache failed",
|
||||
zap.Error(err), zap.Int("count", len(dirty)), zap.String("path_id", st.p.ID))
|
||||
}
|
||||
}
|
||||
|
||||
// walk115Flat 使用 115 开放平台扁平化分页批量拉取机制与目录拓扑缓存(参考 QMediaSync)。
|
||||
// 极大地降低 API 请求次数并支持毫秒级/秒级增量同步。
|
||||
func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
|
||||
@@ -721,7 +759,7 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
|
||||
// 解析相对路径
|
||||
relPath := cleanDirRel(detail.RelativePath(rootCID))
|
||||
st.dirCache.Store(pid, relPath)
|
||||
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, pid, relPath)
|
||||
st.deferDirCacheSave(pid, relPath)
|
||||
|
||||
// 顺便解析并缓存 detail.Paths 中包含的中间各层级目录
|
||||
for _, ancestor := range detail.Paths {
|
||||
@@ -742,7 +780,7 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
|
||||
}
|
||||
ancestorRel := cleanDirRel(subDetail.RelativePath(rootCID))
|
||||
st.dirCache.Store(ancestor.FileId, ancestorRel)
|
||||
_ = st.s.repo.StrmDirCache.Set(ctx, st.p.ID, ancestor.FileId, ancestorRel)
|
||||
st.deferDirCacheSave(ancestor.FileId, ancestorRel)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -757,6 +795,9 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
|
||||
}()
|
||||
}
|
||||
pwg.Wait()
|
||||
// 目录解析阶段结束即批量落库已解析的缓存:失败路径也保留部分成果,
|
||||
// 下次同步可少解析一批目录。
|
||||
st.flushDirCacheSave()
|
||||
if firstErr != nil {
|
||||
// 目录树解析失败会导致 rel 塌缩,若继续处理会让大量本地文件
|
||||
// 被错误判定为"云端不存在"而重复下载/上传,并可能误删本地文件。
|
||||
@@ -767,10 +808,57 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
|
||||
|
||||
st.updateSyncMessage(fmt.Sprintf("正在生成 STRM 与同步文件 (共 %d 个)...", len(allFiles)))
|
||||
|
||||
// 5. 分类处理所有文件
|
||||
// 5. 分类处理所有文件。本地磁盘 I/O(Stat/读内容比对/写盘)远慢于列表
|
||||
// 拉取,串行消化是大库同步的尾部瓶颈(输出目录在网络挂载上尤甚);
|
||||
// processRemoteFile 的共享状态均由 st.mu 保护(walkRemote 已并发调用),
|
||||
// 这里用有界 worker 池并行处理。rel 构建依赖 dirCache 且需在父目录缺失
|
||||
// 时整体中止,保留在生产者侧串行完成。
|
||||
type strmFileTask struct {
|
||||
file cloud115.RemoteFile
|
||||
rel string
|
||||
}
|
||||
fileCh := make(chan strmFileTask)
|
||||
var (
|
||||
procWg sync.WaitGroup
|
||||
procErrMu sync.Mutex
|
||||
procErr error
|
||||
)
|
||||
for i := 0; i < strmProcessWorkers; i++ {
|
||||
procWg.Add(1)
|
||||
go func() {
|
||||
defer procWg.Done()
|
||||
if err := helper.Recover(st.s.log, "strm.sync.walk115.process", func() error {
|
||||
for t := range fileCh {
|
||||
// 中止(ctx 取消)后排空队列即可,不再产生任何写操作
|
||||
if ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
entry := cloud.FileEntry{
|
||||
ID: t.file.FileId,
|
||||
Name: t.file.FileName,
|
||||
IsDir: false,
|
||||
Size: t.file.FileSize,
|
||||
MTime: t.file.Utime,
|
||||
PickCode: t.file.PickCode,
|
||||
Sha1: t.file.Sha1,
|
||||
}
|
||||
st.processRemoteFile(entry, t.rel)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
procErrMu.Lock()
|
||||
if procErr == nil {
|
||||
procErr = err
|
||||
}
|
||||
procErrMu.Unlock()
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
}
|
||||
feed:
|
||||
for _, f := range allFiles {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
break
|
||||
}
|
||||
cleanName := cleanEntryName(f.FileName, false)
|
||||
var rel string
|
||||
@@ -783,21 +871,27 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
|
||||
// 父目录不在目录缓存,无法还原真实相对路径。若继续用塌缩后的
|
||||
// 根路径处理,该文件会被错误判定,导致重复下载/上传或误删本地文件。
|
||||
// 目录树不完整时宁可中止本次同步,也不带着损坏的 rel 继续执行。
|
||||
return fmt.Errorf("115: 文件 %s 的父目录未解析成功,目录树不完整,中止同步以防误删/误传", cleanName)
|
||||
procErrMu.Lock()
|
||||
if procErr == nil {
|
||||
procErr = fmt.Errorf("115: 文件 %s 的父目录未解析成功,目录树不完整,中止同步以防误删/误传", cleanName)
|
||||
}
|
||||
procErrMu.Unlock()
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
}
|
||||
entry := cloud.FileEntry{
|
||||
ID: f.FileId,
|
||||
Name: f.FileName,
|
||||
IsDir: false,
|
||||
Size: f.FileSize,
|
||||
MTime: f.Utime,
|
||||
PickCode: f.PickCode,
|
||||
select {
|
||||
case fileCh <- strmFileTask{file: f, rel: rel}:
|
||||
case <-ctx.Done():
|
||||
break feed
|
||||
}
|
||||
st.processRemoteFile(entry, rel)
|
||||
}
|
||||
|
||||
return nil
|
||||
close(fileCh)
|
||||
procWg.Wait()
|
||||
if procErr != nil {
|
||||
return procErr
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// handleVideo 生成/更新 .strm 文件。
|
||||
@@ -933,11 +1027,46 @@ func (st *strmSyncState) strmPathParam(rel string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// recordRemoteMeta 记录远端存在的元数据索引及文件大小。
|
||||
// usableSha1 归一化远端内容哈希:115 对目录/未完成文件可能返回空串或占位符 "-",均视为不可用。
|
||||
func usableSha1(sha string) string {
|
||||
sha = strings.TrimSpace(sha)
|
||||
if sha == "-" {
|
||||
return ""
|
||||
}
|
||||
return sha
|
||||
}
|
||||
|
||||
// localSha1Matches 计算本地文件 SHA1 并与远端哈希做大小写不敏感比对
|
||||
// (115 列表返回大写 hex,本地计算为小写)。读取/哈希失败按"视为同一文件"
|
||||
// 处理,避免瞬时读文件错误触发大规模重复上传/下载。
|
||||
func (st *strmSyncState) localSha1Matches(path, remoteSha1 string) bool {
|
||||
local, err := cloud115.FileSHA1(path)
|
||||
if err != nil {
|
||||
st.s.log.Warn("strm 计算本地元数据 SHA1 失败,按同一文件处理",
|
||||
zap.String("path", path), zap.Error(err))
|
||||
return true
|
||||
}
|
||||
return strings.EqualFold(local, remoteSha1)
|
||||
}
|
||||
|
||||
// recordRemoteMeta 记录远端存在的元数据索引、文件大小、文件引用及内容 SHA1。
|
||||
func (st *strmSyncState) recordRemoteMeta(entry cloud.FileEntry, rel string) {
|
||||
st.mu.Lock()
|
||||
if st.remoteMeta == nil {
|
||||
st.remoteMeta = map[string]int64{}
|
||||
}
|
||||
if st.remoteMetaRef == nil {
|
||||
st.remoteMetaRef = map[string]string{}
|
||||
}
|
||||
if st.remoteMetaSha1 == nil {
|
||||
st.remoteMetaSha1 = map[string]string{}
|
||||
}
|
||||
st.seenMeta["m:"+rel] = true
|
||||
st.remoteMeta["m:"+rel] = entry.Size
|
||||
st.remoteMetaRef["m:"+rel] = entry.ID
|
||||
if sha := usableSha1(entry.Sha1); sha != "" {
|
||||
st.remoteMetaSha1["m:"+rel] = sha
|
||||
}
|
||||
st.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -971,7 +1100,8 @@ func (st *strmSyncState) flushPendingUploads() {
|
||||
}
|
||||
}
|
||||
|
||||
// handleMeta 元数据入下载队列(本地已存在且大小一致则跳过)。
|
||||
// handleMeta 元数据入下载队列。本地已存在时:开启上传元数据则一律跳过(以本地
|
||||
// 为准);否则大小与 SHA1(115 提供)均一致视为同一文件跳过,内容不同则下载覆盖。
|
||||
func (st *strmSyncState) handleMeta(entry cloud.FileEntry, rel, ext string) {
|
||||
st.recordRemoteMeta(entry, rel)
|
||||
|
||||
@@ -993,9 +1123,22 @@ func (st *strmSyncState) handleMeta(entry cloud.FileEntry, rel, ext string) {
|
||||
st.seenMetaTarget[target] = entry
|
||||
st.mu.Unlock()
|
||||
|
||||
if info, err := os.Stat(target); err == nil && info.Size() == entry.Size {
|
||||
st.touchProgress()
|
||||
return
|
||||
if info, err := os.Stat(target); err == nil {
|
||||
// 开启上传元数据时以本地为准:本地已存在的元数据不再用远端版本覆盖,
|
||||
// 与网盘版本的差异交给上传队列把本地文件推回网盘,避免下载/上传
|
||||
// 两个队列互相覆盖形成回环。
|
||||
if st.cfg.UploadMeta {
|
||||
st.touchProgress()
|
||||
return
|
||||
}
|
||||
// 同名同大小:115 提供远端 SHA1 时做内容级比对,网盘更新了同大小
|
||||
// 元数据也能被下载到本地;无哈希(其他网盘/列表未返回)或哈希一致
|
||||
// 视为同一文件跳过。
|
||||
remoteSha := usableSha1(entry.Sha1)
|
||||
if info.Size() == entry.Size && (remoteSha == "" || st.localSha1Matches(target, remoteSha)) {
|
||||
st.touchProgress()
|
||||
return
|
||||
}
|
||||
}
|
||||
st.mu.Lock()
|
||||
if st.activeDownloadPaths == nil {
|
||||
@@ -1135,6 +1278,8 @@ func (st *strmSyncState) walkLocalSource() error {
|
||||
}
|
||||
|
||||
// scanLocalMetaForUpload 扫描本地元数据,与远端比对后入上传队列。
|
||||
// 以本地为准:网盘端不存在、同名不同大小、或同名同大小但 SHA1 不同(115 提供
|
||||
// 远端哈希时做内容级比对)均入队覆盖上传;同名同大小同内容视为同一文件跳过。
|
||||
func (st *strmSyncState) scanLocalMetaForUpload() error {
|
||||
defer st.flushPendingUploads()
|
||||
if st.activeUploadPaths == nil {
|
||||
@@ -1174,12 +1319,22 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
|
||||
return nil
|
||||
}
|
||||
st.mu.Lock()
|
||||
_, exists := st.remoteMeta["m:"+rel]
|
||||
remoteSize, exists := st.remoteMeta["m:"+rel]
|
||||
remoteRef := st.remoteMetaRef["m:"+rel]
|
||||
remoteSha1 := st.remoteMetaSha1["m:"+rel]
|
||||
st.mu.Unlock()
|
||||
if exists {
|
||||
// 网盘端已存在该元数据文件,跳过上传
|
||||
return nil
|
||||
if exists && remoteSize == info.Size() {
|
||||
// 网盘端同名同大小:候选同一文件。115 提供远端 SHA1 时做内容级比对,
|
||||
// 识别"同大小不同内容"(如 nfo 改一个字符长度不变)避免漏传;
|
||||
// 无哈希(其他网盘/列表未返回)视为同一文件,保持大小比对兜底。
|
||||
if remoteSha1 == "" || st.localSha1Matches(path, remoteSha1) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// 入队上传(以本地为准):网盘端不存在;或同名但大小不同(必然内容不同);
|
||||
// 或同名同大小但 SHA1 不同(精确比对发现的同大小不同内容)。
|
||||
// 115 的上传接口不保证同名覆盖,任务携带远端旧文件 ID(RemoteRef),
|
||||
// 由上传端先删旧文件再上传;WebDAV/OpenList 的 PutFile 本身即覆盖上传。
|
||||
st.mu.Lock()
|
||||
if st.activeUploadPaths != nil && st.activeUploadPaths[path] {
|
||||
st.mu.Unlock()
|
||||
@@ -1200,6 +1355,9 @@ func (st *strmSyncState) scanLocalMetaForUpload() error {
|
||||
Size: info.Size(),
|
||||
Status: model.StrmTaskPending,
|
||||
}
|
||||
if exists && st.p.Provider == model.StrmProvider115 {
|
||||
task.RemoteRef = remoteRef
|
||||
}
|
||||
st.mu.Lock()
|
||||
st.pendingUploads = append(st.pendingUploads, task)
|
||||
shouldFlush := len(st.pendingUploads) >= 100
|
||||
@@ -1260,10 +1418,11 @@ func (st *strmSyncState) taskExists(kind, syncPathID, localPath string) bool {
|
||||
return count > 0
|
||||
}
|
||||
|
||||
// pruneLocal 清理本地多余 .strm 与元数据(远端已不存在),可选删除空目录。
|
||||
// pruneLocal 清理本地多余 .strm(远端已不存在的视频),可选删除空目录。
|
||||
// 元数据文件(nfo/图片/字幕等)一律保留:本地刮削结果不因网盘端缺失而被删除。
|
||||
func (st *strmSyncState) pruneLocal() error {
|
||||
// 增量同步保护:本次远端扫描不完整(目录详情解析失败 / 文件父路径降级)时,
|
||||
// seenVideo/seenMeta 覆盖不全,按"远端不存在"清理会误删刚下载或已存在的本地文件,
|
||||
// seenVideo 覆盖不全,按"远端不存在"清理会误删刚下载或已存在的本地 .strm,
|
||||
// 进而触发"下次增量重新下载"的循环。此时跳过清理,仅做进度落库。
|
||||
if st.syncType == model.StrmSyncTypeIncremental && st.scanIncomplete.Load() {
|
||||
st.s.log.Warn("strm 增量同步跳过清理:本次远端扫描不完整,prune 已禁用",
|
||||
@@ -1295,16 +1454,11 @@ func (st *strmSyncState) pruneLocal() error {
|
||||
rel = filepath.ToSlash(rel)
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
remove := false
|
||||
switch {
|
||||
case ext == ".strm":
|
||||
if ext == ".strm" {
|
||||
relSansExt := rel[:len(rel)-len(ext)]
|
||||
st.mu.Lock()
|
||||
remove = !st.seenVideo["v:"+relSansExt]
|
||||
st.mu.Unlock()
|
||||
case st.isMetaExt(ext) && st.cfg.DownloadMeta && !st.cfg.UploadMeta && st.p.Provider != model.StrmProviderLocal:
|
||||
st.mu.Lock()
|
||||
remove = !st.seenMeta["m:"+rel]
|
||||
st.mu.Unlock()
|
||||
}
|
||||
if remove {
|
||||
if err := os.Remove(path); err == nil {
|
||||
|
||||
@@ -344,14 +344,19 @@ func TestSanitizePathWithSpecialChars(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanLocalMetaForUpload 验证元数据上传比对逻辑:网盘已存在跳过,网盘不存在才入队上传。
|
||||
// TestScanLocalMetaForUpload 验证元数据上传比对逻辑:网盘同名同大小(同一文件)跳过,
|
||||
// 网盘不存在或同名不同大小(内容不同的元数据文件)以本地为准入队上传。
|
||||
func TestScanLocalMetaForUpload(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
|
||||
// 本地有 2 个元数据:poster.jpg 和 fanart.jpg
|
||||
// 本地有 3 个元数据:
|
||||
// poster.jpg — 网盘已有同名同大小 → 同一文件,跳过
|
||||
// fanart.jpg — 网盘没有 → 上传
|
||||
// tvshow.nfo — 网盘已有同名但大小不同(内容不同的元数据文件)→ 以本地为准覆盖上传
|
||||
writeFile(t, filepath.Join(localDir, "动漫", "poster.jpg"), "poster-data")
|
||||
writeFile(t, filepath.Join(localDir, "动漫", "fanart.jpg"), "fanart-data")
|
||||
writeFile(t, filepath.Join(localDir, "动漫", "tvshow.nfo"), "local-nfo-data")
|
||||
|
||||
p := &model.StrmSyncPath{
|
||||
Base: model.Base{ID: "test-path-upload"},
|
||||
@@ -363,35 +368,47 @@ func TestScanLocalMetaForUpload(t *testing.T) {
|
||||
}
|
||||
|
||||
st := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
}
|
||||
|
||||
// 模拟远端已存在 poster.jpg
|
||||
st.remoteMeta["m:动漫/poster.jpg"] = 1000
|
||||
// 模拟远端已存在 poster.jpg(与本地同一文件)和 tvshow.nfo(与本地不同)
|
||||
st.remoteMeta["m:动漫/poster.jpg"] = int64(len("poster-data"))
|
||||
st.remoteMeta["m:动漫/tvshow.nfo"] = 999
|
||||
|
||||
if err := st.scanLocalMetaForUpload(); err != nil {
|
||||
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
|
||||
}
|
||||
|
||||
// 此时应该只有 fanart.jpg 入队上传,poster.jpg 被跳过
|
||||
// fanart.jpg(网盘缺失)与 tvshow.nfo(网盘版本不同)入队,poster.jpg 跳过
|
||||
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("expected 1 upload task (fanart.jpg), got %d", len(tasks))
|
||||
if len(tasks) != 2 {
|
||||
t.Fatalf("expected 2 upload tasks (fanart.jpg, tvshow.nfo), got %d", len(tasks))
|
||||
}
|
||||
if tasks[0].FileName != "fanart.jpg" {
|
||||
t.Errorf("expected upload task for fanart.jpg, got %s", tasks[0].FileName)
|
||||
got := map[string]model.StrmUploadTask{}
|
||||
for _, task := range tasks {
|
||||
got[task.FileName] = task
|
||||
}
|
||||
if _, ok := got["fanart.jpg"]; !ok {
|
||||
t.Errorf("expected upload task for fanart.jpg, got %v", taskNames(tasks))
|
||||
}
|
||||
if _, ok := got["tvshow.nfo"]; !ok {
|
||||
t.Errorf("expected upload task for tvshow.nfo (local wins), got %v", taskNames(tasks))
|
||||
}
|
||||
if _, ok := got["poster.jpg"]; ok {
|
||||
t.Errorf("poster.jpg (same size on remote) should be skipped")
|
||||
}
|
||||
|
||||
// 再次扫描:fanart.jpg 已经在队列中,应自动去重,不重复入队
|
||||
// 再次扫描:已入队的文件自动去重,不重复入队
|
||||
if err := st.scanLocalMetaForUpload(); err != nil {
|
||||
t.Fatalf("second scanLocalMetaForUpload failed: %v", err)
|
||||
}
|
||||
@@ -399,8 +416,313 @@ func TestScanLocalMetaForUpload(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tasks) != 2 {
|
||||
t.Fatalf("expected still 2 upload tasks after dedup, got %d", len(tasks))
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanLocalMetaForUpload115CarriesRemoteRef 验证 115 覆盖上传时任务携带
|
||||
// 网盘旧文件 ID(供上传前删除旧文件,避免同名重复),网盘无同名文件时不携带。
|
||||
func TestScanLocalMetaForUpload115CarriesRemoteRef(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
|
||||
// movie.nfo 网盘已有同名但大小不同(需要删除旧文件后覆盖上传)
|
||||
writeFile(t, filepath.Join(localDir, "movie.nfo"), "local-nfo-data")
|
||||
// fresh.nfo 网盘没有(普通上传,不携带 ref)
|
||||
writeFile(t, filepath.Join(localDir, "fresh.nfo"), "fresh-nfo")
|
||||
|
||||
p := &model.StrmSyncPath{
|
||||
Base: model.Base{ID: "test-path-upload-115"},
|
||||
AccountID: "acct-1",
|
||||
Provider: model.StrmProvider115,
|
||||
RemotePath: "100",
|
||||
LocalPath: localDir,
|
||||
UploadMeta: true,
|
||||
}
|
||||
|
||||
st := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"jpg", "nfo"}},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
}
|
||||
st.remoteMeta["m:movie.nfo"] = 1
|
||||
st.remoteMetaRef["m:movie.nfo"] = "file-42"
|
||||
|
||||
if err := st.scanLocalMetaForUpload(); err != nil {
|
||||
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
|
||||
}
|
||||
|
||||
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tasks) != 2 {
|
||||
t.Fatalf("expected 2 upload tasks, got %d", len(tasks))
|
||||
}
|
||||
for _, task := range tasks {
|
||||
switch task.FileName {
|
||||
case "movie.nfo":
|
||||
if task.RemoteRef != "file-42" {
|
||||
t.Errorf("movie.nfo upload task should carry remote ref %q, got %q", "file-42", task.RemoteRef)
|
||||
}
|
||||
if task.RemotePath != "100" {
|
||||
t.Errorf("movie.nfo upload task remote path = %q, want parent cid %q", task.RemotePath, "100")
|
||||
}
|
||||
case "fresh.nfo":
|
||||
if task.RemoteRef != "" {
|
||||
t.Errorf("fresh.nfo (not on remote) should not carry remote ref, got %q", task.RemoteRef)
|
||||
}
|
||||
default:
|
||||
t.Errorf("unexpected upload task %s", task.FileName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func taskNames(tasks []model.StrmUploadTask) []string {
|
||||
names := make([]string, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
names = append(names, task.FileName)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// TestPruneLocalKeepsLocalMeta 验证清理规则:远端已删除的视频 .strm 仍会被清理,
|
||||
// 但本地元数据一律保留(即使开启"下载元数据"且未开启"上传元数据"、网盘端没有
|
||||
// 该元数据,也不再删除本地刮削好的 nfo/图片/字幕)。
|
||||
func TestPruneLocalKeepsLocalMeta(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
|
||||
// 阿凡达.strm(对应视频已被网盘删除 → 应清理)+ 阿凡达.nfo(网盘没有 → 保留)
|
||||
writeFile(t, filepath.Join(localDir, "电影", "阿凡达.strm"), "http://test.local:8096/x")
|
||||
writeFile(t, filepath.Join(localDir, "电影", "阿凡达.nfo"), "<local scraped meta/>")
|
||||
writeFile(t, filepath.Join(localDir, "电影", "poster.jpg"), "local-poster")
|
||||
|
||||
p := &model.StrmSyncPath{
|
||||
Base: model.Base{ID: "prune-meta-path"},
|
||||
Provider: model.StrmProvider115,
|
||||
RemotePath: "0",
|
||||
LocalPath: localDir,
|
||||
DownloadMeta: true,
|
||||
UploadMeta: false,
|
||||
}
|
||||
|
||||
st := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: false, MetaExt: []string{"nfo", "jpg"}},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
syncType: model.StrmSyncTypeFull,
|
||||
seenVideo: map[string]bool{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
}
|
||||
// 本次远端扫描既没有看到视频,也没有看到任何元数据
|
||||
if err := st.pruneLocal(); err != nil {
|
||||
t.Fatalf("pruneLocal failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(localDir, "电影", "阿凡达.strm")); !os.IsNotExist(err) {
|
||||
t.Fatalf("orphan .strm should be pruned, stat err = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(localDir, "电影", "阿凡达.nfo")); err != nil {
|
||||
t.Fatalf("local meta must be kept even when missing on remote: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(localDir, "电影", "poster.jpg")); err != nil {
|
||||
t.Fatalf("local poster must be kept even when missing on remote: %v", err)
|
||||
}
|
||||
if st.rec.Pruned != 1 {
|
||||
t.Fatalf("expected 1 pruned (strm only), got %d", st.rec.Pruned)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMetaKeepsLocalWhenUploadEnabled 验证下载侧规则:开启"上传元数据"时
|
||||
// 以本地为准——本地已存在的元数据(即使与网盘大小不同)不再入下载队列被网盘版本
|
||||
// 覆盖;本地不存在的元数据仍正常入队下载。
|
||||
func TestHandleMetaKeepsLocalWhenUploadEnabled(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
|
||||
// 本地已有 test.nfo(大小 50,与网盘版本大小 100 不同)
|
||||
writeFile(t, filepath.Join(localDir, "test.nfo"), strings.Repeat("L", 50))
|
||||
|
||||
p := &model.StrmSyncPath{
|
||||
Base: model.Base{ID: "meta-local-wins-path"},
|
||||
Provider: model.StrmProvider115,
|
||||
RemotePath: "0",
|
||||
LocalPath: localDir,
|
||||
DownloadMeta: true,
|
||||
UploadMeta: true,
|
||||
}
|
||||
|
||||
st := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: true, MetaExt: []string{"nfo"}},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
seenMetaTarget: map[string]cloud.FileEntry{},
|
||||
}
|
||||
|
||||
// 网盘版本与本地不同:本地存在 → 不入下载队列(以本地为准)
|
||||
entry := cloud.FileEntry{ID: "r1", Name: "test.nfo", Size: 100, PickCode: "pc1"}
|
||||
st.handleMeta(entry, "test.nfo", ".nfo")
|
||||
// 网盘独有:本地不存在 → 正常入下载队列
|
||||
missing := cloud.FileEntry{ID: "r2", Name: "absent.nfo", Size: 200, PickCode: "pc2"}
|
||||
st.handleMeta(missing, "absent.nfo", ".nfo")
|
||||
st.flushPendingDownloads()
|
||||
|
||||
tasks, _, err := svc.repo.StrmDownload.List(context.Background(), "", 1, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("expected still 1 upload task after dedup, got %d", len(tasks))
|
||||
t.Fatalf("expected only 1 download task (absent.nfo), got %d", len(tasks))
|
||||
}
|
||||
if tasks[0].FileName != "absent.nfo" {
|
||||
t.Fatalf("expected download task for absent.nfo, got %s", tasks[0].FileName)
|
||||
}
|
||||
if st.rec.NewMeta != 1 {
|
||||
t.Fatalf("expected NewMeta = 1, got %d", st.rec.NewMeta)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanLocalMetaForUploadSha1Identity 验证 115 远端 SHA1 可用时按内容精确比对:
|
||||
// 同名同大小同内容(大小写不敏感)跳过;同名同大小不同内容以本地为准覆盖上传,
|
||||
// 且任务携带网盘旧文件 ID。纯大小比对无法识别"同大小不同内容"(如 nfo 改一个字符)。
|
||||
func TestScanLocalMetaForUploadSha1Identity(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
|
||||
// same.nfo / diff.nfo 本地与网盘大小均相同:
|
||||
// same.nfo 网盘内容与本地一致(SHA1 相同)→ 同一文件,跳过;
|
||||
// diff.nfo 网盘上是另一个同大小文件(SHA1 不同)→ 以本地为准覆盖上传。
|
||||
writeFile(t, filepath.Join(localDir, "same.nfo"), "same-content")
|
||||
writeFile(t, filepath.Join(localDir, "diff.nfo"), "diff-content")
|
||||
otherFile := filepath.Join(localDir, "other.tmp")
|
||||
writeFile(t, otherFile, "other-content!")
|
||||
|
||||
sameSha, err := cloud115.FileSHA1(filepath.Join(localDir, "same.nfo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
otherSha, err := cloud115.FileSHA1(otherFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p := &model.StrmSyncPath{
|
||||
Base: model.Base{ID: "test-path-upload-sha1"},
|
||||
AccountID: "acct-1",
|
||||
Provider: model.StrmProvider115,
|
||||
RemotePath: "100",
|
||||
LocalPath: localDir,
|
||||
UploadMeta: true,
|
||||
}
|
||||
st := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
cfg: &strmPathConfig{UploadMeta: true, MetaExt: []string{"nfo"}},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
remoteMetaSha1: map[string]string{},
|
||||
}
|
||||
// 115 返回大写 SHA1,本地计算为小写:同时验证大小写不敏感比对
|
||||
st.remoteMeta["m:same.nfo"] = int64(len("same-content"))
|
||||
st.remoteMetaSha1["m:same.nfo"] = strings.ToUpper(sameSha)
|
||||
st.remoteMeta["m:diff.nfo"] = int64(len("diff-content"))
|
||||
st.remoteMetaSha1["m:diff.nfo"] = strings.ToUpper(otherSha)
|
||||
st.remoteMetaRef["m:diff.nfo"] = "old-diff-1"
|
||||
|
||||
if err := st.scanLocalMetaForUpload(); err != nil {
|
||||
t.Fatalf("scanLocalMetaForUpload failed: %v", err)
|
||||
}
|
||||
|
||||
tasks, _, err := svc.repo.StrmUpload.List(context.Background(), "", 1, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("expected 1 upload task (diff.nfo), got %d: %v", len(tasks), taskNames(tasks))
|
||||
}
|
||||
if tasks[0].FileName != "diff.nfo" {
|
||||
t.Fatalf("expected upload task for diff.nfo, got %s", tasks[0].FileName)
|
||||
}
|
||||
if tasks[0].RemoteRef != "old-diff-1" {
|
||||
t.Fatalf("diff.nfo task should carry remote ref %q, got %q", "old-diff-1", tasks[0].RemoteRef)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMetaSha1Identity 验证下载侧(未开启上传时镜像网盘元数据):
|
||||
// 同名同大小同 SHA1 跳过;网盘更新了同大小元数据(SHA1 不同)仍会下载;
|
||||
// 网盘未返回哈希时退回大小比对。
|
||||
func TestHandleMetaSha1Identity(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
|
||||
writeFile(t, filepath.Join(localDir, "same.nfo"), "identical-data") // 与网盘内容一致
|
||||
writeFile(t, filepath.Join(localDir, "stale.nfo"), "stale-content!!") // 网盘已更新为同大小新内容
|
||||
writeFile(t, filepath.Join(localDir, "nohash.nfo"), "nohash-content") // 网盘未返回 SHA1
|
||||
otherFile := filepath.Join(localDir, "other.tmp")
|
||||
writeFile(t, otherFile, "totally-newdata")
|
||||
|
||||
sameSha, err := cloud115.FileSHA1(filepath.Join(localDir, "same.nfo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
otherSha, err := cloud115.FileSHA1(otherFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p := &model.StrmSyncPath{
|
||||
Base: model.Base{ID: "meta-sha1-path"},
|
||||
Provider: model.StrmProvider115,
|
||||
RemotePath: "0",
|
||||
LocalPath: localDir,
|
||||
DownloadMeta: true,
|
||||
UploadMeta: false,
|
||||
}
|
||||
st := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
cfg: &strmPathConfig{DownloadMeta: true, UploadMeta: false, MetaExt: []string{"nfo"}},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
remoteMetaSha1: map[string]string{},
|
||||
seenMetaTarget: map[string]cloud.FileEntry{},
|
||||
}
|
||||
|
||||
st.handleMeta(cloud.FileEntry{ID: "r1", Name: "same.nfo", Size: int64(len("identical-data")), Sha1: strings.ToUpper(sameSha), PickCode: "pc1"}, "same.nfo", ".nfo")
|
||||
st.handleMeta(cloud.FileEntry{ID: "r2", Name: "stale.nfo", Size: int64(len("stale-content!!")), Sha1: strings.ToUpper(otherSha), PickCode: "pc2"}, "stale.nfo", ".nfo")
|
||||
st.handleMeta(cloud.FileEntry{ID: "r3", Name: "nohash.nfo", Size: int64(len("nohash-content")), PickCode: "pc3"}, "nohash.nfo", ".nfo")
|
||||
st.flushPendingDownloads()
|
||||
|
||||
tasks, _, err := svc.repo.StrmDownload.List(context.Background(), "", 1, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tasks) != 1 {
|
||||
t.Fatalf("expected only 1 download task (stale.nfo), got %d", len(tasks))
|
||||
}
|
||||
if tasks[0].FileName != "stale.nfo" {
|
||||
t.Fatalf("expected download task for stale.nfo, got %s", tasks[0].FileName)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -672,7 +994,7 @@ func TestWalk115FlatAbortsOnDirResolveFailure(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/open/ufile/files":
|
||||
w.Write([]byte(`{"state":true,"count":1,"data":[{"fid":"100","pid":"999","fc":1,"fn":"movie.mkv","pc":"pc1","upt":1700000000,"fs":1024}]}`))
|
||||
w.Write([]byte(`{"state":true,"count":1,"data":[{"fid":"100","pid":"999","fc":"1","fn":"movie.mkv","pc":"pc1","upt":1700000000,"fs":1024}]}`))
|
||||
case "/open/folder/get_info":
|
||||
w.Write([]byte(`{"state":false,"code":40140123,"message":"access_token 格式错误"}`))
|
||||
default:
|
||||
@@ -716,3 +1038,126 @@ func TestWalk115FlatAbortsOnDirResolveFailure(t *testing.T) {
|
||||
t.Fatalf("expected no .strm written after abort, got %d", strmCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWalk115FlatConcurrentProcessing 验证 115 平铺拉取后文件分类处理走并发
|
||||
// worker 池:同一父目录下多个视频的 strm 全部生成,且解析出的目录拓扑缓存
|
||||
// 通过 SetBatch 批量落库,供下次增量同步预加载(省掉重复的 get_info 调用)。
|
||||
func TestWalk115FlatConcurrentProcessing(t *testing.T) {
|
||||
svc := testStrmService(t)
|
||||
localDir := t.TempDir()
|
||||
acct := &model.StrmAccount{Name: "fake115", Provider: "cloud115", Config: "{}", Enabled: true}
|
||||
if err := svc.repo.StrmAccount.Create(context.Background(), acct); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := &model.StrmSyncPath{
|
||||
Base: model.Base{ID: "flat-path"},
|
||||
AccountID: acct.ID,
|
||||
Provider: model.StrmProvider115,
|
||||
RemotePath: "0",
|
||||
LocalPath: localDir,
|
||||
}
|
||||
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/open/ufile/files":
|
||||
w.Write([]byte(`{"state":true,"count":3,"data":[
|
||||
{"fid":"101","pid":"999","fc":"1","fn":"m1.mkv","pc":"pc1","upt":1700000001,"fs":1024},
|
||||
{"fid":"102","pid":"999","fc":"1","fn":"m2.mkv","pc":"pc2","upt":1700000002,"fs":2048},
|
||||
{"fid":"103","pid":"999","fc":"1","fn":"m3.mkv","pc":"pc3","upt":1700000003,"fs":4096}]}`))
|
||||
case "/open/folder/get_info":
|
||||
w.Write([]byte(`{"state":true,"data":{"file_id":"999","file_name":"Movies","file_category":"0",
|
||||
"paths":[{"file_id":"0","file_name":"根目录"},{"file_id":"999","file_name":"Movies"}]}}`))
|
||||
default:
|
||||
t.Errorf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
oldPro := cloud115.ProAPIBase
|
||||
cloud115.ProAPIBase = api.URL
|
||||
defer func() { cloud115.ProAPIBase = oldPro }()
|
||||
|
||||
oc := cloud115.NewOpenClient("app", "at", "rt")
|
||||
st := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
provider: cloud.NewOpenAPI115("app", "at", "rt"),
|
||||
cfg: &strmPathConfig{VideoExt: []string{"mkv"}, MetaExt: []string{"nfo"}, AddPath: 1},
|
||||
rec: &model.StrmSyncRecord{},
|
||||
syncType: model.StrmSyncTypeFull,
|
||||
dirCache: sync.Map{},
|
||||
seenVideo: map[string]bool{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
remoteMetaSha1: map[string]string{},
|
||||
seenMetaTarget: map[string]cloud.FileEntry{},
|
||||
seenVideoTarget: map[string]cloud.FileEntry{},
|
||||
}
|
||||
if err := st.walk115Flat(oc); err != nil {
|
||||
t.Fatalf("walk115Flat: %v", err)
|
||||
}
|
||||
|
||||
// 3 个视频的 strm 全部生成
|
||||
if st.rec.NewStrm != 3 {
|
||||
t.Fatalf("expected 3 strm created, got %d", st.rec.NewStrm)
|
||||
}
|
||||
for _, name := range []string{"m1.strm", "m2.strm", "m3.strm"} {
|
||||
if _, err := os.Stat(filepath.Join(localDir, "Movies", name)); err != nil {
|
||||
t.Fatalf("strm %s missing: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 目录拓扑缓存已批量落库
|
||||
rows, err := svc.repo.StrmDirCache.ListBySyncPathID(context.Background(), p.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].DirID != "999" || rows[0].Path != "Movies" {
|
||||
t.Fatalf("dir cache rows = %#v, want one row for dir 999 -> Movies", rows)
|
||||
}
|
||||
|
||||
// 增量同步复用目录缓存:不再发起 get_info 调用
|
||||
var infoCalls int
|
||||
api.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/open/ufile/files":
|
||||
w.Write([]byte(`{"state":true,"count":3,"data":[
|
||||
{"fid":"101","pid":"999","fc":"1","fn":"m1.mkv","pc":"pc1","upt":1700000001,"fs":1024},
|
||||
{"fid":"102","pid":"999","fc":"1","fn":"m2.mkv","pc":"pc2","upt":1700000002,"fs":2048},
|
||||
{"fid":"103","pid":"999","fc":"1","fn":"m3.mkv","pc":"pc3","upt":1700000003,"fs":4096}]}`))
|
||||
case "/open/folder/get_info":
|
||||
infoCalls++
|
||||
w.Write([]byte(`{"state":true,"data":{"file_id":"999","file_name":"Movies","file_category":"0","paths":[]}}`))
|
||||
default:
|
||||
t.Errorf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
})
|
||||
|
||||
st2 := &strmSyncState{
|
||||
s: svc,
|
||||
ctx: context.Background(),
|
||||
p: p,
|
||||
provider: cloud.NewOpenAPI115("app", "at", "rt"),
|
||||
cfg: st.cfg,
|
||||
rec: &model.StrmSyncRecord{},
|
||||
syncType: model.StrmSyncTypeIncremental,
|
||||
dirCache: sync.Map{},
|
||||
seenVideo: map[string]bool{},
|
||||
seenMeta: map[string]bool{},
|
||||
remoteMeta: map[string]int64{},
|
||||
remoteMetaRef: map[string]string{},
|
||||
remoteMetaSha1: map[string]string{},
|
||||
seenMetaTarget: map[string]cloud.FileEntry{},
|
||||
seenVideoTarget: map[string]cloud.FileEntry{},
|
||||
}
|
||||
if err := st2.walk115Flat(oc); err != nil {
|
||||
t.Fatalf("incremental walk115Flat: %v", err)
|
||||
}
|
||||
if infoCalls != 0 {
|
||||
t.Fatalf("incremental sync should reuse dir cache, got %d get_info calls", infoCalls)
|
||||
}
|
||||
if st2.rec.NewStrm != 0 || st2.rec.Skipped != 3 {
|
||||
t.Fatalf("incremental sync should skip all, new = %d skipped = %d", st2.rec.NewStrm, st2.rec.Skipped)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user