fix(sync): fallback to recursive traversal when 115 flat list hits deep-paging limit

115 API's flat list (search under the hood) enforces a hard limit of offset+limit <= 10000.
When syncing huge directories (e.g. >10000 files), this silently truncates results,
causing remote files to appear as missing locally, leading to infinite metadata
re-upload loops and potential wrongful deletion of strm files.

This patch auto-detects if the total file count >= 9500 and dynamically
falls back to the standard recursive concurrent traversal (walkRemote).
This commit is contained in:
truewhile
2026-09-06 12:27:56 +08:00
parent 4173caac5d
commit fb84c62e9a
+20 -2
View File
@@ -25,6 +25,8 @@ import (
"github.com/truewhile/MeBox/internal/service/cloud115"
)
var errFallbackToWalkRemote = errors.New("fallback to walk remote")
// remoteMetaItem 记录远端存在的单个元数据文件副本信息(大小、文件ID、内容SHA1、修改时间)。
type remoteMetaItem struct {
ID string
@@ -352,8 +354,17 @@ func (st *strmSyncState) run() error {
if st.provider != nil {
if open115, ok := st.provider.(cloud.OpenAPI115Provider); ok && st.p.Provider == model.StrmProvider115 {
if err := st.walk115Flat(open115.OpenClient()); err != nil {
return err
err := st.walk115Flat(open115.OpenClient())
if err != nil {
if errors.Is(err, errFallbackToWalkRemote) {
st.s.log.Warn("115: 扁平列表文件数超限,自动降级为传统并发递归同步",
zap.String("path_id", st.p.ID))
if errWalk := st.walkRemote(); errWalk != nil {
return errWalk
}
} else {
return err
}
}
} else {
if err := st.walkRemote(); err != nil {
@@ -675,6 +686,13 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
return fmt.Errorf("115: 获取文件列表失败:%w", err)
}
// 115 的扁平化列表(搜索底层)对 offset + limit 有 10000 的最大深度限制。
// 当扁平模式下的总文件数 >= 9500 时,强行拒绝继续扁平拉取,而是抛出降级错误,
// 让外层回退到使用普通的按目录并发递归(walkRemote),以免截断导致后排文件被误删/重传。
if totalCount >= 9500 {
return errFallbackToWalkRemote
}
st.updateSyncMessage(fmt.Sprintf("正在拉取远端文件列表 (共 %d 个文件)...", totalCount))
allFiles := make([]cloud115.RemoteFile, 0, totalCount)