mirror of
https://github.com/truewhile/MeBox.git
synced 2026-10-06 21:36:37 +08:00
fix(sync): fallback to recursive traversal when 115 flat list hits deep-paging limit
115 API's flat list (search under the hood) enforces a hard limit of offset+limit <= 10000. When syncing huge directories (e.g. >10000 files), this silently truncates results, causing remote files to appear as missing locally, leading to infinite metadata re-upload loops and potential wrongful deletion of strm files. This patch auto-detects if the total file count >= 9500 and dynamically falls back to the standard recursive concurrent traversal (walkRemote).
This commit is contained in:
@@ -25,6 +25,8 @@ import (
|
|||||||
"github.com/truewhile/MeBox/internal/service/cloud115"
|
"github.com/truewhile/MeBox/internal/service/cloud115"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var errFallbackToWalkRemote = errors.New("fallback to walk remote")
|
||||||
|
|
||||||
// remoteMetaItem 记录远端存在的单个元数据文件副本信息(大小、文件ID、内容SHA1、修改时间)。
|
// remoteMetaItem 记录远端存在的单个元数据文件副本信息(大小、文件ID、内容SHA1、修改时间)。
|
||||||
type remoteMetaItem struct {
|
type remoteMetaItem struct {
|
||||||
ID string
|
ID string
|
||||||
@@ -352,8 +354,17 @@ func (st *strmSyncState) run() error {
|
|||||||
|
|
||||||
if st.provider != nil {
|
if st.provider != nil {
|
||||||
if open115, ok := st.provider.(cloud.OpenAPI115Provider); ok && st.p.Provider == model.StrmProvider115 {
|
if open115, ok := st.provider.(cloud.OpenAPI115Provider); ok && st.p.Provider == model.StrmProvider115 {
|
||||||
if err := st.walk115Flat(open115.OpenClient()); err != nil {
|
err := st.walk115Flat(open115.OpenClient())
|
||||||
return err
|
if err != nil {
|
||||||
|
if errors.Is(err, errFallbackToWalkRemote) {
|
||||||
|
st.s.log.Warn("115: 扁平列表文件数超限,自动降级为传统并发递归同步",
|
||||||
|
zap.String("path_id", st.p.ID))
|
||||||
|
if errWalk := st.walkRemote(); errWalk != nil {
|
||||||
|
return errWalk
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if err := st.walkRemote(); err != nil {
|
if err := st.walkRemote(); err != nil {
|
||||||
@@ -675,6 +686,13 @@ func (st *strmSyncState) walk115Flat(open115 *cloud115.OpenClient) error {
|
|||||||
return fmt.Errorf("115: 获取文件列表失败:%w", err)
|
return fmt.Errorf("115: 获取文件列表失败:%w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 115 的扁平化列表(搜索底层)对 offset + limit 有 10000 的最大深度限制。
|
||||||
|
// 当扁平模式下的总文件数 >= 9500 时,强行拒绝继续扁平拉取,而是抛出降级错误,
|
||||||
|
// 让外层回退到使用普通的按目录并发递归(walkRemote),以免截断导致后排文件被误删/重传。
|
||||||
|
if totalCount >= 9500 {
|
||||||
|
return errFallbackToWalkRemote
|
||||||
|
}
|
||||||
|
|
||||||
st.updateSyncMessage(fmt.Sprintf("正在拉取远端文件列表 (共 %d 个文件)...", totalCount))
|
st.updateSyncMessage(fmt.Sprintf("正在拉取远端文件列表 (共 %d 个文件)...", totalCount))
|
||||||
|
|
||||||
allFiles := make([]cloud115.RemoteFile, 0, totalCount)
|
allFiles := make([]cloud115.RemoteFile, 0, totalCount)
|
||||||
|
|||||||
Reference in New Issue
Block a user