✨ Feature: add concurrency limit for download

README.md

@@ -20,7 +20,7 @@
 - 保留了PicGo的所有功能，兼容已有的PicGo插件系统，包括和typora、obsidian等的搭配
 - 相册中可同步删除云端图片
 - 支持管理所有图床，可以在线进行云端目录查看、文件搜索、批量上传、批量下载、删除文件等
-- 支持预览多种格式的文件，包括图片、视频、纯文本文件和markdown文件等，具体支持的格式请参考[支持的文件格式列表](https://github.com/Kuingsmile/PicList/supported_format.md)
+- 支持预览多种格式的文件，包括图片、视频、纯文本文件和markdown文件等，具体支持的格式请参考[支持的文件格式列表](https://github.com/Kuingsmile/PicList/blob/dev/supported_format.md)
 - 管理界面使用内置数据库缓存目录，加速目录加载速度
 - 对于私有存储桶等支持复制预签名链接进行分享
 - 优化了PicGo的界面，解锁了窗口大小限制，同时美化了部分界面布局

package.json

@@ -52,6 +52,7 @@
     "marked": "^4.2.12",
     "mime-types": "^2.1.35",
     "mitt": "^3.0.0",
+    "nodejs-file-downloader": "^4.10.6",
     "piclist": "^0.0.8",
     "pinia": "^2.0.29",
     "pinia-plugin-persistedstate": "^3.0.2",

src/main/manage/apis/aliyun.ts

@@ -1,7 +1,6 @@
 import axios from 'axios'
-import { hmacSha1Base64, getFileMimeType, gotDownload, formatError } from '../utils/common'
+import { hmacSha1Base64, getFileMimeType, formatError, NewDownloader, ConcurrencyPromisePool } from '../utils/common'
 import { ipcMain, IpcMainEvent } from 'electron'
-import fs from 'fs-extra'
 import { XMLParser } from 'fast-xml-parser'
 import OSS from 'ali-oss'
 import path from 'path'
@@ -548,19 +547,13 @@ class AliyunApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
-    // fileArray = [{
-    //   bucketName: string,
-    //   region: string,
-    //   key: string,
-    //  fileName: string
-    // }]
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName } = item
       const client = this.getNewCtx(region, bucketName)
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -575,8 +568,21 @@ class AliyunApi {
       const preSignedUrl = client.signatureUrl(key, {
         expires: 60 * 60 * 48
       })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
             }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error: any) => {
+      this.logger.error(formatError(error, { class: 'AliyunApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/github.ts

@@ -1,10 +1,10 @@
 import got from 'got'
 import { ManageLogger } from '../utils/logger'
-import { isImage } from '~/renderer/manage/utils/common'
+import { formatHttpProxy, isImage } from '~/renderer/manage/utils/common'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
-import { gotUpload, trimPath, gotDownload, getAgent, getOptions } from '../utils/common'
+import { gotUpload, trimPath, NewDownloader, getAgent, getOptions, ConcurrencyPromisePool, formatError } from '../utils/common'
 import UpDownTaskQueue,
 {
   commonTaskStatus
@@ -17,6 +17,7 @@ class GithubApi {
   username: string
   logger: ManageLogger
   proxy: any
+  proxyStr: string | undefined
   baseUrl = 'https://api.github.com'
   commonHeaders : IStringKeyMap
 
@@ -25,6 +26,7 @@ class GithubApi {
     this.token = token.startsWith('Bearer ') ? token : `Bearer ${token}`.trim()
     this.username = username
     this.proxy = proxy
+    this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined
     this.commonHeaders = {
       Authorization: this.token,
       Accept: 'application/vnd.github+json'
@@ -388,13 +390,13 @@ class GithubApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName: repo, customUrl: branch, key, fileName, githubPrivate, githubUrl } = item
       const id = `${repo}-${branch}-${key}-${fileName}`
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       if (instance.getDownloadTask(id)) {
         continue
       }
@@ -405,7 +407,7 @@ class GithubApi {
         sourceFileName: fileName,
         targetFilePath: savedFilePath
       })
-      let downloadUrl
+      let downloadUrl: string
       if (githubPrivate) {
         const preSignedUrl = await this.getPreSignedUrl({
           bucketName: repo,
@@ -418,17 +420,28 @@ class GithubApi {
       } else {
         downloadUrl = githubUrl
       }
-      gotDownload(
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(
           instance,
           downloadUrl,
-        fileStream,
           id,
           savedFilePath,
           this.logger,
-        undefined,
+          this.proxyStr
-        getAgent(this.proxy)
         )
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
             }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'GithubApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/imgur.ts

@@ -1,10 +1,10 @@
 import got from 'got'
 import ManageLogger from '../utils/logger'
-import { getAgent, getOptions, gotDownload, gotUpload, getFileMimeType } from '../utils/common'
+import { getAgent, getOptions, NewDownloader, gotUpload, getFileMimeType, ConcurrencyPromisePool, formatError } from '../utils/common'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
-import { isImage } from '~/renderer/manage/utils/common'
+import { formatHttpProxy, isImage } from '~/renderer/manage/utils/common'
 import path from 'path'
 import UpDownTaskQueue,
 {
@@ -18,6 +18,7 @@ class ImgurApi {
   accessToken: string
   proxy: any
   logger: ManageLogger
+  proxyStr: string | undefined
   tokenHeaders: any
   idHeaders: any
   baseUrl = 'https://api.imgur.com/3'
@@ -26,6 +27,7 @@ class ImgurApi {
     this.userName = userName
     this.accessToken = accessToken.startsWith('Bearer ') ? accessToken : `Bearer ${accessToken}`
     this.proxy = proxy
+    this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined
     this.logger = logger
     this.tokenHeaders = {
       Authorization: this.accessToken
@@ -227,13 +229,13 @@ class ImgurApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, githubUrl: url } = item
       const id = `${bucketName}-${region}-${key}-${fileName}`
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       if (instance.getDownloadTask(id)) {
         continue
       }
@@ -244,17 +246,28 @@ class ImgurApi {
         sourceFileName: fileName,
         targetFilePath: savedFilePath
       })
-      gotDownload(
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(
           instance,
           url,
-        fileStream,
           id,
           savedFilePath,
           this.logger,
-        undefined,
+          this.proxyStr
-        getAgent(this.proxy)
         )
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
             }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'ImgurApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/qiniu.ts

@@ -1,6 +1,5 @@
 import axios from 'axios'
-import { hmacSha1Base64, getFileMimeType, gotDownload, formatError } from '../utils/common'
+import { hmacSha1Base64, getFileMimeType, NewDownloader, formatError, ConcurrencyPromisePool } from '../utils/common'
-import fs from 'fs-extra'
 import qiniu from 'qiniu/index'
 import path from 'path'
 import { isImage } from '~/renderer/manage/utils/common'
@@ -627,12 +626,12 @@ class QiniuApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, customUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -645,8 +644,21 @@ class QiniuApi {
         targetFilePath: savedFilePath
       })
       const preSignedUrl = await this.getPreSignedUrl({ key, expires: 36000, customUrl })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
             }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'QiniuApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/s3plist.ts

@@ -18,8 +18,8 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
 import https from 'https'
 import http from 'http'
 import { ManageLogger } from '../utils/logger'
-import { formatEndpoint, formatError, getAgent, getFileMimeType, gotDownload } from '../utils/common'
+import { formatEndpoint, formatError, getAgent, getFileMimeType, NewDownloader, ConcurrencyPromisePool } from '../utils/common'
-import { isImage } from '@/manage/utils/common'
+import { isImage, formatHttpProxy } from '@/manage/utils/common'
 import { HttpsProxyAgent, HttpProxyAgent } from 'hpagent'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
@@ -49,6 +49,7 @@ class S3plistApi {
   baseOptions: S3plistApiOptions
   logger: ManageLogger
   agent: any
+  proxy: string | undefined
 
   constructor (
     accessKeyId: string,
@@ -73,6 +74,7 @@ class S3plistApi {
     } as S3plistApiOptions
     this.logger = logger
     this.agent = this.setAgent(proxy, sslEnabled)
+    this.proxy = formatHttpProxy(proxy, 'string') as string | undefined
   }
 
   setAgent (proxy: string | undefined, sslEnabled: boolean) : HttpProxyAgent | HttpsProxyAgent | undefined {
@@ -129,12 +131,22 @@ class S3plistApi {
     const options = Object.assign({}, this.baseOptions) as S3ClientConfig
     options.region = 'us-east-1'
     const result = [] as IStringKeyMap[]
+    const endpoint = options.endpoint as string || '' as string
     try {
       const client = new S3Client(options)
       const command = new ListBucketsCommand({})
       const data = await client.send(command)
       if (data.$metadata.httpStatusCode === 200) {
         if (data.Buckets) {
+          if (endpoint.indexOf('cloudflarestorage') !== -1) {
+            data.Buckets.forEach((bucket) => {
+              result.push({
+                Name: bucket.Name,
+                CreationDate: bucket.CreationDate,
+                Location: 'auto'
+              })
+            })
+          } else {
             for (let i = 0; i < data.Buckets.length; i++) {
               const bucket = data.Buckets[i]
               const bucketName = bucket.Name
@@ -146,7 +158,7 @@ class S3plistApi {
                 result.push({
                   Name: bucketName,
                   CreationDate: bucket.CreationDate,
-                Location: bucketConfig.LocationConstraint || 'us-east-1'
+                  Location: bucketConfig.LocationConstraint?.toLowerCase() || 'us-east-1'
                 })
               } else {
                 this.logParam(bucketConfig, 'getBucketList')
@@ -158,6 +170,7 @@ class S3plistApi {
               }
             }
           }
+        }
       } else {
         this.logParam(data, 'getBucketList')
       }
@@ -573,18 +586,12 @@ class S3plistApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
-    // fileArray = [{
-    //   bucketName: string,
-    //   region: string,
-    //   key: string,
-    //  fileName: string
-    // }]
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, customUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}-${savedFilePath}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -603,8 +610,21 @@ class S3plistApi {
         expires: 36000,
         customUrl
       })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger, this.proxy)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
             }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logParam(error, 'downloadBucketFile')
+    })
     return true
   }
 }

src/main/manage/apis/smms.ts

@@ -5,7 +5,7 @@ import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
 import FormData from 'form-data'
 import fs from 'fs-extra'
-import { getFileMimeType, gotUpload, gotDownload } from '../utils/common'
+import { getFileMimeType, gotUpload, NewDownloader, ConcurrencyPromisePool, formatError } from '../utils/common'
 import path from 'path'
 import UpDownTaskQueue, { commonTaskStatus } from '../datastore/upDownTaskQueue'
 import { ManageLogger } from '../utils/logger'
@@ -227,12 +227,12 @@ class SmmsApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, downloadUrl: preSignedUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -244,8 +244,21 @@ class SmmsApi {
         sourceFileName: fileName,
         targetFilePath: savedFilePath
       })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
             }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'SmmsApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/upyun.ts

@@ -1,6 +1,6 @@
 // @ts-ignore
 import Upyun from 'upyun'
-import { md5, hmacSha1Base64, getFileMimeType, gotDownload, gotUpload } from '../utils/common'
+import { md5, hmacSha1Base64, getFileMimeType, NewDownloader, gotUpload, ConcurrencyPromisePool, formatError } from '../utils/common'
 import { isImage } from '~/renderer/manage/utils/common'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
@@ -359,12 +359,12 @@ class UpyunApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, customUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -377,8 +377,21 @@ class UpyunApi {
         targetFilePath: savedFilePath
       })
       const preSignedUrl = `${customUrl}/${key}`
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
             }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'UpyunApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/webdavplist.ts

@@ -1,12 +1,19 @@
 import ManageLogger from '../utils/logger'
-import { createClient, WebDAVClient, FileStat } from 'webdav'
+import { createClient, WebDAVClient, FileStat, ProgressEvent } from 'webdav'
-import { formatError, formatEndpoint, getInnerAgent } from '../utils/common'
+import { formatError, formatEndpoint, getInnerAgent, NewDownloader, ConcurrencyPromisePool } from '../utils/common'
-import { isImage } from '@/manage/utils/common'
+import { formatHttpProxy, isImage } from '@/manage/utils/common'
 import http from 'http'
 import https from 'https'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
+import UpDownTaskQueue,
+{
+  uploadTaskSpecialStatus,
+  commonTaskStatus
+} from '../datastore/upDownTaskQueue'
+import fs from 'fs-extra'
+import path from 'path'
 
 class WebdavplistApi {
   endpoint: string
@@ -14,6 +21,7 @@ class WebdavplistApi {
   password: string
   sslEnabled: boolean
   proxy: string | undefined
+  proxyStr: string | undefined
   logger: ManageLogger
   agent: https.Agent | http.Agent
   ctx: WebDAVClient
@@ -24,6 +32,7 @@ class WebdavplistApi {
     this.password = password
     this.sslEnabled = sslEnabled
     this.proxy = proxy
+    this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined
     this.logger = logger
     this.agent = getInnerAgent(proxy, sslEnabled).agent
     this.ctx = createClient(
@@ -125,6 +134,173 @@ class WebdavplistApi {
     window.webContents.send('refreshFileTransferList', result)
     ipcMain.removeAllListeners('cancelLoadingFileList')
   }
+
+  async renameBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { oldKey, newKey } = configMap
+    let result = false
+    try {
+      await this.ctx.moveFile(oldKey, newKey)
+      result = true
+    } catch (error) {
+      this.logParam(error, 'renameBucketFile')
+    }
+    return result
+  }
+
+  async deleteBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { key } = configMap
+    let result = false
+    try {
+      await this.ctx.deleteFile(key)
+      result = true
+    } catch (error) {
+      this.logParam(error, 'deleteBucketFile')
+    }
+    return result
+  }
+
+  async deleteBucketFolder (configMap: IStringKeyMap): Promise<boolean> {
+    const { key } = configMap
+    let result = false
+    try {
+      await this.ctx.deleteFile(key)
+      result = true
+    } catch (error) {
+      this.logParam(error, 'deleteBucketFolder')
+    }
+    return result
+  }
+
+  async getPreSignedUrl (configMap: IStringKeyMap): Promise<string> {
+    const { key } = configMap
+    let result = ''
+    try {
+      const res = this.ctx.getFileDownloadLink(key)
+      result = res
+    } catch (error) {
+      this.logParam(error, 'getPreSignedUrl')
+    }
+    return result
+  }
+
+  async uploadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { fileArray } = configMap
+    const instance = UpDownTaskQueue.getInstance()
+    for (const item of fileArray) {
+      const { alias, bucketName, region, key, filePath, fileName } = item
+      const id = `${alias}-${bucketName}-${key}-${filePath}`
+      if (instance.getUploadTask(id)) {
+        continue
+      }
+      instance.addUploadTask({
+        id,
+        progress: 0,
+        status: commonTaskStatus.queuing,
+        sourceFileName: fileName,
+        sourceFilePath: filePath,
+        targetFilePath: key,
+        targetFileBucket: bucketName,
+        targetFileRegion: region,
+        noProgress: true
+      })
+      this.ctx.putFileContents(
+        key,
+        fs.createReadStream(filePath),
+        {
+          overwrite: true,
+          onUploadProgress: (progressEvent: ProgressEvent) => {
+            instance.updateUploadTask({
+              id,
+              progress: Math.floor((progressEvent.loaded / progressEvent.total) * 100),
+              status: uploadTaskSpecialStatus.uploading
+            })
+          }
+        }
+      ).then((res: boolean) => {
+        if (res) {
+          instance.updateUploadTask({
+            id,
+            progress: 100,
+            status: uploadTaskSpecialStatus.uploaded,
+            finishTime: new Date().toLocaleString()
+          })
+        } else {
+          instance.updateUploadTask({
+            id,
+            progress: 0,
+            status: commonTaskStatus.failed,
+            finishTime: new Date().toLocaleString()
+          })
+        }
+      }).catch((error: any) => {
+        this.logParam(error, 'uploadBucketFile')
+        instance.updateUploadTask({
+          id,
+          progress: 0,
+          status: commonTaskStatus.failed,
+          finishTime: new Date().toLocaleString()
+        })
+      })
+    }
+    return true
+  }
+
+  async createBucketFolder (configMap: IStringKeyMap): Promise<boolean> {
+    const { key } = configMap
+    let result = false
+    try {
+      await this.ctx.createDirectory(key, {
+        recursive: true
+      })
+      result = true
+    } catch (error) {
+      this.logParam(error, 'createBucketFolder')
+    }
+    return result
+  }
+
+  async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
+    const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
+    for (const item of fileArray) {
+      const { alias, bucketName, region, key, fileName } = item
+      const savedFilePath = path.join(downloadPath, fileName)
+      const id = `${alias}-${bucketName}-${region}-${key}`
+      if (instance.getDownloadTask(id)) {
+        continue
+      }
+      instance.addDownloadTask({
+        id,
+        progress: 0,
+        status: commonTaskStatus.queuing,
+        sourceFileName: fileName,
+        targetFilePath: savedFilePath
+      })
+      const preSignedUrl = await this.getPreSignedUrl({
+        key
+      })
+      const base64Str = Buffer.from(`${this.username}:${this.password}`).toString('base64')
+      const headers = {
+        Authorization: `Basic ${base64Str}`
+      }
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger, this.proxyStr, headers)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logParam(error, 'downloadBucketFile')
+    })
+    return true
+  }
 }
 
 export default WebdavplistApi

src/main/manage/manageApi.ts

@@ -387,6 +387,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'github':
       case 'imgur':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           const res = await client.deleteBucketFile(param!)
@@ -411,6 +412,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'upyun':
       case 'github':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.deleteBucketFolder(param!)
@@ -433,6 +435,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'qiniu':
       case 'upyun':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.renameBucketFile(param!)
@@ -458,6 +461,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'github':
       case 'imgur':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           const res = await client.downloadBucketFile(param!)
@@ -489,6 +493,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'upyun':
       case 'github':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.createBucketFolder(param!)
@@ -514,6 +519,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'github':
       case 'imgur':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.uploadBucketFile(param!)
@@ -536,6 +542,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'qiniu':
       case 'github':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.getPreSignedUrl(param!)

src/main/manage/utils/common.ts

@@ -18,6 +18,7 @@ import { formatHttpProxy, IHTTPProxy } from '@/manage/utils/common'
 import { HttpsProxyAgent, HttpProxyAgent } from 'hpagent'
 import http from 'http'
 import https from 'https'
+import Downloader from 'nodejs-file-downloader'
 
 export const getFSFile = async (
   filePath: string,
@@ -90,52 +91,57 @@ export const md5 = (str: string, code: 'hex' | 'base64'): string => crypto.creat
 
 export const hmacSha1Base64 = (secretKey: string, stringToSign: string) : string => crypto.createHmac('sha1', secretKey).update(Buffer.from(stringToSign, 'utf8')).digest('base64')
 
-export const gotDownload = async (
+export const NewDownloader = async (
   instance: UpDownTaskQueue,
   preSignedUrl: string,
-  fileStream: fs.WriteStream,
   id : string,
   savedFilePath: string,
   logger?: ManageLogger,
-  param?: any,
+  proxy?: string,
-  agent: any = {}
+  headers?: any
-) => {
+) : Promise<boolean> => {
-  got(
+  const options = {
-    preSignedUrl,
+    url: preSignedUrl,
-    {
+    directory: path.dirname(savedFilePath),
-      isStream: true,
+    fileName: path.basename(savedFilePath),
-      throwHttpErrors: false,
+    cloneFiles: false,
-      searchParams: param,
+    onProgress: (percentage: string) => {
-      agent: agent || {}
-    }
-  )
-    .on('downloadProgress', (progress: any) => {
       instance.updateDownloadTask({
         id,
-        progress: Math.floor(progress.percent * 100),
+        progress: Math.floor(Number(percentage)),
         status: downloadTaskSpecialStatus.downloading
       })
-    })
+    },
-    .pipe(fileStream)
+    maxAttempts: 3
-    .on('close', () => {
+  } as any
+  if (proxy) {
+    options.proxy = proxy
+  }
+  if (headers) {
+    options.headers = headers
+  }
+  const downloader = new Downloader(options)
+  try {
+    await downloader.download()
     instance.updateDownloadTask({
       id,
       progress: 100,
       status: downloadTaskSpecialStatus.downloaded,
       finishTime: new Date().toLocaleString()
     })
-    })
+    return true
-    .on('error', (err: any) => {
+  } catch (e: any) {
-      logger && logger.error(formatError(err, { method: 'gotDownload' }))
+    logger && logger.error(formatError(e, { method: 'NewDownloader' }))
     fs.remove(savedFilePath)
     instance.updateDownloadTask({
       id,
       progress: 0,
       status: commonTaskStatus.failed,
-        response: formatError(err, { method: 'gotDownload' }),
+      response: formatError(e, { method: 'NewDownloader' }),
       finishTime: new Date().toLocaleString()
     })
-    })
+    return false
+  }
 }
 
 export const gotUpload = async (
@@ -320,3 +326,45 @@ export const formatEndpoint = (endpoint: string, sslEnabled: boolean): string =>
     : sslEnabled
       ? endpoint.replace('http://', 'https://')
       : endpoint.replace('https://', 'http://')
+
+export class ConcurrencyPromisePool {
+  limit: number
+  queue: any[]
+  runningNum: number
+  results: any[]
+
+  constructor (limit: number) {
+    this.limit = limit
+    this.queue = []
+    this.runningNum = 0
+    this.results = []
+  }
+
+  all (promises: any[] = []) {
+    return new Promise((resolve, reject) => {
+      for (const promise of promises) {
+        this._run(promise, resolve, reject)
+      }
+    })
+  }
+
+  _run (promise: any, resolve: any, reject: any) {
+    if (this.runningNum >= this.limit) {
+      this.queue.push(promise)
+      return
+    }
+    this.runningNum += 1
+    promise()
+      .then((res: any) => {
+        this.results.push(res)
+        --this.runningNum
+        if (this.queue.length === 0 && this.runningNum === 0) {
+          return resolve(this.results)
+        }
+        if (this.queue.length > 0) {
+          this._run(this.queue.shift(), resolve, reject)
+        }
+      })
+      .catch(reject)
+  }
+}

src/renderer/manage/pages/bucketPage.vue

@@ -234,7 +234,7 @@ ea/*
         </el-breadcrumb-item>
         <template v-if="configMap.prefix !== '/'">
           <el-breadcrumb-item
-            v-for="(item, index) in configMap.prefix.slice(0, configMap.prefix.length - 1).split('/')"
+            v-for="(item, index) in configMap.prefix.replace(/\/$/g, '').split('/')"
             :key="index"
             style="flex-shrink: 0;font-size: 12px;color: #606266;font-family: Arial, Helvetica, sans-serif;cursor: pointer;"
             @click="handleBreadcrumbClick(index)"
@@ -562,9 +562,19 @@ https://www.baidu.com/img/bd_logo1.png"
         stretch
       >
         <el-tab-pane
-          :label="`上传中(${uploadingTaskList.length})`"
           name="uploading"
         >
+          <template #label>
+            <span>
+              上传中
+            </span>
+            <el-badge
+              v-if="uploadingTaskList.length"
+              :value="uploadingTaskList.length"
+              :max="9999"
+              type="primary"
+            />
+          </template>
           <el-button-group size="small">
             <el-button
               type="primary"
@@ -605,9 +615,19 @@ https://www.baidu.com/img/bd_logo1.png"
           </div>
         </el-tab-pane>
         <el-tab-pane
-          :label="`已完成(${uploadedTaskList.length})`"
           name="finished"
         >
+          <template #label>
+            <span>
+              成功
+            </span>
+            <el-badge
+              v-if="uploadedTaskList.filter(item => item.status === 'uploaded').length"
+              :value="uploadedTaskList.filter(item => item.status === 'uploaded').length"
+              :max="9999"
+              type="success"
+            />
+          </template>
           <el-button-group size="small">
             <el-button
               type="primary"
@@ -639,7 +659,60 @@ https://www.baidu.com/img/bd_logo1.png"
               <template #default="{ height, width }">
                 <el-table-v2
                   :columns="uploadedTaskColumns"
-                  :data="uploadedTaskList"
+                  :data="uploadedTaskList.filter(item => item.status === 'uploaded')"
+                  :width="width"
+                  :height="height"
+                />
+              </template>
+            </el-auto-resizer>
+          </div>
+        </el-tab-pane>
+        <el-tab-pane
+          name="failed"
+        >
+          <template #label>
+            <span>
+              失败
+            </span>
+            <el-badge
+              v-if="uploadedTaskList.filter(item => item.status !== 'uploaded').length"
+              :value="uploadedTaskList.filter(item => item.status !== 'uploaded').length"
+              :max="9999"
+              type="danger"
+            />
+          </template>
+          <el-button-group size="small">
+            <el-button
+              type="primary"
+              plain
+              :icon="Document"
+              @click="handelCopyUploadingTaskInfo"
+            >
+              复制上传任务信息
+            </el-button>
+            <el-button
+              type="primary"
+              plain
+              :icon="DeleteFilled"
+              @click="handelDeleteUploadedTask"
+            >
+              清空已完成任务
+            </el-button>
+            <el-button
+              type="primary"
+              plain
+              :icon="DeleteFilled"
+              @click="handelDeleteAllUploadedTask"
+            >
+              清空所有任务
+            </el-button>
+          </el-button-group>
+          <div style="height:500px;">
+            <el-auto-resizer>
+              <template #default="{ height, width }">
+                <el-table-v2
+                  :columns="uploadedTaskColumns"
+                  :data="uploadedTaskList.filter(item => item.status !== 'uploaded')"
                   :width="width"
                   :height="height"
                 />
@@ -661,9 +734,17 @@ https://www.baidu.com/img/bd_logo1.png"
         stretch
       >
         <el-tab-pane
-          :label="`下载中(${downloadingTaskList.length})`"
           name="downloading"
         >
+          <template #label>
+            <span>下载中</span>
+            <el-badge
+              v-if="downloadingTaskList.length"
+              :value="downloadingTaskList.length"
+              type="primary"
+              :max="9999"
+            />
+          </template>
           <el-button-group size="small">
             <el-button
               type="primary"
@@ -712,9 +793,17 @@ https://www.baidu.com/img/bd_logo1.png"
           </div>
         </el-tab-pane>
         <el-tab-pane
-          :label="`已完成(${downloadedTaskList.length})`"
           name="finished"
         >
+          <template #label>
+            <span>成功</span>
+            <el-badge
+              v-if="downloadedTaskList.filter(item => item.status === 'downloaded').length"
+              :value="downloadedTaskList.filter(item => item.status === 'downloaded').length"
+              :max="9999"
+              type="success"
+            />
+          </template>
           <el-button-group size="small">
             <el-button
               type="primary"
@@ -754,7 +843,66 @@ https://www.baidu.com/img/bd_logo1.png"
               <template #default="{ height, width }">
                 <el-table-v2
                   :columns="downloadedTaskColumns"
-                  :data="downloadedTaskList"
+                  :data="downloadedTaskList.filter(item => item.status === 'downloaded')"
+                  :width="width"
+                  :height="height"
+                />
+              </template>
+            </el-auto-resizer>
+          </div>
+        </el-tab-pane>
+        <el-tab-pane
+          name="failed"
+        >
+          <template #label>
+            <span>失败</span>
+            <el-badge
+              v-if="downloadedTaskList.filter(item => item.status !== 'downloaded').length"
+              :value="downloadedTaskList.filter(item => item.status !== 'downloaded').length"
+              :max="9999"
+              type="warning"
+            />
+          </template>
+          <el-button-group size="small">
+            <el-button
+              type="primary"
+              plain
+              :icon="Document"
+              @click="handelCopyDownloadingTaskInfo"
+            >
+              复制下载任务信息
+            </el-button>
+            <el-button
+              type="primary"
+              plain
+              :icon="DeleteFilled"
+              @click="handelDeleteDownloadedTask"
+            >
+              清空已完成任务
+            </el-button>
+            <el-button
+              type="primary"
+              plain
+              :icon="DeleteFilled"
+              @click="handelDeleteAllDownloadedTask"
+            >
+              清空所有任务
+            </el-button>
+            <el-button
+              type="primary"
+              plain
+              :icon="Folder"
+              @click="handelOpenDownloadedFolder"
+            >
+              打开下载目录
+            </el-button>
+          </el-button-group>
+          <div style="height:600px;">
+            <el-auto-resizer>
+              <template #default="{ height, width }">
+                <el-table-v2
+                  :columns="downloadedTaskColumns"
+                  :data="downloadedTaskList.filter(item => item.status !== 'downloaded')"
                   :width="width"
                   :height="height"
                 />
@@ -826,6 +974,7 @@ https://www.baidu.com/img/bd_logo1.png"
     >
       <video-player
         :src="videoFileUrl"
+        :headers="videoPlayerHeaders"
         controls
         :loop="true"
         :volume="0.6"
@@ -958,6 +1107,7 @@ const isShowTextFileDialog = ref(false)
 const textfileContent = ref('')
 const isShowVideoFileDialog = ref(false)
 const videoFileUrl = ref('')
+const videoPlayerHeaders = ref({})
 
 const showCustomUrlSelectList = computed(() => ['tcyun', 'aliyun', 'qiniu', 'github'].includes(currentPicBedName.value))
 
@@ -967,7 +1117,7 @@ const showCreateNewFolder = computed(() => ['tcyun', 'aliyun', 'qiniu', 'upyun',
 
 const showRenameFileIcon = computed(() => ['tcyun', 'aliyun', 'qiniu', 'upyun', 's3plist', 'webdavplist'].includes(currentPicBedName.value))
 
-const showPresignedUrl = computed(() => ['tcyun', 'aliyun', 'qiniu', 'github', 's3plist'].includes(currentPicBedName.value))
+const showPresignedUrl = computed(() => ['tcyun', 'aliyun', 'qiniu', 'github', 's3plist', 'webdavplist'].includes(currentPicBedName.value))
 
 const uploadingTaskList = computed(() => uploadTaskList.value.filter(item => ['uploading', 'queuing', 'paused'].includes(item.status)))
 
@@ -1191,6 +1341,7 @@ function uploadFiles () {
   }
   formateduploadPanelFilesList.forEach((item: any) => {
     param.fileArray.push({
+      alias: configMap.alias,
       bucketName: configMap.bucketName,
       region: configMap.bucketConfig.Location,
       key: item.key,
@@ -1254,6 +1405,12 @@ async function handleBreadcrumbClick (index: number) {
 }
 
 async function handleClickFile (item: any) {
+  const options = {} as any
+  if (currentPicBedName.value === 'webdavplist') {
+    options.headers = {
+      Authorization: `Basic ${Buffer.from(`${manageStore.config.picBed[configMap.alias].username}:${manageStore.config.picBed[configMap.alias].password}`).toString('base64')}`
+    }
+  }
   if (item.isImage) {
     previewedImage.value = item.url
     showImagePreview.value = true
@@ -1274,7 +1431,7 @@ async function handleClickFile (item: any) {
         type: 'success'
       })
       const fileUrl = item.url
-      const res = await axios.get(fileUrl)
+      const res = await axios.get(fileUrl, options)
       const content = res.data
       markDownContent.value = marked(content)
       isShowMarkDownDialog.value = true
@@ -1291,7 +1448,7 @@ async function handleClickFile (item: any) {
         type: 'success'
       })
       const fileUrl = item.url
-      const res = await axios.get(fileUrl)
+      const res = await axios.get(fileUrl, options)
       textfileContent.value = res.data
       isShowTextFileDialog.value = true
     } catch (error) {
@@ -1300,6 +1457,7 @@ async function handleClickFile (item: any) {
   } else if (videoExt.includes(path.extname(item.fileName).toLowerCase())) {
     videoFileUrl.value = item.url
     isShowVideoFileDialog.value = true
+    videoPlayerHeaders.value = options.headers
   }
 }
 
@@ -1763,11 +1921,13 @@ async function handelBatchDownload () {
   const defaultDownloadPath = await ipcRenderer.invoke('getDefaultDownloadFolder')
   const param = {
     downloadPath: manageStore.config.settings.downloadDir ?? defaultDownloadPath,
+    maxDownloadFileCount: manageStore.config.settings.maxDownloadFileCount ? manageStore.config.settings.maxDownloadFileCount : 5,
     fileArray: [] as any[]
   }
   selectedItems.forEach((item: any) => {
     if (!item.isDir) {
       param.fileArray.push({
+        alias: configMap.alias,
         bucketName: configMap.bucketName,
         region: configMap.bucketConfig.Location,
         key: item.key,
@@ -2431,11 +2591,12 @@ const uploadingTaskColumns: Column<any>[] = [
     width: 300,
     cellRenderer: ({ rowData: item }) => (
       <ElProgress
-        percentage={item.progress}
+        percentage={item.progress ? item.progress : 50}
         status="success"
         strokeWidth={20}
         textInside
         style="width: 100%;"
+        indeterminate={!!item.noProgress}
       />
     )
   }

src/renderer/manage/pages/manageSetting.vue

@@ -129,6 +129,31 @@
                 @change="handelIsForceCustomUrlHttpsChange"
               />
             </el-form-item>
+            <el-form-item>
+              <template #label>
+                <span
+                  style="position:absolute;left: 0;"
+                >最大同时下载文件数(1-9999)
+                  <el-tooltip
+                    effect="dark"
+                    content="腾讯云由于后端实现不同，该设置不生效"
+                    placement="right"
+                  >
+                    <el-icon>
+                      <InfoFilled />
+                    </el-icon>
+                  </el-tooltip>
+                </span>
+              </template>
+              <el-input-number
+                v-model="maxDownloadFileCount"
+                style="position:absolute;right: 0;"
+                placeholder="请输入最大同时下载文件数"
+                :min="1"
+                :max="9999"
+                :step="1"
+              />
+            </el-form-item>
             <el-form-item>
               <template #label>
                 <span
@@ -421,6 +446,14 @@ const customRenameFormat = reactive({
   value: '{filename}'
 })
 
+const maxDownloadFileCount = ref(5)
+
+watch(maxDownloadFileCount, (val) => {
+  saveConfig({
+    'settings.maxDownloadFileCount': val
+  })
+})
+
 watch(customRenameFormat, (val) => {
   saveConfig({
     'settings.customRenameFormat': val.value
@@ -468,6 +501,7 @@ async function initData () {
   form.isIgnoreCase = config.settings.isIgnoreCase ?? false
   form.isForceCustomUrlHttps = config.settings.isForceCustomUrlHttps ?? true
   PreSignedExpire.value = config.settings.PreSignedExpire ?? 14400
+  maxDownloadFileCount.value = config.settings.maxDownloadFileCount ?? 5
 }
 
 async function handleDownloadDirClick () {

yarn.lock

@@ -7611,7 +7611,7 @@ follow-redirects@^1.0.0:
   resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.6.tgz#8cfb281bbc035b3c067d6cd975b0f6ade6e855cd"
   integrity sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A==
 
-follow-redirects@^1.14.8, follow-redirects@^1.15.0:
+follow-redirects@^1.14.8, follow-redirects@^1.15.0, follow-redirects@^1.15.1:
   version "1.15.2"
   resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
   integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
@@ -10338,6 +10338,16 @@ node-releases@^2.0.6:
   resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.8.tgz#0f349cdc8fcfa39a92ac0be9bc48b7706292b9ae"
   integrity sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==
 
+nodejs-file-downloader@^4.10.6:
+  version "4.10.6"
+  resolved "https://registry.npmjs.org/nodejs-file-downloader/-/nodejs-file-downloader-4.10.6.tgz#a2b6e2c721de14968cf6f8d7d980aac448217ea3"
+  integrity sha512-n9XK3+h1aSKbnf1dYvEiB6wd4rnzPz40IKBNuqr4zKlzfvc9AT69Vjf/X8+QwyOxsqYLY3/4etDZX97FKh27Jw==
+  dependencies:
+    follow-redirects "^1.15.1"
+    https-proxy-agent "^5.0.0"
+    mime-types "^2.1.27"
+    sanitize-filename "^1.6.3"
+
 normalize-package-data@^2.3.2, normalize-package-data@^2.3.4, normalize-package-data@^2.5.0:
   version "2.5.0"
   resolved "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8"