From 8440b75f1ecbd0ff7801bfa720dfaf1994249827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=8C=E8=90=8C=E5=93=92=E8=B5=AB=E8=90=9D?= Date: Wed, 22 Feb 2023 10:50:13 +0800 Subject: [PATCH] :sparkles: Feature: add concurrency limit for download --- README.md | 2 +- package.json | 1 + src/main/manage/apis/aliyun.ts | 28 +-- src/main/manage/apis/github.ts | 43 +++-- src/main/manage/apis/imgur.ts | 41 +++-- src/main/manage/apis/qiniu.ts | 22 ++- src/main/manage/apis/s3plist.ts | 78 +++++---- src/main/manage/apis/smms.ts | 21 ++- src/main/manage/apis/upyun.ts | 21 ++- src/main/manage/apis/webdavplist.ts | 182 ++++++++++++++++++- src/main/manage/manageApi.ts | 7 + src/main/manage/utils/common.ts | 118 +++++++++---- src/renderer/manage/pages/bucketPage.vue | 183 ++++++++++++++++++-- src/renderer/manage/pages/manageSetting.vue | 34 ++++ yarn.lock | 12 +- 15 files changed, 660 insertions(+), 133 deletions(-) diff --git a/README.md b/README.md index 983eb5a..0b6da21 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ - 保留了PicGo的所有功能,兼容已有的PicGo插件系统,包括和typora、obsidian等的搭配 - 相册中可同步删除云端图片 - 支持管理所有图床,可以在线进行云端目录查看、文件搜索、批量上传、批量下载、删除文件等 -- 支持预览多种格式的文件,包括图片、视频、纯文本文件和markdown文件等,具体支持的格式请参考[支持的文件格式列表](https://github.com/Kuingsmile/PicList/supported_format.md) +- 支持预览多种格式的文件,包括图片、视频、纯文本文件和markdown文件等,具体支持的格式请参考[支持的文件格式列表](https://github.com/Kuingsmile/PicList/blob/dev/supported_format.md) - 管理界面使用内置数据库缓存目录,加速目录加载速度 - 对于私有存储桶等支持复制预签名链接进行分享 - 优化了PicGo的界面,解锁了窗口大小限制,同时美化了部分界面布局 diff --git a/package.json b/package.json index 4e10631..7dc9c6e 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "marked": "^4.2.12", "mime-types": "^2.1.35", "mitt": "^3.0.0", + "nodejs-file-downloader": "^4.10.6", "piclist": "^0.0.8", "pinia": "^2.0.29", "pinia-plugin-persistedstate": "^3.0.2", diff --git a/src/main/manage/apis/aliyun.ts b/src/main/manage/apis/aliyun.ts index a03a25c..6d2dc0f 100644 --- a/src/main/manage/apis/aliyun.ts +++ b/src/main/manage/apis/aliyun.ts @@ -1,7 +1,6 @@ import axios from 'axios' -import { hmacSha1Base64, getFileMimeType, gotDownload, formatError } from '../utils/common' +import { hmacSha1Base64, getFileMimeType, formatError, NewDownloader, ConcurrencyPromisePool } from '../utils/common' import { ipcMain, IpcMainEvent } from 'electron' -import fs from 'fs-extra' import { XMLParser } from 'fast-xml-parser' import OSS from 'ali-oss' import path from 'path' @@ -548,19 +547,13 @@ class AliyunApi { * @param configMap */ async downloadBucketFile (configMap: IStringKeyMap): Promise { - const { downloadPath, fileArray } = configMap - // fileArray = [{ - // bucketName: string, - // region: string, - // key: string, - // fileName: string - // }] + const { downloadPath, fileArray, maxDownloadFileCount } = configMap const instance = UpDownTaskQueue.getInstance() + const promises = [] as any for (const item of fileArray) { const { bucketName, region, key, fileName } = item const client = this.getNewCtx(region, bucketName) const savedFilePath = path.join(downloadPath, fileName) - const fileStream = fs.createWriteStream(savedFilePath) const id = `${bucketName}-${region}-${key}` if (instance.getDownloadTask(id)) { continue @@ -575,8 +568,21 @@ class AliyunApi { const preSignedUrl = client.signatureUrl(key, { expires: 60 * 60 * 48 }) - gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger) + promises.push(() => new Promise((resolve, reject) => { + NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error: any) => { + this.logger.error(formatError(error, { class: 'AliyunApi', method: 'downloadBucketFile' })) + }) return true } } diff --git a/src/main/manage/apis/github.ts b/src/main/manage/apis/github.ts index 877f06b..65fed7f 100644 --- a/src/main/manage/apis/github.ts +++ b/src/main/manage/apis/github.ts @@ -1,10 +1,10 @@ import got from 'got' import { ManageLogger } from '../utils/logger' -import { isImage } from '~/renderer/manage/utils/common' +import { formatHttpProxy, isImage } from '~/renderer/manage/utils/common' import windowManager from 'apis/app/window/windowManager' import { IWindowList } from '#/types/enum' import { ipcMain, IpcMainEvent } from 'electron' -import { gotUpload, trimPath, gotDownload, getAgent, getOptions } from '../utils/common' +import { gotUpload, trimPath, NewDownloader, getAgent, getOptions, ConcurrencyPromisePool, formatError } from '../utils/common' import UpDownTaskQueue, { commonTaskStatus @@ -17,6 +17,7 @@ class GithubApi { username: string logger: ManageLogger proxy: any + proxyStr: string | undefined baseUrl = 'https://api.github.com' commonHeaders : IStringKeyMap @@ -25,6 +26,7 @@ class GithubApi { this.token = token.startsWith('Bearer ') ? token : `Bearer ${token}`.trim() this.username = username this.proxy = proxy + this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined this.commonHeaders = { Authorization: this.token, Accept: 'application/vnd.github+json' @@ -388,13 +390,13 @@ class GithubApi { * @param configMap */ async downloadBucketFile (configMap: IStringKeyMap): Promise { - const { downloadPath, fileArray } = configMap + const { downloadPath, fileArray, maxDownloadFileCount } = configMap const instance = UpDownTaskQueue.getInstance() + const promises = [] as any for (const item of fileArray) { const { bucketName: repo, customUrl: branch, key, fileName, githubPrivate, githubUrl } = item const id = `${repo}-${branch}-${key}-${fileName}` const savedFilePath = path.join(downloadPath, fileName) - const fileStream = fs.createWriteStream(savedFilePath) if (instance.getDownloadTask(id)) { continue } @@ -405,7 +407,7 @@ class GithubApi { sourceFileName: fileName, targetFilePath: savedFilePath }) - let downloadUrl + let downloadUrl: string if (githubPrivate) { const preSignedUrl = await this.getPreSignedUrl({ bucketName: repo, @@ -418,17 +420,28 @@ class GithubApi { } else { downloadUrl = githubUrl } - gotDownload( - instance, - downloadUrl, - fileStream, - id, - savedFilePath, - this.logger, - undefined, - getAgent(this.proxy) - ) + promises.push(() => new Promise((resolve, reject) => { + NewDownloader( + instance, + downloadUrl, + id, + savedFilePath, + this.logger, + this.proxyStr + ) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error) => { + this.logger.error(formatError(error, { class: 'GithubApi', method: 'downloadBucketFile' })) + }) return true } } diff --git a/src/main/manage/apis/imgur.ts b/src/main/manage/apis/imgur.ts index 1577abb..2f65a29 100644 --- a/src/main/manage/apis/imgur.ts +++ b/src/main/manage/apis/imgur.ts @@ -1,10 +1,10 @@ import got from 'got' import ManageLogger from '../utils/logger' -import { getAgent, getOptions, gotDownload, gotUpload, getFileMimeType } from '../utils/common' +import { getAgent, getOptions, NewDownloader, gotUpload, getFileMimeType, ConcurrencyPromisePool, formatError } from '../utils/common' import windowManager from 'apis/app/window/windowManager' import { IWindowList } from '#/types/enum' import { ipcMain, IpcMainEvent } from 'electron' -import { isImage } from '~/renderer/manage/utils/common' +import { formatHttpProxy, isImage } from '~/renderer/manage/utils/common' import path from 'path' import UpDownTaskQueue, { @@ -18,6 +18,7 @@ class ImgurApi { accessToken: string proxy: any logger: ManageLogger + proxyStr: string | undefined tokenHeaders: any idHeaders: any baseUrl = 'https://api.imgur.com/3' @@ -26,6 +27,7 @@ class ImgurApi { this.userName = userName this.accessToken = accessToken.startsWith('Bearer ') ? accessToken : `Bearer ${accessToken}` this.proxy = proxy + this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined this.logger = logger this.tokenHeaders = { Authorization: this.accessToken @@ -227,13 +229,13 @@ class ImgurApi { * @param configMap */ async downloadBucketFile (configMap: IStringKeyMap): Promise { - const { downloadPath, fileArray } = configMap + const { downloadPath, fileArray, maxDownloadFileCount } = configMap const instance = UpDownTaskQueue.getInstance() + const promises = [] as any for (const item of fileArray) { const { bucketName, region, key, fileName, githubUrl: url } = item const id = `${bucketName}-${region}-${key}-${fileName}` const savedFilePath = path.join(downloadPath, fileName) - const fileStream = fs.createWriteStream(savedFilePath) if (instance.getDownloadTask(id)) { continue } @@ -244,17 +246,28 @@ class ImgurApi { sourceFileName: fileName, targetFilePath: savedFilePath }) - gotDownload( - instance, - url, - fileStream, - id, - savedFilePath, - this.logger, - undefined, - getAgent(this.proxy) - ) + promises.push(() => new Promise((resolve, reject) => { + NewDownloader( + instance, + url, + id, + savedFilePath, + this.logger, + this.proxyStr + ) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error) => { + this.logger.error(formatError(error, { class: 'ImgurApi', method: 'downloadBucketFile' })) + }) return true } } diff --git a/src/main/manage/apis/qiniu.ts b/src/main/manage/apis/qiniu.ts index d0e73f9..2a425df 100644 --- a/src/main/manage/apis/qiniu.ts +++ b/src/main/manage/apis/qiniu.ts @@ -1,6 +1,5 @@ import axios from 'axios' -import { hmacSha1Base64, getFileMimeType, gotDownload, formatError } from '../utils/common' -import fs from 'fs-extra' +import { hmacSha1Base64, getFileMimeType, NewDownloader, formatError, ConcurrencyPromisePool } from '../utils/common' import qiniu from 'qiniu/index' import path from 'path' import { isImage } from '~/renderer/manage/utils/common' @@ -627,12 +626,12 @@ class QiniuApi { * @param configMap */ async downloadBucketFile (configMap: IStringKeyMap): Promise { - const { downloadPath, fileArray } = configMap + const { downloadPath, fileArray, maxDownloadFileCount } = configMap const instance = UpDownTaskQueue.getInstance() + const promises = [] as any for (const item of fileArray) { const { bucketName, region, key, fileName, customUrl } = item const savedFilePath = path.join(downloadPath, fileName) - const fileStream = fs.createWriteStream(savedFilePath) const id = `${bucketName}-${region}-${key}` if (instance.getDownloadTask(id)) { continue @@ -645,8 +644,21 @@ class QiniuApi { targetFilePath: savedFilePath }) const preSignedUrl = await this.getPreSignedUrl({ key, expires: 36000, customUrl }) - gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger) + promises.push(() => new Promise((resolve, reject) => { + NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error) => { + this.logger.error(formatError(error, { class: 'QiniuApi', method: 'downloadBucketFile' })) + }) return true } } diff --git a/src/main/manage/apis/s3plist.ts b/src/main/manage/apis/s3plist.ts index f91b6e5..0736523 100644 --- a/src/main/manage/apis/s3plist.ts +++ b/src/main/manage/apis/s3plist.ts @@ -18,8 +18,8 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import https from 'https' import http from 'http' import { ManageLogger } from '../utils/logger' -import { formatEndpoint, formatError, getAgent, getFileMimeType, gotDownload } from '../utils/common' -import { isImage } from '@/manage/utils/common' +import { formatEndpoint, formatError, getAgent, getFileMimeType, NewDownloader, ConcurrencyPromisePool } from '../utils/common' +import { isImage, formatHttpProxy } from '@/manage/utils/common' import { HttpsProxyAgent, HttpProxyAgent } from 'hpagent' import windowManager from 'apis/app/window/windowManager' import { IWindowList } from '#/types/enum' @@ -49,6 +49,7 @@ class S3plistApi { baseOptions: S3plistApiOptions logger: ManageLogger agent: any + proxy: string | undefined constructor ( accessKeyId: string, @@ -73,6 +74,7 @@ class S3plistApi { } as S3plistApiOptions this.logger = logger this.agent = this.setAgent(proxy, sslEnabled) + this.proxy = formatHttpProxy(proxy, 'string') as string | undefined } setAgent (proxy: string | undefined, sslEnabled: boolean) : HttpProxyAgent | HttpsProxyAgent | undefined { @@ -129,32 +131,43 @@ class S3plistApi { const options = Object.assign({}, this.baseOptions) as S3ClientConfig options.region = 'us-east-1' const result = [] as IStringKeyMap[] + const endpoint = options.endpoint as string || '' as string try { const client = new S3Client(options) const command = new ListBucketsCommand({}) const data = await client.send(command) if (data.$metadata.httpStatusCode === 200) { if (data.Buckets) { - for (let i = 0; i < data.Buckets.length; i++) { - const bucket = data.Buckets[i] - const bucketName = bucket.Name - const command = new GetBucketLocationCommand({ - Bucket: bucketName + if (endpoint.indexOf('cloudflarestorage') !== -1) { + data.Buckets.forEach((bucket) => { + result.push({ + Name: bucket.Name, + CreationDate: bucket.CreationDate, + Location: 'auto' + }) }) - const bucketConfig = await client.send(command) - if (bucketConfig.$metadata.httpStatusCode === 200) { - result.push({ - Name: bucketName, - CreationDate: bucket.CreationDate, - Location: bucketConfig.LocationConstraint || 'us-east-1' - }) - } else { - this.logParam(bucketConfig, 'getBucketList') - result.push({ - Name: bucketName, - CreationDate: bucket.CreationDate, - Location: 'us-east-1' + } else { + for (let i = 0; i < data.Buckets.length; i++) { + const bucket = data.Buckets[i] + const bucketName = bucket.Name + const command = new GetBucketLocationCommand({ + Bucket: bucketName }) + const bucketConfig = await client.send(command) + if (bucketConfig.$metadata.httpStatusCode === 200) { + result.push({ + Name: bucketName, + CreationDate: bucket.CreationDate, + Location: bucketConfig.LocationConstraint?.toLowerCase() || 'us-east-1' + }) + } else { + this.logParam(bucketConfig, 'getBucketList') + result.push({ + Name: bucketName, + CreationDate: bucket.CreationDate, + Location: 'us-east-1' + }) + } } } } @@ -573,18 +586,12 @@ class S3plistApi { * @param configMap */ async downloadBucketFile (configMap: IStringKeyMap): Promise { - const { downloadPath, fileArray } = configMap - // fileArray = [{ - // bucketName: string, - // region: string, - // key: string, - // fileName: string - // }] + const { downloadPath, fileArray, maxDownloadFileCount } = configMap const instance = UpDownTaskQueue.getInstance() + const promises = [] as any for (const item of fileArray) { const { bucketName, region, key, fileName, customUrl } = item const savedFilePath = path.join(downloadPath, fileName) - const fileStream = fs.createWriteStream(savedFilePath) const id = `${bucketName}-${region}-${key}-${savedFilePath}` if (instance.getDownloadTask(id)) { continue @@ -603,8 +610,21 @@ class S3plistApi { expires: 36000, customUrl }) - gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger) + promises.push(() => new Promise((resolve, reject) => { + NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger, this.proxy) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error) => { + this.logParam(error, 'downloadBucketFile') + }) return true } } diff --git a/src/main/manage/apis/smms.ts b/src/main/manage/apis/smms.ts index 42db895..2d873f9 100644 --- a/src/main/manage/apis/smms.ts +++ b/src/main/manage/apis/smms.ts @@ -5,7 +5,7 @@ import { IWindowList } from '#/types/enum' import { ipcMain, IpcMainEvent } from 'electron' import FormData from 'form-data' import fs from 'fs-extra' -import { getFileMimeType, gotUpload, gotDownload } from '../utils/common' +import { getFileMimeType, gotUpload, NewDownloader, ConcurrencyPromisePool, formatError } from '../utils/common' import path from 'path' import UpDownTaskQueue, { commonTaskStatus } from '../datastore/upDownTaskQueue' import { ManageLogger } from '../utils/logger' @@ -227,12 +227,12 @@ class SmmsApi { * @param configMap */ async downloadBucketFile (configMap: IStringKeyMap): Promise { - const { downloadPath, fileArray } = configMap + const { downloadPath, fileArray, maxDownloadFileCount } = configMap const instance = UpDownTaskQueue.getInstance() + const promises = [] as any for (const item of fileArray) { const { bucketName, region, key, fileName, downloadUrl: preSignedUrl } = item const savedFilePath = path.join(downloadPath, fileName) - const fileStream = fs.createWriteStream(savedFilePath) const id = `${bucketName}-${region}-${key}` if (instance.getDownloadTask(id)) { continue @@ -244,8 +244,21 @@ class SmmsApi { sourceFileName: fileName, targetFilePath: savedFilePath }) - gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger) + promises.push(() => new Promise((resolve, reject) => { + NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error) => { + this.logger.error(formatError(error, { class: 'SmmsApi', method: 'downloadBucketFile' })) + }) return true } } diff --git a/src/main/manage/apis/upyun.ts b/src/main/manage/apis/upyun.ts index cc51e72..1142336 100644 --- a/src/main/manage/apis/upyun.ts +++ b/src/main/manage/apis/upyun.ts @@ -1,6 +1,6 @@ // @ts-ignore import Upyun from 'upyun' -import { md5, hmacSha1Base64, getFileMimeType, gotDownload, gotUpload } from '../utils/common' +import { md5, hmacSha1Base64, getFileMimeType, NewDownloader, gotUpload, ConcurrencyPromisePool, formatError } from '../utils/common' import { isImage } from '~/renderer/manage/utils/common' import windowManager from 'apis/app/window/windowManager' import { IWindowList } from '#/types/enum' @@ -359,12 +359,12 @@ class UpyunApi { * @param configMap */ async downloadBucketFile (configMap: IStringKeyMap): Promise { - const { downloadPath, fileArray } = configMap + const { downloadPath, fileArray, maxDownloadFileCount } = configMap const instance = UpDownTaskQueue.getInstance() + const promises = [] as any for (const item of fileArray) { const { bucketName, region, key, fileName, customUrl } = item const savedFilePath = path.join(downloadPath, fileName) - const fileStream = fs.createWriteStream(savedFilePath) const id = `${bucketName}-${region}-${key}` if (instance.getDownloadTask(id)) { continue @@ -377,8 +377,21 @@ class UpyunApi { targetFilePath: savedFilePath }) const preSignedUrl = `${customUrl}/${key}` - gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger) + promises.push(() => new Promise((resolve, reject) => { + NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error) => { + this.logger.error(formatError(error, { class: 'UpyunApi', method: 'downloadBucketFile' })) + }) return true } } diff --git a/src/main/manage/apis/webdavplist.ts b/src/main/manage/apis/webdavplist.ts index 58a2757..0b7d607 100644 --- a/src/main/manage/apis/webdavplist.ts +++ b/src/main/manage/apis/webdavplist.ts @@ -1,12 +1,19 @@ import ManageLogger from '../utils/logger' -import { createClient, WebDAVClient, FileStat } from 'webdav' -import { formatError, formatEndpoint, getInnerAgent } from '../utils/common' -import { isImage } from '@/manage/utils/common' +import { createClient, WebDAVClient, FileStat, ProgressEvent } from 'webdav' +import { formatError, formatEndpoint, getInnerAgent, NewDownloader, ConcurrencyPromisePool } from '../utils/common' +import { formatHttpProxy, isImage } from '@/manage/utils/common' import http from 'http' import https from 'https' import windowManager from 'apis/app/window/windowManager' import { IWindowList } from '#/types/enum' import { ipcMain, IpcMainEvent } from 'electron' +import UpDownTaskQueue, +{ + uploadTaskSpecialStatus, + commonTaskStatus +} from '../datastore/upDownTaskQueue' +import fs from 'fs-extra' +import path from 'path' class WebdavplistApi { endpoint: string @@ -14,6 +21,7 @@ class WebdavplistApi { password: string sslEnabled: boolean proxy: string | undefined + proxyStr: string | undefined logger: ManageLogger agent: https.Agent | http.Agent ctx: WebDAVClient @@ -24,6 +32,7 @@ class WebdavplistApi { this.password = password this.sslEnabled = sslEnabled this.proxy = proxy + this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined this.logger = logger this.agent = getInnerAgent(proxy, sslEnabled).agent this.ctx = createClient( @@ -125,6 +134,173 @@ class WebdavplistApi { window.webContents.send('refreshFileTransferList', result) ipcMain.removeAllListeners('cancelLoadingFileList') } + + async renameBucketFile (configMap: IStringKeyMap): Promise { + const { oldKey, newKey } = configMap + let result = false + try { + await this.ctx.moveFile(oldKey, newKey) + result = true + } catch (error) { + this.logParam(error, 'renameBucketFile') + } + return result + } + + async deleteBucketFile (configMap: IStringKeyMap): Promise { + const { key } = configMap + let result = false + try { + await this.ctx.deleteFile(key) + result = true + } catch (error) { + this.logParam(error, 'deleteBucketFile') + } + return result + } + + async deleteBucketFolder (configMap: IStringKeyMap): Promise { + const { key } = configMap + let result = false + try { + await this.ctx.deleteFile(key) + result = true + } catch (error) { + this.logParam(error, 'deleteBucketFolder') + } + return result + } + + async getPreSignedUrl (configMap: IStringKeyMap): Promise { + const { key } = configMap + let result = '' + try { + const res = this.ctx.getFileDownloadLink(key) + result = res + } catch (error) { + this.logParam(error, 'getPreSignedUrl') + } + return result + } + + async uploadBucketFile (configMap: IStringKeyMap): Promise { + const { fileArray } = configMap + const instance = UpDownTaskQueue.getInstance() + for (const item of fileArray) { + const { alias, bucketName, region, key, filePath, fileName } = item + const id = `${alias}-${bucketName}-${key}-${filePath}` + if (instance.getUploadTask(id)) { + continue + } + instance.addUploadTask({ + id, + progress: 0, + status: commonTaskStatus.queuing, + sourceFileName: fileName, + sourceFilePath: filePath, + targetFilePath: key, + targetFileBucket: bucketName, + targetFileRegion: region, + noProgress: true + }) + this.ctx.putFileContents( + key, + fs.createReadStream(filePath), + { + overwrite: true, + onUploadProgress: (progressEvent: ProgressEvent) => { + instance.updateUploadTask({ + id, + progress: Math.floor((progressEvent.loaded / progressEvent.total) * 100), + status: uploadTaskSpecialStatus.uploading + }) + } + } + ).then((res: boolean) => { + if (res) { + instance.updateUploadTask({ + id, + progress: 100, + status: uploadTaskSpecialStatus.uploaded, + finishTime: new Date().toLocaleString() + }) + } else { + instance.updateUploadTask({ + id, + progress: 0, + status: commonTaskStatus.failed, + finishTime: new Date().toLocaleString() + }) + } + }).catch((error: any) => { + this.logParam(error, 'uploadBucketFile') + instance.updateUploadTask({ + id, + progress: 0, + status: commonTaskStatus.failed, + finishTime: new Date().toLocaleString() + }) + }) + } + return true + } + + async createBucketFolder (configMap: IStringKeyMap): Promise { + const { key } = configMap + let result = false + try { + await this.ctx.createDirectory(key, { + recursive: true + }) + result = true + } catch (error) { + this.logParam(error, 'createBucketFolder') + } + return result + } + + async downloadBucketFile (configMap: IStringKeyMap): Promise { + const { downloadPath, fileArray, maxDownloadFileCount } = configMap + const instance = UpDownTaskQueue.getInstance() + const promises = [] as any + for (const item of fileArray) { + const { alias, bucketName, region, key, fileName } = item + const savedFilePath = path.join(downloadPath, fileName) + const id = `${alias}-${bucketName}-${region}-${key}` + if (instance.getDownloadTask(id)) { + continue + } + instance.addDownloadTask({ + id, + progress: 0, + status: commonTaskStatus.queuing, + sourceFileName: fileName, + targetFilePath: savedFilePath + }) + const preSignedUrl = await this.getPreSignedUrl({ + key + }) + const base64Str = Buffer.from(`${this.username}:${this.password}`).toString('base64') + const headers = { + Authorization: `Basic ${base64Str}` + } + promises.push(() => new Promise((resolve, reject) => { + NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger, this.proxyStr, headers) + .then((res: boolean) => { + if (res) { + resolve(res) + } else { + reject(res) + } + }) + })) + } + const pool = new ConcurrencyPromisePool(maxDownloadFileCount) + pool.all(promises).catch((error) => { + this.logParam(error, 'downloadBucketFile') + }) + return true + } } export default WebdavplistApi diff --git a/src/main/manage/manageApi.ts b/src/main/manage/manageApi.ts index 6771915..0d02e7b 100644 --- a/src/main/manage/manageApi.ts +++ b/src/main/manage/manageApi.ts @@ -387,6 +387,7 @@ export class ManageApi extends EventEmitter implements ManageApiType { case 'github': case 'imgur': case 's3plist': + case 'webdavplist': try { client = this.createClient() as any const res = await client.deleteBucketFile(param!) @@ -411,6 +412,7 @@ export class ManageApi extends EventEmitter implements ManageApiType { case 'upyun': case 'github': case 's3plist': + case 'webdavplist': try { client = this.createClient() as any return await client.deleteBucketFolder(param!) @@ -433,6 +435,7 @@ export class ManageApi extends EventEmitter implements ManageApiType { case 'qiniu': case 'upyun': case 's3plist': + case 'webdavplist': try { client = this.createClient() as any return await client.renameBucketFile(param!) @@ -458,6 +461,7 @@ export class ManageApi extends EventEmitter implements ManageApiType { case 'github': case 'imgur': case 's3plist': + case 'webdavplist': try { client = this.createClient() as any const res = await client.downloadBucketFile(param!) @@ -489,6 +493,7 @@ export class ManageApi extends EventEmitter implements ManageApiType { case 'upyun': case 'github': case 's3plist': + case 'webdavplist': try { client = this.createClient() as any return await client.createBucketFolder(param!) @@ -514,6 +519,7 @@ export class ManageApi extends EventEmitter implements ManageApiType { case 'github': case 'imgur': case 's3plist': + case 'webdavplist': try { client = this.createClient() as any return await client.uploadBucketFile(param!) @@ -536,6 +542,7 @@ export class ManageApi extends EventEmitter implements ManageApiType { case 'qiniu': case 'github': case 's3plist': + case 'webdavplist': try { client = this.createClient() as any return await client.getPreSignedUrl(param!) diff --git a/src/main/manage/utils/common.ts b/src/main/manage/utils/common.ts index faa344e..a29cd6d 100644 --- a/src/main/manage/utils/common.ts +++ b/src/main/manage/utils/common.ts @@ -18,6 +18,7 @@ import { formatHttpProxy, IHTTPProxy } from '@/manage/utils/common' import { HttpsProxyAgent, HttpProxyAgent } from 'hpagent' import http from 'http' import https from 'https' +import Downloader from 'nodejs-file-downloader' export const getFSFile = async ( filePath: string, @@ -90,52 +91,57 @@ export const md5 = (str: string, code: 'hex' | 'base64'): string => crypto.creat export const hmacSha1Base64 = (secretKey: string, stringToSign: string) : string => crypto.createHmac('sha1', secretKey).update(Buffer.from(stringToSign, 'utf8')).digest('base64') -export const gotDownload = async ( +export const NewDownloader = async ( instance: UpDownTaskQueue, preSignedUrl: string, - fileStream: fs.WriteStream, id : string, savedFilePath: string, logger?: ManageLogger, - param?: any, - agent: any = {} -) => { - got( - preSignedUrl, - { - isStream: true, - throwHttpErrors: false, - searchParams: param, - agent: agent || {} - } - ) - .on('downloadProgress', (progress: any) => { + proxy?: string, + headers?: any +) : Promise => { + const options = { + url: preSignedUrl, + directory: path.dirname(savedFilePath), + fileName: path.basename(savedFilePath), + cloneFiles: false, + onProgress: (percentage: string) => { instance.updateDownloadTask({ id, - progress: Math.floor(progress.percent * 100), + progress: Math.floor(Number(percentage)), status: downloadTaskSpecialStatus.downloading }) + }, + maxAttempts: 3 + } as any + if (proxy) { + options.proxy = proxy + } + if (headers) { + options.headers = headers + } + const downloader = new Downloader(options) + try { + await downloader.download() + instance.updateDownloadTask({ + id, + progress: 100, + status: downloadTaskSpecialStatus.downloaded, + finishTime: new Date().toLocaleString() }) - .pipe(fileStream) - .on('close', () => { - instance.updateDownloadTask({ - id, - progress: 100, - status: downloadTaskSpecialStatus.downloaded, - finishTime: new Date().toLocaleString() - }) - }) - .on('error', (err: any) => { - logger && logger.error(formatError(err, { method: 'gotDownload' })) - fs.remove(savedFilePath) - instance.updateDownloadTask({ - id, - progress: 0, - status: commonTaskStatus.failed, - response: formatError(err, { method: 'gotDownload' }), - finishTime: new Date().toLocaleString() - }) + return true + } catch (e: any) { + logger && logger.error(formatError(e, { method: 'NewDownloader' })) + fs.remove(savedFilePath) + instance.updateDownloadTask({ + id, + progress: 0, + status: commonTaskStatus.failed, + response: formatError(e, { method: 'NewDownloader' }), + finishTime: new Date().toLocaleString() }) + return false + } } export const gotUpload = async ( @@ -320,3 +326,45 @@ export const formatEndpoint = (endpoint: string, sslEnabled: boolean): string => : sslEnabled ? endpoint.replace('http://', 'https://') : endpoint.replace('https://', 'http://') + +export class ConcurrencyPromisePool { + limit: number + queue: any[] + runningNum: number + results: any[] + + constructor (limit: number) { + this.limit = limit + this.queue = [] + this.runningNum = 0 + this.results = [] + } + + all (promises: any[] = []) { + return new Promise((resolve, reject) => { + for (const promise of promises) { + this._run(promise, resolve, reject) + } + }) + } + + _run (promise: any, resolve: any, reject: any) { + if (this.runningNum >= this.limit) { + this.queue.push(promise) + return + } + this.runningNum += 1 + promise() + .then((res: any) => { + this.results.push(res) + --this.runningNum + if (this.queue.length === 0 && this.runningNum === 0) { + return resolve(this.results) + } + if (this.queue.length > 0) { + this._run(this.queue.shift(), resolve, reject) + } + }) + .catch(reject) + } +} diff --git a/src/renderer/manage/pages/bucketPage.vue b/src/renderer/manage/pages/bucketPage.vue index fe5bf8a..51fc0c7 100644 --- a/src/renderer/manage/pages/bucketPage.vue +++ b/src/renderer/manage/pages/bucketPage.vue @@ -234,7 +234,7 @@ ea/* + + + + + + + + 复制上传任务信息 + + + 清空已完成任务 + + + 清空所有任务 + + +
+ + + +
+
+ + + + + 复制下载任务信息 + + + 清空已完成任务 + + + 清空所有任务 + + + 打开下载目录 + + +
+ +