✨ Feature(custom): improve speed of download config files

src/main/events/ipcList.ts

@@ -93,6 +93,8 @@ import webServer from '../server/webServer'
 import { configPaths } from '~/universal/utils/configPaths'
 
 const STORE_PATH = app.getPath('userData')
+const commonConfigList = ['data.json', 'data.bak.json']
+const manageConfigList = ['manage.json', 'manage.bak.json']
 
 export default {
   listen () {
@@ -251,43 +253,27 @@ export default {
     })
 
     ipcMain.handle('uploadCommonConfig', async () => {
-      const dataResult = await uploadFile('data.json')
+      return await uploadFile(commonConfigList)
-      const bakResult = await uploadFile('data.bak.json')
-      return dataResult + bakResult
     })
 
     ipcMain.handle('downloadCommonConfig', async () => {
-      const dataResult = await downloadFile('data.json')
+      return await downloadFile(commonConfigList)
-      const bakResult = await downloadFile('data.bak.json')
-      return dataResult + bakResult
     })
 
     ipcMain.handle('uploadManageConfig', async () => {
-      const manageResult = await uploadFile('manage.json')
+      return await uploadFile(manageConfigList)
-      const bakResult = await uploadFile('manage.bak.json')
-      return manageResult + bakResult
     })
 
     ipcMain.handle('downloadManageConfig', async () => {
-      const manageResult = await downloadFile('manage.json')
+      return await downloadFile(manageConfigList)
-      const bakResult = await downloadFile('manage.bak.json')
-      return manageResult + bakResult
     })
 
     ipcMain.handle('uploadAllConfig', async () => {
-      const dataResult = await uploadFile('data.json')
+      return await uploadFile([...commonConfigList, ...manageConfigList])
-      const bakResult = await uploadFile('data.bak.json')
-      const manageResult = await uploadFile('manage.json')
-      const manageBakResult = await uploadFile('manage.bak.json')
-      return dataResult + bakResult + manageResult + manageBakResult
     })
 
     ipcMain.handle('downloadAllConfig', async () => {
-      const dataResult = await downloadFile('data.json')
+      return await downloadFile([...commonConfigList, ...manageConfigList])
-      const bakResult = await downloadFile('data.bak.json')
-      const manageResult = await downloadFile('manage.json')
-      const manageBakResult = await downloadFile('manage.bak.json')
-      return dataResult + bakResult + manageResult + manageBakResult
     })
 
     ipcMain.on('toggleMainWindowAlwaysOnTop', () => {

src/main/utils/syncSettings.ts

@@ -10,32 +10,12 @@ import { configPaths } from '~/universal/utils/configPaths'
 
 const STORE_PATH = app.getPath('userData')
 
-const configFileNames = [
+const readFileAsBase64 = (filePath: string) => fs.readFileSync(filePath, { encoding: 'base64' })
-  'data.json',
-  'data.bak.json',
-  'manage.json',
-  'manage.bak.json'
-]
 
-function getOctokit (syncConfig: ISyncConfig) {
+const isHttpResSuccess = (res: any) => res.status >= 200 && res.status < 300
-  const { token, proxy } = syncConfig
+const uploadOrUpdateMsg = (fileName: string, isUpdate: boolean = true) => isUpdate ? `update ${fileName} from PicList` : `upload ${fileName} from PicList`
-  return new Octokit({
-    auth: token,
-    request: {
-      agent: proxy
-        ? new HttpsProxyAgent({
-          keepAlive: true,
-          keepAliveMsecs: 1000,
-          rejectUnauthorized: false,
-          proxy: proxy.replace('127.0.0.1', 'localhost'),
-          scheduling: 'lifo'
-        })
-        : undefined
-    }
-  })
-}
 
-function getSyncConfig () {
+const getSyncConfig = () => {
   return db.get(configPaths.settings.sync) || {
     type: 'github',
     username: '',
@@ -46,8 +26,35 @@ function getSyncConfig () {
   }
 }
 
-function syncConfigValidator (syncConfig: ISyncConfig) {
+const getProxyagent = (proxy: string | undefined) => {
-  const { type, username, repo, branch, token } = syncConfig
+  return proxy
+    ? new HttpsProxyAgent({
+      keepAlive: true,
+      keepAliveMsecs: 1000,
+      rejectUnauthorized: false,
+      proxy: proxy.replace('127.0.0.1', 'localhost'),
+      scheduling: 'lifo'
+    })
+    : undefined
+}
+
+function getOctokit (syncConfig: ISyncConfig) {
+  const { token, proxy } = syncConfig
+  return new Octokit({
+    auth: token,
+    request: {
+      agent: getProxyagent(proxy)
+    }
+  })
+}
+
+const isSyncConfigValidate = ({
+  type,
+  username,
+  repo,
+  branch,
+  token
+}: ISyncConfig) => {
   return type && username && repo && branch && token
 }
 
@@ -57,57 +64,48 @@ async function uploadLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
     return false
   }
   const { username, repo, branch, token, type } = syncConfig
-  if (type === 'gitee') {
+  const defaultConfig = {
+    content: readFileAsBase64(localFilePath),
+    message: uploadOrUpdateMsg(fileName, false),
+    branch
+  }
   try {
+    switch (type) {
+      case 'gitee': {
         const url = `https://gitee.com/api/v5/repos/${username}/${repo}/contents/${fileName}`
         const res = await axios.post(url, {
-        access_token: token,
+          ...defaultConfig,
-        branch,
+          access_token: token
-        content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
-        message: `upload ${fileName} from PicList`
         })
-      return res.status >= 200 && res.status < 300
+        return isHttpResSuccess(res)
-    } catch (error: any) {
-      logger.error(error)
-      return false
       }
-  } else if (type === 'github') {
+      case 'github': {
         const octokit = getOctokit(syncConfig)
-    try {
         const res = await octokit.rest.repos.createOrUpdateFileContents({
+          ...defaultConfig,
           owner: username,
           repo,
-        path: fileName,
+          path: fileName
-        message: `upload ${fileName} from PicList`,
-        content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
-        branch
         })
-      return res.status >= 200 && res.status < 300
+        return isHttpResSuccess(res)
-    } catch (error: any) {
-      logger.error(error)
-      return false
       }
-  } else {
+      case 'gitea': {
         const { endpoint = '' } = syncConfig
         const apiUrl = `${endpoint}/api/v1/repos/${username}/${repo}/contents/${fileName}`
-    try {
         const headers = {
           Authorization: `token ${token}`
         }
-      const res = await axios.post(apiUrl, {
+        const res = await axios.post(apiUrl, defaultConfig, { headers })
-        message: `upload ${fileName} from PicList`,
+        return isHttpResSuccess(res)
-        content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
+      }
-        branch
+      default:
-      }, {
+        return false
-        headers
+    }
-      })
-      return res.status >= 200 && res.status < 300
   } catch (error: any) {
     logger.error(error)
     return false
   }
 }
-}
 
 async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
   const localFilePath = path.join(STORE_PATH, fileName)
@@ -115,7 +113,13 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
     return false
   }
   const { username, repo, branch, token, type } = syncConfig
-  if (type === 'gitee') {
+  const defaultConfig = {
+    branch,
+    message: uploadOrUpdateMsg(fileName),
+    content: readFileAsBase64(localFilePath)
+  }
+  switch (type) {
+    case 'gitee': {
       const url = `https://gitee.com/api/v5/repos/${username}/${repo}/contents/${fileName}`
       const shaRes = await axios.get(url, {
         params: {
@@ -123,25 +127,21 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
           ref: branch
         }
       })
-    if (shaRes.status < 200 || shaRes.status > 300) {
+      if (!isHttpResSuccess(shaRes)) {
         return false
       }
       const sha = shaRes.data.sha
       const res = await axios.put(url, {
+        ...defaultConfig,
         owner: username,
         repo,
         path: fileName,
-      message: `update ${fileName} from PicList`,
-      content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
-      branch,
         sha,
         access_token: token
       })
-    if (res.status >= 200 && res.status < 300) {
+      return isHttpResSuccess(res)
-      return true
     }
-    return false
+    case 'github': {
-  } else if (type === 'github') {
       const octokit = getOctokit(syncConfig)
       const shaRes = await octokit.rest.repos.getContent({
         owner: username,
@@ -155,16 +155,15 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
       const data = shaRes.data as any
       const sha = data.sha
       const res = await octokit.rest.repos.createOrUpdateFileContents({
+        ...defaultConfig,
         owner: username,
         repo,
         path: fileName,
-      message: `update ${fileName} from PicList`,
-      content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
-      branch,
         sha
       })
       return res.status === 200
-  } else {
+    }
+    case 'gitea': {
       const { endpoint = '' } = syncConfig
       const apiUrl = `${endpoint}/api/v1/repos/${username}/${repo}/contents/${fileName}`
       const headers = {
@@ -173,48 +172,74 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
       const shaRes = await axios.get(apiUrl, {
         headers
       })
-    if (shaRes.status < 200 || shaRes.status > 300) {
+      if (!isHttpResSuccess(shaRes)) {
         throw new Error('get sha failed')
       }
       const data = shaRes.data as any
       const sha = data.sha
       const res = await axios.put(apiUrl, {
-      message: `update ${fileName} from PicList`,
+        ...defaultConfig,
-      content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
-      branch,
         sha
       }, {
         headers
       })
-    return res.status >= 200 && res.status < 300
+      return isHttpResSuccess(res)
     }
+    default:
+      return false
+  }
+}
+
+async function uploadFile (fileName: string[]): Promise<number> {
+  const syncConfig = getSyncConfig()
+  if (!isSyncConfigValidate(syncConfig)) {
+    logger.error('sync config is invalid')
+    return 0
+  }
+  const uploadFunc = async (file: string): Promise<number> => {
+    let result = false
+    try {
+      result = await updateLocalToRemote(syncConfig, file)
+    } catch (error: any) {
+      result = await uploadLocalToRemote(syncConfig, file)
+    }
+    logger.info(`upload ${file} ${result ? 'success' : 'failed'}`)
+    return result ? 1 : 0
+  }
+
+  let count = 0
+  for (const file of fileName) {
+    count += await uploadFunc(file)
+  }
+
+  return count
+}
+
+async function downloadAndWriteFile (url: string, localFilePath:string, config: any, isWriteJson = false) {
+  const res = await axios.get(url, config)
+  if (isHttpResSuccess(res)) {
+    await fs.writeFile(localFilePath, isWriteJson ? JSON.stringify(res.data, null, 2) : Buffer.from(res.data.content, 'base64'))
+    return true
+  }
+  return false
 }
 
 async function downloadRemoteToLocal (syncConfig: ISyncConfig, fileName: string) {
   const localFilePath = path.join(STORE_PATH, fileName)
   const { username, repo, branch, token, proxy, type } = syncConfig
-  if (type === 'gitee') {
   try {
+    switch (type) {
+      case 'gitee': {
         const url = `https://gitee.com/api/v5/repos/${username}/${repo}/contents/${fileName}`
-      const res = await axios.get(url, {
+        return downloadAndWriteFile(url, localFilePath, {
           params: {
             access_token: token,
             ref: branch
           }
         })
-      if (res.status >= 200 && res.status < 300) {
-        const content = res.data.content
-        await fs.writeFile(localFilePath, Buffer.from(content, 'base64'))
-        return true
       }
-      return false
+      case 'github': {
-    } catch (error: any) {
-      logger.error(error)
-      return false
-    }
-  } else if (type === 'github') {
         const octokit = getOctokit(syncConfig)
-    try {
         const res = await octokit.rest.repos.getContent({
           owner: username,
           repo,
@@ -224,122 +249,47 @@ async function downloadRemoteToLocal (syncConfig: ISyncConfig, fileName: string)
         if (res.status === 200) {
           const data = res.data as any
           const downloadUrl = data.download_url
-        const downloadRes = await axios.get(downloadUrl, {
+          return downloadAndWriteFile(downloadUrl, localFilePath, {
-          httpsAgent: proxy
+            httpsAgent: getProxyagent(proxy)
-            ? new HttpsProxyAgent({
+          }, true)
-              keepAlive: true,
-              keepAliveMsecs: 1000,
-              rejectUnauthorized: false,
-              proxy: proxy.replace('127.0.0.1', 'localhost'),
-              scheduling: 'lifo'
-            })
-            : undefined
-        })
-        if (downloadRes.status >= 200 && downloadRes.status < 300) {
-          await fs.writeFile(localFilePath, JSON.stringify(downloadRes.data, null, 2))
-          return true
-        }
         }
         return false
-    } catch (error: any) {
-      logger.error(error)
-      return false
       }
-  } else {
+      case 'gitea': {
         const { endpoint = '' } = syncConfig
         const apiUrl = `${endpoint}/api/v1/repos/${username}/${repo}/contents/${fileName}`
-    try {
+        return downloadAndWriteFile(apiUrl, localFilePath, {
-      const headers = {
+          headers: {
             Authorization: `token ${token}`
-      }
+          },
-      const res = await axios.get(apiUrl, {
-        headers,
           params: {
             ref: branch
           }
         })
-      if (res.status >= 200 && res.status < 300) {
-        const content = res.data.content
-        await fs.writeFile(localFilePath, Buffer.from(content, 'base64'))
-        return true
       }
+      default:
         return false
+    }
   } catch (error: any) {
     logger.error(error)
     return false
   }
 }
-}
 
-async function uploadFile (fileName: string, all = false) {
+async function downloadFile (fileName: string[]): Promise<number> {
   const syncConfig = getSyncConfig()
-  if (!syncConfigValidator(syncConfig)) {
+  if (!isSyncConfigValidate(syncConfig)) {
     logger.error('sync config is invalid')
     return 0
   }
-  let successCount = 0
-  if (all) {
-    for (const file of configFileNames) {
-      const result = await uploadFunc(syncConfig, file)
-      if (result) {
-        successCount++
-      }
-    }
-    logger.info(`upload all files at ${new Date().toLocaleString()}`)
-    return successCount
-  } else {
-    const ressult = await uploadFunc(syncConfig, fileName)
-    return ressult ? 1 : 0
-  }
-}
 
-async function uploadFunc (syncConfig: ISyncConfig, fileName: string) {
+  const downloadFunc = async (file: string): Promise<number> => {
-  let result = false
+    const result = await downloadRemoteToLocal(syncConfig, file)
-  try {
+    logger.info(`download ${file} ${result ? 'success' : 'failed'}`)
-    result = await updateLocalToRemote(syncConfig, fileName)
-  } catch (error: any) {
-    result = await uploadLocalToRemote(syncConfig, fileName)
-  }
-  if (!result) {
-    logger.error(`upload ${fileName} failed`)
-    return false
-  } else {
-    logger.info(`upload ${fileName} success`)
-    return true
-  }
-}
-
-async function downloadFile (fileName: string, all = false) {
-  const syncConfig = getSyncConfig()
-  if (!syncConfigValidator(syncConfig)) {
-    logger.error('sync config is invalid')
-    return 0
-  }
-  if (all) {
-    let successCount = 0
-    for (const file of configFileNames) {
-      const result = await downloadFunc(syncConfig, file)
-      if (result) {
-        successCount++
-      }
-    }
-    logger.info(`download all files at ${new Date().toLocaleString()}`)
-    return successCount
-  } else {
-    const result = await downloadFunc(syncConfig, fileName)
     return result ? 1 : 0
   }
-}
 
-async function downloadFunc (syncConfig: ISyncConfig, fileName: string) {
+  return (await Promise.all(fileName.map(downloadFunc))).reduce((a, b) => a + b, 0)
-  const result = await downloadRemoteToLocal(syncConfig, fileName)
-  if (!result) {
-    logger.error(`download ${fileName} failed`)
-    return false
-  } else {
-    logger.info(`download ${fileName} success`)
-    return true
-  }
 }
 
 export {