tcjlj/PicList
1
0
Fork 0
mirror of https://github.com/Kuingsmile/PicList.git synced 2025-01-22 22:28:14 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

✨ Feature(custom): improve speed of download config files

Browse Source
This commit is contained in:
Kuingsmile 2024-05-23 09:23:29 +08:00
parent 4f950148d5
commit e16737e69c
2 changed files with 226 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/main/events/ipcList.ts
View File

@ -93,6 +93,8 @@ import webServer from '../server/webServer'
import { configPaths } from '~/universal/utils/configPaths'
const STORE_PATH = app.getPath('userData')
const commonConfigList = ['data.json', 'data.bak.json']
const manageConfigList = ['manage.json', 'manage.bak.json']
export default {
listen () {
@ -251,43 +253,27 @@ export default {
})
ipcMain.handle('uploadCommonConfig', async () => {
const dataResult = await uploadFile('data.json')
const bakResult = await uploadFile('data.bak.json')
return dataResult + bakResult
return await uploadFile(commonConfigList)
})
ipcMain.handle('downloadCommonConfig', async () => {
const dataResult = await downloadFile('data.json')
const bakResult = await downloadFile('data.bak.json')
return dataResult + bakResult
return await downloadFile(commonConfigList)
})
ipcMain.handle('uploadManageConfig', async () => {
const manageResult = await uploadFile('manage.json')
const bakResult = await uploadFile('manage.bak.json')
return manageResult + bakResult
return await uploadFile(manageConfigList)
})
ipcMain.handle('downloadManageConfig', async () => {
const manageResult = await downloadFile('manage.json')
const bakResult = await downloadFile('manage.bak.json')
return manageResult + bakResult
return await downloadFile(manageConfigList)
})
ipcMain.handle('uploadAllConfig', async () => {
const dataResult = await uploadFile('data.json')
const bakResult = await uploadFile('data.bak.json')
const manageResult = await uploadFile('manage.json')
const manageBakResult = await uploadFile('manage.bak.json')
return dataResult + bakResult + manageResult + manageBakResult
return await uploadFile([...commonConfigList, ...manageConfigList])
})
ipcMain.handle('downloadAllConfig', async () => {
const dataResult = await downloadFile('data.json')
const bakResult = await downloadFile('data.bak.json')
const manageResult = await downloadFile('manage.json')
const manageBakResult = await downloadFile('manage.bak.json')
return dataResult + bakResult + manageResult + manageBakResult
return await downloadFile([...commonConfigList, ...manageConfigList])
})
ipcMain.on('toggleMainWindowAlwaysOnTop', () => {

304
src/main/utils/syncSettings.ts
View File

@ -10,32 +10,12 @@ import { configPaths } from '~/universal/utils/configPaths'
const STORE_PATH = app.getPath('userData')
const configFileNames = [
'data.json',
'data.bak.json',
'manage.json',
'manage.bak.json'
]
const readFileAsBase64 = (filePath: string) => fs.readFileSync(filePath, { encoding: 'base64' })
function getOctokit (syncConfig: ISyncConfig) {
const { token, proxy } = syncConfig
return new Octokit({
auth: token,
request: {
agent: proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 1000,
rejectUnauthorized: false,
proxy: proxy.replace('127.0.0.1', 'localhost'),
scheduling: 'lifo'
})
: undefined
}
})
}
const isHttpResSuccess = (res: any) => res.status >= 200 && res.status < 300
const uploadOrUpdateMsg = (fileName: string, isUpdate: boolean = true) => isUpdate ? `update ${fileName} from PicList` : `upload ${fileName} from PicList`
function getSyncConfig () {
const getSyncConfig = () => {
return db.get(configPaths.settings.sync) || {
type: 'github',
username: '',
@ -46,8 +26,35 @@ function getSyncConfig () {
}
}
function syncConfigValidator (syncConfig: ISyncConfig) {
const { type, username, repo, branch, token } = syncConfig
const getProxyagent = (proxy: string | undefined) => {
return proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 1000,
rejectUnauthorized: false,
proxy: proxy.replace('127.0.0.1', 'localhost'),
scheduling: 'lifo'
})
: undefined
}
function getOctokit (syncConfig: ISyncConfig) {
const { token, proxy } = syncConfig
return new Octokit({
auth: token,
request: {
agent: getProxyagent(proxy)
}
})
}
const isSyncConfigValidate = ({
type,
username,
repo,
branch,
token
}: ISyncConfig) => {
return type && username && repo && branch && token
}
@ -57,57 +64,48 @@ async function uploadLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
return false
}
const { username, repo, branch, token, type } = syncConfig
if (type === 'gitee') {
const defaultConfig = {
content: readFileAsBase64(localFilePath),
message: uploadOrUpdateMsg(fileName, false),
branch
}
try {
switch (type) {
case 'gitee': {
const url = `https://gitee.com/api/v5/repos/${username}/${repo}/contents/${fileName}`
const res = await axios.post(url, {
access_token: token,
branch,
content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
message: `upload ${fileName} from PicList`
...defaultConfig,
access_token: token
})
return res.status >= 200 && res.status < 300
} catch (error: any) {
logger.error(error)
return false
return isHttpResSuccess(res)
}
} else if (type === 'github') {
case 'github': {
const octokit = getOctokit(syncConfig)
try {
const res = await octokit.rest.repos.createOrUpdateFileContents({
...defaultConfig,
owner: username,
repo,
path: fileName,
message: `upload ${fileName} from PicList`,
content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
branch
path: fileName
})
return res.status >= 200 && res.status < 300
} catch (error: any) {
logger.error(error)
return false
return isHttpResSuccess(res)
}
} else {
case 'gitea': {
const { endpoint = '' } = syncConfig
const apiUrl = `${endpoint}/api/v1/repos/${username}/${repo}/contents/${fileName}`
try {
const headers = {
Authorization: `token ${token}`
}
const res = await axios.post(apiUrl, {
message: `upload ${fileName} from PicList`,
content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
branch
}, {
headers
})
return res.status >= 200 && res.status < 300
const res = await axios.post(apiUrl, defaultConfig, { headers })
return isHttpResSuccess(res)
}
default:
return false
}
} catch (error: any) {
logger.error(error)
return false
}
}
}
async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
const localFilePath = path.join(STORE_PATH, fileName)
@ -115,7 +113,13 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
return false
}
const { username, repo, branch, token, type } = syncConfig
if (type === 'gitee') {
const defaultConfig = {
branch,
message: uploadOrUpdateMsg(fileName),
content: readFileAsBase64(localFilePath)
}
switch (type) {
case 'gitee': {
const url = `https://gitee.com/api/v5/repos/${username}/${repo}/contents/${fileName}`
const shaRes = await axios.get(url, {
params: {
@ -123,25 +127,21 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
ref: branch
}
})
if (shaRes.status < 200 || shaRes.status > 300) {
if (!isHttpResSuccess(shaRes)) {
return false
}
const sha = shaRes.data.sha
const res = await axios.put(url, {
...defaultConfig,
owner: username,
repo,
path: fileName,
message: `update ${fileName} from PicList`,
content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
branch,
sha,
access_token: token
})
if (res.status >= 200 && res.status < 300) {
return true
return isHttpResSuccess(res)
}
return false
} else if (type === 'github') {
case 'github': {
const octokit = getOctokit(syncConfig)
const shaRes = await octokit.rest.repos.getContent({
owner: username,
@ -155,16 +155,15 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
const data = shaRes.data as any
const sha = data.sha
const res = await octokit.rest.repos.createOrUpdateFileContents({
...defaultConfig,
owner: username,
repo,
path: fileName,
message: `update ${fileName} from PicList`,
content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
branch,
sha
})
return res.status === 200
} else {
}
case 'gitea': {
const { endpoint = '' } = syncConfig
const apiUrl = `${endpoint}/api/v1/repos/${username}/${repo}/contents/${fileName}`
const headers = {
@ -173,48 +172,74 @@ async function updateLocalToRemote (syncConfig: ISyncConfig, fileName: string) {
const shaRes = await axios.get(apiUrl, {
headers
})
if (shaRes.status < 200 || shaRes.status > 300) {
if (!isHttpResSuccess(shaRes)) {
throw new Error('get sha failed')
}
const data = shaRes.data as any
const sha = data.sha
const res = await axios.put(apiUrl, {
message: `update ${fileName} from PicList`,
content: fs.readFileSync(localFilePath, { encoding: 'base64' }),
branch,
...defaultConfig,
sha
}, {
headers
})
return res.status >= 200 && res.status < 300
return isHttpResSuccess(res)
}
default:
return false
}
}
async function uploadFile (fileName: string[]): Promise<number> {
const syncConfig = getSyncConfig()
if (!isSyncConfigValidate(syncConfig)) {
logger.error('sync config is invalid')
return 0
}
const uploadFunc = async (file: string): Promise<number> => {
let result = false
try {
result = await updateLocalToRemote(syncConfig, file)
} catch (error: any) {
result = await uploadLocalToRemote(syncConfig, file)
}
logger.info(`upload ${file} ${result ? 'success' : 'failed'}`)
return result ? 1 : 0
}
let count = 0
for (const file of fileName) {
count += await uploadFunc(file)
}
return count
}
async function downloadAndWriteFile (url: string, localFilePath:string, config: any, isWriteJson = false) {
const res = await axios.get(url, config)
if (isHttpResSuccess(res)) {
await fs.writeFile(localFilePath, isWriteJson ? JSON.stringify(res.data, null, 2) : Buffer.from(res.data.content, 'base64'))
return true
}
return false
}
async function downloadRemoteToLocal (syncConfig: ISyncConfig, fileName: string) {
const localFilePath = path.join(STORE_PATH, fileName)
const { username, repo, branch, token, proxy, type } = syncConfig
if (type === 'gitee') {
try {
switch (type) {
case 'gitee': {
const url = `https://gitee.com/api/v5/repos/${username}/${repo}/contents/${fileName}`
const res = await axios.get(url, {
return downloadAndWriteFile(url, localFilePath, {
params: {
access_token: token,
ref: branch
}
})
if (res.status >= 200 && res.status < 300) {
const content = res.data.content
await fs.writeFile(localFilePath, Buffer.from(content, 'base64'))
return true
}
return false
} catch (error: any) {
logger.error(error)
return false
}
} else if (type === 'github') {
case 'github': {
const octokit = getOctokit(syncConfig)
try {
const res = await octokit.rest.repos.getContent({
owner: username,
repo,
@ -224,122 +249,47 @@ async function downloadRemoteToLocal (syncConfig: ISyncConfig, fileName: string)
if (res.status === 200) {
const data = res.data as any
const downloadUrl = data.download_url
const downloadRes = await axios.get(downloadUrl, {
httpsAgent: proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 1000,
rejectUnauthorized: false,
proxy: proxy.replace('127.0.0.1', 'localhost'),
scheduling: 'lifo'
})
: undefined
})
if (downloadRes.status >= 200 && downloadRes.status < 300) {
await fs.writeFile(localFilePath, JSON.stringify(downloadRes.data, null, 2))
return true
}
return downloadAndWriteFile(downloadUrl, localFilePath, {
httpsAgent: getProxyagent(proxy)
}, true)
}
return false
} catch (error: any) {
logger.error(error)
return false
}
} else {
case 'gitea': {
const { endpoint = '' } = syncConfig
const apiUrl = `${endpoint}/api/v1/repos/${username}/${repo}/contents/${fileName}`
try {
const headers = {
return downloadAndWriteFile(apiUrl, localFilePath, {
headers: {
Authorization: `token ${token}`
}
const res = await axios.get(apiUrl, {
headers,
},
params: {
ref: branch
}
})
if (res.status >= 200 && res.status < 300) {
const content = res.data.content
await fs.writeFile(localFilePath, Buffer.from(content, 'base64'))
return true
}
default:
return false
}
} catch (error: any) {
logger.error(error)
return false
}
}
}
async function uploadFile (fileName: string, all = false) {
async function downloadFile (fileName: string[]): Promise<number> {
const syncConfig = getSyncConfig()
if (!syncConfigValidator(syncConfig)) {
if (!isSyncConfigValidate(syncConfig)) {
logger.error('sync config is invalid')
return 0
}
let successCount = 0
if (all) {
for (const file of configFileNames) {
const result = await uploadFunc(syncConfig, file)
if (result) {
successCount++
}
}
logger.info(`upload all files at ${new Date().toLocaleString()}`)
return successCount
} else {
const ressult = await uploadFunc(syncConfig, fileName)
return ressult ? 1 : 0
}
}
async function uploadFunc (syncConfig: ISyncConfig, fileName: string) {
let result = false
try {
result = await updateLocalToRemote(syncConfig, fileName)
} catch (error: any) {
result = await uploadLocalToRemote(syncConfig, fileName)
}
if (!result) {
logger.error(`upload ${fileName} failed`)
return false
} else {
logger.info(`upload ${fileName} success`)
return true
}
}
async function downloadFile (fileName: string, all = false) {
const syncConfig = getSyncConfig()
if (!syncConfigValidator(syncConfig)) {
logger.error('sync config is invalid')
return 0
}
if (all) {
let successCount = 0
for (const file of configFileNames) {
const result = await downloadFunc(syncConfig, file)
if (result) {
successCount++
}
}
logger.info(`download all files at ${new Date().toLocaleString()}`)
return successCount
} else {
const result = await downloadFunc(syncConfig, fileName)
const downloadFunc = async (file: string): Promise<number> => {
const result = await downloadRemoteToLocal(syncConfig, file)
logger.info(`download ${file} ${result ? 'success' : 'failed'}`)
return result ? 1 : 0
}
}
async function downloadFunc (syncConfig: ISyncConfig, fileName: string) {
const result = await downloadRemoteToLocal(syncConfig, fileName)
if (!result) {
logger.error(`download ${fileName} failed`)
return false
} else {
logger.info(`download ${fileName} success`)
return true
}
return (await Promise.all(fileName.map(downloadFunc))).reduce((a, b) => a + b, 0)
}
export {