From 75b230c8b28a21eac1f0c636c3423d5d0a054151 Mon Sep 17 00:00:00 2001 From: mrmara Date: Mon, 2 Sep 2024 14:59:59 +0200 Subject: [PATCH 1/4] Removed autochunking --- file.js | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/file.js b/file.js index 682a9c9..f7981cc 100644 --- a/file.js +++ b/file.js @@ -1,15 +1,36 @@ import fs from 'fs'; +import readline from 'readline'; import splitFile from 'split-file'; import md5File from 'md5-file'; import { log, spinner, yellow } from './helper.js'; -import { MIN_CHUNK_SIZE} from './.env'; +import { MIN_CHUNK_SIZE } from './.env'; + +const checkFolderAndPrompt = (dirname) => { + if (fs.existsSync(dirname) && dirname.endsWith('_chunks')) { + log('yellow', '\n\n- Chunks found, skipping rechunking and proceeding to sync...\n\n'); + return false; + } else { + log('yellow', '\n\n- Folder does not exist or does not end with "_chunks".\n\n'); + return true; + } +} const dumpManifest = (fPath, manifest) => { try { fs.writeFileSync(fPath, JSON.stringify(manifest, null, 2)); return JSON.stringify(manifest, null, 2); - } catch(e) { - log('red', 'ERROR: Cannot write chunk manifest...'); + } catch (e) { + log('red', '\nERROR: Cannot write chunk manifest...\n\n'); + return null; + } +} + +const loadManifest = (fPath) => { + try { + const data = fs.readFileSync(fPath, 'utf8'); + return data; + } catch (e) { + log('red', '\nERROR: Cannot read or parse chunk manifest...\n\n'); return null; } } @@ -24,23 +45,23 @@ const buildChunkManifest = async (chunkList, chunkSize) => { chunkSize, chunks: [] } - + //build manifest chunkList.forEach(chunk => { chunkManifest.chunks.push({ - chunk: chunk, + chunk: chunk, hash: md5File.sync(chunk) }); }); - + //dump manifest - let manifestFile = dumpManifest(`${destDir}/manifest.json`, chunkManifest); + let manifestFile = dumpManifest(`${destDir}/manifest.json`, chunkManifest); progress.stop(); return manifestFile; - } catch(e) { + } catch (e) { console.log(e) - log('red', 'ERROR: Cannot generate chunk manifest...'); + log('red', '\nERROR: Cannot generate chunk manifest...'); return null; } } @@ -62,13 +83,16 @@ export const chunkFile = async (file, cs, autocs) => { let fSize = fs.statSync(file).size; let chunkSize = autocs ? Math.max(Math.min(cs, fSize), MIN_CHUNK_SIZE) : cs; let chunkDir = file2chunkDir(file); + let manifestDir = file2Manifest(file); let progress = spinner(yellow('Chunking file... ')); - + if (!checkFolderAndPrompt(chunkDir)) { + return loadManifest(manifestDir); + } try { //create chinks dir and chink file log('yellow', `\n\n- Chunking file into ${chunkSize} byte chunks...\n\n`); progress.start(); - + makeDir(chunkDir); let chunkList = await splitFile.splitFileBySize(file, chunkSize, chunkDir); progress.stop(); From efad39cb3877818ea41929ba667b93b2dec772cb Mon Sep 17 00:00:00 2001 From: mrmara Date: Mon, 2 Sep 2024 15:02:17 +0200 Subject: [PATCH 2/4] RAM usage improvement --- helper.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/helper.js b/helper.js index febce15..c1e3998 100644 --- a/helper.js +++ b/helper.js @@ -54,10 +54,20 @@ export const getS3WriteLocalChunk = async (bucket, chunk) => { writeFile(chunk, chunk_bytes); } +// export const writeManifestChunks = (bucket, manifest) => { +// manifest.forEach(async chunk => { +// log('blue', `Chunk: ${chunk.chunk}\n`); +// await writeObject(bucket, chunk.chunk); +// log('blue', `${chunk.chunk} DONE\n`); +// }); +// } + export const writeManifestChunks = async (bucket, manifest) => { - manifest.forEach(async chunk => { + for (let chunk of manifest){ + log('blue', `Chunk: ${chunk.chunk}\n`); await writeObject(bucket, chunk.chunk); - }); + log('blue', `${chunk.chunk} DONE\n`); + } } // export const writeManifestChunks = async (bucket, manifest) => { From acdb0b4e2ff2a17077bff3a6026e788cb546f9be Mon Sep 17 00:00:00 2001 From: mrmara Date: Mon, 2 Sep 2024 15:03:43 +0200 Subject: [PATCH 3/4] added retry on failure --- s3.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3.js b/s3.js index 13afe64..b5d2eaf 100644 --- a/s3.js +++ b/s3.js @@ -69,8 +69,8 @@ export const writeObject = async (bucket, file) => { }); const response = await client.send(command); } catch (e) { - log("red", "ERROR: Cannot write an object to S3"); - process.exit(1); + log("red", "ERROR: Cannot write an object to S3, retrying..."); + writeObject(bucket, file); } }; From eda0cb5658bd9e70ea51f3ef09299589e507e2ad Mon Sep 17 00:00:00 2001 From: mrmara Date: Mon, 2 Sep 2024 15:03:52 +0200 Subject: [PATCH 4/4] Logs --- s3rsync.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/s3rsync.js b/s3rsync.js index eb27298..22e6770 100644 --- a/s3rsync.js +++ b/s3rsync.js @@ -45,10 +45,13 @@ const s3rsyncTo = async (file, bucket, size) => { let s3Manifest = await getManifest(bucket, file); if(s3Manifest == null) { + log('yellow', '\n\tS3 does not seem to contain a chunk memory, uploading for the first time...') await writeManifestChunks(bucket, localManifest.chunks); + log('yellow', "\nManifest Chunks Written\n") await writeObject(bucket, file2Manifest(file)) uploadedChunks = localManifest.chunks.length; } else { + log('yellow', '\nS3 seems to contain a chunk memory, uploading...') let {mergedLocalManifest, mergedS3Manifest} = mergeManifests(localManifest, s3Manifest) await writeManifestChunks(bucket, mergedLocalManifest); await deleteManifestChunks(bucket, mergedS3Manifest); @@ -59,7 +62,7 @@ const s3rsyncTo = async (file, bucket, size) => { log('yellow', `\nSuccessfully synched file://${file} to s3://${bucket}\n\n`); log('yellow', `Uploaded ${uploadedChunks}/${totalChunks} chunks - Saved ${(totalChunks - uploadedChunks) * chunkSize} bytes\n\n`); } catch(e) { - log('red', `ERROR: Cannot sync file://${file} to s3://${bucket}`); + log('red', `\n ERROR: Cannot sync file://${file} to s3://${bucket} due to ${e} \n\n`); } finally { //cleanup delChunks(file); @@ -102,7 +105,7 @@ const s3rsyncFrom = async (bucket, file, size) => { log('yellow', `\nSuccessfully synched s3://${bucket} to file://${file}\n\n`); log('yellow', `Downloaded ${uploadedChunks}/${totalChunks} chunks - Saved ${(totalChunks - uploadedChunks) * chunkSize} bytes\n\n`); } catch (e) { - log('red', `ERROR: Cannot sync s3://${bucket}/${file} to file://${file}`); + log('red', `\nERROR: Cannot sync s3://${bucket}/${file} to file://${file} due to ${e} \n`); }finally { delChunks(file); }