Feature/no autochunking ram usage improvement

file.js

@@ -1,15 +1,36 @@
 import fs from 'fs';
+import readline from 'readline';
 import splitFile from 'split-file';
 import md5File from 'md5-file';
 import { log, spinner, yellow } from './helper.js';
-import { MIN_CHUNK_SIZE}  from './.env';
+import { MIN_CHUNK_SIZE } from './.env';
+
+const checkFolderAndPrompt = (dirname) => {
+    if (fs.existsSync(dirname) && dirname.endsWith('_chunks')) {
+        log('yellow', '\n\n- Chunks found, skipping rechunking and proceeding to sync...\n\n');
+        return false;
+    } else {
+        log('yellow', '\n\n- Folder does not exist or does not end with "_chunks".\n\n');
+        return true;
+    }
+}
 
 const dumpManifest = (fPath, manifest) => {
     try {
         fs.writeFileSync(fPath, JSON.stringify(manifest, null, 2));
         return JSON.stringify(manifest, null, 2);
-    } catch(e) {
-        log('red', 'ERROR: Cannot write chunk manifest...');
+    } catch (e) {
+        log('red', '\nERROR: Cannot write chunk manifest...\n\n');
+        return null;
+    }
+}
+
+const loadManifest = (fPath) => {
+    try {
+        const data = fs.readFileSync(fPath, 'utf8');
+        return data;
+    } catch (e) {
+        log('red', '\nERROR: Cannot read or parse chunk manifest...\n\n');
         return null;
     }
 }
@@ -24,23 +45,23 @@ const buildChunkManifest = async (chunkList, chunkSize) => {
             chunkSize,
             chunks: []
         }
-    
+
         //build manifest
         chunkList.forEach(chunk => {
             chunkManifest.chunks.push({
-                chunk: chunk, 
+                chunk: chunk,
                 hash: md5File.sync(chunk)
             });
         });
-        
+
         //dump manifest
-        let manifestFile = dumpManifest(`${destDir}/manifest.json`, chunkManifest); 
+        let manifestFile = dumpManifest(`${destDir}/manifest.json`, chunkManifest);
         progress.stop();
 
         return manifestFile;
-    } catch(e) {
+    } catch (e) {
         console.log(e)
-        log('red', 'ERROR: Cannot generate chunk manifest...');
+        log('red', '\nERROR: Cannot generate chunk manifest...');
         return null;
     }
 }
@@ -62,13 +83,16 @@ export const chunkFile = async (file, cs, autocs) => {
     let fSize = fs.statSync(file).size;
     let chunkSize = autocs ? Math.max(Math.min(cs, fSize), MIN_CHUNK_SIZE) : cs;
     let chunkDir = file2chunkDir(file);
+    let manifestDir = file2Manifest(file);
     let progress = spinner(yellow('Chunking file...   '));
-
+    if (!checkFolderAndPrompt(chunkDir)) { 
+        return loadManifest(manifestDir);
+    }
     try {
         //create chinks dir and chink file
         log('yellow', `\n\n- Chunking file into ${chunkSize} byte chunks...\n\n`);
         progress.start();
-        
+
         makeDir(chunkDir);
         let chunkList = await splitFile.splitFileBySize(file, chunkSize, chunkDir);
         progress.stop();

helper.js

@@ -54,10 +54,20 @@ export const getS3WriteLocalChunk = async (bucket, chunk) => {
     writeFile(chunk, chunk_bytes);
 }
 
+// export const writeManifestChunks = (bucket, manifest) => {
+//     manifest.forEach(async chunk => {
+//         log('blue', `Chunk: ${chunk.chunk}\n`);
+//         await writeObject(bucket, chunk.chunk);
+//         log('blue', `${chunk.chunk} DONE\n`);
+//     });
+// }
+
 export const writeManifestChunks = async (bucket, manifest) => {
-    manifest.forEach(async chunk => {
+    for (let chunk of manifest){
+        log('blue', `Chunk: ${chunk.chunk}\n`);
         await writeObject(bucket, chunk.chunk);
-    });
+        log('blue', `${chunk.chunk} DONE\n`);
+    }
 }
 
 // export const writeManifestChunks = async (bucket, manifest) => {

s3.js

@@ -69,8 +69,8 @@ export const writeObject = async (bucket, file) => {
     });
     const response = await client.send(command);
   } catch (e) {
-    log("red", "ERROR: Cannot write an object to S3");
-    process.exit(1);
+    log("red", "ERROR: Cannot write an object to S3, retrying...");
+    writeObject(bucket, file);
   }
 };
 

s3rsync.js

@@ -45,10 +45,13 @@ const s3rsyncTo = async (file, bucket, size) => {
         let s3Manifest = await getManifest(bucket, file);
 
         if(s3Manifest == null) {
+            log('yellow', '\n\tS3 does not seem to contain a chunk memory, uploading for the first time...')
             await writeManifestChunks(bucket, localManifest.chunks);
+            log('yellow', "\nManifest Chunks Written\n")
             await writeObject(bucket, file2Manifest(file))
             uploadedChunks = localManifest.chunks.length;
         } else {
+            log('yellow', '\nS3 seems to contain a chunk memory, uploading...')
             let {mergedLocalManifest, mergedS3Manifest} = mergeManifests(localManifest, s3Manifest)
             await writeManifestChunks(bucket, mergedLocalManifest);
             await deleteManifestChunks(bucket, mergedS3Manifest);
@@ -59,7 +62,7 @@ const s3rsyncTo = async (file, bucket, size) => {
         log('yellow', `\nSuccessfully synched file://${file} to s3://${bucket}\n\n`);
         log('yellow', `Uploaded ${uploadedChunks}/${totalChunks} chunks - Saved ${(totalChunks - uploadedChunks) * chunkSize} bytes\n\n`);
     } catch(e) {
-        log('red', `ERROR: Cannot sync file://${file} to s3://${bucket}`);
+        log('red', `\n ERROR: Cannot sync file://${file} to s3://${bucket} due to ${e} \n\n`);
     } finally {
         //cleanup
         delChunks(file);
@@ -102,7 +105,7 @@ const s3rsyncFrom = async (bucket, file, size) => {
         log('yellow', `\nSuccessfully synched s3://${bucket} to file://${file}\n\n`);
         log('yellow', `Downloaded ${uploadedChunks}/${totalChunks} chunks - Saved ${(totalChunks - uploadedChunks) * chunkSize} bytes\n\n`);
     } catch (e) {
-        log('red', `ERROR: Cannot sync s3://${bucket}/${file} to file://${file}`);
+        log('red', `\nERROR: Cannot sync s3://${bucket}/${file} to file://${file} due to ${e} \n`);
     }finally {
         delChunks(file);
     }