From 3d13fc78d68ac7a67b197c66aec12e4429969e10 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 13:07:04 -0700 Subject: [PATCH 01/10] Set up scaffolding, installed npm dependancies, created .gitignore file --- lab/.gitignore | 127 +++++++++++++++++++++++++++++++++++++++++++++++ lab/index.js | 0 lab/package.json | 19 +++++++ 3 files changed, 146 insertions(+) create mode 100644 lab/.gitignore create mode 100644 lab/index.js create mode 100644 lab/package.json diff --git a/lab/.gitignore b/lab/.gitignore new file mode 100644 index 0000000..fbd29bb --- /dev/null +++ b/lab/.gitignore @@ -0,0 +1,127 @@ + +# Created by https://www.gitignore.io/api/osx,vim,node,windows + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Typescript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Vim ### +# swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-v][a-z] +[._]sw[a-p] +# session +Session.vim +# temporary +.netrwhist +*~ +# auto-generated tag files +tags + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.gitignore.io/api/osx,vim,node,windows diff --git a/lab/index.js b/lab/index.js new file mode 100644 index 0000000..e69de29 diff --git a/lab/package.json b/lab/package.json new file mode 100644 index 0000000..e18610f --- /dev/null +++ b/lab/package.json @@ -0,0 +1,19 @@ +{ + "name": "lab", + "version": "1.0.0", + "description": "", + "main": "index.js", + "directories": { + "test": "test" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "chai": "^4.1.0", + "mocha": "^3.4.2" + } +} From d373b9ea0a9137dde40cdb8303d14cba273618b0 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 13:27:16 -0700 Subject: [PATCH 02/10] Wrote fileReader recurssive funciton draft. Still need to incorporate some methods of fs --- lab/lib/file-reader.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 lab/lib/file-reader.js diff --git a/lab/lib/file-reader.js b/lab/lib/file-reader.js new file mode 100644 index 0000000..ee6090a --- /dev/null +++ b/lab/lib/file-reader.js @@ -0,0 +1,16 @@ +'use strict'; + +const fs = require('fs'); +const errorCheck = ('../my-mods/error-check.js'); + +const fileReader = module.exports = function(path, callback, fileIndex=0) { + let fileName = ''; + + fs.readFile(fileName, function(err, asset) { + if(err) throw err; + callback(null, asset); + fileIndex += 1; + + if(fileName !== 'last file') fileReader(path, callback, fileIndex); + }); +}; From d047e7391c09dea06b94c4dab43f1308c1e7fe11 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 13:37:41 -0700 Subject: [PATCH 03/10] Created recurssive file reader that iterates through a directory and reads contents of each file --- lab/lib/file-reader.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lab/lib/file-reader.js b/lab/lib/file-reader.js index ee6090a..920ec85 100644 --- a/lab/lib/file-reader.js +++ b/lab/lib/file-reader.js @@ -4,13 +4,24 @@ const fs = require('fs'); const errorCheck = ('../my-mods/error-check.js'); const fileReader = module.exports = function(path, callback, fileIndex=0) { - let fileName = ''; - + let allFiles; + fs.readdir(`${__dirname}/../data`, (err, dir) => { + if(err) throw err; + allFiles = dir; + }); + let fileName = allFiles[fileIndex]; + fs.readFile(fileName, function(err, asset) { if(err) throw err; callback(null, asset); - fileIndex += 1; - if(fileName !== 'last file') fileReader(path, callback, fileIndex); + fileIndex += 1; + if(fileName !== allFiles[allFiles.length - 1]) fileReader(path, callback, fileIndex); }); }; + + +fs.readdir(`${__dirname}/../data`, (err, dir) => { + if(err) throw err; + console.log(dir); +}); From c39c376ceeb5b297f6a057bb5b9504bee0145bd8 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 14:17:56 -0700 Subject: [PATCH 04/10] Fixed bug in recurssive file reader --- lab/lib/file-reader.js | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/lab/lib/file-reader.js b/lab/lib/file-reader.js index 920ec85..027c4a2 100644 --- a/lab/lib/file-reader.js +++ b/lab/lib/file-reader.js @@ -1,27 +1,22 @@ 'use strict'; const fs = require('fs'); -const errorCheck = ('../my-mods/error-check.js'); const fileReader = module.exports = function(path, callback, fileIndex=0) { - let allFiles; - fs.readdir(`${__dirname}/../data`, (err, dir) => { - if(err) throw err; - allFiles = dir; - }); - let fileName = allFiles[fileIndex]; - fs.readFile(fileName, function(err, asset) { + fs.readdir(path, (err, dir) => { if(err) throw err; - callback(null, asset); - - fileIndex += 1; - if(fileName !== allFiles[allFiles.length - 1]) fileReader(path, callback, fileIndex); + asyncWrapper(dir); }); -}; + function asyncWrapper(file) { + let fileName = file[fileIndex]; + fs.readFile(`${path}/${fileName}`, function(err, asset) { + if(err) throw err; + callback(null, asset); -fs.readdir(`${__dirname}/../data`, (err, dir) => { - if(err) throw err; - console.log(dir); -}); + fileIndex += 1; + if(fileIndex < file.length) fileReader(path, callback, fileIndex); + }); + } +}; From 3826fb722286add4a03c4f410c181767841d37c5 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 15:27:14 -0700 Subject: [PATCH 05/10] Wrote tests for reading files and error handling. Now I'm trying to get them to work --- lab/data/one.txt | 1 + lab/data/three.txt | 1 + lab/data/two.txt | 1 + lab/lib/file-reader.js | 26 ++++++++++++++++---------- lab/test/file-reader-test.js | 24 ++++++++++++++++++++++++ 5 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 lab/data/one.txt create mode 100644 lab/data/three.txt create mode 100644 lab/data/two.txt create mode 100644 lab/test/file-reader-test.js diff --git a/lab/data/one.txt b/lab/data/one.txt new file mode 100644 index 0000000..2328dc3 --- /dev/null +++ b/lab/data/one.txt @@ -0,0 +1 @@ +This is a test for the first file diff --git a/lab/data/three.txt b/lab/data/three.txt new file mode 100644 index 0000000..90444ee --- /dev/null +++ b/lab/data/three.txt @@ -0,0 +1 @@ +Bacon grease has all sorts of useful purposes... diff --git a/lab/data/two.txt b/lab/data/two.txt new file mode 100644 index 0000000..cd54426 --- /dev/null +++ b/lab/data/two.txt @@ -0,0 +1 @@ +I really enjoy eating cake diff --git a/lab/lib/file-reader.js b/lab/lib/file-reader.js index 027c4a2..5b65ce0 100644 --- a/lab/lib/file-reader.js +++ b/lab/lib/file-reader.js @@ -2,21 +2,27 @@ const fs = require('fs'); -const fileReader = module.exports = function(path, callback, fileIndex=0) { +const fileReader = module.exports = function(path, callback) { - fs.readdir(path, (err, dir) => { - if(err) throw err; - asyncWrapper(dir); - }); - - function asyncWrapper(file) { - let fileName = file[fileIndex]; + function asyncWrapper(folder, callback, fileIndex=0) { + let fileName = folder[fileIndex]; fs.readFile(`${path}/${fileName}`, function(err, asset) { - if(err) throw err; + + if(err) return callback(err); + asset = new Buffer(Array.prototype.slice.call(asset, 0, 8)); + console.log(asset); callback(null, asset); fileIndex += 1; - if(fileIndex < file.length) fileReader(path, callback, fileIndex); + if(fileIndex < folder.length) asyncWrapper(folder, callback, fileIndex); }); } + + return fs.readdir(path, (err, dir) => { + if(err) return callback(err); + //This is a hack to get them to read in order.... + //May the coding gods have mercy on my soul... + dir.sort((a, b) => a.length - b.length); + asyncWrapper(dir, callback); + }); }; diff --git a/lab/test/file-reader-test.js b/lab/test/file-reader-test.js new file mode 100644 index 0000000..1c6eb91 --- /dev/null +++ b/lab/test/file-reader-test.js @@ -0,0 +1,24 @@ +'use strict'; + +const fileReader = require('../lib/file-reader.js'); +const expect = require('chai').expect; + + +describe('File Reader Module', () => { + describe('With improper file path', () => { + it('Should throw an error', (done) => { + fileReader(`${__dirname}/../not-a-file.txt`, (err) => { + expect(err).to.be.an('error'); + }); + done(); + }); + }); + + describe('With proper file path', () => { + it('Should throw console log the first 8 items in the buffer', () => { + fileReader(`${__dirname}/../data`, (err, asset) => { + expect(asset.toString()).to.equal(8); + }); + }); + }); +}); From b64c504eea97207da73e60b6c3a5a96eec29a3a4 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 15:32:59 -0700 Subject: [PATCH 06/10] Still trying to fix this test :( --- lab/lib/file-reader.js | 5 +++-- lab/test/file-reader-test.js | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lab/lib/file-reader.js b/lab/lib/file-reader.js index 5b65ce0..f39d8b6 100644 --- a/lab/lib/file-reader.js +++ b/lab/lib/file-reader.js @@ -9,12 +9,13 @@ const fileReader = module.exports = function(path, callback) { fs.readFile(`${path}/${fileName}`, function(err, asset) { if(err) return callback(err); - asset = new Buffer(Array.prototype.slice.call(asset, 0, 8)); + asset = Array.prototype.slice.call(asset, 0, 8); console.log(asset); callback(null, asset); fileIndex += 1; - if(fileIndex < folder.length) asyncWrapper(folder, callback, fileIndex); + if(fileIndex < folder.length) return asyncWrapper(folder, callback, fileIndex); + return; }); } diff --git a/lab/test/file-reader-test.js b/lab/test/file-reader-test.js index 1c6eb91..8367973 100644 --- a/lab/test/file-reader-test.js +++ b/lab/test/file-reader-test.js @@ -15,10 +15,9 @@ describe('File Reader Module', () => { }); describe('With proper file path', () => { - it('Should throw console log the first 8 items in the buffer', () => { + it('Should throw console log the first 8 items in the buffer', (done) => { fileReader(`${__dirname}/../data`, (err, asset) => { - expect(asset.toString()).to.equal(8); - }); + }) }); }); }); From 192737533fac39f4c0c124e3727d73d533bfefa2 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 16:04:14 -0700 Subject: [PATCH 07/10] Got it to pass the cursed test --- lab/lib/file-reader.js | 22 ++++++++++------------ lab/test/file-reader-test.js | 6 +++++- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lab/lib/file-reader.js b/lab/lib/file-reader.js index f39d8b6..fed9c4e 100644 --- a/lab/lib/file-reader.js +++ b/lab/lib/file-reader.js @@ -4,26 +4,24 @@ const fs = require('fs'); const fileReader = module.exports = function(path, callback) { - function asyncWrapper(folder, callback, fileIndex=0) { - let fileName = folder[fileIndex]; - fs.readFile(`${path}/${fileName}`, function(err, asset) { + function asyncWrapper(folder, callback, buffArray) { + let fileName = folder.pop(); + return fs.readFile(`${path}/${fileName}`, function(err, asset) { if(err) return callback(err); - asset = Array.prototype.slice.call(asset, 0, 8); - console.log(asset); - callback(null, asset); - - fileIndex += 1; - if(fileIndex < folder.length) return asyncWrapper(folder, callback, fileIndex); - return; + buffArray.push(asset.toString('hex', 0, 8)); + console.log(buffArray[buffArray.length -1]); + if(folder.length !== 0) return asyncWrapper(folder, callback, buffArray); + return callback(err, buffArray); }); } return fs.readdir(path, (err, dir) => { if(err) return callback(err); + let buffArray = []; //This is a hack to get them to read in order.... //May the coding gods have mercy on my soul... - dir.sort((a, b) => a.length - b.length); - asyncWrapper(dir, callback); + dir.sort((a, b) => a.length - b.length).reverse(); + asyncWrapper(dir, callback, buffArray); }); }; diff --git a/lab/test/file-reader-test.js b/lab/test/file-reader-test.js index 8367973..0afeb6d 100644 --- a/lab/test/file-reader-test.js +++ b/lab/test/file-reader-test.js @@ -17,7 +17,11 @@ describe('File Reader Module', () => { describe('With proper file path', () => { it('Should throw console log the first 8 items in the buffer', (done) => { fileReader(`${__dirname}/../data`, (err, asset) => { - }) + expect(asset[0]).to.equal('5468697320697320'); + expect(asset[1]).to.equal('49207265616c6c79'); + expect(asset[2]).to.equal('4261636f6e206772'); + done(); + }); }); }); }); From 410e1ab841f7f6f9357b0811c0aa93a4fb59afee Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 17:29:36 -0700 Subject: [PATCH 08/10] Refactored test --- README.md | 46 +----------------------------------- lab/lib/file-reader.js | 3 +-- lab/test/file-reader-test.js | 8 +++---- 3 files changed, 6 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 8cc8c11..18a68a7 100644 --- a/README.md +++ b/README.md @@ -1,45 +1 @@ -![CF](https://camo.githubusercontent.com/70edab54bba80edb7493cad3135e9606781cbb6b/687474703a2f2f692e696d6775722e636f6d2f377635415363382e706e67) 04: Parallel File Processing -=== - -## Submission Instructions - * fork this repository & create a new branch for your work - * write all of your code in a directory named `lab-` + `` **e.g.** `lab-susan` - * push to your repository - * submit a pull request to this repository - * submit a link to your PR in canvas - * write a question and observation on canvas - -## Learning Objectives - * students will be able to create asynchronous programs using the node.js callback pattern - * students will be able to read, write, and encode binary data using the Buffer class - * students will be able to utilize the built-in `fs` module for basic file system I/O operations - * students will be able to use `done` (provided by mocha.js) for creating asynchronous tests - -## Resources - * [fs module docs](https://nodejs.org/api/fs.html) - -## Requirements - -#### Configuration - -* include the following: - * **README.md** - contains documentation about your lab - * **.gitignore** - contains a robust `.gitignore` file - * **.eslintrc** - contains the provided `.eslintrc` file - * **.eslintignore** - contains the provided `.eslintignore` file - * **lib/** - contains your modules - * **test/** - contains your unit tests - * **assets/** - contains the text files used by the program - * **index.js** - contains main program file - -#### Feature Tasks - * for this assignment you will need to read three files and `console.log` the first 8 bytes ***(in hex)*** of each file - * regardless of file size, all three files should be read and logged in the order `'one.txt'`, `'two.txt'`, `'three.txt'` - -#### Testing - * create a test that guarantees that the files are logged in the proper order - * create a test that checks for improper file paths - -#### Documentation - * create a simple description of your project - * create a simple layer of documentation that describes how to use `done` in mocha callbacks +Kind of experimented with recursion on this one so forgive the messiness diff --git a/lab/lib/file-reader.js b/lab/lib/file-reader.js index fed9c4e..390456d 100644 --- a/lab/lib/file-reader.js +++ b/lab/lib/file-reader.js @@ -7,7 +7,6 @@ const fileReader = module.exports = function(path, callback) { function asyncWrapper(folder, callback, buffArray) { let fileName = folder.pop(); return fs.readFile(`${path}/${fileName}`, function(err, asset) { - if(err) return callback(err); buffArray.push(asset.toString('hex', 0, 8)); console.log(buffArray[buffArray.length -1]); @@ -19,7 +18,7 @@ const fileReader = module.exports = function(path, callback) { return fs.readdir(path, (err, dir) => { if(err) return callback(err); let buffArray = []; - //This is a hack to get them to read in order.... + //This is a hacky POS to get them to read in order.... //May the coding gods have mercy on my soul... dir.sort((a, b) => a.length - b.length).reverse(); asyncWrapper(dir, callback, buffArray); diff --git a/lab/test/file-reader-test.js b/lab/test/file-reader-test.js index 0afeb6d..3c26712 100644 --- a/lab/test/file-reader-test.js +++ b/lab/test/file-reader-test.js @@ -3,6 +3,8 @@ const fileReader = require('../lib/file-reader.js'); const expect = require('chai').expect; +//Conatains the first 8 bytes from files one, two, and three in order +let fileBytes = ['5468697320697320', '49207265616c6c79', '4261636f6e206772']; describe('File Reader Module', () => { describe('With improper file path', () => { @@ -15,11 +17,9 @@ describe('File Reader Module', () => { }); describe('With proper file path', () => { - it('Should throw console log the first 8 items in the buffer', (done) => { + it('Should console log files in order', (done) => { fileReader(`${__dirname}/../data`, (err, asset) => { - expect(asset[0]).to.equal('5468697320697320'); - expect(asset[1]).to.equal('49207265616c6c79'); - expect(asset[2]).to.equal('4261636f6e206772'); + asset.forEach((buff, ind) => expect(buff).to.equal(fileBytes[ind])); done(); }); }); From e302bcab03e46e74c196e69fe96a5627233c1fe6 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 17:42:25 -0700 Subject: [PATCH 09/10] Wrote a thorough read me --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 18a68a7..b4b4a2a 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -Kind of experimented with recursion on this one so forgive the messiness +the lib directory contains the following files: + +file-reader.js + +file-reader.js accepts a directory path and a callback as an argument. The directory is then read using fs. An array containing all files is passed to a wrapper function which then uses recursion to iterate through each file. Using the fs module, each file is read as a buffer which is then spliced to the length of 8 bytes. Each of these is console logged and pushed to an array. The containing array will then be returned as a parameter of the callback function. From f93c96d294b641d9dda1e3164548e4caa3f95b61 Mon Sep 17 00:00:00 2001 From: Edwin Del Rio Date: Wed, 19 Jul 2017 17:50:44 -0700 Subject: [PATCH 10/10] Added description for mocha done method --- README.md | 5 ----- lab/.eslintignore | 6 ++++++ lab/.eslintrc | 21 +++++++++++++++++++++ lab/README.md | 16 ++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) delete mode 100644 README.md create mode 100644 lab/.eslintignore create mode 100644 lab/.eslintrc create mode 100644 lab/README.md diff --git a/README.md b/README.md deleted file mode 100644 index b4b4a2a..0000000 --- a/README.md +++ /dev/null @@ -1,5 +0,0 @@ -the lib directory contains the following files: - -file-reader.js - -file-reader.js accepts a directory path and a callback as an argument. The directory is then read using fs. An array containing all files is passed to a wrapper function which then uses recursion to iterate through each file. Using the fs module, each file is read as a buffer which is then spliced to the length of 8 bytes. Each of these is console logged and pushed to an array. The containing array will then be returned as a parameter of the callback function. diff --git a/lab/.eslintignore b/lab/.eslintignore new file mode 100644 index 0000000..ae8da71 --- /dev/null +++ b/lab/.eslintignore @@ -0,0 +1,6 @@ + +**/node_modules/* +**/vendor/* +**/*.min.js +**/coverage/* +**/build/* diff --git a/lab/.eslintrc b/lab/.eslintrc new file mode 100644 index 0000000..8dc6807 --- /dev/null +++ b/lab/.eslintrc @@ -0,0 +1,21 @@ +{ + "rules": { + "no-console": "off", + "indent": [ "error", 2 ], + "quotes": [ "error", "single" ], + "semi": ["error", "always"], + "linebreak-style": [ "error", "unix" ] + }, + "env": { + "es6": true, + "node": true, + "mocha": true, + "jasmine": true + }, + "ecmaFeatures": { + "modules": true, + "experimentalObjectRestSpread": true, + "impliedStrict": true + }, + "extends": "eslint:recommended" +} diff --git a/lab/README.md b/lab/README.md new file mode 100644 index 0000000..0ba30da --- /dev/null +++ b/lab/README.md @@ -0,0 +1,16 @@ +the lib directory contains the following files: + +file-reader.js + +file-reader.js accepts a directory path and a callback as an argument. The directory is then read using fs. An array containing all files is passed to a wrapper function which then uses recursion to iterate through each file. Using the fs module, each file is read as a buffer which is then spliced to the length of 8 bytes. Each of these is console logged and pushed to an array. The containing array will then be returned as a parameter of the callback function. + +the test directory contains the following files: + +file-reader-test.js + +file-reader-test.js contains a test for file-reader.js. It tests if the file-reader module throws an error when an incorrect path is passed. Additionally the test check if file-reader logs the files in the specified order. + +-------------------------------------------- + +--done() explanation +Done is used to prevent the test from completing before an asynchronous operation is done executing. done should be placed in the call back of the asynchronous operations call back. Once the operation is called, the done method will be called. done must be passed as a parameter for the callback of the it method executing the test.