From 648ecb6e69e73d7611e02ef80c5fc3640340549f Mon Sep 17 00:00:00 2001 From: David Perry Date: Thu, 28 May 2020 11:44:37 -0400 Subject: [PATCH 1/4] Initial command line/TCP server script This fairly simple script leverages the Node API provided by CyberChef to allow performing operations on local filesystem objects without needing to copy them into and out of a browser window. It requires a recipe to be specified at runtime as a JSON file. Also includes a simple TCP server for use by other local processes. This server is hard-coded to use localhost in order to discourage production use of the script. --- cli.js | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 4 ++ 2 files changed, 161 insertions(+) create mode 100755 cli.js diff --git a/cli.js b/cli.js new file mode 100755 index 0000000000..0e3350c200 --- /dev/null +++ b/cli.js @@ -0,0 +1,157 @@ +#!/usr/bin/env node +/** + * @author Boolean263 [boolean263@protonmail.com] + * @copyright Crown Copyright 2020 + * @license Apache-2.0 + */ +'use strict'; + +const fs = require("fs"); +const path = require("path"); +const chef = require("cyberchef"); +const program = require("commander"); + +let slurpStream = (istream) => { // {{{1 + var ret = []; + var len = 0; + return new Promise(resolve => { + istream.on('readable', () => { + var chunk; + while ((chunk = istream.read()) !== null) { + ret.push(chunk); + len += chunk.length; + } + resolve(Buffer.concat(ret, len)); + }); + }); +}; // }}}1 + +let slurp = (fname) => { // {{{1 + let istream; + + if (fname === undefined || fname == '-') { + istream = process.stdin; + if (istream.isTTY) { + throw new Error("TTY input not supported"); + } + } + else { + istream = fs.createReadStream(fname, { flags: 'r' }); + } + return slurpStream(istream); +}; // }}}1 + +let getPort = (value, dummyPrevious) => { // {{{1 + // Get a valid port number from the command line + let ret = parseInt(value, 10); + if (ret < 1 || ret > 65535) { + throw new Error("invalid port number"); + } + return ret; +}; +// }}}1 + +///////// MAIN ///////// {{{1 + +program + .version(require('./package.json').version) + .usage('[options] [file [file ...]]') + .requiredOption('-r, --recipe-file ', + 'recipe JSON file') + .option('-l, --listen [port]', + 'listen on TCP port for data (default:random)', getPort, false) + .option('-o, --output ', + 'write result here (not for TCP; default:stdout)') + .parse(process.argv); + +// If we get no inputs and we aren't running a server, +// make stdin our single input +let inputs = program.args; +if (inputs.length == 0 && !program.listen) { + inputs = [ '-' ]; +} + +// Likewise stdout for our output +let ostream; +let outputIsDir = false; +if (program.output === undefined && !program.listen) { + ostream = process.stdout; +} +else if (inputs.length > 0) { + // See if our output is a directory + let st; + try { + st = fs.statSync(program.output); + outputIsDir = st.isDirectory(); + } + catch(err) { + if (err.code != 'ENOENT') throw err; + } + if (!outputIsDir) { + ostream = fs.createWriteStream(program.output); + } +} + +let recipe; +slurp(program.recipeFile).then((data) => { + recipe = JSON.parse(data); +}) +.catch((err) => { + console.error(`Error parsing recipe: ${err}`); + process.exit(1); +}) +.then(() => { + // First, deal with any files we want to read + for(let i of inputs) { + slurp(i).then((data) => { + let output = chef.bake(data, recipe); + if (outputIsDir) { + let outFileName = path.basename(i); + if (outFileName == '-') outFileName = 'from-stdin'; + ostream = fs.createWriteStream( + path.join(program.output, outFileName)); + } + ostream.write(output.presentAs("string", true)); + if (outputIsDir) ostream.end(); + }) + .catch((err) => { + console.error(err); + }); + } + + // Next, listen for TCP requests. + // This is intentionally hardcoded to localhost to discourage + // the use of this script as a production system. + if (program.listen) { + const net = require('net'); + const server = net.createServer((socket) => { + slurpStream(socket).then((data) => { + let output = chef.bake(data, recipe); + socket.write(output.presentAs("string", true)); + socket.end(); + }) + .catch((err) => { + console.error(err); + }); + }); + + // If no port given, let the OS choose one + if (program.listen === true) program.listen = 0; + server.listen(program.listen, '127.0.0.1') + .on('listening', () => { + console.log('Now listening on ' + + server.address().address + + ":" + server.address().port); + }); + + // Exit gracefully + process.on('SIGINT', () => { + console.log("Exiting"); + server.close(); + }); + } +}) +.catch((err) => { + console.error(err); + process.exit(2); +}) diff --git a/package.json b/package.json index a8f4198e32..0e57d647de 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,9 @@ "Firefox >= 38", "node >= 10" ], + "bin": { + "cyberchef": "./cli.js" + }, "devDependencies": { "@babel/core": "^7.8.7", "@babel/plugin-transform-runtime": "^7.8.3", @@ -98,6 +101,7 @@ "bson": "^4.0.3", "chi-squared": "^1.1.0", "codepage": "^1.14.0", + "commander": "^2.14.1", "core-js": "^3.6.4", "crypto-api": "^0.8.5", "crypto-js": "^4.0.0", From d11850beb9dc12d76f1f138b9311fc467e676c11 Mon Sep 17 00:00:00 2001 From: David Perry Date: Thu, 28 May 2020 15:48:24 -0400 Subject: [PATCH 2/4] Improve internal and external documentation --- cli.js | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/cli.js b/cli.js index 0e3350c200..c4b1a5c79e 100755 --- a/cli.js +++ b/cli.js @@ -7,11 +7,11 @@ 'use strict'; const fs = require("fs"); -const path = require("path"); -const chef = require("cyberchef"); -const program = require("commander"); + +///////// Helper Functions ///////// let slurpStream = (istream) => { // {{{1 + // Slurp the contents of a stream up into a Buffer to pass to CyberChef var ret = []; var len = 0; return new Promise(resolve => { @@ -27,12 +27,13 @@ let slurpStream = (istream) => { // {{{1 }; // }}}1 let slurp = (fname) => { // {{{1 + // Slurp the contents of a file (or stdin) into a Buffer let istream; if (fname === undefined || fname == '-') { istream = process.stdin; if (istream.isTTY) { - throw new Error("TTY input not supported"); + return Promise.reject(new Error("TTY input not supported")); } } else { @@ -53,16 +54,30 @@ let getPort = (value, dummyPrevious) => { // {{{1 ///////// MAIN ///////// {{{1 +const chef = require("cyberchef"); +const program = require("commander"); + program .version(require('./package.json').version) + .description('Bake data from files and/or TCP clients ' + + 'using a CyberChef recipe.') .usage('[options] [file [file ...]]') .requiredOption('-r, --recipe-file ', 'recipe JSON file') .option('-l, --listen [port]', - 'listen on TCP port for data (default:random)', getPort, false) + 'listen on TCP port for data (random if not given)', getPort, false) .option('-o, --output ', - 'write result here (not for TCP; default:stdout)') - .parse(process.argv); + 'where to write result (file input only; default:stdout)'); + +try { + program.exitOverride().parse(process.argv); +} +catch (e) { + if (e.code != 'commander.helpDisplayed') { + console.error("Run with '--help' for usage"); + } + process.exit(1); +} // If we get no inputs and we aren't running a server, // make stdin our single input @@ -73,6 +88,7 @@ if (inputs.length == 0 && !program.listen) { // Likewise stdout for our output let ostream; +let path; let outputIsDir = false; if (program.output === undefined && !program.listen) { ostream = process.stdout; @@ -85,20 +101,22 @@ else if (inputs.length > 0) { outputIsDir = st.isDirectory(); } catch(err) { + // We're fine if the output doesn't exist yet if (err.code != 'ENOENT') throw err; } if (!outputIsDir) { ostream = fs.createWriteStream(program.output); } } +if (outputIsDir) path = require("path"); let recipe; slurp(program.recipeFile).then((data) => { recipe = JSON.parse(data); }) .catch((err) => { - console.error(`Error parsing recipe: ${err}`); - process.exit(1); + console.error(`Error parsing recipe: ${err.message}`); + process.exit(2); }) .then(() => { // First, deal with any files we want to read @@ -113,9 +131,14 @@ slurp(program.recipeFile).then((data) => { } ostream.write(output.presentAs("string", true)); if (outputIsDir) ostream.end(); + }, + (err) => { + console.error(err.message); + process.exitCode = 2; }) .catch((err) => { - console.error(err); + console.error(err.message); + process.exitCode = 2; }); } @@ -135,7 +158,7 @@ slurp(program.recipeFile).then((data) => { }); }); - // If no port given, let the OS choose one + // If no port given by user, let the OS choose one if (program.listen === true) program.listen = 0; server.listen(program.listen, '127.0.0.1') .on('listening', () => { @@ -153,5 +176,5 @@ slurp(program.recipeFile).then((data) => { }) .catch((err) => { console.error(err); - process.exit(2); + process.exit(3); }) From 6805f90e05b1bdeb453be40f12c60466c1327be0 Mon Sep 17 00:00:00 2001 From: David Perry Date: Thu, 28 May 2020 16:16:55 -0400 Subject: [PATCH 3/4] Fix 109 `grunt lint` errors --- cli.js | 211 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 111 insertions(+), 100 deletions(-) diff --git a/cli.js b/cli.js index c4b1a5c79e..9581ff2dbe 100755 --- a/cli.js +++ b/cli.js @@ -4,19 +4,23 @@ * @copyright Crown Copyright 2020 * @license Apache-2.0 */ -'use strict'; +"use strict"; const fs = require("fs"); -///////// Helper Functions ///////// +/* * * * * Helper Functions * * * * */ -let slurpStream = (istream) => { // {{{1 - // Slurp the contents of a stream up into a Buffer to pass to CyberChef - var ret = []; - var len = 0; +/** + * Slurp the contents of a stream up into a Buffer to pass to CyberChef + * + * @param {Stream} istream + */ +const slurpStream = (istream) => { // {{{1 + const ret = []; + let len = 0; return new Promise(resolve => { - istream.on('readable', () => { - var chunk; + istream.on("readable", () => { + let chunk; while ((chunk = istream.read()) !== null) { ret.push(chunk); len += chunk.length; @@ -26,25 +30,35 @@ let slurpStream = (istream) => { // {{{1 }); }; // }}}1 -let slurp = (fname) => { // {{{1 - // Slurp the contents of a file (or stdin) into a Buffer +/** + * Slurp the contents of a file (or stdin) into a Buffer + * + * @param {String} fname + */ +const slurp = (fname) => { // {{{1 let istream; - if (fname === undefined || fname == '-') { + if (fname === undefined || fname === "-") { istream = process.stdin; if (istream.isTTY) { return Promise.reject(new Error("TTY input not supported")); } - } - else { - istream = fs.createReadStream(fname, { flags: 'r' }); + } else { + istream = fs.createReadStream(fname, { flags: "r" }); } return slurpStream(istream); }; // }}}1 -let getPort = (value, dummyPrevious) => { // {{{1 - // Get a valid port number from the command line - let ret = parseInt(value, 10); +/** + * Get a valid port number from the command line + * + * Used by commander + * + * @param {String} value + * @param {String} dummyPrevious + */ +const getPort = (value, dummyPrevious) => { // {{{1 + const ret = parseInt(value, 10); if (ret < 1 || ret > 65535) { throw new Error("invalid port number"); } @@ -52,29 +66,28 @@ let getPort = (value, dummyPrevious) => { // {{{1 }; // }}}1 -///////// MAIN ///////// {{{1 +/* * * * * MAIN * * * * */ // {{{1 const chef = require("cyberchef"); const program = require("commander"); program - .version(require('./package.json').version) - .description('Bake data from files and/or TCP clients ' - + 'using a CyberChef recipe.') - .usage('[options] [file [file ...]]') - .requiredOption('-r, --recipe-file ', - 'recipe JSON file') - .option('-l, --listen [port]', - 'listen on TCP port for data (random if not given)', getPort, false) - .option('-o, --output ', - 'where to write result (file input only; default:stdout)'); + .version(require("./package.json").version) + .description("Bake data from files and/or TCP clients " + + "using a CyberChef recipe.") + .usage("[options] [file [file ...]]") + .requiredOption("-r, --recipe-file ", + "recipe JSON file") + .option("-l, --listen [port]", + "listen on TCP port for data (random if not given)", getPort, false) + .option("-o, --output ", + "where to write result (file input only; default:stdout)"); try { program.exitOverride().parse(process.argv); -} -catch (e) { - if (e.code != 'commander.helpDisplayed') { - console.error("Run with '--help' for usage"); +} catch (e) { + if (e.code !== "commander.helpDisplayed") { + console.error("Run with \"--help\" for usage"); } process.exit(1); } @@ -82,8 +95,8 @@ catch (e) { // If we get no inputs and we aren't running a server, // make stdin our single input let inputs = program.args; -if (inputs.length == 0 && !program.listen) { - inputs = [ '-' ]; +if (inputs.length === 0 && !program.listen) { + inputs = ["-"]; } // Likewise stdout for our output @@ -92,17 +105,15 @@ let path; let outputIsDir = false; if (program.output === undefined && !program.listen) { ostream = process.stdout; -} -else if (inputs.length > 0) { +} else if (inputs.length > 0) { // See if our output is a directory let st; try { st = fs.statSync(program.output); outputIsDir = st.isDirectory(); - } - catch(err) { - // We're fine if the output doesn't exist yet - if (err.code != 'ENOENT') throw err; + } catch (err) { + // We"re fine if the output doesn"t exist yet + if (err.code !== "ENOENT") throw err; } if (!outputIsDir) { ostream = fs.createWriteStream(program.output); @@ -114,67 +125,67 @@ let recipe; slurp(program.recipeFile).then((data) => { recipe = JSON.parse(data); }) -.catch((err) => { - console.error(`Error parsing recipe: ${err.message}`); - process.exit(2); -}) -.then(() => { - // First, deal with any files we want to read - for(let i of inputs) { - slurp(i).then((data) => { - let output = chef.bake(data, recipe); - if (outputIsDir) { - let outFileName = path.basename(i); - if (outFileName == '-') outFileName = 'from-stdin'; - ostream = fs.createWriteStream( - path.join(program.output, outFileName)); - } - ostream.write(output.presentAs("string", true)); - if (outputIsDir) ostream.end(); - }, - (err) => { - console.error(err.message); - process.exitCode = 2; - }) - .catch((err) => { - console.error(err.message); - process.exitCode = 2; - }); - } - - // Next, listen for TCP requests. - // This is intentionally hardcoded to localhost to discourage - // the use of this script as a production system. - if (program.listen) { - const net = require('net'); - const server = net.createServer((socket) => { - slurpStream(socket).then((data) => { - let output = chef.bake(data, recipe); - socket.write(output.presentAs("string", true)); - socket.end(); + .catch((err) => { + console.error(`Error parsing recipe: ${err.message}`); + process.exit(2); + }) + .then(() => { + // First, deal with any files we want to read + for (const i of inputs) { + slurp(i).then((data) => { + const output = chef.bake(data, recipe); + if (outputIsDir) { + let outFileName = path.basename(i); + if (outFileName === "-") outFileName = "from-stdin"; + ostream = fs.createWriteStream( + path.join(program.output, outFileName)); + } + ostream.write(output.presentAs("string", true)); + if (outputIsDir) ostream.end(); + }, + (err) => { + console.error(err.message); + process.exitCode = 2; }) - .catch((err) => { - console.error(err); - }); - }); + .catch((err) => { + console.error(err.message); + process.exitCode = 2; + }); + } - // If no port given by user, let the OS choose one - if (program.listen === true) program.listen = 0; - server.listen(program.listen, '127.0.0.1') - .on('listening', () => { - console.log('Now listening on ' - + server.address().address - + ":" + server.address().port); + // Next, listen for TCP requests. + // This is intentionally hardcoded to localhost to discourage + // the use of this script as a production system. + if (program.listen) { + const net = require("net"); + const server = net.createServer((socket) => { + slurpStream(socket).then((data) => { + const output = chef.bake(data, recipe); + socket.write(output.presentAs("string", true)); + socket.end(); + }) + .catch((err) => { + console.error(err); + }); }); - // Exit gracefully - process.on('SIGINT', () => { - console.log("Exiting"); - server.close(); - }); - } -}) -.catch((err) => { - console.error(err); - process.exit(3); -}) + // If no port given by user, let the OS choose one + if (program.listen === true) program.listen = 0; + server.listen(program.listen, "127.0.0.1") + .on("listening", () => { + console.log("Now listening on " + + server.address().address + + ":" + server.address().port); + }); + + // Exit gracefully + process.on("SIGINT", () => { + console.log("Exiting"); + server.close(); + }); + } + }) + .catch((err) => { + console.error(err); + process.exit(3); + }); From 50a66c19100de2115d1da8b3ef62c088352b0917 Mon Sep 17 00:00:00 2001 From: David Perry Date: Fri, 29 May 2020 11:16:12 -0400 Subject: [PATCH 4/4] Address feedback so far in MR #1043 * Remove TCP server feature; this area is better served by the CyberChef-server project * Increase required version of commander to one which provides `requiredOption()` * Print the error message that is captured when failing to parse the command line Also use `fs.readFileSync()` to read the recipe file. --- cli.js | 123 +++++++++++++-------------------------------------- package.json | 2 +- 2 files changed, 32 insertions(+), 93 deletions(-) diff --git a/cli.js b/cli.js index 9581ff2dbe..a8f7771b72 100755 --- a/cli.js +++ b/cli.js @@ -49,26 +49,9 @@ const slurp = (fname) => { // {{{1 return slurpStream(istream); }; // }}}1 -/** - * Get a valid port number from the command line - * - * Used by commander - * - * @param {String} value - * @param {String} dummyPrevious - */ -const getPort = (value, dummyPrevious) => { // {{{1 - const ret = parseInt(value, 10); - if (ret < 1 || ret > 65535) { - throw new Error("invalid port number"); - } - return ret; -}; -// }}}1 - /* * * * * MAIN * * * * */ // {{{1 -const chef = require("cyberchef"); +const chef = require("./src/node/cjs.js"); const program = require("commander"); program @@ -78,24 +61,30 @@ program .usage("[options] [file [file ...]]") .requiredOption("-r, --recipe-file ", "recipe JSON file") - .option("-l, --listen [port]", - "listen on TCP port for data (random if not given)", getPort, false) .option("-o, --output ", "where to write result (file input only; default:stdout)"); try { program.exitOverride().parse(process.argv); } catch (e) { + console.error(e.message); if (e.code !== "commander.helpDisplayed") { console.error("Run with \"--help\" for usage"); } process.exit(1); } -// If we get no inputs and we aren't running a server, -// make stdin our single input +let recipe; +try { + recipe = JSON.parse(fs.readFileSync(program.recipeFile)); +} catch (err) { + console.error(err.message); + process.exit(3); +} + +// If we get no inputs, make stdin our single input let inputs = program.args; -if (inputs.length === 0 && !program.listen) { +if (inputs.length === 0) { inputs = ["-"]; } @@ -103,16 +92,16 @@ if (inputs.length === 0 && !program.listen) { let ostream; let path; let outputIsDir = false; -if (program.output === undefined && !program.listen) { +if (program.output === undefined) { ostream = process.stdout; -} else if (inputs.length > 0) { +} else { // See if our output is a directory let st; try { st = fs.statSync(program.output); outputIsDir = st.isDirectory(); } catch (err) { - // We"re fine if the output doesn"t exist yet + // We"re fine if the output doesn't exist yet if (err.code !== "ENOENT") throw err; } if (!outputIsDir) { @@ -121,71 +110,21 @@ if (program.output === undefined && !program.listen) { } if (outputIsDir) path = require("path"); -let recipe; -slurp(program.recipeFile).then((data) => { - recipe = JSON.parse(data); -}) - .catch((err) => { - console.error(`Error parsing recipe: ${err.message}`); - process.exit(2); - }) - .then(() => { - // First, deal with any files we want to read - for (const i of inputs) { - slurp(i).then((data) => { - const output = chef.bake(data, recipe); - if (outputIsDir) { - let outFileName = path.basename(i); - if (outFileName === "-") outFileName = "from-stdin"; - ostream = fs.createWriteStream( - path.join(program.output, outFileName)); - } - ostream.write(output.presentAs("string", true)); - if (outputIsDir) ostream.end(); - }, - (err) => { - console.error(err.message); - process.exitCode = 2; - }) - .catch((err) => { - console.error(err.message); - process.exitCode = 2; - }); - } - - // Next, listen for TCP requests. - // This is intentionally hardcoded to localhost to discourage - // the use of this script as a production system. - if (program.listen) { - const net = require("net"); - const server = net.createServer((socket) => { - slurpStream(socket).then((data) => { - const output = chef.bake(data, recipe); - socket.write(output.presentAs("string", true)); - socket.end(); - }) - .catch((err) => { - console.error(err); - }); - }); - - // If no port given by user, let the OS choose one - if (program.listen === true) program.listen = 0; - server.listen(program.listen, "127.0.0.1") - .on("listening", () => { - console.log("Now listening on " + - server.address().address + - ":" + server.address().port); - }); - - // Exit gracefully - process.on("SIGINT", () => { - console.log("Exiting"); - server.close(); - }); +// Deal with any files we want to read +for (const i of inputs) { + slurp(i).then((data) => { + const output = chef.bake(data, recipe); + if (outputIsDir) { + let outFileName = path.basename(i); + if (outFileName === "-") outFileName = "from-stdin"; + ostream = fs.createWriteStream( + path.join(program.output, outFileName)); } + ostream.write(output.presentAs("string", true)); + if (outputIsDir) ostream.end(); }) - .catch((err) => { - console.error(err); - process.exit(3); - }); + .catch((err) => { + console.error(err.message); + process.exitCode = 2; + }); +} diff --git a/package.json b/package.json index 0e57d647de..370fad3dd7 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "bson": "^4.0.3", "chi-squared": "^1.1.0", "codepage": "^1.14.0", - "commander": "^2.14.1", + "commander": "^5.1.0", "core-js": "^3.6.4", "crypto-api": "^0.8.5", "crypto-js": "^4.0.0",