From d3592ae388eb8570572783ed6efb5ab42af8cccc Mon Sep 17 00:00:00 2001 From: kyriakos Date: Sat, 18 Feb 2017 01:32:52 +0200 Subject: [PATCH 01/11] app.js promisified. generators are used. --- app.js | 85 ++++++++++++++++++++++++++-------------------------- package.json | 5 ++-- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/app.js b/app.js index 002f017..8184a43 100644 --- a/app.js +++ b/app.js @@ -1,6 +1,8 @@ const tar = require('tar-stream'); const fs = require('fs'); const commandLineArgs = require('command-line-args'); +const Promise = require('bluebird'); +const coroutine = Promise.coroutine; /** @@ -38,12 +40,22 @@ const server = require('./server')( */ const javaBox = require('./javaBox'); + /** - * Specifying the server and javaBox control flow + * Given a .tar pack and a file, return a promise to insert the file into the tar. + * + * @param file {Object}: the file to be inserted into the tar. + * @param file.name {string}: the filename. + * @param file.data {string}: data the file contains. + * @param tarPack {Object}: A tar-stream to be used for inserting the file into the tar. + * @return {Promise}: a promise to insert the file. */ -server.on('runJunit', runJunit); -server.on('runJava', runJava); -javaBox.on('result', giveFeedBack); +const insertFileToTar = coroutine(function*(file, tarPack) { + return new Promise(function (reject, resolve) { + tarPack.entry({name: file.name}, file.data); // no need for callback there + resolve(); + }); +}); /** @@ -52,30 +64,23 @@ javaBox.on('result', giveFeedBack); * * @param messageId {String}: id of the given message/request. * @param main {String}: entry point class name. - * @param files [{name: {String}, data: {String]: array of objects with filename and its content + * @param files {{name: String, data: String}[]}: array of objects with filename and its content * @param timeLimitCompileMs {Number}: Compilation timeout. * @param timeLimitExecutionMs {Number}: Execution timeout. */ -function runJava(messageId, main, files, timeLimitCompileMs, timeLimitExecutionMs) { - - let filesToAdd = files.length; +const runJava = coroutine(function*(messageId, main, files, timeLimitCompileMs, timeLimitExecutionMs) { const pack = tar.pack(); - - const tryTarAndRun = () => { - - filesToAdd--; - - if (filesToAdd === 0) { - const tarBuffer = pack.read(); - javaBox.emit('runJava', messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); - } - }; - - files.forEach((file) => { - pack.entry({ name: file.name }, file.data, tryTarAndRun); + const fileEntriesPromises = []; + for (let file in files) { + fileEntriesPromises.push(yield insertFileToTar(file, pack)); + } + + Promise.all(fileEntriesPromises).then(() => { + const tarBuffer = pack.read(); + javaBox.emit('runJava', messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); }); -} +}); /** * Given code to execute and to test and info about it, @@ -87,30 +92,23 @@ function runJava(messageId, main, files, timeLimitCompileMs, timeLimitExecutionM * @param timeLimitCompileMs {Number}: Compilation timeout. * @param timeLimitExecutionMs {Number}: Execution timeout. */ -function runJunit(messageId, tests, files, timeLimitCompileMs, timeLimitExecutionMs) { +const runJunit = coroutine(function *(messageId, tests, files, timeLimitCompileMs, timeLimitExecutionMs) { - let filesToAdd = files.length + tests.length; + files = files.concat(tests); const pack = tar.pack(); + const fileEntriesPromises = []; + for (let file in files) { + fileEntriesPromises.push(yield insertFileToTar(file, pack)); - const tryTarAndRun = () => { + } - filesToAdd--; - - if (filesToAdd === 0) { - const tarBuffer = pack.read(); - javaBox.emit('runJunit', messageId, tests, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); - } - }; - - files.forEach((file) => { - pack.entry({ name: file.name }, file.data, tryTarAndRun); + Promise.all(fileEntriesPromises).then(() => { + const tarBuffer = pack.read(); + javaBox.emit('runJunit', messageId, tests, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); }); - tests.forEach((test) => { - pack.entry({ name: test.name }, test.data, tryTarAndRun); - }); -} +}); /** * Given the feedback object passes it to the server. @@ -121,6 +119,9 @@ function giveFeedBack(feedback) { server.emit('result', feedback); } - - - +/** + * Specifying the server and javaBox control flow + */ +server.on('runJunit', runJunit); +server.on('runJava', runJava); +javaBox.on('result', giveFeedBack); diff --git a/package.json b/package.json index 368bc85..b829294 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "javabox", - "version": "1.0.0", + "version": "1.1.0", "description": "Server to procede java code", "main": "index.js", "scripts": { @@ -15,6 +15,7 @@ "concat-stream": "^1.5.2", "dockerode": "^2.3.1", "json-socket": "^0.2.0", - "tar-stream": "^1.5.2" + "tar-stream": "^1.5.2", + "bluebird": "^2.8.2" } } From ca13ca187b63e1e9f4d7ee8ec00a5ec2cdd93327 Mon Sep 17 00:00:00 2001 From: kyriakos Date: Tue, 21 Feb 2017 02:51:35 +0200 Subject: [PATCH 02/11] created queue.js for modularity. Promises are used (in queue.js). First layer of generators is used in server.js --- .gitignore | 1 + app.js | 46 ++++++------------ queue.js | 71 ++++++++++++++++++++++++++++ server.js | 133 +++++++++++++++++++++++++++++------------------------ 4 files changed, 160 insertions(+), 91 deletions(-) create mode 100644 queue.js diff --git a/.gitignore b/.gitignore index 3f0a142..015dacc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ + # Compiled source # ################### *.com diff --git a/app.js b/app.js index 8184a43..2dd3686 100644 --- a/app.js +++ b/app.js @@ -33,31 +33,14 @@ const server = require('./server')( commandLine.mongoAddress, commandLine.mongoCollection, commandLine.defaultConcurrency, - commandLine.maxConcurrency); + commandLine.maxConcurrency +); /** * Object controlling docker specified in javaBox.js * @type {EventEmitter} */ const javaBox = require('./javaBox'); - -/** - * Given a .tar pack and a file, return a promise to insert the file into the tar. - * - * @param file {Object}: the file to be inserted into the tar. - * @param file.name {string}: the filename. - * @param file.data {string}: data the file contains. - * @param tarPack {Object}: A tar-stream to be used for inserting the file into the tar. - * @return {Promise}: a promise to insert the file. - */ -const insertFileToTar = coroutine(function*(file, tarPack) { - return new Promise(function (reject, resolve) { - tarPack.entry({name: file.name}, file.data); // no need for callback there - resolve(); - }); -}); - - /** * Given code to execute and info about it, * creates a tar with the code and passed the info and the tar to javaBox. @@ -71,17 +54,18 @@ const insertFileToTar = coroutine(function*(file, tarPack) { const runJava = coroutine(function*(messageId, main, files, timeLimitCompileMs, timeLimitExecutionMs) { const pack = tar.pack(); - const fileEntriesPromises = []; - for (let file in files) { - fileEntriesPromises.push(yield insertFileToTar(file, pack)); - } + const packEntry = Promise.promisify(pack.entry, {context: pack}); + + Promise.map(files, function (file) { + return packEntry({name: file.name}, file.data); - Promise.all(fileEntriesPromises).then(() => { + }).then(function () { const tarBuffer = pack.read(); javaBox.emit('runJava', messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + }); -}); +}); /** * Given code to execute and to test and info about it, * creates a tar with the code and passed the info and the tar to javaBox. @@ -97,15 +81,15 @@ const runJunit = coroutine(function *(messageId, tests, files, timeLimitCompileM files = files.concat(tests); const pack = tar.pack(); - const fileEntriesPromises = []; - for (let file in files) { - fileEntriesPromises.push(yield insertFileToTar(file, pack)); + const packEntry = Promise.promisify(pack.entry, {context: pack}); - } + Promise.map(files, function (file) { + return packEntry({name: file.name}, file.data); - Promise.all(fileEntriesPromises).then(() => { + }).then(function () { const tarBuffer = pack.read(); - javaBox.emit('runJunit', messageId, tests, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + javaBox.emit('runJava', messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + }); }); diff --git a/queue.js b/queue.js new file mode 100644 index 0000000..f28beb4 --- /dev/null +++ b/queue.js @@ -0,0 +1,71 @@ +const Agenda = require('agenda'); + +/** + * @typedef {Agenda} queue + */ +const queue = new Agenda(); + +/** + * @typedef {Object} JsonSocket + * @typedef {Object} Request + * + * @typedef {Object} StoredMessage + * @type StoredMessage.socket: {JsonSocket} + * @type StoredMessage.request: {Request} + * @type StoredMessage.done: Function + */ + +/** + * Initialize queue, connect to the database and define actions + * @param queueParams {Object} + * @param queueParams.mongoFullAddress {string} + * @param queueParams.mongoCollection {string} + * @param queueParams.maxConcurrency {int} + * @param queueParams.defaultConcurrency {int} + * @param processMessageJob {Function}: A callback for Agenda.now() for 'process_message' action + * + * @return {Promise}: A promise for starting the queue + */ +queue.prototype.initialize = function (queueParams, processMessageJob) { + + return new Promise((resolve, reject) => { + + queue.database(queueParams.mongoFullAddress, queueParams.mongoCollection) + .defaultConcurrency(queueParams.defaultConcurrency) + .maxConcurrency(queueParams.maxConcurrency); + + queue.define('process_message', processMessageJob); + + queue.on('ready', resolve); // use .then(queue.start()) + queue.on('error', reject); + }); +}; + +queue.prototype.addMessage = function (request) { + + const messageId = createMessageId(request); + const jobData = { + messageId: messageId, + request: request + }; + queue.now('process_message', jobData); + return messageId; +}; + +/** + * Given a clientId creates an unique messageId. + * + * @param message {Object}: connection message + * @param message.clientId: client UID + * @return {string} clientId + ::: + current date in milliseconds + */ +function createMessageId(message) { + return `${message.clientId}:::${Date.now()}`; +} + +module.exports = queue; + + + + + diff --git a/server.js b/server.js index 32fc7bd..1db1332 100644 --- a/server.js +++ b/server.js @@ -1,7 +1,7 @@ const net = require('net'); const JsonSocket = require('json-socket'); -const Agenda = require('agenda'); - +const Promise = require('bluebird'); +const coroutine = Promise.coroutine; /** * Contains preceding messages and its related information as such @@ -13,17 +13,23 @@ const messages = {}; * Agenda object, will be initialised in init function. * @type {Agenda} */ -const queue = new Agenda(); +const queue = require('./queue'); + +/** + * @typedef {Object} Server + */ /** * Tcp server. * @type {Server} */ const server = net.createServer(); +//TODO: is this correct? +server.listen = Promise.promisify(server.listen); + /** - * Initialises server object, connects to mongo database, - * and starts the server on given port. + * Initialises server and queue for storing requests. * * @param port {Number}: tcp port number. * @param mongoAddress {String}: mongo database url. @@ -33,20 +39,27 @@ const server = net.createServer(); * * @return {Server}: server event emitter object () */ -function init(port, mongoAddress, mongoCollection, defaultConcurrency, maxConcurrency) { +const init = coroutine(function*(port, mongoAddress, mongoCollection, defaultConcurrency, maxConcurrency) { const mongoFullAddress = `mongodb://${mongoAddress}`; - queue.database(mongoFullAddress, mongoCollection) - .defaultConcurrency(defaultConcurrency) - .maxConcurrency(maxConcurrency); - queue.define('process_message', processMessageJob); - queue.on('ready', () => queue.start()); + const queueInitParams = { + mongoFullAddress, + mongoCollection, + maxConcurrency, + defaultConcurrency + }; + yield queue.initialize(queueInitParams, processMessageJob); + queue.start(); + server.on('connection', initSocket); server.on('result', sendResult); - server.listen(port); + + yield server.listen(port); + console.log(`server listening on port ${port}...`); + return server; -} +}); /** @@ -58,34 +71,59 @@ function init(port, mongoAddress, mongoCollection, defaultConcurrency, maxConcur function initSocket(connection) { const socket = new JsonSocket(connection); + socket.on('message', addMessageInQueue.bind(null, socket)); +} - const putMessageInQueue = (request) => { - - const messageId = createMessageId(request); - const jobData = { - messageId: messageId, - request: request - }; - messages[messageId] = {socket: socket, request: request}; - queue.now('process_message', jobData); +function addMessageInQueue(socket, request) { - }; - socket.on('message', putMessageInQueue); + const messageId = queue.addMessage(request); + messages[messageId] = {socket: socket, request: request}; } -/** - * Given a job and done function, stores done function, - * reads jobs attributes and call parseRequestAndSend function on them. - * @param job {Object}: Agenda job. - * @param done {Object}: job terminating function. - */ + function processMessageJob(job, done) { const request = job.attrs.data.request; const messageId = job.attrs.data.messageId; messages[messageId]['done'] = done; - parseRequestAndSend(request, messageId); + + if (isRequestValid(request)) { + executeRequest(request, messageId); + + } else { + sendResultForBadRequest(messageId); + } + + done(); // not needed, useful for async code +} + +function isRequestValid(request) { + const main = request.submission.main; + const files = request.submission.files; + const tests = request.submission.tests; + const timeLimitCompileMs = request.compileTimeoutMs; + const timeLimitExecutionMs = request.executionTimeoutMs; + const charactersMaxLength = request.charactersMaxLength; + + if (!((main || tests) && files && timeLimitCompileMs && timeLimitExecutionMs && charactersMaxLength)) { // absence of parameter + return false; + } else if (!(timeLimitCompileMs > 0 && timeLimitExecutionMs > 0)) { // illogical values for timeout + return false; + } else { + return true; + } +} + +function sendResultForBadRequest(messageId) { + const message = messages[messageId]; + const socket = message.socket; + const clientId = message.request.clientId; + try { + socket.sendEndMessage({}); //TODO: define output for bad requests + } catch (e) { + console.log(`73: Socket with client ${clientId} closed before sending result back.`); + } } /** @@ -94,26 +132,13 @@ function processMessageJob(job, done) { * @param request {Object}, request object as specified in readme. * @param messageId {String}, id of the given message/request. */ -function parseRequestAndSend(request, messageId) { +function executeRequest(request, messageId) { - const clientId = request.clientId; const main = request.submission.main; const files = request.submission.files; const tests = request.submission.tests; const timeLimitCompileMs = request.compileTimeoutMs; const timeLimitExecutionMs = request.executionTimeoutMs; - const charactersMaxLength = request.charactersMaxLength; - - - if (!((main || tests) && files && timeLimitCompileMs && timeLimitExecutionMs && charactersMaxLength)) { - sendResult({clientId: clientId}); - return; - } - - if (!(timeLimitCompileMs > 0 && timeLimitExecutionMs > 0)) { // illogical values for timeout - sendResult({clientId: clientId}); - return; - } if (tests && Array.isArray(tests) && tests.length > 0) { // run junit tests @@ -126,22 +151,10 @@ function parseRequestAndSend(request, messageId) { } } - -/** - * Given a clientId creates an unique messageId. - * - * @param message {string} Some random clientId - * @return {string} clientId + ::: + current date in milliseconds - */ -function createMessageId(message) { - - return `${message.clientId}:::${Date.now()}`; -} - - /** * Correct and truncates the feedback and sends it back. - * @param feedback {Object}, feedback/result object as specified in readme or javaBox.js + * @param feedback {Object}: feedback/result object as specified in readme or javaBox.js + * @param feedback.messageId {string} */ function sendResult(feedback) { @@ -163,7 +176,7 @@ function sendResult(feedback) { try { socket.sendEndMessage(feedback); } catch (e) { - console.log('socket closed before sending result back'); + console.log(`73: Socket with client ${clientId} closed before sending result back.`); } done(); From 9804c851bcbe60733341f9591edb9e8f716dae8b Mon Sep 17 00:00:00 2001 From: kyriakos Date: Tue, 21 Feb 2017 14:03:33 +0200 Subject: [PATCH 03/11] Added JsDocs --- server.js | 50 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/server.js b/server.js index 1db1332..fd7a981 100644 --- a/server.js +++ b/server.js @@ -4,14 +4,23 @@ const Promise = require('bluebird'); const coroutine = Promise.coroutine; /** - * Contains preceding messages and its related information as such - * {messageId = {socket: JsonSocket, request: Object, done: Function}, ...}. + * @typdef {Object} JsonSocket + * @typedef {Object} Request + */ + +/** + * "Dictionary" containing preceding messages and its related information as such + * "messageId" {String} -> { + * socket: {JsonSocket}, + * request: {Request}, + * done: Function + * } * @type {Object} */ const messages = {}; /** * Agenda object, will be initialised in init function. - * @type {Agenda} + * @type {Queue} */ const queue = require('./queue'); @@ -63,9 +72,9 @@ const init = coroutine(function*(port, mongoAddress, mongoCollection, defaultCon /** - * Given a connection creates putMessageInQueue function, - * creates a json socket and initialises it to accept - * the request with putMessageInQueue function. + * When a new connection is established, + * creates a new socket + * and inserts request object into the queue for execution. * @param connection {Object}: A tcp connection. */ function initSocket(connection) { @@ -74,13 +83,22 @@ function initSocket(connection) { socket.on('message', addMessageInQueue.bind(null, socket)); } +/** + * Adds a request from a socket to the queue for execution. + * @param socket: {JsonSocket} + * @param request: {Request} + */ function addMessageInQueue(socket, request) { const messageId = queue.addMessage(request); messages[messageId] = {socket: socket, request: request}; } - +/** + * Process the execution of a job-request. + * @param job {Object}: An `Agenda` job. + * @param done {Function}: A callback to execute when finished, in case the function is asynchronous. + */ function processMessageJob(job, done) { const request = job.attrs.data.request; @@ -97,7 +115,12 @@ function processMessageJob(job, done) { done(); // not needed, useful for async code } - +/** + * Checks whether a request is valid or not, given the request specification + *(See: https://github.com/ASQ-USI/asq-java-q-backend/blob/master/README.md#communication-api) + * @param request {Request}: The request to be checked. + * @return {boolean} : Returns `false` if request has one or more missing properties or illogical values, `true` otherwise. + */ function isRequestValid(request) { const main = request.submission.main; const files = request.submission.files; @@ -115,6 +138,10 @@ function isRequestValid(request) { } } +/** + * Sends back to corresponding client error code and closes connection. + * @param messageId {String}: The id in the queue of the bad request message. + */ function sendResultForBadRequest(messageId) { const message = messages[messageId]; const socket = message.socket; @@ -127,9 +154,8 @@ function sendResultForBadRequest(messageId) { } /** - * Given a request and its id, parses the request, checks if it's ok, - * and emits the right event. - * @param request {Object}, request object as specified in readme. + * Executes the request by emitting 'runJunit' or 'runJava' event. + * @param request {Request} * @param messageId {String}, id of the given message/request. */ function executeRequest(request, messageId) { @@ -152,7 +178,7 @@ function executeRequest(request, messageId) { } } /** - * Correct and truncates the feedback and sends it back. + * Correct and truncates the feedback and sends it back. Also closes connection. * @param feedback {Object}: feedback/result object as specified in readme or javaBox.js * @param feedback.messageId {string} */ From 7aaad335f04aa5a43b1cdbe0aad071cbe6fc759f Mon Sep 17 00:00:00 2001 From: kyriakos Date: Thu, 23 Feb 2017 21:11:42 +0200 Subject: [PATCH 04/11] New version of javabox partially implemented --- javaBox2.js | 233 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 javaBox2.js diff --git a/javaBox2.js b/javaBox2.js new file mode 100644 index 0000000..b314bf9 --- /dev/null +++ b/javaBox2.js @@ -0,0 +1,233 @@ +const EventEmitter = require('events'); +const Promise = require('bluebird'); +const coroutine = Promise.coroutine; +const Docker = Promise.promisifyAll(require('dockerode')); +//TODO: is promisfy applied recursively????? +const concat = require('concat-stream'); + +//TODO: forgot about archives???? + +/** + * @typedef {Object} ExecutionOutput + * @type ExecutionOutput.messageId {String} + * @type ExecutionOutput.success {Boolean} + * @type ExecutionOutput.output {String} + * @type ExecutionOutput.errorMessage {String} + * @type ExecutionOutput.timeout {Boolean} + * + */ + +/** + * Docker object, connected on /var/run/docker.socket or default localhost docker port. + * @type {Docker} + */ +const docker = new Docker(); + +/** + * JavaBox eventEmitter. + * @type {EventEmitter} + */ +const javaBox = new EventEmitter(); +/** + * Initialising javaBox. + */ +javaBox.on('runJava', runJava); +javaBox.on('runJunit', runJunit); + + +/** + * Creates a docker container with newly created execution + * to run the Main.java specified inside the tar. + * + * @param messageId {String}: id of the given message/request. + * @param main {String}: entry point class name. + * @param tarBuffer {String}: the buffer of the tar containing java files + * @param timeLimitCompileMs + * @param timeLimitExecutionMs + */ +const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { + + const className = main.split('.')[0]; + + let container = yield initializeContainer(tarBuffer, messageId); + + const javacCmd = ['javac', '-cp', 'home', `home/${main}`]; + const javaCmd = ['java', '-Djava.security.manager', '-cp', 'home', className]; + + try { + //TODO: Subcalls of coroutines + const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); + if (executedCorrectly(compileOutput)) { + + const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); + if (executedCorrectly(runtimeOutput)) { + emitSuccess(runtimeOutput); + + } else { + if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); + else emitWrong(runtimeOutput, 'Runtime'); + } + + } else { + if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); + else emitWrong(compileOutput, 'Compile'); + } + + + yield container.kill(); + yield container.remove({v: true}); + + } catch (e) { // internal error + //TODO: specify javaBox behavior for internal error + console.log('500: Internal error with Docker!'); + emitServerError(); + yield container.kill(); + yield container.remove({v: true}); + } + +}); + + +const initializeContainer = coroutine(function*(tarBuffer, messageId) { + + const createOpts = {Image: 'openjdk:8u111-jdk', Tty: true, Cmd: ['/bin/bash']}; + let container = yield docker.createContainerAsync(createOpts); + container['messageId'] = messageId; + + const startOps = {}; + yield container.startAsync(startOps); + + const tarOptsSource = {path: 'home'}; + yield container.putArchiveAsync(tarBuffer, tarOptsSource); + + return container; +}); + + +/** + * Run a command inside a container, capture and return output streams and successful execution. + * + * @param command {String[]}: A command with arguments to execute + * @param container {Container}: The container, properly initialised (from initializeContainer), for execution + * @param executionTimeLimit {Number} [Optional]: The number in milliseconds after which, execution should stop + * @return The result of the execution indicating if the execution was successful and the output of the stdout/ stderr. + * @return result {ExecutionOutput} + * @throws Error + * + */ +const runCommand = coroutine(function *(command, container, executionTimeLimit) { + + let timeout = null; + let stdoutData = ''; + let stderrData = ''; + + try { // possible errors with docker container + const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: false}; // execution options + const exec = yield container.execAsync(execOpts); // create execution of command + + const stream = yield container.attach({stream: true, stdout: true, stderr: true}); // prepare streams + container.modem.demuxStream(stream, (d) => { + stdoutData += d + }, (d) => { + stderrData += d + }); // get output chunks // get stdout and stderr + + yield exec.startAsync(); // start execution + + if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time + + yield stream.onAsync('end'); //will this work???????????? // wait until finished + + const executionData = yield exec.inspectAsync(); // get data of execution + const executedSuccessfully = (!executionData.Running) ? (executionData.ExitCode == 0) : null; // set successful execution mark + // should never be null + + if (executionTimeLimit) clearTimeout(timeout); + + return { + messageId: container.messageId, + success: executedSuccessfully, + output: stdoutData, + errorMessage: stderrData, + timeout: false + } + + } catch (e) { + container.stop(); + + if (e.name == 'timeout') { + return { + messageId: container.messageId, + success: false, + output: stdoutData, + errorMessage: stderrData, + timeout: true + } + } else { + throw e; + } + } +}); + +/** + * Check if a command executed correctly, by analyzing the result object. + * + * @param result {Object || ExecutionOutput} + * @param result.timeout: {Boolean} + * @param result.success: {Boolean} + * @return {boolean}: `true` if command exited with code 0 and no time out occurred, `false` otherwise. + */ +function executedCorrectly(result) { + return (result.success && !result.timeout); +} + +/** + * Dummy function to raise (throw) a custom timeout error + * @param stage {String} [Optional]: The stage in which timeout happened. + * @throws timeout {Object} + * @type timeout.name {string} + * @type timeout.stage {string} + */ +function throwTimeOutError(stage) { + stage = stage || ''; + throw { + name: 'timeout', + stage: stage + }; +} + + +function emitServerError() { + //TODO: implement +} + +/** + * Emit `result` event with success values. + * + * @param executionOutput {ExecutionOutput}: Output of a run command execution + */ +function emitSuccess(executionOutput) { + const feedback = executionOutput; + javaBox.emit('result', feedback); +} + +/** + * Emit `result` event with timeout values. + * + * @param executionOutput {ExecutionOutput}: Output of a run command execution + */ +function emitTimeout(executionOutput, stage) { + const feedback = executionOutput; + feedback.errorMessage = `${stage} timeout reached.`; + javaBox.emit('result', feedback); +} + +/** + * Emit `result` event with error values. + * + * @param executionOutput {ExecutionOutput}: Output of a run command execution + */ +function emitWrong(executionOutput, stage) { + const feedback = executionOutput; + javaBox.emit('result', feedback); +} \ No newline at end of file From 66e70fae0610eff25cb8e1928f2aac41d9f6b988 Mon Sep 17 00:00:00 2001 From: kyriakos Date: Thu, 23 Feb 2017 21:16:19 +0200 Subject: [PATCH 05/11] Fixed bug in 'runjunit' event emitter in app.js --- app.js | 2 +- javaBox2.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/app.js b/app.js index 2dd3686..316be17 100644 --- a/app.js +++ b/app.js @@ -88,7 +88,7 @@ const runJunit = coroutine(function *(messageId, tests, files, timeLimitCompileM }).then(function () { const tarBuffer = pack.read(); - javaBox.emit('runJava', messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + javaBox.emit('runJunit', messageId, junitFileNames, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); }); diff --git a/javaBox2.js b/javaBox2.js index b314bf9..5457aa8 100644 --- a/javaBox2.js +++ b/javaBox2.js @@ -88,6 +88,48 @@ const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompile }); +const runJunit = coroutine(function*(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { + + const className = main.split('.')[0]; + + let container = yield initializeContainer(tarBuffer, messageId); + + const javacCmd = ['javac', '-cp', 'home', `home/${main}`]; + const javaCmd = ['java', '-Djava.security.manager', '-cp', 'home', className]; + + try { + //TODO: Subcalls of coroutines + const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); + if (executedCorrectly(compileOutput)) { + + const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); + if (executedCorrectly(runtimeOutput)) { + emitSuccess(runtimeOutput); + + } else { + if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); + else emitWrong(runtimeOutput, 'Runtime'); + } + + } else { + if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); + else emitWrong(compileOutput, 'Compile'); + } + + + yield container.kill(); + yield container.remove({v: true}); + + } catch (e) { // internal error + //TODO: specify javaBox behavior for internal error + console.log('500: Internal error with Docker!'); + emitServerError(); + yield container.kill(); + yield container.remove({v: true}); + } + +}); + const initializeContainer = coroutine(function*(tarBuffer, messageId) { const createOpts = {Image: 'openjdk:8u111-jdk', Tty: true, Cmd: ['/bin/bash']}; From 0655d928633f711246f0aa87cf1d23bdb25fb461 Mon Sep 17 00:00:00 2001 From: kyriakos Date: Thu, 23 Feb 2017 21:17:15 +0200 Subject: [PATCH 06/11] Fixed bug in 'runjunit' event emitter in app.js --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index 316be17..7be40cb 100644 --- a/app.js +++ b/app.js @@ -88,7 +88,7 @@ const runJunit = coroutine(function *(messageId, tests, files, timeLimitCompileM }).then(function () { const tarBuffer = pack.read(); - javaBox.emit('runJunit', messageId, junitFileNames, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + javaBox.emit('runJunit', messageId, tests, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); }); From 92b72384ae71b5fc0f833c17cfea7cf65c89b408 Mon Sep 17 00:00:00 2001 From: kyriakos Date: Fri, 24 Feb 2017 01:53:52 +0200 Subject: [PATCH 07/11] JavaBox version with promises. Code is SID, untested and unchecked. --- javaBox.js | 445 +++++++++++++++++++++++++--------------------------- javaBox2.js | 275 -------------------------------- 2 files changed, 212 insertions(+), 508 deletions(-) delete mode 100644 javaBox2.js diff --git a/javaBox.js b/javaBox.js index 7dac31c..afb87cd 100644 --- a/javaBox.js +++ b/javaBox.js @@ -1,20 +1,26 @@ const EventEmitter = require('events'); -const Docker = require('dockerode'); +const Promise = require('bluebird'); +const coroutine = Promise.coroutine; +const Docker = Promise.promisifyAll(require('dockerode')); +//TODO: is promisfy applied recursively????? const concat = require('concat-stream'); /** - * Docker object, connected on /var/run/docker.socket or default localhost docker port. - * @type {Docker} + * @typedef {Object} ExecutionOutput + * @type ExecutionOutput.messageId {String} + * @type ExecutionOutput.success {Boolean} + * @type ExecutionOutput.output {String} + * @type ExecutionOutput.errorMessage {String} + * @type ExecutionOutput.timeout {Boolean} + * */ -const docker = new Docker(); /** - * Time to wait between checking that the command has been executed (milliseconds). - * @type {number} + * Docker object, connected on /var/run/docker.socket or default localhost docker port. + * @type {Docker} */ -const EXEC_WAIT_TIME_MS = 250; - +const docker = new Docker(); /** * JavaBox eventEmitter. @@ -38,29 +44,50 @@ javaBox.on('runJunit', runJunit); * @param timeLimitCompileMs * @param timeLimitExecutionMs */ -function runJava(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { +const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { const className = main.split('.')[0]; + let container = yield initializeContainer(tarBuffer, messageId); + const javacCmd = ['javac', '-cp', 'home', `home/${main}`]; const javaCmd = ['java', '-Djava.security.manager', '-cp', 'home', className]; - const sourceLocation = tarBuffer; - const execution = dockerCommand(javacCmd, timeLimitCompileMs, dockerCommand(javaCmd, timeLimitExecutionMs)); + try { + //TODO: Subcalls of coroutines + const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); + if (executedCorrectly(compileOutput)) { + + const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); + if (executedCorrectly(runtimeOutput)) { + emitSuccess(runtimeOutput); + + } else { + if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); + else emitWrong(runtimeOutput, 'Runtime'); + } + + } else { + if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); + else emitWrong(compileOutput, 'Compile'); + } + + + yield container.kill(); + yield container.remove({v: true}); + + } catch (e) { // internal error + //TODO: specify javaBox behavior for internal error + console.log('500: Internal error with Docker!'); + emitServerError(); + yield container.kill(); + yield container.remove({v: true}); + } + +}); - createJContainer(messageId, sourceLocation, false, execution); -} -/** - * Creates a docker container with newly created execution - * to run the tests on files to test specified inside the tar. - * - * @param messageId {String}: id of the given message/request. - * @param junitFileNames [String]: array of different junit tests filename. - * @param tarBuffer {String}: the buffer of the tar containing java files (both test and testing). - * @param timeLimitCompileMs - * @param timeLimitExecutionMs - */ -function runJunit(messageId, junitFileNames, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { + +const runJunit = coroutine(function*(messageId, junitFileNames, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { let junitFiles = []; junitFileNames.forEach((f)=>{ @@ -69,6 +96,8 @@ function runJunit(messageId, junitFileNames, tarBuffer, timeLimitCompileMs, time const className = 'TestRunner'; + let container = yield initializeContainer(tarBuffer, messageId, true); + const javacCmd = ['javac', '-cp', 'home:libs/junit-4.12:libs/hamcrest-core-1.3:libs/json-simple-1.1.1']; let javaCmd = ['java', '-cp', 'home:libs/junit-4.12:libs/hamcrest-core-1.3:libs/json-simple-1.1.1', className]; @@ -77,248 +106,126 @@ function runJunit(messageId, junitFileNames, tarBuffer, timeLimitCompileMs, time javaCmd.push(file); }); + try { + //TODO: Subcalls of coroutines + const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); + if (executedCorrectly(compileOutput)) { - const sourceLocation = tarBuffer; - const execution = dockerCommand(javacCmd, timeLimitCompileMs, dockerCommand(javaCmd, timeLimitExecutionMs)); - - createJContainer(messageId, sourceLocation, true, execution); -} - - -/** - * Creates and starts a container with bash, JDK SE, maybe junit and executes the callback - * passing the container or error to it. - * - * @param messageId {String}: id of the given message/request. - * @param tarBuffer {String}: the buffer of the tar containing java files. - * @param isJunit {Boolean}: true if need to create container with junit support. - * @param callback {function(error, container)}: callback to operate on error and container or data in case of error, - * should accept two arguments. - */ -function createJContainer(messageId, tarBuffer, isJunit, callback) { - - let copyToCall = 4; - - const tryCallback = (container) => { + const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); + if (executedCorrectly(runtimeOutput)) { + emitSuccess(runtimeOutput, true); - if (copyToCall === 0) { + } else { + if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); + else emitWrong(runtimeOutput, 'Runtime'); + } - callback(null, container); + } else { + if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); + else emitWrong(compileOutput, 'Compile'); } - }; - const createOpts = {Image: 'openjdk:8u111-jdk', Tty: true, Cmd: ['/bin/bash']}; - docker.createContainer(createOpts, (err, container) => { - if (err) {callback(err, container); return} + yield container.kill(); + yield container.remove({v: true}); - container['messageId'] = messageId; + } catch (e) { // internal error + //TODO: specify javaBox behavior for internal error + console.log('500: Internal error with Docker!'); + emitServerError(); + yield container.kill(); + yield container.remove({v: true}); + } - const startOpts = {}; - container.start(startOpts, (err, data) => { +}); - if (err) {callback(err, data); return} +const initializeContainer = coroutine(function*(tarBuffer, messageId, junit) { - if (isJunit) { + junit = junit || false; - const tarOptsRunner = {path: 'home'}; - container.putArchive('./archives/TestRunner.class.tar', tarOptsRunner, (err, data) => { - copyToCall--; - if (err) callback(err, data); - else tryCallback(container); + const createOpts = {Image: 'openjdk:8u111-jdk', Tty: true, Cmd: ['/bin/bash']}; + let container = yield docker.createContainerAsync(createOpts); + container['messageId'] = messageId; - }); + const startOps = {}; + yield container.startAsync(startOps); - const tarOptsSecureTest = {path: 'home'}; - container.putArchive('./archives/SecureTest.class.tar', tarOptsSecureTest, (err, data) => { - copyToCall--; - if (err) callback(err, data); - else tryCallback(container); + const tarOptsSource = {path: 'home'}; + yield container.putArchiveAsync(tarBuffer, tarOptsSource); - }); + if (junit) { + yield container.putArchiveAsync('./archives/SecureTest.class.tar', {path: 'home'}); + yield container.putArchive('./archives/libs.tar', {path: '/'}); + } - const tarOptsLibs = {path: '/'}; - container.putArchive('./archives/libs.tar', tarOptsLibs, (err, data) => { - copyToCall--; - if (err) callback(err, data); - else tryCallback(container); - }); - } else { - copyToCall = 1; - } + return container; +}); - const tarOptsSource = {path: 'home'}; - container.putArchive(tarBuffer, tarOptsSource, (err, data) => { - copyToCall--; - if (err) callback(err, data); - else tryCallback(container); - }); - }); - }); -} /** - * Given a command, the timeout and the callback, returns a function that on some given container, - * executes the command, attaches the output listener and runs waitCmdExit. + * Run a command inside a container, capture and return output streams and successful execution. * - * @param command {String}: command to be passed to container runtime environment. - * @param commandTimeLimitMs {Number}: execution timeout. - * @param callback {function(err, container)}: function to be executed after the command has finished. + * @param command {String[]}: A command with arguments to execute + * @param container {Container}: The container, properly initialised (from initializeContainer), for execution + * @param executionTimeLimit {Number} [Optional]: The number in milliseconds after which, execution should stop + * @return The result of the execution indicating if the execution was successful and the output of the stdout/ stderr. + * @return result {ExecutionOutput} + * @throws Error * - * @return {function(err, container)}: function that executes the command on a container. */ -function dockerCommand(command, commandTimeLimitMs, callback) { - - const opts = {Cmd: command, AttachStdout: true, AttachStderr: true}; - - const execution = (err, container) => { - - if (!err) container.exec(opts, (err, exec) => { - exec.start((err, stream) => { - - let stdOut = ''; - let stdErr = ''; - const stdOutConcat = concat({}, (data) => { - try { - stdOut += data; - } catch (e) { - stdOut = 'Output larger than 268435440 bytes.'; - } - }).on('error', (err) => {}); - const stdErrConcat = concat({}, (data) => { - try { - stdErr += data; - } catch (e) { - stdOut = 'Output larger than 268435440 bytes.'; - } - }).on('error', (err) => {}); - - container.modem.demuxStream(stream, stdOutConcat, stdErrConcat); - - - const streamInfo = { - - getOut: () => {return stdOut}, - getErr: () => {return stdErr}, - endStream: () => { - stdOutConcat.end(); - stdErrConcat.end(); - } - }; - - waitCmdExit(container, exec, callback, streamInfo, commandTimeLimitMs); - }); - }); - }; +const runCommand = coroutine(function *(command, container, executionTimeLimit) { - return execution; -} + let timeout = null; + let stdoutData = ''; + let stderrData = ''; -/** - * Given an execution of a command on a container, a stream handler and command timeout - * waits for the command to be finished or timeout to be expired and calls the callback - * if it isn't null, otherwise it calls the feedbackAndClose function. - * - * @param container {Container}: Active docker container. - * @param exec {Object}: Docker execution object. - * @param callback {function(err, container)}: function to be executed after the command has finished. - * @param streamInfo {Object}: returns stdOut, stdIn and closes concat stream - * @param commandTimeOutMs {Number}: execution timeout in ms. - * @param previousTimeMs {Number}: ms already spent on this execution, called when the function is called - * recursively, should not be passed otherwise. - */ -function waitCmdExit(container, exec, callback, streamInfo, commandTimeOutMs, previousTimeMs){ + try { // possible errors with docker container + const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: false}; // execution options + const exec = yield container.execAsync(execOpts); // create execution of command - let timeSpentMs = previousTimeMs || 0; + const stream = yield container.attach({stream: true, stdout: true, stderr: true}); // prepare streams + container.modem.demuxStream(stream, (d) => { + stdoutData += d + }, (d) => { + stderrData += d + }); // get output chunks // get stdout and stderr - const checkExit = (err, data) => { + yield exec.startAsync(); // start execution - if (data.Running) { // command is still running, check later or send time out + if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time - timeSpentMs += EXEC_WAIT_TIME_MS; // count time spent + yield stream.onAsync('end'); //will this work???????????? // wait until finished - if (timeSpentMs >= commandTimeOutMs){ // time period expired + const executionData = yield exec.inspectAsync(); // get data of execution + const executedSuccessfully = (!executionData.Running) ? (executionData.ExitCode == 0) : null; // set successful execution mark + // should never be null - feedbackAndClose(container, streamInfo, false, true); + if (executionTimeLimit) clearTimeout(timeout); - } else { + return { + messageId: container.messageId, + success: executedSuccessfully, + output: stdoutData, + errorMessage: stderrData, + timeout: false + } - waitCmdExit(container, exec, callback, streamInfo, commandTimeOutMs, timeSpentMs); + } catch (e) { + container.stop(); + if (e.name == 'timeout') { + return { + messageId: container.messageId, + success: false, + output: stdoutData, + errorMessage: stderrData, + timeout: true } - - } else if ((data.ExitCode === 0) && (callback)) { // command successful, has next command - - callback(null, container); - - } else if (data.ExitCode === 0) { // command successful, it was the last command - - feedbackAndClose(container, streamInfo, true, false) - - } else { // command failed - - feedbackAndClose(container, streamInfo, false, false); + } else { + throw e; } - }; - - setTimeout(() => exec.inspect(checkExit), EXEC_WAIT_TIME_MS); -} - -/** - * Closes the stream, parses it assuming it could be a specific junit output, - * accordingly creates a feedback and emits it, closing and deleting docker container - * at the end. - * - * @param container {Container}: Active docker container. - * @param streamInfo Object, returns stdOut, stdIn and closes concat stream - * @param passed Boolean, true if no compile or runtime error during normal execution - * @param timeOut Boolean, true if timeout time elapsed - * - * @feedback - * if test files exist (junit output) and passed is true: - * { - * messageId, - * passed: Boolean (false if compile/runtime errors true otherwise), - * output: String, - * errorMessage: String (empty if `passed` is true), - * timeOut: Boolean, - * totalNumberOfTests: Integer, - * numberOfTestsPassed: Integer, - * testsOutput: String (output of all failed tests) - * } - * otherwise: - * { - * messageId, - * passed: Boolean (false if compile/runtime errors true otherwise), - * output: String, - * errorMessage: String (empty if `passed` is false), - * timeOut: Boolean - * } - */ -function feedbackAndClose(container, streamInfo, passed, timeOut) { - - streamInfo.endStream(); - - const feedback = { - messageId: container.messageId, - passed: passed, - output: (!timeOut) ? streamInfo.getOut() : '', - errorMessage: (!timeOut) ? streamInfo.getErr() : "Reached maximum time limit", - timeOut: timeOut - - }; - - const parsed = parseOutput(feedback.output); - feedback.output = parsed.normalOutput; - feedback.totalNumberOfTests = parsed.totalNumberOfTests; - feedback.numberOfTestsPassed = parsed.numberOfTestsPassed; - feedback.testsOutput = parsed.testsOutput; - - javaBox.emit('result', feedback); - - container.kill({}, () => - container.remove({v: true}, () => {})); -} + } +}); /** * Given a possibly junit output, parses it and if it's junit, adds @@ -352,5 +259,77 @@ function parseOutput(output){ return outputObject; } +/** + * Check if a command executed correctly, by analyzing the result object. + * + * @param result {Object || ExecutionOutput} + * @param result.timeout: {Boolean} + * @param result.success: {Boolean} + * @return {boolean}: `true` if command exited with code 0 and no time out occurred, `false` otherwise. + */ +function executedCorrectly(result) { + return (result.success && !result.timeout); +} + +/** + * Dummy function to raise (throw) a custom timeout error + * @param stage {String} [Optional]: The stage in which timeout happened. + * @throws timeout {Object} + * @type timeout.name {string} + * @type timeout.stage {string} + */ +function throwTimeOutError(stage) { + stage = stage || ''; + throw { + name: 'timeout', + stage: stage + }; +} + + +function emitServerError() { + //TODO: implement +} + +/** + * Emit `result` event with success values. + * + * @param executionOutput {ExecutionOutput}: Output of a run command execution + * @param junit {Boolean} [Optional][Default: false]: Set to true if execution output is from junit orchestrator class for parsing. + */ +function emitSuccess(executionOutput, junit) { + junit = junit || false; + const feedback = executionOutput; + + if (junit) { + const parsed = parseOutput(feedback.output); + feedback.output = parsed.normalOutput; + feedback.totalNumberOfTests = parsed.totalNumberOfTests; + feedback.numberOfTestsPassed = parsed.numberOfTestsPassed; + feedback.testsOutput = parsed.testsOutput; + } + javaBox.emit('result', feedback); +} + +/** + * Emit `result` event with timeout values. + * + * @param executionOutput {ExecutionOutput}: Output of a run command execution + * @param stage {'compile' || 'runtime'}[Optional]: In which stage this function is called + */ +function emitTimeout(executionOutput, stage) { + const feedback = executionOutput; + feedback.errorMessage = `${stage} timeout reached.`; + javaBox.emit('result', feedback); +} -module.exports = javaBox; +/** + * Emit `result` event with error values. + * + * @param executionOutput {ExecutionOutput}: Output of a run command execution + * @param stage {'compile' || 'runtime'}[Optional]: In which stage this function is called + */ +function emitWrong(executionOutput, stage) { + const feedback = executionOutput; + javaBox.emit('result', feedback); +} \ No newline at end of file diff --git a/javaBox2.js b/javaBox2.js deleted file mode 100644 index 5457aa8..0000000 --- a/javaBox2.js +++ /dev/null @@ -1,275 +0,0 @@ -const EventEmitter = require('events'); -const Promise = require('bluebird'); -const coroutine = Promise.coroutine; -const Docker = Promise.promisifyAll(require('dockerode')); -//TODO: is promisfy applied recursively????? -const concat = require('concat-stream'); - -//TODO: forgot about archives???? - -/** - * @typedef {Object} ExecutionOutput - * @type ExecutionOutput.messageId {String} - * @type ExecutionOutput.success {Boolean} - * @type ExecutionOutput.output {String} - * @type ExecutionOutput.errorMessage {String} - * @type ExecutionOutput.timeout {Boolean} - * - */ - -/** - * Docker object, connected on /var/run/docker.socket or default localhost docker port. - * @type {Docker} - */ -const docker = new Docker(); - -/** - * JavaBox eventEmitter. - * @type {EventEmitter} - */ -const javaBox = new EventEmitter(); -/** - * Initialising javaBox. - */ -javaBox.on('runJava', runJava); -javaBox.on('runJunit', runJunit); - - -/** - * Creates a docker container with newly created execution - * to run the Main.java specified inside the tar. - * - * @param messageId {String}: id of the given message/request. - * @param main {String}: entry point class name. - * @param tarBuffer {String}: the buffer of the tar containing java files - * @param timeLimitCompileMs - * @param timeLimitExecutionMs - */ -const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { - - const className = main.split('.')[0]; - - let container = yield initializeContainer(tarBuffer, messageId); - - const javacCmd = ['javac', '-cp', 'home', `home/${main}`]; - const javaCmd = ['java', '-Djava.security.manager', '-cp', 'home', className]; - - try { - //TODO: Subcalls of coroutines - const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); - if (executedCorrectly(compileOutput)) { - - const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); - if (executedCorrectly(runtimeOutput)) { - emitSuccess(runtimeOutput); - - } else { - if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); - else emitWrong(runtimeOutput, 'Runtime'); - } - - } else { - if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); - else emitWrong(compileOutput, 'Compile'); - } - - - yield container.kill(); - yield container.remove({v: true}); - - } catch (e) { // internal error - //TODO: specify javaBox behavior for internal error - console.log('500: Internal error with Docker!'); - emitServerError(); - yield container.kill(); - yield container.remove({v: true}); - } - -}); - - -const runJunit = coroutine(function*(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { - - const className = main.split('.')[0]; - - let container = yield initializeContainer(tarBuffer, messageId); - - const javacCmd = ['javac', '-cp', 'home', `home/${main}`]; - const javaCmd = ['java', '-Djava.security.manager', '-cp', 'home', className]; - - try { - //TODO: Subcalls of coroutines - const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); - if (executedCorrectly(compileOutput)) { - - const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); - if (executedCorrectly(runtimeOutput)) { - emitSuccess(runtimeOutput); - - } else { - if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); - else emitWrong(runtimeOutput, 'Runtime'); - } - - } else { - if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); - else emitWrong(compileOutput, 'Compile'); - } - - - yield container.kill(); - yield container.remove({v: true}); - - } catch (e) { // internal error - //TODO: specify javaBox behavior for internal error - console.log('500: Internal error with Docker!'); - emitServerError(); - yield container.kill(); - yield container.remove({v: true}); - } - -}); - -const initializeContainer = coroutine(function*(tarBuffer, messageId) { - - const createOpts = {Image: 'openjdk:8u111-jdk', Tty: true, Cmd: ['/bin/bash']}; - let container = yield docker.createContainerAsync(createOpts); - container['messageId'] = messageId; - - const startOps = {}; - yield container.startAsync(startOps); - - const tarOptsSource = {path: 'home'}; - yield container.putArchiveAsync(tarBuffer, tarOptsSource); - - return container; -}); - - -/** - * Run a command inside a container, capture and return output streams and successful execution. - * - * @param command {String[]}: A command with arguments to execute - * @param container {Container}: The container, properly initialised (from initializeContainer), for execution - * @param executionTimeLimit {Number} [Optional]: The number in milliseconds after which, execution should stop - * @return The result of the execution indicating if the execution was successful and the output of the stdout/ stderr. - * @return result {ExecutionOutput} - * @throws Error - * - */ -const runCommand = coroutine(function *(command, container, executionTimeLimit) { - - let timeout = null; - let stdoutData = ''; - let stderrData = ''; - - try { // possible errors with docker container - const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: false}; // execution options - const exec = yield container.execAsync(execOpts); // create execution of command - - const stream = yield container.attach({stream: true, stdout: true, stderr: true}); // prepare streams - container.modem.demuxStream(stream, (d) => { - stdoutData += d - }, (d) => { - stderrData += d - }); // get output chunks // get stdout and stderr - - yield exec.startAsync(); // start execution - - if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time - - yield stream.onAsync('end'); //will this work???????????? // wait until finished - - const executionData = yield exec.inspectAsync(); // get data of execution - const executedSuccessfully = (!executionData.Running) ? (executionData.ExitCode == 0) : null; // set successful execution mark - // should never be null - - if (executionTimeLimit) clearTimeout(timeout); - - return { - messageId: container.messageId, - success: executedSuccessfully, - output: stdoutData, - errorMessage: stderrData, - timeout: false - } - - } catch (e) { - container.stop(); - - if (e.name == 'timeout') { - return { - messageId: container.messageId, - success: false, - output: stdoutData, - errorMessage: stderrData, - timeout: true - } - } else { - throw e; - } - } -}); - -/** - * Check if a command executed correctly, by analyzing the result object. - * - * @param result {Object || ExecutionOutput} - * @param result.timeout: {Boolean} - * @param result.success: {Boolean} - * @return {boolean}: `true` if command exited with code 0 and no time out occurred, `false` otherwise. - */ -function executedCorrectly(result) { - return (result.success && !result.timeout); -} - -/** - * Dummy function to raise (throw) a custom timeout error - * @param stage {String} [Optional]: The stage in which timeout happened. - * @throws timeout {Object} - * @type timeout.name {string} - * @type timeout.stage {string} - */ -function throwTimeOutError(stage) { - stage = stage || ''; - throw { - name: 'timeout', - stage: stage - }; -} - - -function emitServerError() { - //TODO: implement -} - -/** - * Emit `result` event with success values. - * - * @param executionOutput {ExecutionOutput}: Output of a run command execution - */ -function emitSuccess(executionOutput) { - const feedback = executionOutput; - javaBox.emit('result', feedback); -} - -/** - * Emit `result` event with timeout values. - * - * @param executionOutput {ExecutionOutput}: Output of a run command execution - */ -function emitTimeout(executionOutput, stage) { - const feedback = executionOutput; - feedback.errorMessage = `${stage} timeout reached.`; - javaBox.emit('result', feedback); -} - -/** - * Emit `result` event with error values. - * - * @param executionOutput {ExecutionOutput}: Output of a run command execution - */ -function emitWrong(executionOutput, stage) { - const feedback = executionOutput; - javaBox.emit('result', feedback); -} \ No newline at end of file From 6491097f42cfa7705a4e9b964f7d214a52d7008c Mon Sep 17 00:00:00 2001 From: StKyr Date: Fri, 23 Jun 2017 02:53:06 +0300 Subject: [PATCH 08/11] Promisifed version compiles and runs. Minor bugfixes needed. --- app.js | 199 +++++++++++++++++++++++++------------------------ javaBox.js | 110 +++++++++++++++------------ package.json | 2 +- queue.js | 4 +- server.js | 2 +- test/client.js | 9 +-- 6 files changed, 173 insertions(+), 153 deletions(-) diff --git a/app.js b/app.js index 7be40cb..1afe2d9 100644 --- a/app.js +++ b/app.js @@ -5,107 +5,114 @@ const Promise = require('bluebird'); const coroutine = Promise.coroutine; -/** - * Command line arguments definition - * @type {[*]} - */ -const commandLineDef = [ - { name: 'port', alias: 'p', type: Number, defaultValue: 5016}, - { name: 'mongoAddress', alias: 'a', type: String, defaultValue: '127.0.0.1/queue'}, - { name: 'mongoCollection', alias: 'c', type: String, defaultValue: 'agendaJobs'}, - { name: 'defaultConcurrency', alias: 'd', type: Number, defaultValue: 40}, - { name: 'maxConcurrency', alias: 'm', type: Number, defaultValue: 70} -]; -/** - * Object that for keys has command line argument names - * and for values its value. - * @type {Object} - */ -const commandLine = commandLineArgs(commandLineDef); - - -/** - * Server specified in server.js - * @type {Server} - */ -const server = require('./server')( - commandLine.port, - commandLine.mongoAddress, - commandLine.mongoCollection, - commandLine.defaultConcurrency, - commandLine.maxConcurrency -); -/** - * Object controlling docker specified in javaBox.js - * @type {EventEmitter} - */ -const javaBox = require('./javaBox'); - -/** - * Given code to execute and info about it, - * creates a tar with the code and passed the info and the tar to javaBox. - * - * @param messageId {String}: id of the given message/request. - * @param main {String}: entry point class name. - * @param files {{name: String, data: String}[]}: array of objects with filename and its content - * @param timeLimitCompileMs {Number}: Compilation timeout. - * @param timeLimitExecutionMs {Number}: Execution timeout. - */ -const runJava = coroutine(function*(messageId, main, files, timeLimitCompileMs, timeLimitExecutionMs) { - - const pack = tar.pack(); - const packEntry = Promise.promisify(pack.entry, {context: pack}); - - Promise.map(files, function (file) { - return packEntry({name: file.name}, file.data); - - }).then(function () { - const tarBuffer = pack.read(); - javaBox.emit('runJava', messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); +const main = coroutine(function*() { + /** + * Command line arguments definition + * @type {[*]} + */ + const commandLineDef = [ + {name: 'port', alias: 'p', type: Number, defaultValue: 5016}, + {name: 'mongoAddress', alias: 'a', type: String, defaultValue: '127.0.0.1/queue'}, + {name: 'mongoCollection', alias: 'c', type: String, defaultValue: 'agendaJobs'}, + {name: 'defaultConcurrency', alias: 'd', type: Number, defaultValue: 40}, + {name: 'maxConcurrency', alias: 'm', type: Number, defaultValue: 70} + ]; + /** + * Object that for keys has command line argument names + * and for values its value. + * @type {Object} + */ + const commandLine = commandLineArgs(commandLineDef); + + + /** + * Server specified in server.js + * @type {Server} + */ + const server = yield require('./server')( + commandLine.port, + commandLine.mongoAddress, + commandLine.mongoCollection, + commandLine.defaultConcurrency, + commandLine.maxConcurrency + ); + + /** + * Object controlling docker specified in javaBox.js + * @type {EventEmitter} + */ + const javaBox = require('./javaBox'); + + /** + * Given code to execute and info about it, + * creates a tar with the code and passed the info and the tar to javaBox. + * + * @param messageId {String}: id of the given message/request. + * @param main {String}: entry point class name. + * @param files {{name: String, data: String}[]}: array of objects with filename and its content + * @param timeLimitCompileMs {Number}: Compilation timeout. + * @param timeLimitExecutionMs {Number}: Execution timeout. + */ + const runJava = coroutine(function*(messageId, main, files, timeLimitCompileMs, timeLimitExecutionMs) { + + const pack = tar.pack(); + //const packEntry = Promise.promisify(pack.entry, {context: pack}); + + Promise.map(files, function (file) { + //return packEntry({name: file.name}, file.data); + return pack.entry({name: file.name}, file.data); + + }).then(function () { + const tarBuffer = pack.read(); + javaBox.emit('runJava', messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + + }); }); + /** + * Given code to execute and to test and info about it, + * creates a tar with the code and passed the info and the tar to javaBox. + * + * @param messageId {String}: id of the given message/request. + * @param tests [{name: {String}, data: {String]: array of objects with filename and its content. + * @param files [{name: {String}, data: {String]: array of objects with filename and its content. + * @param timeLimitCompileMs {Number}: Compilation timeout. + * @param timeLimitExecutionMs {Number}: Execution timeout. + */ + const runJunit = coroutine(function *(messageId, tests, files, timeLimitCompileMs, timeLimitExecutionMs) { -}); -/** - * Given code to execute and to test and info about it, - * creates a tar with the code and passed the info and the tar to javaBox. - * - * @param messageId {String}: id of the given message/request. - * @param tests [{name: {String}, data: {String]: array of objects with filename and its content. - * @param files [{name: {String}, data: {String]: array of objects with filename and its content. - * @param timeLimitCompileMs {Number}: Compilation timeout. - * @param timeLimitExecutionMs {Number}: Execution timeout. - */ -const runJunit = coroutine(function *(messageId, tests, files, timeLimitCompileMs, timeLimitExecutionMs) { - - files = files.concat(tests); - - const pack = tar.pack(); - const packEntry = Promise.promisify(pack.entry, {context: pack}); - - Promise.map(files, function (file) { - return packEntry({name: file.name}, file.data); - - }).then(function () { - const tarBuffer = pack.read(); - javaBox.emit('runJunit', messageId, tests, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + files = files.concat(tests); + + const pack = tar.pack(); + const packEntry = Promise.promisify(pack.entry, {context: pack}); + + Promise.map(files, function (file) { + return packEntry({name: file.name}, file.data); + + }).then(function () { + const tarBuffer = pack.read(); + javaBox.emit('runJunit', messageId, tests, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs); + + }); }); + /** + * Given the feedback object passes it to the server. + * @param feedback {Object}, feedback/result object as specified in readme or javaBox.js + */ + function giveFeedBack(feedback) { + + server.emit('result', feedback); + } + + /** + * Specifying the server and javaBox control flow + */ + server.on('runJunit', runJunit); + server.on('runJava', runJava); + javaBox.on('result', giveFeedBack); + }); -/** - * Given the feedback object passes it to the server. - * @param feedback {Object}, feedback/result object as specified in readme or javaBox.js - */ -function giveFeedBack(feedback) { - - server.emit('result', feedback); -} - -/** - * Specifying the server and javaBox control flow - */ -server.on('runJunit', runJunit); -server.on('runJava', runJava); -javaBox.on('result', giveFeedBack); +main(); \ No newline at end of file diff --git a/javaBox.js b/javaBox.js index afb87cd..fb10ded 100644 --- a/javaBox.js +++ b/javaBox.js @@ -1,8 +1,8 @@ const EventEmitter = require('events'); const Promise = require('bluebird'); const coroutine = Promise.coroutine; -const Docker = Promise.promisifyAll(require('dockerode')); -//TODO: is promisfy applied recursively????? +//const Docker = Promise.promisifyAll(require('dockerode')); +const Docker = require('dockerode'); const concat = require('concat-stream'); @@ -20,29 +20,25 @@ const concat = require('concat-stream'); * Docker object, connected on /var/run/docker.socket or default localhost docker port. * @type {Docker} */ -const docker = new Docker(); +const docker = Promise.promisifyAll(new Docker()); /** * JavaBox eventEmitter. * @type {EventEmitter} */ const javaBox = new EventEmitter(); -/** - * Initialising javaBox. - */ -javaBox.on('runJava', runJava); -javaBox.on('runJunit', runJunit); + /** * Creates a docker container with newly created execution * to run the Main.java specified inside the tar. * - * @param messageId {String}: id of the given message/request. - * @param main {String}: entry point class name. - * @param tarBuffer {String}: the buffer of the tar containing java files - * @param timeLimitCompileMs - * @param timeLimitExecutionMs + * @param {String} messageId : id of the given message/request. + * @param {String} main : entry point class name. + * @param {String} tarBuffer : the buffer of the tar containing java files + * @param {Number} timeLimitCompileMs + * @param {Number} timeLimitExecutionMs */ const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { @@ -54,32 +50,37 @@ const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompile const javaCmd = ['java', '-Djava.security.manager', '-cp', 'home', className]; try { - //TODO: Subcalls of coroutines - const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); + const compileOutput = yield runCommand(javacCmd, container, timeLimitCompileMs); + if (executedCorrectly(compileOutput)) { - const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); + + const runtimeOutput = yield runCommand(javaCmd, container, timeLimitExecutionMs); + if (executedCorrectly(runtimeOutput)) { + //TODO: maybe also return compileOutput for warnings? emitSuccess(runtimeOutput); } else { + if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); else emitWrong(runtimeOutput, 'Runtime'); } } else { + if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); else emitWrong(compileOutput, 'Compile'); } - yield container.kill(); - yield container.remove({v: true}); + yield container.killAsync(); + yield container.removeAsync({v: true}); } catch (e) { // internal error //TODO: specify javaBox behavior for internal error console.log('500: Internal error with Docker!'); - emitServerError(); + emitServerError(e); yield container.kill(); yield container.remove({v: true}); } @@ -107,11 +108,10 @@ const runJunit = coroutine(function*(messageId, junitFileNames, tarBuffer, timeL }); try { - //TODO: Subcalls of coroutines - const compileOutput = runCommand(javacCmd, container, timeLimitCompileMs); + const compileOutput = yield runCommand(javacCmd, container, timeLimitCompileMs); if (executedCorrectly(compileOutput)) { - const runtimeOutput = runCommand(javaCmd, container, timeLimitExecutionMs); + const runtimeOutput = yield runCommand(javaCmd, container, timeLimitExecutionMs); if (executedCorrectly(runtimeOutput)) { emitSuccess(runtimeOutput, true); @@ -126,13 +126,13 @@ const runJunit = coroutine(function*(messageId, junitFileNames, tarBuffer, timeL } - yield container.kill(); - yield container.remove({v: true}); + yield container.killAsync(); + yield container.removeAsync({v: true}); } catch (e) { // internal error //TODO: specify javaBox behavior for internal error - console.log('500: Internal error with Docker!'); - emitServerError(); + console.log('501: Internal error with Docker!'); + emitServerError(e); yield container.kill(); yield container.remove({v: true}); } @@ -144,7 +144,7 @@ const initializeContainer = coroutine(function*(tarBuffer, messageId, junit) { junit = junit || false; const createOpts = {Image: 'openjdk:8u111-jdk', Tty: true, Cmd: ['/bin/bash']}; - let container = yield docker.createContainerAsync(createOpts); + let container = Promise.promisifyAll(yield docker.createContainerAsync(createOpts)); container['messageId'] = messageId; const startOps = {}; @@ -178,27 +178,33 @@ const runCommand = coroutine(function *(command, container, executionTimeLimit) let timeout = null; let stdoutData = ''; let stderrData = ''; + const re = /[\0-\x1F\x7F-\x9F\xAD\u0378\u0379\u037F-\u0383\u038B\u038D\u03A2\u0528-\u0530\u0557\u0558\u0560\u0588\u058B-\u058E\u0590\u05C8-\u05CF\u05EB-\u05EF\u05F5-\u0605\u061C\u061D\u06DD\u070E\u070F\u074B\u074C\u07B2-\u07BF\u07FB-\u07FF\u082E\u082F\u083F\u085C\u085D\u085F-\u089F\u08A1\u08AD-\u08E3\u08FF\u0978\u0980\u0984\u098D\u098E\u0991\u0992\u09A9\u09B1\u09B3-\u09B5\u09BA\u09BB\u09C5\u09C6\u09C9\u09CA\u09CF-\u09D6\u09D8-\u09DB\u09DE\u09E4\u09E5\u09FC-\u0A00\u0A04\u0A0B-\u0A0E\u0A11\u0A12\u0A29\u0A31\u0A34\u0A37\u0A3A\u0A3B\u0A3D\u0A43-\u0A46\u0A49\u0A4A\u0A4E-\u0A50\u0A52-\u0A58\u0A5D\u0A5F-\u0A65\u0A76-\u0A80\u0A84\u0A8E\u0A92\u0AA9\u0AB1\u0AB4\u0ABA\u0ABB\u0AC6\u0ACA\u0ACE\u0ACF\u0AD1-\u0ADF\u0AE4\u0AE5\u0AF2-\u0B00\u0B04\u0B0D\u0B0E\u0B11\u0B12\u0B29\u0B31\u0B34\u0B3A\u0B3B\u0B45\u0B46\u0B49\u0B4A\u0B4E-\u0B55\u0B58-\u0B5B\u0B5E\u0B64\u0B65\u0B78-\u0B81\u0B84\u0B8B-\u0B8D\u0B91\u0B96-\u0B98\u0B9B\u0B9D\u0BA0-\u0BA2\u0BA5-\u0BA7\u0BAB-\u0BAD\u0BBA-\u0BBD\u0BC3-\u0BC5\u0BC9\u0BCE\u0BCF\u0BD1-\u0BD6\u0BD8-\u0BE5\u0BFB-\u0C00\u0C04\u0C0D\u0C11\u0C29\u0C34\u0C3A-\u0C3C\u0C45\u0C49\u0C4E-\u0C54\u0C57\u0C5A-\u0C5F\u0C64\u0C65\u0C70-\u0C77\u0C80\u0C81\u0C84\u0C8D\u0C91\u0CA9\u0CB4\u0CBA\u0CBB\u0CC5\u0CC9\u0CCE-\u0CD4\u0CD7-\u0CDD\u0CDF\u0CE4\u0CE5\u0CF0\u0CF3-\u0D01\u0D04\u0D0D\u0D11\u0D3B\u0D3C\u0D45\u0D49\u0D4F-\u0D56\u0D58-\u0D5F\u0D64\u0D65\u0D76-\u0D78\u0D80\u0D81\u0D84\u0D97-\u0D99\u0DB2\u0DBC\u0DBE\u0DBF\u0DC7-\u0DC9\u0DCB-\u0DCE\u0DD5\u0DD7\u0DE0-\u0DF1\u0DF5-\u0E00\u0E3B-\u0E3E\u0E5C-\u0E80\u0E83\u0E85\u0E86\u0E89\u0E8B\u0E8C\u0E8E-\u0E93\u0E98\u0EA0\u0EA4\u0EA6\u0EA8\u0EA9\u0EAC\u0EBA\u0EBE\u0EBF\u0EC5\u0EC7\u0ECE\u0ECF\u0EDA\u0EDB\u0EE0-\u0EFF\u0F48\u0F6D-\u0F70\u0F98\u0FBD\u0FCD\u0FDB-\u0FFF\u10C6\u10C8-\u10CC\u10CE\u10CF\u1249\u124E\u124F\u1257\u1259\u125E\u125F\u1289\u128E\u128F\u12B1\u12B6\u12B7\u12BF\u12C1\u12C6\u12C7\u12D7\u1311\u1316\u1317\u135B\u135C\u137D-\u137F\u139A-\u139F\u13F5-\u13FF\u169D-\u169F\u16F1-\u16FF\u170D\u1715-\u171F\u1737-\u173F\u1754-\u175F\u176D\u1771\u1774-\u177F\u17DE\u17DF\u17EA-\u17EF\u17FA-\u17FF\u180F\u181A-\u181F\u1878-\u187F\u18AB-\u18AF\u18F6-\u18FF\u191D-\u191F\u192C-\u192F\u193C-\u193F\u1941-\u1943\u196E\u196F\u1975-\u197F\u19AC-\u19AF\u19CA-\u19CF\u19DB-\u19DD\u1A1C\u1A1D\u1A5F\u1A7D\u1A7E\u1A8A-\u1A8F\u1A9A-\u1A9F\u1AAE-\u1AFF\u1B4C-\u1B4F\u1B7D-\u1B7F\u1BF4-\u1BFB\u1C38-\u1C3A\u1C4A-\u1C4C\u1C80-\u1CBF\u1CC8-\u1CCF\u1CF7-\u1CFF\u1DE7-\u1DFB\u1F16\u1F17\u1F1E\u1F1F\u1F46\u1F47\u1F4E\u1F4F\u1F58\u1F5A\u1F5C\u1F5E\u1F7E\u1F7F\u1FB5\u1FC5\u1FD4\u1FD5\u1FDC\u1FF0\u1FF1\u1FF5\u1FFF\u200B-\u200F\u202A-\u202E\u2060-\u206F\u2072\u2073\u208F\u209D-\u209F\u20BB-\u20CF\u20F1-\u20FF\u218A-\u218F\u23F4-\u23FF\u2427-\u243F\u244B-\u245F\u2700\u2B4D-\u2B4F\u2B5A-\u2BFF\u2C2F\u2C5F\u2CF4-\u2CF8\u2D26\u2D28-\u2D2C\u2D2E\u2D2F\u2D68-\u2D6E\u2D71-\u2D7E\u2D97-\u2D9F\u2DA7\u2DAF\u2DB7\u2DBF\u2DC7\u2DCF\u2DD7\u2DDF\u2E3C-\u2E7F\u2E9A\u2EF4-\u2EFF\u2FD6-\u2FEF\u2FFC-\u2FFF\u3040\u3097\u3098\u3100-\u3104\u312E-\u3130\u318F\u31BB-\u31BF\u31E4-\u31EF\u321F\u32FF\u4DB6-\u4DBF\u9FCD-\u9FFF\uA48D-\uA48F\uA4C7-\uA4CF\uA62C-\uA63F\uA698-\uA69E\uA6F8-\uA6FF\uA78F\uA794-\uA79F\uA7AB-\uA7F7\uA82C-\uA82F\uA83A-\uA83F\uA878-\uA87F\uA8C5-\uA8CD\uA8DA-\uA8DF\uA8FC-\uA8FF\uA954-\uA95E\uA97D-\uA97F\uA9CE\uA9DA-\uA9DD\uA9E0-\uA9FF\uAA37-\uAA3F\uAA4E\uAA4F\uAA5A\uAA5B\uAA7C-\uAA7F\uAAC3-\uAADA\uAAF7-\uAB00\uAB07\uAB08\uAB0F\uAB10\uAB17-\uAB1F\uAB27\uAB2F-\uABBF\uABEE\uABEF\uABFA-\uABFF\uD7A4-\uD7AF\uD7C7-\uD7CA\uD7FC-\uF8FF\uFA6E\uFA6F\uFADA-\uFAFF\uFB07-\uFB12\uFB18-\uFB1C\uFB37\uFB3D\uFB3F\uFB42\uFB45\uFBC2-\uFBD2\uFD40-\uFD4F\uFD90\uFD91\uFDC8-\uFDEF\uFDFE\uFDFF\uFE1A-\uFE1F\uFE27-\uFE2F\uFE53\uFE67\uFE6C-\uFE6F\uFE75\uFEFD-\uFF00\uFFBF-\uFFC1\uFFC8\uFFC9\uFFD0\uFFD1\uFFD8\uFFD9\uFFDD-\uFFDF\uFFE7\uFFEF-\uFFFB\uFFFE\uFFFF]/g; + try { // possible errors with docker container - const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: false}; // execution options - const exec = yield container.execAsync(execOpts); // create execution of command + const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: true}; // execution options + const exec = Promise.promisifyAll(yield container.execAsync(execOpts)); // create execution of command + + const stream = yield exec.startAsync(); // start execution (get an output stream) - const stream = yield container.attach({stream: true, stdout: true, stderr: true}); // prepare streams - container.modem.demuxStream(stream, (d) => { - stdoutData += d - }, (d) => { - stderrData += d - }); // get output chunks // get stdout and stderr + if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time - yield exec.startAsync(); // start execution - if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time + let data = []; + stream.on('data', chunk => { + data.push(chunk); + }); // get each output chunk + stream.on('end', () => { + stdoutData = Buffer.concat(data).toString().replace(re, "") + }); // finalize them into a string + // no need for promisified code here. It will introduce loops making it less readable - yield stream.onAsync('end'); //will this work???????????? // wait until finished + let executionData = yield exec.inspectAsync(); + while (executionData.Running) { // loop (asynchronously) until execution stops + executionData = yield exec.inspectAsync(); + } - const executionData = yield exec.inspectAsync(); // get data of execution - const executedSuccessfully = (!executionData.Running) ? (executionData.ExitCode == 0) : null; // set successful execution mark - // should never be null + const executedSuccessfully = (executionData.ExitCode == 0); // set successful execution mark if (executionTimeLimit) clearTimeout(timeout); @@ -211,7 +217,7 @@ const runCommand = coroutine(function *(command, container, executionTimeLimit) } } catch (e) { - container.stop(); + container.stopAsync(); if (e.name == 'timeout') { return { @@ -287,8 +293,8 @@ function throwTimeOutError(stage) { } -function emitServerError() { - //TODO: implement +function emitServerError(e) { + throw e; } /** @@ -315,7 +321,7 @@ function emitSuccess(executionOutput, junit) { * Emit `result` event with timeout values. * * @param executionOutput {ExecutionOutput}: Output of a run command execution - * @param stage {'compile' || 'runtime'}[Optional]: In which stage this function is called + * @param stage {"compile" | "runtime"}[Optional]: In which stage this function is called */ function emitTimeout(executionOutput, stage) { const feedback = executionOutput; @@ -330,6 +336,14 @@ function emitTimeout(executionOutput, stage) { * @param stage {'compile' || 'runtime'}[Optional]: In which stage this function is called */ function emitWrong(executionOutput, stage) { - const feedback = executionOutput; - javaBox.emit('result', feedback); -} \ No newline at end of file + javaBox.emit('result', executionOutput); +} + + +/** + * Initialising javaBox. + */ +javaBox.on('runJava', runJava); +javaBox.on('runJunit', runJunit); + +module.exports = javaBox; \ No newline at end of file diff --git a/package.json b/package.json index b829294..eecceff 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "javabox", "version": "1.1.0", - "description": "Server to procede java code", + "description": "Server for compiling and running Java code", "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", diff --git a/queue.js b/queue.js index f28beb4..e14ad93 100644 --- a/queue.js +++ b/queue.js @@ -26,7 +26,7 @@ const queue = new Agenda(); * * @return {Promise}: A promise for starting the queue */ -queue.prototype.initialize = function (queueParams, processMessageJob) { +Agenda.prototype.initialize = function (queueParams, processMessageJob) { return new Promise((resolve, reject) => { @@ -41,7 +41,7 @@ queue.prototype.initialize = function (queueParams, processMessageJob) { }); }; -queue.prototype.addMessage = function (request) { +Agenda.prototype.addMessage = function (request) { const messageId = createMessageId(request); const jobData = { diff --git a/server.js b/server.js index fd7a981..de2f462 100644 --- a/server.js +++ b/server.js @@ -65,7 +65,7 @@ const init = coroutine(function*(port, mongoAddress, mongoCollection, defaultCon server.on('result', sendResult); yield server.listen(port); - console.log(`server listening on port ${port}...`); + console.log(`Server listening on port ${port}...`); return server; }); diff --git a/test/client.js b/test/client.js index 6513e6e..689d082 100644 --- a/test/client.js +++ b/test/client.js @@ -110,8 +110,8 @@ function makeConnection(clientId) { const message = { clientId : clientId, submission : submission, - compileTimeoutMs : 60000, - executionTimeoutMs : 60000, + compileTimeoutMs: 6000, + executionTimeoutMs: 600, charactersMaxLength: 1000 }; @@ -131,9 +131,8 @@ function makeConnection(clientId) { console.log(`Clients left: ${clientsLeft}\n \n`); }); }); -}; - +} for (var i = 0; i < clientsNumber; i++) { const clientId = 'client' + i; makeConnection(clientId); -}; +} From d24fde9cded42f3c7a39a12870a7d680ab7b0403 Mon Sep 17 00:00:00 2001 From: StKyr Date: Wed, 5 Jul 2017 04:42:40 +0300 Subject: [PATCH 09/11] Stdout and stderr demuxed and returned properly --- containerOutputStream.js | 30 ++++++++++++++++++++++++++++++ javaBox.js | 34 +++++++++++++++------------------- server.js | 2 +- 3 files changed, 46 insertions(+), 20 deletions(-) create mode 100644 containerOutputStream.js diff --git a/containerOutputStream.js b/containerOutputStream.js new file mode 100644 index 0000000..638fa18 --- /dev/null +++ b/containerOutputStream.js @@ -0,0 +1,30 @@ +/** + * Module for creating custom writable streams in order to get the stdout and stderr from the containers + */ + +const stream = require('stream'); +const util = require('util'); +const Writable = stream.Writable; + + +function OutputStream(options) { + if (!(this instanceof OutputStream)) return new OutputStream(options); + + Writable.call(this, options); + this.memStore = new Buffer(''); + + this.toString = function () { + return this.memStore.toString() + } +} + +util.inherits(OutputStream, Writable); + +OutputStream.prototype._write = function (chunk, enc, cb) { + + const buffer = (Buffer.isBuffer(chunk)) ? chunk : new Buffer(chunk, enc); + this.memStore = Buffer.concat([this.memStore, buffer]); + cb(); +}; + +module.exports = OutputStream; \ No newline at end of file diff --git a/javaBox.js b/javaBox.js index fb10ded..6e41671 100644 --- a/javaBox.js +++ b/javaBox.js @@ -4,6 +4,10 @@ const coroutine = Promise.coroutine; //const Docker = Promise.promisifyAll(require('dockerode')); const Docker = require('dockerode'); const concat = require('concat-stream'); +const OutputStream = require('./containerOutputStream'); + + + /** @@ -139,6 +143,7 @@ const runJunit = coroutine(function*(messageId, junitFileNames, tarBuffer, timeL }); + const initializeContainer = coroutine(function*(tarBuffer, messageId, junit) { junit = junit || false; @@ -176,34 +181,25 @@ const initializeContainer = coroutine(function*(tarBuffer, messageId, junit) { const runCommand = coroutine(function *(command, container, executionTimeLimit) { let timeout = null; - let stdoutData = ''; - let stderrData = ''; - const re = /[\0-\x1F\x7F-\x9F\xAD\u0378\u0379\u037F-\u0383\u038B\u038D\u03A2\u0528-\u0530\u0557\u0558\u0560\u0588\u058B-\u058E\u0590\u05C8-\u05CF\u05EB-\u05EF\u05F5-\u0605\u061C\u061D\u06DD\u070E\u070F\u074B\u074C\u07B2-\u07BF\u07FB-\u07FF\u082E\u082F\u083F\u085C\u085D\u085F-\u089F\u08A1\u08AD-\u08E3\u08FF\u0978\u0980\u0984\u098D\u098E\u0991\u0992\u09A9\u09B1\u09B3-\u09B5\u09BA\u09BB\u09C5\u09C6\u09C9\u09CA\u09CF-\u09D6\u09D8-\u09DB\u09DE\u09E4\u09E5\u09FC-\u0A00\u0A04\u0A0B-\u0A0E\u0A11\u0A12\u0A29\u0A31\u0A34\u0A37\u0A3A\u0A3B\u0A3D\u0A43-\u0A46\u0A49\u0A4A\u0A4E-\u0A50\u0A52-\u0A58\u0A5D\u0A5F-\u0A65\u0A76-\u0A80\u0A84\u0A8E\u0A92\u0AA9\u0AB1\u0AB4\u0ABA\u0ABB\u0AC6\u0ACA\u0ACE\u0ACF\u0AD1-\u0ADF\u0AE4\u0AE5\u0AF2-\u0B00\u0B04\u0B0D\u0B0E\u0B11\u0B12\u0B29\u0B31\u0B34\u0B3A\u0B3B\u0B45\u0B46\u0B49\u0B4A\u0B4E-\u0B55\u0B58-\u0B5B\u0B5E\u0B64\u0B65\u0B78-\u0B81\u0B84\u0B8B-\u0B8D\u0B91\u0B96-\u0B98\u0B9B\u0B9D\u0BA0-\u0BA2\u0BA5-\u0BA7\u0BAB-\u0BAD\u0BBA-\u0BBD\u0BC3-\u0BC5\u0BC9\u0BCE\u0BCF\u0BD1-\u0BD6\u0BD8-\u0BE5\u0BFB-\u0C00\u0C04\u0C0D\u0C11\u0C29\u0C34\u0C3A-\u0C3C\u0C45\u0C49\u0C4E-\u0C54\u0C57\u0C5A-\u0C5F\u0C64\u0C65\u0C70-\u0C77\u0C80\u0C81\u0C84\u0C8D\u0C91\u0CA9\u0CB4\u0CBA\u0CBB\u0CC5\u0CC9\u0CCE-\u0CD4\u0CD7-\u0CDD\u0CDF\u0CE4\u0CE5\u0CF0\u0CF3-\u0D01\u0D04\u0D0D\u0D11\u0D3B\u0D3C\u0D45\u0D49\u0D4F-\u0D56\u0D58-\u0D5F\u0D64\u0D65\u0D76-\u0D78\u0D80\u0D81\u0D84\u0D97-\u0D99\u0DB2\u0DBC\u0DBE\u0DBF\u0DC7-\u0DC9\u0DCB-\u0DCE\u0DD5\u0DD7\u0DE0-\u0DF1\u0DF5-\u0E00\u0E3B-\u0E3E\u0E5C-\u0E80\u0E83\u0E85\u0E86\u0E89\u0E8B\u0E8C\u0E8E-\u0E93\u0E98\u0EA0\u0EA4\u0EA6\u0EA8\u0EA9\u0EAC\u0EBA\u0EBE\u0EBF\u0EC5\u0EC7\u0ECE\u0ECF\u0EDA\u0EDB\u0EE0-\u0EFF\u0F48\u0F6D-\u0F70\u0F98\u0FBD\u0FCD\u0FDB-\u0FFF\u10C6\u10C8-\u10CC\u10CE\u10CF\u1249\u124E\u124F\u1257\u1259\u125E\u125F\u1289\u128E\u128F\u12B1\u12B6\u12B7\u12BF\u12C1\u12C6\u12C7\u12D7\u1311\u1316\u1317\u135B\u135C\u137D-\u137F\u139A-\u139F\u13F5-\u13FF\u169D-\u169F\u16F1-\u16FF\u170D\u1715-\u171F\u1737-\u173F\u1754-\u175F\u176D\u1771\u1774-\u177F\u17DE\u17DF\u17EA-\u17EF\u17FA-\u17FF\u180F\u181A-\u181F\u1878-\u187F\u18AB-\u18AF\u18F6-\u18FF\u191D-\u191F\u192C-\u192F\u193C-\u193F\u1941-\u1943\u196E\u196F\u1975-\u197F\u19AC-\u19AF\u19CA-\u19CF\u19DB-\u19DD\u1A1C\u1A1D\u1A5F\u1A7D\u1A7E\u1A8A-\u1A8F\u1A9A-\u1A9F\u1AAE-\u1AFF\u1B4C-\u1B4F\u1B7D-\u1B7F\u1BF4-\u1BFB\u1C38-\u1C3A\u1C4A-\u1C4C\u1C80-\u1CBF\u1CC8-\u1CCF\u1CF7-\u1CFF\u1DE7-\u1DFB\u1F16\u1F17\u1F1E\u1F1F\u1F46\u1F47\u1F4E\u1F4F\u1F58\u1F5A\u1F5C\u1F5E\u1F7E\u1F7F\u1FB5\u1FC5\u1FD4\u1FD5\u1FDC\u1FF0\u1FF1\u1FF5\u1FFF\u200B-\u200F\u202A-\u202E\u2060-\u206F\u2072\u2073\u208F\u209D-\u209F\u20BB-\u20CF\u20F1-\u20FF\u218A-\u218F\u23F4-\u23FF\u2427-\u243F\u244B-\u245F\u2700\u2B4D-\u2B4F\u2B5A-\u2BFF\u2C2F\u2C5F\u2CF4-\u2CF8\u2D26\u2D28-\u2D2C\u2D2E\u2D2F\u2D68-\u2D6E\u2D71-\u2D7E\u2D97-\u2D9F\u2DA7\u2DAF\u2DB7\u2DBF\u2DC7\u2DCF\u2DD7\u2DDF\u2E3C-\u2E7F\u2E9A\u2EF4-\u2EFF\u2FD6-\u2FEF\u2FFC-\u2FFF\u3040\u3097\u3098\u3100-\u3104\u312E-\u3130\u318F\u31BB-\u31BF\u31E4-\u31EF\u321F\u32FF\u4DB6-\u4DBF\u9FCD-\u9FFF\uA48D-\uA48F\uA4C7-\uA4CF\uA62C-\uA63F\uA698-\uA69E\uA6F8-\uA6FF\uA78F\uA794-\uA79F\uA7AB-\uA7F7\uA82C-\uA82F\uA83A-\uA83F\uA878-\uA87F\uA8C5-\uA8CD\uA8DA-\uA8DF\uA8FC-\uA8FF\uA954-\uA95E\uA97D-\uA97F\uA9CE\uA9DA-\uA9DD\uA9E0-\uA9FF\uAA37-\uAA3F\uAA4E\uAA4F\uAA5A\uAA5B\uAA7C-\uAA7F\uAAC3-\uAADA\uAAF7-\uAB00\uAB07\uAB08\uAB0F\uAB10\uAB17-\uAB1F\uAB27\uAB2F-\uABBF\uABEE\uABEF\uABFA-\uABFF\uD7A4-\uD7AF\uD7C7-\uD7CA\uD7FC-\uF8FF\uFA6E\uFA6F\uFADA-\uFAFF\uFB07-\uFB12\uFB18-\uFB1C\uFB37\uFB3D\uFB3F\uFB42\uFB45\uFBC2-\uFBD2\uFD40-\uFD4F\uFD90\uFD91\uFDC8-\uFDEF\uFDFE\uFDFF\uFE1A-\uFE1F\uFE27-\uFE2F\uFE53\uFE67\uFE6C-\uFE6F\uFE75\uFEFD-\uFF00\uFFBF-\uFFC1\uFFC8\uFFC9\uFFD0\uFFD1\uFFD8\uFFD9\uFFDD-\uFFDF\uFFE7\uFFEF-\uFFFB\uFFFE\uFFFF]/g; - + const stdoutStream = new OutputStream(); + const stderrStream = new OutputStream(); try { // possible errors with docker container - const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: true}; // execution options - const exec = Promise.promisifyAll(yield container.execAsync(execOpts)); // create execution of command - - const stream = yield exec.startAsync(); // start execution (get an output stream) + const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: false}; // execution options + const exec = Promise.promisifyAll(yield container.execAsync(execOpts)); // create execution of command - if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time + const stream = yield exec.startAsync(); // start execution (get an output stream) + container.modem.demuxStream(stream, stdoutStream, stderrStream); // intercept container output to our streams + if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time - let data = []; - stream.on('data', chunk => { - data.push(chunk); - }); // get each output chunk - stream.on('end', () => { - stdoutData = Buffer.concat(data).toString().replace(re, "") - }); // finalize them into a string - // no need for promisified code here. It will introduce loops making it less readable let executionData = yield exec.inspectAsync(); while (executionData.Running) { // loop (asynchronously) until execution stops executionData = yield exec.inspectAsync(); } + const executedSuccessfully = (executionData.ExitCode == 0); // set successful execution mark if (executionTimeLimit) clearTimeout(timeout); @@ -211,8 +207,8 @@ const runCommand = coroutine(function *(command, container, executionTimeLimit) return { messageId: container.messageId, success: executedSuccessfully, - output: stdoutData, - errorMessage: stderrData, + output: stdoutStream.toString(), + errorMessage: stderrStream.toString(), timeout: false } diff --git a/server.js b/server.js index de2f462..815b10b 100644 --- a/server.js +++ b/server.js @@ -113,7 +113,7 @@ function processMessageJob(job, done) { sendResultForBadRequest(messageId); } - done(); // not needed, useful for async code + //done(); // not needed, useful for async code } /** * Checks whether a request is valid or not, given the request specification From c9e0922b98249ec1486dedf15193a51a380f70fd Mon Sep 17 00:00:00 2001 From: StKyr Date: Wed, 5 Jul 2017 04:43:08 +0300 Subject: [PATCH 10/11] Promisifed version compiles and runs. Minor bugfixes needed. --- test/client.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/client.js b/test/client.js index 689d082..a185b68 100644 --- a/test/client.js +++ b/test/client.js @@ -12,7 +12,7 @@ const host = '127.0.0.1'; */ const commandLineDef = [ { name: 'clients', alias: 'c', type: Number, defaultValue: 1}, - { name: 'submission', alias: 's', type: String, defaultValue: 'hwSub'} + {name: 'submission', alias: 's', type: String, defaultValue: 'exceptionPlusOutput'} ]; /** * Object that for keys has command line argument names @@ -53,6 +53,7 @@ function simpleSubmission(command) { const submissions = { hwSub: simpleSubmission(`System.out.println("Hello world."); `), infiniteSub: simpleSubmission(`while (true) { System.out.println("To infinity and beyond!"); }`), + exceptionPlusOutput: simpleSubmission("System.out.println(\"Standard output message\" ); String s=null; s.toString();"), rmSub: { main: 'RemoveSub.java', files: [{ @@ -79,7 +80,8 @@ const submissions = { name: 'TestJunit2.java', data: fs.readFileSync('src/TestJunit2.java', 'utf8'), }] - } + }, + }; From 09d3dc0296ef494660feb58c2ac1f8001644ebd0 Mon Sep 17 00:00:00 2001 From: StKyr Date: Thu, 6 Jul 2017 06:10:38 +0300 Subject: [PATCH 11/11] Timeouts (and corresponding messages) supported, containers are now stopped (better) --- app.js | 1 + javaBox.js | 137 ++++++++++++++++++++++--------------------------- server.js | 6 ++- test/client.js | 8 +-- 4 files changed, 70 insertions(+), 82 deletions(-) diff --git a/app.js b/app.js index 1afe2d9..fd9bb88 100644 --- a/app.js +++ b/app.js @@ -109,6 +109,7 @@ const main = coroutine(function*() { /** * Specifying the server and javaBox control flow */ + server.on('runJunit', runJunit); server.on('runJava', runJava); javaBox.on('result', giveFeedBack); diff --git a/javaBox.js b/javaBox.js index 6e41671..fdbaa7a 100644 --- a/javaBox.js +++ b/javaBox.js @@ -46,6 +46,7 @@ const javaBox = new EventEmitter(); */ const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompileMs, timeLimitExecutionMs) { + const className = main.split('.')[0]; let container = yield initializeContainer(tarBuffer, messageId); @@ -54,39 +55,37 @@ const runJava = coroutine(function*(messageId, main, tarBuffer, timeLimitCompile const javaCmd = ['java', '-Djava.security.manager', '-cp', 'home', className]; try { - const compileOutput = yield runCommand(javacCmd, container, timeLimitCompileMs); + const compileOutput = yield runCommand(javacCmd, container, timeLimitCompileMs, 'compilation'); if (executedCorrectly(compileOutput)) { - const runtimeOutput = yield runCommand(javaCmd, container, timeLimitExecutionMs); + const runtimeOutput = yield runCommand(javaCmd, container, timeLimitExecutionMs, 'execution'); if (executedCorrectly(runtimeOutput)) { //TODO: maybe also return compileOutput for warnings? emitSuccess(runtimeOutput); } else { - - if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); + if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'execution'); else emitWrong(runtimeOutput, 'Runtime'); } } else { - - if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); + if (compileOutput.timeout) emitTimeout(compileOutput, 'compilation'); else emitWrong(compileOutput, 'Compile'); } + yield container.stopAsync(); + yield container.removeAsync({f: true}); - yield container.killAsync(); - yield container.removeAsync({v: true}); } catch (e) { // internal error //TODO: specify javaBox behavior for internal error console.log('500: Internal error with Docker!'); emitServerError(e); - yield container.kill(); - yield container.remove({v: true}); + yield container.killAsync({t: 0}); + yield container.removeAsync({f: true}); } }); @@ -112,33 +111,33 @@ const runJunit = coroutine(function*(messageId, junitFileNames, tarBuffer, timeL }); try { - const compileOutput = yield runCommand(javacCmd, container, timeLimitCompileMs); + const compileOutput = yield runCommand(javacCmd, container, timeLimitCompileMs, 'compilation'); if (executedCorrectly(compileOutput)) { - const runtimeOutput = yield runCommand(javaCmd, container, timeLimitExecutionMs); + const runtimeOutput = yield runCommand(javaCmd, container, timeLimitExecutionMs, 'execution'); if (executedCorrectly(runtimeOutput)) { emitSuccess(runtimeOutput, true); } else { - if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'Runtime'); + if (runtimeOutput.timeout) emitTimeout(runtimeOutput, 'execution'); else emitWrong(runtimeOutput, 'Runtime'); } } else { - if (compileOutput.timeout) emitTimeout(compileOutput, 'Compile'); + if (compileOutput.timeout) emitTimeout(compileOutput, 'compilation'); else emitWrong(compileOutput, 'Compile'); } - yield container.killAsync(); - yield container.removeAsync({v: true}); + yield container.stopAsync(); + yield container.removeAsync({f: true}); } catch (e) { // internal error //TODO: specify javaBox behavior for internal error - console.log('501: Internal error with Docker!'); + console.log('500: Internal error with Docker!'); emitServerError(e); - yield container.kill(); - yield container.remove({v: true}); + yield container.killAsync({t: 0}); + yield container.removeAsync({f: true}); } }); @@ -180,53 +179,44 @@ const initializeContainer = coroutine(function*(tarBuffer, messageId, junit) { */ const runCommand = coroutine(function *(command, container, executionTimeLimit) { - let timeout = null; + let timeoutCallback = null; + let timeoutHappend = false; const stdoutStream = new OutputStream(); const stderrStream = new OutputStream(); - try { // possible errors with docker container - const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: false}; // execution options - const exec = Promise.promisifyAll(yield container.execAsync(execOpts)); // create execution of command - const stream = yield exec.startAsync(); // start execution (get an output stream) - container.modem.demuxStream(stream, stdoutStream, stderrStream); // intercept container output to our streams + const execOpts = {Cmd: command, AttachStdout: true, AttachStderr: true, Tty: false}; // execution options + const exec = Promise.promisifyAll(yield container.execAsync(execOpts)); // create execution of command - if (executionTimeLimit) timeout = setTimeout(throwTimeOutError, executionTimeLimit); // start keeping time + const stream = yield exec.startAsync(); // start execution (get an output stream) + container.modem.demuxStream(stream, stdoutStream, stderrStream); // intercept container output to our streams - - let executionData = yield exec.inspectAsync(); - while (executionData.Running) { // loop (asynchronously) until execution stops - executionData = yield exec.inspectAsync(); - } + if (executionTimeLimit) { // start keeping time + timeoutCallback = setTimeout(() => { + timeoutHappend = true + }, executionTimeLimit); // prepare timeout termination + } - const executedSuccessfully = (executionData.ExitCode == 0); // set successful execution mark + let executionData = yield exec.inspectAsync(); + while (executionData.Running && timeoutHappend == false) { // loop (asynchronously) until execution stops + executionData = yield exec.inspectAsync(); + } - if (executionTimeLimit) clearTimeout(timeout); - return { - messageId: container.messageId, - success: executedSuccessfully, - output: stdoutStream.toString(), - errorMessage: stderrStream.toString(), - timeout: false - } + const executedSuccessfully = (executionData.ExitCode == 0 && timeoutHappend == false); // set successful execution mark - } catch (e) { - container.stopAsync(); + if (executionTimeLimit) clearTimeout(timeoutCallback); - if (e.name == 'timeout') { - return { - messageId: container.messageId, - success: false, - output: stdoutData, - errorMessage: stderrData, - timeout: true - } - } else { - throw e; - } + return { + messageId: container.messageId, + success: executedSuccessfully, + output: stdoutStream.toString(), + errorMessage: stderrStream.toString(), + timeout: timeoutHappend } + + }); /** @@ -273,22 +263,6 @@ function executedCorrectly(result) { return (result.success && !result.timeout); } -/** - * Dummy function to raise (throw) a custom timeout error - * @param stage {String} [Optional]: The stage in which timeout happened. - * @throws timeout {Object} - * @type timeout.name {string} - * @type timeout.stage {string} - */ -function throwTimeOutError(stage) { - stage = stage || ''; - throw { - name: 'timeout', - stage: stage - }; -} - - function emitServerError(e) { throw e; } @@ -314,22 +288,31 @@ function emitSuccess(executionOutput, junit) { } /** - * Emit `result` event with timeout values. + * Emit `result` event with timeout values and error message. * - * @param executionOutput {ExecutionOutput}: Output of a run command execution - * @param stage {"compile" | "runtime"}[Optional]: In which stage this function is called + * @param feedback {Object}: The feedback with all the info to be emitted back. + * @param stage {String}[Optional]: In which stage this function is called. */ -function emitTimeout(executionOutput, stage) { - const feedback = executionOutput; - feedback.errorMessage = `${stage} timeout reached.`; +function emitTimeout(feedback, stage) {//messageId, stdoutStream, stderrStream, stage) { + + + feedback.errorMessage += '\n>>>JavaBox: Timeout reached'; + + if (stage) { + feedback.errorMessage += ` during ${stage}.`; + } else { + feedback.errorMessage += '.'; + } + javaBox.emit('result', feedback); + } /** * Emit `result` event with error values. * - * @param executionOutput {ExecutionOutput}: Output of a run command execution - * @param stage {'compile' || 'runtime'}[Optional]: In which stage this function is called + * @param executionOutput {ExecutionOutput}: Output of a run command execution. + * @param stage {String}[Optional]: In which stage this function is called. */ function emitWrong(executionOutput, stage) { javaBox.emit('result', executionOutput); diff --git a/server.js b/server.js index 815b10b..7b918b0 100644 --- a/server.js +++ b/server.js @@ -65,7 +65,7 @@ const init = coroutine(function*(port, mongoAddress, mongoCollection, defaultCon server.on('result', sendResult); yield server.listen(port); - console.log(`Server listening on port ${port}...`); + console.log(`JavaBox listening on port ${port}...`); return server; }); @@ -79,6 +79,7 @@ const init = coroutine(function*(port, mongoAddress, mongoCollection, defaultCon */ function initSocket(connection) { + const socket = new JsonSocket(connection); socket.on('message', addMessageInQueue.bind(null, socket)); } @@ -160,6 +161,7 @@ function sendResultForBadRequest(messageId) { */ function executeRequest(request, messageId) { + console.log('executing request for ' + messageId); const main = request.submission.main; const files = request.submission.files; const tests = request.submission.tests; @@ -184,6 +186,8 @@ function executeRequest(request, messageId) { */ function sendResult(feedback) { + console.log('>>sending back to ' + feedback.messageId); + const message = messages[feedback.messageId]; const socket = message.socket; const clientId = message.request.clientId; diff --git a/test/client.js b/test/client.js index a185b68..66718ed 100644 --- a/test/client.js +++ b/test/client.js @@ -11,8 +11,8 @@ const host = '127.0.0.1'; * @type {[*]} */ const commandLineDef = [ - { name: 'clients', alias: 'c', type: Number, defaultValue: 1}, - {name: 'submission', alias: 's', type: String, defaultValue: 'exceptionPlusOutput'} + {name: 'clients', alias: 'c', type: Number, defaultValue: 50}, + {name: 'submission', alias: 's', type: String, defaultValue: 'infiniteSub'} ]; /** * Object that for keys has command line argument names @@ -112,8 +112,8 @@ function makeConnection(clientId) { const message = { clientId : clientId, submission : submission, - compileTimeoutMs: 6000, - executionTimeoutMs: 600, + compileTimeoutMs: 15000, + executionTimeoutMs: 6000, charactersMaxLength: 1000 };