diff --git a/AVAVerify.js b/AVAVerify.js index dc9b384..46e4cfd 100644 --- a/AVAVerify.js +++ b/AVAVerify.js @@ -1,6 +1,8 @@ const AVAVerifyConfig = require("./config"); const jsverify = require("jsverify"); const assert = require("assert"); +const FailedTestError = require("./lib/FailedTestError"); +const TestDetails = require("./lib/TestDetails"); let unnamedSuites = 0; @@ -59,16 +61,60 @@ class AVAVerify { }; } + /** + * Shrink and retry a failing test. + * @param {Test} t the `Test` object from an AVA test + * @param {TestDetails} details the details about the current test. + * @return {Promise} resolves when testing is complete. + * @todo Implement. + */ + retryBody(t, details) { + + } + /** * The contents to use inside an AVA `test` block to run a test instance. * @param {Number} i the test index - * @todo Implement. + * @todo Publicize `TestDetails` object being in `t.context` (warn that it is unstable) * @return {Function} the `test` block contents. */ testBody(i) { return (t) => { - return this.constructor.genArbs(this.opts.size, this.arbs) - .then(vals => this.body(t, ...vals)); + const details = new TestDetails({ + verify: this, + index: i, + }); + t.context._avaVerify = details; + return this.constructor + .genArbs(this.opts.size, this.arbs) + .then(vals => details.storeFirstValues(vals)) + .then(vals => this.body(t, ...vals)) + .then(() => { + if(t.assertError || t.calledEnd) { + throw new FailedTestError(); + } + // TODO: broadcast test success + }) + .catch(err => { + details.failed = true; + details.failureDetails = this.constructor.getFailureDetails(t); + // TODO: catch any other test failures? err.name === AssertionError? + const retriable = err instanceof FailedTestError; + if(details.shrink && retriable) { + return this + .retryBody(t, details) + .then(() => this.constructor.restoreFailureDetails(t, details.failureDetails)) + .then(() => { + //TODO: broadcast test finish + //TODO: if test rejected above, reject again? + return true; + }); + } + else { + //TODO: broadcast test finish + throw err; + } + }); }; } diff --git a/lib/Assert.js b/lib/Assert.js deleted file mode 100644 index 07170da..0000000 --- a/lib/Assert.js +++ /dev/null @@ -1,34 +0,0 @@ -const assert = require("assert"); - -/** - * An internal assertion library. Allows custom assertions (like using AVA's `t` object). -*/ -class Assert { - - /** - * Assert that `val` is `true`. - * @param {Boolean} val will ensure value is `true` - * @param {String} [msg] optional message to include. - */ - true(val, msg) { - return assert.ok(val, msg); - } - -} - -class AVAAssert extends Assert { - - /** - * @param {Test} t AVA's `t` object. - */ - constructor(t) { - this.t = t; - } - - true(val, msg) { - this.t.true(val, msg); - } - -} - -module.exports = { Assert, AVAAssert }; diff --git a/lib/ExtendableError.js b/lib/ExtendableError.js new file mode 100644 index 0000000..2e73294 --- /dev/null +++ b/lib/ExtendableError.js @@ -0,0 +1,17 @@ +/** + * From https://stackoverflow.com/a/32749533/666727 +*/ +class ExtendableError extends Error { + constructor(msg) { + super(message); + this.name = this.constructor.name; + if(typeof Error.captureStackTrace === "function") { + Error.captureStackTrace(this, this.constructor); + } + else { + this.stack = (new Error(message)).stack; + } + } +} + +module.exports = ExtendableError; diff --git a/lib/FailedTestError.js b/lib/FailedTestError.js new file mode 100644 index 0000000..c71e4d9 --- /dev/null +++ b/lib/FailedTestError.js @@ -0,0 +1,6 @@ +const ExtendableError = require("./ExtendableError"); + +class FailedTestError extends ExtendableError { +} + +module.exports = FailedTestError; diff --git a/lib/FailureDetails.js b/lib/FailureDetails.js new file mode 100644 index 0000000..8be43c9 --- /dev/null +++ b/lib/FailureDetails.js @@ -0,0 +1,77 @@ +/** + * Stores details about an AVA test failure, which can be used to save and reset variables when re-running tests. + * @property {Number} assertCount + * @property {Error} [assertError] + * @property {Boolean} calledEnd + * @property {Number} [duration] + * @property {Number} pendingAssertionCount + * @property {Number} [planCount] + * @property {Number} startedAt + * @see https://github.com/avajs/ava/blob/854203a728c4dff3ea61a3bcf49f1dfae04c7657/lib/test.js#L103 + * @todo Write unit tests +*/ +class FailureDetails { + + /** + * The test properties that need to be saved and reset between test attempts. + */ + static get testProperties() { + return [ + "assertCount", + "assertError", + "calledEnd", + "duration", + "pendingAssertionCount", + "planCount", + "startedAt", + ]; + } + + /** + * Copy the test details that should be saved between test attempts. + * @param {Test|FailureDetails} source the object to copy details from + * @param {Test|FailureDetails} dest the object to store details in + * @return {Test|FailureDetails} the destination object + */ + static copyTestDetails(source, dest) { + for(const property of this.testProperties) { + dest[property] = source[property]; + } + return dest; + } + + /** + * Extract the internal AVA details after a test failure. + * @param {Test} t the current Test object (`t` in AVA) + * @return {FailureDetails} details about the test failure. + * @todo Create and use a `typedef` for the return type. Use in other methods. + */ + static getFailureDetails(t) { + return this.copyTestDetails(t, new FailureDetails()); + } + + /** + * Restore the internal AVA details after retrying tests. + * @param {Test} t the current Test object (`t` in AVA) + * @param {FailureDetails} failureDetails the saved details about the test failure + * @return {Test} the given Test object. + */ + static restoreFailureDetails(t, failureDetails) { + return this.copyTestDetails(failureDetails, t); + } + + /** + * Reset the internal AVA details to prepare a clean testing environment. + * Creates a `{Test}` object from AVA, and copies it's default properties into the given `{Test}`. + * @param {Test} t the current Test object (`t` in AVA) + * @return {Test} the given Test object. + */ + static resetTest(t) { + const AVATest = require("ava/lib/test"); + const _test = new AVATest(); + return this.copyTestDetails(_test, t); + } + +} + +module.exports = FailureDetails; diff --git a/lib/TestDetails.js b/lib/TestDetails.js new file mode 100644 index 0000000..3eff831 --- /dev/null +++ b/lib/TestDetails.js @@ -0,0 +1,41 @@ +const merge = require("lodash.merge"); +const FailureDetails = require("./FailureDetails"); + +/** + * Internal data about a single test instance. + * @property {Number} index the index of this test in the {@link AVAVerify} suite. + * @property {Number} attempt the number of times this test instance has been tried. `0` for the first run, and `1` for + * the first retry. Defaults to `0`. + * @property {Boolean} shrink user can set this to `false` to disable shrinking. Shrinking will not be attempted if + * either this is set, or other internal checks fail. Defaults to `true`. + * @property {AVAVerify} [verify] a link back to the {@link AVAVerify} instance. Not to be used inside AVAVerify, but + * can be used by user tests if needed. + * @property {Boolean} failed `true` if the first attempt failed. + * @property {FailureDetails} failureDetails details from the first failing run. + * @property {Any[]} firstValues the generated values for the first run of this test. +*/ +class TestDetails { + + constructor(opts) { + merge(this, { + attempt: 0, + shrink: true, + failed: false, + failureDetails: new FailureDetails(), + }, opts); + if(typeof this.index !== "number") { throw new TypeError(`'index' is a required field. Given ${typeof index}.`); } + } + + /** + * Store the initial generated values for this test. + * @param {Any[]} vals the generated values + * @return {Any[]} the given values + */ + storeFirstValues(vals) { + this.firstValues = vals; + return vals; + } + +} + +module.exports = TestDetails; diff --git a/package.json b/package.json index e43e754..6cfa3a4 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "jsverify": "^0.8.2" }, "dependencies": { + "lodash.merge": "^4.6.0", "modconf": "0.0.1" }, "devDependencies": {