diff --git a/src/classes/TestResult.js b/src/classes/TestResult.js index 76a11f2..78ade2a 100644 --- a/src/classes/TestResult.js +++ b/src/classes/TestResult.js @@ -83,37 +83,59 @@ export default class TestResult extends BubblingEventTarget { * Run the test(s) */ async run () { - this.messages = await interceptConsole(async () => { - if (!this.parent) { - // We are running the test in isolation, so we need to run beforeAll (if it exists) - await this.test.beforeAll?.(); - } + let test = this.test; + + // By default, give the test 10 seconds to run + let timeout = 10000; + if (test.maxTime && ("expect" in test || test.throws !== undefined)) { + // For result-based and error-based tests, maxTime is the timeout + timeout = test.maxTime; + } - await this.test.beforeEach?.(); + let timeoutId; + this.messages = await Promise.race([ + new Promise(resolve => { + timeoutId = setTimeout(() => { + this.error = new Error(`Test timed out after ${ timeout }ms`); + this.timeTaken = timeout; + resolve([]); + }, timeout); + }), + + interceptConsole(async () => { + if (!this.parent) { + // We are running the test in isolation, so we need to run beforeAll (if it exists) + await test.beforeAll?.(); + } - let start = performance.now(); + await test.beforeEach?.(); - try { - this.actual = this.test.run ? this.test.run.apply(this.test, this.test.args) : this.test.args[0]; - this.timeTaken = performance.now() - start; + let start = performance.now(); - if (this.actual instanceof Promise) { - this.actual = await this.actual; - this.timeTakenAsync = performance.now() - start; + try { + this.actual = test.run ? test.run.apply(test, test.args) : test.args[0]; + this.timeTaken = performance.now() - start; + + if (this.actual instanceof Promise) { + this.actual = await this.actual; + this.timeTakenAsync = performance.now() - start; + } } - } - catch (e) { - this.error = e; - } - finally { - await this.test.afterEach?.(); + catch (e) { + this.error = e; + } + finally { + await test.afterEach?.(); - if (!this.parent) { + if (!this.parent) { // We are running the test in isolation, so we need to run afterAll - await this.test.afterAll?.(); + await test.afterAll?.(); + } } - } - }); + }), + ]); + + clearTimeout(timeoutId); this.evaluate(); } @@ -183,13 +205,14 @@ export default class TestResult extends BubblingEventTarget { evaluate () { let test = this.test; + if (test.maxTime || test.maxTimeAsync) { + Object.assign(this, this.evaluateTimeTaken()); + } + if (test.throws !== undefined) { Object.assign(this, this.evaluateThrown()); } - else if (test.maxTime || test.maxTimeAsync) { - Object.assign(this, this.evaluateTimeTaken()); - } - else { + else if ("expect" in test) { Object.assign(this, this.evaluateResult()); } @@ -209,7 +232,7 @@ export default class TestResult extends BubblingEventTarget { */ evaluateThrown () { let test = this.test; - let ret = {pass: !!this.error, details: []}; + let ret = {pass: (this.pass ?? true) && !!this.error, details: this.details ?? []}; // We may have more picky criteria for the error if (ret.pass) { @@ -251,38 +274,42 @@ export default class TestResult extends BubblingEventTarget { */ evaluateResult () { let test = this.test; - let ret = {pass: true, details: []}; - if (test.map) { - try { - this.mapped = { - actual: Array.isArray(this.actual) ? this.actual.map(test.map) : test.map(this.actual), - expect: Array.isArray(test.expect) ? test.expect.map(test.map) : test.map(test.expect), - }; + // If we are here and there is an error (e.g., the test timed out), we consider the test failed + let ret = {pass: (this.pass ?? true) && !this.error, details: this.details ?? []}; + if (ret.pass) { + if (test.map) { try { - ret.pass = test.check(this.mapped.actual, this.mapped.expect); + this.mapped = { + actual: Array.isArray(this.actual) ? this.actual.map(test.map) : test.map(this.actual), + expect: Array.isArray(test.expect) ? test.expect.map(test.map) : test.map(test.expect), + }; + + try { + ret.pass = test.check(this.mapped.actual, this.mapped.expect); + } + catch (e) { + this.error = new Error(`check() failed (working with mapped values). ${ e.message }`); + } } catch (e) { - this.error = new Error(`check() failed (working with mapped values). ${ e.message }`); + this.error = new Error(`map() failed. ${ e.message }`); } } - catch (e) { - this.error = new Error(`map() failed. ${ e.message }`); - } - } - else { - try { - ret.pass = test.check(this.actual, test.expect); - } - catch (e) { - this.error = new Error(`check() failed. ${ e.message }`); + else { + try { + ret.pass = test.check(this.actual, test.expect); + } + catch (e) { + this.error = new Error(`check() failed. ${ e.message }`); + } } - } - // If `map()` or `check()` errors, consider the test failed - if (this.error) { - ret.pass = false; + // If `map()` or `check()` errors, consider the test failed + if (this.error) { + ret.pass = false; + } } if (!ret.pass) { diff --git a/tests/failing-tests.js b/tests/failing-tests.js index 9e5828d..3098499 100644 --- a/tests/failing-tests.js +++ b/tests/failing-tests.js @@ -1,6 +1,7 @@ export default { name: "Failing tests", description: "These tests are designed to fail and should not break the test runner", + expect: 42, tests: [ { name: "map() fails", @@ -15,7 +16,6 @@ export default { map: arg => undefined, check: (actual, expected) => actual.length < expected.length, arg: 42, - expect: 42, }, ], }; diff --git a/tests/timeout.js b/tests/timeout.js new file mode 100644 index 0000000..5bcce24 --- /dev/null +++ b/tests/timeout.js @@ -0,0 +1,25 @@ +export default { + name: "Tests for timeout", + description: "These tests are designed to fail.", + run: () => new Promise(resolve => setTimeout(resolve, 200, "foo")), + maxTime: 100, + tests: [ + { + name: "Result-based test", + expect: "bar", + }, + { + name: "Error-based test", + throws: error => !error.message.startsWith("Test timed out"), + }, + { + name: "Time-based test", + maxTimeAsync: 100, + }, + { + name: "Default timeout", + run: () => new Promise(resolve => setTimeout(resolve, 10200)), + skip: true, // Comment this line out to see the test fail after 10 seconds + }, + ], +};