diff --git a/.github/workflows/tests-and-coverage.yml b/.github/workflows/tests-and-coverage.yml index afd6cf6..cd7b5d1 100644 --- a/.github/workflows/tests-and-coverage.yml +++ b/.github/workflows/tests-and-coverage.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [linux, macos, macos-arm64] - lua: [lua=5.1, lua=5.2, lua=5.3, lua=5.4, luajit=@v2.0, luajit=@v2.1] + lua: [lua=5.1, lua=5.2, lua=5.4, luajit=@v2.1] include: - os: linux runner: ubuntu-latest @@ -77,7 +77,7 @@ jobs: strategy: fail-fast: false matrix: - lua: [lua=5.1, lua=5.2, lua=5.3, lua=5.4, luajit=@v2.0, luajit=@v2.1] + lua: [lua=5.1, lua=5.2, lua=5.4, luajit=@v2.1] target: [mingw, vs] runs-on: windows-2022 steps: diff --git a/README.md b/README.md index 90d33e7..d34c704 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,6 @@ Parts of the following are included in the source code present in this repo: - Bundles a slightly modified [inspect.lua](https://github.com/kikito/inspect.lua) for table diffing and viewing - MIT - Also bundles a slightly modified [ansicolors.lua](https://github.com/kikito/ansicolors.lua) - MIT - A function from [Luacov](https://github.com/lunarmodules/luacov) code to help merge stats files in process - MIT +- Bundles [json.lua](https://github.com/rxi/json.lua)'s encoder for writing file output to json Major thanks to hishamhm, kikito, and benoit-germain for their work in the Lua space. Without them, `tested` wouldn't be possible. diff --git a/build/tested.lua b/build/tested.lua index c1a33b6..541aff4 100644 --- a/build/tested.lua +++ b/build/tested.lua @@ -3,6 +3,8 @@ local assert_table = require("tested.assert_table") local tested = { tests = {}, run_only_tests = false } + + local function validate_options(test_name, options, test_src) local error_prefix = test_src .. " in \"" .. test_name .. "\": " if options.expected ~= nil then @@ -32,6 +34,8 @@ local function extract_fn_and_options(test_name, fn_or_options, fn, test_src) fn = fn end + tested.filename = test_src + validate_options(test_name, options, test_src or "?") return fn, options @@ -160,10 +164,16 @@ local function should_skip_test(test, run_only, options) return nil, nil end +local function captured_os_exit(code) + local prefix = "os.exit()" + if code then prefix = "os.exit(" .. tostring(code) .. ")" end + error(prefix .. " intercepted — something tried to exit out of the process", 0) +end + local function set_result(ok, err, total_assertions, assert_failed_count, test_output) if ok == false then test_output.result = "EXCEPTION" - test_output.message = err .. "\n" .. debug.traceback() + test_output.message = err elseif total_assertions == 0 then test_output.result = "UNKNOWN" @@ -238,6 +248,11 @@ function tested:run(filename, options) tested.before_fn() end + local function xpcall_handler(e) + local msg = type(e) == "string" and (e) or tostring(e) + return msg .. debug.traceback("", 2) + end + for i, test in ipairs(self.tests) do test_results.tests[i] = { assertion_results = {}, name = test.name } @@ -264,8 +279,8 @@ function tested:run(filename, options) total_assertions = total_assertions + 1 local assertion_result = {} - local file_info = debug.getinfo(2, "Sl") - assertion_result.filename = file_info.short_src + local file_info = debug.getinfo(2, "l") + assertion_result.filename = tested.filename assertion_result.line_number = file_info.currentline assertion_result.given = assertion.given @@ -283,12 +298,34 @@ function tested:run(filename, options) return ok, err end + local start = os.clock() - local ok, err = pcall(test.fn) + + local original_os_exit + if _VERSION == "Lua 5.1" then + + original_os_exit = getfenv(test.fn).os.exit + getfenv(test.fn).os.exit = captured_os_exit + else + original_os_exit = os.exit + os.exit = captured_os_exit + end + + + local ok, err = xpcall(test.fn, xpcall_handler) test_results.tests[i].time = os.clock() - start test_results.total_time = test_results.total_time + test_results.tests[i].time + + self.assert = original_assert + + if _VERSION == "Lua 5.1" then + getfenv(test.fn).os.exit = original_os_exit + else + os.exit = original_os_exit + end + set_result(ok, err, total_assertions, assert_failed_count, test_results.tests[i]) diff --git a/build/tested/cli.lua b/build/tested/cli.lua index 68d5807..3a52df3 100644 --- a/build/tested/cli.lua +++ b/build/tested/cli.lua @@ -3,6 +3,7 @@ local lfs = require("lfs") local logging = require("tested.libs.logging") local logger = logging.get_logger("tested.cli") +local util = require("tested.util") @@ -47,6 +48,7 @@ local cli = { CLIOptions = {} } + local cli_to_display = { @@ -79,10 +81,13 @@ function cli.parse_args(version) choices({ "terminal", "plain", "tap" }): default("terminal"), parser:option("-z --custom-formatter"): - description("File that loads a custom formatter to use for output")) + description("File that loads a custom formatter to use for terminal output")) + parser:option("-o --output-file"): + description("Output file to save test results in (currently supported extensions: '.txt' and '.json')"): + count("*") parser:option("-n --threads"): - description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable."): + description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable. Test files are split amongst the threads."): default(4): convert(tonumber) parser:option("-x --format-handler"): @@ -119,6 +124,14 @@ function cli.set_defaults(args) local show_all = false for _, display_option in ipairs(args.show) do if display_option == "all" then show_all = true; break end end if show_all then args.show = { "skip", "pass", "fail", "exception", "unknown", "expected", "unexpected" } end + + if #args.output_file > 0 then + for _, output_file in ipairs(args.output_file) do + if not (util.get_file_extension(output_file) == ".txt" or util.get_file_extension(output_file) == ".json") then + error("The given output file does not have a supported file extension: '" .. output_file .. "'. Supported file extensions are: '.txt', '.json'", 0) + end + end + end end function cli.validate_args(args) diff --git a/build/tested/file_output/json.lua b/build/tested/file_output/json.lua new file mode 100644 index 0000000..5426b2b --- /dev/null +++ b/build/tested/file_output/json.lua @@ -0,0 +1,142 @@ + + + + + + +local encode + + + + + + + + + + + +local escape_char_map = { + ["\\"] = "\\", + ["\""] = "\"", + ["\b"] = "b", + ["\f"] = "f", + ["\n"] = "n", + ["\r"] = "r", + ["\t"] = "t", +} + +local escape_char_map_inv = { ["/"] = "/" } +for k, v in pairs(escape_char_map) do + escape_char_map_inv[v] = k +end + + +local function escape_char(c) + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + + +local function encode_nil(_val) + return "null" +end + + +local function encode_table(val, stack) + local res = {} + stack = stack or {} + + + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val then + error("invalid table: sparse array") + end + + for _, v in ipairs(val) do + table.insert(res, encode(v, stack)) + end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + + else + + for k, v in pairs(val) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + + +local function encode_string(val) + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + + +local function encode_number(val) + + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + + +local type_func_map = { + ["nil"] = encode_nil, + ["table"] = encode_table, + ["string"] = encode_string, + ["number"] = encode_number, + ["boolean"] = tostring, +} + + +encode = function(val, stack) + local t = type(val) + local f = type_func_map[t] + if f then + return f(val, stack) + end + error("unexpected type '" .. t .. "'") +end + + + + + + +local json = {} + +json.format = "json" + + +function json.header(_version, _filepaths, _comments) + return "" +end + +function json.results(_tested_result, _test_types_to_display) + return "" +end + +function json.summary(runner_output) + return encode(runner_output) +end + + +return json diff --git a/build/tested/file_output/txt.lua b/build/tested/file_output/txt.lua new file mode 100644 index 0000000..c7a57a7 --- /dev/null +++ b/build/tested/file_output/txt.lua @@ -0,0 +1,12 @@ + +local plain = require("tested.results.plain") + + +local txt = {} + +txt.format = "txt" +txt.header = plain.header +txt.results = plain.results +txt.summary = plain.summary + +return txt diff --git a/build/tested/main.lua b/build/tested/main.lua index 072cd60..468823f 100644 --- a/build/tested/main.lua +++ b/build/tested/main.lua @@ -1,17 +1,17 @@ local lfs = require("lfs") -local file_loader = require("tested.file_loader") +local cli = require("tested.cli") +local file_loader = require("tested.file_loader") +local logging = require("tested.libs.logging") local test_runner = require("tested.test_runner") -local TestRunner, run_parallel_tests = test_runner[1], test_runner[2] -local logging = require("tested.libs.logging") +local util = require("tested.util") + local logger = logging.get_logger("tested.main") +local TestRunner, run_parallel_tests = test_runner[1], test_runner[2] local TESTED_VERSION = "tested v0.2.0" - -local cli = require("tested.cli") - local function load_result_formatter(args) if args.custom_formatter then logger:info("Loading custom formatter: %s", args.custom_formatter) @@ -54,6 +54,7 @@ local function register_format_handler(handlers) file_loader.register_handler(module_format_handler.extension, module_format_handler.loader) end + local info, err = lfs.attributes(handler) if err then error("Unable to load format handler, the file/module '" .. handler .. "' was not able to be loaded.") end if info.mode ~= "file" then error("The custom format loader should point to a file, but currently appears to be a: " .. info.mode, 0) end @@ -83,14 +84,10 @@ local function find_tests(files, test_path) logger:info("Found %d test files to run in %s", #files, test_path) end -local function get_file_extension(str) - return str:match("^.+(%..+)$") -end - local function get_all_test_files(args) local all_files = {} for _, test_file in ipairs(args.test_files) do - if file_loader.loader[get_file_extension(test_file)] then + if file_loader.loader[util.get_file_extension(test_file)] then table.insert(all_files, test_file) end end @@ -110,7 +107,7 @@ local function run_tests(formatter, args, test_files) } local display_results = function(test_output) - formatter.results(test_output, cli.display_types(args.show)) + print(formatter.results(test_output, cli.display_types(args.show))) end if args.threads == 0 or #test_files <= 1 then @@ -129,6 +126,24 @@ local function run_tests(formatter, args, test_files) return runner_output end +local function write_output_files(args, header_comments, runner_output) + + for _, file_output in ipairs(args.output_file) do + local extension = util.get_file_extension(file_output) + local output_formatter = require("tested.file_output" .. extension) + local file_to_write, err = io.open(file_output, "w") + if not file_to_write then error("Unable to open file for writing: " .. err, 0) end + file_to_write:write(output_formatter.header(TESTED_VERSION, args.paths, header_comments)) + + for _, test_output in ipairs(runner_output.module_results) do + file_to_write:write(output_formatter.results(test_output, cli.display_types(args.show))) + end + + file_to_write:write(output_formatter.summary(runner_output)) + file_to_write:close() + end +end + local function main() local args = cli.parse_args(TESTED_VERSION) @@ -153,10 +168,13 @@ local function main() end - formatter.header(TESTED_VERSION, args.paths, header_comments) + print(formatter.header(TESTED_VERSION, args.paths, header_comments)) local runner_output = run_tests(formatter, args, test_files) - formatter.summary(runner_output) + runner_output.tested_version = TESTED_VERSION + print(formatter.summary(runner_output)) + + write_output_files(args, header_comments, runner_output) if runner_output.all_fully_tested then diff --git a/build/tested/results/plain.lua b/build/tested/results/plain.lua index 9f80dc0..af48bd2 100644 --- a/build/tested/results/plain.lua +++ b/build/tested/results/plain.lua @@ -3,8 +3,6 @@ local terminal = require("tested.results.terminal") local plain = {} - - plain.format = "plain" diff --git a/build/tested/results/tap.lua b/build/tested/results/tap.lua index 6cf03da..2b0b1ab 100644 --- a/build/tested/results/tap.lua +++ b/build/tested/results/tap.lua @@ -7,7 +7,7 @@ tap.allow_filtering = false tap.format = "tap" function tap.header(_version_info, _filepaths, _comments) - print("TAP version 14") + return "TAP version 14" end function tap.results(tested_result, _test_types_to_display) @@ -23,7 +23,7 @@ function tap.results(tested_result, _test_types_to_display) tadd.add(tap_result, test.name) if test.result == "SKIP" or test.result == "CONDITIONAL_SKIP" then - tadd.add(" # SKIP" or "", "\n") + tadd.add(" # SKIP", "\n") else tadd.add("\n") @@ -48,11 +48,11 @@ function tap.results(tested_result, _test_types_to_display) end end - print(tadd.tostring()) + return tadd.tostring() end function tap.summary(output) - print("1.." .. output.total_tests) + return "1.." .. output.total_tests end return tap diff --git a/build/tested/results/terminal.lua b/build/tested/results/terminal.lua index ced29fc..b46de42 100644 --- a/build/tested/results/terminal.lua +++ b/build/tested/results/terminal.lua @@ -42,11 +42,12 @@ terminal.allow_filtering = true terminal.colors = colors function terminal.header(version_info, filepaths, comments) - print(colors("%{bright}" .. version_info .. " " .. table.concat(filepaths, " "))) + tadd.new("%{bright}", version_info, " ", table.concat(filepaths, " "), "\n") for _, comment in ipairs(comments) do - print(comment) + tadd.add(comment, "\n") end - print() + tadd.add("\n") + return colors(tadd.tostring()) end local function to_ms(time, add_color) @@ -109,7 +110,7 @@ function terminal.results(tested_result, test_types_to_display) end end end - print(colors(tadd.tostring())) + return colors(tadd.tostring()) end @@ -155,7 +156,7 @@ function terminal.summary(output) tadd.add("\n%{bright}Fully Tested!%{reset}\n") end - print(colors(tadd.tostring())) + return colors(tadd.tostring()) end return terminal diff --git a/build/tested/types.lua b/build/tested/types.lua index b60afa8..b94e656 100644 --- a/build/tested/types.lua +++ b/build/tested/types.lua @@ -155,6 +155,8 @@ local types = {} + + diff --git a/build/tested/util.lua b/build/tested/util.lua new file mode 100644 index 0000000..98dcf59 --- /dev/null +++ b/build/tested/util.lua @@ -0,0 +1,9 @@ +local util = {} + + + +function util.get_file_extension(str) + return str:match("^.+(%..+)$") +end + +return util diff --git a/docs/cli.md b/docs/cli.md index 8cd20c6..eaac8a5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -48,6 +48,18 @@ Specify the number of threads `tested` should use. If set to `0`, will not use a ## `tested -z/--custom-formatter` `tested` supports loading a [custom result formatter](./custom-formatter.md) from the commandline. It tries to load what's passed in initially as a Lua module, and then as filepath, doing some basic checks to ensure the object returned appears to be a formatter. Only one custom formatter can be loaded and will be used to display results. +## `tested -o/--output-file` +Output file to save results to a specified file. It loads a formatter based on the file extension of the file that's passed in. The currently supported extensions are: + +- `.txt` - Outputs _exactly_ what is shown in the terminal (includes any filtering) +- `.json` - Outputs the full [TestRunnerOutput](./custom-formatter.md#testrunneroutput). Everything - it does not filter anything. We're still pre v1, so the output here _could_ change. + +Multiple files can be specified and all will be written to at the end of the test run: + +```bash +tested -o ./terminal_output.txt -o ./full_output.json +``` + ## `tested -x/--format-handler` `tested` also supports loading [custom format handlers](./additional-formats.md). These are used to extend the functionality of tested and tap into other languages that can embed into Lua. Similar to custom output formatters, the format handler will first try and load from a Lua module and then from a filepath. This allows flexibility in distribution in how folks may want to support a custom format. Multiple format handlers can be loaded, and afterward can be used for custom formatters or the tests themselves. @@ -57,8 +69,8 @@ Specify the number of threads `tested` should use. If set to `0`, will not use a Usage: tested ([-f {terminal,plain,tap}] | [-z ]) [-h] [-c] [-r] [-F ] [-s {all,valid,invalid,skip,pass,fail,exception,unknown,expected,unexpected}] - [-n ] [-x ] [-d {DEBUG,INFO,WARNING}] - [--version] [] ... + [-o ] [-n ] [-x ] + [-d {DEBUG,INFO,WARNING}] [--version] [] ... A Lua/Teal Unit Testing Framework @@ -79,14 +91,17 @@ Options: What format to output the results in (default: 'terminal') (default: terminal) -z , --custom-formatter - File that loads a custom formatter to use for output - -n , Set the number of threads to run the tests with (default: 4). Set to 0 to disable. + File that loads a custom formatter to use for terminal output + -o , + --output-file + Output file to save test results in (currently supported extensions: '.txt' and '.json') + -n , Set the number of threads to run the tests with (default: 4). Set to 0 to disable. Test files are split amongst the threads. --threads -x , --format-handler File that loads custom formats that are Lua-compatible -d {DEBUG,INFO,WARNING}, --debug {DEBUG,INFO,WARNING} - Set the log level - mostly for debugging purposes (default: 'WARNING') (default: WARNING) + Set the log level - mostly for debugging issues with tested (default: 'WARNING') (default: WARNING) --version Show version information ``` diff --git a/docs/custom-formatter.md b/docs/custom-formatter.md index aab4e35..679cf1d 100644 --- a/docs/custom-formatter.md +++ b/docs/custom-formatter.md @@ -1,5 +1,5 @@ # Custom Formatters -As of now (1/2026), `tested` currently supports a "terminal" output and a "plain" output (which is just the terminal output without colors), with plans for a few more [in the future](https://github.com/FourierTransformer/tested/issues/21)! But we've tried to make it easy to create your own formatter for `tested`. +As of now (1/2026), `tested` currently supports printing out a "terminal", "plain" (which is just the terminal output without colors), and "tap" output. We've tried to make it easy to create your own formatter for `tested`. You can only create a custom formatter for _display_ purposes. If you want to create a custom formatter for _file saving_ purposes, please [create an issue](https://github.com/FourierTransformer/tested/issues)! ## A basic formatter @@ -16,17 +16,17 @@ local custom_formatter = { -- Runs after performing all the setups and tests are about to run! -- version: "tested v0.0.0" -- filepaths: list of filepaths passed into tested. -function custom_formatter.header(version: string, filepaths: {string}) end +function custom_formatter.header(version: string, filepaths: {string}): string end -- Displays results after a test has been run function custom_formatter.results( tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean} -) +): string end -- Outputs a summary at the end -function custom_formatter.summary(output: types.TestRunnerOutput) end +function custom_formatter.summary(output: types.TestRunnerOutput): string end return custom_formatter @@ -128,7 +128,8 @@ An example of what `types.TestRunnerOutput` looks like. `types.TestedOutput` is skipped = 0 }, total_tests = 5, - total_time = 2.1999999999966e-05 + total_time = 2.1999999999966e-05, + tested_version = "tested v0.2.0" } ``` @@ -140,7 +141,7 @@ enum TestResult "SKIP" "CONDITIONAL_SKIP" "EXCEPTION" - "TIMEOUT" + -- "TIMEOUT" -- NYI "UNKNOWN" "EXPECTED_FAIL" "EXPECTED_EXCEPTION" @@ -187,14 +188,15 @@ interface TestRunnerOutput total_time: number total_tests: integer all_fully_tested: boolean + tested_version: string end -- The actual formatter itself interface ResultFormatter format: string allow_filtering: boolean - header: function(version: string, filepaths: {string}) - results: function(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}) - summary: function(runner_output: types.TestRunnerOutput) + header: function(version: string, filepaths: {string}, comments: {string}): string + results: function(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}): string + summary: function(runner_output: types.TestRunnerOutput): string end ``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 92365ed..0283c1f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -137,7 +137,8 @@ There are a couple CLI commands that are good to know when you get started: - `tested -F ` or `--filter ` will filter tests based on a `string.find` pattern. It can just be the test name, a couple words from the test name (in order), or a full on Lua pattern! - `tested -s