From 7c2e718b55d68733b79e0126dae2135e5a453637 Mon Sep 17 00:00:00 2001 From: FourierTransformer Date: Wed, 13 May 2026 12:37:06 -0500 Subject: [PATCH 1/6] make exception output easier to debug --- build/tested.lua | 17 ++++++++++-- src/tested.tl | 17 ++++++++++-- tests/exit_test.lua | 46 +++++++++++++++++++++++++++++++ tests/lifecycle_broken_before.lua | 17 ++++++++++++ 4 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 tests/exit_test.lua create mode 100644 tests/lifecycle_broken_before.lua diff --git a/build/tested.lua b/build/tested.lua index c1a33b6..f86590d 100644 --- a/build/tested.lua +++ b/build/tested.lua @@ -163,7 +163,7 @@ end local function set_result(ok, err, total_assertions, assert_failed_count, test_output) if ok == false then test_output.result = "EXCEPTION" - test_output.message = err .. "\n" .. debug.traceback() + test_output.message = err elseif total_assertions == 0 then test_output.result = "UNKNOWN" @@ -238,6 +238,11 @@ function tested:run(filename, options) tested.before_fn() end + local function xpcall_handler(e) + local msg = type(e) == "string" and (e) or tostring(e) + return msg .. debug.traceback("", 2) + end + for i, test in ipairs(self.tests) do test_results.tests[i] = { assertion_results = {}, name = test.name } @@ -283,11 +288,19 @@ function tested:run(filename, options) return ok, err end + local original_os_exit = os.exit + os.exit = function(code) + local prefix = "os.exit()" + if code then prefix = "os.exit(" .. tostring(code) .. ")" end + error(prefix .. " intercepted — something tried to exit out of the process", 0) + end + local start = os.clock() - local ok, err = pcall(test.fn) + local ok, err = xpcall(test.fn, xpcall_handler) test_results.tests[i].time = os.clock() - start test_results.total_time = test_results.total_time + test_results.tests[i].time self.assert = original_assert + os.exit = original_os_exit set_result(ok, err, total_assertions, assert_failed_count, test_results.tests[i]) diff --git a/src/tested.tl b/src/tested.tl index bfd9f5b..a693f57 100644 --- a/src/tested.tl +++ b/src/tested.tl @@ -163,7 +163,7 @@ end local function set_result(ok: boolean, err: string, total_assertions: integer, assert_failed_count: integer, test_output: types.TestOutput) if ok == false then test_output.result = "EXCEPTION" - test_output.message = err .. "\n" .. debug.traceback() + test_output.message = err elseif total_assertions == 0 then test_output.result = "UNKNOWN" @@ -238,6 +238,11 @@ function tested:run(filename: string, options: types.TestRunnerOptions): types.T tested.before_fn() end + local function xpcall_handler(e: any): string + local msg = type(e) == "string" and (e as string) or tostring(e) + return msg .. debug.traceback("", 2) + end + for i, test in ipairs(self.tests) do test_results.tests[i] = {assertion_results = {}, name = test.name} @@ -283,11 +288,19 @@ function tested:run(filename: string, options: types.TestRunnerOptions): types.T return ok, err end + local original_os_exit = os.exit + os.exit = function(code: integer | boolean | nil) + local prefix = "os.exit()" + if code then prefix = "os.exit(" .. tostring(code) .. ")" end + error(prefix .. " intercepted — something tried to exit out of the process", 0) + end + local start = os.clock() - local ok, err = pcall(test.fn) as (boolean, string) + local ok, err = xpcall(test.fn, xpcall_handler) as (boolean, string) test_results.tests[i].time = os.clock() - start test_results.total_time = test_results.total_time + test_results.tests[i].time self.assert = original_assert + os.exit = original_os_exit set_result(ok, err, total_assertions, assert_failed_count, test_results.tests[i]) diff --git a/tests/exit_test.lua b/tests/exit_test.lua new file mode 100644 index 0000000..3136b80 --- /dev/null +++ b/tests/exit_test.lua @@ -0,0 +1,46 @@ +local tested = require("tested") + +-- Natural completion with no assertions: expect UNKNOWN +tested.test("natural completion - no assertions", {expected="UNKNOWN"}, function() + local x = 1 + 1 +end) + +-- error() unhandled: pcall in tested catches it → EXCEPTION +tested.test("unhandled error() call", {expected="EXCEPTION"}, function() + error("something went wrong") +end) + +-- Lua assert() with nil: same as error() under the hood → EXCEPTION +tested.test("assert() with nil condition", {expected="EXCEPTION"}, function() + local file = io.open("/nonexistent/path/file.txt", "r") + assert(file, "could not open file") +end) + +-- Indexing nil: runtime error, pcall catches it → EXCEPTION +tested.test("nil indexing runtime crash", {expected="EXCEPTION"}, function() + local a = nil + print(a.value) +end) + +-- load() returns nil + message for bad syntax; calling nil is itself a runtime error → EXCEPTION +tested.test("calling result of load() with syntax error", {expected="EXCEPTION"}, function() + local chunk = load("if true print('hello') end") + chunk() +end) + +-- error() with a table value: pcall catches it but the error object is not a string +tested.test("error() with non-string table object", {expected="EXCEPTION"}, function() + error({ code = 42, msg = "structured error" }) +end) + +-- os.exit() is now intercepted and raised as an error so tested can report it. +tested.test("os.exit(0) terminates the host process", {expected="EXCEPTION"}, function() + os.exit(0) +end) + +-- os.exit() is now intercepted and raised as an error so tested can report it. +tested.test("os.exit() terminates the host process", {expected="EXCEPTION"}, function() + os.exit() +end) + +return tested diff --git a/tests/lifecycle_broken_before.lua b/tests/lifecycle_broken_before.lua new file mode 100644 index 0000000..84fbdd4 --- /dev/null +++ b/tests/lifecycle_broken_before.lua @@ -0,0 +1,17 @@ +local tested = require("tested") + +-- before_fn is called without pcall in tested:run(), so this error unwinds +-- through run_with_cleanup() with no catch — none of the tests below will run. +tested.before(function() + error("before hook is broken") +end) + +tested.test("this test never runs", function() + tested.assert({ expected = true, actual = true }) +end) + +tested.test("neither does this one", function() + tested.assert({ expected = 1, actual = 1 }) +end) + +return tested From 43602277efbb425ebb32aebe5805e53441445156 Mon Sep 17 00:00:00 2001 From: FourierTransformer Date: Wed, 13 May 2026 14:39:32 -0500 Subject: [PATCH 2/6] added a file output with plain txt and json output --- README.md | 1 + build/tested.lua | 4 + build/tested/cli.lua | 15 +++- build/tested/file_output/json.lua | 142 ++++++++++++++++++++++++++++++ build/tested/file_output/txt.lua | 12 +++ build/tested/main.lua | 57 +++++++----- build/tested/results/plain.lua | 2 - build/tested/results/tap.lua | 8 +- build/tested/results/terminal.lua | 11 +-- build/tested/types.lua | 1 + build/tested/util.lua | 9 ++ docs/roadmap.md | 6 +- src/tested.tl | 4 + src/tested/cli.tl | 15 +++- src/tested/file_output/json.tl | 142 ++++++++++++++++++++++++++++++ src/tested/file_output/txt.tl | 12 +++ src/tested/main.tl | 57 +++++++----- src/tested/results/plain.tl | 4 +- src/tested/results/tap.tl | 14 +-- src/tested/results/terminal.tl | 19 ++-- src/tested/types.tl | 7 +- src/tested/util.tl | 9 ++ tested-dev-1.rockspec | 7 ++ 23 files changed, 480 insertions(+), 78 deletions(-) create mode 100644 build/tested/file_output/json.lua create mode 100644 build/tested/file_output/txt.lua create mode 100644 build/tested/util.lua create mode 100644 src/tested/file_output/json.tl create mode 100644 src/tested/file_output/txt.tl create mode 100644 src/tested/util.tl diff --git a/README.md b/README.md index 90d33e7..d34c704 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,6 @@ Parts of the following are included in the source code present in this repo: - Bundles a slightly modified [inspect.lua](https://github.com/kikito/inspect.lua) for table diffing and viewing - MIT - Also bundles a slightly modified [ansicolors.lua](https://github.com/kikito/ansicolors.lua) - MIT - A function from [Luacov](https://github.com/lunarmodules/luacov) code to help merge stats files in process - MIT +- Bundles [json.lua](https://github.com/rxi/json.lua)'s encoder for writing file output to json Major thanks to hishamhm, kikito, and benoit-germain for their work in the Lua space. Without them, `tested` wouldn't be possible. diff --git a/build/tested.lua b/build/tested.lua index f86590d..87da790 100644 --- a/build/tested.lua +++ b/build/tested.lua @@ -288,6 +288,7 @@ function tested:run(filename, options) return ok, err end + local original_os_exit = os.exit os.exit = function(code) local prefix = "os.exit()" @@ -295,10 +296,13 @@ function tested:run(filename, options) error(prefix .. " intercepted — something tried to exit out of the process", 0) end + local start = os.clock() local ok, err = xpcall(test.fn, xpcall_handler) test_results.tests[i].time = os.clock() - start test_results.total_time = test_results.total_time + test_results.tests[i].time + + self.assert = original_assert os.exit = original_os_exit diff --git a/build/tested/cli.lua b/build/tested/cli.lua index 68d5807..9ce2599 100644 --- a/build/tested/cli.lua +++ b/build/tested/cli.lua @@ -3,6 +3,7 @@ local lfs = require("lfs") local logging = require("tested.libs.logging") local logger = logging.get_logger("tested.cli") +local util = require("tested.util") @@ -47,6 +48,7 @@ local cli = { CLIOptions = {} } + local cli_to_display = { @@ -81,8 +83,11 @@ function cli.parse_args(version) parser:option("-z --custom-formatter"): description("File that loads a custom formatter to use for output")) + parser:option("-o --output"): + description("Output file to save test results in (currently supported extensions: '.txt' and '.json')"): + count("*") parser:option("-n --threads"): - description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable."): + description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable. Test files are split amongst the threads."): default(4): convert(tonumber) parser:option("-x --format-handler"): @@ -119,6 +124,14 @@ function cli.set_defaults(args) local show_all = false for _, display_option in ipairs(args.show) do if display_option == "all" then show_all = true; break end end if show_all then args.show = { "skip", "pass", "fail", "exception", "unknown", "expected", "unexpected" } end + + if #args.output > 0 then + for _, output_file in ipairs(args.output) do + if not (util.get_file_extension(output_file) == ".txt" or util.get_file_extension(output_file) == ".json") then + error("The given output file does not have a supported file extension: '" .. output_file .. "'. Supported file extensions are: '.txt', '.json'", 0) + end + end + end end function cli.validate_args(args) diff --git a/build/tested/file_output/json.lua b/build/tested/file_output/json.lua new file mode 100644 index 0000000..5426b2b --- /dev/null +++ b/build/tested/file_output/json.lua @@ -0,0 +1,142 @@ + + + + + + +local encode + + + + + + + + + + + +local escape_char_map = { + ["\\"] = "\\", + ["\""] = "\"", + ["\b"] = "b", + ["\f"] = "f", + ["\n"] = "n", + ["\r"] = "r", + ["\t"] = "t", +} + +local escape_char_map_inv = { ["/"] = "/" } +for k, v in pairs(escape_char_map) do + escape_char_map_inv[v] = k +end + + +local function escape_char(c) + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + + +local function encode_nil(_val) + return "null" +end + + +local function encode_table(val, stack) + local res = {} + stack = stack or {} + + + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val then + error("invalid table: sparse array") + end + + for _, v in ipairs(val) do + table.insert(res, encode(v, stack)) + end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + + else + + for k, v in pairs(val) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + + +local function encode_string(val) + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + + +local function encode_number(val) + + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + + +local type_func_map = { + ["nil"] = encode_nil, + ["table"] = encode_table, + ["string"] = encode_string, + ["number"] = encode_number, + ["boolean"] = tostring, +} + + +encode = function(val, stack) + local t = type(val) + local f = type_func_map[t] + if f then + return f(val, stack) + end + error("unexpected type '" .. t .. "'") +end + + + + + + +local json = {} + +json.format = "json" + + +function json.header(_version, _filepaths, _comments) + return "" +end + +function json.results(_tested_result, _test_types_to_display) + return "" +end + +function json.summary(runner_output) + return encode(runner_output) +end + + +return json diff --git a/build/tested/file_output/txt.lua b/build/tested/file_output/txt.lua new file mode 100644 index 0000000..c7a57a7 --- /dev/null +++ b/build/tested/file_output/txt.lua @@ -0,0 +1,12 @@ + +local plain = require("tested.results.plain") + + +local txt = {} + +txt.format = "txt" +txt.header = plain.header +txt.results = plain.results +txt.summary = plain.summary + +return txt diff --git a/build/tested/main.lua b/build/tested/main.lua index 072cd60..eb598ee 100644 --- a/build/tested/main.lua +++ b/build/tested/main.lua @@ -1,17 +1,17 @@ local lfs = require("lfs") -local file_loader = require("tested.file_loader") +local cli = require("tested.cli") +local file_loader = require("tested.file_loader") +local logging = require("tested.libs.logging") local test_runner = require("tested.test_runner") -local TestRunner, run_parallel_tests = test_runner[1], test_runner[2] -local logging = require("tested.libs.logging") +local util = require("tested.util") + local logger = logging.get_logger("tested.main") +local TestRunner, run_parallel_tests = test_runner[1], test_runner[2] local TESTED_VERSION = "tested v0.2.0" - -local cli = require("tested.cli") - local function load_result_formatter(args) if args.custom_formatter then logger:info("Loading custom formatter: %s", args.custom_formatter) @@ -52,13 +52,13 @@ local function register_format_handler(handlers) local ok, module_format_handler = pcall(require, handler) if ok then file_loader.register_handler(module_format_handler.extension, module_format_handler.loader) - end - - local info, err = lfs.attributes(handler) - if err then error("Unable to load format handler, the file/module '" .. handler .. "' was not able to be loaded.") end - if info.mode ~= "file" then error("The custom format loader should point to a file, but currently appears to be a: " .. info.mode, 0) end + else + local info, err = lfs.attributes(handler) + if err then error("Unable to load format handler, the file/module '" .. handler .. "' was not able to be loaded.") end + if info.mode ~= "file" then error("The custom format loader should point to a file, but currently appears to be a: " .. info.mode, 0) end - file_loader.load_and_register_handler(handler) + file_loader.load_and_register_handler(handler) + end end end @@ -83,14 +83,10 @@ local function find_tests(files, test_path) logger:info("Found %d test files to run in %s", #files, test_path) end -local function get_file_extension(str) - return str:match("^.+(%..+)$") -end - local function get_all_test_files(args) local all_files = {} for _, test_file in ipairs(args.test_files) do - if file_loader.loader[get_file_extension(test_file)] then + if file_loader.loader[util.get_file_extension(test_file)] then table.insert(all_files, test_file) end end @@ -110,7 +106,7 @@ local function run_tests(formatter, args, test_files) } local display_results = function(test_output) - formatter.results(test_output, cli.display_types(args.show)) + print(formatter.results(test_output, cli.display_types(args.show))) end if args.threads == 0 or #test_files <= 1 then @@ -129,6 +125,24 @@ local function run_tests(formatter, args, test_files) return runner_output end +local function write_output_files(args, header_comments, runner_output) + + for _, file_output in ipairs(args.output) do + local extension = util.get_file_extension(file_output) + local output_formatter = require("tested.file_output" .. extension) + local file_to_write, err = io.open(file_output, "w") + if not file_to_write then error("Unable to open file for writing: " .. err, 0) end + file_to_write:write(output_formatter.header(TESTED_VERSION, args.paths, header_comments)) + + for _, test_output in ipairs(runner_output.module_results) do + file_to_write:write(output_formatter.results(test_output, cli.display_types(args.show))) + end + + file_to_write:write(output_formatter.summary(runner_output)) + file_to_write:close() + end +end + local function main() local args = cli.parse_args(TESTED_VERSION) @@ -153,10 +167,13 @@ local function main() end - formatter.header(TESTED_VERSION, args.paths, header_comments) + print(formatter.header(TESTED_VERSION, args.paths, header_comments)) local runner_output = run_tests(formatter, args, test_files) - formatter.summary(runner_output) + runner_output.tested_version = TESTED_VERSION + print(formatter.summary(runner_output)) + + write_output_files(args, header_comments, runner_output) if runner_output.all_fully_tested then diff --git a/build/tested/results/plain.lua b/build/tested/results/plain.lua index 9f80dc0..af48bd2 100644 --- a/build/tested/results/plain.lua +++ b/build/tested/results/plain.lua @@ -3,8 +3,6 @@ local terminal = require("tested.results.terminal") local plain = {} - - plain.format = "plain" diff --git a/build/tested/results/tap.lua b/build/tested/results/tap.lua index 6cf03da..2b0b1ab 100644 --- a/build/tested/results/tap.lua +++ b/build/tested/results/tap.lua @@ -7,7 +7,7 @@ tap.allow_filtering = false tap.format = "tap" function tap.header(_version_info, _filepaths, _comments) - print("TAP version 14") + return "TAP version 14" end function tap.results(tested_result, _test_types_to_display) @@ -23,7 +23,7 @@ function tap.results(tested_result, _test_types_to_display) tadd.add(tap_result, test.name) if test.result == "SKIP" or test.result == "CONDITIONAL_SKIP" then - tadd.add(" # SKIP" or "", "\n") + tadd.add(" # SKIP", "\n") else tadd.add("\n") @@ -48,11 +48,11 @@ function tap.results(tested_result, _test_types_to_display) end end - print(tadd.tostring()) + return tadd.tostring() end function tap.summary(output) - print("1.." .. output.total_tests) + return "1.." .. output.total_tests end return tap diff --git a/build/tested/results/terminal.lua b/build/tested/results/terminal.lua index ced29fc..b46de42 100644 --- a/build/tested/results/terminal.lua +++ b/build/tested/results/terminal.lua @@ -42,11 +42,12 @@ terminal.allow_filtering = true terminal.colors = colors function terminal.header(version_info, filepaths, comments) - print(colors("%{bright}" .. version_info .. " " .. table.concat(filepaths, " "))) + tadd.new("%{bright}", version_info, " ", table.concat(filepaths, " "), "\n") for _, comment in ipairs(comments) do - print(comment) + tadd.add(comment, "\n") end - print() + tadd.add("\n") + return colors(tadd.tostring()) end local function to_ms(time, add_color) @@ -109,7 +110,7 @@ function terminal.results(tested_result, test_types_to_display) end end end - print(colors(tadd.tostring())) + return colors(tadd.tostring()) end @@ -155,7 +156,7 @@ function terminal.summary(output) tadd.add("\n%{bright}Fully Tested!%{reset}\n") end - print(colors(tadd.tostring())) + return colors(tadd.tostring()) end return terminal diff --git a/build/tested/types.lua b/build/tested/types.lua index b60afa8..cb59008 100644 --- a/build/tested/types.lua +++ b/build/tested/types.lua @@ -156,6 +156,7 @@ local types = {} + return types diff --git a/build/tested/util.lua b/build/tested/util.lua new file mode 100644 index 0000000..98dcf59 --- /dev/null +++ b/build/tested/util.lua @@ -0,0 +1,9 @@ +local util = {} + + + +function util.get_file_extension(str) + return str:match("^.+(%..+)$") +end + +return util diff --git a/docs/roadmap.md b/docs/roadmap.md index 8d3b998..98c7ac8 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -10,14 +10,14 @@ Things that I am one day planning to add (in no particular order): - [ ] `retries` and (maybe) `retry_timeout` for automatically retrying failing tests - [ ] tags for filtering - [x] Lifecycle management (`before`, `after`, `before_each`, `after_each`) -- [ ] Table driven assertion (no more for loops around asserts!) +- [ ] Spies - [ ] Stubbing - [ ] Mocking - [ ] A [pure Lua](./pure-lua.md) single-file (maybe two files) distribution [#20](https://github.com/FourierTransformer/tested/issues/20) - Should allow for embedding (on devices, maybe with Neovim and Love2d? ) -- [ ] File output (alongside terminal) +- [x] File output (alongside terminal) - A cool fancy HTML output with the tests and coverage could be fun [#14](https://github.com/FourierTransformer/tested/issues/14) - - Likely JSON as well + - [x] Likely JSON as well - [ ] Test timeouts [#3](https://github.com/FourierTransformer/tested/issues/3) If there are any things you would really want to see added to a Unit testing framework, feel free to [open up a discussion](https://github.com/FourierTransformer/tested/discussions/new/choose). I'm currently open to new ideas! \ No newline at end of file diff --git a/src/tested.tl b/src/tested.tl index a693f57..6c0943f 100644 --- a/src/tested.tl +++ b/src/tested.tl @@ -288,6 +288,7 @@ function tested:run(filename: string, options: types.TestRunnerOptions): types.T return ok, err end + -- also wrap os.exit before running local original_os_exit = os.exit os.exit = function(code: integer | boolean | nil) local prefix = "os.exit()" @@ -295,10 +296,13 @@ function tested:run(filename: string, options: types.TestRunnerOptions): types.T error(prefix .. " intercepted — something tried to exit out of the process", 0) end + -- run the test local start = os.clock() local ok, err = xpcall(test.fn, xpcall_handler) as (boolean, string) test_results.tests[i].time = os.clock() - start test_results.total_time = test_results.total_time + test_results.tests[i].time + + -- put things back self.assert = original_assert os.exit = original_os_exit diff --git a/src/tested/cli.tl b/src/tested/cli.tl index b3c28c9..0862110 100644 --- a/src/tested/cli.tl +++ b/src/tested/cli.tl @@ -3,6 +3,7 @@ local lfs = require("lfs") local logging = require("tested.libs.logging") local logger = logging.get_logger("tested.cli") +local util = require("tested.util") local type types = require("tested.types") @@ -30,6 +31,7 @@ local record cli random: boolean show: {DisplayOptions} display_format: DisplayFormat + output: {string} custom_formatter: string format_handler: {string} paths: {string} @@ -81,8 +83,11 @@ function cli.parse_args(version: string): cli.CLIOptions parser:option("-z --custom-formatter") :description("File that loads a custom formatter to use for output") ) + parser:option("-o --output") + :description("Output file to save test results in (currently supported extensions: '.txt' and '.json')") + :count("*") parser:option("-n --threads") - :description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable.") + :description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable. Test files are split amongst the threads.") :default(4) :convert(tonumber) parser:option("-x --format-handler") @@ -119,6 +124,14 @@ function cli.set_defaults(args: cli.CLIOptions) local show_all = false for _, display_option in ipairs(args.show) do if display_option == "all" then show_all = true break end end if show_all then args.show = {"skip", "pass", "fail", "exception", "unknown", "expected", "unexpected"} end -- NYI: timeout + + if #args.output > 0 then + for _, output_file in ipairs(args.output) do + if not (util.get_file_extension(output_file) == ".txt" or util.get_file_extension(output_file) == ".json") then + error("The given output file does not have a supported file extension: '" .. output_file .. "'. Supported file extensions are: '.txt', '.json'", 0) + end + end + end end function cli.validate_args(args: cli.CLIOptions) diff --git a/src/tested/file_output/json.tl b/src/tested/file_output/json.tl new file mode 100644 index 0000000..7499bc5 --- /dev/null +++ b/src/tested/file_output/json.tl @@ -0,0 +1,142 @@ +local type types = require("tested.types") + +------------------------------------------------------------------------------- +-- JSON Encoder from rxi/json.lua - used under MIT License +------------------------------------------------------------------------------- + +local encode: function(val: table, stack?: table): string + +local enum ESCAPE_CHAR + "\\" + "\"" + "\b" + "\f" + "\n" + "\r" + "\t" +end + +local escape_char_map: {ESCAPE_CHAR:string} = { + [ "\\" ] = "\\", + [ "\"" ] = "\"", + [ "\b" ] = "b", + [ "\f" ] = "f", + [ "\n" ] = "n", + [ "\r" ] = "r", + [ "\t" ] = "t", +} + +local escape_char_map_inv: {string:string} = { [ "/" ] = "/" } +for k, v in pairs(escape_char_map) do + escape_char_map_inv[v] = k +end + + +local function escape_char(c: ESCAPE_CHAR): string + return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte())) +end + + +local function encode_nil(_val: nil): string + return "null" +end + + +local function encode_table(val: table, stack: table): string + local res = {} + stack = stack or {} + + -- Circular reference? + if stack[val] then error("circular reference") end + + stack[val] = true + + if rawget(val, 1) ~= nil or next(val) == nil then + -- Treat as array -- check keys are valid and it is not sparse + local n = 0 + for k in pairs(val) do + if type(k) ~= "number" then + error("invalid table: mixed or invalid key types") + end + n = n + 1 + end + if n ~= #val as {any} then + error("invalid table: sparse array") + end + -- Encode + for _, v in ipairs(val as {table}) do + table.insert(res, encode(v, stack)) + end + stack[val] = nil + return "[" .. table.concat(res, ",") .. "]" + + else + -- Treat as an object + for k, v in pairs(val as {table:table}) do + if type(k) ~= "string" then + error("invalid table: mixed or invalid key types") + end + table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) + end + stack[val] = nil + return "{" .. table.concat(res, ",") .. "}" + end +end + + +local function encode_string(val: string): string + return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"' +end + + +local function encode_number(val: number): string + -- Check for NaN, -inf and inf + if val ~= val or val <= -math.huge or val >= math.huge then + error("unexpected number value '" .. tostring(val) .. "'") + end + return string.format("%.14g", val) +end + + +local type_func_map: {string:any} = { + [ "nil" ] = encode_nil, + [ "table" ] = encode_table, + [ "string" ] = encode_string, + [ "number" ] = encode_number, + [ "boolean" ] = tostring, +} + + +encode = function(val: table, stack?: table): string + local t = type(val) + local f = type_func_map[t] as function(table, table) + if f then + return f(val, stack) as string + end + error("unexpected type '" .. t .. "'") +end + + +------------------------------------------------------------------------------- +-- Below is standard `tested` licensed code +------------------------------------------------------------------------------- + +local record json is types.ResultFormatter where self.format == "json" end + +json.format = "json" + + +function json.header(_version: string, _filepaths: {string}, _comments: {string}): string + return "" +end + +function json.results(_tested_result: types.TestedOutput, _test_types_to_display: {types.TestResult: boolean}): string + return "" +end + +function json.summary(runner_output: types.TestRunnerOutput): string + return encode(runner_output as table) +end + + +return json \ No newline at end of file diff --git a/src/tested/file_output/txt.tl b/src/tested/file_output/txt.tl new file mode 100644 index 0000000..5e17b49 --- /dev/null +++ b/src/tested/file_output/txt.tl @@ -0,0 +1,12 @@ +-- this is just the plain results formatter +local plain = require("tested.results.plain") +local type types = require("tested.types") + +local record txt is types.ResultFormatter where self.format == "txt" end + +txt.format = "txt" +txt.header = plain.header +txt.results = plain.results +txt.summary = plain.summary + +return txt \ No newline at end of file diff --git a/src/tested/main.tl b/src/tested/main.tl index c7257e1..a5ee6c7 100644 --- a/src/tested/main.tl +++ b/src/tested/main.tl @@ -1,17 +1,17 @@ local lfs = require("lfs") -local file_loader = require("tested.file_loader") +local cli = require("tested.cli") +local file_loader = require("tested.file_loader") +local logging = require("tested.libs.logging") local test_runner = require("tested.test_runner") -local TestRunner, run_parallel_tests = test_runner[1], test_runner[2] +local type types = require("tested.types") +local util = require("tested.util") -local logging = require("tested.libs.logging") local logger = logging.get_logger("tested.main") +local TestRunner, run_parallel_tests = test_runner[1], test_runner[2] local TESTED_VERSION = "tested v0.2.0" -local type types = require("tested.types") -local cli = require("tested.cli") - local function load_result_formatter(args: cli.CLIOptions): types.ResultFormatter if args.custom_formatter then logger:info("Loading custom formatter: %s", args.custom_formatter) @@ -52,13 +52,13 @@ local function register_format_handler(handlers: {string}) local ok, module_format_handler = pcall(require, handler) as (boolean, types.FileLoaderHandler) if ok then file_loader.register_handler(module_format_handler.extension, module_format_handler.loader) - end - - local info, err = lfs.attributes(handler) - if err then error("Unable to load format handler, the file/module '" .. handler .."' was not able to be loaded.") end - if info.mode ~= "file" then error("The custom format loader should point to a file, but currently appears to be a: " .. info.mode, 0) end + else + local info, err = lfs.attributes(handler) + if err then error("Unable to load format handler, the file/module '" .. handler .."' was not able to be loaded.") end + if info.mode ~= "file" then error("The custom format loader should point to a file, but currently appears to be a: " .. info.mode, 0) end - file_loader.load_and_register_handler(handler) + file_loader.load_and_register_handler(handler) + end end end @@ -83,14 +83,10 @@ local function find_tests(files: {string}, test_path: string): {string} logger:info("Found %d test files to run in %s", #files, test_path) end -local function get_file_extension(str: string): string - return str:match("^.+(%..+)$") -end - local function get_all_test_files(args: cli.CLIOptions): {string} local all_files = {} for _, test_file in ipairs(args.test_files) do - if file_loader.loader[get_file_extension(test_file)] then + if file_loader.loader[util.get_file_extension(test_file)] then table.insert(all_files, test_file) end end @@ -110,7 +106,7 @@ local function run_tests(formatter: types.ResultFormatter, args: cli.CLIOptions, } local display_results = function(test_output: types.TestedOutput) - formatter.results(test_output, cli.display_types(args.show)) + print(formatter.results(test_output, cli.display_types(args.show))) end if args.threads == 0 or #test_files <= 1 then @@ -129,6 +125,24 @@ local function run_tests(formatter: types.ResultFormatter, args: cli.CLIOptions, return runner_output end +local function write_output_files(args: cli.CLIOptions, header_comments: {string}, runner_output: types.TestRunnerOutput) + -- write out all the files at the end + for _, file_output in ipairs(args.output) do + local extension = util.get_file_extension(file_output) + local output_formatter = require("tested.file_output" .. extension) as types.ResultFormatter + local file_to_write, err = io.open(file_output, "w") + if not file_to_write then error("Unable to open file for writing: " .. err, 0) end + file_to_write:write(output_formatter.header(TESTED_VERSION, args.paths, header_comments)) + + for _, test_output in ipairs(runner_output.module_results) do + file_to_write:write(output_formatter.results(test_output, cli.display_types(args.show))) + end + + file_to_write:write(output_formatter.summary(runner_output)) + file_to_write:close() + end +end + local function main() -- setting up defaults and registering things... local args = cli.parse_args(TESTED_VERSION) @@ -153,10 +167,13 @@ local function main() end -- running the tests - formatter.header(TESTED_VERSION, args.paths, header_comments) + print(formatter.header(TESTED_VERSION, args.paths, header_comments)) local runner_output = run_tests(formatter, args, test_files) - formatter.summary(runner_output) + runner_output.tested_version = TESTED_VERSION + print(formatter.summary(runner_output)) + + write_output_files(args, header_comments, runner_output) -- exiting cleanly if runner_output.all_fully_tested then diff --git a/src/tested/results/plain.tl b/src/tested/results/plain.tl index 1c6203d..e87aa66 100644 --- a/src/tested/results/plain.tl +++ b/src/tested/results/plain.tl @@ -1,9 +1,7 @@ local terminal = require("tested.results.terminal") local type types = require("tested.types") -local record plain is types.ResultFormatter where self.format == "plain" - -end +local record plain is types.ResultFormatter where self.format == "plain" end plain.format = "plain" diff --git a/src/tested/results/tap.tl b/src/tested/results/tap.tl index 18f2d45..fb7c70d 100644 --- a/src/tested/results/tap.tl +++ b/src/tested/results/tap.tl @@ -6,11 +6,11 @@ local record tap is types.ResultFormatter where self.format == "tap" end tap.allow_filtering = false tap.format = "tap" -function tap.header(_version_info: string, _filepaths: {string}, _comments: {string}) - print("TAP version 14") +function tap.header(_version_info: string, _filepaths: {string}, _comments: {string}): string + return "TAP version 14" end -function tap.results(tested_result: types.TestedOutput, _test_types_to_display: {types.TestResult: boolean}) +function tap.results(tested_result: types.TestedOutput, _test_types_to_display: {types.TestResult: boolean}): string tadd.new("# ", tested_result.filename, "\n") local tap_result: string @@ -23,7 +23,7 @@ function tap.results(tested_result: types.TestedOutput, _test_types_to_display: tadd.add(tap_result, test.name) if test.result == "SKIP" or test.result == "CONDITIONAL_SKIP" then - tadd.add(" # SKIP" or "", "\n") + tadd.add(" # SKIP", "\n") else tadd.add("\n") @@ -48,11 +48,11 @@ function tap.results(tested_result: types.TestedOutput, _test_types_to_display: end end - print(tadd.tostring()) + return tadd.tostring() end -function tap.summary(output: types.TestRunnerOutput) - print("1.." .. output.total_tests) +function tap.summary(output: types.TestRunnerOutput): string + return "1.." .. output.total_tests end return tap \ No newline at end of file diff --git a/src/tested/results/terminal.tl b/src/tested/results/terminal.tl index d371bca..5c166e0 100644 --- a/src/tested/results/terminal.tl +++ b/src/tested/results/terminal.tl @@ -41,12 +41,13 @@ terminal.allow_filtering = true -- used to disable the colors in the "plain" formatter terminal.colors = colors -function terminal.header(version_info: string, filepaths: {string}, comments: {string}) - print(colors("%{bright}" .. version_info .. " " .. table.concat(filepaths, " "))) +function terminal.header(version_info: string, filepaths: {string}, comments: {string}): string + tadd.new("%{bright}", version_info, " ", table.concat(filepaths, " "), "\n") for _, comment in ipairs(comments) do - print(comment) + tadd.add(comment, "\n") end - print() + tadd.add("\n") + return colors(tadd.tostring()) end local function to_ms(time: number, add_color?: boolean): string @@ -81,7 +82,7 @@ local function format_assertion_result(assertion_result: types.AssertionResult): tadd.add("\n") end -function terminal.results(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}) +function terminal.results(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}): string tadd.new("%{bright}- ", tested_result.filename, to_ms(tested_result.total_time), "%{reset}\n") for _, test_result in ipairs(tested_result.tests) do @@ -103,13 +104,13 @@ function terminal.results(tested_result: types.TestedOutput, test_types_to_displ if extra_newline then tadd.add("\n") end end - if test_result.result == "EXCEPTION" or test_result.result == "UNKNOWN" or test_result.result == "UNEXPECTED" or test_result.result == "EXPECTED_EXCEPTION" or test_result.result == "EXPECTED_UNKNOWN"then + if test_result.result == "EXCEPTION" or test_result.result == "UNKNOWN" or test_result.result == "UNEXPECTED" or test_result.result == "EXPECTED_EXCEPTION" or test_result.result == "EXPECTED_UNKNOWN" then tadd.add(" ", (test_result.message:gsub("\n", "\n ")), "\n") tadd.add("\n") end end end - print(colors(tadd.tostring())) + return colors(tadd.tostring()) end -- just being particular! @@ -121,7 +122,7 @@ local function test_counts_s(test_count: integer): string end end -function terminal.summary(output: types.TestRunnerOutput) +function terminal.summary(output: types.TestRunnerOutput): string tadd.new("%{bright}Test Summary for ", test_counts_s(output.total_tests), to_ms(output.total_time, false), ":%{reset}\n") tadd.add( @@ -155,7 +156,7 @@ function terminal.summary(output: types.TestRunnerOutput) tadd.add("\n%{bright}Fully Tested!%{reset}\n") end - print(colors(tadd.tostring())) + return colors(tadd.tostring()) end return terminal diff --git a/src/tested/types.tl b/src/tested/types.tl index d402133..96f48f0 100644 --- a/src/tested/types.tl +++ b/src/tested/types.tl @@ -75,15 +75,16 @@ local record types total_time: number total_tests: integer all_fully_tested: boolean + tested_version: string end -- needed for result handling interface ResultFormatter format: string allow_filtering: boolean - header: function(version: string, filepaths: {string}, comments: {string}) - results: function(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}) - summary: function(runner_output: types.TestRunnerOutput) + header: function(version: string, filepaths: {string}, comments: {string}): string + results: function(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}): string + summary: function(runner_output: types.TestRunnerOutput): string end -- needed for registering new file types diff --git a/src/tested/util.tl b/src/tested/util.tl new file mode 100644 index 0000000..c65ad47 --- /dev/null +++ b/src/tested/util.tl @@ -0,0 +1,9 @@ +local record util end + +-- i generally don't like utils, but so it goes! if we get enough and they're groupable enough i'll +-- create a utils/str.tl or something +function util.get_file_extension(str: string): string + return str:match("^.+(%..+)$") +end + +return util \ No newline at end of file diff --git a/tested-dev-1.rockspec b/tested-dev-1.rockspec index 00bece2..a6dcfcf 100644 --- a/tested-dev-1.rockspec +++ b/tested-dev-1.rockspec @@ -37,6 +37,7 @@ build = { ["tested.main"] = "build/tested/main.lua", ["tested.test_runner"] = "build/tested/test_runner.lua", ["tested.types"] = "build/tested/types.lua", + ["tested.util"] = "build/tested/util.lua", ["tested.libs.ansicolors"] = "build/tested/libs/ansicolors.lua", ["tested.libs.inspect"] = "build/tested/libs/inspect.lua", @@ -44,6 +45,9 @@ build = { ["tested.libs.tadd"] = "build/tested/libs/tadd.lua", ["tested.libs.ThreadPool"] = "build/tested/libs/ThreadPool.lua", + ["tested.file_output.txt"] = "build/tested/file_output/txt.lua", + ["tested.file_output.json"] = "build/tested/file_output/json.lua", + ["tested.results.plain"] = "build/tested/results/plain.lua", ["tested.results.tap"] = "build/tested/results/tap.lua", ["tested.results.terminal"] = "build/tested/results/terminal.lua", @@ -68,6 +72,9 @@ build = { ["tested.libs.tadd"] = "src/tested/libs/tadd.tl", ["tested.libs.ThreadPool"] = "src/tested/libs/ThreadPool.tl", + ["tested.file_output.txt"] = "src/tested/file_output/txt.tl", + ["tested.file_output.json"] = "src/tested/file_output/json.tl", + ["tested.results.plain"] = "src/tested/results/plain.tl", ["tested.results.tap"] = "src/tested/results/tap.tl", ["tested.results.terminal"] = "src/tested/results/terminal.tl", From e3545c09a14c910f9708ee48cc50a5e0765a7bad Mon Sep 17 00:00:00 2001 From: FourierTransformer Date: Wed, 13 May 2026 18:30:57 -0500 Subject: [PATCH 3/6] docs updated and cleared up output a bit --- build/tested/cli.lua | 8 +-- build/tested/main.lua | 2 +- docs/cli.md | 25 ++++++-- docs/custom-formatter.md | 14 +++-- docs/index.md | 3 +- docs/unit-testing.md | 130 +++++++++++++++++++++++++++++++++++++++ src/tested/cli.tl | 10 +-- src/tested/main.tl | 2 +- 8 files changed, 171 insertions(+), 23 deletions(-) diff --git a/build/tested/cli.lua b/build/tested/cli.lua index 9ce2599..3a52df3 100644 --- a/build/tested/cli.lua +++ b/build/tested/cli.lua @@ -81,9 +81,9 @@ function cli.parse_args(version) choices({ "terminal", "plain", "tap" }): default("terminal"), parser:option("-z --custom-formatter"): - description("File that loads a custom formatter to use for output")) + description("File that loads a custom formatter to use for terminal output")) - parser:option("-o --output"): + parser:option("-o --output-file"): description("Output file to save test results in (currently supported extensions: '.txt' and '.json')"): count("*") parser:option("-n --threads"): @@ -125,8 +125,8 @@ function cli.set_defaults(args) for _, display_option in ipairs(args.show) do if display_option == "all" then show_all = true; break end end if show_all then args.show = { "skip", "pass", "fail", "exception", "unknown", "expected", "unexpected" } end - if #args.output > 0 then - for _, output_file in ipairs(args.output) do + if #args.output_file > 0 then + for _, output_file in ipairs(args.output_file) do if not (util.get_file_extension(output_file) == ".txt" or util.get_file_extension(output_file) == ".json") then error("The given output file does not have a supported file extension: '" .. output_file .. "'. Supported file extensions are: '.txt', '.json'", 0) end diff --git a/build/tested/main.lua b/build/tested/main.lua index eb598ee..4d86eac 100644 --- a/build/tested/main.lua +++ b/build/tested/main.lua @@ -127,7 +127,7 @@ end local function write_output_files(args, header_comments, runner_output) - for _, file_output in ipairs(args.output) do + for _, file_output in ipairs(args.output_file) do local extension = util.get_file_extension(file_output) local output_formatter = require("tested.file_output" .. extension) local file_to_write, err = io.open(file_output, "w") diff --git a/docs/cli.md b/docs/cli.md index 8cd20c6..eaac8a5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -48,6 +48,18 @@ Specify the number of threads `tested` should use. If set to `0`, will not use a ## `tested -z/--custom-formatter` `tested` supports loading a [custom result formatter](./custom-formatter.md) from the commandline. It tries to load what's passed in initially as a Lua module, and then as filepath, doing some basic checks to ensure the object returned appears to be a formatter. Only one custom formatter can be loaded and will be used to display results. +## `tested -o/--output-file` +Output file to save results to a specified file. It loads a formatter based on the file extension of the file that's passed in. The currently supported extensions are: + +- `.txt` - Outputs _exactly_ what is shown in the terminal (includes any filtering) +- `.json` - Outputs the full [TestRunnerOutput](./custom-formatter.md#testrunneroutput). Everything - it does not filter anything. We're still pre v1, so the output here _could_ change. + +Multiple files can be specified and all will be written to at the end of the test run: + +```bash +tested -o ./terminal_output.txt -o ./full_output.json +``` + ## `tested -x/--format-handler` `tested` also supports loading [custom format handlers](./additional-formats.md). These are used to extend the functionality of tested and tap into other languages that can embed into Lua. Similar to custom output formatters, the format handler will first try and load from a Lua module and then from a filepath. This allows flexibility in distribution in how folks may want to support a custom format. Multiple format handlers can be loaded, and afterward can be used for custom formatters or the tests themselves. @@ -57,8 +69,8 @@ Specify the number of threads `tested` should use. If set to `0`, will not use a Usage: tested ([-f {terminal,plain,tap}] | [-z ]) [-h] [-c] [-r] [-F ] [-s {all,valid,invalid,skip,pass,fail,exception,unknown,expected,unexpected}] - [-n ] [-x ] [-d {DEBUG,INFO,WARNING}] - [--version] [] ... + [-o ] [-n ] [-x ] + [-d {DEBUG,INFO,WARNING}] [--version] [] ... A Lua/Teal Unit Testing Framework @@ -79,14 +91,17 @@ Options: What format to output the results in (default: 'terminal') (default: terminal) -z , --custom-formatter - File that loads a custom formatter to use for output - -n , Set the number of threads to run the tests with (default: 4). Set to 0 to disable. + File that loads a custom formatter to use for terminal output + -o , + --output-file + Output file to save test results in (currently supported extensions: '.txt' and '.json') + -n , Set the number of threads to run the tests with (default: 4). Set to 0 to disable. Test files are split amongst the threads. --threads -x , --format-handler File that loads custom formats that are Lua-compatible -d {DEBUG,INFO,WARNING}, --debug {DEBUG,INFO,WARNING} - Set the log level - mostly for debugging purposes (default: 'WARNING') (default: WARNING) + Set the log level - mostly for debugging issues with tested (default: 'WARNING') (default: WARNING) --version Show version information ``` diff --git a/docs/custom-formatter.md b/docs/custom-formatter.md index aab4e35..2d17b39 100644 --- a/docs/custom-formatter.md +++ b/docs/custom-formatter.md @@ -1,5 +1,5 @@ # Custom Formatters -As of now (1/2026), `tested` currently supports a "terminal" output and a "plain" output (which is just the terminal output without colors), with plans for a few more [in the future](https://github.com/FourierTransformer/tested/issues/21)! But we've tried to make it easy to create your own formatter for `tested`. +As of now (1/2026), `tested` currently supports printing out a "terminal", "plain" (which is just the terminal output without colors), and "tap" output. We've tried to make it easy to create your own formatter for `tested`. You can only create a custom formatter for _display_ purposes. If you want to create a custom formatter for _file saving_ purposes, please [create an issue](https://github.com/FourierTransformer/tested/issues)! ## A basic formatter @@ -128,7 +128,8 @@ An example of what `types.TestRunnerOutput` looks like. `types.TestedOutput` is skipped = 0 }, total_tests = 5, - total_time = 2.1999999999966e-05 + total_time = 2.1999999999966e-05, + tested_version = "tested v0.2.0" } ``` @@ -140,7 +141,7 @@ enum TestResult "SKIP" "CONDITIONAL_SKIP" "EXCEPTION" - "TIMEOUT" + -- "TIMEOUT" -- NYI "UNKNOWN" "EXPECTED_FAIL" "EXPECTED_EXCEPTION" @@ -187,14 +188,15 @@ interface TestRunnerOutput total_time: number total_tests: integer all_fully_tested: boolean + tested_version: string end -- The actual formatter itself interface ResultFormatter format: string allow_filtering: boolean - header: function(version: string, filepaths: {string}) - results: function(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}) - summary: function(runner_output: types.TestRunnerOutput) + header: function(version: string, filepaths: {string}, comments: {string}): string + results: function(tested_result: types.TestedOutput, test_types_to_display: {types.TestResult: boolean}): string + summary: function(runner_output: types.TestRunnerOutput): string end ``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 92365ed..0283c1f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -137,7 +137,8 @@ There are a couple CLI commands that are good to know when you get started: - `tested -F ` or `--filter ` will filter tests based on a `string.find` pattern. It can just be the test name, a couple words from the test name (in order), or a full on Lua pattern! - `tested -s