Add file output, clean up stack traces, and update docs

.github/workflows/tests-and-coverage.yml

@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [linux, macos, macos-arm64]
-        lua: [lua=5.1, lua=5.2, lua=5.3, lua=5.4, luajit=@v2.0, luajit=@v2.1]
+        lua: [lua=5.1, lua=5.2, lua=5.4, luajit=@v2.1]
         include:
           - os: linux
             runner: ubuntu-latest
@@ -77,7 +77,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        lua: [lua=5.1, lua=5.2, lua=5.3, lua=5.4, luajit=@v2.0, luajit=@v2.1]
+        lua: [lua=5.1, lua=5.2, lua=5.4, luajit=@v2.1]
         target: [mingw, vs]
     runs-on: windows-2022
     steps:

README.md

@@ -55,5 +55,6 @@ Parts of the following are included in the source code present in this repo:
 - Bundles a slightly modified [inspect.lua](https://github.com/kikito/inspect.lua) for table diffing and viewing - MIT
 - Also bundles a slightly modified [ansicolors.lua](https://github.com/kikito/ansicolors.lua) - MIT
 - A function from [Luacov](https://github.com/lunarmodules/luacov) code to help merge stats files in process - MIT
+- Bundles [json.lua](https://github.com/rxi/json.lua)'s encoder for writing file output to json
 
 Major thanks to hishamhm, kikito, and benoit-germain for their work in the Lua space. Without them, `tested` wouldn't be possible.

build/tested.lua

@@ -3,6 +3,8 @@ local assert_table = require("tested.assert_table")
 
 local tested = { tests = {}, run_only_tests = false }
 
+
+
 local function validate_options(test_name, options, test_src)
    local error_prefix = test_src .. " in \"" .. test_name .. "\": "
    if options.expected ~= nil then
@@ -32,6 +34,8 @@ local function extract_fn_and_options(test_name, fn_or_options, fn, test_src)
       fn = fn
    end
 
+   tested.filename = test_src
+
    validate_options(test_name, options, test_src or "?")
 
    return fn, options
@@ -160,10 +164,16 @@ local function should_skip_test(test, run_only, options)
    return nil, nil
 end
 
+local function captured_os_exit(code)
+   local prefix = "os.exit()"
+   if code then prefix = "os.exit(" .. tostring(code) .. ")" end
+   error(prefix .. " intercepted — something tried to exit out of the process", 0)
+end
+
 local function set_result(ok, err, total_assertions, assert_failed_count, test_output)
    if ok == false then
       test_output.result = "EXCEPTION"
-      test_output.message = err .. "\n" .. debug.traceback()
+      test_output.message = err
 
    elseif total_assertions == 0 then
       test_output.result = "UNKNOWN"
@@ -238,6 +248,11 @@ function tested:run(filename, options)
       tested.before_fn()
    end
 
+   local function xpcall_handler(e)
+      local msg = type(e) == "string" and (e) or tostring(e)
+      return msg .. debug.traceback("", 2)
+   end
+
    for i, test in ipairs(self.tests) do
 
       test_results.tests[i] = { assertion_results = {}, name = test.name }
@@ -264,8 +279,8 @@ function tested:run(filename, options)
             total_assertions = total_assertions + 1
 
             local assertion_result = {}
-            local file_info = debug.getinfo(2, "Sl")
-            assertion_result.filename = file_info.short_src
+            local file_info = debug.getinfo(2, "l")
+            assertion_result.filename = tested.filename
             assertion_result.line_number = file_info.currentline
 
             assertion_result.given = assertion.given
@@ -283,12 +298,34 @@ function tested:run(filename, options)
             return ok, err
          end
 
+
          local start = os.clock()
-         local ok, err = pcall(test.fn)
+
+         local original_os_exit
+         if _VERSION == "Lua 5.1" then
+
+            original_os_exit = getfenv(test.fn).os.exit
+            getfenv(test.fn).os.exit = captured_os_exit
+         else
+            original_os_exit = os.exit
+            os.exit = captured_os_exit
+         end
+
+
+         local ok, err = xpcall(test.fn, xpcall_handler)
          test_results.tests[i].time = os.clock() - start
          test_results.total_time = test_results.total_time + test_results.tests[i].time
+
+
          self.assert = original_assert
 
+
+         if _VERSION == "Lua 5.1" then
+            getfenv(test.fn).os.exit = original_os_exit
+         else
+            os.exit = original_os_exit
+         end
+
          set_result(ok, err, total_assertions, assert_failed_count, test_results.tests[i])
 
 

build/tested/cli.lua

@@ -3,6 +3,7 @@ local lfs = require("lfs")
 
 local logging = require("tested.libs.logging")
 local logger = logging.get_logger("tested.cli")
+local util = require("tested.util")
 
 
 
@@ -47,6 +48,7 @@ local cli = { CLIOptions = {} }
 
 
 
+
 
 
 local cli_to_display = {
@@ -79,10 +81,13 @@ function cli.parse_args(version)
    choices({ "terminal", "plain", "tap" }):
    default("terminal"),
    parser:option("-z --custom-formatter"):
-   description("File that loads a custom formatter to use for output"))
+   description("File that loads a custom formatter to use for terminal output"))
 
+   parser:option("-o --output-file"):
+   description("Output file to save test results in (currently supported extensions: '.txt' and '.json')"):
+   count("*")
    parser:option("-n --threads"):
-   description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable."):
+   description("Set the number of threads to run the tests with (default: 4). Set to 0 to disable. Test files are split amongst the threads."):
    default(4):
    convert(tonumber)
    parser:option("-x --format-handler"):
@@ -119,6 +124,14 @@ function cli.set_defaults(args)
    local show_all = false
    for _, display_option in ipairs(args.show) do if display_option == "all" then show_all = true; break end end
    if show_all then args.show = { "skip", "pass", "fail", "exception", "unknown", "expected", "unexpected" } end
+
+   if #args.output_file > 0 then
+      for _, output_file in ipairs(args.output_file) do
+         if not (util.get_file_extension(output_file) == ".txt" or util.get_file_extension(output_file) == ".json") then
+            error("The given output file does not have a supported file extension: '" .. output_file .. "'. Supported file extensions are: '.txt', '.json'", 0)
+         end
+      end
+   end
 end
 
 function cli.validate_args(args)

build/tested/file_output/json.lua

@@ -0,0 +1,142 @@
+
+
+
+
+
+
+local encode
+
+
+
+
+
+
+
+
+
+
+
+local escape_char_map = {
+   ["\\"] = "\\",
+   ["\""] = "\"",
+   ["\b"] = "b",
+   ["\f"] = "f",
+   ["\n"] = "n",
+   ["\r"] = "r",
+   ["\t"] = "t",
+}
+
+local escape_char_map_inv = { ["/"] = "/" }
+for k, v in pairs(escape_char_map) do
+   escape_char_map_inv[v] = k
+end
+
+
+local function escape_char(c)
+   return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte()))
+end
+
+
+local function encode_nil(_val)
+   return "null"
+end
+
+
+local function encode_table(val, stack)
+   local res = {}
+   stack = stack or {}
+
+
+   if stack[val] then error("circular reference") end
+
+   stack[val] = true
+
+   if rawget(val, 1) ~= nil or next(val) == nil then
+
+      local n = 0
+      for k in pairs(val) do
+         if type(k) ~= "number" then
+            error("invalid table: mixed or invalid key types")
+         end
+         n = n + 1
+      end
+      if n ~= #val then
+         error("invalid table: sparse array")
+      end
+
+      for _, v in ipairs(val) do
+         table.insert(res, encode(v, stack))
+      end
+      stack[val] = nil
+      return "[" .. table.concat(res, ",") .. "]"
+
+   else
+
+      for k, v in pairs(val) do
+         if type(k) ~= "string" then
+            error("invalid table: mixed or invalid key types")
+         end
+         table.insert(res, encode(k, stack) .. ":" .. encode(v, stack))
+      end
+      stack[val] = nil
+      return "{" .. table.concat(res, ",") .. "}"
+   end
+end
+
+
+local function encode_string(val)
+   return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"'
+end
+
+
+local function encode_number(val)
+
+   if val ~= val or val <= -math.huge or val >= math.huge then
+      error("unexpected number value '" .. tostring(val) .. "'")
+   end
+   return string.format("%.14g", val)
+end
+
+
+local type_func_map = {
+   ["nil"] = encode_nil,
+   ["table"] = encode_table,
+   ["string"] = encode_string,
+   ["number"] = encode_number,
+   ["boolean"] = tostring,
+}
+
+
+encode = function(val, stack)
+   local t = type(val)
+   local f = type_func_map[t]
+   if f then
+      return f(val, stack)
+   end
+   error("unexpected type '" .. t .. "'")
+end
+
+
+
+
+
+
+local json = {}
+
+json.format = "json"
+
+
+function json.header(_version, _filepaths, _comments)
+   return ""
+end
+
+function json.results(_tested_result, _test_types_to_display)
+   return ""
+end
+
+function json.summary(runner_output)
+   return encode(runner_output)
+end
+
+
+return json

build/tested/file_output/txt.lua

@@ -0,0 +1,12 @@
+
+local plain = require("tested.results.plain")
+
+
+local txt = {}
+
+txt.format = "txt"
+txt.header = plain.header
+txt.results = plain.results
+txt.summary = plain.summary
+
+return txt