estest

#!/usr/bin/env es
# vi: ft=es

fmt = %dict(
	bold => <={%string \x01 \x1b '[1m' \x02}
	redbold => <={%string \x01 \x1b '[' 31 ';' 1 'm' \x02}
	yellow => <={%string \x01 \x1b '[33m' \x02}
	green => <={%string \x01 \x1b '[' 32 'm' \x02}
	greenbold => <={%string \x01 \x1b '[' 32 ';1' 'm' \x02}
	nocolor => <={%string \x01 \x1b '[' 0 'm' \x02}
)

fn info {
	echo '>>> '^$fmt(bold)^'info:'^$fmt(nocolor) $*
}

fn warning {
	echo '>>> '^$fmt(redbold)^'warning:'^$fmt(nocolor) $*
}

fn lastelem elems {
	result $elems($#elems)
}

fn output {
	if {$print_failure_stderr} {
		echo >[1=2] $*
	} {
		echo $*
	}
}

dollar0 = $0
runbin = false
testoutput = false
estest_debugging = false
testpath = 'tests'
print_failure_output = false
print_failure_stderr = false
extra_es_args = false
always_yes = false
srcdir = '.'
fullhelp = false
verbose = false
subprocess_default = false
libdir = 'libraries'
testlibs = false
testcore = true
commit = <={%fsplit ' ' $buildstring |> %elem 3 |> %fsplit '-' |> %last}
run_one = false
test_to_run = ()
keep_test_logs = false
disabled_tests = %dict()

noexport += disabled_tests # there seems to be a bug when importing empty dicts

(_ _ restargs) = <={parseargs @ arg {
	match $arg (
		(-h) { fullhelp = true ; usage }
		(-x) {
			runbin = true
			if {has_argument} {
				esbin = $flagarg
			}
		}
		(-b) {
			if {access -xf $flagarg} {
				esbin = $flagarg
			}
		}
		(-v) { verbose = true }
		(-V) { testoutput = true }
		(-d) { estest_debugging = true }
		(-E) { print_failure_output = true }
		(-A) {
			extra_es_args = true; done
		}
		(-y) { always_yes = true }
		(-s) { subprocess_default = true }
		(-S) {
			srcdir = <={
				if {~ $flagarg */ } {
					result <={~~ $flagarg */}
				} {
					result $flagarg
				}
			}
			if {~ <={access -r1d $srcdir |> %count} 0 } {
				throw error $dollar0 $srcdir^' is not accessible'
			}
			if {~ $testpath 'tests'} {
				testpath = $srcdir/tests
			}
			if {~ $libdir 'libraries'} {
				libdir = $srcdir/libraries
			}
		}
		(-C) { testcore = false }
		(-l) { testlibs = true }
		(-L) {
			libdir = <={
				if {~ $flagarg */ } {
					result <={~~ $flagarg */}
				} {
					result $flagarg
				}
			}
			if {~ <={access -r1d $libdir |> %count} 0 } {
				throw error $dollar0 $libdir^' is no accessible'
			}
			testlibs = true
		}
		(-K) { keep_test_logs = true }
		(-p) {
			testpath = <={
				if {~ $flagarg */} {
					result <={~~ $flagarg */}
				} {
					result $flagarg
				}
			}
		}
		(-R) {
			run_one = true
			if {~ $flagarg *^','^*} {
				test_to_run = $test_to_run <={%fsplit ',' $flagarg}
			} {
				test_to_run = $test_to_run $flagarg
			}
		}
		(-D) {
			if {~ $flagarg *^','^*} {
				for (t = <={%split ',' $flagarg}) {
					disabled_tests := $t => true
				}
			} {
				disabled_tests := $flagarg => true
			}
		}
		* { usage }
	)
} @{
	echo 'usage: '^$dollar0^' [-dvVqyslC] [-x [es binary]] [-S srcdir] [-p path] [-R test,...] [-D test,...] [-L libdir] [-A es arguments]'
	if {$fullhelp} {
		cat << '%%end'
    flags:
        -d -- enable debugging
        -v -- verbose
        -V -- show test output in all cases
        -E -- show test output on failures
        -y -- automatically answer yes
        -h -- show this message
        -s -- default to using subprocesses for tests
        -C -- do not run core tests
        -l -- also test libraries (implicit for -L)
        -K -- keep test logs

        -x [es binary]  -- run estest in a different instance of es
        -R test,...     -- run selected tests
        -D test,...     -- disable selected tests
        -S srcdir       -- es source directory
        -p path         -- test cases path
        -L path         -- libraries path
        -A es arguments -- arguments to pass to instances of es

        -b es binary    -- set es binary
%%end
		exit 0
	} {
		exit 1
	}
} $*}

if {! access -rd $testpath} {
	throw error $dollar0 $testpath^' is not accessible'
}

if {~ $#esbin 0} {
	esbin = <={
		if {access -xf $srcdir/es} {
			result $srcdir/es
		} {
			result <={access -1n es -xf $path}
		}
	}
}

fn run_subproc_test testname testfn test_log tfp libdir {
	libraries = $libdir
	info 'running '^$testname^' in pid '^$pid > $test_log
	catch @ e {
		if {~ $e abort} {
			echo 'got abort'
			exit 2
		}
		if {! $$tfp} {
			echo 'got exception: '^$^e >> $test_log
		}
		exit 1
	} {
		local(es_assert_def = $fn-assert; es_assert2_def = $fn-assert2) local (
			fn-assert = @{ echo 'try assert:' $* ; $es_assert_def $*}
			fn-assert2 = @{ echo 'try assert2:' $*(1) ':' $*(2 ...) ; $es_assert2_def $*}
			fn-abort = @{ throw abort }
			es = $esbin
		) {
			$testfn >> $test_log
		}
	}
	exit 0
}

fn run_test_file file {
	assert2 $0 {eq $#file 1}
	lets (
		islibtest = <={if {~ $file */libtest_*.es} { result true } { result false }}
		testname = <={
			if {$islibtest} {
				result <={~~ <={%split '/' $file |> lastelem} libtest_*.es}
			} {
				result <={~~ <={%split '/' $file |> lastelem} test_*.es}
			}
		}
		passed = false
		skipped = false
		test_exception=
		test_log='/tmp/es_'^$pid^'_test_'^$testname^'.log'
		tfp = <={
			if {$islibtest} {
				result 'libtest_'^$testname^'_fail_pass'
			} {
				result 'test_'^$testname^'_fail_pass'
			}
		}
		tsp = <={
			if {$islibtest} {
				result 'libtest_'^$testname^'_subprocess'
			} {
				result 'test_'^$testname^'_subprocess'
			}
		}
		tspargs = <={
			if {$islibtest} {
				result 'libtest_'^$testname^'_subprocess_arguments'
			} {
				result 'test_'^$testname^'_subprocess_arguments'
			}
		}
		tdisabled = <={
			if {$islibtest} {
				result 'libtest_'^$testname^'_disable'
			} {
				result 'test_'^$testname^'_disable'
			}
		}
	) {
		. $file
		if {~ <={%count $$tfp} 0} {
			$tfp = false
		}
		if {~ <={%count $$tsp} 0} {
			$tsp = $subprocess_default
		}
		if {~ <={%count $$tdisabled} 0 || ~ $$tdisabled false} {
			if {dicthaskey $disabled_tests $testname} {
				$tdisabled = true
			} {
				$tdisabled = false
			}
		}
		if {$islibtest} {
			echo -n 'library test '^$testname^': '
		} {
			if {$testlibs} {
				echo -n 'core test '^$testname^': '
			} {
				echo -n 'test '^$testname^': '
			}
		}
		if {$$tdisabled} {
			echo $fmt(redbold)^'disabled'^$fmt(nocolor)
			return true
		}
		catch @ e {
			if {~ $e abort} {
				skipped = true
			} {
				test_exception = $e
				if {$$tfp} {
					passed = true
				} {
					passed = false
				}
			}
		} {
			if {$$tsp} {
				escmd = $esbin
				if {! ~ <={%count $$tspargs} 0} {
					escmd = $escmd $$tspargs
				}
				if {$extra_es_args} { escmd = $escmd $restargs }
				escmd = $escmd -c 'run_subproc_test '^$testname^' run_test_'^$testname^' '^$test_log^' '^$tfp^' '^$libdir
				if {$estest_debugging} { info 'running: ' $escmd }
				match <={$escmd} (
					1 { throw error run_subproc 'return false' }
					2 { throw abort }
				)
			} {
				local(es_assert_def = $fn-assert; es_assert2_def = $fn-assert2) local(
					fn-assert = @{echo 'try assert:' $* ; $es_assert_def $*}
					fn-assert2 = @{ echo 'try assert2:' $*(1) ':' $*(2 ...) ; $es_assert2_def $*}
					fn-abort = @{ throw abort }
					es = $esbin
				) {
					run_test_$testname > $test_log
				}
			}
			if {$$tfp} {
				passed = false
			} {
				passed = true
			}
		}

		if {$skipped} {
			echo 'skipped.'
		} {$passed} {
			echo $fmt(green)^'passed!'^$fmt(nocolor)
			if {$testoutput} {
				output '>>> '^$testname^' output:'
				for(outline = ``(\n) {cat $test_log}) {
					output '>>>>>> '^$^outline
				}
			}
		} {
			echo $fmt(redbold)^'failed!'^$fmt(nocolor)
			echo '>>> '^$fmt(redbold)^'exception:'^$fmt(nocolor)^' '^$^test_exception
			if {$print_failure_output} {
				output '>>> '^$testname^' output:'
				for(outline = ``(\n) {cat $test_log}) {
					output '>>>>>> '^$^outline
				}
			}
		}
	if {! $keep_test_logs} {
		rm $test_log
	}
	result $passed
	}
}

fn run_all_tests {
	for(i = $testfiles) {
		run_test_file $i
	}
}

fn run_one_test single_test {
	for (i = $testfiles) {
		if {~ $i *^'/test_'^$single_test^'.es'} {
			run_test_file $i
		}
	}
}

fn get-git-rev dir {
	let (cwd = `{pwd}; rev=) {
		cd $dir
		rev = `{git rev-parse --verify --short=12 HEAD >[2]/dev/null}
		cd $cwd
		result $rev
	}
}

if {$runbin} {
	debugarg = <={ if {$estest_debugging} { result '-d' } { result '' }}
	testoutputarg = <={ if {$testoutput} { result '-v' } { result '' }}
	cmdline = $esbin -Np
	if {$extra_es_args} { cmdline = $cmdline $restargs }
	cmdline = $cmdline '--' $0
	if {$estest_debugging} { cmdline = $cmdline '-d' }
	if {$verbose} { cmdline = $cmdline '-v' }
	if {$testoutput} { cmdline = $cmdline '-V' }
	if {$always_yes} { cmdline = $cmdline '-y' }
	if {$subprocess_default} { cmdline = $cmdline '-s' }
	if {! $testcore} { cmdline = $cmdline '-C' }
	if {! ~ $srcdir '.'} {cmdline = $cmdline '-S' $srcdir}
	cmdline = $cmdline '-b' $esbin
	cmdline = $cmdline '-p' $testpath
	if {$testlibs} { cmdline = $cmdline '-L' $libdir }
	if {$run_one} { for (i = $test_to_run) { cmdline = $cmdline '-R' $i } }
	if {gt $#disabled_tests 0} { cmdline = $cmdline '-D' <={dictkeys $disabled_tests |> %flatten ','} }
	if {$extra_es_args} { cmdline = $cmdline '-A' $restargs }
	info 'parent build: '^$buildstring
	info 'running:' $cmdline
	exit <={$cmdline}
} {
	if {! $testcore && ! $testlibs} {
		echo 'error: must run core and/or library tests'
		exit 1
	}
	info 'test build: '^$buildstring
	if {! ~ <={get-git-rev $srcdir} $commit} {
		warning 'es commit and estest commit do not match (es '^$commit^', estest '^<={get-git-rev $srcdir}^')'
	}
	info 'using '^<={%gcstats |> %first}^' garbage collector'
	if {! $testcore} {
		info 'only running library tests'
	}
	libraries = $libdir
	es_conf_dynlib-local-search = true
	info 'using libraries in '^$libraries^'/'
	info 'running tests in '^$testpath^'/'

	inside_estest = true

	testfiles = <={
		let (core_tests =; lib_tests=) {
			process $testpath/*.es (
				(*/test_*) {
					if {$testcore} {
						if {access -xf $matchexpr} {
							if {$estest_debugging || $verbose} {
								info 'found '^$matchexpr
							}
							core_tests = $core_tests $matchexpr
						} {
							if {$estest_debugging || $verbose} {
								info 'ignoring '^$matchexpr
							}
						}
					}
				}
				(*/libtest_*) {
					if {$testlibs} {
						if {access -xf $matchexpr} {
							if {$estest_debugging || $verbose} {
								info 'found '^$matchexpr
							}
							lib_tests = $lib_tests $matchexpr
						} {
							if {$estest_debugging || $verbose} {
								info 'ignoring '^$matchexpr
							}
						}
					}
				}
				(*/util_*) {
					if {access -xf $matchexpr} {
						if {$estest_debugging || $verbose} {
							info 'loading '^$matchexpr
						}
						. $matchexpr
					}
				}
			)
			result $core_tests $lib_tests
		}
	}

	match $#testfiles (
		0 { info 'found no tests' }
		1 { info 'found 1 test' }
		* { info 'found '^$matchexpr^' tests' }
	)
	if {gt $#disabled_tests 0} {
		info 'temporarily disabled tests: '^<={dictkeys $disabled_tests |> %flatten ' '}
	}
	if {$run_one} {
		run_one_test $test_to_run
	} {
		if {$always_yes} {
			run_all_tests
		} {
			echo -n 'run all tests [yes]? '
			r = <=%read
			if {~ $r '' || ~ $r [yY]*} {
				run_all_tests
			}
		}
	}
	exit <=true
}