diff --git a/src/SnapshotTesting.jl b/src/SnapshotTesting.jl index becfb66..6c501d7 100644 --- a/src/SnapshotTesting.jl +++ b/src/SnapshotTesting.jl @@ -5,5 +5,15 @@ import DeepDiffs using Test include("snapshots.jl") +include("suite.jl") + +export SnapshotTestSuite, + snapshot_tests, + snapshot_expected_dir, + snapshot_produce, + snapshot_test_extras, + snapshot_allow_additions, + run_snapshot_tests, + run_snapshot_test end diff --git a/src/suite.jl b/src/suite.jl new file mode 100644 index 0000000..9c971f2 --- /dev/null +++ b/src/suite.jl @@ -0,0 +1,116 @@ +""" + SnapshotTestSuite{X} + +Parameterized type for dispatch-based snapshot test suites (following the Salsa +`Keyspace{X}` pattern). Packages pick a unique symbol `X` and overload the +interface functions to get automatic test iteration, filtering, and plumbing. + +# Required methods +- `snapshot_tests(suite)` — return a `Vector{Pair{String,String}}` of `(name, path)` pairs +- `snapshot_expected_dir(suite)` — return the path to the expected snapshots directory +- `snapshot_produce(suite, name, path, dir)` — produce output files in `dir` + +# Optional methods (have defaults) +- `snapshot_test_extras(suite, name, path)` — run assertions before the snapshot comparison (default: no-op) +- `snapshot_allow_additions(suite)` — whether to allow new files in snapshots (default: `true`) + +# Example + +```julia +const MySuite = SnapshotTestSuite{:my_package} + +SnapshotTesting.snapshot_tests(::MySuite) = [("test1" => "path/to/test1"), ...] +SnapshotTesting.snapshot_expected_dir(::MySuite) = joinpath(@__DIR__, "expected") +function SnapshotTesting.snapshot_produce(::MySuite, name, path, dir) + write(joinpath(dir, "out.txt"), run_my_code(path)) +end + +run_snapshot_tests(MySuite()) +run_snapshot_test(MySuite(), "test1") # run single test +``` +""" +struct SnapshotTestSuite{X} end + +""" + snapshot_tests(suite::SnapshotTestSuite) -> Vector{Pair{String,String}} + +Return the list of test cases as `name => path` pairs. +""" +function snapshot_tests end + +""" + snapshot_expected_dir(suite::SnapshotTestSuite) -> String + +Return the path to the directory containing expected snapshot outputs. +""" +function snapshot_expected_dir end + +""" + snapshot_produce(suite::SnapshotTestSuite, name::String, path::String, dir::String) + +Produce snapshot output files in `dir`. +""" +function snapshot_produce end + +""" + snapshot_test_extras(suite::SnapshotTestSuite, name::String, path::String) + +Run additional assertions before the snapshot comparison. Defaults to no-op. +""" +snapshot_test_extras(::SnapshotTestSuite, name, path) = nothing + +""" + snapshot_allow_additions(suite::SnapshotTestSuite) -> Bool + +Whether to allow new files in snapshot output that aren't in the expected directory. +Defaults to `true`. +""" +snapshot_allow_additions(::SnapshotTestSuite) = true + +""" + run_snapshot_tests(suite::SnapshotTestSuite; filter=nothing, skip=String[]) + +Run all snapshot tests defined by `suite`, with optional filtering and skipping. + +- `filter=nothing`: run all tests +- `filter="name"`: run only the test with exactly this name +- `filter=f::Function`: run tests where `f(name)` returns `true` +- `skip=["name1", ...]`: skip tests with these names +""" +function run_snapshot_tests(suite::SnapshotTestSuite; filter=nothing, skip=String[]) + cases = snapshot_tests(suite) + expected = snapshot_expected_dir(suite) + allow = snapshot_allow_additions(suite) + pred = _make_filter(filter) + skip_set = Set{String}(skip) + + matched = false + for (name, path) in cases + pred(name) || continue + name in skip_set && continue + matched = true + @testset "$name" begin + snapshot_test_extras(suite, name, path) + test_snapshot(expected, name; allow_additions=allow) do dir + snapshot_produce(suite, name, path, dir) + end + end + end + + if !matched && filter !== nothing + @warn "No snapshot tests matched the filter" filter + end +end + +""" + run_snapshot_test(suite::SnapshotTestSuite, name::AbstractString) + +Run a single snapshot test by name. Convenience wrapper around +`run_snapshot_tests(suite; filter=name)`. +""" +run_snapshot_test(suite::SnapshotTestSuite, name::AbstractString) = + run_snapshot_tests(suite; filter=name) + +_make_filter(::Nothing) = _ -> true +_make_filter(name::AbstractString) = n -> n == name +_make_filter(f::Function) = f diff --git a/test/runtests.jl b/test/runtests.jl index 3a03c52..3186508 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,4 +8,7 @@ using Test @testset "update modes" begin include("test_update_modes.jl") end + @testset "suite" begin + include("test_suite.jl") + end end diff --git a/test/test_suite.jl b/test/test_suite.jl new file mode 100644 index 0000000..c33759e --- /dev/null +++ b/test/test_suite.jl @@ -0,0 +1,217 @@ +using SnapshotTesting +using Test + +# --- Helper to pre-create expected snapshot directories --- + +function setup_expected!(expected_dir, name, content) + d = joinpath(expected_dir, name) + mkpath(d) + write(joinpath(d, "out.txt"), content) +end + +# --- Suite definitions at top level (required by Julia scoping rules) --- + +# :basic_all — basic suite that runs all tests +const _basic_all_cases = Ref{Vector{Pair{String,String}}}() +const _basic_all_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:basic_all}) = _basic_all_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:basic_all}) = _basic_all_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:basic_all}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :filter_str — filter by exact string +const _filter_str_cases = Ref{Vector{Pair{String,String}}}() +const _filter_str_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_str}) = _filter_str_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_str}) = _filter_str_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_str}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :filter_pred — filter by predicate function +const _filter_pred_cases = Ref{Vector{Pair{String,String}}}() +const _filter_pred_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_pred}) = _filter_pred_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_pred}) = _filter_pred_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_pred}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :skip_list — skip list +const _skip_list_cases = Ref{Vector{Pair{String,String}}}() +const _skip_list_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:skip_list}) = _skip_list_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:skip_list}) = _skip_list_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:skip_list}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :empty_suite — empty suite +const _empty_suite_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:empty_suite}) = Pair{String,String}[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:empty_suite}) = _empty_suite_expected[] + +# :filter_warn — filter matching nothing emits warning +const _filter_warn_cases = Ref{Vector{Pair{String,String}}}() +const _filter_warn_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_warn}) = _filter_warn_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_warn}) = _filter_warn_expected[] + +# :extras_test — snapshot_test_extras is called +const _extras_test_cases = Ref{Vector{Pair{String,String}}}() +const _extras_test_expected = Ref{String}() +const _extras_test_called = String[] +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:extras_test}) = _extras_test_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:extras_test}) = _extras_test_expected[] +function SnapshotTesting.snapshot_test_extras(::SnapshotTestSuite{:extras_test}, name, _path) + push!(_extras_test_called, name) +end +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:extras_test}, name, _path, dir) + write(joinpath(dir, "out.txt"), "extras output for $name") +end + +# :strict_suite — snapshot_allow_additions=false +const _strict_suite_cases = Ref{Vector{Pair{String,String}}}() +const _strict_suite_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:strict_suite}) = _strict_suite_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:strict_suite}) = _strict_suite_expected[] +SnapshotTesting.snapshot_allow_additions(::SnapshotTestSuite{:strict_suite}) = false +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:strict_suite}, name, _path, dir) + write(joinpath(dir, "out.txt"), "output for $name") +end + +# :singular_test — run_snapshot_test singular convenience +const _singular_test_cases = Ref{Vector{Pair{String,String}}}() +const _singular_test_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:singular_test}) = _singular_test_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:singular_test}) = _singular_test_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:singular_test}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# --- Tests --- + +@testset "SnapshotTestSuite" begin + + @testset "basic suite runs all tests" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "output for test_a from /path/test_a") + setup_expected!(expected, "test_b", "output for test_b from /path/test_b") + + _basic_all_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _basic_all_expected[] = expected + run_snapshot_tests(SnapshotTestSuite{:basic_all}()) + end + end + + @testset "filter by exact string" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "output for test_a from /path/test_a") + + _filter_str_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _filter_str_expected[] = expected + run_snapshot_tests(SnapshotTestSuite{:filter_str}(); filter="test_a") + # test_b should not have been run (no directory created) + @test !isdir(joinpath(expected, "test_b")) + end + end + + @testset "filter by predicate function" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "csv_one", "output for csv_one from /path/csv_one") + setup_expected!(expected, "csv_two", "output for csv_two from /path/csv_two") + + _filter_pred_cases[] = [ + "csv_one" => "/path/csv_one", + "csv_two" => "/path/csv_two", + "bin_one" => "/path/bin_one", + ] + _filter_pred_expected[] = expected + run_snapshot_tests(SnapshotTestSuite{:filter_pred}(); filter=n -> startswith(n, "csv_")) + @test !isdir(joinpath(expected, "bin_one")) + end + end + + @testset "skip list" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_b", "output for test_b from /path/test_b") + + _skip_list_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _skip_list_expected[] = expected + run_snapshot_tests(SnapshotTestSuite{:skip_list}(); skip=["test_a"]) + @test !isdir(joinpath(expected, "test_a")) + end + end + + @testset "empty suite does not error" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + _empty_suite_expected[] = expected + run_snapshot_tests(SnapshotTestSuite{:empty_suite}()) + end + end + + @testset "filter matching nothing emits warning" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + _filter_warn_cases[] = ["test_a" => "/path/test_a"] + _filter_warn_expected[] = expected + @test_logs (:warn, "No snapshot tests matched the filter") run_snapshot_tests( + SnapshotTestSuite{:filter_warn}(); filter="nonexistent" + ) + end + end + + @testset "snapshot_test_extras is called before produce" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "extras output for test_a") + setup_expected!(expected, "test_b", "extras output for test_b") + + empty!(_extras_test_called) + _extras_test_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _extras_test_expected[] = expected + run_snapshot_tests(SnapshotTestSuite{:extras_test}()) + @test _extras_test_called == ["test_a", "test_b"] + end + end + + @testset "snapshot_allow_additions=false is respected" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "output for test_a") + + _strict_suite_cases[] = ["test_a" => "/path/test_a"] + _strict_suite_expected[] = expected + run_snapshot_tests(SnapshotTestSuite{:strict_suite}()) + end + end + + @testset "run_snapshot_test singular convenience" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "only_this", "output for only_this from /path/only_this") + + _singular_test_cases[] = ["only_this" => "/path/only_this", "not_this" => "/path/not_this"] + _singular_test_expected[] = expected + run_snapshot_test(SnapshotTestSuite{:singular_test}(), "only_this") + # not_this should not have been run + @test !isdir(joinpath(expected, "not_this")) + end + end + +end