From f4bc035e2a0933ccf360215922011a93a90622d2 Mon Sep 17 00:00:00 2001 From: tf Date: Tue, 10 Feb 2026 23:16:27 +0000 Subject: [PATCH 1/3] Add dispatch-based SnapshotTestSuite interface Add an abstract type `SnapshotTestSuite` with a `run_snapshot_tests` driver that handles test iteration, filtering, and skipping. Packages define a marker type, overload a few functions (snapshot_tests, snapshot_expected_dir, snapshot_produce), and get automatic plumbing. Supports filtering by exact name or predicate function, skip lists, and optional pre-snapshot assertions via snapshot_test_extras. The existing test_snapshot API is untouched. Co-Authored-By: Claude Opus 4.6 --- src/SnapshotTesting.jl | 9 +++ src/suite.jl | 107 +++++++++++++++++++++++++++ test/runtests.jl | 3 + test/test_suite.jl | 159 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 278 insertions(+) create mode 100644 src/suite.jl create mode 100644 test/test_suite.jl diff --git a/src/SnapshotTesting.jl b/src/SnapshotTesting.jl index becfb66..4dd5890 100644 --- a/src/SnapshotTesting.jl +++ b/src/SnapshotTesting.jl @@ -5,5 +5,14 @@ import DeepDiffs using Test include("snapshots.jl") +include("suite.jl") + +export SnapshotTestSuite, + snapshot_tests, + snapshot_expected_dir, + snapshot_produce, + snapshot_test_extras, + snapshot_allow_additions, + run_snapshot_tests end diff --git a/src/suite.jl b/src/suite.jl new file mode 100644 index 0000000..093c838 --- /dev/null +++ b/src/suite.jl @@ -0,0 +1,107 @@ +""" + SnapshotTestSuite + +Abstract type for dispatch-based snapshot test suites. Packages define a concrete +subtype and overload the interface functions to get automatic test iteration, +filtering, and plumbing. + +# Required methods +- `snapshot_tests(suite)` — return a `Vector{Pair{String,String}}` of `(name, path)` pairs +- `snapshot_expected_dir(suite)` — return the path to the expected snapshots directory +- `snapshot_produce(suite, name, path, dir)` — produce output files in `dir` + +# Optional methods (have defaults) +- `snapshot_test_extras(suite, name, path)` — run assertions before the snapshot comparison (default: no-op) +- `snapshot_allow_additions(suite)` — whether to allow new files in snapshots (default: `true`) + +# Example + +```julia +struct MySnapshots <: SnapshotTestSuite end + +SnapshotTesting.snapshot_tests(::MySnapshots) = [("test1" => "path/to/test1"), ...] +SnapshotTesting.snapshot_expected_dir(::MySnapshots) = joinpath(@__DIR__, "expected") +function SnapshotTesting.snapshot_produce(::MySnapshots, name, path, dir) + write(joinpath(dir, "out.txt"), run_my_code(path)) +end + +run_snapshot_tests(MySnapshots()) +run_snapshot_tests(MySnapshots(); filter="test1") # run single test +``` +""" +abstract type SnapshotTestSuite end + +""" + snapshot_tests(suite::SnapshotTestSuite) -> Vector{Pair{String,String}} + +Return the list of test cases as `name => path` pairs. +""" +function snapshot_tests end + +""" + snapshot_expected_dir(suite::SnapshotTestSuite) -> String + +Return the path to the directory containing expected snapshot outputs. +""" +function snapshot_expected_dir end + +""" + snapshot_produce(suite::SnapshotTestSuite, name::String, path::String, dir::String) + +Produce snapshot output files in `dir`. +""" +function snapshot_produce end + +""" + snapshot_test_extras(suite::SnapshotTestSuite, name::String, path::String) + +Run additional assertions before the snapshot comparison. Defaults to no-op. +""" +snapshot_test_extras(::SnapshotTestSuite, name, path) = nothing + +""" + snapshot_allow_additions(suite::SnapshotTestSuite) -> Bool + +Whether to allow new files in snapshot output that aren't in the expected directory. +Defaults to `true`. +""" +snapshot_allow_additions(::SnapshotTestSuite) = true + +""" + run_snapshot_tests(suite::SnapshotTestSuite; filter=nothing, skip=String[]) + +Run all snapshot tests defined by `suite`, with optional filtering and skipping. + +- `filter=nothing`: run all tests +- `filter="name"`: run only the test with exactly this name +- `filter=f::Function`: run tests where `f(name)` returns `true` +- `skip=["name1", ...]`: skip tests with these names +""" +function run_snapshot_tests(suite::SnapshotTestSuite; filter=nothing, skip=String[]) + cases = snapshot_tests(suite) + expected = snapshot_expected_dir(suite) + allow = snapshot_allow_additions(suite) + pred = _make_filter(filter) + skip_set = Set{String}(skip) + + matched = false + for (name, path) in cases + pred(name) || continue + name in skip_set && continue + matched = true + @testset "$name" begin + snapshot_test_extras(suite, name, path) + test_snapshot(expected, name; allow_additions=allow) do dir + snapshot_produce(suite, name, path, dir) + end + end + end + + if !matched && filter !== nothing + @warn "No snapshot tests matched the filter" filter + end +end + +_make_filter(::Nothing) = _ -> true +_make_filter(name::AbstractString) = n -> n == name +_make_filter(f::Function) = f diff --git a/test/runtests.jl b/test/runtests.jl index 3a03c52..3186508 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,4 +8,7 @@ using Test @testset "update modes" begin include("test_update_modes.jl") end + @testset "suite" begin + include("test_suite.jl") + end end diff --git a/test/test_suite.jl b/test/test_suite.jl new file mode 100644 index 0000000..22bf35e --- /dev/null +++ b/test/test_suite.jl @@ -0,0 +1,159 @@ +using SnapshotTesting +using Test + +# --- Test suite types --- + +struct BasicSuite <: SnapshotTestSuite + cases::Vector{Pair{String,String}} + expected_dir::String +end +SnapshotTesting.snapshot_tests(s::BasicSuite) = s.cases +SnapshotTesting.snapshot_expected_dir(s::BasicSuite) = s.expected_dir +function SnapshotTesting.snapshot_produce(::BasicSuite, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +mutable struct ExtrasSuite <: SnapshotTestSuite + cases::Vector{Pair{String,String}} + expected_dir::String + extras_called::Vector{String} + ExtrasSuite(cases, dir) = new(cases, dir, String[]) +end +SnapshotTesting.snapshot_tests(s::ExtrasSuite) = s.cases +SnapshotTesting.snapshot_expected_dir(s::ExtrasSuite) = s.expected_dir +function SnapshotTesting.snapshot_test_extras(s::ExtrasSuite, name, _path) + push!(s.extras_called, name) +end +function SnapshotTesting.snapshot_produce(::ExtrasSuite, name, _path, dir) + write(joinpath(dir, "out.txt"), "extras output for $name") +end + +struct StrictSuite <: SnapshotTestSuite + cases::Vector{Pair{String,String}} + expected_dir::String +end +SnapshotTesting.snapshot_tests(s::StrictSuite) = s.cases +SnapshotTesting.snapshot_expected_dir(s::StrictSuite) = s.expected_dir +SnapshotTesting.snapshot_allow_additions(::StrictSuite) = false +function SnapshotTesting.snapshot_produce(::StrictSuite, name, _path, dir) + write(joinpath(dir, "out.txt"), "output for $name") +end + +# --- Helper to pre-create expected snapshot directories --- + +function setup_expected!(expected_dir, name, content) + d = joinpath(expected_dir, name) + mkpath(d) + write(joinpath(d, "out.txt"), content) +end + +# --- Tests --- + +@testset "SnapshotTestSuite" begin + + @testset "basic suite runs all tests" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "output for test_a from /path/test_a") + setup_expected!(expected, "test_b", "output for test_b from /path/test_b") + + cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + suite = BasicSuite(cases, expected) + run_snapshot_tests(suite) + end + end + + @testset "filter by exact string" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "output for test_a from /path/test_a") + + cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + suite = BasicSuite(cases, expected) + run_snapshot_tests(suite; filter="test_a") + # test_b should not have been run (no directory created) + @test !isdir(joinpath(expected, "test_b")) + end + end + + @testset "filter by predicate function" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "csv_one", "output for csv_one from /path/csv_one") + setup_expected!(expected, "csv_two", "output for csv_two from /path/csv_two") + + cases = [ + "csv_one" => "/path/csv_one", + "csv_two" => "/path/csv_two", + "bin_one" => "/path/bin_one", + ] + suite = BasicSuite(cases, expected) + run_snapshot_tests(suite; filter=n -> startswith(n, "csv_")) + @test !isdir(joinpath(expected, "bin_one")) + end + end + + @testset "skip list" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_b", "output for test_b from /path/test_b") + + cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + suite = BasicSuite(cases, expected) + run_snapshot_tests(suite; skip=["test_a"]) + @test !isdir(joinpath(expected, "test_a")) + end + end + + @testset "empty suite does not error" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + suite = BasicSuite(Pair{String,String}[], expected) + run_snapshot_tests(suite) + end + end + + @testset "filter matching nothing emits warning" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + cases = ["test_a" => "/path/test_a"] + suite = BasicSuite(cases, expected) + @test_logs (:warn, "No snapshot tests matched the filter") run_snapshot_tests( + suite; filter="nonexistent" + ) + end + end + + @testset "snapshot_test_extras is called before produce" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "extras output for test_a") + setup_expected!(expected, "test_b", "extras output for test_b") + + cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + suite = ExtrasSuite(cases, expected) + run_snapshot_tests(suite) + @test suite.extras_called == ["test_a", "test_b"] + end + end + + @testset "snapshot_allow_additions=false is respected" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "test_a", "output for test_a") + + cases = ["test_a" => "/path/test_a"] + suite = StrictSuite(cases, expected) + run_snapshot_tests(suite) + end + end + +end From d8903a01f90259531064a59de909dcfe204213ea Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Feb 2026 03:38:15 +0000 Subject: [PATCH 2/3] Address PR review: parameterized SnapshotTestSuite{X} + run_snapshot_test - Change SnapshotTestSuite from abstract type to parameterized struct SnapshotTestSuite{X}, following the Salsa Keyspace{X} pattern. Users dispatch on SnapshotTestSuite{:my_symbol} instead of defining/exporting concrete subtypes. - Add run_snapshot_test(suite, name) convenience function for running a single snapshot test by name. - Update tests to use parameterized types with unique symbols per testset. Co-Authored-By: Claude Opus 4.6 --- src/SnapshotTesting.jl | 3 +- src/suite.jl | 31 ++++++---- test/test_suite.jl | 130 ++++++++++++++++++++++++----------------- 3 files changed, 97 insertions(+), 67 deletions(-) diff --git a/src/SnapshotTesting.jl b/src/SnapshotTesting.jl index 4dd5890..6c501d7 100644 --- a/src/SnapshotTesting.jl +++ b/src/SnapshotTesting.jl @@ -13,6 +13,7 @@ export SnapshotTestSuite, snapshot_produce, snapshot_test_extras, snapshot_allow_additions, - run_snapshot_tests + run_snapshot_tests, + run_snapshot_test end diff --git a/src/suite.jl b/src/suite.jl index 093c838..9c971f2 100644 --- a/src/suite.jl +++ b/src/suite.jl @@ -1,9 +1,9 @@ """ - SnapshotTestSuite + SnapshotTestSuite{X} -Abstract type for dispatch-based snapshot test suites. Packages define a concrete -subtype and overload the interface functions to get automatic test iteration, -filtering, and plumbing. +Parameterized type for dispatch-based snapshot test suites (following the Salsa +`Keyspace{X}` pattern). Packages pick a unique symbol `X` and overload the +interface functions to get automatic test iteration, filtering, and plumbing. # Required methods - `snapshot_tests(suite)` — return a `Vector{Pair{String,String}}` of `(name, path)` pairs @@ -17,19 +17,19 @@ filtering, and plumbing. # Example ```julia -struct MySnapshots <: SnapshotTestSuite end +const MySuite = SnapshotTestSuite{:my_package} -SnapshotTesting.snapshot_tests(::MySnapshots) = [("test1" => "path/to/test1"), ...] -SnapshotTesting.snapshot_expected_dir(::MySnapshots) = joinpath(@__DIR__, "expected") -function SnapshotTesting.snapshot_produce(::MySnapshots, name, path, dir) +SnapshotTesting.snapshot_tests(::MySuite) = [("test1" => "path/to/test1"), ...] +SnapshotTesting.snapshot_expected_dir(::MySuite) = joinpath(@__DIR__, "expected") +function SnapshotTesting.snapshot_produce(::MySuite, name, path, dir) write(joinpath(dir, "out.txt"), run_my_code(path)) end -run_snapshot_tests(MySnapshots()) -run_snapshot_tests(MySnapshots(); filter="test1") # run single test +run_snapshot_tests(MySuite()) +run_snapshot_test(MySuite(), "test1") # run single test ``` """ -abstract type SnapshotTestSuite end +struct SnapshotTestSuite{X} end """ snapshot_tests(suite::SnapshotTestSuite) -> Vector{Pair{String,String}} @@ -102,6 +102,15 @@ function run_snapshot_tests(suite::SnapshotTestSuite; filter=nothing, skip=Strin end end +""" + run_snapshot_test(suite::SnapshotTestSuite, name::AbstractString) + +Run a single snapshot test by name. Convenience wrapper around +`run_snapshot_tests(suite; filter=name)`. +""" +run_snapshot_test(suite::SnapshotTestSuite, name::AbstractString) = + run_snapshot_tests(suite; filter=name) + _make_filter(::Nothing) = _ -> true _make_filter(name::AbstractString) = n -> n == name _make_filter(f::Function) = f diff --git a/test/test_suite.jl b/test/test_suite.jl index 22bf35e..99b94f7 100644 --- a/test/test_suite.jl +++ b/test/test_suite.jl @@ -1,44 +1,6 @@ using SnapshotTesting using Test -# --- Test suite types --- - -struct BasicSuite <: SnapshotTestSuite - cases::Vector{Pair{String,String}} - expected_dir::String -end -SnapshotTesting.snapshot_tests(s::BasicSuite) = s.cases -SnapshotTesting.snapshot_expected_dir(s::BasicSuite) = s.expected_dir -function SnapshotTesting.snapshot_produce(::BasicSuite, name, path, dir) - write(joinpath(dir, "out.txt"), "output for $name from $path") -end - -mutable struct ExtrasSuite <: SnapshotTestSuite - cases::Vector{Pair{String,String}} - expected_dir::String - extras_called::Vector{String} - ExtrasSuite(cases, dir) = new(cases, dir, String[]) -end -SnapshotTesting.snapshot_tests(s::ExtrasSuite) = s.cases -SnapshotTesting.snapshot_expected_dir(s::ExtrasSuite) = s.expected_dir -function SnapshotTesting.snapshot_test_extras(s::ExtrasSuite, name, _path) - push!(s.extras_called, name) -end -function SnapshotTesting.snapshot_produce(::ExtrasSuite, name, _path, dir) - write(joinpath(dir, "out.txt"), "extras output for $name") -end - -struct StrictSuite <: SnapshotTestSuite - cases::Vector{Pair{String,String}} - expected_dir::String -end -SnapshotTesting.snapshot_tests(s::StrictSuite) = s.cases -SnapshotTesting.snapshot_expected_dir(s::StrictSuite) = s.expected_dir -SnapshotTesting.snapshot_allow_additions(::StrictSuite) = false -function SnapshotTesting.snapshot_produce(::StrictSuite, name, _path, dir) - write(joinpath(dir, "out.txt"), "output for $name") -end - # --- Helper to pre-create expected snapshot directories --- function setup_expected!(expected_dir, name, content) @@ -59,8 +21,13 @@ end setup_expected!(expected, "test_b", "output for test_b from /path/test_b") cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - suite = BasicSuite(cases, expected) - run_snapshot_tests(suite) + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:basic_all}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:basic_all}) = expected + function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:basic_all}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") + end + + run_snapshot_tests(SnapshotTestSuite{:basic_all}()) end end @@ -71,8 +38,13 @@ end setup_expected!(expected, "test_a", "output for test_a from /path/test_a") cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - suite = BasicSuite(cases, expected) - run_snapshot_tests(suite; filter="test_a") + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_str}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_str}) = expected + function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_str}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") + end + + run_snapshot_tests(SnapshotTestSuite{:filter_str}(); filter="test_a") # test_b should not have been run (no directory created) @test !isdir(joinpath(expected, "test_b")) end @@ -90,8 +62,13 @@ end "csv_two" => "/path/csv_two", "bin_one" => "/path/bin_one", ] - suite = BasicSuite(cases, expected) - run_snapshot_tests(suite; filter=n -> startswith(n, "csv_")) + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_pred}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_pred}) = expected + function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_pred}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") + end + + run_snapshot_tests(SnapshotTestSuite{:filter_pred}(); filter=n -> startswith(n, "csv_")) @test !isdir(joinpath(expected, "bin_one")) end end @@ -103,8 +80,13 @@ end setup_expected!(expected, "test_b", "output for test_b from /path/test_b") cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - suite = BasicSuite(cases, expected) - run_snapshot_tests(suite; skip=["test_a"]) + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:skip_list}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:skip_list}) = expected + function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:skip_list}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") + end + + run_snapshot_tests(SnapshotTestSuite{:skip_list}(); skip=["test_a"]) @test !isdir(joinpath(expected, "test_a")) end end @@ -113,8 +95,10 @@ end mktempdir() do tmpdir expected = joinpath(tmpdir, "expected") mkpath(expected) - suite = BasicSuite(Pair{String,String}[], expected) - run_snapshot_tests(suite) + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:empty_suite}) = Pair{String,String}[] + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:empty_suite}) = expected + + run_snapshot_tests(SnapshotTestSuite{:empty_suite}()) end end @@ -123,9 +107,11 @@ end expected = joinpath(tmpdir, "expected") mkpath(expected) cases = ["test_a" => "/path/test_a"] - suite = BasicSuite(cases, expected) + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_warn}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_warn}) = expected + @test_logs (:warn, "No snapshot tests matched the filter") run_snapshot_tests( - suite; filter="nonexistent" + SnapshotTestSuite{:filter_warn}(); filter="nonexistent" ) end end @@ -137,10 +123,19 @@ end setup_expected!(expected, "test_a", "extras output for test_a") setup_expected!(expected, "test_b", "extras output for test_b") + extras_called = String[] cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - suite = ExtrasSuite(cases, expected) - run_snapshot_tests(suite) - @test suite.extras_called == ["test_a", "test_b"] + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:extras_test}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:extras_test}) = expected + function SnapshotTesting.snapshot_test_extras(::SnapshotTestSuite{:extras_test}, name, _path) + push!(extras_called, name) + end + function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:extras_test}, name, _path, dir) + write(joinpath(dir, "out.txt"), "extras output for $name") + end + + run_snapshot_tests(SnapshotTestSuite{:extras_test}()) + @test extras_called == ["test_a", "test_b"] end end @@ -151,8 +146,33 @@ end setup_expected!(expected, "test_a", "output for test_a") cases = ["test_a" => "/path/test_a"] - suite = StrictSuite(cases, expected) - run_snapshot_tests(suite) + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:strict_suite}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:strict_suite}) = expected + SnapshotTesting.snapshot_allow_additions(::SnapshotTestSuite{:strict_suite}) = false + function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:strict_suite}, name, _path, dir) + write(joinpath(dir, "out.txt"), "output for $name") + end + + run_snapshot_tests(SnapshotTestSuite{:strict_suite}()) + end + end + + @testset "run_snapshot_test singular convenience" begin + mktempdir() do tmpdir + expected = joinpath(tmpdir, "expected") + mkpath(expected) + setup_expected!(expected, "only_this", "output for only_this from /path/only_this") + + cases = ["only_this" => "/path/only_this", "not_this" => "/path/not_this"] + SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:singular_test}) = cases + SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:singular_test}) = expected + function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:singular_test}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") + end + + run_snapshot_test(SnapshotTestSuite{:singular_test}(), "only_this") + # not_this should not have been run + @test !isdir(joinpath(expected, "not_this")) end end From be458768f8da15c72dd037447f5b8b6c33c88f83 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Feb 2026 03:42:47 +0000 Subject: [PATCH 3/3] Fix test_suite.jl: move method definitions to top level Julia requires global method definitions to be at the top level, not inside local scopes like `mktempdir() do ... end` blocks. Use Ref variables at module level to pass test-specific data (cases, expected dirs) into the parameterized type method implementations. Co-Authored-By: Claude Opus 4.6 --- test/test_suite.jl | 162 ++++++++++++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 62 deletions(-) diff --git a/test/test_suite.jl b/test/test_suite.jl index 99b94f7..c33759e 100644 --- a/test/test_suite.jl +++ b/test/test_suite.jl @@ -9,6 +9,87 @@ function setup_expected!(expected_dir, name, content) write(joinpath(d, "out.txt"), content) end +# --- Suite definitions at top level (required by Julia scoping rules) --- + +# :basic_all — basic suite that runs all tests +const _basic_all_cases = Ref{Vector{Pair{String,String}}}() +const _basic_all_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:basic_all}) = _basic_all_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:basic_all}) = _basic_all_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:basic_all}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :filter_str — filter by exact string +const _filter_str_cases = Ref{Vector{Pair{String,String}}}() +const _filter_str_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_str}) = _filter_str_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_str}) = _filter_str_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_str}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :filter_pred — filter by predicate function +const _filter_pred_cases = Ref{Vector{Pair{String,String}}}() +const _filter_pred_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_pred}) = _filter_pred_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_pred}) = _filter_pred_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_pred}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :skip_list — skip list +const _skip_list_cases = Ref{Vector{Pair{String,String}}}() +const _skip_list_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:skip_list}) = _skip_list_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:skip_list}) = _skip_list_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:skip_list}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + +# :empty_suite — empty suite +const _empty_suite_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:empty_suite}) = Pair{String,String}[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:empty_suite}) = _empty_suite_expected[] + +# :filter_warn — filter matching nothing emits warning +const _filter_warn_cases = Ref{Vector{Pair{String,String}}}() +const _filter_warn_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_warn}) = _filter_warn_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_warn}) = _filter_warn_expected[] + +# :extras_test — snapshot_test_extras is called +const _extras_test_cases = Ref{Vector{Pair{String,String}}}() +const _extras_test_expected = Ref{String}() +const _extras_test_called = String[] +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:extras_test}) = _extras_test_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:extras_test}) = _extras_test_expected[] +function SnapshotTesting.snapshot_test_extras(::SnapshotTestSuite{:extras_test}, name, _path) + push!(_extras_test_called, name) +end +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:extras_test}, name, _path, dir) + write(joinpath(dir, "out.txt"), "extras output for $name") +end + +# :strict_suite — snapshot_allow_additions=false +const _strict_suite_cases = Ref{Vector{Pair{String,String}}}() +const _strict_suite_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:strict_suite}) = _strict_suite_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:strict_suite}) = _strict_suite_expected[] +SnapshotTesting.snapshot_allow_additions(::SnapshotTestSuite{:strict_suite}) = false +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:strict_suite}, name, _path, dir) + write(joinpath(dir, "out.txt"), "output for $name") +end + +# :singular_test — run_snapshot_test singular convenience +const _singular_test_cases = Ref{Vector{Pair{String,String}}}() +const _singular_test_expected = Ref{String}() +SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:singular_test}) = _singular_test_cases[] +SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:singular_test}) = _singular_test_expected[] +function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:singular_test}, name, path, dir) + write(joinpath(dir, "out.txt"), "output for $name from $path") +end + # --- Tests --- @testset "SnapshotTestSuite" begin @@ -20,13 +101,8 @@ end setup_expected!(expected, "test_a", "output for test_a from /path/test_a") setup_expected!(expected, "test_b", "output for test_b from /path/test_b") - cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:basic_all}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:basic_all}) = expected - function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:basic_all}, name, path, dir) - write(joinpath(dir, "out.txt"), "output for $name from $path") - end - + _basic_all_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _basic_all_expected[] = expected run_snapshot_tests(SnapshotTestSuite{:basic_all}()) end end @@ -37,13 +113,8 @@ end mkpath(expected) setup_expected!(expected, "test_a", "output for test_a from /path/test_a") - cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_str}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_str}) = expected - function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_str}, name, path, dir) - write(joinpath(dir, "out.txt"), "output for $name from $path") - end - + _filter_str_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _filter_str_expected[] = expected run_snapshot_tests(SnapshotTestSuite{:filter_str}(); filter="test_a") # test_b should not have been run (no directory created) @test !isdir(joinpath(expected, "test_b")) @@ -57,17 +128,12 @@ end setup_expected!(expected, "csv_one", "output for csv_one from /path/csv_one") setup_expected!(expected, "csv_two", "output for csv_two from /path/csv_two") - cases = [ + _filter_pred_cases[] = [ "csv_one" => "/path/csv_one", "csv_two" => "/path/csv_two", "bin_one" => "/path/bin_one", ] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_pred}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_pred}) = expected - function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_pred}, name, path, dir) - write(joinpath(dir, "out.txt"), "output for $name from $path") - end - + _filter_pred_expected[] = expected run_snapshot_tests(SnapshotTestSuite{:filter_pred}(); filter=n -> startswith(n, "csv_")) @test !isdir(joinpath(expected, "bin_one")) end @@ -79,13 +145,8 @@ end mkpath(expected) setup_expected!(expected, "test_b", "output for test_b from /path/test_b") - cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:skip_list}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:skip_list}) = expected - function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:skip_list}, name, path, dir) - write(joinpath(dir, "out.txt"), "output for $name from $path") - end - + _skip_list_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _skip_list_expected[] = expected run_snapshot_tests(SnapshotTestSuite{:skip_list}(); skip=["test_a"]) @test !isdir(joinpath(expected, "test_a")) end @@ -95,9 +156,7 @@ end mktempdir() do tmpdir expected = joinpath(tmpdir, "expected") mkpath(expected) - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:empty_suite}) = Pair{String,String}[] - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:empty_suite}) = expected - + _empty_suite_expected[] = expected run_snapshot_tests(SnapshotTestSuite{:empty_suite}()) end end @@ -106,10 +165,8 @@ end mktempdir() do tmpdir expected = joinpath(tmpdir, "expected") mkpath(expected) - cases = ["test_a" => "/path/test_a"] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_warn}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_warn}) = expected - + _filter_warn_cases[] = ["test_a" => "/path/test_a"] + _filter_warn_expected[] = expected @test_logs (:warn, "No snapshot tests matched the filter") run_snapshot_tests( SnapshotTestSuite{:filter_warn}(); filter="nonexistent" ) @@ -123,19 +180,11 @@ end setup_expected!(expected, "test_a", "extras output for test_a") setup_expected!(expected, "test_b", "extras output for test_b") - extras_called = String[] - cases = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:extras_test}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:extras_test}) = expected - function SnapshotTesting.snapshot_test_extras(::SnapshotTestSuite{:extras_test}, name, _path) - push!(extras_called, name) - end - function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:extras_test}, name, _path, dir) - write(joinpath(dir, "out.txt"), "extras output for $name") - end - + empty!(_extras_test_called) + _extras_test_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"] + _extras_test_expected[] = expected run_snapshot_tests(SnapshotTestSuite{:extras_test}()) - @test extras_called == ["test_a", "test_b"] + @test _extras_test_called == ["test_a", "test_b"] end end @@ -145,14 +194,8 @@ end mkpath(expected) setup_expected!(expected, "test_a", "output for test_a") - cases = ["test_a" => "/path/test_a"] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:strict_suite}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:strict_suite}) = expected - SnapshotTesting.snapshot_allow_additions(::SnapshotTestSuite{:strict_suite}) = false - function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:strict_suite}, name, _path, dir) - write(joinpath(dir, "out.txt"), "output for $name") - end - + _strict_suite_cases[] = ["test_a" => "/path/test_a"] + _strict_suite_expected[] = expected run_snapshot_tests(SnapshotTestSuite{:strict_suite}()) end end @@ -163,13 +206,8 @@ end mkpath(expected) setup_expected!(expected, "only_this", "output for only_this from /path/only_this") - cases = ["only_this" => "/path/only_this", "not_this" => "/path/not_this"] - SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:singular_test}) = cases - SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:singular_test}) = expected - function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:singular_test}, name, path, dir) - write(joinpath(dir, "out.txt"), "output for $name from $path") - end - + _singular_test_cases[] = ["only_this" => "/path/only_this", "not_this" => "/path/not_this"] + _singular_test_expected[] = expected run_snapshot_test(SnapshotTestSuite{:singular_test}(), "only_this") # not_this should not have been run @test !isdir(joinpath(expected, "not_this"))