Add dispatch-based SnapshotTestSuite interface

src/SnapshotTesting.jl

@@ -5,5 +5,15 @@ import DeepDiffs
 using Test
 
 include("snapshots.jl")
+include("suite.jl")
+
+export SnapshotTestSuite,
+    snapshot_tests,
+    snapshot_expected_dir,
+    snapshot_produce,
+    snapshot_test_extras,
+    snapshot_allow_additions,
+    run_snapshot_tests,
+    run_snapshot_test
 
 end

src/suite.jl

@@ -0,0 +1,116 @@
+"""
+    SnapshotTestSuite{X}
+
+Parameterized type for dispatch-based snapshot test suites (following the Salsa
+`Keyspace{X}` pattern). Packages pick a unique symbol `X` and overload the
+interface functions to get automatic test iteration, filtering, and plumbing.
+
+# Required methods
+- `snapshot_tests(suite)` — return a `Vector{Pair{String,String}}` of `(name, path)` pairs
+- `snapshot_expected_dir(suite)` — return the path to the expected snapshots directory
+- `snapshot_produce(suite, name, path, dir)` — produce output files in `dir`
+
+# Optional methods (have defaults)
+- `snapshot_test_extras(suite, name, path)` — run assertions before the snapshot comparison (default: no-op)
+- `snapshot_allow_additions(suite)` — whether to allow new files in snapshots (default: `true`)
+
+# Example
+
+```julia
+const MySuite = SnapshotTestSuite{:my_package}
+
+SnapshotTesting.snapshot_tests(::MySuite) = [("test1" => "path/to/test1"), ...]
+SnapshotTesting.snapshot_expected_dir(::MySuite) = joinpath(@__DIR__, "expected")
+function SnapshotTesting.snapshot_produce(::MySuite, name, path, dir)
+    write(joinpath(dir, "out.txt"), run_my_code(path))
+end
+
+run_snapshot_tests(MySuite())
+run_snapshot_test(MySuite(), "test1")  # run single test
+```
+"""
+struct SnapshotTestSuite{X} end
+
+"""
+    snapshot_tests(suite::SnapshotTestSuite) -> Vector{Pair{String,String}}
+
+Return the list of test cases as `name => path` pairs.
+"""
+function snapshot_tests end
+
+"""
+    snapshot_expected_dir(suite::SnapshotTestSuite) -> String
+
+Return the path to the directory containing expected snapshot outputs.
+"""
+function snapshot_expected_dir end
+
+"""
+    snapshot_produce(suite::SnapshotTestSuite, name::String, path::String, dir::String)
+
+Produce snapshot output files in `dir`.
+"""
+function snapshot_produce end
+
+"""
+    snapshot_test_extras(suite::SnapshotTestSuite, name::String, path::String)
+
+Run additional assertions before the snapshot comparison. Defaults to no-op.
+"""
+snapshot_test_extras(::SnapshotTestSuite, name, path) = nothing
+
+"""
+    snapshot_allow_additions(suite::SnapshotTestSuite) -> Bool
+
+Whether to allow new files in snapshot output that aren't in the expected directory.
+Defaults to `true`.
+"""
+snapshot_allow_additions(::SnapshotTestSuite) = true
+
+"""
+    run_snapshot_tests(suite::SnapshotTestSuite; filter=nothing, skip=String[])
+
+Run all snapshot tests defined by `suite`, with optional filtering and skipping.
+
+- `filter=nothing`: run all tests
+- `filter="name"`: run only the test with exactly this name
+- `filter=f::Function`: run tests where `f(name)` returns `true`
+- `skip=["name1", ...]`: skip tests with these names
+"""
+function run_snapshot_tests(suite::SnapshotTestSuite; filter=nothing, skip=String[])
+    cases = snapshot_tests(suite)
+    expected = snapshot_expected_dir(suite)
+    allow = snapshot_allow_additions(suite)
+    pred = _make_filter(filter)
+    skip_set = Set{String}(skip)
+
+    matched = false
+    for (name, path) in cases
+        pred(name) || continue
+        name in skip_set && continue
+        matched = true
+        @testset "$name" begin
+            snapshot_test_extras(suite, name, path)
+            test_snapshot(expected, name; allow_additions=allow) do dir
+                snapshot_produce(suite, name, path, dir)
+            end
+        end
+    end
+
+    if !matched && filter !== nothing
+        @warn "No snapshot tests matched the filter" filter
+    end
+end
+
+"""
+    run_snapshot_test(suite::SnapshotTestSuite, name::AbstractString)
+
+Run a single snapshot test by name. Convenience wrapper around
+`run_snapshot_tests(suite; filter=name)`.
+"""
+run_snapshot_test(suite::SnapshotTestSuite, name::AbstractString) =
+    run_snapshot_tests(suite; filter=name)
+
+_make_filter(::Nothing) = _ -> true
+_make_filter(name::AbstractString) = n -> n == name
+_make_filter(f::Function) = f

test/runtests.jl

@@ -8,4 +8,7 @@ using Test
     @testset "update modes" begin
         include("test_update_modes.jl")
     end
+    @testset "suite" begin
+        include("test_suite.jl")
+    end
 end

test/test_suite.jl

@@ -0,0 +1,217 @@
+using SnapshotTesting
+using Test
+
+# --- Helper to pre-create expected snapshot directories ---
+
+function setup_expected!(expected_dir, name, content)
+    d = joinpath(expected_dir, name)
+    mkpath(d)
+    write(joinpath(d, "out.txt"), content)
+end
+
+# --- Suite definitions at top level (required by Julia scoping rules) ---
+
+# :basic_all — basic suite that runs all tests
+const _basic_all_cases = Ref{Vector{Pair{String,String}}}()
+const _basic_all_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:basic_all}) = _basic_all_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:basic_all}) = _basic_all_expected[]
+function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:basic_all}, name, path, dir)
+    write(joinpath(dir, "out.txt"), "output for $name from $path")
+end
+
+# :filter_str — filter by exact string
+const _filter_str_cases = Ref{Vector{Pair{String,String}}}()
+const _filter_str_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_str}) = _filter_str_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_str}) = _filter_str_expected[]
+function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_str}, name, path, dir)
+    write(joinpath(dir, "out.txt"), "output for $name from $path")
+end
+
+# :filter_pred — filter by predicate function
+const _filter_pred_cases = Ref{Vector{Pair{String,String}}}()
+const _filter_pred_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_pred}) = _filter_pred_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_pred}) = _filter_pred_expected[]
+function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:filter_pred}, name, path, dir)
+    write(joinpath(dir, "out.txt"), "output for $name from $path")
+end
+
+# :skip_list — skip list
+const _skip_list_cases = Ref{Vector{Pair{String,String}}}()
+const _skip_list_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:skip_list}) = _skip_list_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:skip_list}) = _skip_list_expected[]
+function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:skip_list}, name, path, dir)
+    write(joinpath(dir, "out.txt"), "output for $name from $path")
+end
+
+# :empty_suite — empty suite
+const _empty_suite_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:empty_suite}) = Pair{String,String}[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:empty_suite}) = _empty_suite_expected[]
+
+# :filter_warn — filter matching nothing emits warning
+const _filter_warn_cases = Ref{Vector{Pair{String,String}}}()
+const _filter_warn_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:filter_warn}) = _filter_warn_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:filter_warn}) = _filter_warn_expected[]
+
+# :extras_test — snapshot_test_extras is called
+const _extras_test_cases = Ref{Vector{Pair{String,String}}}()
+const _extras_test_expected = Ref{String}()
+const _extras_test_called = String[]
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:extras_test}) = _extras_test_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:extras_test}) = _extras_test_expected[]
+function SnapshotTesting.snapshot_test_extras(::SnapshotTestSuite{:extras_test}, name, _path)
+    push!(_extras_test_called, name)
+end
+function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:extras_test}, name, _path, dir)
+    write(joinpath(dir, "out.txt"), "extras output for $name")
+end
+
+# :strict_suite — snapshot_allow_additions=false
+const _strict_suite_cases = Ref{Vector{Pair{String,String}}}()
+const _strict_suite_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:strict_suite}) = _strict_suite_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:strict_suite}) = _strict_suite_expected[]
+SnapshotTesting.snapshot_allow_additions(::SnapshotTestSuite{:strict_suite}) = false
+function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:strict_suite}, name, _path, dir)
+    write(joinpath(dir, "out.txt"), "output for $name")
+end
+
+# :singular_test — run_snapshot_test singular convenience
+const _singular_test_cases = Ref{Vector{Pair{String,String}}}()
+const _singular_test_expected = Ref{String}()
+SnapshotTesting.snapshot_tests(::SnapshotTestSuite{:singular_test}) = _singular_test_cases[]
+SnapshotTesting.snapshot_expected_dir(::SnapshotTestSuite{:singular_test}) = _singular_test_expected[]
+function SnapshotTesting.snapshot_produce(::SnapshotTestSuite{:singular_test}, name, path, dir)
+    write(joinpath(dir, "out.txt"), "output for $name from $path")
+end
+
+# --- Tests ---
+
+@testset "SnapshotTestSuite" begin
+
+    @testset "basic suite runs all tests" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            setup_expected!(expected, "test_a", "output for test_a from /path/test_a")
+            setup_expected!(expected, "test_b", "output for test_b from /path/test_b")
+
+            _basic_all_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"]
+            _basic_all_expected[] = expected
+            run_snapshot_tests(SnapshotTestSuite{:basic_all}())
+        end
+    end
+
+    @testset "filter by exact string" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            setup_expected!(expected, "test_a", "output for test_a from /path/test_a")
+
+            _filter_str_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"]
+            _filter_str_expected[] = expected
+            run_snapshot_tests(SnapshotTestSuite{:filter_str}(); filter="test_a")
+            # test_b should not have been run (no directory created)
+            @test !isdir(joinpath(expected, "test_b"))
+        end
+    end
+
+    @testset "filter by predicate function" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            setup_expected!(expected, "csv_one", "output for csv_one from /path/csv_one")
+            setup_expected!(expected, "csv_two", "output for csv_two from /path/csv_two")
+
+            _filter_pred_cases[] = [
+                "csv_one" => "/path/csv_one",
+                "csv_two" => "/path/csv_two",
+                "bin_one" => "/path/bin_one",
+            ]
+            _filter_pred_expected[] = expected
+            run_snapshot_tests(SnapshotTestSuite{:filter_pred}(); filter=n -> startswith(n, "csv_"))
+            @test !isdir(joinpath(expected, "bin_one"))
+        end
+    end
+
+    @testset "skip list" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            setup_expected!(expected, "test_b", "output for test_b from /path/test_b")
+
+            _skip_list_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"]
+            _skip_list_expected[] = expected
+            run_snapshot_tests(SnapshotTestSuite{:skip_list}(); skip=["test_a"])
+            @test !isdir(joinpath(expected, "test_a"))
+        end
+    end
+
+    @testset "empty suite does not error" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            _empty_suite_expected[] = expected
+            run_snapshot_tests(SnapshotTestSuite{:empty_suite}())
+        end
+    end
+
+    @testset "filter matching nothing emits warning" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            _filter_warn_cases[] = ["test_a" => "/path/test_a"]
+            _filter_warn_expected[] = expected
+            @test_logs (:warn, "No snapshot tests matched the filter") run_snapshot_tests(
+                SnapshotTestSuite{:filter_warn}(); filter="nonexistent"
+            )
+        end
+    end
+
+    @testset "snapshot_test_extras is called before produce" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            setup_expected!(expected, "test_a", "extras output for test_a")
+            setup_expected!(expected, "test_b", "extras output for test_b")
+
+            empty!(_extras_test_called)
+            _extras_test_cases[] = ["test_a" => "/path/test_a", "test_b" => "/path/test_b"]
+            _extras_test_expected[] = expected
+            run_snapshot_tests(SnapshotTestSuite{:extras_test}())
+            @test _extras_test_called == ["test_a", "test_b"]
+        end
+    end
+
+    @testset "snapshot_allow_additions=false is respected" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            setup_expected!(expected, "test_a", "output for test_a")
+
+            _strict_suite_cases[] = ["test_a" => "/path/test_a"]
+            _strict_suite_expected[] = expected
+            run_snapshot_tests(SnapshotTestSuite{:strict_suite}())
+        end
+    end
+
+    @testset "run_snapshot_test singular convenience" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+            setup_expected!(expected, "only_this", "output for only_this from /path/only_this")
+
+            _singular_test_cases[] = ["only_this" => "/path/only_this", "not_this" => "/path/not_this"]
+            _singular_test_expected[] = expected
+            run_snapshot_test(SnapshotTestSuite{:singular_test}(), "only_this")
+            # not_this should not have been run
+            @test !isdir(joinpath(expected, "not_this"))
+        end
+    end
+
+end