diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bec83446..0603da7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,8 +100,10 @@ jobs: # because its fixtures bind DateOnly, a .NET 6+ type absent from net472. RequestBinder is still floored here # through its property tests, and Dummies through its own contract suite (Dummies.UnitTests), running on the # netstandard2.0 asset .NET Framework consumers load — its net8-only tests are conditioned out (issue #215). + # Dummies.Xunit is floored the same way: it ships netstandard2.0, so a .NET Framework consumer loads that + # asset, and its adapter suite must prove it works there and not only on net10. # A per-project loop (not a solution-wide -f net472, which would force the TFM onto the net10-only projects - # and fail) keeps the net472 scope exactly these four. + # and fail) keeps the net472 scope exactly these five. - name: Test the netstandard2.0 libraries on .NET Framework 4.7.2 shell: bash run: | @@ -110,7 +112,8 @@ jobs: FirstClassErrors.UnitTests \ FirstClassErrors.PropertyTests \ FirstClassErrors.RequestBinder.PropertyTests \ - Dummies.UnitTests ; do + Dummies.UnitTests \ + Dummies.Xunit.UnitTests ; do echo "::group::$proj (net472)" dotnet test "$proj/$proj.csproj" -c Release -f net472 -p:EnableNet472Floor=true \ --logger "console;verbosity=normal" diff --git a/Directory.Packages.props b/Directory.Packages.props index b203e811..1489e63f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -42,6 +42,7 @@ + diff --git a/Dummies.UnitTests/AmbientSeedScopeTests.cs b/Dummies.UnitTests/AmbientSeedScopeTests.cs new file mode 100644 index 00000000..2739fdfe --- /dev/null +++ b/Dummies.UnitTests/AmbientSeedScopeTests.cs @@ -0,0 +1,209 @@ +#region Usings declarations + +using JetBrains.Annotations; + +using NFluent; + +#endregion + +namespace Dummies.UnitTests; + +/// +/// The scope form of reproducibility: a handle a caller opens and disposes itself, for a test-framework adapter +/// that observes a test through before/after hooks and therefore has no delegate to wrap (ADR-0035). The seed +/// behaviour must match Any.Reproducibly; what the scope adds is the replay instruction a +/// generation-failure diagnostic names, so a run pinned from outside the test body never advertises a call the +/// test does not contain. +/// +[TestSubject(typeof(Any))] +public sealed class AmbientSeedScopeTests { + + #region Statics members declarations + + private static (int, string) Batch() { + return (Any.Int32().Generate(), Any.String().NonEmpty().Generate()); + } + + #endregion + + [Fact(DisplayName = "UseSeed pins the ambient context, so the same seed yields the same values.")] + public void UseSeedPinsTheAmbientContext() { + (int, string) first; + (int, string) second; + + using (Any.UseSeed(1234)) { first = Batch(); } + using (Any.UseSeed(1234)) { second = Batch(); } + + Check.That(second).IsEqualTo(first); + } + + [Fact(DisplayName = "UseSeed with different seeds produces different sequences.")] + public void DifferentSeedsDiffer() { + (int, string) fromOne; + (int, string) fromTwo; + + using (Any.UseSeed(1)) { fromOne = Batch(); } + using (Any.UseSeed(2)) { fromTwo = Batch(); } + + Check.That(fromTwo).IsNotEqualTo(fromOne); + } + + [Fact(DisplayName = "UseSeed pins the same sequence as Reproducibly for the same seed.")] + public void UseSeedMatchesReproducibly() { + (int, string) fromScope; + (int, string) fromRunner = default; + + using (Any.UseSeed(4242)) { fromScope = Batch(); } + Any.Reproducibly(4242, () => { fromRunner = Batch(); }); + + Check.That(fromScope).IsEqualTo(fromRunner); + } + + [Fact(DisplayName = "Disposing the scope restores the previous context with its draw sequence intact.")] + public void DisposingRestoresThePreviousContext() { + (int, string) first; + (int, string) second; + (int, string) restoredFirst; + (int, string) restoredSecond; + + using (Any.UseSeed(7)) { + first = Batch(); + second = Batch(); + } + + using (Any.UseSeed(7)) { + restoredFirst = Batch(); + // A nested scope draws from its own generator; the outer one must neither be consumed nor reset by + // it, so the outer sequence resumes exactly where it was interrupted. + using (Any.UseSeed(99)) { Batch(); } + restoredSecond = Batch(); + } + + Check.That(restoredFirst).IsEqualTo(first); + Check.That(restoredSecond).IsEqualTo(second); + } + + [Fact(DisplayName = "Nested scopes pin the inner seed while they are open.")] + public void NestedScopesPinTheInnerSeed() { + (int, string) standalone; + (int, string) nested; + + using (Any.UseSeed(555)) { standalone = Batch(); } + + using (Any.UseSeed(111)) { + using (Any.UseSeed(555)) { nested = Batch(); } + } + + Check.That(nested).IsEqualTo(standalone); + } + + [Fact(DisplayName = "Disposing the scope twice is harmless.")] + public void DisposingTwiceIsHarmless() { + IDisposable scope = Any.UseSeed(31); + + scope.Dispose(); + + Check.ThatCode(() => scope.Dispose()).DoesNotThrow(); + } + + [Fact(DisplayName = "The scope does not leak across parallel execution contexts.")] + public async Task TheScopeDoesNotLeakAcrossExecutionContexts() { + (int, string) inside; + (int, string) outside = default; + + using (Any.UseSeed(2026)) { + inside = Batch(); + + // A task started inside the scope inherits it, so run the probe on a context that never saw it. + await Task.Run(() => { outside = Batch(); }, TestContext.Current.CancellationToken); + } + + // The probe drew from a context whose scope was never entered, so it cannot have replayed the pinned + // sequence. (An unseeded draw could coincide, but not across both components of the batch.) + Check.That(outside).IsNotEqualTo(inside); + } + + [Fact(DisplayName = "Without a replay instruction, a generation failure names Any.Reproducibly.")] + public void WithoutAnInstructionTheFailureNamesTheDelegateRunner() { + AnyGenerationException caught; + + using (Any.UseSeed(1234)) { + caught = Assert.Throws( + () => Any.Int32().As(_ => throw new InvalidOperationException("rejected")).Generate()); + } + + Check.That(caught.Seed).IsEqualTo(1234); + Check.That(caught.Message).Contains("The arbitrary values were seeded with 1234"); + Check.That(caught.Message).Contains("Any.Reproducibly(1234, ...)"); + } + + [Fact(DisplayName = "With a replay instruction, a generation failure names it instead of Any.Reproducibly.")] + public void WithAnInstructionTheFailureNamesIt() { + AnyGenerationException caught; + + using (Any.UseSeed(1234, "[Reproducible(Seed = 1234)]")) { + caught = Assert.Throws( + () => Any.Int32().As(_ => throw new InvalidOperationException("rejected")).Generate()); + } + + Check.That(caught.Message).Contains("The arbitrary values were seeded with 1234"); + Check.That(caught.Message).Contains("[Reproducible(Seed = 1234)]"); + // The whole point: the reader is never pointed at a call their test does not contain. + Check.That(caught.Message).Not.Contains("Any.Reproducibly"); + } + + [Fact(DisplayName = "The replay instruction also reaches the partial-replay guidance.")] + public void TheInstructionReachesThePartialReplayGuidance() { + AnyGenerationException caught; + + using (Any.UseSeed(777, "[Reproducible(Seed = 777)]")) { + IAny foreign = new ForeignAny(); + caught = Assert.Throws( + () => Any.Combine(Any.Int32(), foreign, (_, _) => throw new InvalidOperationException("rejected")).Generate()); + } + + Check.That(caught.Message).Contains("not reproducible from this seed alone"); + Check.That(caught.Message).Contains("[Reproducible(Seed = 777)]"); + Check.That(caught.Message).Not.Contains("Any.Reproducibly"); + } + + [Fact(DisplayName = "The replay instruction is scoped: it does not outlive the scope that supplied it.")] + public void TheInstructionDoesNotOutliveItsScope() { + using (Any.UseSeed(1, "[Reproducible(Seed = 1)]")) { } + + AnyGenerationException caught; + using (Any.UseSeed(2)) { + caught = Assert.Throws( + () => Any.Int32().As(_ => throw new InvalidOperationException("rejected")).Generate()); + } + + Check.That(caught.Message).Contains("Any.Reproducibly(2, ...)"); + Check.That(caught.Message).Not.Contains("[Reproducible"); + } + + [Fact(DisplayName = "UseSeed rejects a null replay instruction.")] + public void UseSeedRejectsANullInstruction() { + Check.ThatCode(() => Any.UseSeed(1, null!)).Throws(); + } + + [Theory(DisplayName = "UseSeed rejects a blank replay instruction.")] + [InlineData("")] + [InlineData(" ")] + public void UseSeedRejectsABlankInstruction(string instruction) { + Check.ThatCode(() => Any.UseSeed(1, instruction)).Throws(); + } + + #region Nested types + + /// A generator carrying no random source, so a derivation over it cannot promise a full replay. + private sealed class ForeignAny : IAny { + + public int Generate() { + return 42; + } + + } + + #endregion + +} diff --git a/Dummies.UnitTests/SurfaceParityTests.cs b/Dummies.UnitTests/SurfaceParityTests.cs index 360bb3b9..89861550 100644 --- a/Dummies.UnitTests/SurfaceParityTests.cs +++ b/Dummies.UnitTests/SurfaceParityTests.cs @@ -57,8 +57,10 @@ public void AnyAndAnyContextExposeTheSameScalarFactories() { // A scalar factory produces a generator from the context's own source: it returns a builder and takes no // IAny<> operand. That excludes the composition/collection factories that live only on Any (Combine, ListOf, - // SetOf, DictionaryOf, PairOf, ...), as well as WithSeed (returns AnyContext) and Reproducibly (returns - // void/Task) — none of which AnyContext is meant to mirror. + // SetOf, DictionaryOf, PairOf, ...), as well as the three ways to control seeding — WithSeed (returns + // AnyContext), Reproducibly (returns void/Task) and UseSeed (returns IDisposable). None of those is a + // generator factory, and AnyContext is not meant to mirror them: it already *is* an explicit deterministic + // context, so pinning a seed on one would be meaningless. private static bool IsScalarFactory(MethodInfo method) { if (method.GetParameters().Any(parameter => IsAny(parameter.ParameterType))) { return false; } @@ -66,6 +68,7 @@ private static bool IsScalarFactory(MethodInfo method) { return returnType != typeof(AnyContext) && returnType != typeof(void) + && returnType != typeof(IDisposable) && !typeof(Task).IsAssignableFrom(returnType); } diff --git a/Dummies.Xunit.UnitTests/ArchitectureTests.cs b/Dummies.Xunit.UnitTests/ArchitectureTests.cs new file mode 100644 index 00000000..6188de40 --- /dev/null +++ b/Dummies.Xunit.UnitTests/ArchitectureTests.cs @@ -0,0 +1,44 @@ +#region Usings declarations + +using System.Reflection; + +using NFluent; + +#endregion + +namespace Dummies.Xunit.UnitTests; + +/// +/// Guards the boundary of the companion package. Dummies itself may depend on nothing beyond the standard +/// library (ADR-0011), which is precisely why the xUnit adapter is a separate package (ADR-0036): it exists to +/// carry the one dependency Dummies cannot. What it must never carry is a FirstClassErrors dependency — the +/// error-agnostic promise applies to the whole Dummies line, not just its core assembly. +/// +public sealed class ArchitectureTests { + + [Fact(DisplayName = "Dummies.Xunit references no FirstClassErrors assembly.")] + public void DummiesXunitReferencesNoFirstClassErrorsAssembly() { + AssemblyName[] references = typeof(ReproducibleAttribute).Assembly.GetReferencedAssemblies(); + + foreach (AssemblyName reference in references) { + Check.WithCustomMessage($"Unexpected assembly reference: {reference.Name}") + .That(reference.Name!.StartsWith("FirstClassErrors", StringComparison.Ordinal)).IsFalse(); + } + } + + [Fact(DisplayName = "Dummies.Xunit depends on nothing beyond the standard library, Dummies and xUnit.")] + public void DummiesXunitDependsOnlyOnDummiesAndXunit() { + AssemblyName[] references = typeof(ReproducibleAttribute).Assembly.GetReferencedAssemblies(); + + foreach (AssemblyName reference in references) { + // The exact facade split varies with the SDK, so the guard checks the intent — the standard + // library, the library being adapted, and the framework it is adapted to — not a fixed list. + bool expected = reference.Name is "netstandard" or "mscorlib" or "Dummies" + || reference.Name!.StartsWith("System.", StringComparison.Ordinal) + || reference.Name.StartsWith("xunit.", StringComparison.Ordinal); + + Check.WithCustomMessage($"Unexpected assembly reference: {reference.Name}").That(expected).IsTrue(); + } + } + +} diff --git a/Dummies.Xunit.UnitTests/Dummies.Xunit.UnitTests.csproj b/Dummies.Xunit.UnitTests/Dummies.Xunit.UnitTests.csproj new file mode 100644 index 00000000..3e8604c9 --- /dev/null +++ b/Dummies.Xunit.UnitTests/Dummies.Xunit.UnitTests.csproj @@ -0,0 +1,40 @@ + + + + + + + enable + enable + false + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + diff --git a/Dummies.Xunit.UnitTests/ReproducibleAttributeTests.cs b/Dummies.Xunit.UnitTests/ReproducibleAttributeTests.cs new file mode 100644 index 00000000..3f71c549 --- /dev/null +++ b/Dummies.Xunit.UnitTests/ReproducibleAttributeTests.cs @@ -0,0 +1,200 @@ +#region Usings declarations + +using System.Collections.Concurrent; +using System.Reflection; + +using JetBrains.Annotations; + +using NFluent; + +using Xunit.v3; + +#endregion + +namespace Dummies.Xunit.UnitTests; + +/// +/// The adapter is exercised through the real xUnit pipeline wherever a behaviour is observable from inside a +/// test: pinning, per-case seeds and scope closing all show up in the values a decorated test draws. Two things +/// cannot be observed that way — the outcome-dependent report, which needs a test that has already finished, and +/// the framework contract the adapter reads — so each gets its own guard: the reporting rule is decided by a seam +/// proved directly, and the contract is asserted from an after-hook, where a violation fails the test carrying it. +/// +[TestSubject(typeof(ReproducibleAttribute))] +public sealed class ReproducibleAttributeTests { + + #region Statics members declarations + + private static readonly ConcurrentDictionary DrawnByCase = new(); + + internal static (int, string) Batch() { + return (Any.Int32().Generate(), Any.String().NonEmpty().Generate()); + } + + #endregion + + [Fact(DisplayName = "A pinned seed yields the values that seed produces.")] + [Reproducible(Seed = 1234)] + public void APinnedSeedYieldsThatSeedsValues() { + (int, string) drawn = Batch(); + + (int, string) expected; + // The attribute pinned 1234 for this test; an explicit scope over the same seed must agree, which is + // only true if the attribute really pinned the ambient source the static Any entry points draw from. + using (Any.UseSeed(1234)) { expected = Batch(); } + + Check.That(drawn).IsEqualTo(expected); + } + + [Fact(DisplayName = "A pinned seed of zero is honoured, not treated as unset.")] + [Reproducible(Seed = 0)] + public void APinnedSeedOfZeroIsHonoured() { + (int, string) drawn = Batch(); + + (int, string) expected; + using (Any.UseSeed(0)) { expected = Batch(); } + + Check.That(drawn).IsEqualTo(expected); + } + + [Theory(DisplayName = "Each theory case draws its own seed, not one shared with its siblings.")] + [Reproducible] + [InlineData(1)] + [InlineData(2)] + public void EachTheoryCaseDrawsItsOwnSeed(int which) { + DrawnByCase[which] = Batch(); + + // Both cases run the same code under the same attribute instance. Sharing one seed would make their + // values identical; a seed drawn per case makes them differ. The check runs on whichever case lands + // second, so it does not depend on the order the two are executed in. + if (DrawnByCase.Count == 2) { + Check.That(DrawnByCase[1]).IsNotEqualTo(DrawnByCase[2]); + } + } + + [Fact(DisplayName = "The scope stays open for the whole test and restores after a nested one.")] + [Reproducible(Seed = 99)] + public void TheAttributeSeedSurvivesANestedScope() { + (int, string) first = Batch(); + + using (Any.UseSeed(11)) { Batch(); } + + (int, string) afterNesting = Batch(); + + (int, string) expectedFirst; + (int, string) expectedSecond; + using (Any.UseSeed(99)) { + expectedFirst = Batch(); + expectedSecond = Batch(); + } + + Check.That(first).IsEqualTo(expectedFirst); + Check.That(afterNesting).IsEqualTo(expectedSecond); + } + + [Fact(DisplayName = "A generation failure names the attribute, not the delegate runner.")] + [Reproducible(Seed = 2026)] + public void AGenerationFailureNamesTheAttribute() { + AnyGenerationException caught = Assert.Throws( + () => Any.Int32().As(_ => throw new InvalidOperationException("rejected")).Generate()); + + Check.That(caught.Seed).IsEqualTo(2026); + Check.That(caught.Message).Contains("[Reproducible(Seed = 2026)]"); + // The whole point of the replay instruction: this test contains no Any.Reproducibly call, so naming + // one would send the reader to a call that is not there. + Check.That(caught.Message).Not.Contains("Any.Reproducibly"); + } + + [Fact(DisplayName = "A failing test is told its seed and how to replay it.")] + public void AFailingTestIsToldItsSeed() { + string? report = ReproducibleAttribute.ReportFor(failed: true, seed: 1234); + + Check.That(report).IsNotNull(); + Check.That(report).Contains("seeded with 1234"); + Check.That(report).Contains("[Reproducible(Seed = 1234)]"); + Check.That(report).Not.Contains("Any.Reproducibly"); + } + + [Fact(DisplayName = "A passing test is told nothing.")] + public void APassingTestIsToldNothing() { + Check.That(ReproducibleAttribute.ReportFor(failed: false, seed: 1234)).IsNull(); + } + + [Fact(DisplayName = "xUnit still exposes a finished test's outcome to an after-hook.")] + [OutcomeContract] + public void TheFrameworkStillExposesTheOutcome() { + // The assertion lives in OutcomeContractAttribute.After: by the time it runs, this test has finished, + // which is the only moment the outcome exists. If a future xUnit stops populating it -- the contract + // the whole "report only on failure" rule rests on -- this test fails. + Check.That(true).IsTrue(); + } + + #region Nested types + + /// + /// Asserts, from the one place where a finished test's outcome exists, that the framework still reports it. + /// Throwing here fails the test that carries the attribute, which is exactly the signal wanted. + /// + [AttributeUsage(AttributeTargets.Method)] + private sealed class OutcomeContractAttribute : BeforeAfterTestAttribute { + + public override void After(MethodInfo methodUnderTest, IXunitTest test) { + TestResultState? state = TestContext.Current.TestState; + + Check.WithCustomMessage("xUnit no longer exposes a finished test's state to an after-hook; the Reproducible attribute cannot decide whether to report the seed.") + .That(state).IsNotNull(); + Check.WithCustomMessage("xUnit no longer reports a passing test as Passed; the failure-only rule cannot be trusted.") + .That(state!.Result).IsEqualTo(TestResult.Passed); + } + + } + + #endregion + +} + +/// +/// A class-level application: every test the class declares is reproducible without repeating the attribute, and +/// a method-level declaration overrides it for the test that carries one. +/// +[Reproducible(Seed = 7)] +public sealed class ClassLevelReproducibleTests { + + [Fact(DisplayName = "A class-level attribute pins every test the class declares.")] + public void AClassLevelAttributePinsEveryTest() { + (int, string) drawn = ReproducibleAttributeTests.Batch(); + + (int, string) expected; + using (Any.UseSeed(7)) { expected = ReproducibleAttributeTests.Batch(); } + + Check.That(drawn).IsEqualTo(expected); + } + + [Fact(DisplayName = "A method-level attribute wins over the class-level one.")] + [Reproducible(Seed = 4242)] + public void AMethodLevelAttributeWins() { + (int, string) drawn = ReproducibleAttributeTests.Batch(); + + (int, string) expected; + using (Any.UseSeed(4242)) { expected = ReproducibleAttributeTests.Batch(); } + + Check.That(drawn).IsEqualTo(expected); + } + +} + +/// +/// Without the attribute, nothing is pinned: two runs of the same draw differ. This is the arbitrary-by-default +/// behaviour the attribute is opt-in over, and the guard that a scope opened for another test never leaks here. +/// +public sealed class UndecoratedTests { + + [Fact(DisplayName = "Without the attribute the ambient source stays unpinned.")] + public void WithoutTheAttributeNothingIsPinned() { + (int, string) first = ReproducibleAttributeTests.Batch(); + (int, string) second = ReproducibleAttributeTests.Batch(); + + Check.That(second).IsNotEqualTo(first); + } + +} diff --git a/Dummies.Xunit/Dummies.Xunit.csproj b/Dummies.Xunit/Dummies.Xunit.csproj new file mode 100644 index 00000000..1b990d31 --- /dev/null +++ b/Dummies.Xunit/Dummies.Xunit.csproj @@ -0,0 +1,80 @@ + + + + + + + + netstandard2.0 + enable + enable + latest + + + true + + + 0.1.0-dev + + + Dummies.Xunit + Sylvain AURAT + Reefact + + + + The xUnit v3 companion of Dummies: mark a test, a class or an assembly [Reproducible] and its arbitrary values are drawn from a pinned seed, reported only when the test fails. Removes the per-test Any.Reproducibly ceremony without changing how values are generated. + + + + testing;test-data;dummies;arbitrary;xunit;xunit-v3;deterministic;seed;reproducible + Apache-2.0 + false + + + https://github.com/Reefact/first-class-errors + https://github.com/Reefact/first-class-errors.git + git + + © Reefact 2026 + + + icon.png + readme.md + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Dummies.Xunit/PublicAPI/netstandard2.0/PublicAPI.Shipped.txt b/Dummies.Xunit/PublicAPI/netstandard2.0/PublicAPI.Shipped.txt new file mode 100644 index 00000000..7dc5c581 --- /dev/null +++ b/Dummies.Xunit/PublicAPI/netstandard2.0/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/Dummies.Xunit/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/Dummies.Xunit/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt new file mode 100644 index 00000000..22f58aee --- /dev/null +++ b/Dummies.Xunit/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -0,0 +1,7 @@ +#nullable enable +Dummies.Xunit.ReproducibleAttribute +Dummies.Xunit.ReproducibleAttribute.ReproducibleAttribute() -> void +Dummies.Xunit.ReproducibleAttribute.Seed.get -> int +Dummies.Xunit.ReproducibleAttribute.Seed.set -> void +override Dummies.Xunit.ReproducibleAttribute.After(System.Reflection.MethodInfo! methodUnderTest, Xunit.v3.IXunitTest! test) -> void +override Dummies.Xunit.ReproducibleAttribute.Before(System.Reflection.MethodInfo! methodUnderTest, Xunit.v3.IXunitTest! test) -> void diff --git a/Dummies.Xunit/README.nuget.md b/Dummies.Xunit/README.nuget.md new file mode 100644 index 00000000..6e92aa09 --- /dev/null +++ b/Dummies.Xunit/README.nuget.md @@ -0,0 +1,72 @@ +# Dummies.Xunit + +The [xUnit v3](https://xunit.net) companion of [Dummies](https://www.nuget.org/packages/Dummies). +Mark a test `[Reproducible]` and its arbitrary values are drawn from a pinned +seed — reported **only when the test fails**, so a red test names the exact seed +to replay while a green one stays silent. + +## Why + +`Dummies` makes a run reproducible by wrapping the test body in a delegate: + + [Fact] + public void Order_reference_is_accepted() { + Any.Reproducibly(() => { + string reference = Any.String().StartingWith("ORD-").WithLength(12).Generate(); + // ... act, assert ... + }); + } + +That works on every test framework, and stays the portable form. This package +removes the ceremony for xUnit v3: + + [Fact, Reproducible] + public void Order_reference_is_accepted() { + string reference = Any.String().StartingWith("ORD-").WithLength(12).Generate(); + // ... act, assert ... + } + +Values still vary between runs — which is what surfaces a test secretly +depending on one — but a failure is now recoverable even though the body was +never wrapped in advance. + +## Replaying a failure + +A failing test writes its seed to the test output: + + [Dummies] These arbitrary values were seeded with 1234. Reproduce this run with [Reproducible(Seed = 1234)]. + +Pin it to replay: + + [Fact, Reproducible(Seed = 1234)] + public void Order_reference_is_accepted() { /* ... */ } + +The same instruction is what a generation failure names, so a diagnostic never +points at a call the test does not contain. + +## Where it applies + +- **A test**: `[Fact, Reproducible]` or `[Theory, Reproducible]`. +- **A class**: `[Reproducible]` on the class covers every test it declares. +- **A whole suite**: `[assembly: Reproducible]`. + +The hooks run once per test *case*, so each case of a theory draws its own seed +rather than sharing one with its siblings. When several levels apply, the most +specific one wins for the duration of the test and the outer ones are restored +after it — an assembly-wide `[Reproducible]` can pin the suite while one test +replays a particular seed. + +## Notes + +- Values drawn from an explicit `Any.WithSeed(...)` context are unaffected: that + context is isolated by design and does not draw from the ambient source this + attribute pins. +- The seed is pinned through `Any.UseSeed(...)`, a public handle any test-framework + adapter can use — this package holds no privileged access to `Dummies`. +- xUnit v3 only. On xUnit v2, NUnit, MSTest or anything else, use + `Any.Reproducibly(...)`: it is unaffected by this package and works everywhere. + +## Links + +- [Repository](https://github.com/Reefact/first-class-errors) +- [Dummies](https://www.nuget.org/packages/Dummies) diff --git a/Dummies.Xunit/ReproducibleAttribute.cs b/Dummies.Xunit/ReproducibleAttribute.cs new file mode 100644 index 00000000..56e00cbd --- /dev/null +++ b/Dummies.Xunit/ReproducibleAttribute.cs @@ -0,0 +1,158 @@ +#region Usings declarations + +using System.Reflection; + +using Xunit; +using Xunit.v3; + +#endregion + +namespace Dummies.Xunit; + +/// +/// Makes a test's arbitrary values reproducible: the ambient context is pinned to a seed for +/// the duration of the test, and that seed is reported only when the test fails — so a red test names the +/// exact seed to replay while a green one stays silent. This is the declarative form of +/// Any.Reproducibly(() => { ... }): the values still vary between runs, which is what surfaces a test +/// secretly depending on one, but a failure is recoverable without the body having been wrapped in advance. +/// +/// +/// +/// Apply it next to [Fact] or [Theory], on a class to cover every test it declares, or on the +/// assembly to cover a whole suite. The hooks run once per test case, so each case of a theory gets its +/// own seed rather than sharing one with its siblings. When several levels apply, the most specific one wins +/// for the duration of the test and the outer ones are restored after it — so an assembly-wide +/// [Reproducible] can pin the suite while one test replays a particular seed. +/// +/// +/// Pin to replay a reported run. Left unset, a fresh seed is drawn for every test case. +/// +/// +/// +/// [Fact, Reproducible] +/// public void Order_reference_is_accepted() { +/// string reference = Any.String().StartingWith("ORD-").WithLength(12).Generate(); +/// // ... act, assert ... +/// } +/// +/// // Replay the seed a failing run reported: +/// [Fact, Reproducible(Seed = 1234)] +/// public void Order_reference_is_accepted() { /* ... */ } +/// +/// +/// +/// The seed reaches the test's output, which xUnit attaches to the failing test's result. Values drawn from an +/// explicit Any.WithSeed(...) context are unaffected: that context is isolated by design and does not +/// draw from the ambient source this attribute pins. +/// +/// +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = false)] +public sealed class ReproducibleAttribute : BeforeAfterTestAttribute { + + #region Statics members declarations + + // The scopes opened for the current test, innermost first. An AsyncLocal rather than an instance field + // because one attribute instance serves every test it is applied to, including tests running in parallel; + // the ambient context this pins is itself AsyncLocal-backed, so the two flow together. A stack rather than + // a single slot because the method, class and assembly levels nest, and xUnit closes them in reverse. + private static readonly AsyncLocal Open = new(); + + #endregion + + #region Fields declarations + + // Nullable behind a non-nullable property: an attribute argument cannot be an int?, but the setter running + // at all is what distinguishes "Seed = 0" (a legitimate seed) from a seed that was never declared. + private int? _seed; + + #endregion + + /// + /// The seed to pin, to replay a run a previous failure reported. Left unset, every test case draws a fresh + /// seed — the arbitrary-by-default behaviour that surfaces a test depending on one particular value. + /// + public int Seed { + get => _seed ?? 0; + set => _seed = value; + } + + /// + public override void Before(MethodInfo methodUnderTest, IXunitTest test) { + int seed = _seed ?? NewSeed(); + + Open.Value = new Scope(Any.UseSeed(seed, ReplayInstruction(seed)), seed, Open.Value); + } + + /// + public override void After(MethodInfo methodUnderTest, IXunitTest test) { + Scope? scope = Open.Value; + if (scope is null) { return; } + + Open.Value = scope.Outer; + + try { + string? report = ReportFor(HasFailed(), scope.Seed); + if (report is not null) { TestContext.Current.TestOutputHelper?.WriteLine(report); } + } finally { + // Restoring the ambient context must happen even if reporting throws: a scope left open would pin + // the seed for whatever runs next in this execution context. + scope.Handle.Dispose(); + } + } + + /// + /// What the reader must write to replay the run — the attribute with its seed, not the delegate runner the + /// ambient source names by default, because a test carrying this attribute contains no such call. + /// + private static string ReplayInstruction(int seed) { + return $"[Reproducible(Seed = {seed})]"; + } + + /// + /// Whether the test that just ran failed. Read from the ambient test context, which carries the finished + /// test's outcome by the time the after-hook runs. A context that cannot be read is treated as a pass: a + /// spurious seed on a green test is noise, and silence is the safer default for a diagnostic aid. + /// + private static bool HasFailed() { + return TestContext.Current.TestState?.Result == TestResult.Failed; + } + + /// + /// What to tell the reader once the outcome is known: the seed and how to replay it when the test failed, + /// nothing at all when it passed. Kept apart from reading the outcome so the rule — report only on failure, + /// and name the attribute rather than the delegate runner — is verifiable without a failing test. + /// + internal static string? ReportFor(bool failed, int seed) { + return failed + ? $"[Dummies] These arbitrary values were seeded with {seed}. Reproduce this run with {ReplayInstruction(seed)}." + : null; + } + + /// + /// A fresh seed per test case. Collision-tolerant by construction: the seed identifies a run to replay, it is + /// never asserted on, so two runs coinciding is harmless. + /// + private static int NewSeed() { + return Guid.NewGuid().GetHashCode(); + } + + #region Nested types + + /// One pinned ambient context and the one it displaced, so nested levels unwind in order. + private sealed class Scope { + + internal Scope(IDisposable handle, int seed, Scope? outer) { + Handle = handle; + Seed = seed; + Outer = outer; + } + + internal IDisposable Handle { get; } + internal int Seed { get; } + internal Scope? Outer { get; } + + } + + #endregion + +} diff --git a/Dummies/Any.cs b/Dummies/Any.cs index 1252a644..8d08e318 100644 --- a/Dummies/Any.cs +++ b/Dummies/Any.cs @@ -401,6 +401,63 @@ public static AnyContext WithSeed(int seed) { return new AnyContext(seed); } + /// + /// Pins the ambient random context to until the returned handle is disposed — the + /// scope form of , for a caller that cannot wrap the + /// code it pins in a delegate. A test-framework adapter is the case this exists for: it observes a test through + /// hooks that run before and after it, so it opens the scope in one and disposes it in the other. + /// + /// + /// + /// Inside a test body, prefer : it also reports the seed + /// when the body fails, which this handle does not — whoever opens the scope owns telling the reader which + /// seed to replay. Prefer when an explicit generator object fits better than an + /// ambient scope. + /// + /// + /// Like the ambient context itself, the scope flows with the current execution context, so it never leaks + /// across tests running in parallel. Scopes nest: disposing restores whatever was pinned before, and + /// disposing twice is harmless. Failing to dispose leaves the seed pinned for whatever runs next in the + /// same execution context. + /// + /// + /// The seed pinning the ambient context's value sequence. + /// A handle that restores the previous ambient context when disposed. + public static IDisposable UseSeed(int seed) { + return AmbientRandomSource.UseSeed(seed); + } + + /// + /// Pins the ambient random context to and names, through + /// , what a reader must write to replay this run — the form a + /// test-framework adapter uses. A generation failure appends a reproduction guidance naming the mechanism that + /// actually replays the run; the default names Any.Reproducibly(seed, ...), which is the wrong + /// instruction for a run pinned from outside the test body, where replaying means changing what the adapter + /// reads instead. + /// + /// + /// + /// The instruction is quoted verbatim into the guidance, so pass what the reader must write — an attribute + /// with its seed argument, a runner setting — not a sentence about it. It is not validated beyond being + /// non-blank: a badly phrased instruction degrades the very diagnostic it is meant to improve. + /// + /// + /// Everything else matches : the scope flows with the execution context, nests, + /// and restores the previous ambient context when disposed. + /// + /// + /// The seed pinning the ambient context's value sequence. + /// What a reader must write to replay this run, quoted verbatim into generation-failure guidance. + /// A handle that restores the previous ambient context when disposed. + /// Thrown when is null. + /// Thrown when is empty or white space. + public static IDisposable UseSeed(int seed, string replayInstruction) { + if (replayInstruction is null) { throw new ArgumentNullException(nameof(replayInstruction)); } + if (replayInstruction.Trim().Length == 0) { throw new ArgumentException("The replay instruction must name what a reader writes to replay the run; pass a non-blank instruction, or use the overload without one to name Any.Reproducibly(seed, ...).", nameof(replayInstruction)); } + + return AmbientRandomSource.UseSeed(seed, replayInstruction); + } + /// /// Runs with the ambient random context pinned to a fresh seed and, if the body /// throws, reports that seed before letting the exception propagate. This is how a test that draws on diff --git a/Dummies/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/Dummies/PublicAPI/net8.0/PublicAPI.Unshipped.txt index 81c04079..cd6cc5fe 100644 --- a/Dummies/PublicAPI/net8.0/PublicAPI.Unshipped.txt +++ b/Dummies/PublicAPI/net8.0/PublicAPI.Unshipped.txt @@ -491,6 +491,8 @@ static Dummies.Any.UInt16() -> Dummies.AnyUInt16! static Dummies.Any.UInt32() -> Dummies.AnyUInt32! static Dummies.Any.UInt64() -> Dummies.AnyUInt64! static Dummies.Any.Uri() -> Dummies.AnyUri! +static Dummies.Any.UseSeed(int seed) -> System.IDisposable! +static Dummies.Any.UseSeed(int seed, string! replayInstruction) -> System.IDisposable! static Dummies.Any.WithSeed(int seed) -> Dummies.AnyContext! static Dummies.AnyExtensions.As(this Dummies.IAny! generator, System.Func! factory) -> Dummies.IAny! static Dummies.NullableExtensions.OrNull(this Dummies.IAny! generator) -> Dummies.IAny! diff --git a/Dummies/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/Dummies/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt index c15a0d34..c6b1c1cd 100644 --- a/Dummies/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt +++ b/Dummies/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -418,6 +418,8 @@ static Dummies.Any.UInt16() -> Dummies.AnyUInt16! static Dummies.Any.UInt32() -> Dummies.AnyUInt32! static Dummies.Any.UInt64() -> Dummies.AnyUInt64! static Dummies.Any.Uri() -> Dummies.AnyUri! +static Dummies.Any.UseSeed(int seed) -> System.IDisposable! +static Dummies.Any.UseSeed(int seed, string! replayInstruction) -> System.IDisposable! static Dummies.Any.WithSeed(int seed) -> Dummies.AnyContext! static Dummies.AnyExtensions.As(this Dummies.IAny! generator, System.Func! factory) -> Dummies.IAny! static Dummies.NullableExtensions.OrNull(this Dummies.IAny! generator) -> Dummies.IAny! diff --git a/Dummies/README.nuget.md b/Dummies/README.nuget.md index da5fe8f0..1f5743b7 100644 --- a/Dummies/README.nuget.md +++ b/Dummies/README.nuget.md @@ -112,7 +112,11 @@ matter — and that is the point. for value types (`int?`, `Guid?`, ...) and reference types alike. - **Reproducible runs**: wrap a test in `Any.Reproducibly(...)` and a failing run reports the seed to replay; `Any.WithSeed(seed)` gives an isolated, deterministic - context. + context; `Any.UseSeed(seed)` pins the ambient one until the handle is disposed, for + a caller that has no body to wrap — a test-framework adapter driving the seed from + before/after hooks. Its second overload names what the reader must write to replay, + so a run pinned from outside the test body never points at a call the test does not + contain. ## Example diff --git a/Dummies/RandomSource.cs b/Dummies/RandomSource.cs index 8e84d3f6..e3bcac67 100644 --- a/Dummies/RandomSource.cs +++ b/Dummies/RandomSource.cs @@ -14,9 +14,11 @@ internal abstract class RandomSource { /// /// The reproduction guidance to append to a generation-failure message, phrased for this kind of source. The - /// ambient source points at Any.Reproducibly(seed, ...); a fixed Any.WithSeed(...) context replays - /// deterministically on its own, so pinning the ambient source would not apply — naming the wrong instruction is - /// exactly the misleading diagnostic this method exists to avoid. + /// ambient source points at Any.Reproducibly(seed, ...) — or at whatever instruction the opener of the + /// current scope supplied, since a run pinned by a test-framework + /// adapter is replayed by changing what the adapter reads, not by adding a call the test never had; a fixed + /// Any.WithSeed(...) context replays deterministically on its own, so pinning the ambient source would + /// not apply — naming the wrong instruction is exactly the misleading diagnostic this method exists to avoid. /// internal abstract string ReplayHint(int seed); @@ -48,7 +50,7 @@ internal SeededRandom(int seed) { /// /// The default random context behind the static entry points. The state is stored in an /// , so it flows with the current execution context and never leaks across tests -/// running in parallel. Outside an scope it lazily seeds itself with a fresh seed — every +/// running in parallel. Outside an scope it lazily seeds itself with a fresh seed — every /// run differs, which surfaces a test that secretly depends on a value — and that seed is remembered, so a /// generation failure can still report it. Inside a scope (how Any.Reproducibly(...) pins a run) it is /// deterministic. @@ -59,15 +61,19 @@ internal sealed class AmbientRandomSource : RandomSource { internal static readonly AmbientRandomSource Instance = new(); - private static readonly AsyncLocal State = new(); + private static readonly AsyncLocal State = new(); internal static int NewSeed() { return Guid.NewGuid().GetHashCode(); } internal static IDisposable UseSeed(int seed) { - SeededRandom? previous = State.Value; - State.Value = new SeededRandom(seed); + return UseSeed(seed, null); + } + + internal static IDisposable UseSeed(int seed, string? replayInstruction) { + AmbientState? previous = State.Value; + State.Value = new AmbientState(new SeededRandom(seed), replayInstruction); return new SeedScope(previous); } @@ -78,32 +84,54 @@ private AmbientRandomSource() { } internal override SeededRandom Current { get { - SeededRandom? current = State.Value; + AmbientState? current = State.Value; if (current is null) { - current = new SeededRandom(NewSeed()); + current = new AmbientState(new SeededRandom(NewSeed()), null); State.Value = current; } - return current; + return current.Random; } } internal override string ReplayHint(int seed) { - return $"The arbitrary values were seeded with {seed}; reproduce this run with Any.Reproducibly({seed}, ...)."; + return $"The arbitrary values were seeded with {seed}; reproduce this run with {ReplayInstruction(seed)}."; } internal override string PartialReplayHint(int seed) { - return $"The seeded draws were made with {seed} (Any.Reproducibly({seed}, ...)), but some values come from a generator that does not draw from this source, so they are not reproducible from this seed alone."; + return $"The seeded draws were made with {seed} ({ReplayInstruction(seed)}), but some values come from a generator that does not draw from this source, so they are not reproducible from this seed alone."; + } + + /// + /// What the reader must write to replay the current run: the instruction the opener of the scope supplied, or + /// the delegate runner when nothing was supplied. Read from the scope rather than fixed on the source, because + /// the ambient source is pinned by several mechanisms and each is replayed differently. + /// + private static string ReplayInstruction(int seed) { + return State.Value?.ReplayInstruction ?? $"Any.Reproducibly({seed}, ...)"; } #region Nested types + /// The ambient state a seed scope installs: the seeded generator, and how to replay the run that uses it. + private sealed class AmbientState { + + internal AmbientState(SeededRandom random, string? replayInstruction) { + Random = random; + ReplayInstruction = replayInstruction; + } + + internal SeededRandom Random { get; } + internal string? ReplayInstruction { get; } + + } + private sealed class SeedScope : IDisposable { - private readonly SeededRandom? _previous; + private readonly AmbientState? _previous; private bool _disposed; - internal SeedScope(SeededRandom? previous) { + internal SeedScope(AmbientState? previous) { _previous = previous; } diff --git a/FirstClassErrors.sln b/FirstClassErrors.sln index 3a7f5fd7..a72f33a5 100644 --- a/FirstClassErrors.sln +++ b/FirstClassErrors.sln @@ -59,6 +59,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FirstClassErrors.RequestBin EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FirstClassErrors.Testing.UnitTests", "FirstClassErrors.Testing.UnitTests\FirstClassErrors.Testing.UnitTests.csproj", "{74E68697-7C00-401C-84A9-7BF8DAFD9D34}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Dummies.Xunit", "Dummies.Xunit\Dummies.Xunit.csproj", "{BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Dummies.Xunit.UnitTests", "Dummies.Xunit.UnitTests\Dummies.Xunit.UnitTests.csproj", "{75FD5E2C-A871-4659-8A01-13461FED84EB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -331,6 +335,30 @@ Global {74E68697-7C00-401C-84A9-7BF8DAFD9D34}.Release|x64.Build.0 = Release|Any CPU {74E68697-7C00-401C-84A9-7BF8DAFD9D34}.Release|x86.ActiveCfg = Release|Any CPU {74E68697-7C00-401C-84A9-7BF8DAFD9D34}.Release|x86.Build.0 = Release|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Debug|x64.ActiveCfg = Debug|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Debug|x64.Build.0 = Debug|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Debug|x86.ActiveCfg = Debug|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Debug|x86.Build.0 = Debug|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Release|Any CPU.Build.0 = Release|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Release|x64.ActiveCfg = Release|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Release|x64.Build.0 = Release|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Release|x86.ActiveCfg = Release|Any CPU + {BF67DBAE-EBE1-45CE-86E5-B2E9E3D4EF6B}.Release|x86.Build.0 = Release|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Debug|x64.ActiveCfg = Debug|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Debug|x64.Build.0 = Debug|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Debug|x86.ActiveCfg = Debug|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Debug|x86.Build.0 = Debug|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Release|Any CPU.Build.0 = Release|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Release|x64.ActiveCfg = Release|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Release|x64.Build.0 = Release|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Release|x86.ActiveCfg = Release|Any CPU + {75FD5E2C-A871-4659-8A01-13461FED84EB}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -351,12 +379,12 @@ Global {BB28120C-1D68-469D-928A-51AE454D2487} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} {7991C766-2720-43E5-A9CB-2F3D6C2025FC} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {75EA57DD-0128-4EB9-ADBE-567BFF602A93} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} + {F2C739E7-6F2E-4256-9C87-9D6F1168DF4A} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} + {BC4F2BE8-55C9-4AA4-90FA-699EC83A8985} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} {1FC2C501-8842-4ABE-845E-000ED6575DD6} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} {D12EE45B-78F8-4CD4-8E67-E4A9DDA01AE6} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} {04707ACA-FB2E-43BF-A12C-28E01BF5F60D} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} {1201EA34-8F50-41B8-B829-FCCB62A5F14B} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} - {F2C739E7-6F2E-4256-9C87-9D6F1168DF4A} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} - {BC4F2BE8-55C9-4AA4-90FA-699EC83A8985} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} {74E68697-7C00-401C-84A9-7BF8DAFD9D34} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/doc/handwritten/for-maintainers/adr/0038-open-the-ambient-seed-scope-to-adapters.fr.md b/doc/handwritten/for-maintainers/adr/0038-open-the-ambient-seed-scope-to-adapters.fr.md new file mode 100644 index 00000000..f0856c40 --- /dev/null +++ b/doc/handwritten/for-maintainers/adr/0038-open-the-ambient-seed-scope-to-adapters.fr.md @@ -0,0 +1,192 @@ +# ADR-0038 | Ouvrir la portée de graine ambiante aux adaptateurs de framework de test + +🌍 🇬🇧 [English](0038-open-the-ambient-seed-scope-to-adapters.md) · 🇫🇷 Français (ce fichier) + +**Statut :** Accepté +**Date :** 2026-07-26 +**Décideurs :** Reefact + +## Contexte + +`Dummies` tire chaque valeur arbitraire d'une source aléatoire. Les points +d'entrée statiques `Any` tirent d'une source **ambiante** qui suit le contexte +d'exécution, si bien qu'elle ne fuit jamais entre des tests exécutés en +parallèle. Le déterminisme sur cette source ambiante est optionnel, et +aujourd'hui seuls deux chemins publics y accèdent : + +* `Any.Reproducibly(...)`, qui fixe une graine pour la durée d'un **délégué dont + il est propriétaire**, exécute ce délégué et rapporte la graine si celui-ci + lève. L'ADR-0026 en a fait le récit unique de graine du dépôt. +* `Any.WithSeed(...)`, qui crée un contexte **isolé**. Les points d'entrée + statiques `Any` n'y tirent pas, donc il ne fixe rien pour du code qui les + utilise. + +La poignée qui ouvre et ferme une portée de graine ambiante existe, mais elle est +interne. + +Un adaptateur de framework de test — le package compagnon xUnit considéré +séparément, ou tout futur adaptateur pour un autre framework — ne possède pas de +délégué enveloppant le corps du test. La couture qu'offre un framework est une +paire de points d'accroche exécutés *avant* et *après* la méthode de test. Un +adaptateur doit donc ouvrir la portée ambiante dans l'un et la fermer dans +l'autre, ce qu'aucun chemin public ne permet. + +Deux faits supplémentaires pèsent sur la forme de cette ouverture. + +* **Les échecs de génération portent une instruction de rejeu.** Lorsqu'un + générateur échoue — typiquement une fabrique rejetant une valeur tirée — le + message d'exception ajoute une indication nommant le mécanisme qui rejoue + réellement l'exécution. Cette indication est choisie par type de source : la + source ambiante nomme l'exécuteur à délégué, un contexte isolé se nomme + lui-même, parce que les deux se rejouent différemment. Nommer une instruction + que le code de l'appelant n'utilise pas est un diagnostic trompeur, et l'éviter + est la raison même pour laquelle l'indication varie. +* **Aucune des formulations existantes ne convient à une exécution fixée par un + adaptateur.** Un test dont la graine a été fixée par un adaptateur ne contient + aucun appel à l'exécuteur à délégué, et le rejouer signifie modifier ce que + l'adaptateur lit — un argument d'attribut, un réglage d'exécuteur — et non + ajouter un appel que le test n'a jamais eu. + +Le dépôt dispose déjà d'un idiome établi pour les surcharges locales au contexte, +consigné dans l'ADR-0006 et employé par les coutures d'horloge et d'identifiants +d'instance du package de test : la surcharge est ouverte par un appel `Use…` et +fermée en disposant ce qu'il retourne. + +`Dummies` est en pré-1.0 et n'est pas encore publié sur NuGet (ADR-0011), donc sa +surface publique peut encore grandir sans cérémonie de compatibilité. L'identité +de la bibliothèque est de ne dépendre de rien au-delà de la bibliothèque +standard, une frontière qu'un test d'architecture vérifie sur son propre +assembly. + +Le chemin d'accès alternatif — accorder à un package compagnon nommé l'accès aux +membres internes de `Dummies` — est disponible : la bibliothèque ne déclare +aucune autorisation de ce type aujourd'hui. + +## Décision + +`Dummies` expose la portée de graine ambiante sous forme de poignée publique et +disposable, dont l'ouvreur peut fournir l'instruction de rejeu que les +diagnostics d'échec de génération nommeront. + +## Justification + +* **La forme d'un adaptateur est avant/après, pas autour d'un délégué.** + L'exécuteur à délégué ne peut pas servir un appelant qui n'a aucun délégué à + envelopper, et le contexte isolé est la mauvaise source — le code sous test + tire de la source ambiante. Une portée que l'appelant ouvre et ferme lui-même + est la seule forme qui épouse la couture qu'un framework de test offre + réellement. +* **C'est le caractère public, et non une autorisation d'accès aux internes, qui + garde tous les adaptateurs possibles.** Une autorisation d'accès privilégie un + compagnon nommé et exclut les autres : un adaptateur tiers, ou un adaptateur + interne pour un autre framework, exigerait chacun sa propre autorisation et sa + propre modification de `Dummies`. Une poignée publique fait de « adapter un + autre framework plus tard » une décision additive ne touchant rien ici — ce qui + est précisément la propriété qui permet de prendre la décision xUnit de façon + étroite, sans trancher le reste. +* **Porter l'instruction de rejeu préserve un invariant que la bibliothèque + applique déjà.** Le diagnostic nomme le mécanisme qui s'applique ; c'est + d'ailleurs pourquoi l'indication varie selon la source. Un adaptateur introduit + une troisième manière de fixer la source ambiante, et sans moyen de le dire il + hériterait de la formulation de l'exécuteur à délégué — annonçant, à un + développeur dont le test ne contient aucun appel de ce genre, exactement + l'instruction trompeuse que le mécanisme existe pour empêcher. Laisser + l'ouvreur nommer l'instruction prolonge ce design au lieu de le contourner. +* **La forme « portée disposable » est déjà l'idiome maison.** L'ADR-0006 l'a + établie pour les surcharges d'horloge et d'identifiants d'instance, si bien que + l'ajout se reconnaît comme la même chose plutôt que comme un second mécanisme + sans rapport. +* **C'est le moment le moins coûteux.** Le package est en pré-1.0 et non publié, + donc la surface peut être façonnée maintenant ; un paramètre ajouté plus tard à + un membre publié est plus perturbateur qu'un paramètre présent dès l'origine. + +## Alternatives considérées + +### Accorder au package compagnon l'accès aux membres internes de Dummies + +Considérée parce qu'elle n'ajoute aucune surface publique : l'adaptateur +utiliserait la poignée interne existante telle quelle. Rejetée parce qu'elle +privilégie un compagnon nommé — tout autre adaptateur, interne ou tiers, aurait +besoin de sa propre autorisation et donc de sa propre modification de `Dummies` — +et parce qu'elle couple les identités d'assembly des deux packages pour une +capacité qui n'est pas, en elle-même, privée. + +### Exposer la portée sans instruction de rejeu + +Considérée comme le plus petit ajout possible, reportant la question du +diagnostic jusqu'à l'existence d'un adaptateur. Rejetée parce qu'elle livre le +diagnostic trompeur que le mécanisme d'indication existe pour empêcher : toute +exécution fixée par un adaptateur dont la génération échoue dirait au +développeur d'utiliser un appel que son test ne contient pas. Elle reporte en +outre l'ajout d'un paramètre sur un membre publié, ce qui est l'ordre le plus +perturbateur. + +### Formuler l'indication ambiante de façon neutre, pour qu'elle ne soit jamais fausse + +Considérée parce qu'une indication ne nommant aucun mécanisme ne peut pas nommer +le mauvais. Rejetée parce qu'elle fait payer le cas rare par le cas dominant : +les utilisateurs de l'exécuteur à délégué perdraient une instruction actionnable +— l'appel exact à écrire — pour accommoder un appelant qui peut simplement +énoncer la sienne. + +### Laisser les adaptateurs réutiliser l'exécuteur à délégué + +Considérée parce qu'elle n'exige rien de nouveau. Rejetée parce que les points +d'accroche avant/après d'un framework ne donnent à un adaptateur aucun délégué à +passer : il observe le test, il ne l'invoque pas. + +## Conséquences + +### Positives + +* Tout framework de test peut être adapté sans accès privilégié à `Dummies` et + sans modification supplémentaire de celui-ci, si bien que chaque adaptateur + supplémentaire est une décision indépendante et additive. +* Une exécution dont un adaptateur a fixé la graine rapporte une instruction de + rejeu correspondant au code de l'appelant, ce qui préserve la garantie qu'un + diagnostic ne nomme jamais un mécanisme que le lecteur n'utilise pas. +* L'ajout réutilise l'idiome établi de portée disposable au lieu d'introduire une + seconde forme pour le même concept. + +### Négatives + +* Une troisième manière publique de contrôler la graine, aux côtés de l'exécuteur + à délégué et du contexte isolé. La documentation doit garder les trois + distinctes et dire laquelle le lecteur cherche. +* L'instruction de rejeu est fournie par l'appelant et ne peut pas être validée + par `Dummies` ; une formulation maladroite dégrade donc le diagnostic qu'elle + devait améliorer. + +### Risques + +* Un appelant qui ouvre la portée sans la fermer laisse fuir une graine fixée + vers ce qui s'exécute ensuite dans le même contexte d'exécution. Le risque est + borné par l'idiome — la propriété appartient à qui a ouvert la portée — et + c'est le même contrat que portent déjà les surcharges d'horloge et + d'identifiants d'instance. +* Une poignée publique invite à un usage hors adaptateur de framework de test, là + où l'exécuteur à délégué servirait mieux. C'est une affaire de documentation, + pas de justesse : la portée se comporte identiquement quelle que soit la + manière dont elle est ouverte. + +## Actions de suivi + +* Documenter l'ajout dans le guide utilisateur, en anglais et en français de + concert, en distinguant les trois manières de contrôler la graine et en + désignant le cas de l'adaptateur comme celui pour lequel cette poignée existe. +* Réexaminer la forme portant l'instruction si un second adaptateur montre + qu'elle reste habituellement inutilisée. + +## Références + +* ADR-0006 — Fournir les valeurs de test arbitraires depuis une source unique + semable : l'idiome de portée disposable que cet ajout réutilise, et le suivi + anticipant un adaptateur de framework de test. +* ADR-0011 — Héberger Dummies comme package autonome : l'identité zéro-dépendance + et la latitude pré-1.0 sur lesquelles cette décision s'appuie. +* ADR-0026 — Rebaser les valeurs arbitraires du package de test sur Dummies : le + récit unique de graine que cette source ambiante porte désormais. +* ADR-0039 — Adapter Dummies à xUnit v3 via un package compagnon : le premier + consommateur de cette poignée. +* Issue #226 — le backlog des « nice-to-have » de Dummies où l'adaptateur est + suivi. diff --git a/doc/handwritten/for-maintainers/adr/0038-open-the-ambient-seed-scope-to-adapters.md b/doc/handwritten/for-maintainers/adr/0038-open-the-ambient-seed-scope-to-adapters.md new file mode 100644 index 00000000..6b357dde --- /dev/null +++ b/doc/handwritten/for-maintainers/adr/0038-open-the-ambient-seed-scope-to-adapters.md @@ -0,0 +1,174 @@ +# ADR-0038 | Open the ambient seed scope to test-framework adapters + +🌍 🇬🇧 English (this file) · 🇫🇷 [Français](0038-open-the-ambient-seed-scope-to-adapters.fr.md) + +**Status:** Accepted +**Date:** 2026-07-26 +**Decision Makers:** Reefact + +## Context + +`Dummies` draws every arbitrary value from a random source. The static `Any` +entry points draw from an **ambient** source that flows with the execution +context, so it never leaks across tests running in parallel. Determinism over +that ambient source is opt-in, and today only two public paths reach it: + +* `Any.Reproducibly(...)`, which pins a seed for the duration of a **delegate it + owns**, runs that delegate, and reports the seed if it throws. ADR-0026 made + this the repository's single seed story. +* `Any.WithSeed(...)`, which creates an **isolated** context. The static `Any` + entry points do not draw from it, so it pins nothing for code that uses them. + +The handle that opens and closes an ambient seed scope exists, but is internal. + +A test-framework adapter — the xUnit companion package considered separately, or +any future adapter for another framework — does not own a delegate wrapping the +test body. The seam a framework offers is a pair of hooks that run *before* and +*after* the test method. An adapter must therefore open the ambient scope in one +hook and close it in the other, which no public path allows. + +Two further facts bear on the shape of that opening. + +* **Generation failures carry a replay instruction.** When a generator fails — + typically a factory rejecting a drawn value — the exception message appends a + hint naming the mechanism that actually replays the run. That hint is chosen + per kind of source: the ambient source names the delegate runner, an isolated + context names itself, because the two are replayed differently. Naming an + instruction the caller's code does not use is a misleading diagnostic, and + avoiding it is the reason the hint varies at all. +* **Neither existing phrasing fits an adapter-pinned run.** A test whose seed was + pinned by an adapter contains no call to the delegate runner, and replaying it + means changing whatever the adapter reads — an attribute argument, a runner + setting — not adding a call the test never had. + +The repository already has an established idiom for context-local overrides, +recorded in ADR-0006 and used by the testing package's clock and instance-id +seams: the override is opened by a `Use…` call and closed by disposing what it +returns. + +`Dummies` is pre-1.0 and not yet published to NuGet (ADR-0011), so its public +surface can still grow without a compatibility ceremony. The library's identity +is that it depends on nothing beyond the standard library, a boundary an +architecture test asserts over its own assembly. + +The alternative access path — granting a named companion package access to +`Dummies`' internals — is available: the library declares no such grant today. + +## Decision + +`Dummies` exposes the ambient seed scope as a public, disposable handle whose +opener may supply the replay instruction that generation-failure diagnostics will +name. + +## Rationale + +* **An adapter's shape is before/after, not around a delegate.** The delegate + runner cannot serve a caller that has no delegate to wrap, and the isolated + context is the wrong source — the code under test draws from the ambient one. + A scope the caller opens and closes itself is the only shape that fits the seam + a test framework actually offers. +* **Public, rather than an internals grant, is what keeps every adapter + possible.** An internals grant privileges one named companion and forecloses + the others: a third-party adapter, or a first-party one for another framework, + would each need their own grant and their own change to `Dummies`. A public + handle makes "adapt another framework later" an additive decision that touches + nothing here — which is precisely the property that lets the xUnit decision be + taken narrowly, without deciding the rest. +* **Carrying the replay instruction preserves an invariant the library already + enforces.** The diagnostic names the mechanism that applies; that is why the + hint varies by source in the first place. An adapter introduces a third way of + pinning the ambient source, and without a way to say so it would inherit the + delegate runner's phrasing — advertising, to a developer whose test contains no + such call, exactly the misleading instruction the mechanism exists to prevent. + Letting the opener name the instruction extends that design rather than + working around it. +* **The disposable-scope shape is already the house idiom.** ADR-0006 established + it for the clock and instance-id overrides, so the addition is recognizable as + the same thing rather than a second, unrelated mechanism. +* **This is the cheapest moment.** The package is pre-1.0 and unpublished, so the + surface can be shaped now; a parameter added to a published member later is + more disruptive than one present from the start. + +## Alternatives Considered + +### Grant the companion package access to Dummies' internals + +Considered because it adds no public surface at all: the adapter would use the +existing internal handle unchanged. Rejected because it privileges one named +companion — every other adapter, first-party or third-party, would need its own +grant and therefore its own change to `Dummies` — and because it couples the two +packages' assembly identities for a capability that is not, in itself, private. + +### Expose the scope without a replay instruction + +Considered as the smallest possible addition, deferring the diagnostic question +until an adapter exists. Rejected because it ships the misleading diagnostic the +hint mechanism exists to prevent: every adapter-pinned run whose generation fails +would tell the developer to use a call their test does not contain. It also +defers a parameter onto a published member, which is the more disruptive order. + +### Phrase the ambient hint neutrally, so it is never wrong + +Considered because a hint that names no mechanism cannot name the wrong one. +Rejected because it pays for the rarer case with the dominant one: the delegate +runner's users would lose an actionable instruction — the exact call to write — +to accommodate a caller that can simply state its own. + +### Let adapters reuse the delegate runner + +Considered because it needs nothing new. Rejected because a framework's +before/after hooks give an adapter no delegate to pass: it observes the test, it +does not invoke it. + +## Consequences + +### Positive + +* Any test framework can be adapted without privileged access to `Dummies` and + without a further change to it, so each additional adapter is an independent, + additive decision. +* A run whose seed an adapter pinned reports a replay instruction that matches + the caller's own code, keeping the guarantee that a diagnostic never names a + mechanism the reader does not use. +* The addition reuses the established disposable-scope idiom instead of + introducing a second shape for the same concept. + +### Negative + +* A third public way to control seeding, alongside the delegate runner and the + isolated context. The documentation must keep the three distinct and say which + one a reader wants. +* The replay instruction is supplied by the caller and cannot be validated by + `Dummies`, so a badly phrased one degrades the diagnostic it was meant to + improve. + +### Risks + +* A caller that opens the scope and fails to close it leaks a pinned seed into + whatever runs next in the same execution context. The risk is bounded by the + idiom — ownership belongs to whoever opened the scope — and is the same + contract the clock and instance-id overrides already carry. +* A public handle invites use outside a test-framework adapter, where the + delegate runner would serve better. This is a documentation matter, not a + correctness one: the scope behaves identically however it is opened. + +## Follow-up Actions + +* Document the addition in the user guide, in English and French in lockstep, + distinguishing the three ways to control seeding and naming the adapter case as + the one this handle exists for. +* Revisit the instruction-carrying form if a second adapter shows it is + habitually left unused. + +## References + +* ADR-0006 — Supply arbitrary test values from a single seedable source: the + disposable-scope idiom this addition reuses, and the follow-up anticipating a + test-framework adapter. +* ADR-0011 — Host Dummies as a standalone package: the zero-dependency identity + and the pre-1.0 latitude this decision relies on. +* ADR-0026 — Rebase the testing package's arbitrary values on Dummies: the single + seed story this ambient source now carries. +* ADR-0039 — Adapt Dummies to xUnit v3 through a companion package: the first + consumer of this handle. +* Issue #226 — the Dummies nice-to-have backlog where the adapter is tracked. diff --git a/doc/handwritten/for-maintainers/adr/0039-adapt-dummies-to-xunit-v3-through-a-companion-package.fr.md b/doc/handwritten/for-maintainers/adr/0039-adapt-dummies-to-xunit-v3-through-a-companion-package.fr.md new file mode 100644 index 00000000..0b820f46 --- /dev/null +++ b/doc/handwritten/for-maintainers/adr/0039-adapt-dummies-to-xunit-v3-through-a-companion-package.fr.md @@ -0,0 +1,213 @@ +# ADR-0039 | Adapter Dummies à xUnit v3 via un package compagnon + +🌍 🇬🇧 [English](0039-adapt-dummies-to-xunit-v3-through-a-companion-package.md) · 🇫🇷 Français (ce fichier) + +**Statut :** Accepté +**Date :** 2026-07-26 +**Décideurs :** Reefact + +## Contexte + +Un test qui tire des valeurs arbitraires n'est reproductible que si une graine +est fixée et rapportée. `Dummies` fournit cela via un exécuteur qui fixe une +graine pour la durée d'un délégué et la rapporte lorsque ce délégué lève ; chaque +test sensible aux valeurs doit donc envelopper son corps dans ce délégué. La +cérémonie est reconstituée à la main dans chaque consommateur. + +Un adaptateur qui la supprime a été anticipé puis perdu. Les suivis de l'ADR-0006 +appelaient un adaptateur optionnel de framework de test « pour que la graine soit +exposée automatiquement, sans envelopper chaque corps » ; l'ADR-0026 a rebasé le +moteur de valeurs sur `Dummies` sans reprendre ce suivi. La capacité est donc +anticipée par une ADR acceptée et remplacée par rien. L'audit d'architecture et +de conception de `Dummies` du 2026-07-20 demande un oui ou un non explicite +plutôt qu'un silence prolongé, et place la décision dans le premier cycle stable. + +La capacité qu'un adaptateur doit fournir est étroite : fixer une graine pour la +durée d'un test, et exposer cette graine au développeur **uniquement lorsque le +test échoue**. Une graine rapportée à chaque exécution est du bruit ; une graine +jamais rapportée laisse un échec irreproductible. + +L'identité de `Dummies` est de ne dépendre de rien au-delà de la bibliothèque +standard (ADR-0011), une frontière qu'un test d'architecture vérifie sur son +propre assembly. Elle ne peut donc pas référencer un framework de test, si bien +que tout adaptateur est un package compagnon distinct — l'arrangement que +`FirstClassErrors.Testing` établit déjà comme précédent dans ce dépôt. + +Les frameworks diffèrent par ce qu'expose leur extensibilité supportée, et la +différence est décisive pour la condition « uniquement en cas d'échec » : + +* **xUnit v3.** Son point d'accroche avant/après reçoit le test lui-même, et le + contexte de test ambiant expose l'issue du test terminé — succès ou échec, sa + cause d'échec et le détail de son exception — ainsi que le puits de sortie du + test. La condition « uniquement en cas d'échec » est donc exprimable dans + l'extensibilité documentée, sans aucune implication dans la découverte ni + l'exécution des tests. Le même point d'accroche est collecté depuis la méthode, + la classe et l'assembly, si bien qu'un attribut unique sert un test, une classe + entière ou une suite complète, et il s'exécute une fois par cas de théorie + plutôt qu'une fois par méthode de théorie. +* **xUnit v2.** Son point d'accroche équivalent ne reçoit que la méthode sous + test, et son assembly ne porte aucun contexte de test. Un attribut v2 ne peut + pas observer si le test a réussi ou échoué. Y exprimer la condition « uniquement + en cas d'échec » exige de remplacer la chaîne de découverte et d'exécution des + cas de test. + +Les deux versions livrent des assemblies et des espaces de noms distincts, si +bien qu'un seul assembly ne peut pas référencer les deux ; supporter chacune +signifierait de toute façon un package séparé. + +Les projets de test de ce dépôt s'exécutent déjà sur xUnit v3, si bien qu'un +adaptateur v3 est éprouvé par son auteur autant que documenté. + +L'exécuteur à délégué continue de fonctionner sur tous les frameworks et n'est +pas affecté par cette décision ; les utilisateurs de tout autre framework ne +perdent donc aucune capacité — ils conservent la forme qui existe aujourd'hui. + +L'ADR-0038 ouvre la portée de graine ambiante sous forme de poignée publique, si +bien qu'un adaptateur n'a besoin d'aucun accès privilégié à `Dummies` et +qu'ajouter plus tard un adaptateur pour un autre framework n'exige aucune +modification de celui-ci. + +`Dummies` est publié sur le train de release `dum`, qui ne porte actuellement +qu'un seul package. La bibliothèque a vocation à rejoindre son propre dépôt à +terme. + +## Décision + +`Dummies` reçoit un package compagnon qui fixe et rapporte automatiquement la +graine pour les tests xUnit v3, et ne vise aucun autre framework de test. + +## Justification + +* **Le rapport « uniquement en cas d'échec » est toute la capacité, et seul v3 + sait l'exprimer.** Fixer une graine est facile partout ; décider s'il faut + l'exposer est ce qui sépare un adaptateur utile du bruit. xUnit v3 expose + l'issue du test terminé dans son extensibilité documentée, si bien que + l'adaptateur est une petite quantité de code au-dessus d'un contrat supporté. + En v2 la même condition est inatteignable depuis le point d'accroche + correspondant et exige de s'approprier la découverte et l'exécution sur une + surface semi-interne — un coût permanent et fragile, assumé pour un package + explicitement non destiné à être rouvert. +* **Un seul point d'accroche couvre toute la surface.** Parce que le framework + collecte le point d'accroche depuis la méthode, la classe et l'assembly et + l'exécute par cas de théorie, un attribut unique sert un test, une classe, une + suite entière et chaque cas d'une théorie. C'est toute la surface dont la + capacité a besoin, sans second type par sorte de test et sans toucher à la + manière dont les tests sont découverts. +* **Rien n'est retiré à personne d'autre.** L'exécuteur à délégué reste la forme + portable et continue de fonctionner sur tous les frameworks ; choisir un + framework pour l'adaptateur ne prive donc les utilisateurs des autres d'aucune + capacité, mais seulement de la commodité. +* **Un package compagnon est imposé par l'identité de la bibliothèque.** La + frontière zéro-dépendance rend impossible une référence à un framework de test + à l'intérieur de `Dummies`, et le dépôt livre déjà un package compagnon pour + exactement cette raison. +* **Le choix est éprouvé par son auteur.** Les suites de ce dépôt s'exécutent sur + xUnit v3, si bien que l'adaptateur est utilisé là où il est maintenu plutôt que + livré sans usage. +* **L'étroitesse est délibérée et peu coûteuse à réexaminer.** Parce que + l'ADR-0038 rend la portée de graine publiquement atteignable, un adaptateur + pour un autre framework est une décision additive n'exigeant aucune + modification ici — décliner les autres maintenant n'exclut donc rien. + +## Alternatives considérées + +### Ne rien livrer et conserver l'exécuteur à délégué + +Considérée parce qu'elle ne coûte rien, fonctionne sur tous les frameworks et +correspond déjà à ce que font les consommateurs. Rejetée parce que c'est ce que +le silence a déjà produit une fois : le suivi consigné par l'ADR-0006 a été +abandonné lors du rebase et remplacé par rien, et l'audit demande que la question +soit tranchée plutôt que laissée ouverte. La cérémonie ainsi préservée est +reconstituée à la main dans chaque consommateur, ce qui est précisément le coût +que l'adaptateur existe pour supprimer. + +### Supporter aussi xUnit v2, dans un second package + +Considérée parce que v2 reste une base installée importante, et que « facilement +adoptable » plaide pour rejoindre les consommateurs là où ils sont. Rejetée parce +que la condition « uniquement en cas d'échec » n'est pas exprimable dans le point +d'accroche avant/après de v2 : la livrer signifie remplacer la chaîne de +découverte et d'exécution des cas de test et la maintenir indéfiniment contre une +surface semi-interne. C'est un coût disproportionné et permanent pour une +commodité dont l'absence laisse les utilisateurs v2 exactement où ils sont +aujourd'hui, avec l'exécuteur à délégué portable. + +### Dériver des attributs de fait et de théorie du framework + +Considérée parce qu'elle produit un attribut unique et auto-descriptif par sorte +de test, correspondant au nom qu'avait esquissé le suivi de l'ADR-0006. Rejetée +parce qu'elle coûte un type par sorte de test, ne se compose pas avec des +attributs de fait tiers, ne peut être appliquée ni à une classe ni à un assembly, +et achète une exposition aux internes de la découverte en échange d'aucune +capacité qui manquerait au point d'accroche avant/après. + +### Construire un adaptateur agnostique du framework + +Considérée parce qu'elle servirait tous les frameworks d'un coup et rendrait le +choix sans objet. Rejetée parce qu'il n'existe aucun point d'accroche +inter-frameworks sur lequel la bâtir : la capacité est définie par ce que chaque +framework expose d'un test terminé, et ces surfaces n'ont ni forme ni vocabulaire +communs. + +## Conséquences + +### Positives + +* La reproductibilité devient déclarative et optionnelle à la granularité que + l'auteur choisit — un test, une classe ou une suite entière — au lieu d'un + délégué enveloppant chaque corps sensible aux valeurs. +* Une exécution en échec nomme sa graine sans que l'auteur ait anticipé l'échec, + ce qui est le cas que l'exécuteur à délégué ne sert que lorsqu'il a été + appliqué à l'avance. +* L'adaptateur est éprouvé par les suites de ce dépôt. + +### Négatives + +* Un nouveau package publié, avec sa propre documentation en anglais et en + français, sa propre référence d'API publique et sa propre place dans la chaîne + de build et de release. +* La commodité n'atteint que les utilisateurs de xUnit v3 ; tout autre framework + conserve l'exécuteur à délégué. +* Le plancher de frameworks supportés du package est celui du framework de test, + au-dessus du plancher que `Dummies` conserve — les deux ne peuvent donc pas + partager une même liste de cibles. + +### Risques + +* L'adaptateur dépend du contrat avant/après du framework et de son exposition de + l'issue d'un test terminé. Une future version majeure pourrait changer l'un ou + l'autre. L'exposition est bornée : l'adaptateur utilise l'extensibilité + documentée, pas les internes, si bien qu'un changement se manifesterait comme + un échec de compilation ou de comportement dans sa propre suite plutôt que + silencieusement. +* Une portée de graine ouverte avant un test doit être fermée même lorsque le + test lève, sans quoi la graine fixée fuit vers ce qui s'exécute ensuite dans le + même contexte d'exécution. + +## Actions de suivi + +* Publier le package sur le train `dum` existant dans un premier temps, pour + qu'il soit versionné avec `Dummies`. **Lorsque `Dummies` rejoindra son propre + dépôt, réexaminer si le package compagnon a besoin de son propre train** — un + train partagé n'est approprié que tant que les deux sont livrés depuis le même + endroit et à la même cadence. +* Documenter l'adaptateur dans le guide utilisateur et le readme du package, en + anglais et en français de concert, en présentant l'exécuteur à délégué comme la + forme portable et l'adaptateur comme la commodité xUnit v3. +* Ne réexaminer un adaptateur pour un autre framework que sur demande démontrée ; + l'ADR-0038 garde chacun additif. + +## Références + +* ADR-0038 — Ouvrir la portée de graine ambiante aux adaptateurs de framework de + test : la poignée publique dont ce package est le premier consommateur. +* ADR-0006 — Fournir les valeurs de test arbitraires depuis une source unique + semable : le suivi qui a anticipé cet adaptateur. +* ADR-0011 — Héberger Dummies comme package autonome : la frontière + zéro-dépendance qui impose un package compagnon. +* ADR-0026 — Rebaser les valeurs arbitraires du package de test sur Dummies : le + rebase dans lequel le suivi anticipé a été abandonné. +* `doc/handwritten/for-maintainers/audit/2026-07-20-dummies-architecture-and-design-audit.md` + — l'audit demandant une décision explicite. +* Issue #226 — le backlog des « nice-to-have » de Dummies où l'adaptateur est + suivi. diff --git a/doc/handwritten/for-maintainers/adr/0039-adapt-dummies-to-xunit-v3-through-a-companion-package.md b/doc/handwritten/for-maintainers/adr/0039-adapt-dummies-to-xunit-v3-through-a-companion-package.md new file mode 100644 index 00000000..0f9a5d9e --- /dev/null +++ b/doc/handwritten/for-maintainers/adr/0039-adapt-dummies-to-xunit-v3-through-a-companion-package.md @@ -0,0 +1,195 @@ +# ADR-0039 | Adapt Dummies to xUnit v3 through a companion package + +🌍 🇬🇧 English (this file) · 🇫🇷 [Français](0039-adapt-dummies-to-xunit-v3-through-a-companion-package.fr.md) + +**Status:** Accepted +**Date:** 2026-07-26 +**Decision Makers:** Reefact + +## Context + +A test that draws arbitrary values is reproducible only if a seed is pinned and +reported. `Dummies` supplies that through a runner that pins a seed for the +duration of a delegate and reports it when the delegate throws, so every +value-sensitive test must wrap its body in that delegate. The ceremony is +re-derived by hand in every consumer. + +An adapter that removes it was anticipated and then lost. ADR-0006's follow-ups +called for an optional test-framework adapter "so the seed is surfaced +automatically, without wrapping each body"; ADR-0026 rebased the value engine +onto `Dummies` and did not carry that follow-up forward. The capability is +therefore anticipated by one accepted ADR and replaced by nothing. The +2026-07-20 `Dummies` architecture and design audit asks for an explicit yes or no +on it rather than continued silence, and places the decision in the first stable +cycle. + +The capability an adapter must provide is narrow: pin a seed for the duration of +a test, and surface that seed to the developer **only when the test fails**. A +seed reported on every run is noise; a seed never reported leaves a failure +unreproducible. + +`Dummies`' identity is that it depends on nothing beyond the standard library +(ADR-0011), a boundary an architecture test asserts over its own assembly. It +therefore cannot reference a test framework, so any adapter is a separate, +companion package — the arrangement `FirstClassErrors.Testing` already +establishes as a precedent in this repository. + +The frameworks differ in what their supported extensibility exposes, and the +difference is decisive for the failure-only condition: + +* **xUnit v3.** Its before/after test hook receives the test itself, and the + ambient test context exposes the finished test's outcome — pass or fail, its + failure cause and its exception detail — together with the test's output sink. + The failure-only condition is therefore expressible in documented + extensibility, with no involvement in test discovery or execution. The same + hook is collected from the method, the class and the assembly, so a single + attribute serves one test, a whole class or an entire suite, and it runs once + per theory case rather than once per theory method. +* **xUnit v2.** Its equivalent hook receives only the method under test, and its + assembly carries no test context at all. A v2 attribute cannot observe whether + the test passed or failed. Expressing the failure-only condition there requires + replacing the test-case discovery and execution chain. + +The two versions ship distinct assemblies and namespaces, so one assembly cannot +reference both; supporting each would mean a separate package either way. + +This repository's own test projects already run on xUnit v3, so a v3 adapter is +exercised by its author as well as documented. + +The delegate runner keeps working on every framework and is unaffected by this +decision, so users of any other framework lose no capability — they keep the +form that exists today. + +ADR-0038 opens the ambient seed scope as a public handle, so an adapter needs no +privileged access to `Dummies` and adding an adapter for another framework later +requires no change to it. + +`Dummies` is published on the `dum` release train, which currently carries a +single package. The library is expected to move to its own repository in time. + +## Decision + +`Dummies` gains a companion package that pins and reports the seed automatically +for xUnit v3 tests, and targets no other test framework. + +## Rationale + +* **The failure-only report is the whole capability, and only v3 can express + it.** Pinning a seed is easy everywhere; deciding whether to surface it is what + separates a useful adapter from noise. xUnit v3 exposes the finished test's + outcome in its documented extensibility, so the adapter is a small amount of + code over a supported contract. In v2 the same condition is unreachable from + the corresponding hook and requires owning discovery and execution over + semi-internal surface — a permanent, fragile cost, taken on for a package + explicitly not expected to be revisited. +* **One hook covers the whole surface.** Because the framework collects the hook + from method, class and assembly and runs it per theory case, a single attribute + serves a test, a class, a whole suite, and every case of a theory. That is the + full surface the capability needs, without a second type per kind of test and + without touching how tests are discovered. +* **Nothing is taken away from anyone else.** The delegate runner remains the + portable form and keeps working on every framework, so choosing one framework + for the adapter withholds no capability from users of the others; it withholds + only the convenience. +* **A companion package is forced by the library's identity.** The zero-dependency + boundary makes a test-framework reference impossible inside `Dummies`, and the + repository already ships a companion package for exactly this reason. +* **The choice is dogfooded.** The repository's own suites run on xUnit v3, so + the adapter is used where it is maintained rather than shipped untried. +* **Narrowness is deliberate and cheap to revisit.** Because ADR-0038 makes the + seed scope publicly reachable, an adapter for another framework is an additive + decision requiring no change here — so declining the others now forecloses + nothing. + +## Alternatives Considered + +### Ship nothing and keep the delegate runner + +Considered because it costs nothing, works on every framework, and is already +what consumers do. Rejected because it is what silence has already produced once: +the follow-up ADR-0006 recorded was dropped in the rebase and replaced by +nothing, and the audit asks for the question to be answered rather than left +open. The ceremony it preserves is re-derived by hand in every consumer, which is +the cost the adapter exists to remove. + +### Support xUnit v2 as well, in a second package + +Considered because v2 remains a large installed base, and "easily adopted" argues +for meeting consumers where they are. Rejected because the failure-only condition +is not expressible in v2's before/after hook: delivering it means replacing the +test-case discovery and execution chain and maintaining that against +semi-internal surface indefinitely. That is a disproportionate, permanent cost +for a convenience whose absence leaves v2 users exactly where they are today, +with the portable delegate runner. + +### Derive from the framework's fact and theory attributes + +Considered because it yields a single self-describing attribute per kind of test, +matching the name ADR-0006's follow-up had sketched. Rejected because it costs one +type per kind of test, does not compose with third-party fact attributes, cannot +be applied to a class or an assembly, and buys exposure to the discovery +internals in exchange for no capability the before/after hook lacks. + +### Build one framework-agnostic adapter + +Considered because it would serve every framework at once and make the choice +moot. Rejected because there is no cross-framework hook to build it on: the +capability is defined by what each framework exposes about a finished test, and +those surfaces have neither a common shape nor a common vocabulary. + +## Consequences + +### Positive + +* Reproducibility becomes declarative and opt-in at the granularity the author + chooses — a test, a class, or an entire suite — instead of a delegate wrapped + around every value-sensitive body. +* A failing run names its seed without the author having anticipated the failure, + which is the case the delegate runner serves only when it was applied in + advance. +* The adapter is exercised by this repository's own suites. + +### Negative + +* A new published package, with its own documentation in English and French, its + own public-API baseline, and its own place in the build and release pipeline. +* The convenience reaches xUnit v3 users only; every other framework keeps the + delegate runner. +* The package's supported-framework floor is the test framework's, which is above + the floor `Dummies` itself keeps — so the two cannot share one target list. + +### Risks + +* The adapter depends on the framework's before/after contract and on its + exposure of a finished test's outcome. A future major version could change + either. The exposure is bounded: the adapter uses documented extensibility, not + internals, so a change would surface as a compilation or behavioural failure in + the adapter's own suite rather than silently. +* A seed scope opened before a test must be closed even when the test throws, or + the pinned seed leaks into whatever runs next in the same execution context. + +## Follow-up Actions + +* Publish the package on the existing `dum` train initially, so it versions with + `Dummies`. **When `Dummies` moves to its own repository, revisit whether the + companion package needs a train of its own** — a shared train is only + appropriate while the two ship from the same place and cadence. +* Document the adapter in the user guide and the package readme, in English and + French in lockstep, presenting the delegate runner as the portable form and the + adapter as the xUnit v3 convenience. +* Revisit an adapter for another framework only on demonstrated demand; ADR-0038 + keeps each one additive. + +## References + +* ADR-0038 — Open the ambient seed scope to test-framework adapters: the public + handle this package is the first consumer of. +* ADR-0006 — Supply arbitrary test values from a single seedable source: the + follow-up that anticipated this adapter. +* ADR-0011 — Host Dummies as a standalone package: the zero-dependency boundary + that forces a companion package. +* ADR-0026 — Rebase the testing package's arbitrary values on Dummies: the rebase + in which the anticipated follow-up was dropped. +* `doc/handwritten/for-maintainers/audit/2026-07-20-dummies-architecture-and-design-audit.md` + — the audit asking for an explicit decision. +* Issue #226 — the Dummies nice-to-have backlog where the adapter is tracked. diff --git a/doc/handwritten/for-maintainers/adr/README.md b/doc/handwritten/for-maintainers/adr/README.md index fa4072cd..aa3c1055 100644 --- a/doc/handwritten/for-maintainers/adr/README.md +++ b/doc/handwritten/for-maintainers/adr/README.md @@ -220,3 +220,5 @@ Optional supporting material: | [ADR-0035](0035-enforce-structural-any-conflicts-at-compile-time.md) | Enforce structural Any conflicts at compile time, value-dependent ones at run time | Accepted | | [ADR-0036](0036-draw-lattice-constrained-scalars-on-the-grid.md) | Draw lattice-constrained scalars on the grid | Proposed | | [ADR-0037](0037-vary-the-datetimeoffset-offset-dimension.md) | Vary the DateTimeOffset offset dimension | Proposed | +| [ADR-0038](0038-open-the-ambient-seed-scope-to-adapters.md) | Open the ambient seed scope to test-framework adapters | Accepted | +| [ADR-0039](0039-adapt-dummies-to-xunit-v3-through-a-companion-package.md) | Adapt Dummies to xUnit v3 through a companion package | Accepted | diff --git a/doc/handwritten/for-users/ArbitraryTestValues.en.md b/doc/handwritten/for-users/ArbitraryTestValues.en.md index 45894314..8a34e527 100644 --- a/doc/handwritten/for-users/ArbitraryTestValues.en.md +++ b/doc/handwritten/for-users/ArbitraryTestValues.en.md @@ -87,6 +87,39 @@ Dummies.Any.Reproducibly(1234, () => { Reproducing a run needs the same sequence of draws, so a body whose order depends on non-deterministic external state is not fully replayable from the seed alone. There is also an asynchronous overload, `Dummies.Any.Reproducibly(Func)`, for `async` test bodies. Because the factories, the primitives, and the clock and id seams below all draw from the same ambient source, one `Reproducibly` scope replays them together. +### Pinning the seed without a body to wrap + +`Reproducibly` needs a delegate. A caller that observes a test from the outside — a test-framework adapter running code *before* and *after* the test method — has no such delegate, so it pins the ambient source with a scope it opens and disposes itself: + +```csharp +IDisposable scope = Dummies.Any.UseSeed(1234); +// ... the test runs ... +scope.Dispose(); +``` + +The scope flows with the execution context and nests exactly like `Reproducibly`, and disposing restores whatever was pinned before. What it does **not** do is report the seed when the test fails: whoever opens the scope owns telling the reader which seed to replay. + +That ownership extends to the replay instruction. When a generator itself fails, the `AnyGenerationException` message names how to replay the run — by default `Any.Reproducibly(1234, ...)`, which is the wrong instruction for a test that contains no such call. A caller that pins the seed from outside says so, and its instruction is quoted verbatim instead: + +```csharp +Dummies.Any.UseSeed(1234, "[Reproducible(Seed = 1234)]"); +``` + +Inside a test body, prefer `Reproducibly`: it reports the seed for you. Reach for `UseSeed` only when there is no body to wrap. + +### On xUnit v3: `[Reproducible]` + +The `Dummies.Xunit` companion package does the wrapping for you. Mark a test, a class, or the whole assembly, and its arbitrary values are drawn from a pinned seed reported **only when the test fails**: + +```csharp +[Fact, Reproducible] +public void Some_value_sensitive_test() { + // ... arrange with the factories and Dummies.Any, act, assert ... +} +``` + +A failing run writes `Reproduce this run with [Reproducible(Seed = 1234)]` to the test output; pin `[Reproducible(Seed = 1234)]` to replay it. Each case of a theory draws its own seed, and a method-level declaration overrides a class- or assembly-level one. This is convenience only: `Reproducibly` remains the portable form and works on every framework. + ## Arbitrary `OccurredAt` and `InstanceId` Occurrence data is arbitrary in the same sense: a test often needs it stable without asserting the exact instant or id. The clock and instance-id seams therefore pair a `UseAny` with their `UseFixed`. `Clock.UseAny()` freezes a single arbitrary instant for the scope, while `InstanceIds.UseAny()` hands each error its own distinct arbitrary id: diff --git a/doc/handwritten/for-users/ArbitraryTestValues.fr.md b/doc/handwritten/for-users/ArbitraryTestValues.fr.md index e5b707a3..e5e54e68 100644 --- a/doc/handwritten/for-users/ArbitraryTestValues.fr.md +++ b/doc/handwritten/for-users/ArbitraryTestValues.fr.md @@ -87,6 +87,39 @@ Dummies.Any.Reproducibly(1234, () => { Reproduire une exécution nécessite la **même séquence** de tirages : un corps dont l’ordre dépend d’un état externe non déterministe n’est pas entièrement rejouable à partir de la seule graine. Une surcharge asynchrone, `Dummies.Any.Reproducibly(Func)`, existe pour les corps de test `async`. Comme les fabriques, les primitives et les seams d’horloge et d’identifiants ci-dessous tirent tous de la même source ambiante, un seul `Reproducibly` les rejoue ensemble. +### Fixer la graine sans corps à envelopper + +`Reproducibly` exige un délégué. Un appelant qui observe un test depuis l’extérieur — un adaptateur de framework de test exécutant du code *avant* et *après* la méthode de test — n’en possède aucun : il fixe donc la source ambiante avec une portée qu’il ouvre et dispose lui-même : + +```csharp +IDisposable scope = Dummies.Any.UseSeed(1234); +// ... le test s’exécute ... +scope.Dispose(); +``` + +La portée suit le contexte d’exécution et s’imbrique exactement comme `Reproducibly`, et la disposer restaure ce qui était fixé auparavant. Ce qu’elle ne fait **pas**, c’est rapporter la graine quand le test échoue : c’est à celui qui ouvre la portée de dire au lecteur quelle graine rejouer. + +Cette responsabilité s’étend à l’instruction de rejeu. Lorsqu’un générateur échoue lui-même, le message de l’`AnyGenerationException` nomme la façon de rejouer l’exécution — par défaut `Any.Reproducibly(1234, ...)`, ce qui est la mauvaise instruction pour un test ne contenant aucun appel de ce genre. Un appelant qui fixe la graine depuis l’extérieur l’énonce, et son instruction est citée telle quelle à la place : + +```csharp +Dummies.Any.UseSeed(1234, "[Reproducible(Seed = 1234)]"); +``` + +Dans un corps de test, préférez `Reproducibly` : il rapporte la graine pour vous. Ne recourez à `UseSeed` que lorsqu’il n’y a aucun corps à envelopper. + +### Sur xUnit v3 : `[Reproducible]` + +Le package compagnon `Dummies.Xunit` fait l’enveloppement pour vous. Marquez un test, une classe ou l’assembly entier, et ses valeurs arbitraires sont tirées d’une graine fixée, rapportée **uniquement lorsque le test échoue** : + +```csharp +[Fact, Reproducible] +public void Some_value_sensitive_test() { + // ... arrange avec les fabriques et Dummies.Any, act, assert ... +} +``` + +Une exécution en échec écrit `Reproduce this run with [Reproducible(Seed = 1234)]` dans la sortie du test ; fixez `[Reproducible(Seed = 1234)]` pour la rejouer. Chaque cas d’une théorie tire sa propre graine, et une déclaration au niveau méthode l’emporte sur une déclaration au niveau classe ou assembly. Ce n’est qu’une commodité : `Reproducibly` reste la forme portable et fonctionne sur tous les frameworks. + ## `OccurredAt` et `InstanceId` arbitraires Les données d’occurrence sont arbitraires au même sens : un test a souvent besoin qu’elles soient stables sans en vérifier l’instant ou l’identifiant exact. Les seams de l’horloge et des identifiants proposent donc un `UseAny` en pendant de leur `UseFixed`. `Clock.UseAny()` fige un unique instant arbitraire pour la portée, tandis que `InstanceIds.UseAny()` attribue à chaque erreur son propre identifiant arbitraire distinct : diff --git a/tools/packaging/pack.sh b/tools/packaging/pack.sh index 5db987de..a9513369 100755 --- a/tools/packaging/pack.sh +++ b/tools/packaging/pack.sh @@ -47,10 +47,12 @@ case "$scope" in projects='FirstClassErrors.Cli/FirstClassErrors.Cli.csproj' ;; dum) - # Dummies, the standalone arbitrary-test-value library. Deliberately independent of everything else in - # this repository (ADR-0011): it references no FirstClassErrors project, so it releases on its own - # train and its package must declare no FirstClassErrors dependency -- asserted below. - projects='Dummies/Dummies.csproj' + # Dummies, the standalone arbitrary-test-value library, and its xUnit v3 companion. Deliberately + # independent of everything else in this repository (ADR-0011): neither references a FirstClassErrors + # project, so they release on their own train and their packages must declare no FirstClassErrors + # dependency -- asserted below. Dummies.Xunit rides this train because it versions with the library it + # adapts (ADR-0036); if Dummies ever moves to its own repository, that pairing is worth revisiting. + projects='Dummies/Dummies.csproj Dummies.Xunit/Dummies.Xunit.csproj' ;; *) echo "error: unknown scope '$scope' (expected 'lib', 'cli' or 'dum')" >&2